From fb5a6115a6f3ea0216e3ca0645ba1eb31fb02876 Mon Sep 17 00:00:00 2001 From: Nick Mathewson Date: Tue, 14 Oct 2008 05:04:40 +0000 Subject: Implement all of signing-side and server-side for updater, and the core loops of client-side. More downloader support is needed, and more polishing. See TODO for details. This no longer matches glider-spec.txt exactly. Notably, it uses json instead of sexp. git-svn-id: file:///home/or/svnrepo/updater/trunk@17084 55e972cd-5a19-0410-ae62-a4d7a52db4cd --- Makefile | 2 +- TODO | 63 ++-- lib/glider/ClientCLI.py | 66 ++++ lib/glider/ServerCLI.py | 187 +++++++++++ lib/glider/SignerCLI.py | 313 ++++++++++++++++++ lib/glider/__init__.py | 32 ++ lib/glider/checkJson.py | 274 ++++++++++++++++ lib/glider/download.py | 127 ++++++++ lib/glider/formats.py | 763 +++++++++++++++++++++++++++++++++++--------- lib/glider/keys.py | 232 ++++++++++---- lib/glider/master_keys.py | 5 + lib/glider/repository.py | 294 ++++++++++++++--- lib/glider/tests.py | 40 ++- lib/glider/util.py | 73 +++++ samples/bundle.cfg | 15 + samples/example-mirrors.txt | 5 + samples/example-package.cfg | 24 ++ samples/testpackage.cfg | 1 + specs/glider-spec.txt | 69 +++- 19 files changed, 2297 insertions(+), 288 deletions(-) create mode 100644 lib/glider/ClientCLI.py create mode 100644 lib/glider/ServerCLI.py create mode 100644 lib/glider/SignerCLI.py create mode 100644 lib/glider/checkJson.py create mode 100644 lib/glider/download.py create mode 100644 lib/glider/master_keys.py create mode 100644 lib/glider/util.py create mode 100644 samples/bundle.cfg create mode 100644 samples/example-mirrors.txt create mode 100644 samples/example-package.cfg create mode 100644 samples/testpackage.cfg diff --git a/Makefile b/Makefile index 0d58eb4..e904f9a 100644 --- a/Makefile +++ b/Makefile @@ -2,5 +2,5 @@ export PYTHONPATH=./lib test: - python -m sexp.tests + #python -m sexp.tests python -m glider.tests diff --git a/TODO b/TODO index 7e49aa6..a2140c9 100644 --- a/TODO +++ b/TODO @@ -1,39 +1,54 @@ - o Write spec . Write server-side code (python) - o S-expression lib - . Code to manage data formats - . Parse + X S-expression lib + o Code to manage data formats + o Parse o Validate - - Code to wrangle private keys + o Code to wrangle private keys o Generate - . Store, load (password-protected) - - Print for posterity + o Store, load (password-protected) + o Print for posterity + o Associate with roles. + o Code to generate timestamp files + o Code to generate mirror files + o Code to generate keylist files + o and add new keys to them + o and remove keys. - - Code to generate timestamp files - - Code to generate mirror files - - Code to generate keylist files, and add new keys to them, and - remove keys. +o Repository code. + o make access functions independent of consistency/loading functions - Write client-side code - - Decide early if a python implementation will do for v1. - IF SO: + . Decide early if a python implementation will do for v1. - Adjust httplib, urllib2 to use socks4a. - Check SOCKS package for suitability as basis for socks4a support? - Look into best packageing practices - IF NOT: - - Maybe use curllib for downloading, unless there's something - better. - - Check out Ron's reference code for s-expression handling. - - - Write cacheing code - - Write code to pick a mirror - - Write code to grab a timestamp file and figure out what to do. - - Write code to update other files - - Write code to run, telling another process about status, + + o Write cacheing code + o Write code to pick a mirror + o Write code to grab a timestamp file and figure out what to do. + o Write code to update other files + . Write code to run, telling another process about status, eventually coming up with a list of packages to install or an "A-OK" signal. - - GUI + D GUI + + - DL-via-Tor + - Install-when-done + - Verbose output + - Rendezvous-back with Tor when done. + +- Wrapping + - More unit tests + - Setup.py script to install everything + +- Documentation + - HOWTO guides + . full pydoc + - revise spec + +- Testing + - Much bigger unit tests. diff --git a/lib/glider/ClientCLI.py b/lib/glider/ClientCLI.py new file mode 100644 index 0000000..c6f13fd --- /dev/null +++ b/lib/glider/ClientCLI.py @@ -0,0 +1,66 @@ + +import os +import sys +import getopt + +import glider.util +import glider.repository +import glider.download + +def update(args): + repoRoot = glider.util.userFilename("cache") + options, args = getopt.getopt(args, "", [ "repo=", "no-download" ]) + download = True + + for o, v in options: + if o == '--repo': + repoRoot = v + elif o == "--no-download": + download = False + + repo = glider.repository.LocalRepository(repoRoot) + + files = repo.getFilesToUpdate(trackingBundles=args) + + if not download: + return + + mirrorlist = repo.getMirrorlistFile().get() + + downloader = glider.download.Downloads() + downloader.start() + + for f in files: + # XXXX Use hash. + dj = glider.download.DownloadJob(f, repo.getFilename(f), + mirrorlist) + downloader.addDownloadJob(dj) + # XXXX replace file in repository if ok; reload; see what changed. + + # Wait for in-progress jobs + +# Check my repository + +# Tell me what I need to download + +# Download stuff + +# Tell me what to install. + +def usage(): + print "Known commands:" + print " update [--repo=repository] [--no-download]" + sys.exit(1) + +def main(): + if len(sys.argv) < 2: + usage() + cmd = sys.argv[1] + args = sys.argv[2:] + if cmd in [ "update" ]: + globals()[cmd](args) + else: + usage() + +if __name__ == '__main__': + main() diff --git a/lib/glider/ServerCLI.py b/lib/glider/ServerCLI.py new file mode 100644 index 0000000..c564f5b --- /dev/null +++ b/lib/glider/ServerCLI.py @@ -0,0 +1,187 @@ + +import os +import sys +import getopt +import time + +import simplejson + +import glider.formats +import glider.util +import glider.keys + +def tstamp(): + return time.strftime("%Y%m%d_%H%M%S", time.localtime()) + +def snarf(fname): + f = open(fname, 'rb') + try: + return f.read() + finally: + f.close() + +def snarfObj(fname): + f = open(fname, 'r') + try: + return simplejson.load(f) + finally: + f.close() + +def insert(args): + repo = os.environ.get("THANDY_MASTER_REPO") + backupDir = glider.util.userFilename("old_files") + checkSigs = True + + options, args = getopt.getopt(args, "", ["repo=", "no-check"]) + for o,v in options: + if o == "--repo": + repo = v + elif o == "--no-check": + checkSigs = False + + if not repo: + print "No repository specified." + usage() + if not os.path.exists(repo): + print "No such repository as %r"%repo + usage() + + if not os.path.exists(backupDir): + os.makedirs(backupDir, 0700) + + if checkSigs: + keys = glider.util.getKeylist(os.path.join(repo, "meta/keys.txt")) + else: + keys = None + + n_ok = 0 + for fn in args: + print "Loading %s..."%fn + try: + content = snarf(fn) + except OSError, e: + print "Couldn't open %s: %s"%(fn, e) + continue + + try: + obj = simplejson.loads(content) + except ValueError, e: + print "Couldn't decode %s: %s"%(fn, e) + continue + + try: + ss, r, path = glider.formats.checkSignedObj(obj, keys) + except glider.FormatException, e: + print "Bad format on %s: %s"%(fn, e) + continue + if checkSigs and not ss.isValid(): + print "Not enough valid signatures on %s"%fn + continue + + print " Looks okay. It goes in %s"%path + assert path.startswith("/") + targetPath = os.path.join(repo, path[1:]) + if os.path.exists(targetPath): + oldContents = snarf(targetPath) + if oldContents == content: + print " File unchanged!" + n_ok += 1 + continue + + baseFname = "%s_%s" % (tstamp(), os.path.split(path)[1]) + backupFname = os.path.join(backupDir, baseFname) + print " Copying old file to %s"%backupFname + glider.util.replaceFile(backupFname, oldContents) + + parentDir = os.path.split(targetPath)[0] + if not os.path.exists(parentDir): + print " Making %s"%parentDir + os.makedirs(parentDir, 0755) + print " Replacing file..." + glider.util.replaceFile(targetPath, content) + print " Done." + n_ok += 1 + if n_ok != len(args): + sys.exit(1) + +def timestamp(args): + repo = os.environ.get("THANDY_MASTER_REPO") + ts_keyfile = glider.util.userFilename("timestamp_key") + + options, args = getopt.getopt(args, "", ["repo=", "ts-key="]) + for o,v in options: + if o == "--repo": + repo = v + elif o == "--ts-key": + ts_keyfile = v + + if repo == None: + print "No repository specified." + usage() + if not os.path.exists(repo): + print "No such repository as %r"%repo + usage() + + tsFname = os.path.join(repo, "meta/timestamp.txt") + + try: + mObj = snarfObj(os.path.join(repo, "meta/mirrors.txt")) + except OSError: + print "No mirror list!" + sys.exit(1) + try: + kObj = snarfObj(os.path.join(repo, "meta/keys.txt")) + except OSError: + print "No key list!" + sys.exit(1) + + bundles = [] + for dirpath, dirname, fns in os.walk(os.path.join(repo, "bundleinfo")): + for fn in fns: + try: + bObj = snarfObj(fn) + except (ValueError, OSError), e: + print "(Couldn't read bundle-like %s)"%fn + continue + try: + _, r, _ = glider.formats.checkSignedObj(bObj) + except glider.FormatException, e: + print "Problem reading object from %s"%fn + continue + if r != "bundle": + print "%s was not a good bundle"%fn + continue + bundles.append(bObj['signed']) + + timestamp = glider.formats.makeTimestampObj( + mObj['signed'], kObj['signed'], bundles) + signable = glider.formats.makeSignable(timestamp) + + keydb = glider.formats.Keylist() + #XXXX Still a roundabout way to do this. + keylist = glider.formats.makeKeylistObj(ts_keyfile, True) + keydb.addFromKeylist(keylist) + for k in keydb.iterkeys(): + glider.formats.sign(signable, k) + + content = simplejson.dumps(signable, sort_keys=True) + glider.util.replaceFile(tsFname, content) + +def usage(): + print "Known commands:" + print " insert [--no-check] [--repo=repository] file ..." + print " timestamp [--repo=repository]" + sys.exit(1) + +def main(): + if len(sys.argv) < 2: + usage() + cmd = sys.argv[1] + args = sys.argv[2:] + if cmd in [ "insert", "timestamp" ]: + globals()[cmd](args) + else: + usage() + +if __name__ == '__main__': + main() diff --git a/lib/glider/SignerCLI.py b/lib/glider/SignerCLI.py new file mode 100644 index 0000000..bf5b73a --- /dev/null +++ b/lib/glider/SignerCLI.py @@ -0,0 +1,313 @@ + +import os +import getopt +import sys +import logging +import simplejson + +import glider.keys +import glider.formats + +def getKeyStore(): + return glider.keys.KeyStore(glider.util.userFilename("secret_keys")) + +def dumpKey(key, indent=0): + i = " "*indent + print "%s%s"%(i, key.getKeyID()) + for r, p in key.getRoles(): + print " %s%s\t%s"%(i, r, p) + +def getKey(ks, keyid=None, role=None, path=None): + if keyid is not None: + keys = ks.getKeysFuzzy(keyid) + if None not in (role, path): + keys = [ k for k in keys if k.hasRole(role, path) ] + elif None not in (role, path): + keys = ks.getKeysByRole(role, path) + else: + assert False + if len(keys) < 1: + print "No such key.\nI wanted", + if keyid: print "keyid='%s...'"%keyid, + if None not in (role, path): print "role=%s, path=%s"%(role,path), + print + print "I only know about:" + for k in ks.iterkeys(): + dumpKey(k) + sys.exit(1) + elif len(keys) > 1: + print "Multiple keys match. Possibilities are:" + for k in keys: + dumpKey(k) + sys.exit(1) + else: + return keys[0] + +# ------------------------------ + +def makepackage(args): + options, args = getopt.getopt(args, "", "keyid=") + keyid = None + for o,v in options: + if o == "--keyid": + keyid = v + + if len(args) < 2: + usage() + + configFile = args[0] + dataFile = args[1] + print "Generating package." + package = glider.formats.makePackageObj(configFile, dataFile) + relpath = package['location'] + print "need a key with role matching [package %s]"%relpath + ks = getKeyStore() + ks.load() + key = getKey(ks, keyid=keyid, role='package', path=relpath) + signable = glider.formats.makeSignable(package) + glider.formats.sign(signable, key) + + if 1: + ss, r, p = glider.formats.checkSignedObj(signable, ks) + assert ss.isValid() + + location = os.path.split(package['location'])[-1] + print "Writing signed package to %s"%location + f = open(location, 'w') + simplejson.dump(signable, f, indent=1) + f.close() + +def makebundle(args): + options, args = getopt.getopt(args, "", "keyid=") + keyid = None + for o,v in options: + if o == "--keyid": + keyid = v + + if len(args) < 2: + usage() + + configFile = args[0] + packages = {} + for pkgFile in args[1:]: + print "Loading", pkgFile + f = open(pkgFile, 'r') + p = simplejson.load(f) + f.close() + _, r, _ = glider.formats.checkSignedObj(p) + if r != 'package': + print pkgFile, "was not a package" + packages[p['signed']['location']] = p + + def getHash(path): + p = packages[path] + return glider.formats.getDigest(p['signed']) + + bundleObj = glider.formats.makeBundleObj(configFile, getHash) + signable = glider.formats.makeSignable(bundleObj) + + ks = getKeyStore() + ks.load() + key = getKey(ks, keyid=keyid, role="bundle", path=bundleObj['location']) + glider.formats.sign(signable, key) + + if 1: + ss, r, p = glider.formats.checkSignedObj(signable, ks) + assert ss.isValid() + + location = os.path.split(bundleObj['location'])[-1] + print "Writing signed bundle to %s"%location + f = open(location, 'w') + simplejson.dump(signable, f, indent=1) + f.close() + +# ------------------------------ +def makekeylist(args): + options, args = getopt.getopt(args, "", "keyid=") + keyid = None + for o,v in options: + if o == "--keyid": + keyid = v + + if len(args) < 1: + usage() + + keylist = glider.formats.makeKeylistObj(args[0]) + signable = glider.formats.makeSignable(keylist) + + ks = getKeyStore() + ks.load() + key = getKey(ks, keyid=keyid, role="master", path="/meta/keys.txt") + glider.formats.sign(signable, key) + + if 1: + ss, r, p = glider.formats.checkSignedObj(signable, ks) + assert ss.isValid() + + print "writing signed keylist to keys.txt" + glider.util.replaceFile("keys.txt", + simplejson.dumps(signable, indent=1, sort_keys=True), + textMode=True) + +def signkeylist(args): + if len(args) != 1: + usage() + + keylist = simplejson.load(open(args[0], 'r')) + glider.formats.SIGNED_SCHEMA.checkMatch(keylist) + glider.formats.KEYLIST_SCHEMA.checkMatch(keylist['signed']) + + ks = getKeyStore() + ks.load() + keys = ks.getKeysByRole("master", "/meta/keys.txt") + for k in keys: + glider.formats.sign(keylist, k) + + print "writing signed keylist to keys.txt" + glider.util.replaceFile("keys.txt", + simplejson.dumps(keylist, indent=1, sort_keys=True), + textMode=True) + +def makemirrorlist(args): + options, args = getopt.getopt(args, "", "keyid=") + keyid = None + for o,v in options: + if o == "--keyid": + keyid = v + + if len(args) < 1: + usage() + + mirrorlist = glider.formats.makeMirrorListObj(args[0]) + signable = glider.formats.makeSignable(mirrorlist) + + ks = getKeyStore() + ks.load() + key = getKey(ks, keyid=keyid, role='mirrors', path="/meta/mirrors.txt") + glider.formats.sign(signable, key) + + if 1: + ss, r, p = glider.formats.checkSignedObj(signable, ks) + assert ss.isValid() + + print "writing signed mirrorlist to mirrors.txt" + glider.util.replaceFile("mirrors.txt", + simplejson.dumps(signable, indent=1, sort_keys=True), + textMode=True) + +# ------------------------------ + +def keygen(args): + k = getKeyStore() + k.load() + print "Generating key. This will be slow." + key = glider.keys.RSAKey.generate() + print "Generated new key: %s" % key.getKeyID() + k.addKey(key) + k.save() + +def listkeys(args): + k = getKeyStore() + k.load() + for k in k.iterkeys(): + print k.getKeyID() + for r, p in k.getRoles(): + print " ", r, p + +def addrole(args): + if len(args) < 3: + usage() + ks = getKeyStore() + ks.load() + k = getKey(ks, args[0]) + r = args[1] + if r not in glider.formats.ALL_ROLES: + print "Unrecognized role %r. Known roles are %s"%( + r,", ".join(glider.format.ALL_ROLES)) + sys.exit(1) + p = args[2] + k.addRole(r, p) + ks.save() + +def delrole(args): + if len(args) < 3: + usage() + ks = getKeyStore() + ks.load() + k = getKey(ks, args[0]) + r = args[1] + if r not in glider.formats.ALL_ROLES: + print "Unrecognized role %r. Known roles are %s"%( + r,", ".join(glider.format.ALL_ROLES)) + sys.exit(1) + p = args[2] + + #XXXX rep. + origLen = len(k._roles) + k._roles = [ (role,path) for role,path in k._roles + if (role,path) != (r,p) ] + removed = origLen - len(k._roles) + print removed, "roles removed" + if removed: + ks.save() + +def chpass(args): + ks = getKeyStore() + print "Old password." + ks.load() + print "New password." + ks.clearPassword() + ks.save() + +def dumpkey(args): + options, args = getopt.getopt(args, "", ["include-secret", "passwd="]) + + includeSecret = False + for o,v in options: + if o == '--include-secret': + includeSecret = True + else: + print "Unexpected %r"%o + + ks = getKeyStore() + ks.load() + + keys = [] + if len(args): + keys = [ getKey(ks, a) for a in args ] + else: + keys = list(ks.iterkeys()) + + for k in keys: + data = k.format(private=includeSecret, includeRoles=True) + print "Key(", simplejson.dumps(data, indent=2), ")" + +def usage(): + print "Known commands:" + print " keygen" + print " listkeys" + print " chpass" + print " addrole keyid role path" + print " delrole keyid role path" + print " dumpkey [--include-secret] keyid" + print " makepackage config datafile" + print " makebundle config packagefile ..." + print " signkeylist keylist" + print " makekeylist keylist" + print " makemirrorlist config" + sys.exit(1) + +def main(): + if len(sys.argv) < 2: + usage() + cmd = sys.argv[1] + args = sys.argv[2:] + if cmd in [ "keygen", "listkeys", "addrole", "delrole", "chpass", + "dumpkey", "makepackage", "makebundle", "signkeylist", + "makekeylist", "signkeylist", "makemirrorlist", ]: + globals()[cmd](args) + else: + usage() + +if __name__ == '__main__': + main() diff --git a/lib/glider/__init__.py b/lib/glider/__init__.py index e05bafe..87fd983 100644 --- a/lib/glider/__init__.py +++ b/lib/glider/__init__.py @@ -1,3 +1,35 @@ __all__ = [ 'formats' ] +_BaseException = Exception + +class Exception(_BaseException): + pass + +class FormatException(Exception): + pass + +class UnknownFormat(FormatException): + pass + +class BadSignature(Exception): + pass + +class BadPassword(Exception): + pass + +class InternalError(Exception): + pass + +class RepoError(InternalError): + pass + +class CryptoError(Exception): + pass + +class PubkeyFormatException(FormatException): + pass + +class UnknownMethod(CryptoError): + pass + diff --git a/lib/glider/checkJson.py b/lib/glider/checkJson.py new file mode 100644 index 0000000..0c5eed6 --- /dev/null +++ b/lib/glider/checkJson.py @@ -0,0 +1,274 @@ + +import re +import sys + +import glider + +class Schema: + def matches(self, obj): + try: + self.checkMatch(obj) + except glider.FormatException: + return False + else: + return True + + def checkMatch(self, obj): + raise NotImplemented() + +class Any(Schema): + """ + >>> s = Any() + >>> s.matches("A String") + True + >>> s.matches([1, "list"]) + True + """ + def checkMatch(self, obj): + pass + +class RE(Schema): + """ + >>> s = RE("h.*d") + >>> s.matches("hello world") + True + >>> s.matches("Hello World") + False + >>> s.matches("hello world!") + False + >>> s.matches([33, "Hello"]) + False + """ + def __init__(self, pat=None, modifiers=0, reObj=None, reName="pattern"): + if not reObj: + if not pat.endswith("$"): + pat += "$" + reObj = re.compile(pat, modifiers) + self._re = reObj + self._reName = reName + def checkMatch(self, obj): + if not isinstance(obj, basestring) or not self._re.match(obj): + raise glider.FormatException("%r did not match %s" + %(obj,self._reName)) + +class Str(Schema): + """ + >>> s = Str("Hi") + >>> s.matches("Hi") + True + >>> s.matches("Not hi") + False + """ + def __init__(self, val): + self._str = val + def checkMatch(self, obj): + if self._str != obj: + raise glider.FormatException("Expected %r; got %r"%(self._str, obj)) + +class AnyStr(Schema): + """ + >>> s = AnyStr() + >>> s.matches("") + True + >>> s.matches("a string") + True + >>> s.matches(["a"]) + False + >>> s.matches(3) + False + >>> s.matches(u"a unicode string") + True + >>> s.matches({}) + False + """ + def __init__(self): + pass + def checkMatch(self, obj): + if not isinstance(obj, basestring): + raise glider.FormatException("Expected a string; got %r"%obj) + +class ListOf(Schema): + """ + >>> s = ListOf(RE("(?:..)*")) + >>> s.matches("hi") + False + >>> s.matches([]) + True + >>> s.matches({}) + False + >>> s.matches(["Hi", "this", "list", "is", "full", "of", "even", "strs"]) + True + >>> s.matches(["This", "one", "is not"]) + False + """ + def __init__(self, schema, minCount=0, maxCount=sys.maxint,listName="list"): + self._schema = schema + self._minCount = minCount + self._maxCount = maxCount + self._listName = listName + def checkMatch(self, obj): + if not isinstance(obj, (list, tuple)): + raise glider.FormatException("Expected %s; got %r" + %(self._listName,obj)) + for item in obj: + try: + self._schema.checkMatch(item) + except glider.FormatException, e: + raise glider.FormatException("%s in %s"%(e, self._listName)) + + if not (self._minCount <= len(obj) <= self._maxCount): + raise glider.FormatException("Length of %s out of range" + %self._listName) + +class Struct(Schema): + """ + >>> s = Struct([ListOf(AnyStr()), AnyStr(), Str("X")]) + >>> s.matches(False) + False + >>> s.matches("Foo") + False + >>> s.matches([[], "Q", "X"]) + True + >>> s.matches([[], "Q", "D"]) + False + >>> s.matches([[3], "Q", "X"]) + False + >>> s.matches([[], "Q", "X", "Y"]) + False + """ + def __init__(self, subschemas, allowMore=False, structName="list"): + self._subschemas = subschemas[:] + self._allowMore = allowMore + self._structName = structName + def checkMatch(self, obj): + if not isinstance(obj, (list, tuple)): + raise glider.FormatException("Expected %s; got %r" + %(self._structName,obj)) + elif len(obj) < len(self._subschemas): + raise glider.FormatException( + "Too few fields in %s"%self._structName) + elif len(obj) > len(self._subschemas) and not self._allowMore: + raise glider.FormatException( + "Too many fields in %s"%self._structName) + for item, schema in zip(obj, self._subschemas): + schema.checkMatch(item) + +class DictOf(Schema): + """ + >>> s = DictOf(RE(r'[aeiou]+'), Struct([AnyStr(), AnyStr()])) + >>> s.matches("") + False + >>> s.matches({}) + True + >>> s.matches({"a": ["x", "y"], "e" : ["", ""]}) + True + >>> s.matches({"a": ["x", 3], "e" : ["", ""]}) + False + >>> s.matches({"a": ["x", "y"], "e" : ["", ""], "d" : ["a", "b"]}) + False + """ + def __init__(self, keySchema, valSchema): + self._keySchema = keySchema + self._valSchema = valSchema + def checkMatch(self, obj): + try: + iter = obj.iteritems() + except AttributeError: + raise glider.FormatException("Expected a dict; got %r"%obj) + + for k,v in iter: + self._keySchema.checkMatch(k) + self._valSchema.checkMatch(v) + +class Opt: + """Helper; applied to a value in Obj to mark it optional. + + >>> s = Obj(k1=Str("X"), k2=Opt(Str("Y"))) + >>> s.matches({'k1': "X", 'k2': "Y"}) + True + >>> s.matches({'k1': "X", 'k2': "Z"}) + False + >>> s.matches({'k1': "X"}) + True + """ + def __init__(self, schema): + self._schema = schema + def checkMatch(self, obj): + self._schema.checkMatch(obj) + +class Obj(Schema): + """ + >>> s = Obj(a=AnyStr(), bc=Struct([Int(), Int()])) + >>> s.matches({'a':"ZYYY", 'bc':[5,9]}) + True + >>> s.matches({'a':"ZYYY", 'bc':[5,9], 'xx':5}) + True + >>> s.matches({'a':"ZYYY", 'bc':[5,9,3]}) + False + >>> s.matches({'a':"ZYYY"}) + False + + """ + def __init__(self, _objname="object", **d): + self._objname = _objname + self._required = d.items() + + + def checkMatch(self, obj): + for k,schema in self._required: + try: + item = obj[k] + except KeyError: + if not isinstance(schema, Opt): + raise glider.FormatException("Missing key %s in %s" + %(k,self._objname)) + + else: + try: + schema.checkMatch(item) + except glider.FormatException, e: + raise glider.FormatException("%s in %s.%s" + %(e,self._objname,k)) + + +class Int(Schema): + """ + >>> s = Int() + >>> s.matches(99) + True + >>> s.matches(False) + False + >>> s.matches(0L) + True + >>> s.matches("a string") + False + >>> Int(lo=10, hi=30).matches(25) + True + >>> Int(lo=10, hi=30).matches(5) + False + """ + def __init__(self, lo=-sys.maxint, hi=sys.maxint): + self._lo = lo + self._hi = hi + def checkMatch(self, obj): + if isinstance(obj, bool) or not isinstance(obj, (int, long)): + # We need to check for bool as a special case, since bool + # is for historical reasons a subtype of int. + raise glider.FormatException("Got %r instead of an integer"%obj) + elif not (self._lo <= obj <= self._hi): + raise glider.FormatException("%r not in range [%r,%r]" + %(obj, self._lo, self._hi)) + +class Bool(Schema): + """ + >>> s = Bool() + >>> s.matches(True) and s.matches(False) + True + >>> s.matches(11) + False + """ + def __init__(self): + pass + def checkMatch(self, obj): + if not isinstance(obj, bool): + raise glider.FormatException("Got %r instead of a boolean"%obj) diff --git a/lib/glider/download.py b/lib/glider/download.py new file mode 100644 index 0000000..3135e39 --- /dev/null +++ b/lib/glider/download.py @@ -0,0 +1,127 @@ + + +import urllib2 +import httplib +import random + +import threading, Queue + +import glider.util + +class Downloads: + def __init__(self, n_threads=2): + self._lock = threading.RLock() + self.downloads = {} + self.haveDownloaded = {} + self.downloadQueue = Queue.Queue() + self.threads = [ threading.Thread(target=self._thread) ] + for t in self.threads: + t.setDaemon(True) + + def start(self): + for t in self.threads: + t.start() + + def isCurrentlyDownloading(self, relPath): + self._lock.acquire() + try: + return self.downloads.has_key(relPath) + finally: + self._lock.release() + + def isRedundant(self, relPath): + self._lock.acquire() + try: + return (self.downloads.has_key(relPath) or + self.haveDownloaded.has_key(relPath)) + finally: + self._lock.release() + + def addDownloadJob(self, job): + rp = job.getRelativePath() + self._lock.acquire() + self.downloads[rp] = job + self._lock.release() + self.downloadQueue.put(job) + + def _thread(self): + while True: + job = self.downloadQueue.get() + job.download() + rp = job.getRelativePath() + self._lock.acquire() + try: + del self.downloads[rp] + self.haveDownloaded[rp] = True + finally: + self._lock.release() + +class DownloadJob: + def __init__(self, relPath, destPath, mirrorlist=None, + wantHash=None, canStall=False): + self._relPath = relPath + self._wantHash = wantHash + self._mirrorList = mirrorlist + self._destPath = destPath + + tmppath = glider.util.userFilename("tmp") + if relPath.startswith("/"): + relPath = relPath[1:] + self._tmppath = os.path.join(tmppath, relPath) + + d = os.path.dirname(self._tmppath) + if not os.path.exists(d): + os.makedirs(d, 0700) + + def getRelativePath(self): + return self._relPath + + def haveStalledFile(self): + return os.path.exists(self._tmppath) + + def getURL(self, mirrorlist=None): + if mirrorlist is None: + mirrorlist = self._mirrorList + weightSoFar = 0 + usable = [] + + for m in mirrorlist['mirrors']: + for c in m['contents']: + # CHECK FOR URL SUITABILITY XXXXX + + if glider.formats.rolePathMatches(c, self._relPath): + weightSoFar += m['weight'] + usable.append( (weightSoFar, m) ) + break + + wTarget = random.randint(0, weightSoFar) + mirror = None + # Could use bisect here instead + for w, m in mirrorlist: + if w >= wTarget: + mirror = m + break + + return m['urlbase'] + self._relPath + + def download(self): + # XXXX RESUME + + f_in = urllib2.urlopen(self.getURL()) + f_out = open(self._tmpPath, 'w') + while True: + c = f_in.read(1024) + if not c: + break + f_out.write(c) + f_in.close() + f_out.close() + # XXXXX retry on failure + + if self._wantHash: + gotHash = glider.formats.getFileDigest(self._tmpPath) + if gotHash != self._wantHash: + # XXXX Corrupt file. + pass + + glider.utils.moveFile(self._tmpPath, self._destPath) diff --git a/lib/glider/formats.py b/lib/glider/formats.py index 34ece7d..9846c50 100644 --- a/lib/glider/formats.py +++ b/lib/glider/formats.py @@ -1,28 +1,62 @@ -import sexp.access -import sexp.encode +import simplejson import time import re +import binascii +import calendar -class FormatException(Exception): - pass +import glider.checkJson + +import Crypto.Hash.SHA256 class KeyDB: + """A KeyDB holds public keys, indexed by their key IDs.""" def __init__(self): - self.keys = {} + self._keys = {} def addKey(self, k): - self.keys[k.getKeyID()] = k + keyid = k.getKeyID() + try: + oldkey = self._keys[keyid] + for r, p in oldkey.getRoles(): + if (r, p) not in k.getRoles(): + k.addRole(r,p) + except KeyError: + pass + self._keys[k.getKeyID()] = k def getKey(self, keyid): - return self.keys[keyid] + return self._keys[keyid] + def getKeysByRole(self, role, path): + results = [] + for key in self._keys.itervalues(): + for r,p in key.getRoles(): + if r == role: + if rolePathMatches(p, path): + results.append(key) + return results + + def getKeysFuzzy(self, keyid): + r = [] + for k,v in self._keys.iteritems(): + if k.startswith(keyid): + r.append(v) + return r + def iterkeys(self): + return self._keys.itervalues() _rolePathCache = {} def rolePathMatches(rolePath, path): - """ + """Return true iff the relative path in the filesystem 'path' conforms + to the pattern 'rolePath': a path that a given key is + authorized to sign. Patterns are allowed to contain * to + represent one or more characters in a filename, and ** to + represent any level of directory structure. >>> rolePathMatches("a/b/c/", "a/b/c/") True >>> rolePathMatches("**/c.*", "a/b/c.txt") True + >>> rolePathMatches("**/c.*", "a/b/ctxt") + False >>> rolePathMatches("**/c.*", "a/b/c.txt/foo") False >>> rolePathMatches("a/*/c", "a/b/c") @@ -35,40 +69,74 @@ def rolePathMatches(rolePath, path): try: regex = _rolePathCache[rolePath] except KeyError: + orig = rolePath + # remove duplicate slashes. rolePath = re.sub(r'/+', '/', rolePath) + # escape, then ** becomes .* rolePath = re.escape(rolePath).replace(r'\*\*', r'.*') + # * becomes [^/]* rolePath = rolePath.replace(r'\*', r'[^/]*') + # and no extra text is allowed. rolePath += "$" - regex = _rolePathCache[rolePath] = re.compile(rolePath) + regex = _rolePathCache[orig] = re.compile(rolePath) return regex.match(path) != None -def checkSignatures(signed, keyDB, role, path): +class SignatureStatus: + """Represents the outcome of checking signature(s) on an object.""" + def __init__(self, good, bad, unrecognized, unauthorized): + # keyids for all the valid signatures + self._good = good[:] + # keyids for the invalid signatures (we had the key, and it failed). + self._bad = bad[:] + # keyids for signatures where we didn't recognize the key + self._unrecognized = unrecognized[:] + # keyids for signatures where we recognized the key, but it doesn't + # seem to be allowed to sign this kind of document. + self._unauthorized = unauthorized[:] + + def isValid(self, threshold=1): + """Return true iff we got at least 'threshold' good signatures.""" + return len(self._good) >= threshold + + def mayNeedNewKeys(self): + """Return true iff downloading a new set of keys might tip this + signature status over to 'valid.'""" + return len(self._unrecognized) or len(self._unauthorized) + +def checkSignatures(signed, keyDB, role=None, path=None): + """Given an object conformant to SIGNED_SCHEMA and a set of public keys + in keyDB, verify the signed object in 'signed'.""" + + SIGNED_SCHEMA.checkMatch(signed) + goodSigs = [] badSigs = [] unknownSigs = [] tangentialSigs = [] - assert signed[0] == "signed" - data = signed[1] + signable = signed['signed'] + signatures = signed['signatures'] d_obj = Crypto.Hash.SHA256.new() - sexp.encode.hash_canonical(data, d_obj) + getDigest(signable, d_obj) digest = d_obj.digest() - for signature in sexp.access.s_children(signed, "signature"): - attrs = signature[1] - sig = attrs[2] - keyid = s_child(attrs, "keyid")[1] + for signature in signatures: + sig = signature['sig'] + keyid = signature['keyid'] + method = signature['method'] + try: key = keyDB.getKey(keyid) except KeyError: unknownSigs.append(keyid) continue - method = s_child(attrs, "method")[1] + try: result = key.checkSignature(method, sig, digest=digest) - except UnknownMethod: + except glider.UnknownMethod: continue + if result == True: if role is not None: for r,p in key.getRoles(): @@ -82,132 +150,264 @@ def checkSignatures(signed, keyDB, role, path): else: badSigs.append(keyid) - return goodSigs, badSigs, unknownSigs, tangentialSigs + return SignatureStatus(goodSigs, badSigs, unknownSigs, tangentialSigs) + +def encodeCanonical(obj, outf=None): + """Encode the object obj in canoncial JSon form, as specified at + http://wiki.laptop.org/go/Canonical_JSON . It's a restricted + dialect of json in which keys are always lexically sorted, + there is no whitespace, floats aren't allowed, and only quote + and backslash get escaped. The result is encoded in UTF-8, + and the resulting bits are passed to outf (if provided), or joined + into a string and returned. + + >>> encodeCanonical("") + '""' + >>> encodeCanonical([1, 2, 3]) + '[1,2,3]' + >>> encodeCanonical({"x" : 3, "y" : 2}) + '{"x":3,"y":2}' + """ + def default(o): + raise TypeError("Can't encode %r", o) + def floatstr(o): + raise TypeError("Floats not allowed.") + def canonical_str_encoder(s): + return '"%s"' % re.sub(r'(["\\])', r'\\\1', s) + + # XXX This is, alas, a hack. I'll submit a canonical JSon patch to + # the simplejson folks. + + iterator = simplejson.encoder._make_iterencode( + None, default, canonical_str_encoder, None, floatstr, + ":", ",", True, False, True)(obj, 0) + + result = None + if outf == None: + result = [ ] + outf = result.append + + for u in iterator: + outf(u.encode("utf-8")) + if result is not None: + return "".join(result) + +def getDigest(obj, digestObj=None): + """Update 'digestObj' (typically a SHA256 object) with the digest of + the canonical json encoding of obj. If digestObj is none, + compute the SHA256 hash and return it. + + DOCDOC string equivalence. + """ + useTempDigestObj = (digestObj == None) + if useTempDigestObj: + digestObj = Crypto.Hash.SHA256.new() + + if isinstance(obj, str): + digestObj.update(obj) + elif isinstance(obj, unicode): + digestObj.update(obj.encode("utf-8")) + else: + encodeCanonical(obj, digestObj.update) + + if useTempDigestObj: + return digestObj.digest() + +def getFileDigest(f, digestObj=None): + """Update 'digestObj' (typically a SHA256 object) with the digest of + the file object in f. If digestObj is none, compute the SHA256 + hash and return it. + + >>> s = "here is a long string"*1000 + >>> import cStringIO, Crypto.Hash.SHA256 + >>> h1 = Crypto.Hash.SHA256.new() + >>> h2 = Crypto.Hash.SHA256.new() + >>> getFileDigest(cStringIO.StringIO(s), h1) + >>> h2.update(s) + >>> h1.digest() == h2.digest() + True + """ + useTempDigestObj = (digestObj == None) + if useTempDigestObj: + digestObj = Crypto.Hash.SHA256.new() -def sign(signed, key): - assert sexp.access.s_tag(signed) == 'signed' - s = signed[1] - keyid = key.keyID() + while 1: + s = f.read(4096) + if not s: + break + digestObj.update(s) - oldsignatures = [ s for s in signed[2:] if s_child(s[1], "keyid") != keyid ] - signed[2:] = oldsignatures + if useTempDigestObj: + return digestObj.digest() - for method, sig in key.sign(s): - signed.append(['signature', [['keyid', keyid], ['method', method]], - sig]) +def makeSignable(obj): + return { 'signed' : obj, 'signatures' : [] } -def formatTime(t): - """ - >>> formatTime(1221265172) - '2008-09-13 00:19:32' +def sign(signed, key): + """Add an element to the signatures of 'signed', containing a new signature + of the "signed" part. """ - return time.strftime("%Y-%m-%d %H:%M:%S", time.gmtime(t)) - -def parseTime(s): - return time.timegm(time.strptime(s, "%Y-%m-%d %H:%M:%S")) -def _parseSchema(s, t=None): - sexpr = sexp.parse.parse(s) - schema = sexp.access.parseSchema(sexpr, t) - return schema + SIGNED_SCHEMA.checkMatch(signed) -SCHEMA_TABLE = { } + signable = signed["signed"] + signatures = signed['signatures'] -PUBKEY_TEMPLATE = r""" - (=pubkey ((:unordered (=type .) (:anyof (. _)))) _) -""" + keyid = key.getKeyID() -SCHEMA_TABLE['PUBKEY'] = _parseSchema(PUBKEY_TEMPLATE) + signatures = [ s for s in signatures if s['keyid'] != keyid ] -TIME_TEMPLATE = r"""/\{d}4-\d{2}-\d{2} \d{2}:\d{2}:\d{2}/""" + method, sig = key.sign(signable) + signatures.append({ 'keyid' : keyid, + 'method' : method, + 'sig' : sig }) + signed['signatures'] = signatures -SCHEMA_TABLE['TIME'] = sexp.access.parseSchema(TIME_TEMPLATE) - -ATTRS_TEMPLATE = r"""(:anyof (_ *))""" +def formatTime(t): + """Encode the time 't' in YYYY-MM-DD HH:MM:SS format. -SCHEMA_TABLE['ATTRS'] = _parseSchema(ATTRS_TEMPLATE) + >>> formatTime(1221265172) + '2008-09-13 00:19:32' + """ + return time.strftime("%Y-%m-%d %H:%M:%S", time.gmtime(t)) -SIGNED_TEMPLATE = r""" - (=signed - _ - (:someof - (=signature ((:unordered - (=keyid _) (=method _) .ATTRS)) _) - ) - )""" - -SIGNED_SCHEMA = _parseSchema(SIGNED_TEMPLATE, SCHEMA_TABLE) - -KEYLIST_TEMPLATE = r""" - (=keylist - (=ts .TIME) - (=keys - (:anyof - (=key ((:unordered (=roles (:someof (. .))) .ATTRS)) _) - )) - * - )""" - -KEYLIST_SCHEMA = _parseSchema(KEYLIST_TEMPLATE, SCHEMA_TABLE) - -MIRRORLIST_TEMPLATE = r""" - (=mirrorlist - (=ts .TIME) - (=mirrors (:anyof - (=mirror ((:unordered (=name .) (=urlbase .) (=contents (:someof .)) - .ATTRS))))) - *) -""" - -MIRRORLIST_SCHEMA = _parseSchema(MIRRORLIST_TEMPLATE, SCHEMA_TABLE) - -TIMESTAMP_TEMPLATE = r""" - (=ts - ((:unordered (=at .TIME) (=m .TIME .) (=k .TIME .) - (:anyof (=b . . .TIME . .)) .ATTRS)) - )""" - -TIMESTAMP_SCHEMA = _parseSchema(TIMESTAMP_TEMPLATE, SCHEMA_TABLE) - -BUNDLE_TEMPLATE = r""" - (=bundle - (=at .TIME) - (=os .) - (:maybe (=arch .)) - (=packages - (:someof - (. . . . ((:unordered - (:maybe (=order . . .)) - (:maybe (=optional)) - (:anyof (=gloss . .)) - (:anyof (=longgloss . .)) - .ATTRS))) - ) +def parseTime(s): + """Parse a time 's' in YYYY-MM-DD HH:MM:SS format.""" + try: + return calendar.timegm(time.strptime(s, "%Y-%m-%d %H:%M:%S")) + except ValueError: + raise glider.FormatError("Malformed time %r", s) + +def formatBase64(h): + """Return the base64 encoding of h with whitespace and = signs omitted.""" + return binascii.b2a_base64(h).rstrip("=\n ") + +formatHash = formatBase64 + +def parseBase64(s): + """Parse a base64 encoding with whitespace and = signs omitted. """ + extra = len(s) % 4 + if extra: + padding = "=" * (4 - extra) + s += padding + try: + return binascii.a2b_base64(s) + except binascii.Error: + raise glider.FormatError("Invalid base64 encoding") + +def parseHash(s): + h = parseBase64(s) + if len(h) != Crypto.Hash.SHA256.digest_size: + raise glider.FormatError("Bad hash length") + return h + +S = glider.checkJson + +# A date, in YYYY-MM-DD HH:MM:SS format. +TIME_SCHEMA = S.RE(r'\d{4}-\d{2}-\d{2} \d{2}:\d{2}:\d{2}') +# A hash, base64-encoded +HASH_SCHEMA = S.RE(r'[a-zA-Z0-9\+\/]{43}') + +# A hexadecimal value. +HEX_SCHEMA = S.RE(r'[a-fA-F0-9]+') +# A base-64 encoded value +BASE64_SCHEMA = S.RE(r'[a-zA-Z0-9\+\/]+') +# An RSA key; subtype of PUBKEY_SCHEMA. +RSAKEY_SCHEMA = S.Obj( + _keytype=S.Str("rsa"), + e=BASE64_SCHEMA, + n=BASE64_SCHEMA) +# Any public key. +PUBKEY_SCHEMA = S.Obj( + _keytype=S.AnyStr()) + +KEYID_SCHEMA = HASH_SCHEMA +SIG_METHOD_SCHEMA = S.AnyStr() +RELPATH_SCHEMA = PATH_PATTERN_SCHEMA = S.AnyStr() +URL_SCHEMA = S.AnyStr() +VERSION_SCHEMA = S.ListOf(S.Any()) #XXXX WRONG + +# A single signature of an object. Indicates the signature, the id of the +# signing key, and the signing method. +SIGNATURE_SCHEMA = S.Obj( + keyid=KEYID_SCHEMA, + method=SIG_METHOD_SCHEMA, + sig=BASE64_SCHEMA) + +# A signed object. +SIGNED_SCHEMA = S.Obj( + signed=S.Any(), + signatures=S.ListOf(SIGNATURE_SCHEMA)) + +ROLENAME_SCHEMA = S.AnyStr() + +# A role: indicates that a key is allowed to certify a kind of +# document at a certain place in the repo. +ROLE_SCHEMA = S.Struct([ROLENAME_SCHEMA, PATH_PATTERN_SCHEMA]) + +# A Keylist: indicates a list of live keys and their roles. +KEYLIST_SCHEMA = S.Obj( + _type=S.Str("Keylist"), + ts=TIME_SCHEMA, + keys=S.ListOf(S.Obj(key=PUBKEY_SCHEMA, roles=S.ListOf(ROLE_SCHEMA)))) + +# A Mirrorlist: indicates all the live mirrors, and what documents they +# serve. +MIRRORLIST_SCHEMA = S.Obj( + _type=S.Str("Mirrorlist"), + ts=TIME_SCHEMA, + mirrors=S.ListOf(S.Obj(name=S.AnyStr(), + urlbase=URL_SCHEMA, + contents=S.ListOf(PATH_PATTERN_SCHEMA), + weight=S.Int(lo=0), + ))) + +# A timestamp: indicates the lastest versions of all top-level signed objects. +TIMESTAMP_SCHEMA = S.Obj( + _type = S.Str("Timestamp"), + at = TIME_SCHEMA, + m = S.Struct([TIME_SCHEMA, HASH_SCHEMA]), + k = S.Struct([TIME_SCHEMA, HASH_SCHEMA]), + b = S.DictOf(keySchema=S.AnyStr(), + valSchema= + S.Struct([ VERSION_SCHEMA, RELPATH_SCHEMA, TIME_SCHEMA, HASH_SCHEMA ])) ) - * - )""" - -BUNDLE_SCHEMA = _parseSchema(BUNDLE_TEMPLATE, SCHEMA_TABLE) - -PACKAGE_TEMPLATE = r""" - (=package - ((:unordered (=name .) - (=version .) - (=format . (.ATTRS)) - (=path .) - (=ts .TIME) - (=digest .) - (:anyof (=shortdesc . .)) - (:anyof (=longdesc . .)) - .ATTRS))) -""" - -PACKAGE_SCHEMA = _parseSchema(PACKAGE_TEMPLATE, SCHEMA_TABLE) + +# A Bundle: lists a bunch of packages that should be updated in tandem +BUNDLE_SCHEMA = S.Obj( + _type=S.Str("Bundle"), + at=TIME_SCHEMA, + name=S.AnyStr(), + os=S.AnyStr(), + arch=S.Opt(S.AnyStr()), + version=VERSION_SCHEMA, + location=RELPATH_SCHEMA, + packages=S.ListOf(S.Obj( + name=S.AnyStr(), + version=VERSION_SCHEMA, + path=RELPATH_SCHEMA, + hash=HASH_SCHEMA, + order=S.Struct([S.Int(), S.Int(), S.Int()]), + optional=S.Opt(S.Bool()), + gloss=S.DictOf(S.AnyStr(), S.AnyStr()), + longgloss=S.DictOf(S.AnyStr(), S.AnyStr())))) + +PACKAGE_SCHEMA = S.Obj( + _type=S.Str("Package"), + name=S.AnyStr(), + location=RELPATH_SCHEMA, + version=VERSION_SCHEMA, + format=S.Obj(), + ts=TIME_SCHEMA, + files=S.ListOf(S.Struct([RELPATH_SCHEMA, HASH_SCHEMA])), + shortdesc=S.DictOf(S.AnyStr(), S.AnyStr()), + longdesc=S.DictOf(S.AnyStr(), S.AnyStr())) ALL_ROLES = ('timestamp', 'mirrors', 'bundle', 'package', 'master') class Key: - def __init__(self, key, roles): + #XXXX UNUSED. + def __init__(self, key, roles=()): self.key = key self.roles = [] for r,p in roles: @@ -215,15 +415,18 @@ class Key: def addRole(self, role, path): assert role in ALL_ROLES - self.roles.append(role, path) + self.roles.append((role, path)) def getRoles(self): - return self.rules + return self.roles @staticmethod - def fromSExpression(sexpr): + def fromJSon(obj): # must match PUBKEY_SCHEMA - typeattr = sexp.access.s_attr(sexpr[1], "type") + keytype = obj['_keytype'] + if keytype == 'rsa': + return Key(glider.keys.RSAKey.fromJSon(obj)) + if typeattr == 'rsa': key = glider.keys.RSAKey.fromSExpression(sexpr) if key is not None: @@ -240,29 +443,27 @@ class Key: def sign(self, sexpr=None, digest=None): return self.key.sign(sexpr, digest=digest) - def checkSignature(self, method, sexpr=None, digest=None): - if digest == None: - _, digest = self.key._digest(sexpr, method) - ok = self.key.checkSignature(method, digest=digest) + def checkSignature(self, method, data, signatute): + ok = self.key.checkSignature(method, data, signature) # XXXX CACHE HERE. return ok -class Keystore(KeyDB): +class Keylist(KeyDB): def __init__(self): KeyDB.__init__(self) - @staticmethod - def addFromKeylist(sexpr, allowMasterKeys=False): - # Don't do this until we have validated the structure. - for ks in sexpr.access.s_lookup_all("keys.key"): - attrs = ks[1] - key_s = ks[2] - roles = s_attr(attrs, "roles") - #XXXX Use interface of Key, not RSAKey. - key = Key.fromSExpression(key_s) - if not key: + def addFromKeylist(self, obj, allowMasterKeys=False): + for keyitem in obj['keys']: + key = keyitem['key'] + roles = keyitem['roles'] + + try: + key = glider.keys.RSAKey.fromJSon(key) + except glider.FormatException, e: + print e #LOG skipping key. continue + for r,p in roles: if r == 'master' and not allowMasterKeys: #LOG @@ -273,4 +474,274 @@ class Keystore(KeyDB): self.addKey(key) +class StampedInfo: + def __init__(self, ts, hash, version=None, relpath=None): + self._ts = ts + self._hash = hash + self._version = version + self._relpath = relpath + + @staticmethod + def fromJSonFields(timeStr, hashStr): + t = parseTime(timeStr) + h = parseHash(hashStr) + return StampedInfo(t, h) + + def getHash(self): + return self._hash + + def getRelativePath(self): + return self._relpath + +class TimestampFile: + def __init__(self, at, mirrorlistinfo, keylistinfo, bundleinfo): + self._time = at + self._mirrorListInfo = mirrorlistinfo + self._keyListInfo = keylistinfo + self._bundleInfo = bundleinfo + + @staticmethod + def fromJSon(obj): + # must be validated. + at = parseTime(obj['at']) + m = StampedInfo.fromJSonFields(*obj['m'][:2]) + k = StampedInfo.fromJSonFields(*obj['k'][:2]) + b = {} + for name, bundle in obj['b'].iteritems(): + v = bundle[0] + rp = bundle[1] + t = parseTime(bundle[2]) + h = parseHash(bundle[3]) + b[name] = StampedInfo(t, h, v, rp) + + return TimestampFile(at, m, k, b) + + def getTime(self): + return self._time + + def getMirrorlistInfo(self): + return self._mirrorListInfo + + def getKeylistInfo(self): + return self._keyListInfo + + def getBundleInfo(self, name): + return self._bundleInfo[name] + +def readConfigFile(fname, needKeys=(), optKeys=(), preload={}): + parsed = preload.copy() + result = {} + execfile(fname, parsed) + + for k in needKeys: + try: + result[k] = parsed[k] + except KeyError: + raise glider.FormatError("Missing value for %s in %s"%k,fname) + + for k in optKeys: + try: + result[k] = parsed[k] + except KeyError: + pass + + return result + +def makePackageObj(config_fname, package_fname): + preload = {} + shortDescs = {} + longDescs = {} + def ShortDesc(lang, val): shortDescs[lang] = val + def LongDesc(lang, val): longDescs[lang] = val + preload = { 'ShortDesc' : ShortDesc, 'LongDesc' : LongDesc } + r = readConfigFile(config_fname, + ['name', + 'version', + 'format', + 'location', + 'relpath', + ], (), preload) + + f = open(package_fname, 'rb') + digest = getFileDigest(f) + + # Check fields! + result = { '_type' : "Package", + 'ts' : formatTime(time.time()), + 'name' : r['name'], + 'location' : r['location'], #DOCDOC + 'version' : r['version'], + 'format' : r['format'], + 'files' : [ [ r['relpath'], formatHash(digest) ] ], + 'shortdesc' : shortDescs, + 'longdesc' : longDescs + } + + PACKAGE_SCHEMA.checkMatch(result) + + return result + +def makeBundleObj(config_fname, getPackageHash): + packages = [] + def ShortGloss(lang, val): packages[-1]['gloss'][lang] = val + def LongGloss(lang, val): packages[-1]['longgloss'][lang] = val + def Package(name, version, path, order, optional=False): + packages.append({'name' : name, + 'version' : version, + 'path' : path, + 'order' : order, + 'optional' : optional, + 'gloss' : {}, + 'longgloss' : {} }) + preload = { 'ShortGloss' : ShortGloss, 'LongGloss' : LongGloss, + 'Package' : Package } + r = readConfigFile(config_fname, + ['name', + 'os', + 'version', + 'location', + ], ['arch'], preload) + + result = { '_type' : "Bundle", + 'at' : formatTime(time.time()), + 'name' : r['name'], + 'os' : r['os'], + 'version' : r['version'], + 'location' : r['location'], + 'packages' : packages } + if r.has_key('arch'): + result['arch'] = r['arch'] + + for p in packages: + try: + p['hash'] = formatHash(getPackageHash(p['path'])) + except KeyError: + raise glider.FormatException("No such package as %s"%p['path']) + + BUNDLE_SCHEMA.checkMatch(result) + return result + +def versionIsNewer(v1, v2): + return v1 > v2 + +def makeTimestampObj(mirrorlist_obj, keylist_obj, + bundle_objs): + result = { '_type' : 'Timestamp', + 'at' : formatTime(time.time()) } + result['m'] = [ mirrorlist_obj['ts'], + formatHash(getDigest(mirrorlist_obj)) ] + result['k'] = [ keylist_obj['ts'], + formatHash(getDigest(keylist_obj)) ] + result['b'] = bundles = {} + for bundle in bundle_objs: + name = bundle['name'] + v = bundle['version'] + entry = [ v, bundle['location'], bundle['at'], formatHash(getDigest(bundle)) ] + if not bundles.has_key(name) or versionIsNewer(v, bundles[name][0]): + bundles[name] = entry + + TIMESTAMP_SCHEMA.checkMatch(result) + + return result + +class MirrorInfo: + def __init__(self, name, urlbase, contents, weight): + self._name = name + self._urlbase = urlbase + self._contents = contents + self._weight = weight + + def canServeFile(self, fname): + for c in self._contents: + if rolePathMatches(c, fname): + return True + return False + + def getFileURL(self, fname): + if self._urlbase[-1] == '/': + return self._urlbase+fname + else: + return "%s/%s" % (self._urlbase, fname) + def format(self): + return { 'name' : self._name, + 'urlbase' : self._urlbase, + 'contents' : self._contents, + 'weight' : self._weight } + +def makeMirrorListObj(mirror_fname): + mirrors = [] + def Mirror(*a, **kw): mirrors.append(MirrorInfo(*a, **kw)) + preload = {'Mirror' : Mirror} + r = readConfigFile(mirror_fname, (), (), preload) + result = { '_type' : "Mirrorlist", + 'ts' : formatTime(time.time()), + 'mirrors' : [ m.format() for m in mirrors ] } + + MIRRORLIST_SCHEMA.checkMatch(result) + return result + +def makeKeylistObj(keylist_fname, includePrivate=False): + keys = [] + def Key(obj): keys.append(obj) + preload = {'Key': Key} + r = readConfigFile(keylist_fname, (), (), preload) + + klist = [] + for k in keys: + k = glider.keys.RSAKey.fromJSon(k) + klist.append({'key': k.format(private=includePrivate), 'roles' : k.getRoles() }) + + result = { '_type' : "Keylist", + 'ts' : formatTime(time.time()), + 'keys' : klist } + + KEYLIST_SCHEMA.checkMatch(result) + return result + +SCHEMAS_BY_TYPE = { + 'Keylist' : KEYLIST_SCHEMA, + 'Mirrorlist' : MIRRORLIST_SCHEMA, + 'Timestamp' : TIMESTAMP_SCHEMA, + 'Bundle' : BUNDLE_SCHEMA, + 'Package' : PACKAGE_SCHEMA, + } + +def checkSignedObj(obj, keydb=None): + # Returns signaturestatus, role, path on sucess. + + SIGNED_SCHEMA.checkMatch(obj) + try: + tp = obj['signed']['_type'] + except KeyError: + raise glider.FormatException("Untyped object") + try: + schema = SCHEMAS_BY_TYPE[tp] + except KeyError: + raise glider.FormatException("Unrecognized type %r" % tp) + schema.checkMatch(obj['signed']) + + if tp == 'Keylist': + role = "master" + path = "/meta/keys.txt" + elif tp == 'Mirrorlist': + role = "mirrors" + path = "/meta/mirrors.txt" + elif tp == "Timestamp": + role = 'timestamp' + path = "/meta/timestamp.txt" + elif tp == 'Bundle': + role = 'bundle' + path = obj['signed']['location'] + elif tp == 'Package': + role = 'package' + path = obj['signed']['location'] + else: + print tp + raise "Foo" + + ss = None + if keydb is not None: + ss = checkSignatures(obj, keydb, role, path) + + return ss, role, path diff --git a/lib/glider/keys.py b/lib/glider/keys.py index e2b2736..fefff6c 100644 --- a/lib/glider/keys.py +++ b/lib/glider/keys.py @@ -4,25 +4,23 @@ import Crypto.PublicKey.RSA import Crypto.Hash.SHA256 import Crypto.Cipher.AES -import sexp.access -import sexp.encode -import sexp.parse - import cPickle as pickle import binascii +import logging import os import struct +import sys +import simplejson +import getpass -class CryptoError(Exception): - pass - -class PubkeyFormatException(Exception): - pass - -class UnknownMethod(Exception): - pass +import glider.formats +import glider.util class PublicKey: + def __init__(self): + # Confusingly, these roles are the ones used for a private key to + # remember what we're willing to do with it. + self._roles = [] def format(self): raise NotImplemented() def sign(self, data): @@ -34,7 +32,17 @@ class PublicKey: def getKeyID(self): raise NotImplemented() def getRoles(self): - raise NotImplemented() + return self._roles + def addRole(self, role, path): + assert role in glider.formats.ALL_ROLES + self._roles.append((role, path)) + def clearRoles(self): + del self._roles[:] + def hasRole(self, role, path): + for r, p in self._roles: + if r == role and glider.formats.rolePathMatches(p, path): + return True + return False if hex(1L).upper() == "0X1L": def intToBinary(number): @@ -64,8 +72,13 @@ def binaryToInt(binary): """ return long(binascii.b2a_hex(binary), 16) -def _pkcs1_padding(m, size): +def intToBase64(number): + return glider.formats.formatBase64(intToBinary(number)) + +def base64ToInt(number): + return binaryToInt(glider.formats.parseBase64(number)) +def _pkcs1_padding(m, size): # I'd rather use OAEP+, but apparently PyCrypto barely supports # signature verification, and doesn't seem to support signature # verification with nondeterministic padding. "argh." @@ -83,25 +96,28 @@ def _xor(a,b): class RSAKey(PublicKey): """ >>> k = RSAKey.generate(bits=512) - >>> sexpr = k.format() - >>> sexpr[:2] - ('pubkey', [('type', 'rsa')]) - >>> k1 = RSAKey.fromSExpression(sexpr) + >>> obj = k.format() + >>> obj['_keytype'] + 'rsa' + >>> base64ToInt(obj['e']) + 65537L + >>> k1 = RSAKey.fromJSon(obj) >>> k1.key.e == k.key.e True >>> k1.key.n == k.key.n True >>> k.getKeyID() == k1.getKeyID() True - >>> s = ['tag1', ['foobar'], [['foop', 'bar']], 'baz'] - >>> method, sig = k.sign(sexpr=s) - >>> k.checkSignature(method, sig, sexpr=s) + >>> s = { 'A B C' : "D", "E" : [ "F", "g", 99] } + >>> method, sig = k.sign(obj=s) + >>> k.checkSignature(method, sig, obj=s) True >>> s2 = [ s ] - >>> k.checkSignature(method, sig, sexpr=s2) + >>> k.checkSignature(method, sig, obj=s2) False """ def __init__(self, key): + PublicKey.__init__(self) self.key = key self.keyid = None @@ -111,58 +127,79 @@ class RSAKey(PublicKey): return RSAKey(key) @staticmethod - def fromSExpression(sexpr): - # sexpr must match PUBKEY_SCHEMA - typeattr = sexp.access.s_attr(sexpr[1], "type") - if typeattr != "rsa": - return None - if len(sexpr[2]) != 2: - raise PubkeyFormatException("RSA keys must have an e,n pair") - e,n = sexpr[2] - key = Crypto.PublicKey.RSA.construct((binaryToInt(n), binaryToInt(e))) - return RSAKey(key) - - def format(self): - n = intToBinary(self.key.n) - e = intToBinary(self.key.e) - return ("pubkey", [("type", "rsa")], (e, n)) + def fromJSon(obj): + # obj must match RSAKEY_SCHEMA + + glider.formats.RSAKEY_SCHEMA.checkMatch(obj) + n = base64ToInt(obj['n']) + e = base64ToInt(obj['e']) + if obj.has_key('d'): + d = base64ToInt(obj['d']) + p = base64ToInt(obj['p']) + q = base64ToInt(obj['q']) + u = base64ToInt(obj['u']) + key = Crypto.PublicKey.RSA.construct((n, e, d, p, q, u)) + else: + key = Crypto.PublicKey.RSA.construct((n, e)) + + result = RSAKey(key) + if obj.has_key('roles'): + for r, p in obj['roles']: + result.addRole(r,p) + + return result + + def isPrivateKey(self): + return hasattr(self.key, 'd') + + def format(self, private=False, includeRoles=False): + n = intToBase64(self.key.n) + e = intToBase64(self.key.e) + result = { '_keytype' : 'rsa', + 'e' : e, + 'n' : n } + if private: + result['d'] = intToBase64(self.key.d) + result['p'] = intToBase64(self.key.p) + result['q'] = intToBase64(self.key.q) + result['u'] = intToBase64(self.key.u) + if includeRoles: + result['roles'] = self.getRoles() + return result def getKeyID(self): if self.keyid == None: - n = intToBinary(self.key.n) - e = intToBinary(self.key.e) - keyval = (e,n) d_obj = Crypto.Hash.SHA256.new() - sexp.encode.hash_canonical(keyval, d_obj) - self.keyid = ("rsa", d_obj.digest()) + glider.formats.getDigest(self.format(), d_obj) + self.keyid = glider.formats.formatHash(d_obj.digest()) return self.keyid - def _digest(self, sexpr, method=None): + def _digest(self, obj, method=None): if method in (None, "sha256-pkcs1"): d_obj = Crypto.Hash.SHA256.new() - sexp.encode.hash_canonical(sexpr, d_obj) + glider.formats.getDigest(obj, d_obj) digest = d_obj.digest() return ("sha256-pkcs1", digest) raise UnknownMethod(method) - def sign(self, sexpr=None, digest=None): - assert _xor(sexpr == None, digest == None) + def sign(self, obj=None, digest=None): + assert _xor(obj == None, digest == None) if digest == None: - method, digest = self._digest(sexpr) + method, digest = self._digest(obj) m = _pkcs1_padding(digest, (self.key.size()+1) // 8) - sig = intToBinary(self.key.sign(m, "")[0]) + sig = intToBase64(self.key.sign(m, "")[0]) return (method, sig) - def checkSignature(self, method, sig, sexpr=None, digest=None): - assert _xor(sexpr == None, digest == None) + def checkSignature(self, method, sig, obj=None, digest=None): + assert _xor(obj == None, digest == None) if method != "sha256-pkcs1": raise UnknownMethod("method") if digest == None: - method, digest = self._digest(sexpr, method) - sig = binaryToInt(sig) + method, digest = self._digest(obj, method) + sig = base64ToInt(sig) m = _pkcs1_padding(digest, (self.key.size()+1) // 8) - return self.key.verify(m, (sig,)) + return bool(self.key.verify(m, (sig,))) SALTLEN=16 @@ -244,15 +281,21 @@ def encryptSecret(secret, password, difficulty=0x80): pad = '\x00' * padlen slen = struct.pack("!L",len(secret)) - encrypted = e.encrypt("%s%s%s%s" % (slen, secret, d, pad))[:-padlen] + encrypted = e.encrypt("%s%s%s%s" % (slen, secret, d, pad)) + if padlen: + encrypted = encrypted[:-padlen] return "GKEY1%s%s%s"%(salt, iv, encrypted) def decryptSecret(encrypted, password): + """Decrypt a value encrypted with encryptSecret. Raises UnknownFormat + or FormatError if 'encrypted' was not generated with encryptSecret. + Raises BadPassword if the password was not correct. + """ if encrypted[:5] != "GKEY1": - raise UnknownFormat() + raise glider.UnknownFormat() encrypted = encrypted[5:] if len(encrypted) < SALTLEN+1+16: - raise FormatError() + raise glider.FormatException() salt = encrypted[:SALTLEN+1] iv = encrypted[SALTLEN+1:SALTLEN+1+16] @@ -276,8 +319,81 @@ def decryptSecret(encrypted, password): d.update(salt) if d.digest() != hash: - print repr(decrypted) - raise BadPassword() + raise glider.BadPassword() return secret +class KeyStore(glider.formats.KeyDB): + def __init__(self, fname, encrypted=True): + glider.formats.KeyDB.__init__(self) + + self._loaded = None + self._fname = fname + self._passwd = None + self._encrypted = encrypted + + def getpass(self, reprompt=False): + if self._passwd != None: + return self._passwd + while 1: + pwd = getpass.getpass("Password: ", sys.stderr) + if not reprompt: + return pwd + + pwd2 = getpass.getpass("Confirm: ", sys.stderr) + if pwd == pwd2: + return pwd + else: + print "Mismatch; try again." + + def load(self, password=None): + logging.info("Loading private keys from %r...", self._fname) + if not os.path.exists(self._fname): + logging.info("...no such file.") + self._loaded = True + return + + if password is None and self._encrypted: + password = self.getpass() + + contents = open(self._fname, 'rb').read() + if self._encrypted: + contents = decryptSecret(contents, password) + + listOfKeys = simplejson.loads(contents) + self._passwd = password # It worked. + if not listOfKeys.has_key('keys'): + listOfKeys['keys'] = [] + for obj in listOfKeys['keys']: + key = RSAKey.fromJSon(obj) + self.addKey(key) + logging.info("Loaded key %s", key.getKeyID()) + + self._loaded = True + + def setPassword(self, passwd): + self._passwd = passwd + + def clearPassword(self): + self._passwd = None + + def save(self, password=None): + if not self._loaded and self._encrypted: + self.load(password) + + if password is None: + password = self.getpass(True) + + logging.info("Saving private keys into %r...", self._fname) + listOfKeys = { 'keys' : + [ key.format(private=True, includeRoles=True) for key in + self._keys.values() ] + } + contents = simplejson.dumps(listOfKeys) + if self._encrypted: + contents = encryptSecret(contents, password) + glider.util.replaceFile(self._fname, contents) + self._passwd = password # It worked. + logging.info("Done.") + + diff --git a/lib/glider/master_keys.py b/lib/glider/master_keys.py new file mode 100644 index 0000000..0d455d1 --- /dev/null +++ b/lib/glider/master_keys.py @@ -0,0 +1,5 @@ + + +MASTER_KEYS = [ + +] diff --git a/lib/glider/repository.py b/lib/glider/repository.py index 9ec1dc6..b956993 100644 --- a/lib/glider/repository.py +++ b/lib/glider/repository.py @@ -1,10 +1,14 @@ -import sexp.parse -import sexp.access import glider.formats +import glider.util +import simplejson +import logging import os import threading +import time + +MAX_TIMESTAMP_AGE = 24*60*60 class RepositoryFile: def __init__(self, repository, relativePath, schema, @@ -16,12 +20,15 @@ class RepositoryFile: self._signedFormat = signedFormat self._needSigs = needSigs - self._signed_sexpr = None - self._main_sexpr = None + self._signed_obj = self._main_obj = None + self._sigStatus = None self._mtime = None + def getRelativePath(self): + return self._relativePath + def getPath(self): - return os.path.join(self._repository._root, self._relativePath) + return self._repository.getFilename(self._relativePath) def _load(self): fname = self.getPath() @@ -40,82 +47,267 @@ class RepositoryFile: finally: f.close() - signed_sexpr,main_sexpr = self._checkContent(content) + signed_obj,main_obj = self._checkContent(content) - self._signed_sexpr = signed_sexpr - self._main_sexpr = main_sexpr + self._signed_obj = signed_obj + self._main_obj = main_obj self._mtime = mtime def _save(self, content=None): if content == None: content = sexpr.encode - signed_sexpr,main_sexpr = self._checkContent(content) + signed_obj,main_obj = self._checkContent(content) fname = self.getPath() - fname_tmp = fname+"_tmp" + glider.util.replaceFile(fname, contents) - fd = os.open(fname_tmp, os.WRONLY|os.O_CREAT|os.O_TRUNC, 0644) - try: - os.write(fd, contents) - finally: - os.close(fd) - if sys.platform in ('cygwin', 'win32'): - # Win32 doesn't let rename replace an existing file. - try: - os.unlink(fname) - except OSError: - pass - os.rename(fname_tmp, fname) - - self._signed_sexpr = signed_sexpr - self._main_sexpr = main_sexpr + self._signed_obj = signed_obj + self._main_obj = main_obj self._mtime = mtime def _checkContent(self, content): - sexpr = sexp.parse.parse(content) - if not sexpr: - raise ParseError() + + try: + obj = simplejson.loads(content) + except ValueError, e: + raise glider.FormatException("Couldn't decode content: %s"%e) if self._signedFormat: - if not glider.formats.SIGNED_SCHEMA.matches(sexpr): - raise FormatError() - - sigs = checkSignatures(sexpr, self._repository._keyDB, - self._needRole, self._relativePath) - good = sigs[0] - # XXXX If good is too low but unknown is high, we may need - # a new key file. - if len(good) < 1: - raise SignatureError() - - main_sexpr = sexpr[1] - signed_sexpr = sexpr + # This is supposed to be signed. + glider.formats.SIGNED_SCHEMA.checkMatch(obj) + + main_obj = obj['signed'] + signed_obj = obj else: - signed_sexpr = None - main_sexpr = sexpr + signed_obj = None + main_obj = obj - if self._schema != None and not self._schema.matches(main_sexpr): - raise FormatError() + if self._schema != None: + self._schema.checkMatch(main_obj) - return signed_sexpr, main_sexpr + return signed_obj, main_obj def load(self): - if self._main_sexpr == None: + if self._main_obj == None: self._load() + def get(self): + return self._main_obj + + def isLoaded(self): + return self._main_obj != None + + def getContent(self): + self.load() + return self._main_obj + + def _checkSignatures(self): + self.load() + sigStatus = glider.formats.checkSignatures(self._signed_obj, + self._repository._keyDB, + self._needRole, self._relativePath) + self._sigStatus = sigStatus + + def checkSignatures(self): + if self._sigStatus is None: + self._checkSignatures() + return self._sigStatus + class LocalRepository: def __init__(self, root): self._root = root - self._keyDB = None + self._keyDB = glider.util.getKeylist(None) self._keylistFile = RepositoryFile( - self, "meta/keys.txt", glider.formats.KEYLIST_SCHEMA, + self, "/meta/keys.txt", glider.formats.KEYLIST_SCHEMA, needRole="master") self._timestampFile = RepositoryFile( - self, "meta/timestamp.txt", glider.formats.TIMESTAMP_SCHEMA, + self, "/meta/timestamp.txt", glider.formats.TIMESTAMP_SCHEMA, needRole="timestamp") self._mirrorlistFile = RepositoryFile( - self, "meta/mirrors.txt", glider.formats.MIRRORLIST_SCHEMA, + self, "/meta/mirrors.txt", glider.formats.MIRRORLIST_SCHEMA, needRole="mirrors") + self._metaFiles = [ self._keylistFile, + self._timestampFile, + self._mirrorlistFile ] + + self._packageFiles = {} + self._bundleFiles = {} + + def getFilename(self, relativePath): + if relativePath.startswith("/"): + relativePath = relativePath[1:] + return os.path.join(self._root, relativePath) + + def getKeylistFile(self): + return self._keylistFile + + def getTimestampFile(self): + return self._timestampFile + + def getMirrorlistFile(self): + return self._mirrorlistFile + + def getPackageFile(self, relPath): + try: + return self._packageFiles[relPath] + except KeyError: + self._packageFiles[relPath] = pkg = RepositoryFile( + self, relPath, glider.formats.PACKAGE_SCHEMA, + needRole='package') + return pkg + + def getBundleFile(self, relPath): + try: + return self._bundleFiles[relPath] + except KeyError: + self._bundleFiles[relPath] = pkg = RepositoryFile( + self, relPath, glider.formats.BUNDLE_SCHEMA, + needRole='bundle') + return pkg + + def getFilesToUpdate(self, now=None, trackingBundles=()): + if now == None: + now = time.time() + + need = set() + + # Fetch missing metafiles. + for f in self._metaFiles: + try: + f.load() + except OSError, e: + print "need", f.getPath() + logging.info("Couldn't load %s: %s. Must fetch it.", + f.getPath(), e) + need.add(f.getRelativePath()) + + # If the timestamp file is out of date, we need to fetch it no + # matter what. (Even if it is isn't signed, it can't possibly + # be good.) + ts = self._timestampFile.get() + if ts: + age = now - glider.formats.parseTime(ts['at']) + ts = glider.formats.TimestampFile.fromJSon(ts) + if age > MAX_TIMESTAMP_AGE: + need.add(self._timestampFile.getRelativePath()) + + # If the keylist isn't signed right, we can't check the + # signatures on anything else. + if self._keylistFile.get(): + s = self._keylistFile.checkSignatures() + if not s.isValid(): # For now only require one master key. + need.add(self._keylistFile.getRelativePath()) + + if need: + return need + + # Import the keys from the keylist. + self._keyDB.addFromKeylist(self._keylistFile.get()) + + # If the timestamp isn't signed right, get a new timestamp and a + # new keylist. + s = self._timestampFile.checkSignatures() + if not s.isValid(): + need.add(self._keylistFile.getRelativePath()) + need.add(self._timestampFile.getRelativePath()) + return need + + # FINALLY, we know we have an up-to-date, signed timestamp + # file. Check whether the keys and mirrors file are as + # authenticated. + h_kf = glider.formats.getDigest(self._keylistFile.get()) + h_expected = ts.getKeylistInfo().getHash() + if h_kf != h_expected: + need.add(self._keylistFile.getRelativePath()) + + if need: + return need + + s = self._mirrorlistFile.checkSignatures() + if not s.isValid(): + need.add(self._mirrorlistFile.getRelativePath()) + + h_mf = glider.formats.getDigest(self._mirrorlistFile.get()) + h_expected = ts.getMirrorlistInfo().getHash() + if h_mf != h_expected: + need.add(self._mirrorlistFile.getRelativePath()) + + if need: + return need + + # Okay; that's it for the metadata. Do we have the right + # bundles? + bundles = {} + for b in trackingBundles: + try: + binfo = ts.getBundleInfo(b) + except KeyError: + logging.warn("Unrecognized bundle %s"%b) + continue + + rp = binfo.getRelativePath() + bfile = self.getBundleFile(rp) + try: + bfile.load() + except OSError: + need.add(rp) + continue + + h_b = glider.formats.getDigest(bfile.get()) + h_expected = binfo.getHash() + if h_b != h_expected: + need.add(rp) + continue + + s = bfile.checkSignatures() + if not s.isValid(): + # Can't actually use it. + continue + + bundles[rp] = bfile + + # Okay. So we have some bundles. See if we have their packages. + packages = {} + for bfile in bundles.values(): + bundle = bfile.get() + for pkginfo in bundle['packages']: + rp = pkginfo['path'] + pfile = self.getPackageFile(rp) + try: + pfile.load() + except OSError: + need.add(rp) + continue + + h_p = glider.formats.getDigest(pfile.get()) + h_expected = glider.formats.parseHash(pkginfo['hash']) + if h_p != h_expected: + need.add(rp) + continue + + s = pfile.checkSignatures() + if not s.isValid(): + # Can't use it. + continue + packages[rp] = pfile + + # Finally, we have some packages. Do we have their underlying + # files? + for pfile in packages.values(): + package = pfile.get() + for f in package['files']: + rp, h = f[:2] + h_expected = glider.formats.parseHash(h) + fn = self.getFilename(rp) + try: + h_got = glider.formats.getFileDigest(fn) + except OSError: + need.add(rp) + continue + if h_got != h_expected: + need.add(rp) + # Okay; these are the files we need. + return need diff --git a/lib/glider/tests.py b/lib/glider/tests.py index 1c392e8..7aeb4f6 100644 --- a/lib/glider/tests.py +++ b/lib/glider/tests.py @@ -1,21 +1,57 @@ import unittest import doctest +import os +import tempfile import glider.keys import glider.formats import glider.repository +import glider.checkJson import glider.tests -class EncryptionTest(unittest.TestCase): - pass +class CanonicalEncodingTest(unittest.TestCase): + def test_encode(self): + enc = glider.formats.encodeCanonical + self.assertEquals(enc(''), '""') + self.assertEquals(enc('"'), '"\\""') + self.assertEquals(enc('\t\\\n"\r'), + '"\t\\\\\n\\"\r"') + +class CryptoTests(unittest.TestCase): + def test_encrypt(self): + s = "The Secret words are marzipan habidashery zeugma." + password = "the password is swordfish." + encrypted = glider.keys.encryptSecret(s, password) + self.assertNotEquals(encrypted, s) + self.assert_(encrypted.startswith("GKEY1")) + self.assertEquals(s, glider.keys.decryptSecret(encrypted, password)) + self.assertRaises(glider.BadPassword, glider.keys.decryptSecret, + encrypted, "password") + self.assertRaises(glider.UnknownFormat, glider.keys.decryptSecret, + "foobar", password) + + def test_keystore(self): + passwd = "umfitty noonah" + fname = tempfile.mktemp() + ks = glider.keys.KeyStore(fname) + key1 = glider.keys.RSAKey.generate(512) + key2 = glider.keys.RSAKey.generate(512) + ks.addKey(key1) + ks.addKey(key2) + ks.save(passwd) + + ks2 = glider.keys.KeyStore(fname) + ks2.load(passwd) + self.assertEquals(key1.key.n, ks2.getKey(key1.getKeyID()).key.n) def suite(): suite = unittest.TestSuite() suite.addTest(doctest.DocTestSuite(glider.formats)) suite.addTest(doctest.DocTestSuite(glider.keys)) + suite.addTest(doctest.DocTestSuite(glider.checkJson)) loader = unittest.TestLoader() suite.addTest(loader.loadTestsFromModule(glider.tests)) diff --git a/lib/glider/util.py b/lib/glider/util.py new file mode 100644 index 0000000..e2fb57a --- /dev/null +++ b/lib/glider/util.py @@ -0,0 +1,73 @@ + +import os +import sys +import tempfile + +import simplejson + +import glider.formats +import glider.keys +import glider.master_keys + +def moveFile(fromLocation, toLocation): + if sys.platform in ('cygwin', 'win32'): + # Win32 doesn't let rename replace an existing file. + try: + os.unlink(toLocation) + except OSError: + pass + os.rename(fromLocation, toLocation) + + +def replaceFile(fname, contents, textMode=False): + """overwrite the file in 'fname' atomically with the content of 'contents' + """ + dir, prefix = os.path.split(fname) + fd, fname_tmp = tempfile.mkstemp(prefix=prefix, dir=dir, text=textMode) + + try: + os.write(fd, contents) + finally: + os.close(fd) + + moveFile(fname_tmp, fname) + +def userFilename(name): + try: + base = os.environ["THANDY_HOME"] + except KeyError: + base = "~/.thandy" + base = os.path.expanduser(base) + if not os.path.exists(base): + os.makedirs(base, 0700) + return os.path.join(base, name) + +def getKeylist(keys_fname, checkKeys=True): + import glider.master_keys + + keydb = glider.formats.Keylist() + + for key in glider.master_keys.MASTER_KEYS: + keydb.addKey(key) + + user_keys = userFilename("preload_keys") + if os.path.exists(user_keys): + #XXXX somewhat roundabout. + keylist = glider.formats.makeKeylistObj(user_keys) + keydb.addFromKeylist(keylist, allowMasterKeys=True) + + if keys_fname and os.path.exists(keys_fname): + f = open(keys_fname, 'r') + try: + obj = simplejson.load(f) + finally: + f.close() + ss, role, path = glider.formats.checkSignedObj(obj, keydb) + if role != 'master': + raise glider.FormatException("%s wasn't a keylist."%keys_fname) + if checkKeys and not ss.isValid(): + raise glider.FormatException("%s not signed by enough master keys"% + keys_fname) + keydb.addFromKeylist(obj['signed'], allowMasterKeys=False) + + return keydb diff --git a/samples/bundle.cfg b/samples/bundle.cfg new file mode 100644 index 0000000..d2c78b3 --- /dev/null +++ b/samples/bundle.cfg @@ -0,0 +1,15 @@ + +name = "example-bundle" +version = [0,1,10] +location = "/bundleinfo/example/os-arch/example-os-arch-0.1.10.txt" +os = "os" +arch = "arch" + +Package(name="example", + version=[0,1,2], + path="/pkginfo/example/rpm/example.txt", + order=(10,10,10), + optional=False) +ShortGloss("en", "Example package is needed to make the ossifrage squeamish.") +LongGloss("en", "You wouldn't want an unsqueamish ossifrage, would you?") + diff --git a/samples/example-mirrors.txt b/samples/example-mirrors.txt new file mode 100644 index 0000000..c02583e --- /dev/null +++ b/samples/example-mirrors.txt @@ -0,0 +1,5 @@ + +Mirror(name="moria", + urlbase="http://moria.seul.org/thandy-example/", + contents=[ "**" ], + weight=1) diff --git a/samples/example-package.cfg b/samples/example-package.cfg new file mode 100644 index 0000000..527211b --- /dev/null +++ b/samples/example-package.cfg @@ -0,0 +1,24 @@ + + +# This is package is name 'example' +name = "example" + +# Encodes version 0.1.2. +version = [0, 1, 2] + +# What kind of package is this? +format = "rpm" + +# Where in the repository does it go? +location = "/pkginfo/example/rpm/example.txt" + +# Where in the repository does its underlying rpm file go? +relpath = "/data/example-0.1.2.rpm" + +# Decriptions of the package. +ShortDesc('en', "Example package") +LongDesc('en', +"""This is an example package. + +Its description is not quite so long as it might be, but hey.""") + diff --git a/samples/testpackage.cfg b/samples/testpackage.cfg new file mode 100644 index 0000000..8b13789 --- /dev/null +++ b/samples/testpackage.cfg @@ -0,0 +1 @@ + diff --git a/specs/glider-spec.txt b/specs/glider-spec.txt index cf0bc49..b940e49 100644 --- a/specs/glider-spec.txt +++ b/specs/glider-spec.txt @@ -307,6 +307,14 @@ (signature ({(keyid K) (method M) (ATTR VAL)*}) SIG)+ ) + { "_type" : "Signed", + "signed" : X, + "sigatures" : [ + { "keyid" : K, + "method" : M, + ... + "sig" : S } ] + where: X is a list whose first element describes the signed object. K is the identifier of a key signing the document M is the method to be used to make the signature @@ -324,6 +332,11 @@ All keys are of the format: (pubkey ({(type TYPE) (ATTR VAL)*}) KEYVAL) + + { "_keytype" : TYPE, + ... + "keyval" : KEYVAL } + where TYPE is a string describing the type of the key and how it's used to sign documents. The type determines the interpretation of KEYVAL. @@ -336,6 +349,10 @@ binary format. [This makes keys 45-60% more compact than using decimal integers.] + {Values given as integers.} + + {'e' : e, 'n' : n, big-endian hex. } + All RSA keys must be at least 2048 bits long. @@ -370,6 +387,13 @@ ... ) + { "_type" : "Keylist", + "ts" : TIME, + "keys" : [ + { "roles" : [ [ ROLE, PATH ], ... ], + ... + "key" : KEY }, ... ] } + The "ts" line describes when the keys file was updated. Clients MUST NOT replace a file with an older one, and SHOULD NOT accept a file too far in the future. @@ -392,7 +416,18 @@ ( (mirror ({(name N) (urlbase U) (contents PATH+) (weight W) (official)? (ATTR VAL)})) * ) ... - ) + ) + + { "_type" : "Mirrorlist", + "mirrors" : [ + { "name" : N, + "urlbase" : U, + "contents" : [PATH ... ] , + "weight" : W, + "official" : BOOL, + ... + }, ... ] + } Every mirror is a copy of some or all of the directory hierarchy containing at least the /meta, /bundles/, and /pkginfo directories. @@ -417,12 +452,20 @@ ({(at TIME) (m TIME MIRRORLISTHASH) (k TIME KEYLISTHASH) - (b NAME VERSION TIME PATH HASH)*}) + (b NAME VERSION PATH TIME HASH)*}) ) + { "_type" : Timestamp, + "at" : TIME, + "m" : [ TIME, HASH ], + "k" : [ TIME, HASH ], + "b" : { NAME : + [ [ Version, Path, Time, Hash ] ] } + } + TIME is when the timestamp was signed. MIRRORLISTHASH is the digest of the mirror-list file; KEYLISTHASH is the digest of the key list - file; and the 'b' entries are a list of the latest version of each + file; and the 'b' entries are a list of the latest version of all bundles and their locations and hashes. 3.6. File formats: bundle files @@ -440,6 +483,23 @@ (ATTR VAL)*})? )* ) ) + { "_type" : "Bundle", + "name" : NAME, + "at" : TIME, + "os" : OS, + [ "arch" : ARCH, ] + "version" : V + "packages" : + [ { "name" : NAME, + "version" : VERSION, + "path" : PATH, + "hash" : HASH, + "order" : [ INST, UPDATE, REMOVE ], + [ "optional : BOOL, ] + "gloss" : { LANG : TEXT }, + "longgloss" : { LANG : TEXT }, + } ] } + Most elements are self-explanatory; the INST, UPDATE, and REMOVE elements of the order element are numbers defining the order in which the packages are installed, updated, and removed respectively. @@ -648,6 +708,3 @@ R.2. Integration with existing GPG signatures have to be mad to touch it. - - - -- cgit v1.2.3