From d90990ee3ecd09a2725b8051759a900ebd488b8c Mon Sep 17 00:00:00 2001 From: Nick Mathewson Date: Tue, 14 Oct 2008 05:10:30 +0000 Subject: Rename glider to thandy, based on discussions on #nottor. Please let me know ASAP if there is another program Thandy, or if it means something rude, or whatever. git-svn-id: file:///home/or/svnrepo/updater/trunk@17085 55e972cd-5a19-0410-ae62-a4d7a52db4cd --- lib/glider/ClientCLI.py | 66 ---- lib/glider/ServerCLI.py | 187 ------------ lib/glider/SignerCLI.py | 313 ------------------- lib/glider/__init__.py | 35 --- lib/glider/checkJson.py | 274 ----------------- lib/glider/download.py | 127 -------- lib/glider/formats.py | 747 ---------------------------------------------- lib/glider/keys.py | 399 ------------------------- lib/glider/master_keys.py | 5 - lib/glider/repository.py | 313 ------------------- lib/glider/tests.py | 64 ---- lib/glider/util.py | 73 ----- lib/thandy/ClientCLI.py | 66 ++++ lib/thandy/ServerCLI.py | 187 ++++++++++++ lib/thandy/SignerCLI.py | 313 +++++++++++++++++++ lib/thandy/__init__.py | 35 +++ lib/thandy/checkJson.py | 274 +++++++++++++++++ lib/thandy/download.py | 127 ++++++++ lib/thandy/formats.py | 747 ++++++++++++++++++++++++++++++++++++++++++++++ lib/thandy/keys.py | 399 +++++++++++++++++++++++++ lib/thandy/master_keys.py | 5 + lib/thandy/repository.py | 313 +++++++++++++++++++ lib/thandy/tests.py | 64 ++++ lib/thandy/util.py | 73 +++++ 24 files changed, 2603 insertions(+), 2603 deletions(-) delete mode 100644 lib/glider/ClientCLI.py delete mode 100644 lib/glider/ServerCLI.py delete mode 100644 lib/glider/SignerCLI.py delete mode 100644 lib/glider/__init__.py delete mode 100644 lib/glider/checkJson.py delete mode 100644 lib/glider/download.py delete mode 100644 lib/glider/formats.py delete mode 100644 lib/glider/keys.py delete mode 100644 lib/glider/master_keys.py delete mode 100644 lib/glider/repository.py delete mode 100644 lib/glider/tests.py delete mode 100644 lib/glider/util.py create mode 100644 lib/thandy/ClientCLI.py create mode 100644 lib/thandy/ServerCLI.py create mode 100644 lib/thandy/SignerCLI.py create mode 100644 lib/thandy/__init__.py create mode 100644 lib/thandy/checkJson.py create mode 100644 lib/thandy/download.py create mode 100644 lib/thandy/formats.py create mode 100644 lib/thandy/keys.py create mode 100644 lib/thandy/master_keys.py create mode 100644 lib/thandy/repository.py create mode 100644 lib/thandy/tests.py create mode 100644 lib/thandy/util.py (limited to 'lib') diff --git a/lib/glider/ClientCLI.py b/lib/glider/ClientCLI.py deleted file mode 100644 index c6f13fd..0000000 --- a/lib/glider/ClientCLI.py +++ /dev/null @@ -1,66 +0,0 @@ - -import os -import sys -import getopt - -import glider.util -import glider.repository -import glider.download - -def update(args): - repoRoot = glider.util.userFilename("cache") - options, args = getopt.getopt(args, "", [ "repo=", "no-download" ]) - download = True - - for o, v in options: - if o == '--repo': - repoRoot = v - elif o == "--no-download": - download = False - - repo = glider.repository.LocalRepository(repoRoot) - - files = repo.getFilesToUpdate(trackingBundles=args) - - if not download: - return - - mirrorlist = repo.getMirrorlistFile().get() - - downloader = glider.download.Downloads() - downloader.start() - - for f in files: - # XXXX Use hash. - dj = glider.download.DownloadJob(f, repo.getFilename(f), - mirrorlist) - downloader.addDownloadJob(dj) - # XXXX replace file in repository if ok; reload; see what changed. - - # Wait for in-progress jobs - -# Check my repository - -# Tell me what I need to download - -# Download stuff - -# Tell me what to install. - -def usage(): - print "Known commands:" - print " update [--repo=repository] [--no-download]" - sys.exit(1) - -def main(): - if len(sys.argv) < 2: - usage() - cmd = sys.argv[1] - args = sys.argv[2:] - if cmd in [ "update" ]: - globals()[cmd](args) - else: - usage() - -if __name__ == '__main__': - main() diff --git a/lib/glider/ServerCLI.py b/lib/glider/ServerCLI.py deleted file mode 100644 index c564f5b..0000000 --- a/lib/glider/ServerCLI.py +++ /dev/null @@ -1,187 +0,0 @@ - -import os -import sys -import getopt -import time - -import simplejson - -import glider.formats -import glider.util -import glider.keys - -def tstamp(): - return time.strftime("%Y%m%d_%H%M%S", time.localtime()) - -def snarf(fname): - f = open(fname, 'rb') - try: - return f.read() - finally: - f.close() - -def snarfObj(fname): - f = open(fname, 'r') - try: - return simplejson.load(f) - finally: - f.close() - -def insert(args): - repo = os.environ.get("THANDY_MASTER_REPO") - backupDir = glider.util.userFilename("old_files") - checkSigs = True - - options, args = getopt.getopt(args, "", ["repo=", "no-check"]) - for o,v in options: - if o == "--repo": - repo = v - elif o == "--no-check": - checkSigs = False - - if not repo: - print "No repository specified." - usage() - if not os.path.exists(repo): - print "No such repository as %r"%repo - usage() - - if not os.path.exists(backupDir): - os.makedirs(backupDir, 0700) - - if checkSigs: - keys = glider.util.getKeylist(os.path.join(repo, "meta/keys.txt")) - else: - keys = None - - n_ok = 0 - for fn in args: - print "Loading %s..."%fn - try: - content = snarf(fn) - except OSError, e: - print "Couldn't open %s: %s"%(fn, e) - continue - - try: - obj = simplejson.loads(content) - except ValueError, e: - print "Couldn't decode %s: %s"%(fn, e) - continue - - try: - ss, r, path = glider.formats.checkSignedObj(obj, keys) - except glider.FormatException, e: - print "Bad format on %s: %s"%(fn, e) - continue - if checkSigs and not ss.isValid(): - print "Not enough valid signatures on %s"%fn - continue - - print " Looks okay. It goes in %s"%path - assert path.startswith("/") - targetPath = os.path.join(repo, path[1:]) - if os.path.exists(targetPath): - oldContents = snarf(targetPath) - if oldContents == content: - print " File unchanged!" - n_ok += 1 - continue - - baseFname = "%s_%s" % (tstamp(), os.path.split(path)[1]) - backupFname = os.path.join(backupDir, baseFname) - print " Copying old file to %s"%backupFname - glider.util.replaceFile(backupFname, oldContents) - - parentDir = os.path.split(targetPath)[0] - if not os.path.exists(parentDir): - print " Making %s"%parentDir - os.makedirs(parentDir, 0755) - print " Replacing file..." - glider.util.replaceFile(targetPath, content) - print " Done." - n_ok += 1 - if n_ok != len(args): - sys.exit(1) - -def timestamp(args): - repo = os.environ.get("THANDY_MASTER_REPO") - ts_keyfile = glider.util.userFilename("timestamp_key") - - options, args = getopt.getopt(args, "", ["repo=", "ts-key="]) - for o,v in options: - if o == "--repo": - repo = v - elif o == "--ts-key": - ts_keyfile = v - - if repo == None: - print "No repository specified." - usage() - if not os.path.exists(repo): - print "No such repository as %r"%repo - usage() - - tsFname = os.path.join(repo, "meta/timestamp.txt") - - try: - mObj = snarfObj(os.path.join(repo, "meta/mirrors.txt")) - except OSError: - print "No mirror list!" - sys.exit(1) - try: - kObj = snarfObj(os.path.join(repo, "meta/keys.txt")) - except OSError: - print "No key list!" - sys.exit(1) - - bundles = [] - for dirpath, dirname, fns in os.walk(os.path.join(repo, "bundleinfo")): - for fn in fns: - try: - bObj = snarfObj(fn) - except (ValueError, OSError), e: - print "(Couldn't read bundle-like %s)"%fn - continue - try: - _, r, _ = glider.formats.checkSignedObj(bObj) - except glider.FormatException, e: - print "Problem reading object from %s"%fn - continue - if r != "bundle": - print "%s was not a good bundle"%fn - continue - bundles.append(bObj['signed']) - - timestamp = glider.formats.makeTimestampObj( - mObj['signed'], kObj['signed'], bundles) - signable = glider.formats.makeSignable(timestamp) - - keydb = glider.formats.Keylist() - #XXXX Still a roundabout way to do this. - keylist = glider.formats.makeKeylistObj(ts_keyfile, True) - keydb.addFromKeylist(keylist) - for k in keydb.iterkeys(): - glider.formats.sign(signable, k) - - content = simplejson.dumps(signable, sort_keys=True) - glider.util.replaceFile(tsFname, content) - -def usage(): - print "Known commands:" - print " insert [--no-check] [--repo=repository] file ..." - print " timestamp [--repo=repository]" - sys.exit(1) - -def main(): - if len(sys.argv) < 2: - usage() - cmd = sys.argv[1] - args = sys.argv[2:] - if cmd in [ "insert", "timestamp" ]: - globals()[cmd](args) - else: - usage() - -if __name__ == '__main__': - main() diff --git a/lib/glider/SignerCLI.py b/lib/glider/SignerCLI.py deleted file mode 100644 index bf5b73a..0000000 --- a/lib/glider/SignerCLI.py +++ /dev/null @@ -1,313 +0,0 @@ - -import os -import getopt -import sys -import logging -import simplejson - -import glider.keys -import glider.formats - -def getKeyStore(): - return glider.keys.KeyStore(glider.util.userFilename("secret_keys")) - -def dumpKey(key, indent=0): - i = " "*indent - print "%s%s"%(i, key.getKeyID()) - for r, p in key.getRoles(): - print " %s%s\t%s"%(i, r, p) - -def getKey(ks, keyid=None, role=None, path=None): - if keyid is not None: - keys = ks.getKeysFuzzy(keyid) - if None not in (role, path): - keys = [ k for k in keys if k.hasRole(role, path) ] - elif None not in (role, path): - keys = ks.getKeysByRole(role, path) - else: - assert False - if len(keys) < 1: - print "No such key.\nI wanted", - if keyid: print "keyid='%s...'"%keyid, - if None not in (role, path): print "role=%s, path=%s"%(role,path), - print - print "I only know about:" - for k in ks.iterkeys(): - dumpKey(k) - sys.exit(1) - elif len(keys) > 1: - print "Multiple keys match. Possibilities are:" - for k in keys: - dumpKey(k) - sys.exit(1) - else: - return keys[0] - -# ------------------------------ - -def makepackage(args): - options, args = getopt.getopt(args, "", "keyid=") - keyid = None - for o,v in options: - if o == "--keyid": - keyid = v - - if len(args) < 2: - usage() - - configFile = args[0] - dataFile = args[1] - print "Generating package." - package = glider.formats.makePackageObj(configFile, dataFile) - relpath = package['location'] - print "need a key with role matching [package %s]"%relpath - ks = getKeyStore() - ks.load() - key = getKey(ks, keyid=keyid, role='package', path=relpath) - signable = glider.formats.makeSignable(package) - glider.formats.sign(signable, key) - - if 1: - ss, r, p = glider.formats.checkSignedObj(signable, ks) - assert ss.isValid() - - location = os.path.split(package['location'])[-1] - print "Writing signed package to %s"%location - f = open(location, 'w') - simplejson.dump(signable, f, indent=1) - f.close() - -def makebundle(args): - options, args = getopt.getopt(args, "", "keyid=") - keyid = None - for o,v in options: - if o == "--keyid": - keyid = v - - if len(args) < 2: - usage() - - configFile = args[0] - packages = {} - for pkgFile in args[1:]: - print "Loading", pkgFile - f = open(pkgFile, 'r') - p = simplejson.load(f) - f.close() - _, r, _ = glider.formats.checkSignedObj(p) - if r != 'package': - print pkgFile, "was not a package" - packages[p['signed']['location']] = p - - def getHash(path): - p = packages[path] - return glider.formats.getDigest(p['signed']) - - bundleObj = glider.formats.makeBundleObj(configFile, getHash) - signable = glider.formats.makeSignable(bundleObj) - - ks = getKeyStore() - ks.load() - key = getKey(ks, keyid=keyid, role="bundle", path=bundleObj['location']) - glider.formats.sign(signable, key) - - if 1: - ss, r, p = glider.formats.checkSignedObj(signable, ks) - assert ss.isValid() - - location = os.path.split(bundleObj['location'])[-1] - print "Writing signed bundle to %s"%location - f = open(location, 'w') - simplejson.dump(signable, f, indent=1) - f.close() - -# ------------------------------ -def makekeylist(args): - options, args = getopt.getopt(args, "", "keyid=") - keyid = None - for o,v in options: - if o == "--keyid": - keyid = v - - if len(args) < 1: - usage() - - keylist = glider.formats.makeKeylistObj(args[0]) - signable = glider.formats.makeSignable(keylist) - - ks = getKeyStore() - ks.load() - key = getKey(ks, keyid=keyid, role="master", path="/meta/keys.txt") - glider.formats.sign(signable, key) - - if 1: - ss, r, p = glider.formats.checkSignedObj(signable, ks) - assert ss.isValid() - - print "writing signed keylist to keys.txt" - glider.util.replaceFile("keys.txt", - simplejson.dumps(signable, indent=1, sort_keys=True), - textMode=True) - -def signkeylist(args): - if len(args) != 1: - usage() - - keylist = simplejson.load(open(args[0], 'r')) - glider.formats.SIGNED_SCHEMA.checkMatch(keylist) - glider.formats.KEYLIST_SCHEMA.checkMatch(keylist['signed']) - - ks = getKeyStore() - ks.load() - keys = ks.getKeysByRole("master", "/meta/keys.txt") - for k in keys: - glider.formats.sign(keylist, k) - - print "writing signed keylist to keys.txt" - glider.util.replaceFile("keys.txt", - simplejson.dumps(keylist, indent=1, sort_keys=True), - textMode=True) - -def makemirrorlist(args): - options, args = getopt.getopt(args, "", "keyid=") - keyid = None - for o,v in options: - if o == "--keyid": - keyid = v - - if len(args) < 1: - usage() - - mirrorlist = glider.formats.makeMirrorListObj(args[0]) - signable = glider.formats.makeSignable(mirrorlist) - - ks = getKeyStore() - ks.load() - key = getKey(ks, keyid=keyid, role='mirrors', path="/meta/mirrors.txt") - glider.formats.sign(signable, key) - - if 1: - ss, r, p = glider.formats.checkSignedObj(signable, ks) - assert ss.isValid() - - print "writing signed mirrorlist to mirrors.txt" - glider.util.replaceFile("mirrors.txt", - simplejson.dumps(signable, indent=1, sort_keys=True), - textMode=True) - -# ------------------------------ - -def keygen(args): - k = getKeyStore() - k.load() - print "Generating key. This will be slow." - key = glider.keys.RSAKey.generate() - print "Generated new key: %s" % key.getKeyID() - k.addKey(key) - k.save() - -def listkeys(args): - k = getKeyStore() - k.load() - for k in k.iterkeys(): - print k.getKeyID() - for r, p in k.getRoles(): - print " ", r, p - -def addrole(args): - if len(args) < 3: - usage() - ks = getKeyStore() - ks.load() - k = getKey(ks, args[0]) - r = args[1] - if r not in glider.formats.ALL_ROLES: - print "Unrecognized role %r. Known roles are %s"%( - r,", ".join(glider.format.ALL_ROLES)) - sys.exit(1) - p = args[2] - k.addRole(r, p) - ks.save() - -def delrole(args): - if len(args) < 3: - usage() - ks = getKeyStore() - ks.load() - k = getKey(ks, args[0]) - r = args[1] - if r not in glider.formats.ALL_ROLES: - print "Unrecognized role %r. Known roles are %s"%( - r,", ".join(glider.format.ALL_ROLES)) - sys.exit(1) - p = args[2] - - #XXXX rep. - origLen = len(k._roles) - k._roles = [ (role,path) for role,path in k._roles - if (role,path) != (r,p) ] - removed = origLen - len(k._roles) - print removed, "roles removed" - if removed: - ks.save() - -def chpass(args): - ks = getKeyStore() - print "Old password." - ks.load() - print "New password." - ks.clearPassword() - ks.save() - -def dumpkey(args): - options, args = getopt.getopt(args, "", ["include-secret", "passwd="]) - - includeSecret = False - for o,v in options: - if o == '--include-secret': - includeSecret = True - else: - print "Unexpected %r"%o - - ks = getKeyStore() - ks.load() - - keys = [] - if len(args): - keys = [ getKey(ks, a) for a in args ] - else: - keys = list(ks.iterkeys()) - - for k in keys: - data = k.format(private=includeSecret, includeRoles=True) - print "Key(", simplejson.dumps(data, indent=2), ")" - -def usage(): - print "Known commands:" - print " keygen" - print " listkeys" - print " chpass" - print " addrole keyid role path" - print " delrole keyid role path" - print " dumpkey [--include-secret] keyid" - print " makepackage config datafile" - print " makebundle config packagefile ..." - print " signkeylist keylist" - print " makekeylist keylist" - print " makemirrorlist config" - sys.exit(1) - -def main(): - if len(sys.argv) < 2: - usage() - cmd = sys.argv[1] - args = sys.argv[2:] - if cmd in [ "keygen", "listkeys", "addrole", "delrole", "chpass", - "dumpkey", "makepackage", "makebundle", "signkeylist", - "makekeylist", "signkeylist", "makemirrorlist", ]: - globals()[cmd](args) - else: - usage() - -if __name__ == '__main__': - main() diff --git a/lib/glider/__init__.py b/lib/glider/__init__.py deleted file mode 100644 index 87fd983..0000000 --- a/lib/glider/__init__.py +++ /dev/null @@ -1,35 +0,0 @@ - -__all__ = [ 'formats' ] - -_BaseException = Exception - -class Exception(_BaseException): - pass - -class FormatException(Exception): - pass - -class UnknownFormat(FormatException): - pass - -class BadSignature(Exception): - pass - -class BadPassword(Exception): - pass - -class InternalError(Exception): - pass - -class RepoError(InternalError): - pass - -class CryptoError(Exception): - pass - -class PubkeyFormatException(FormatException): - pass - -class UnknownMethod(CryptoError): - pass - diff --git a/lib/glider/checkJson.py b/lib/glider/checkJson.py deleted file mode 100644 index 0c5eed6..0000000 --- a/lib/glider/checkJson.py +++ /dev/null @@ -1,274 +0,0 @@ - -import re -import sys - -import glider - -class Schema: - def matches(self, obj): - try: - self.checkMatch(obj) - except glider.FormatException: - return False - else: - return True - - def checkMatch(self, obj): - raise NotImplemented() - -class Any(Schema): - """ - >>> s = Any() - >>> s.matches("A String") - True - >>> s.matches([1, "list"]) - True - """ - def checkMatch(self, obj): - pass - -class RE(Schema): - """ - >>> s = RE("h.*d") - >>> s.matches("hello world") - True - >>> s.matches("Hello World") - False - >>> s.matches("hello world!") - False - >>> s.matches([33, "Hello"]) - False - """ - def __init__(self, pat=None, modifiers=0, reObj=None, reName="pattern"): - if not reObj: - if not pat.endswith("$"): - pat += "$" - reObj = re.compile(pat, modifiers) - self._re = reObj - self._reName = reName - def checkMatch(self, obj): - if not isinstance(obj, basestring) or not self._re.match(obj): - raise glider.FormatException("%r did not match %s" - %(obj,self._reName)) - -class Str(Schema): - """ - >>> s = Str("Hi") - >>> s.matches("Hi") - True - >>> s.matches("Not hi") - False - """ - def __init__(self, val): - self._str = val - def checkMatch(self, obj): - if self._str != obj: - raise glider.FormatException("Expected %r; got %r"%(self._str, obj)) - -class AnyStr(Schema): - """ - >>> s = AnyStr() - >>> s.matches("") - True - >>> s.matches("a string") - True - >>> s.matches(["a"]) - False - >>> s.matches(3) - False - >>> s.matches(u"a unicode string") - True - >>> s.matches({}) - False - """ - def __init__(self): - pass - def checkMatch(self, obj): - if not isinstance(obj, basestring): - raise glider.FormatException("Expected a string; got %r"%obj) - -class ListOf(Schema): - """ - >>> s = ListOf(RE("(?:..)*")) - >>> s.matches("hi") - False - >>> s.matches([]) - True - >>> s.matches({}) - False - >>> s.matches(["Hi", "this", "list", "is", "full", "of", "even", "strs"]) - True - >>> s.matches(["This", "one", "is not"]) - False - """ - def __init__(self, schema, minCount=0, maxCount=sys.maxint,listName="list"): - self._schema = schema - self._minCount = minCount - self._maxCount = maxCount - self._listName = listName - def checkMatch(self, obj): - if not isinstance(obj, (list, tuple)): - raise glider.FormatException("Expected %s; got %r" - %(self._listName,obj)) - for item in obj: - try: - self._schema.checkMatch(item) - except glider.FormatException, e: - raise glider.FormatException("%s in %s"%(e, self._listName)) - - if not (self._minCount <= len(obj) <= self._maxCount): - raise glider.FormatException("Length of %s out of range" - %self._listName) - -class Struct(Schema): - """ - >>> s = Struct([ListOf(AnyStr()), AnyStr(), Str("X")]) - >>> s.matches(False) - False - >>> s.matches("Foo") - False - >>> s.matches([[], "Q", "X"]) - True - >>> s.matches([[], "Q", "D"]) - False - >>> s.matches([[3], "Q", "X"]) - False - >>> s.matches([[], "Q", "X", "Y"]) - False - """ - def __init__(self, subschemas, allowMore=False, structName="list"): - self._subschemas = subschemas[:] - self._allowMore = allowMore - self._structName = structName - def checkMatch(self, obj): - if not isinstance(obj, (list, tuple)): - raise glider.FormatException("Expected %s; got %r" - %(self._structName,obj)) - elif len(obj) < len(self._subschemas): - raise glider.FormatException( - "Too few fields in %s"%self._structName) - elif len(obj) > len(self._subschemas) and not self._allowMore: - raise glider.FormatException( - "Too many fields in %s"%self._structName) - for item, schema in zip(obj, self._subschemas): - schema.checkMatch(item) - -class DictOf(Schema): - """ - >>> s = DictOf(RE(r'[aeiou]+'), Struct([AnyStr(), AnyStr()])) - >>> s.matches("") - False - >>> s.matches({}) - True - >>> s.matches({"a": ["x", "y"], "e" : ["", ""]}) - True - >>> s.matches({"a": ["x", 3], "e" : ["", ""]}) - False - >>> s.matches({"a": ["x", "y"], "e" : ["", ""], "d" : ["a", "b"]}) - False - """ - def __init__(self, keySchema, valSchema): - self._keySchema = keySchema - self._valSchema = valSchema - def checkMatch(self, obj): - try: - iter = obj.iteritems() - except AttributeError: - raise glider.FormatException("Expected a dict; got %r"%obj) - - for k,v in iter: - self._keySchema.checkMatch(k) - self._valSchema.checkMatch(v) - -class Opt: - """Helper; applied to a value in Obj to mark it optional. - - >>> s = Obj(k1=Str("X"), k2=Opt(Str("Y"))) - >>> s.matches({'k1': "X", 'k2': "Y"}) - True - >>> s.matches({'k1': "X", 'k2': "Z"}) - False - >>> s.matches({'k1': "X"}) - True - """ - def __init__(self, schema): - self._schema = schema - def checkMatch(self, obj): - self._schema.checkMatch(obj) - -class Obj(Schema): - """ - >>> s = Obj(a=AnyStr(), bc=Struct([Int(), Int()])) - >>> s.matches({'a':"ZYYY", 'bc':[5,9]}) - True - >>> s.matches({'a':"ZYYY", 'bc':[5,9], 'xx':5}) - True - >>> s.matches({'a':"ZYYY", 'bc':[5,9,3]}) - False - >>> s.matches({'a':"ZYYY"}) - False - - """ - def __init__(self, _objname="object", **d): - self._objname = _objname - self._required = d.items() - - - def checkMatch(self, obj): - for k,schema in self._required: - try: - item = obj[k] - except KeyError: - if not isinstance(schema, Opt): - raise glider.FormatException("Missing key %s in %s" - %(k,self._objname)) - - else: - try: - schema.checkMatch(item) - except glider.FormatException, e: - raise glider.FormatException("%s in %s.%s" - %(e,self._objname,k)) - - -class Int(Schema): - """ - >>> s = Int() - >>> s.matches(99) - True - >>> s.matches(False) - False - >>> s.matches(0L) - True - >>> s.matches("a string") - False - >>> Int(lo=10, hi=30).matches(25) - True - >>> Int(lo=10, hi=30).matches(5) - False - """ - def __init__(self, lo=-sys.maxint, hi=sys.maxint): - self._lo = lo - self._hi = hi - def checkMatch(self, obj): - if isinstance(obj, bool) or not isinstance(obj, (int, long)): - # We need to check for bool as a special case, since bool - # is for historical reasons a subtype of int. - raise glider.FormatException("Got %r instead of an integer"%obj) - elif not (self._lo <= obj <= self._hi): - raise glider.FormatException("%r not in range [%r,%r]" - %(obj, self._lo, self._hi)) - -class Bool(Schema): - """ - >>> s = Bool() - >>> s.matches(True) and s.matches(False) - True - >>> s.matches(11) - False - """ - def __init__(self): - pass - def checkMatch(self, obj): - if not isinstance(obj, bool): - raise glider.FormatException("Got %r instead of a boolean"%obj) diff --git a/lib/glider/download.py b/lib/glider/download.py deleted file mode 100644 index 3135e39..0000000 --- a/lib/glider/download.py +++ /dev/null @@ -1,127 +0,0 @@ - - -import urllib2 -import httplib -import random - -import threading, Queue - -import glider.util - -class Downloads: - def __init__(self, n_threads=2): - self._lock = threading.RLock() - self.downloads = {} - self.haveDownloaded = {} - self.downloadQueue = Queue.Queue() - self.threads = [ threading.Thread(target=self._thread) ] - for t in self.threads: - t.setDaemon(True) - - def start(self): - for t in self.threads: - t.start() - - def isCurrentlyDownloading(self, relPath): - self._lock.acquire() - try: - return self.downloads.has_key(relPath) - finally: - self._lock.release() - - def isRedundant(self, relPath): - self._lock.acquire() - try: - return (self.downloads.has_key(relPath) or - self.haveDownloaded.has_key(relPath)) - finally: - self._lock.release() - - def addDownloadJob(self, job): - rp = job.getRelativePath() - self._lock.acquire() - self.downloads[rp] = job - self._lock.release() - self.downloadQueue.put(job) - - def _thread(self): - while True: - job = self.downloadQueue.get() - job.download() - rp = job.getRelativePath() - self._lock.acquire() - try: - del self.downloads[rp] - self.haveDownloaded[rp] = True - finally: - self._lock.release() - -class DownloadJob: - def __init__(self, relPath, destPath, mirrorlist=None, - wantHash=None, canStall=False): - self._relPath = relPath - self._wantHash = wantHash - self._mirrorList = mirrorlist - self._destPath = destPath - - tmppath = glider.util.userFilename("tmp") - if relPath.startswith("/"): - relPath = relPath[1:] - self._tmppath = os.path.join(tmppath, relPath) - - d = os.path.dirname(self._tmppath) - if not os.path.exists(d): - os.makedirs(d, 0700) - - def getRelativePath(self): - return self._relPath - - def haveStalledFile(self): - return os.path.exists(self._tmppath) - - def getURL(self, mirrorlist=None): - if mirrorlist is None: - mirrorlist = self._mirrorList - weightSoFar = 0 - usable = [] - - for m in mirrorlist['mirrors']: - for c in m['contents']: - # CHECK FOR URL SUITABILITY XXXXX - - if glider.formats.rolePathMatches(c, self._relPath): - weightSoFar += m['weight'] - usable.append( (weightSoFar, m) ) - break - - wTarget = random.randint(0, weightSoFar) - mirror = None - # Could use bisect here instead - for w, m in mirrorlist: - if w >= wTarget: - mirror = m - break - - return m['urlbase'] + self._relPath - - def download(self): - # XXXX RESUME - - f_in = urllib2.urlopen(self.getURL()) - f_out = open(self._tmpPath, 'w') - while True: - c = f_in.read(1024) - if not c: - break - f_out.write(c) - f_in.close() - f_out.close() - # XXXXX retry on failure - - if self._wantHash: - gotHash = glider.formats.getFileDigest(self._tmpPath) - if gotHash != self._wantHash: - # XXXX Corrupt file. - pass - - glider.utils.moveFile(self._tmpPath, self._destPath) diff --git a/lib/glider/formats.py b/lib/glider/formats.py deleted file mode 100644 index 9846c50..0000000 --- a/lib/glider/formats.py +++ /dev/null @@ -1,747 +0,0 @@ - -import simplejson -import time -import re -import binascii -import calendar - -import glider.checkJson - -import Crypto.Hash.SHA256 - -class KeyDB: - """A KeyDB holds public keys, indexed by their key IDs.""" - def __init__(self): - self._keys = {} - def addKey(self, k): - keyid = k.getKeyID() - try: - oldkey = self._keys[keyid] - for r, p in oldkey.getRoles(): - if (r, p) not in k.getRoles(): - k.addRole(r,p) - except KeyError: - pass - self._keys[k.getKeyID()] = k - def getKey(self, keyid): - return self._keys[keyid] - def getKeysByRole(self, role, path): - results = [] - for key in self._keys.itervalues(): - for r,p in key.getRoles(): - if r == role: - if rolePathMatches(p, path): - results.append(key) - return results - - def getKeysFuzzy(self, keyid): - r = [] - for k,v in self._keys.iteritems(): - if k.startswith(keyid): - r.append(v) - return r - def iterkeys(self): - return self._keys.itervalues() - -_rolePathCache = {} -def rolePathMatches(rolePath, path): - """Return true iff the relative path in the filesystem 'path' conforms - to the pattern 'rolePath': a path that a given key is - authorized to sign. Patterns are allowed to contain * to - represent one or more characters in a filename, and ** to - represent any level of directory structure. - - >>> rolePathMatches("a/b/c/", "a/b/c/") - True - >>> rolePathMatches("**/c.*", "a/b/c.txt") - True - >>> rolePathMatches("**/c.*", "a/b/ctxt") - False - >>> rolePathMatches("**/c.*", "a/b/c.txt/foo") - False - >>> rolePathMatches("a/*/c", "a/b/c") - True - >>> rolePathMatches("a/*/c", "a/b/c.txt") - False - >>> rolePathMatches("a/*/c", "a/b/c.txt") #Check cache - False - """ - try: - regex = _rolePathCache[rolePath] - except KeyError: - orig = rolePath - # remove duplicate slashes. - rolePath = re.sub(r'/+', '/', rolePath) - # escape, then ** becomes .* - rolePath = re.escape(rolePath).replace(r'\*\*', r'.*') - # * becomes [^/]* - rolePath = rolePath.replace(r'\*', r'[^/]*') - # and no extra text is allowed. - rolePath += "$" - regex = _rolePathCache[orig] = re.compile(rolePath) - return regex.match(path) != None - -class SignatureStatus: - """Represents the outcome of checking signature(s) on an object.""" - def __init__(self, good, bad, unrecognized, unauthorized): - # keyids for all the valid signatures - self._good = good[:] - # keyids for the invalid signatures (we had the key, and it failed). - self._bad = bad[:] - # keyids for signatures where we didn't recognize the key - self._unrecognized = unrecognized[:] - # keyids for signatures where we recognized the key, but it doesn't - # seem to be allowed to sign this kind of document. - self._unauthorized = unauthorized[:] - - def isValid(self, threshold=1): - """Return true iff we got at least 'threshold' good signatures.""" - return len(self._good) >= threshold - - def mayNeedNewKeys(self): - """Return true iff downloading a new set of keys might tip this - signature status over to 'valid.'""" - return len(self._unrecognized) or len(self._unauthorized) - -def checkSignatures(signed, keyDB, role=None, path=None): - """Given an object conformant to SIGNED_SCHEMA and a set of public keys - in keyDB, verify the signed object in 'signed'.""" - - SIGNED_SCHEMA.checkMatch(signed) - - goodSigs = [] - badSigs = [] - unknownSigs = [] - tangentialSigs = [] - - signable = signed['signed'] - signatures = signed['signatures'] - - d_obj = Crypto.Hash.SHA256.new() - getDigest(signable, d_obj) - digest = d_obj.digest() - - for signature in signatures: - sig = signature['sig'] - keyid = signature['keyid'] - method = signature['method'] - - try: - key = keyDB.getKey(keyid) - except KeyError: - unknownSigs.append(keyid) - continue - - try: - result = key.checkSignature(method, sig, digest=digest) - except glider.UnknownMethod: - continue - - if result == True: - if role is not None: - for r,p in key.getRoles(): - if r == role and rolePathMatches(p, path): - break - else: - tangentialSigs.append(sig) - continue - - goodSigs.append(keyid) - else: - badSigs.append(keyid) - - return SignatureStatus(goodSigs, badSigs, unknownSigs, tangentialSigs) - -def encodeCanonical(obj, outf=None): - """Encode the object obj in canoncial JSon form, as specified at - http://wiki.laptop.org/go/Canonical_JSON . It's a restricted - dialect of json in which keys are always lexically sorted, - there is no whitespace, floats aren't allowed, and only quote - and backslash get escaped. The result is encoded in UTF-8, - and the resulting bits are passed to outf (if provided), or joined - into a string and returned. - - >>> encodeCanonical("") - '""' - >>> encodeCanonical([1, 2, 3]) - '[1,2,3]' - >>> encodeCanonical({"x" : 3, "y" : 2}) - '{"x":3,"y":2}' - """ - def default(o): - raise TypeError("Can't encode %r", o) - def floatstr(o): - raise TypeError("Floats not allowed.") - def canonical_str_encoder(s): - return '"%s"' % re.sub(r'(["\\])', r'\\\1', s) - - # XXX This is, alas, a hack. I'll submit a canonical JSon patch to - # the simplejson folks. - - iterator = simplejson.encoder._make_iterencode( - None, default, canonical_str_encoder, None, floatstr, - ":", ",", True, False, True)(obj, 0) - - result = None - if outf == None: - result = [ ] - outf = result.append - - for u in iterator: - outf(u.encode("utf-8")) - if result is not None: - return "".join(result) - -def getDigest(obj, digestObj=None): - """Update 'digestObj' (typically a SHA256 object) with the digest of - the canonical json encoding of obj. If digestObj is none, - compute the SHA256 hash and return it. - - DOCDOC string equivalence. - """ - useTempDigestObj = (digestObj == None) - if useTempDigestObj: - digestObj = Crypto.Hash.SHA256.new() - - if isinstance(obj, str): - digestObj.update(obj) - elif isinstance(obj, unicode): - digestObj.update(obj.encode("utf-8")) - else: - encodeCanonical(obj, digestObj.update) - - if useTempDigestObj: - return digestObj.digest() - -def getFileDigest(f, digestObj=None): - """Update 'digestObj' (typically a SHA256 object) with the digest of - the file object in f. If digestObj is none, compute the SHA256 - hash and return it. - - >>> s = "here is a long string"*1000 - >>> import cStringIO, Crypto.Hash.SHA256 - >>> h1 = Crypto.Hash.SHA256.new() - >>> h2 = Crypto.Hash.SHA256.new() - >>> getFileDigest(cStringIO.StringIO(s), h1) - >>> h2.update(s) - >>> h1.digest() == h2.digest() - True - """ - useTempDigestObj = (digestObj == None) - if useTempDigestObj: - digestObj = Crypto.Hash.SHA256.new() - - while 1: - s = f.read(4096) - if not s: - break - digestObj.update(s) - - if useTempDigestObj: - return digestObj.digest() - -def makeSignable(obj): - return { 'signed' : obj, 'signatures' : [] } - -def sign(signed, key): - """Add an element to the signatures of 'signed', containing a new signature - of the "signed" part. - """ - - SIGNED_SCHEMA.checkMatch(signed) - - signable = signed["signed"] - signatures = signed['signatures'] - - keyid = key.getKeyID() - - signatures = [ s for s in signatures if s['keyid'] != keyid ] - - method, sig = key.sign(signable) - signatures.append({ 'keyid' : keyid, - 'method' : method, - 'sig' : sig }) - signed['signatures'] = signatures - -def formatTime(t): - """Encode the time 't' in YYYY-MM-DD HH:MM:SS format. - - >>> formatTime(1221265172) - '2008-09-13 00:19:32' - """ - return time.strftime("%Y-%m-%d %H:%M:%S", time.gmtime(t)) - -def parseTime(s): - """Parse a time 's' in YYYY-MM-DD HH:MM:SS format.""" - try: - return calendar.timegm(time.strptime(s, "%Y-%m-%d %H:%M:%S")) - except ValueError: - raise glider.FormatError("Malformed time %r", s) - -def formatBase64(h): - """Return the base64 encoding of h with whitespace and = signs omitted.""" - return binascii.b2a_base64(h).rstrip("=\n ") - -formatHash = formatBase64 - -def parseBase64(s): - """Parse a base64 encoding with whitespace and = signs omitted. """ - extra = len(s) % 4 - if extra: - padding = "=" * (4 - extra) - s += padding - try: - return binascii.a2b_base64(s) - except binascii.Error: - raise glider.FormatError("Invalid base64 encoding") - -def parseHash(s): - h = parseBase64(s) - if len(h) != Crypto.Hash.SHA256.digest_size: - raise glider.FormatError("Bad hash length") - return h - -S = glider.checkJson - -# A date, in YYYY-MM-DD HH:MM:SS format. -TIME_SCHEMA = S.RE(r'\d{4}-\d{2}-\d{2} \d{2}:\d{2}:\d{2}') -# A hash, base64-encoded -HASH_SCHEMA = S.RE(r'[a-zA-Z0-9\+\/]{43}') - -# A hexadecimal value. -HEX_SCHEMA = S.RE(r'[a-fA-F0-9]+') -# A base-64 encoded value -BASE64_SCHEMA = S.RE(r'[a-zA-Z0-9\+\/]+') -# An RSA key; subtype of PUBKEY_SCHEMA. -RSAKEY_SCHEMA = S.Obj( - _keytype=S.Str("rsa"), - e=BASE64_SCHEMA, - n=BASE64_SCHEMA) -# Any public key. -PUBKEY_SCHEMA = S.Obj( - _keytype=S.AnyStr()) - -KEYID_SCHEMA = HASH_SCHEMA -SIG_METHOD_SCHEMA = S.AnyStr() -RELPATH_SCHEMA = PATH_PATTERN_SCHEMA = S.AnyStr() -URL_SCHEMA = S.AnyStr() -VERSION_SCHEMA = S.ListOf(S.Any()) #XXXX WRONG - -# A single signature of an object. Indicates the signature, the id of the -# signing key, and the signing method. -SIGNATURE_SCHEMA = S.Obj( - keyid=KEYID_SCHEMA, - method=SIG_METHOD_SCHEMA, - sig=BASE64_SCHEMA) - -# A signed object. -SIGNED_SCHEMA = S.Obj( - signed=S.Any(), - signatures=S.ListOf(SIGNATURE_SCHEMA)) - -ROLENAME_SCHEMA = S.AnyStr() - -# A role: indicates that a key is allowed to certify a kind of -# document at a certain place in the repo. -ROLE_SCHEMA = S.Struct([ROLENAME_SCHEMA, PATH_PATTERN_SCHEMA]) - -# A Keylist: indicates a list of live keys and their roles. -KEYLIST_SCHEMA = S.Obj( - _type=S.Str("Keylist"), - ts=TIME_SCHEMA, - keys=S.ListOf(S.Obj(key=PUBKEY_SCHEMA, roles=S.ListOf(ROLE_SCHEMA)))) - -# A Mirrorlist: indicates all the live mirrors, and what documents they -# serve. -MIRRORLIST_SCHEMA = S.Obj( - _type=S.Str("Mirrorlist"), - ts=TIME_SCHEMA, - mirrors=S.ListOf(S.Obj(name=S.AnyStr(), - urlbase=URL_SCHEMA, - contents=S.ListOf(PATH_PATTERN_SCHEMA), - weight=S.Int(lo=0), - ))) - -# A timestamp: indicates the lastest versions of all top-level signed objects. -TIMESTAMP_SCHEMA = S.Obj( - _type = S.Str("Timestamp"), - at = TIME_SCHEMA, - m = S.Struct([TIME_SCHEMA, HASH_SCHEMA]), - k = S.Struct([TIME_SCHEMA, HASH_SCHEMA]), - b = S.DictOf(keySchema=S.AnyStr(), - valSchema= - S.Struct([ VERSION_SCHEMA, RELPATH_SCHEMA, TIME_SCHEMA, HASH_SCHEMA ])) - ) - -# A Bundle: lists a bunch of packages that should be updated in tandem -BUNDLE_SCHEMA = S.Obj( - _type=S.Str("Bundle"), - at=TIME_SCHEMA, - name=S.AnyStr(), - os=S.AnyStr(), - arch=S.Opt(S.AnyStr()), - version=VERSION_SCHEMA, - location=RELPATH_SCHEMA, - packages=S.ListOf(S.Obj( - name=S.AnyStr(), - version=VERSION_SCHEMA, - path=RELPATH_SCHEMA, - hash=HASH_SCHEMA, - order=S.Struct([S.Int(), S.Int(), S.Int()]), - optional=S.Opt(S.Bool()), - gloss=S.DictOf(S.AnyStr(), S.AnyStr()), - longgloss=S.DictOf(S.AnyStr(), S.AnyStr())))) - -PACKAGE_SCHEMA = S.Obj( - _type=S.Str("Package"), - name=S.AnyStr(), - location=RELPATH_SCHEMA, - version=VERSION_SCHEMA, - format=S.Obj(), - ts=TIME_SCHEMA, - files=S.ListOf(S.Struct([RELPATH_SCHEMA, HASH_SCHEMA])), - shortdesc=S.DictOf(S.AnyStr(), S.AnyStr()), - longdesc=S.DictOf(S.AnyStr(), S.AnyStr())) - -ALL_ROLES = ('timestamp', 'mirrors', 'bundle', 'package', 'master') - -class Key: - #XXXX UNUSED. - def __init__(self, key, roles=()): - self.key = key - self.roles = [] - for r,p in roles: - self.addRole(r,p) - - def addRole(self, role, path): - assert role in ALL_ROLES - self.roles.append((role, path)) - - def getRoles(self): - return self.roles - - @staticmethod - def fromJSon(obj): - # must match PUBKEY_SCHEMA - keytype = obj['_keytype'] - if keytype == 'rsa': - return Key(glider.keys.RSAKey.fromJSon(obj)) - - if typeattr == 'rsa': - key = glider.keys.RSAKey.fromSExpression(sexpr) - if key is not None: - return Key(key) - else: - return None - - def format(self): - return self.key.format() - - def getKeyID(self): - return self.key.getKeyID() - - def sign(self, sexpr=None, digest=None): - return self.key.sign(sexpr, digest=digest) - - def checkSignature(self, method, data, signatute): - ok = self.key.checkSignature(method, data, signature) - # XXXX CACHE HERE. - return ok - -class Keylist(KeyDB): - def __init__(self): - KeyDB.__init__(self) - - def addFromKeylist(self, obj, allowMasterKeys=False): - for keyitem in obj['keys']: - key = keyitem['key'] - roles = keyitem['roles'] - - try: - key = glider.keys.RSAKey.fromJSon(key) - except glider.FormatException, e: - print e - #LOG skipping key. - continue - - for r,p in roles: - if r == 'master' and not allowMasterKeys: - #LOG - continue - if r not in ALL_ROLES: - continue - key.addRole(r,p) - - self.addKey(key) - -class StampedInfo: - def __init__(self, ts, hash, version=None, relpath=None): - self._ts = ts - self._hash = hash - self._version = version - self._relpath = relpath - - @staticmethod - def fromJSonFields(timeStr, hashStr): - t = parseTime(timeStr) - h = parseHash(hashStr) - return StampedInfo(t, h) - - def getHash(self): - return self._hash - - def getRelativePath(self): - return self._relpath - -class TimestampFile: - def __init__(self, at, mirrorlistinfo, keylistinfo, bundleinfo): - self._time = at - self._mirrorListInfo = mirrorlistinfo - self._keyListInfo = keylistinfo - self._bundleInfo = bundleinfo - - @staticmethod - def fromJSon(obj): - # must be validated. - at = parseTime(obj['at']) - m = StampedInfo.fromJSonFields(*obj['m'][:2]) - k = StampedInfo.fromJSonFields(*obj['k'][:2]) - b = {} - for name, bundle in obj['b'].iteritems(): - v = bundle[0] - rp = bundle[1] - t = parseTime(bundle[2]) - h = parseHash(bundle[3]) - b[name] = StampedInfo(t, h, v, rp) - - return TimestampFile(at, m, k, b) - - def getTime(self): - return self._time - - def getMirrorlistInfo(self): - return self._mirrorListInfo - - def getKeylistInfo(self): - return self._keyListInfo - - def getBundleInfo(self, name): - return self._bundleInfo[name] - -def readConfigFile(fname, needKeys=(), optKeys=(), preload={}): - parsed = preload.copy() - result = {} - execfile(fname, parsed) - - for k in needKeys: - try: - result[k] = parsed[k] - except KeyError: - raise glider.FormatError("Missing value for %s in %s"%k,fname) - - for k in optKeys: - try: - result[k] = parsed[k] - except KeyError: - pass - - return result - -def makePackageObj(config_fname, package_fname): - preload = {} - shortDescs = {} - longDescs = {} - def ShortDesc(lang, val): shortDescs[lang] = val - def LongDesc(lang, val): longDescs[lang] = val - preload = { 'ShortDesc' : ShortDesc, 'LongDesc' : LongDesc } - r = readConfigFile(config_fname, - ['name', - 'version', - 'format', - 'location', - 'relpath', - ], (), preload) - - f = open(package_fname, 'rb') - digest = getFileDigest(f) - - # Check fields! - result = { '_type' : "Package", - 'ts' : formatTime(time.time()), - 'name' : r['name'], - 'location' : r['location'], #DOCDOC - 'version' : r['version'], - 'format' : r['format'], - 'files' : [ [ r['relpath'], formatHash(digest) ] ], - 'shortdesc' : shortDescs, - 'longdesc' : longDescs - } - - PACKAGE_SCHEMA.checkMatch(result) - - return result - -def makeBundleObj(config_fname, getPackageHash): - packages = [] - def ShortGloss(lang, val): packages[-1]['gloss'][lang] = val - def LongGloss(lang, val): packages[-1]['longgloss'][lang] = val - def Package(name, version, path, order, optional=False): - packages.append({'name' : name, - 'version' : version, - 'path' : path, - 'order' : order, - 'optional' : optional, - 'gloss' : {}, - 'longgloss' : {} }) - preload = { 'ShortGloss' : ShortGloss, 'LongGloss' : LongGloss, - 'Package' : Package } - r = readConfigFile(config_fname, - ['name', - 'os', - 'version', - 'location', - ], ['arch'], preload) - - result = { '_type' : "Bundle", - 'at' : formatTime(time.time()), - 'name' : r['name'], - 'os' : r['os'], - 'version' : r['version'], - 'location' : r['location'], - 'packages' : packages } - if r.has_key('arch'): - result['arch'] = r['arch'] - - for p in packages: - try: - p['hash'] = formatHash(getPackageHash(p['path'])) - except KeyError: - raise glider.FormatException("No such package as %s"%p['path']) - - BUNDLE_SCHEMA.checkMatch(result) - return result - -def versionIsNewer(v1, v2): - return v1 > v2 - -def makeTimestampObj(mirrorlist_obj, keylist_obj, - bundle_objs): - result = { '_type' : 'Timestamp', - 'at' : formatTime(time.time()) } - result['m'] = [ mirrorlist_obj['ts'], - formatHash(getDigest(mirrorlist_obj)) ] - result['k'] = [ keylist_obj['ts'], - formatHash(getDigest(keylist_obj)) ] - result['b'] = bundles = {} - for bundle in bundle_objs: - name = bundle['name'] - v = bundle['version'] - entry = [ v, bundle['location'], bundle['at'], formatHash(getDigest(bundle)) ] - if not bundles.has_key(name) or versionIsNewer(v, bundles[name][0]): - bundles[name] = entry - - TIMESTAMP_SCHEMA.checkMatch(result) - - return result - -class MirrorInfo: - def __init__(self, name, urlbase, contents, weight): - self._name = name - self._urlbase = urlbase - self._contents = contents - self._weight = weight - - def canServeFile(self, fname): - for c in self._contents: - if rolePathMatches(c, fname): - return True - return False - - def getFileURL(self, fname): - if self._urlbase[-1] == '/': - return self._urlbase+fname - else: - return "%s/%s" % (self._urlbase, fname) - - def format(self): - return { 'name' : self._name, - 'urlbase' : self._urlbase, - 'contents' : self._contents, - 'weight' : self._weight } - -def makeMirrorListObj(mirror_fname): - mirrors = [] - def Mirror(*a, **kw): mirrors.append(MirrorInfo(*a, **kw)) - preload = {'Mirror' : Mirror} - r = readConfigFile(mirror_fname, (), (), preload) - result = { '_type' : "Mirrorlist", - 'ts' : formatTime(time.time()), - 'mirrors' : [ m.format() for m in mirrors ] } - - MIRRORLIST_SCHEMA.checkMatch(result) - return result - -def makeKeylistObj(keylist_fname, includePrivate=False): - keys = [] - def Key(obj): keys.append(obj) - preload = {'Key': Key} - r = readConfigFile(keylist_fname, (), (), preload) - - klist = [] - for k in keys: - k = glider.keys.RSAKey.fromJSon(k) - klist.append({'key': k.format(private=includePrivate), 'roles' : k.getRoles() }) - - result = { '_type' : "Keylist", - 'ts' : formatTime(time.time()), - 'keys' : klist } - - KEYLIST_SCHEMA.checkMatch(result) - return result - -SCHEMAS_BY_TYPE = { - 'Keylist' : KEYLIST_SCHEMA, - 'Mirrorlist' : MIRRORLIST_SCHEMA, - 'Timestamp' : TIMESTAMP_SCHEMA, - 'Bundle' : BUNDLE_SCHEMA, - 'Package' : PACKAGE_SCHEMA, - } - -def checkSignedObj(obj, keydb=None): - # Returns signaturestatus, role, path on sucess. - - SIGNED_SCHEMA.checkMatch(obj) - try: - tp = obj['signed']['_type'] - except KeyError: - raise glider.FormatException("Untyped object") - try: - schema = SCHEMAS_BY_TYPE[tp] - except KeyError: - raise glider.FormatException("Unrecognized type %r" % tp) - schema.checkMatch(obj['signed']) - - if tp == 'Keylist': - role = "master" - path = "/meta/keys.txt" - elif tp == 'Mirrorlist': - role = "mirrors" - path = "/meta/mirrors.txt" - elif tp == "Timestamp": - role = 'timestamp' - path = "/meta/timestamp.txt" - elif tp == 'Bundle': - role = 'bundle' - path = obj['signed']['location'] - elif tp == 'Package': - role = 'package' - path = obj['signed']['location'] - else: - print tp - raise "Foo" - - ss = None - if keydb is not None: - ss = checkSignatures(obj, keydb, role, path) - - return ss, role, path diff --git a/lib/glider/keys.py b/lib/glider/keys.py deleted file mode 100644 index fefff6c..0000000 --- a/lib/glider/keys.py +++ /dev/null @@ -1,399 +0,0 @@ - -# These require PyCrypto. -import Crypto.PublicKey.RSA -import Crypto.Hash.SHA256 -import Crypto.Cipher.AES - -import cPickle as pickle -import binascii -import logging -import os -import struct -import sys -import simplejson -import getpass - -import glider.formats -import glider.util - -class PublicKey: - def __init__(self): - # Confusingly, these roles are the ones used for a private key to - # remember what we're willing to do with it. - self._roles = [] - def format(self): - raise NotImplemented() - def sign(self, data): - # returns a list of method,signature tuples. - raise NotImplemented() - def checkSignature(self, method, data, signature): - # returns True, False, or raises UnknownMethod. - raise NotImplemented() - def getKeyID(self): - raise NotImplemented() - def getRoles(self): - return self._roles - def addRole(self, role, path): - assert role in glider.formats.ALL_ROLES - self._roles.append((role, path)) - def clearRoles(self): - del self._roles[:] - def hasRole(self, role, path): - for r, p in self._roles: - if r == role and glider.formats.rolePathMatches(p, path): - return True - return False - -if hex(1L).upper() == "0X1L": - def intToBinary(number): - """Convert an int or long into a big-endian series of bytes. - """ - # This "convert-to-hex, then use binascii" approach may look silly, - # but it's over 10x faster than the Crypto.Util.number approach. - h = hex(long(number)) - h = h[2:-1] - if len(h)%2: - h = "0"+h - return binascii.a2b_hex(h) -elif hex(1L).upper() == "0X1": - def intToBinary(number): - h = hex(long(number)) - h = h[2:] - if len(h)%2: - h = "0"+h - return binascii.a2b_hex(h) -else: - import Crypto.Util.number - intToBinary = Crypto.Util.number.long_to_bytes - assert None - -def binaryToInt(binary): - """Convert a big-endian series of bytes into a long. - """ - return long(binascii.b2a_hex(binary), 16) - -def intToBase64(number): - return glider.formats.formatBase64(intToBinary(number)) - -def base64ToInt(number): - return binaryToInt(glider.formats.parseBase64(number)) - -def _pkcs1_padding(m, size): - # I'd rather use OAEP+, but apparently PyCrypto barely supports - # signature verification, and doesn't seem to support signature - # verification with nondeterministic padding. "argh." - - s = [ "\x00\x01", "\xff"* (size-3-len(m)), "\x00", m ] - r = "".join(s) - return r - -def _xor(a,b): - if a: - return not b - else: - return b - -class RSAKey(PublicKey): - """ - >>> k = RSAKey.generate(bits=512) - >>> obj = k.format() - >>> obj['_keytype'] - 'rsa' - >>> base64ToInt(obj['e']) - 65537L - >>> k1 = RSAKey.fromJSon(obj) - >>> k1.key.e == k.key.e - True - >>> k1.key.n == k.key.n - True - >>> k.getKeyID() == k1.getKeyID() - True - >>> s = { 'A B C' : "D", "E" : [ "F", "g", 99] } - >>> method, sig = k.sign(obj=s) - >>> k.checkSignature(method, sig, obj=s) - True - >>> s2 = [ s ] - >>> k.checkSignature(method, sig, obj=s2) - False - """ - def __init__(self, key): - PublicKey.__init__(self) - self.key = key - self.keyid = None - - @staticmethod - def generate(bits=2048): - key = Crypto.PublicKey.RSA.generate(bits=bits, randfunc=os.urandom) - return RSAKey(key) - - @staticmethod - def fromJSon(obj): - # obj must match RSAKEY_SCHEMA - - glider.formats.RSAKEY_SCHEMA.checkMatch(obj) - n = base64ToInt(obj['n']) - e = base64ToInt(obj['e']) - if obj.has_key('d'): - d = base64ToInt(obj['d']) - p = base64ToInt(obj['p']) - q = base64ToInt(obj['q']) - u = base64ToInt(obj['u']) - key = Crypto.PublicKey.RSA.construct((n, e, d, p, q, u)) - else: - key = Crypto.PublicKey.RSA.construct((n, e)) - - result = RSAKey(key) - if obj.has_key('roles'): - for r, p in obj['roles']: - result.addRole(r,p) - - return result - - def isPrivateKey(self): - return hasattr(self.key, 'd') - - def format(self, private=False, includeRoles=False): - n = intToBase64(self.key.n) - e = intToBase64(self.key.e) - result = { '_keytype' : 'rsa', - 'e' : e, - 'n' : n } - if private: - result['d'] = intToBase64(self.key.d) - result['p'] = intToBase64(self.key.p) - result['q'] = intToBase64(self.key.q) - result['u'] = intToBase64(self.key.u) - if includeRoles: - result['roles'] = self.getRoles() - return result - - def getKeyID(self): - if self.keyid == None: - d_obj = Crypto.Hash.SHA256.new() - glider.formats.getDigest(self.format(), d_obj) - self.keyid = glider.formats.formatHash(d_obj.digest()) - return self.keyid - - def _digest(self, obj, method=None): - if method in (None, "sha256-pkcs1"): - d_obj = Crypto.Hash.SHA256.new() - glider.formats.getDigest(obj, d_obj) - digest = d_obj.digest() - return ("sha256-pkcs1", digest) - - raise UnknownMethod(method) - - def sign(self, obj=None, digest=None): - assert _xor(obj == None, digest == None) - if digest == None: - method, digest = self._digest(obj) - m = _pkcs1_padding(digest, (self.key.size()+1) // 8) - sig = intToBase64(self.key.sign(m, "")[0]) - return (method, sig) - - def checkSignature(self, method, sig, obj=None, digest=None): - assert _xor(obj == None, digest == None) - if method != "sha256-pkcs1": - raise UnknownMethod("method") - if digest == None: - method, digest = self._digest(obj, method) - sig = base64ToInt(sig) - m = _pkcs1_padding(digest, (self.key.size()+1) // 8) - return bool(self.key.verify(m, (sig,))) - -SALTLEN=16 - -def secretToKey(salt, secret): - """Convert 'secret' to a 32-byte key, using a version of the algorithm - from RFC2440. The salt must be SALTLEN+1 bytes long, and should - be random, except for the last byte, which encodes how time- - consuming the computation should be. - - (The goal is to make offline password-guessing attacks harder by - increasing the time required to convert a password to a key, and to - make precomputed password tables impossible to generate by ) - """ - assert len(salt) == SALTLEN+1 - - # The algorithm is basically, 'call the last byte of the salt the - # "difficulty", and all other bytes of the salt S. Now make - # an infinite stream of S|secret|S|secret|..., and hash the - # first N bytes of that, where N is determined by the difficulty. - # - # Obviously, this wants a hash algorithm that's tricky to - # parallelize. - # - # Unlike RFC2440, we use a 16-byte salt. Because CPU times - # have improved, we start at 16 times the previous minimum. - - difficulty = ord(salt[-1]) - count = (16L+(difficulty & 15)) << ((difficulty >> 4) + 10) - - # Make 'data' nice and long, so that we don't need to call update() - # a zillion times. - data = salt[:-1]+secret - if len(data)<1024: - data *= (1024 // len(data))+1 - - d = Crypto.Hash.SHA256.new() - iters, leftover = divmod(count, len(data)) - for _ in xrange(iters): - d.update(data) - #count -= len(data) - if leftover: - d.update(data[:leftover]) - #count -= leftover - #assert count == 0 - - return d.digest() - -def encryptSecret(secret, password, difficulty=0x80): - """Encrypt the secret 'secret' using the password 'password', - and return the encrypted result.""" - # The encrypted format is: - # "GKEY1" -- 5 octets, fixed, denotes data format. - # SALT -- 17 bytes, used to hash password - # IV -- 16 bytes; salt for encryption - # ENCRYPTED IN AES256-OFB, using a key=s2k(password, salt) and IV=IV: - # SLEN -- 4 bytes; length of secret, big-endian. - # SECRET -- len(secret) bytes - # D -- 32 bytes; SHA256 hash of (salt|secret|salt). - # - # This format leaks the secret length, obviously. - assert 0 <= difficulty < 256 - salt = os.urandom(SALTLEN)+chr(difficulty) - key = secretToKey(salt, password) - - d_obj = Crypto.Hash.SHA256.new() - d_obj.update(salt) - d_obj.update(secret) - d_obj.update(salt) - d = d_obj.digest() - - iv = os.urandom(16) - e = Crypto.Cipher.AES.new(key, Crypto.Cipher.AES.MODE_OFB, iv) - - # Stupidly, pycrypto doesn't accept that stream ciphers don't need to - # take their input in blocks. So pad it, then ignore the padded output. - - padlen = 16-((len(secret)+len(d)+4) % 16) - if padlen == 16: padlen = 0 - pad = '\x00' * padlen - - slen = struct.pack("!L",len(secret)) - encrypted = e.encrypt("%s%s%s%s" % (slen, secret, d, pad)) - if padlen: - encrypted = encrypted[:-padlen] - return "GKEY1%s%s%s"%(salt, iv, encrypted) - -def decryptSecret(encrypted, password): - """Decrypt a value encrypted with encryptSecret. Raises UnknownFormat - or FormatError if 'encrypted' was not generated with encryptSecret. - Raises BadPassword if the password was not correct. - """ - if encrypted[:5] != "GKEY1": - raise glider.UnknownFormat() - encrypted = encrypted[5:] - if len(encrypted) < SALTLEN+1+16: - raise glider.FormatException() - - salt = encrypted[:SALTLEN+1] - iv = encrypted[SALTLEN+1:SALTLEN+1+16] - encrypted = encrypted[SALTLEN+1+16:] - - key = secretToKey(salt, password) - - e = Crypto.Cipher.AES.new(key, Crypto.Cipher.AES.MODE_OFB, iv) - padlen = 16-(len(encrypted) % 16) - if padlen == 16: padlen = 0 - pad = '\x00' * padlen - - decrypted = e.decrypt("%s%s"%(encrypted,pad)) - slen = struct.unpack("!L", decrypted[:4])[0] - secret = decrypted[4:4+slen] - hash = decrypted[4+slen:4+slen+Crypto.Hash.SHA256.digest_size] - - d = Crypto.Hash.SHA256.new() - d.update(salt) - d.update(secret) - d.update(salt) - - if d.digest() != hash: - raise glider.BadPassword() - - return secret - -class KeyStore(glider.formats.KeyDB): - def __init__(self, fname, encrypted=True): - glider.formats.KeyDB.__init__(self) - - self._loaded = None - self._fname = fname - self._passwd = None - self._encrypted = encrypted - - def getpass(self, reprompt=False): - if self._passwd != None: - return self._passwd - while 1: - pwd = getpass.getpass("Password: ", sys.stderr) - if not reprompt: - return pwd - - pwd2 = getpass.getpass("Confirm: ", sys.stderr) - if pwd == pwd2: - return pwd - else: - print "Mismatch; try again." - - def load(self, password=None): - logging.info("Loading private keys from %r...", self._fname) - if not os.path.exists(self._fname): - logging.info("...no such file.") - self._loaded = True - return - - if password is None and self._encrypted: - password = self.getpass() - - contents = open(self._fname, 'rb').read() - if self._encrypted: - contents = decryptSecret(contents, password) - - listOfKeys = simplejson.loads(contents) - self._passwd = password # It worked. - if not listOfKeys.has_key('keys'): - listOfKeys['keys'] = [] - for obj in listOfKeys['keys']: - key = RSAKey.fromJSon(obj) - self.addKey(key) - logging.info("Loaded key %s", key.getKeyID()) - - self._loaded = True - - def setPassword(self, passwd): - self._passwd = passwd - - def clearPassword(self): - self._passwd = None - - def save(self, password=None): - if not self._loaded and self._encrypted: - self.load(password) - - if password is None: - password = self.getpass(True) - - logging.info("Saving private keys into %r...", self._fname) - listOfKeys = { 'keys' : - [ key.format(private=True, includeRoles=True) for key in - self._keys.values() ] - } - contents = simplejson.dumps(listOfKeys) - if self._encrypted: - contents = encryptSecret(contents, password) - glider.util.replaceFile(self._fname, contents) - self._passwd = password # It worked. - logging.info("Done.") - - diff --git a/lib/glider/master_keys.py b/lib/glider/master_keys.py deleted file mode 100644 index 0d455d1..0000000 --- a/lib/glider/master_keys.py +++ /dev/null @@ -1,5 +0,0 @@ - - -MASTER_KEYS = [ - -] diff --git a/lib/glider/repository.py b/lib/glider/repository.py deleted file mode 100644 index b956993..0000000 --- a/lib/glider/repository.py +++ /dev/null @@ -1,313 +0,0 @@ - -import glider.formats -import glider.util - -import simplejson -import logging -import os -import threading -import time - -MAX_TIMESTAMP_AGE = 24*60*60 - -class RepositoryFile: - def __init__(self, repository, relativePath, schema, - needRole=None, signedFormat=True, needSigs=1): - self._repository = repository - self._relativePath = relativePath - self._schema = schema - self._needRole = needRole - self._signedFormat = signedFormat - self._needSigs = needSigs - - self._signed_obj = self._main_obj = None - self._sigStatus = None - self._mtime = None - - def getRelativePath(self): - return self._relativePath - - def getPath(self): - return self._repository.getFilename(self._relativePath) - - def _load(self): - fname = self.getPath() - - # Propagate OSError - f = None - fd = os.open(fname, os.O_RDONLY) - try: - f = os.fdopen(fd, 'r') - except: - os.close(fd) - raise - try: - mtime = os.fstat(fd).st_mtime - content = f.read() - finally: - f.close() - - signed_obj,main_obj = self._checkContent(content) - - self._signed_obj = signed_obj - self._main_obj = main_obj - self._mtime = mtime - - def _save(self, content=None): - if content == None: - content = sexpr.encode - - signed_obj,main_obj = self._checkContent(content) - - fname = self.getPath() - glider.util.replaceFile(fname, contents) - - self._signed_obj = signed_obj - self._main_obj = main_obj - self._mtime = mtime - - def _checkContent(self, content): - - try: - obj = simplejson.loads(content) - except ValueError, e: - raise glider.FormatException("Couldn't decode content: %s"%e) - - if self._signedFormat: - # This is supposed to be signed. - glider.formats.SIGNED_SCHEMA.checkMatch(obj) - - main_obj = obj['signed'] - signed_obj = obj - else: - signed_obj = None - main_obj = obj - - if self._schema != None: - self._schema.checkMatch(main_obj) - - return signed_obj, main_obj - - def load(self): - if self._main_obj == None: - self._load() - - def get(self): - return self._main_obj - - def isLoaded(self): - return self._main_obj != None - - def getContent(self): - self.load() - return self._main_obj - - def _checkSignatures(self): - self.load() - sigStatus = glider.formats.checkSignatures(self._signed_obj, - self._repository._keyDB, - self._needRole, self._relativePath) - self._sigStatus = sigStatus - - def checkSignatures(self): - if self._sigStatus is None: - self._checkSignatures() - return self._sigStatus - -class LocalRepository: - def __init__(self, root): - self._root = root - self._keyDB = glider.util.getKeylist(None) - - self._keylistFile = RepositoryFile( - self, "/meta/keys.txt", glider.formats.KEYLIST_SCHEMA, - needRole="master") - self._timestampFile = RepositoryFile( - self, "/meta/timestamp.txt", glider.formats.TIMESTAMP_SCHEMA, - needRole="timestamp") - self._mirrorlistFile = RepositoryFile( - self, "/meta/mirrors.txt", glider.formats.MIRRORLIST_SCHEMA, - needRole="mirrors") - self._metaFiles = [ self._keylistFile, - self._timestampFile, - self._mirrorlistFile ] - - self._packageFiles = {} - self._bundleFiles = {} - - def getFilename(self, relativePath): - if relativePath.startswith("/"): - relativePath = relativePath[1:] - return os.path.join(self._root, relativePath) - - def getKeylistFile(self): - return self._keylistFile - - def getTimestampFile(self): - return self._timestampFile - - def getMirrorlistFile(self): - return self._mirrorlistFile - - def getPackageFile(self, relPath): - try: - return self._packageFiles[relPath] - except KeyError: - self._packageFiles[relPath] = pkg = RepositoryFile( - self, relPath, glider.formats.PACKAGE_SCHEMA, - needRole='package') - return pkg - - def getBundleFile(self, relPath): - try: - return self._bundleFiles[relPath] - except KeyError: - self._bundleFiles[relPath] = pkg = RepositoryFile( - self, relPath, glider.formats.BUNDLE_SCHEMA, - needRole='bundle') - return pkg - - def getFilesToUpdate(self, now=None, trackingBundles=()): - if now == None: - now = time.time() - - need = set() - - # Fetch missing metafiles. - for f in self._metaFiles: - try: - f.load() - except OSError, e: - print "need", f.getPath() - logging.info("Couldn't load %s: %s. Must fetch it.", - f.getPath(), e) - need.add(f.getRelativePath()) - - # If the timestamp file is out of date, we need to fetch it no - # matter what. (Even if it is isn't signed, it can't possibly - # be good.) - ts = self._timestampFile.get() - if ts: - age = now - glider.formats.parseTime(ts['at']) - ts = glider.formats.TimestampFile.fromJSon(ts) - if age > MAX_TIMESTAMP_AGE: - need.add(self._timestampFile.getRelativePath()) - - # If the keylist isn't signed right, we can't check the - # signatures on anything else. - if self._keylistFile.get(): - s = self._keylistFile.checkSignatures() - if not s.isValid(): # For now only require one master key. - need.add(self._keylistFile.getRelativePath()) - - if need: - return need - - # Import the keys from the keylist. - self._keyDB.addFromKeylist(self._keylistFile.get()) - - # If the timestamp isn't signed right, get a new timestamp and a - # new keylist. - s = self._timestampFile.checkSignatures() - if not s.isValid(): - need.add(self._keylistFile.getRelativePath()) - need.add(self._timestampFile.getRelativePath()) - return need - - # FINALLY, we know we have an up-to-date, signed timestamp - # file. Check whether the keys and mirrors file are as - # authenticated. - h_kf = glider.formats.getDigest(self._keylistFile.get()) - h_expected = ts.getKeylistInfo().getHash() - if h_kf != h_expected: - need.add(self._keylistFile.getRelativePath()) - - if need: - return need - - s = self._mirrorlistFile.checkSignatures() - if not s.isValid(): - need.add(self._mirrorlistFile.getRelativePath()) - - h_mf = glider.formats.getDigest(self._mirrorlistFile.get()) - h_expected = ts.getMirrorlistInfo().getHash() - if h_mf != h_expected: - need.add(self._mirrorlistFile.getRelativePath()) - - if need: - return need - - # Okay; that's it for the metadata. Do we have the right - # bundles? - bundles = {} - for b in trackingBundles: - try: - binfo = ts.getBundleInfo(b) - except KeyError: - logging.warn("Unrecognized bundle %s"%b) - continue - - rp = binfo.getRelativePath() - bfile = self.getBundleFile(rp) - try: - bfile.load() - except OSError: - need.add(rp) - continue - - h_b = glider.formats.getDigest(bfile.get()) - h_expected = binfo.getHash() - if h_b != h_expected: - need.add(rp) - continue - - s = bfile.checkSignatures() - if not s.isValid(): - # Can't actually use it. - continue - - bundles[rp] = bfile - - # Okay. So we have some bundles. See if we have their packages. - packages = {} - for bfile in bundles.values(): - bundle = bfile.get() - for pkginfo in bundle['packages']: - rp = pkginfo['path'] - pfile = self.getPackageFile(rp) - try: - pfile.load() - except OSError: - need.add(rp) - continue - - h_p = glider.formats.getDigest(pfile.get()) - h_expected = glider.formats.parseHash(pkginfo['hash']) - if h_p != h_expected: - need.add(rp) - continue - - s = pfile.checkSignatures() - if not s.isValid(): - # Can't use it. - continue - packages[rp] = pfile - - # Finally, we have some packages. Do we have their underlying - # files? - for pfile in packages.values(): - package = pfile.get() - for f in package['files']: - rp, h = f[:2] - h_expected = glider.formats.parseHash(h) - fn = self.getFilename(rp) - try: - h_got = glider.formats.getFileDigest(fn) - except OSError: - need.add(rp) - continue - if h_got != h_expected: - need.add(rp) - - # Okay; these are the files we need. - return need diff --git a/lib/glider/tests.py b/lib/glider/tests.py deleted file mode 100644 index 7aeb4f6..0000000 --- a/lib/glider/tests.py +++ /dev/null @@ -1,64 +0,0 @@ - -import unittest -import doctest -import os -import tempfile - -import glider.keys -import glider.formats -import glider.repository -import glider.checkJson - -import glider.tests - -class CanonicalEncodingTest(unittest.TestCase): - def test_encode(self): - enc = glider.formats.encodeCanonical - self.assertEquals(enc(''), '""') - self.assertEquals(enc('"'), '"\\""') - self.assertEquals(enc('\t\\\n"\r'), - '"\t\\\\\n\\"\r"') - -class CryptoTests(unittest.TestCase): - def test_encrypt(self): - s = "The Secret words are marzipan habidashery zeugma." - password = "the password is swordfish." - encrypted = glider.keys.encryptSecret(s, password) - self.assertNotEquals(encrypted, s) - self.assert_(encrypted.startswith("GKEY1")) - self.assertEquals(s, glider.keys.decryptSecret(encrypted, password)) - self.assertRaises(glider.BadPassword, glider.keys.decryptSecret, - encrypted, "password") - self.assertRaises(glider.UnknownFormat, glider.keys.decryptSecret, - "foobar", password) - - def test_keystore(self): - passwd = "umfitty noonah" - fname = tempfile.mktemp() - ks = glider.keys.KeyStore(fname) - key1 = glider.keys.RSAKey.generate(512) - key2 = glider.keys.RSAKey.generate(512) - ks.addKey(key1) - ks.addKey(key2) - ks.save(passwd) - - ks2 = glider.keys.KeyStore(fname) - ks2.load(passwd) - self.assertEquals(key1.key.n, ks2.getKey(key1.getKeyID()).key.n) - -def suite(): - suite = unittest.TestSuite() - - suite.addTest(doctest.DocTestSuite(glider.formats)) - suite.addTest(doctest.DocTestSuite(glider.keys)) - suite.addTest(doctest.DocTestSuite(glider.checkJson)) - - loader = unittest.TestLoader() - suite.addTest(loader.loadTestsFromModule(glider.tests)) - - return suite - - -if __name__ == '__main__': - - unittest.TextTestRunner(verbosity=1).run(suite()) diff --git a/lib/glider/util.py b/lib/glider/util.py deleted file mode 100644 index e2fb57a..0000000 --- a/lib/glider/util.py +++ /dev/null @@ -1,73 +0,0 @@ - -import os -import sys -import tempfile - -import simplejson - -import glider.formats -import glider.keys -import glider.master_keys - -def moveFile(fromLocation, toLocation): - if sys.platform in ('cygwin', 'win32'): - # Win32 doesn't let rename replace an existing file. - try: - os.unlink(toLocation) - except OSError: - pass - os.rename(fromLocation, toLocation) - - -def replaceFile(fname, contents, textMode=False): - """overwrite the file in 'fname' atomically with the content of 'contents' - """ - dir, prefix = os.path.split(fname) - fd, fname_tmp = tempfile.mkstemp(prefix=prefix, dir=dir, text=textMode) - - try: - os.write(fd, contents) - finally: - os.close(fd) - - moveFile(fname_tmp, fname) - -def userFilename(name): - try: - base = os.environ["THANDY_HOME"] - except KeyError: - base = "~/.thandy" - base = os.path.expanduser(base) - if not os.path.exists(base): - os.makedirs(base, 0700) - return os.path.join(base, name) - -def getKeylist(keys_fname, checkKeys=True): - import glider.master_keys - - keydb = glider.formats.Keylist() - - for key in glider.master_keys.MASTER_KEYS: - keydb.addKey(key) - - user_keys = userFilename("preload_keys") - if os.path.exists(user_keys): - #XXXX somewhat roundabout. - keylist = glider.formats.makeKeylistObj(user_keys) - keydb.addFromKeylist(keylist, allowMasterKeys=True) - - if keys_fname and os.path.exists(keys_fname): - f = open(keys_fname, 'r') - try: - obj = simplejson.load(f) - finally: - f.close() - ss, role, path = glider.formats.checkSignedObj(obj, keydb) - if role != 'master': - raise glider.FormatException("%s wasn't a keylist."%keys_fname) - if checkKeys and not ss.isValid(): - raise glider.FormatException("%s not signed by enough master keys"% - keys_fname) - keydb.addFromKeylist(obj['signed'], allowMasterKeys=False) - - return keydb diff --git a/lib/thandy/ClientCLI.py b/lib/thandy/ClientCLI.py new file mode 100644 index 0000000..702ebbf --- /dev/null +++ b/lib/thandy/ClientCLI.py @@ -0,0 +1,66 @@ + +import os +import sys +import getopt + +import thandy.util +import thandy.repository +import thandy.download + +def update(args): + repoRoot = thandy.util.userFilename("cache") + options, args = getopt.getopt(args, "", [ "repo=", "no-download" ]) + download = True + + for o, v in options: + if o == '--repo': + repoRoot = v + elif o == "--no-download": + download = False + + repo = thandy.repository.LocalRepository(repoRoot) + + files = repo.getFilesToUpdate(trackingBundles=args) + + if not download: + return + + mirrorlist = repo.getMirrorlistFile().get() + + downloader = thandy.download.Downloads() + downloader.start() + + for f in files: + # XXXX Use hash. + dj = thandy.download.DownloadJob(f, repo.getFilename(f), + mirrorlist) + downloader.addDownloadJob(dj) + # XXXX replace file in repository if ok; reload; see what changed. + + # Wait for in-progress jobs + +# Check my repository + +# Tell me what I need to download + +# Download stuff + +# Tell me what to install. + +def usage(): + print "Known commands:" + print " update [--repo=repository] [--no-download]" + sys.exit(1) + +def main(): + if len(sys.argv) < 2: + usage() + cmd = sys.argv[1] + args = sys.argv[2:] + if cmd in [ "update" ]: + globals()[cmd](args) + else: + usage() + +if __name__ == '__main__': + main() diff --git a/lib/thandy/ServerCLI.py b/lib/thandy/ServerCLI.py new file mode 100644 index 0000000..47fa912 --- /dev/null +++ b/lib/thandy/ServerCLI.py @@ -0,0 +1,187 @@ + +import os +import sys +import getopt +import time + +import simplejson + +import thandy.formats +import thandy.util +import thandy.keys + +def tstamp(): + return time.strftime("%Y%m%d_%H%M%S", time.localtime()) + +def snarf(fname): + f = open(fname, 'rb') + try: + return f.read() + finally: + f.close() + +def snarfObj(fname): + f = open(fname, 'r') + try: + return simplejson.load(f) + finally: + f.close() + +def insert(args): + repo = os.environ.get("THANDY_MASTER_REPO") + backupDir = thandy.util.userFilename("old_files") + checkSigs = True + + options, args = getopt.getopt(args, "", ["repo=", "no-check"]) + for o,v in options: + if o == "--repo": + repo = v + elif o == "--no-check": + checkSigs = False + + if not repo: + print "No repository specified." + usage() + if not os.path.exists(repo): + print "No such repository as %r"%repo + usage() + + if not os.path.exists(backupDir): + os.makedirs(backupDir, 0700) + + if checkSigs: + keys = thandy.util.getKeylist(os.path.join(repo, "meta/keys.txt")) + else: + keys = None + + n_ok = 0 + for fn in args: + print "Loading %s..."%fn + try: + content = snarf(fn) + except OSError, e: + print "Couldn't open %s: %s"%(fn, e) + continue + + try: + obj = simplejson.loads(content) + except ValueError, e: + print "Couldn't decode %s: %s"%(fn, e) + continue + + try: + ss, r, path = thandy.formats.checkSignedObj(obj, keys) + except thandy.FormatException, e: + print "Bad format on %s: %s"%(fn, e) + continue + if checkSigs and not ss.isValid(): + print "Not enough valid signatures on %s"%fn + continue + + print " Looks okay. It goes in %s"%path + assert path.startswith("/") + targetPath = os.path.join(repo, path[1:]) + if os.path.exists(targetPath): + oldContents = snarf(targetPath) + if oldContents == content: + print " File unchanged!" + n_ok += 1 + continue + + baseFname = "%s_%s" % (tstamp(), os.path.split(path)[1]) + backupFname = os.path.join(backupDir, baseFname) + print " Copying old file to %s"%backupFname + thandy.util.replaceFile(backupFname, oldContents) + + parentDir = os.path.split(targetPath)[0] + if not os.path.exists(parentDir): + print " Making %s"%parentDir + os.makedirs(parentDir, 0755) + print " Replacing file..." + thandy.util.replaceFile(targetPath, content) + print " Done." + n_ok += 1 + if n_ok != len(args): + sys.exit(1) + +def timestamp(args): + repo = os.environ.get("THANDY_MASTER_REPO") + ts_keyfile = thandy.util.userFilename("timestamp_key") + + options, args = getopt.getopt(args, "", ["repo=", "ts-key="]) + for o,v in options: + if o == "--repo": + repo = v + elif o == "--ts-key": + ts_keyfile = v + + if repo == None: + print "No repository specified." + usage() + if not os.path.exists(repo): + print "No such repository as %r"%repo + usage() + + tsFname = os.path.join(repo, "meta/timestamp.txt") + + try: + mObj = snarfObj(os.path.join(repo, "meta/mirrors.txt")) + except OSError: + print "No mirror list!" + sys.exit(1) + try: + kObj = snarfObj(os.path.join(repo, "meta/keys.txt")) + except OSError: + print "No key list!" + sys.exit(1) + + bundles = [] + for dirpath, dirname, fns in os.walk(os.path.join(repo, "bundleinfo")): + for fn in fns: + try: + bObj = snarfObj(fn) + except (ValueError, OSError), e: + print "(Couldn't read bundle-like %s)"%fn + continue + try: + _, r, _ = thandy.formats.checkSignedObj(bObj) + except thandy.FormatException, e: + print "Problem reading object from %s"%fn + continue + if r != "bundle": + print "%s was not a good bundle"%fn + continue + bundles.append(bObj['signed']) + + timestamp = thandy.formats.makeTimestampObj( + mObj['signed'], kObj['signed'], bundles) + signable = thandy.formats.makeSignable(timestamp) + + keydb = thandy.formats.Keylist() + #XXXX Still a roundabout way to do this. + keylist = thandy.formats.makeKeylistObj(ts_keyfile, True) + keydb.addFromKeylist(keylist) + for k in keydb.iterkeys(): + thandy.formats.sign(signable, k) + + content = simplejson.dumps(signable, sort_keys=True) + thandy.util.replaceFile(tsFname, content) + +def usage(): + print "Known commands:" + print " insert [--no-check] [--repo=repository] file ..." + print " timestamp [--repo=repository]" + sys.exit(1) + +def main(): + if len(sys.argv) < 2: + usage() + cmd = sys.argv[1] + args = sys.argv[2:] + if cmd in [ "insert", "timestamp" ]: + globals()[cmd](args) + else: + usage() + +if __name__ == '__main__': + main() diff --git a/lib/thandy/SignerCLI.py b/lib/thandy/SignerCLI.py new file mode 100644 index 0000000..4facc2e --- /dev/null +++ b/lib/thandy/SignerCLI.py @@ -0,0 +1,313 @@ + +import os +import getopt +import sys +import logging +import simplejson + +import thandy.keys +import thandy.formats + +def getKeyStore(): + return thandy.keys.KeyStore(thandy.util.userFilename("secret_keys")) + +def dumpKey(key, indent=0): + i = " "*indent + print "%s%s"%(i, key.getKeyID()) + for r, p in key.getRoles(): + print " %s%s\t%s"%(i, r, p) + +def getKey(ks, keyid=None, role=None, path=None): + if keyid is not None: + keys = ks.getKeysFuzzy(keyid) + if None not in (role, path): + keys = [ k for k in keys if k.hasRole(role, path) ] + elif None not in (role, path): + keys = ks.getKeysByRole(role, path) + else: + assert False + if len(keys) < 1: + print "No such key.\nI wanted", + if keyid: print "keyid='%s...'"%keyid, + if None not in (role, path): print "role=%s, path=%s"%(role,path), + print + print "I only know about:" + for k in ks.iterkeys(): + dumpKey(k) + sys.exit(1) + elif len(keys) > 1: + print "Multiple keys match. Possibilities are:" + for k in keys: + dumpKey(k) + sys.exit(1) + else: + return keys[0] + +# ------------------------------ + +def makepackage(args): + options, args = getopt.getopt(args, "", "keyid=") + keyid = None + for o,v in options: + if o == "--keyid": + keyid = v + + if len(args) < 2: + usage() + + configFile = args[0] + dataFile = args[1] + print "Generating package." + package = thandy.formats.makePackageObj(configFile, dataFile) + relpath = package['location'] + print "need a key with role matching [package %s]"%relpath + ks = getKeyStore() + ks.load() + key = getKey(ks, keyid=keyid, role='package', path=relpath) + signable = thandy.formats.makeSignable(package) + thandy.formats.sign(signable, key) + + if 1: + ss, r, p = thandy.formats.checkSignedObj(signable, ks) + assert ss.isValid() + + location = os.path.split(package['location'])[-1] + print "Writing signed package to %s"%location + f = open(location, 'w') + simplejson.dump(signable, f, indent=1) + f.close() + +def makebundle(args): + options, args = getopt.getopt(args, "", "keyid=") + keyid = None + for o,v in options: + if o == "--keyid": + keyid = v + + if len(args) < 2: + usage() + + configFile = args[0] + packages = {} + for pkgFile in args[1:]: + print "Loading", pkgFile + f = open(pkgFile, 'r') + p = simplejson.load(f) + f.close() + _, r, _ = thandy.formats.checkSignedObj(p) + if r != 'package': + print pkgFile, "was not a package" + packages[p['signed']['location']] = p + + def getHash(path): + p = packages[path] + return thandy.formats.getDigest(p['signed']) + + bundleObj = thandy.formats.makeBundleObj(configFile, getHash) + signable = thandy.formats.makeSignable(bundleObj) + + ks = getKeyStore() + ks.load() + key = getKey(ks, keyid=keyid, role="bundle", path=bundleObj['location']) + thandy.formats.sign(signable, key) + + if 1: + ss, r, p = thandy.formats.checkSignedObj(signable, ks) + assert ss.isValid() + + location = os.path.split(bundleObj['location'])[-1] + print "Writing signed bundle to %s"%location + f = open(location, 'w') + simplejson.dump(signable, f, indent=1) + f.close() + +# ------------------------------ +def makekeylist(args): + options, args = getopt.getopt(args, "", "keyid=") + keyid = None + for o,v in options: + if o == "--keyid": + keyid = v + + if len(args) < 1: + usage() + + keylist = thandy.formats.makeKeylistObj(args[0]) + signable = thandy.formats.makeSignable(keylist) + + ks = getKeyStore() + ks.load() + key = getKey(ks, keyid=keyid, role="master", path="/meta/keys.txt") + thandy.formats.sign(signable, key) + + if 1: + ss, r, p = thandy.formats.checkSignedObj(signable, ks) + assert ss.isValid() + + print "writing signed keylist to keys.txt" + thandy.util.replaceFile("keys.txt", + simplejson.dumps(signable, indent=1, sort_keys=True), + textMode=True) + +def signkeylist(args): + if len(args) != 1: + usage() + + keylist = simplejson.load(open(args[0], 'r')) + thandy.formats.SIGNED_SCHEMA.checkMatch(keylist) + thandy.formats.KEYLIST_SCHEMA.checkMatch(keylist['signed']) + + ks = getKeyStore() + ks.load() + keys = ks.getKeysByRole("master", "/meta/keys.txt") + for k in keys: + thandy.formats.sign(keylist, k) + + print "writing signed keylist to keys.txt" + thandy.util.replaceFile("keys.txt", + simplejson.dumps(keylist, indent=1, sort_keys=True), + textMode=True) + +def makemirrorlist(args): + options, args = getopt.getopt(args, "", "keyid=") + keyid = None + for o,v in options: + if o == "--keyid": + keyid = v + + if len(args) < 1: + usage() + + mirrorlist = thandy.formats.makeMirrorListObj(args[0]) + signable = thandy.formats.makeSignable(mirrorlist) + + ks = getKeyStore() + ks.load() + key = getKey(ks, keyid=keyid, role='mirrors', path="/meta/mirrors.txt") + thandy.formats.sign(signable, key) + + if 1: + ss, r, p = thandy.formats.checkSignedObj(signable, ks) + assert ss.isValid() + + print "writing signed mirrorlist to mirrors.txt" + thandy.util.replaceFile("mirrors.txt", + simplejson.dumps(signable, indent=1, sort_keys=True), + textMode=True) + +# ------------------------------ + +def keygen(args): + k = getKeyStore() + k.load() + print "Generating key. This will be slow." + key = thandy.keys.RSAKey.generate() + print "Generated new key: %s" % key.getKeyID() + k.addKey(key) + k.save() + +def listkeys(args): + k = getKeyStore() + k.load() + for k in k.iterkeys(): + print k.getKeyID() + for r, p in k.getRoles(): + print " ", r, p + +def addrole(args): + if len(args) < 3: + usage() + ks = getKeyStore() + ks.load() + k = getKey(ks, args[0]) + r = args[1] + if r not in thandy.formats.ALL_ROLES: + print "Unrecognized role %r. Known roles are %s"%( + r,", ".join(thandy.format.ALL_ROLES)) + sys.exit(1) + p = args[2] + k.addRole(r, p) + ks.save() + +def delrole(args): + if len(args) < 3: + usage() + ks = getKeyStore() + ks.load() + k = getKey(ks, args[0]) + r = args[1] + if r not in thandy.formats.ALL_ROLES: + print "Unrecognized role %r. Known roles are %s"%( + r,", ".join(thandy.format.ALL_ROLES)) + sys.exit(1) + p = args[2] + + #XXXX rep. + origLen = len(k._roles) + k._roles = [ (role,path) for role,path in k._roles + if (role,path) != (r,p) ] + removed = origLen - len(k._roles) + print removed, "roles removed" + if removed: + ks.save() + +def chpass(args): + ks = getKeyStore() + print "Old password." + ks.load() + print "New password." + ks.clearPassword() + ks.save() + +def dumpkey(args): + options, args = getopt.getopt(args, "", ["include-secret", "passwd="]) + + includeSecret = False + for o,v in options: + if o == '--include-secret': + includeSecret = True + else: + print "Unexpected %r"%o + + ks = getKeyStore() + ks.load() + + keys = [] + if len(args): + keys = [ getKey(ks, a) for a in args ] + else: + keys = list(ks.iterkeys()) + + for k in keys: + data = k.format(private=includeSecret, includeRoles=True) + print "Key(", simplejson.dumps(data, indent=2), ")" + +def usage(): + print "Known commands:" + print " keygen" + print " listkeys" + print " chpass" + print " addrole keyid role path" + print " delrole keyid role path" + print " dumpkey [--include-secret] keyid" + print " makepackage config datafile" + print " makebundle config packagefile ..." + print " signkeylist keylist" + print " makekeylist keylist" + print " makemirrorlist config" + sys.exit(1) + +def main(): + if len(sys.argv) < 2: + usage() + cmd = sys.argv[1] + args = sys.argv[2:] + if cmd in [ "keygen", "listkeys", "addrole", "delrole", "chpass", + "dumpkey", "makepackage", "makebundle", "signkeylist", + "makekeylist", "signkeylist", "makemirrorlist", ]: + globals()[cmd](args) + else: + usage() + +if __name__ == '__main__': + main() diff --git a/lib/thandy/__init__.py b/lib/thandy/__init__.py new file mode 100644 index 0000000..87fd983 --- /dev/null +++ b/lib/thandy/__init__.py @@ -0,0 +1,35 @@ + +__all__ = [ 'formats' ] + +_BaseException = Exception + +class Exception(_BaseException): + pass + +class FormatException(Exception): + pass + +class UnknownFormat(FormatException): + pass + +class BadSignature(Exception): + pass + +class BadPassword(Exception): + pass + +class InternalError(Exception): + pass + +class RepoError(InternalError): + pass + +class CryptoError(Exception): + pass + +class PubkeyFormatException(FormatException): + pass + +class UnknownMethod(CryptoError): + pass + diff --git a/lib/thandy/checkJson.py b/lib/thandy/checkJson.py new file mode 100644 index 0000000..e6b6c6e --- /dev/null +++ b/lib/thandy/checkJson.py @@ -0,0 +1,274 @@ + +import re +import sys + +import thandy + +class Schema: + def matches(self, obj): + try: + self.checkMatch(obj) + except thandy.FormatException: + return False + else: + return True + + def checkMatch(self, obj): + raise NotImplemented() + +class Any(Schema): + """ + >>> s = Any() + >>> s.matches("A String") + True + >>> s.matches([1, "list"]) + True + """ + def checkMatch(self, obj): + pass + +class RE(Schema): + """ + >>> s = RE("h.*d") + >>> s.matches("hello world") + True + >>> s.matches("Hello World") + False + >>> s.matches("hello world!") + False + >>> s.matches([33, "Hello"]) + False + """ + def __init__(self, pat=None, modifiers=0, reObj=None, reName="pattern"): + if not reObj: + if not pat.endswith("$"): + pat += "$" + reObj = re.compile(pat, modifiers) + self._re = reObj + self._reName = reName + def checkMatch(self, obj): + if not isinstance(obj, basestring) or not self._re.match(obj): + raise thandy.FormatException("%r did not match %s" + %(obj,self._reName)) + +class Str(Schema): + """ + >>> s = Str("Hi") + >>> s.matches("Hi") + True + >>> s.matches("Not hi") + False + """ + def __init__(self, val): + self._str = val + def checkMatch(self, obj): + if self._str != obj: + raise thandy.FormatException("Expected %r; got %r"%(self._str, obj)) + +class AnyStr(Schema): + """ + >>> s = AnyStr() + >>> s.matches("") + True + >>> s.matches("a string") + True + >>> s.matches(["a"]) + False + >>> s.matches(3) + False + >>> s.matches(u"a unicode string") + True + >>> s.matches({}) + False + """ + def __init__(self): + pass + def checkMatch(self, obj): + if not isinstance(obj, basestring): + raise thandy.FormatException("Expected a string; got %r"%obj) + +class ListOf(Schema): + """ + >>> s = ListOf(RE("(?:..)*")) + >>> s.matches("hi") + False + >>> s.matches([]) + True + >>> s.matches({}) + False + >>> s.matches(["Hi", "this", "list", "is", "full", "of", "even", "strs"]) + True + >>> s.matches(["This", "one", "is not"]) + False + """ + def __init__(self, schema, minCount=0, maxCount=sys.maxint,listName="list"): + self._schema = schema + self._minCount = minCount + self._maxCount = maxCount + self._listName = listName + def checkMatch(self, obj): + if not isinstance(obj, (list, tuple)): + raise thandy.FormatException("Expected %s; got %r" + %(self._listName,obj)) + for item in obj: + try: + self._schema.checkMatch(item) + except thandy.FormatException, e: + raise thandy.FormatException("%s in %s"%(e, self._listName)) + + if not (self._minCount <= len(obj) <= self._maxCount): + raise thandy.FormatException("Length of %s out of range" + %self._listName) + +class Struct(Schema): + """ + >>> s = Struct([ListOf(AnyStr()), AnyStr(), Str("X")]) + >>> s.matches(False) + False + >>> s.matches("Foo") + False + >>> s.matches([[], "Q", "X"]) + True + >>> s.matches([[], "Q", "D"]) + False + >>> s.matches([[3], "Q", "X"]) + False + >>> s.matches([[], "Q", "X", "Y"]) + False + """ + def __init__(self, subschemas, allowMore=False, structName="list"): + self._subschemas = subschemas[:] + self._allowMore = allowMore + self._structName = structName + def checkMatch(self, obj): + if not isinstance(obj, (list, tuple)): + raise thandy.FormatException("Expected %s; got %r" + %(self._structName,obj)) + elif len(obj) < len(self._subschemas): + raise thandy.FormatException( + "Too few fields in %s"%self._structName) + elif len(obj) > len(self._subschemas) and not self._allowMore: + raise thandy.FormatException( + "Too many fields in %s"%self._structName) + for item, schema in zip(obj, self._subschemas): + schema.checkMatch(item) + +class DictOf(Schema): + """ + >>> s = DictOf(RE(r'[aeiou]+'), Struct([AnyStr(), AnyStr()])) + >>> s.matches("") + False + >>> s.matches({}) + True + >>> s.matches({"a": ["x", "y"], "e" : ["", ""]}) + True + >>> s.matches({"a": ["x", 3], "e" : ["", ""]}) + False + >>> s.matches({"a": ["x", "y"], "e" : ["", ""], "d" : ["a", "b"]}) + False + """ + def __init__(self, keySchema, valSchema): + self._keySchema = keySchema + self._valSchema = valSchema + def checkMatch(self, obj): + try: + iter = obj.iteritems() + except AttributeError: + raise thandy.FormatException("Expected a dict; got %r"%obj) + + for k,v in iter: + self._keySchema.checkMatch(k) + self._valSchema.checkMatch(v) + +class Opt: + """Helper; applied to a value in Obj to mark it optional. + + >>> s = Obj(k1=Str("X"), k2=Opt(Str("Y"))) + >>> s.matches({'k1': "X", 'k2': "Y"}) + True + >>> s.matches({'k1': "X", 'k2': "Z"}) + False + >>> s.matches({'k1': "X"}) + True + """ + def __init__(self, schema): + self._schema = schema + def checkMatch(self, obj): + self._schema.checkMatch(obj) + +class Obj(Schema): + """ + >>> s = Obj(a=AnyStr(), bc=Struct([Int(), Int()])) + >>> s.matches({'a':"ZYYY", 'bc':[5,9]}) + True + >>> s.matches({'a':"ZYYY", 'bc':[5,9], 'xx':5}) + True + >>> s.matches({'a':"ZYYY", 'bc':[5,9,3]}) + False + >>> s.matches({'a':"ZYYY"}) + False + + """ + def __init__(self, _objname="object", **d): + self._objname = _objname + self._required = d.items() + + + def checkMatch(self, obj): + for k,schema in self._required: + try: + item = obj[k] + except KeyError: + if not isinstance(schema, Opt): + raise thandy.FormatException("Missing key %s in %s" + %(k,self._objname)) + + else: + try: + schema.checkMatch(item) + except thandy.FormatException, e: + raise thandy.FormatException("%s in %s.%s" + %(e,self._objname,k)) + + +class Int(Schema): + """ + >>> s = Int() + >>> s.matches(99) + True + >>> s.matches(False) + False + >>> s.matches(0L) + True + >>> s.matches("a string") + False + >>> Int(lo=10, hi=30).matches(25) + True + >>> Int(lo=10, hi=30).matches(5) + False + """ + def __init__(self, lo=-sys.maxint, hi=sys.maxint): + self._lo = lo + self._hi = hi + def checkMatch(self, obj): + if isinstance(obj, bool) or not isinstance(obj, (int, long)): + # We need to check for bool as a special case, since bool + # is for historical reasons a subtype of int. + raise thandy.FormatException("Got %r instead of an integer"%obj) + elif not (self._lo <= obj <= self._hi): + raise thandy.FormatException("%r not in range [%r,%r]" + %(obj, self._lo, self._hi)) + +class Bool(Schema): + """ + >>> s = Bool() + >>> s.matches(True) and s.matches(False) + True + >>> s.matches(11) + False + """ + def __init__(self): + pass + def checkMatch(self, obj): + if not isinstance(obj, bool): + raise thandy.FormatException("Got %r instead of a boolean"%obj) diff --git a/lib/thandy/download.py b/lib/thandy/download.py new file mode 100644 index 0000000..0b22cfa --- /dev/null +++ b/lib/thandy/download.py @@ -0,0 +1,127 @@ + + +import urllib2 +import httplib +import random + +import threading, Queue + +import thandy.util + +class Downloads: + def __init__(self, n_threads=2): + self._lock = threading.RLock() + self.downloads = {} + self.haveDownloaded = {} + self.downloadQueue = Queue.Queue() + self.threads = [ threading.Thread(target=self._thread) ] + for t in self.threads: + t.setDaemon(True) + + def start(self): + for t in self.threads: + t.start() + + def isCurrentlyDownloading(self, relPath): + self._lock.acquire() + try: + return self.downloads.has_key(relPath) + finally: + self._lock.release() + + def isRedundant(self, relPath): + self._lock.acquire() + try: + return (self.downloads.has_key(relPath) or + self.haveDownloaded.has_key(relPath)) + finally: + self._lock.release() + + def addDownloadJob(self, job): + rp = job.getRelativePath() + self._lock.acquire() + self.downloads[rp] = job + self._lock.release() + self.downloadQueue.put(job) + + def _thread(self): + while True: + job = self.downloadQueue.get() + job.download() + rp = job.getRelativePath() + self._lock.acquire() + try: + del self.downloads[rp] + self.haveDownloaded[rp] = True + finally: + self._lock.release() + +class DownloadJob: + def __init__(self, relPath, destPath, mirrorlist=None, + wantHash=None, canStall=False): + self._relPath = relPath + self._wantHash = wantHash + self._mirrorList = mirrorlist + self._destPath = destPath + + tmppath = thandy.util.userFilename("tmp") + if relPath.startswith("/"): + relPath = relPath[1:] + self._tmppath = os.path.join(tmppath, relPath) + + d = os.path.dirname(self._tmppath) + if not os.path.exists(d): + os.makedirs(d, 0700) + + def getRelativePath(self): + return self._relPath + + def haveStalledFile(self): + return os.path.exists(self._tmppath) + + def getURL(self, mirrorlist=None): + if mirrorlist is None: + mirrorlist = self._mirrorList + weightSoFar = 0 + usable = [] + + for m in mirrorlist['mirrors']: + for c in m['contents']: + # CHECK FOR URL SUITABILITY XXXXX + + if thandy.formats.rolePathMatches(c, self._relPath): + weightSoFar += m['weight'] + usable.append( (weightSoFar, m) ) + break + + wTarget = random.randint(0, weightSoFar) + mirror = None + # Could use bisect here instead + for w, m in mirrorlist: + if w >= wTarget: + mirror = m + break + + return m['urlbase'] + self._relPath + + def download(self): + # XXXX RESUME + + f_in = urllib2.urlopen(self.getURL()) + f_out = open(self._tmpPath, 'w') + while True: + c = f_in.read(1024) + if not c: + break + f_out.write(c) + f_in.close() + f_out.close() + # XXXXX retry on failure + + if self._wantHash: + gotHash = thandy.formats.getFileDigest(self._tmpPath) + if gotHash != self._wantHash: + # XXXX Corrupt file. + pass + + thandy.utils.moveFile(self._tmpPath, self._destPath) diff --git a/lib/thandy/formats.py b/lib/thandy/formats.py new file mode 100644 index 0000000..42d1086 --- /dev/null +++ b/lib/thandy/formats.py @@ -0,0 +1,747 @@ + +import simplejson +import time +import re +import binascii +import calendar + +import thandy.checkJson + +import Crypto.Hash.SHA256 + +class KeyDB: + """A KeyDB holds public keys, indexed by their key IDs.""" + def __init__(self): + self._keys = {} + def addKey(self, k): + keyid = k.getKeyID() + try: + oldkey = self._keys[keyid] + for r, p in oldkey.getRoles(): + if (r, p) not in k.getRoles(): + k.addRole(r,p) + except KeyError: + pass + self._keys[k.getKeyID()] = k + def getKey(self, keyid): + return self._keys[keyid] + def getKeysByRole(self, role, path): + results = [] + for key in self._keys.itervalues(): + for r,p in key.getRoles(): + if r == role: + if rolePathMatches(p, path): + results.append(key) + return results + + def getKeysFuzzy(self, keyid): + r = [] + for k,v in self._keys.iteritems(): + if k.startswith(keyid): + r.append(v) + return r + def iterkeys(self): + return self._keys.itervalues() + +_rolePathCache = {} +def rolePathMatches(rolePath, path): + """Return true iff the relative path in the filesystem 'path' conforms + to the pattern 'rolePath': a path that a given key is + authorized to sign. Patterns are allowed to contain * to + represent one or more characters in a filename, and ** to + represent any level of directory structure. + + >>> rolePathMatches("a/b/c/", "a/b/c/") + True + >>> rolePathMatches("**/c.*", "a/b/c.txt") + True + >>> rolePathMatches("**/c.*", "a/b/ctxt") + False + >>> rolePathMatches("**/c.*", "a/b/c.txt/foo") + False + >>> rolePathMatches("a/*/c", "a/b/c") + True + >>> rolePathMatches("a/*/c", "a/b/c.txt") + False + >>> rolePathMatches("a/*/c", "a/b/c.txt") #Check cache + False + """ + try: + regex = _rolePathCache[rolePath] + except KeyError: + orig = rolePath + # remove duplicate slashes. + rolePath = re.sub(r'/+', '/', rolePath) + # escape, then ** becomes .* + rolePath = re.escape(rolePath).replace(r'\*\*', r'.*') + # * becomes [^/]* + rolePath = rolePath.replace(r'\*', r'[^/]*') + # and no extra text is allowed. + rolePath += "$" + regex = _rolePathCache[orig] = re.compile(rolePath) + return regex.match(path) != None + +class SignatureStatus: + """Represents the outcome of checking signature(s) on an object.""" + def __init__(self, good, bad, unrecognized, unauthorized): + # keyids for all the valid signatures + self._good = good[:] + # keyids for the invalid signatures (we had the key, and it failed). + self._bad = bad[:] + # keyids for signatures where we didn't recognize the key + self._unrecognized = unrecognized[:] + # keyids for signatures where we recognized the key, but it doesn't + # seem to be allowed to sign this kind of document. + self._unauthorized = unauthorized[:] + + def isValid(self, threshold=1): + """Return true iff we got at least 'threshold' good signatures.""" + return len(self._good) >= threshold + + def mayNeedNewKeys(self): + """Return true iff downloading a new set of keys might tip this + signature status over to 'valid.'""" + return len(self._unrecognized) or len(self._unauthorized) + +def checkSignatures(signed, keyDB, role=None, path=None): + """Given an object conformant to SIGNED_SCHEMA and a set of public keys + in keyDB, verify the signed object in 'signed'.""" + + SIGNED_SCHEMA.checkMatch(signed) + + goodSigs = [] + badSigs = [] + unknownSigs = [] + tangentialSigs = [] + + signable = signed['signed'] + signatures = signed['signatures'] + + d_obj = Crypto.Hash.SHA256.new() + getDigest(signable, d_obj) + digest = d_obj.digest() + + for signature in signatures: + sig = signature['sig'] + keyid = signature['keyid'] + method = signature['method'] + + try: + key = keyDB.getKey(keyid) + except KeyError: + unknownSigs.append(keyid) + continue + + try: + result = key.checkSignature(method, sig, digest=digest) + except thandy.UnknownMethod: + continue + + if result == True: + if role is not None: + for r,p in key.getRoles(): + if r == role and rolePathMatches(p, path): + break + else: + tangentialSigs.append(sig) + continue + + goodSigs.append(keyid) + else: + badSigs.append(keyid) + + return SignatureStatus(goodSigs, badSigs, unknownSigs, tangentialSigs) + +def encodeCanonical(obj, outf=None): + """Encode the object obj in canoncial JSon form, as specified at + http://wiki.laptop.org/go/Canonical_JSON . It's a restricted + dialect of json in which keys are always lexically sorted, + there is no whitespace, floats aren't allowed, and only quote + and backslash get escaped. The result is encoded in UTF-8, + and the resulting bits are passed to outf (if provided), or joined + into a string and returned. + + >>> encodeCanonical("") + '""' + >>> encodeCanonical([1, 2, 3]) + '[1,2,3]' + >>> encodeCanonical({"x" : 3, "y" : 2}) + '{"x":3,"y":2}' + """ + def default(o): + raise TypeError("Can't encode %r", o) + def floatstr(o): + raise TypeError("Floats not allowed.") + def canonical_str_encoder(s): + return '"%s"' % re.sub(r'(["\\])', r'\\\1', s) + + # XXX This is, alas, a hack. I'll submit a canonical JSon patch to + # the simplejson folks. + + iterator = simplejson.encoder._make_iterencode( + None, default, canonical_str_encoder, None, floatstr, + ":", ",", True, False, True)(obj, 0) + + result = None + if outf == None: + result = [ ] + outf = result.append + + for u in iterator: + outf(u.encode("utf-8")) + if result is not None: + return "".join(result) + +def getDigest(obj, digestObj=None): + """Update 'digestObj' (typically a SHA256 object) with the digest of + the canonical json encoding of obj. If digestObj is none, + compute the SHA256 hash and return it. + + DOCDOC string equivalence. + """ + useTempDigestObj = (digestObj == None) + if useTempDigestObj: + digestObj = Crypto.Hash.SHA256.new() + + if isinstance(obj, str): + digestObj.update(obj) + elif isinstance(obj, unicode): + digestObj.update(obj.encode("utf-8")) + else: + encodeCanonical(obj, digestObj.update) + + if useTempDigestObj: + return digestObj.digest() + +def getFileDigest(f, digestObj=None): + """Update 'digestObj' (typically a SHA256 object) with the digest of + the file object in f. If digestObj is none, compute the SHA256 + hash and return it. + + >>> s = "here is a long string"*1000 + >>> import cStringIO, Crypto.Hash.SHA256 + >>> h1 = Crypto.Hash.SHA256.new() + >>> h2 = Crypto.Hash.SHA256.new() + >>> getFileDigest(cStringIO.StringIO(s), h1) + >>> h2.update(s) + >>> h1.digest() == h2.digest() + True + """ + useTempDigestObj = (digestObj == None) + if useTempDigestObj: + digestObj = Crypto.Hash.SHA256.new() + + while 1: + s = f.read(4096) + if not s: + break + digestObj.update(s) + + if useTempDigestObj: + return digestObj.digest() + +def makeSignable(obj): + return { 'signed' : obj, 'signatures' : [] } + +def sign(signed, key): + """Add an element to the signatures of 'signed', containing a new signature + of the "signed" part. + """ + + SIGNED_SCHEMA.checkMatch(signed) + + signable = signed["signed"] + signatures = signed['signatures'] + + keyid = key.getKeyID() + + signatures = [ s for s in signatures if s['keyid'] != keyid ] + + method, sig = key.sign(signable) + signatures.append({ 'keyid' : keyid, + 'method' : method, + 'sig' : sig }) + signed['signatures'] = signatures + +def formatTime(t): + """Encode the time 't' in YYYY-MM-DD HH:MM:SS format. + + >>> formatTime(1221265172) + '2008-09-13 00:19:32' + """ + return time.strftime("%Y-%m-%d %H:%M:%S", time.gmtime(t)) + +def parseTime(s): + """Parse a time 's' in YYYY-MM-DD HH:MM:SS format.""" + try: + return calendar.timegm(time.strptime(s, "%Y-%m-%d %H:%M:%S")) + except ValueError: + raise thandy.FormatError("Malformed time %r", s) + +def formatBase64(h): + """Return the base64 encoding of h with whitespace and = signs omitted.""" + return binascii.b2a_base64(h).rstrip("=\n ") + +formatHash = formatBase64 + +def parseBase64(s): + """Parse a base64 encoding with whitespace and = signs omitted. """ + extra = len(s) % 4 + if extra: + padding = "=" * (4 - extra) + s += padding + try: + return binascii.a2b_base64(s) + except binascii.Error: + raise thandy.FormatError("Invalid base64 encoding") + +def parseHash(s): + h = parseBase64(s) + if len(h) != Crypto.Hash.SHA256.digest_size: + raise thandy.FormatError("Bad hash length") + return h + +S = thandy.checkJson + +# A date, in YYYY-MM-DD HH:MM:SS format. +TIME_SCHEMA = S.RE(r'\d{4}-\d{2}-\d{2} \d{2}:\d{2}:\d{2}') +# A hash, base64-encoded +HASH_SCHEMA = S.RE(r'[a-zA-Z0-9\+\/]{43}') + +# A hexadecimal value. +HEX_SCHEMA = S.RE(r'[a-fA-F0-9]+') +# A base-64 encoded value +BASE64_SCHEMA = S.RE(r'[a-zA-Z0-9\+\/]+') +# An RSA key; subtype of PUBKEY_SCHEMA. +RSAKEY_SCHEMA = S.Obj( + _keytype=S.Str("rsa"), + e=BASE64_SCHEMA, + n=BASE64_SCHEMA) +# Any public key. +PUBKEY_SCHEMA = S.Obj( + _keytype=S.AnyStr()) + +KEYID_SCHEMA = HASH_SCHEMA +SIG_METHOD_SCHEMA = S.AnyStr() +RELPATH_SCHEMA = PATH_PATTERN_SCHEMA = S.AnyStr() +URL_SCHEMA = S.AnyStr() +VERSION_SCHEMA = S.ListOf(S.Any()) #XXXX WRONG + +# A single signature of an object. Indicates the signature, the id of the +# signing key, and the signing method. +SIGNATURE_SCHEMA = S.Obj( + keyid=KEYID_SCHEMA, + method=SIG_METHOD_SCHEMA, + sig=BASE64_SCHEMA) + +# A signed object. +SIGNED_SCHEMA = S.Obj( + signed=S.Any(), + signatures=S.ListOf(SIGNATURE_SCHEMA)) + +ROLENAME_SCHEMA = S.AnyStr() + +# A role: indicates that a key is allowed to certify a kind of +# document at a certain place in the repo. +ROLE_SCHEMA = S.Struct([ROLENAME_SCHEMA, PATH_PATTERN_SCHEMA]) + +# A Keylist: indicates a list of live keys and their roles. +KEYLIST_SCHEMA = S.Obj( + _type=S.Str("Keylist"), + ts=TIME_SCHEMA, + keys=S.ListOf(S.Obj(key=PUBKEY_SCHEMA, roles=S.ListOf(ROLE_SCHEMA)))) + +# A Mirrorlist: indicates all the live mirrors, and what documents they +# serve. +MIRRORLIST_SCHEMA = S.Obj( + _type=S.Str("Mirrorlist"), + ts=TIME_SCHEMA, + mirrors=S.ListOf(S.Obj(name=S.AnyStr(), + urlbase=URL_SCHEMA, + contents=S.ListOf(PATH_PATTERN_SCHEMA), + weight=S.Int(lo=0), + ))) + +# A timestamp: indicates the lastest versions of all top-level signed objects. +TIMESTAMP_SCHEMA = S.Obj( + _type = S.Str("Timestamp"), + at = TIME_SCHEMA, + m = S.Struct([TIME_SCHEMA, HASH_SCHEMA]), + k = S.Struct([TIME_SCHEMA, HASH_SCHEMA]), + b = S.DictOf(keySchema=S.AnyStr(), + valSchema= + S.Struct([ VERSION_SCHEMA, RELPATH_SCHEMA, TIME_SCHEMA, HASH_SCHEMA ])) + ) + +# A Bundle: lists a bunch of packages that should be updated in tandem +BUNDLE_SCHEMA = S.Obj( + _type=S.Str("Bundle"), + at=TIME_SCHEMA, + name=S.AnyStr(), + os=S.AnyStr(), + arch=S.Opt(S.AnyStr()), + version=VERSION_SCHEMA, + location=RELPATH_SCHEMA, + packages=S.ListOf(S.Obj( + name=S.AnyStr(), + version=VERSION_SCHEMA, + path=RELPATH_SCHEMA, + hash=HASH_SCHEMA, + order=S.Struct([S.Int(), S.Int(), S.Int()]), + optional=S.Opt(S.Bool()), + gloss=S.DictOf(S.AnyStr(), S.AnyStr()), + longgloss=S.DictOf(S.AnyStr(), S.AnyStr())))) + +PACKAGE_SCHEMA = S.Obj( + _type=S.Str("Package"), + name=S.AnyStr(), + location=RELPATH_SCHEMA, + version=VERSION_SCHEMA, + format=S.Obj(), + ts=TIME_SCHEMA, + files=S.ListOf(S.Struct([RELPATH_SCHEMA, HASH_SCHEMA])), + shortdesc=S.DictOf(S.AnyStr(), S.AnyStr()), + longdesc=S.DictOf(S.AnyStr(), S.AnyStr())) + +ALL_ROLES = ('timestamp', 'mirrors', 'bundle', 'package', 'master') + +class Key: + #XXXX UNUSED. + def __init__(self, key, roles=()): + self.key = key + self.roles = [] + for r,p in roles: + self.addRole(r,p) + + def addRole(self, role, path): + assert role in ALL_ROLES + self.roles.append((role, path)) + + def getRoles(self): + return self.roles + + @staticmethod + def fromJSon(obj): + # must match PUBKEY_SCHEMA + keytype = obj['_keytype'] + if keytype == 'rsa': + return Key(thandy.keys.RSAKey.fromJSon(obj)) + + if typeattr == 'rsa': + key = thandy.keys.RSAKey.fromSExpression(sexpr) + if key is not None: + return Key(key) + else: + return None + + def format(self): + return self.key.format() + + def getKeyID(self): + return self.key.getKeyID() + + def sign(self, sexpr=None, digest=None): + return self.key.sign(sexpr, digest=digest) + + def checkSignature(self, method, data, signatute): + ok = self.key.checkSignature(method, data, signature) + # XXXX CACHE HERE. + return ok + +class Keylist(KeyDB): + def __init__(self): + KeyDB.__init__(self) + + def addFromKeylist(self, obj, allowMasterKeys=False): + for keyitem in obj['keys']: + key = keyitem['key'] + roles = keyitem['roles'] + + try: + key = thandy.keys.RSAKey.fromJSon(key) + except thandy.FormatException, e: + print e + #LOG skipping key. + continue + + for r,p in roles: + if r == 'master' and not allowMasterKeys: + #LOG + continue + if r not in ALL_ROLES: + continue + key.addRole(r,p) + + self.addKey(key) + +class StampedInfo: + def __init__(self, ts, hash, version=None, relpath=None): + self._ts = ts + self._hash = hash + self._version = version + self._relpath = relpath + + @staticmethod + def fromJSonFields(timeStr, hashStr): + t = parseTime(timeStr) + h = parseHash(hashStr) + return StampedInfo(t, h) + + def getHash(self): + return self._hash + + def getRelativePath(self): + return self._relpath + +class TimestampFile: + def __init__(self, at, mirrorlistinfo, keylistinfo, bundleinfo): + self._time = at + self._mirrorListInfo = mirrorlistinfo + self._keyListInfo = keylistinfo + self._bundleInfo = bundleinfo + + @staticmethod + def fromJSon(obj): + # must be validated. + at = parseTime(obj['at']) + m = StampedInfo.fromJSonFields(*obj['m'][:2]) + k = StampedInfo.fromJSonFields(*obj['k'][:2]) + b = {} + for name, bundle in obj['b'].iteritems(): + v = bundle[0] + rp = bundle[1] + t = parseTime(bundle[2]) + h = parseHash(bundle[3]) + b[name] = StampedInfo(t, h, v, rp) + + return TimestampFile(at, m, k, b) + + def getTime(self): + return self._time + + def getMirrorlistInfo(self): + return self._mirrorListInfo + + def getKeylistInfo(self): + return self._keyListInfo + + def getBundleInfo(self, name): + return self._bundleInfo[name] + +def readConfigFile(fname, needKeys=(), optKeys=(), preload={}): + parsed = preload.copy() + result = {} + execfile(fname, parsed) + + for k in needKeys: + try: + result[k] = parsed[k] + except KeyError: + raise thandy.FormatError("Missing value for %s in %s"%k,fname) + + for k in optKeys: + try: + result[k] = parsed[k] + except KeyError: + pass + + return result + +def makePackageObj(config_fname, package_fname): + preload = {} + shortDescs = {} + longDescs = {} + def ShortDesc(lang, val): shortDescs[lang] = val + def LongDesc(lang, val): longDescs[lang] = val + preload = { 'ShortDesc' : ShortDesc, 'LongDesc' : LongDesc } + r = readConfigFile(config_fname, + ['name', + 'version', + 'format', + 'location', + 'relpath', + ], (), preload) + + f = open(package_fname, 'rb') + digest = getFileDigest(f) + + # Check fields! + result = { '_type' : "Package", + 'ts' : formatTime(time.time()), + 'name' : r['name'], + 'location' : r['location'], #DOCDOC + 'version' : r['version'], + 'format' : r['format'], + 'files' : [ [ r['relpath'], formatHash(digest) ] ], + 'shortdesc' : shortDescs, + 'longdesc' : longDescs + } + + PACKAGE_SCHEMA.checkMatch(result) + + return result + +def makeBundleObj(config_fname, getPackageHash): + packages = [] + def ShortGloss(lang, val): packages[-1]['gloss'][lang] = val + def LongGloss(lang, val): packages[-1]['longgloss'][lang] = val + def Package(name, version, path, order, optional=False): + packages.append({'name' : name, + 'version' : version, + 'path' : path, + 'order' : order, + 'optional' : optional, + 'gloss' : {}, + 'longgloss' : {} }) + preload = { 'ShortGloss' : ShortGloss, 'LongGloss' : LongGloss, + 'Package' : Package } + r = readConfigFile(config_fname, + ['name', + 'os', + 'version', + 'location', + ], ['arch'], preload) + + result = { '_type' : "Bundle", + 'at' : formatTime(time.time()), + 'name' : r['name'], + 'os' : r['os'], + 'version' : r['version'], + 'location' : r['location'], + 'packages' : packages } + if r.has_key('arch'): + result['arch'] = r['arch'] + + for p in packages: + try: + p['hash'] = formatHash(getPackageHash(p['path'])) + except KeyError: + raise thandy.FormatException("No such package as %s"%p['path']) + + BUNDLE_SCHEMA.checkMatch(result) + return result + +def versionIsNewer(v1, v2): + return v1 > v2 + +def makeTimestampObj(mirrorlist_obj, keylist_obj, + bundle_objs): + result = { '_type' : 'Timestamp', + 'at' : formatTime(time.time()) } + result['m'] = [ mirrorlist_obj['ts'], + formatHash(getDigest(mirrorlist_obj)) ] + result['k'] = [ keylist_obj['ts'], + formatHash(getDigest(keylist_obj)) ] + result['b'] = bundles = {} + for bundle in bundle_objs: + name = bundle['name'] + v = bundle['version'] + entry = [ v, bundle['location'], bundle['at'], formatHash(getDigest(bundle)) ] + if not bundles.has_key(name) or versionIsNewer(v, bundles[name][0]): + bundles[name] = entry + + TIMESTAMP_SCHEMA.checkMatch(result) + + return result + +class MirrorInfo: + def __init__(self, name, urlbase, contents, weight): + self._name = name + self._urlbase = urlbase + self._contents = contents + self._weight = weight + + def canServeFile(self, fname): + for c in self._contents: + if rolePathMatches(c, fname): + return True + return False + + def getFileURL(self, fname): + if self._urlbase[-1] == '/': + return self._urlbase+fname + else: + return "%s/%s" % (self._urlbase, fname) + + def format(self): + return { 'name' : self._name, + 'urlbase' : self._urlbase, + 'contents' : self._contents, + 'weight' : self._weight } + +def makeMirrorListObj(mirror_fname): + mirrors = [] + def Mirror(*a, **kw): mirrors.append(MirrorInfo(*a, **kw)) + preload = {'Mirror' : Mirror} + r = readConfigFile(mirror_fname, (), (), preload) + result = { '_type' : "Mirrorlist", + 'ts' : formatTime(time.time()), + 'mirrors' : [ m.format() for m in mirrors ] } + + MIRRORLIST_SCHEMA.checkMatch(result) + return result + +def makeKeylistObj(keylist_fname, includePrivate=False): + keys = [] + def Key(obj): keys.append(obj) + preload = {'Key': Key} + r = readConfigFile(keylist_fname, (), (), preload) + + klist = [] + for k in keys: + k = thandy.keys.RSAKey.fromJSon(k) + klist.append({'key': k.format(private=includePrivate), 'roles' : k.getRoles() }) + + result = { '_type' : "Keylist", + 'ts' : formatTime(time.time()), + 'keys' : klist } + + KEYLIST_SCHEMA.checkMatch(result) + return result + +SCHEMAS_BY_TYPE = { + 'Keylist' : KEYLIST_SCHEMA, + 'Mirrorlist' : MIRRORLIST_SCHEMA, + 'Timestamp' : TIMESTAMP_SCHEMA, + 'Bundle' : BUNDLE_SCHEMA, + 'Package' : PACKAGE_SCHEMA, + } + +def checkSignedObj(obj, keydb=None): + # Returns signaturestatus, role, path on sucess. + + SIGNED_SCHEMA.checkMatch(obj) + try: + tp = obj['signed']['_type'] + except KeyError: + raise thandy.FormatException("Untyped object") + try: + schema = SCHEMAS_BY_TYPE[tp] + except KeyError: + raise thandy.FormatException("Unrecognized type %r" % tp) + schema.checkMatch(obj['signed']) + + if tp == 'Keylist': + role = "master" + path = "/meta/keys.txt" + elif tp == 'Mirrorlist': + role = "mirrors" + path = "/meta/mirrors.txt" + elif tp == "Timestamp": + role = 'timestamp' + path = "/meta/timestamp.txt" + elif tp == 'Bundle': + role = 'bundle' + path = obj['signed']['location'] + elif tp == 'Package': + role = 'package' + path = obj['signed']['location'] + else: + print tp + raise "Foo" + + ss = None + if keydb is not None: + ss = checkSignatures(obj, keydb, role, path) + + return ss, role, path diff --git a/lib/thandy/keys.py b/lib/thandy/keys.py new file mode 100644 index 0000000..5b4e072 --- /dev/null +++ b/lib/thandy/keys.py @@ -0,0 +1,399 @@ + +# These require PyCrypto. +import Crypto.PublicKey.RSA +import Crypto.Hash.SHA256 +import Crypto.Cipher.AES + +import cPickle as pickle +import binascii +import logging +import os +import struct +import sys +import simplejson +import getpass + +import thandy.formats +import thandy.util + +class PublicKey: + def __init__(self): + # Confusingly, these roles are the ones used for a private key to + # remember what we're willing to do with it. + self._roles = [] + def format(self): + raise NotImplemented() + def sign(self, data): + # returns a list of method,signature tuples. + raise NotImplemented() + def checkSignature(self, method, data, signature): + # returns True, False, or raises UnknownMethod. + raise NotImplemented() + def getKeyID(self): + raise NotImplemented() + def getRoles(self): + return self._roles + def addRole(self, role, path): + assert role in thandy.formats.ALL_ROLES + self._roles.append((role, path)) + def clearRoles(self): + del self._roles[:] + def hasRole(self, role, path): + for r, p in self._roles: + if r == role and thandy.formats.rolePathMatches(p, path): + return True + return False + +if hex(1L).upper() == "0X1L": + def intToBinary(number): + """Convert an int or long into a big-endian series of bytes. + """ + # This "convert-to-hex, then use binascii" approach may look silly, + # but it's over 10x faster than the Crypto.Util.number approach. + h = hex(long(number)) + h = h[2:-1] + if len(h)%2: + h = "0"+h + return binascii.a2b_hex(h) +elif hex(1L).upper() == "0X1": + def intToBinary(number): + h = hex(long(number)) + h = h[2:] + if len(h)%2: + h = "0"+h + return binascii.a2b_hex(h) +else: + import Crypto.Util.number + intToBinary = Crypto.Util.number.long_to_bytes + assert None + +def binaryToInt(binary): + """Convert a big-endian series of bytes into a long. + """ + return long(binascii.b2a_hex(binary), 16) + +def intToBase64(number): + return thandy.formats.formatBase64(intToBinary(number)) + +def base64ToInt(number): + return binaryToInt(thandy.formats.parseBase64(number)) + +def _pkcs1_padding(m, size): + # I'd rather use OAEP+, but apparently PyCrypto barely supports + # signature verification, and doesn't seem to support signature + # verification with nondeterministic padding. "argh." + + s = [ "\x00\x01", "\xff"* (size-3-len(m)), "\x00", m ] + r = "".join(s) + return r + +def _xor(a,b): + if a: + return not b + else: + return b + +class RSAKey(PublicKey): + """ + >>> k = RSAKey.generate(bits=512) + >>> obj = k.format() + >>> obj['_keytype'] + 'rsa' + >>> base64ToInt(obj['e']) + 65537L + >>> k1 = RSAKey.fromJSon(obj) + >>> k1.key.e == k.key.e + True + >>> k1.key.n == k.key.n + True + >>> k.getKeyID() == k1.getKeyID() + True + >>> s = { 'A B C' : "D", "E" : [ "F", "g", 99] } + >>> method, sig = k.sign(obj=s) + >>> k.checkSignature(method, sig, obj=s) + True + >>> s2 = [ s ] + >>> k.checkSignature(method, sig, obj=s2) + False + """ + def __init__(self, key): + PublicKey.__init__(self) + self.key = key + self.keyid = None + + @staticmethod + def generate(bits=2048): + key = Crypto.PublicKey.RSA.generate(bits=bits, randfunc=os.urandom) + return RSAKey(key) + + @staticmethod + def fromJSon(obj): + # obj must match RSAKEY_SCHEMA + + thandy.formats.RSAKEY_SCHEMA.checkMatch(obj) + n = base64ToInt(obj['n']) + e = base64ToInt(obj['e']) + if obj.has_key('d'): + d = base64ToInt(obj['d']) + p = base64ToInt(obj['p']) + q = base64ToInt(obj['q']) + u = base64ToInt(obj['u']) + key = Crypto.PublicKey.RSA.construct((n, e, d, p, q, u)) + else: + key = Crypto.PublicKey.RSA.construct((n, e)) + + result = RSAKey(key) + if obj.has_key('roles'): + for r, p in obj['roles']: + result.addRole(r,p) + + return result + + def isPrivateKey(self): + return hasattr(self.key, 'd') + + def format(self, private=False, includeRoles=False): + n = intToBase64(self.key.n) + e = intToBase64(self.key.e) + result = { '_keytype' : 'rsa', + 'e' : e, + 'n' : n } + if private: + result['d'] = intToBase64(self.key.d) + result['p'] = intToBase64(self.key.p) + result['q'] = intToBase64(self.key.q) + result['u'] = intToBase64(self.key.u) + if includeRoles: + result['roles'] = self.getRoles() + return result + + def getKeyID(self): + if self.keyid == None: + d_obj = Crypto.Hash.SHA256.new() + thandy.formats.getDigest(self.format(), d_obj) + self.keyid = thandy.formats.formatHash(d_obj.digest()) + return self.keyid + + def _digest(self, obj, method=None): + if method in (None, "sha256-pkcs1"): + d_obj = Crypto.Hash.SHA256.new() + thandy.formats.getDigest(obj, d_obj) + digest = d_obj.digest() + return ("sha256-pkcs1", digest) + + raise UnknownMethod(method) + + def sign(self, obj=None, digest=None): + assert _xor(obj == None, digest == None) + if digest == None: + method, digest = self._digest(obj) + m = _pkcs1_padding(digest, (self.key.size()+1) // 8) + sig = intToBase64(self.key.sign(m, "")[0]) + return (method, sig) + + def checkSignature(self, method, sig, obj=None, digest=None): + assert _xor(obj == None, digest == None) + if method != "sha256-pkcs1": + raise UnknownMethod("method") + if digest == None: + method, digest = self._digest(obj, method) + sig = base64ToInt(sig) + m = _pkcs1_padding(digest, (self.key.size()+1) // 8) + return bool(self.key.verify(m, (sig,))) + +SALTLEN=16 + +def secretToKey(salt, secret): + """Convert 'secret' to a 32-byte key, using a version of the algorithm + from RFC2440. The salt must be SALTLEN+1 bytes long, and should + be random, except for the last byte, which encodes how time- + consuming the computation should be. + + (The goal is to make offline password-guessing attacks harder by + increasing the time required to convert a password to a key, and to + make precomputed password tables impossible to generate by ) + """ + assert len(salt) == SALTLEN+1 + + # The algorithm is basically, 'call the last byte of the salt the + # "difficulty", and all other bytes of the salt S. Now make + # an infinite stream of S|secret|S|secret|..., and hash the + # first N bytes of that, where N is determined by the difficulty. + # + # Obviously, this wants a hash algorithm that's tricky to + # parallelize. + # + # Unlike RFC2440, we use a 16-byte salt. Because CPU times + # have improved, we start at 16 times the previous minimum. + + difficulty = ord(salt[-1]) + count = (16L+(difficulty & 15)) << ((difficulty >> 4) + 10) + + # Make 'data' nice and long, so that we don't need to call update() + # a zillion times. + data = salt[:-1]+secret + if len(data)<1024: + data *= (1024 // len(data))+1 + + d = Crypto.Hash.SHA256.new() + iters, leftover = divmod(count, len(data)) + for _ in xrange(iters): + d.update(data) + #count -= len(data) + if leftover: + d.update(data[:leftover]) + #count -= leftover + #assert count == 0 + + return d.digest() + +def encryptSecret(secret, password, difficulty=0x80): + """Encrypt the secret 'secret' using the password 'password', + and return the encrypted result.""" + # The encrypted format is: + # "GKEY1" -- 5 octets, fixed, denotes data format. + # SALT -- 17 bytes, used to hash password + # IV -- 16 bytes; salt for encryption + # ENCRYPTED IN AES256-OFB, using a key=s2k(password, salt) and IV=IV: + # SLEN -- 4 bytes; length of secret, big-endian. + # SECRET -- len(secret) bytes + # D -- 32 bytes; SHA256 hash of (salt|secret|salt). + # + # This format leaks the secret length, obviously. + assert 0 <= difficulty < 256 + salt = os.urandom(SALTLEN)+chr(difficulty) + key = secretToKey(salt, password) + + d_obj = Crypto.Hash.SHA256.new() + d_obj.update(salt) + d_obj.update(secret) + d_obj.update(salt) + d = d_obj.digest() + + iv = os.urandom(16) + e = Crypto.Cipher.AES.new(key, Crypto.Cipher.AES.MODE_OFB, iv) + + # Stupidly, pycrypto doesn't accept that stream ciphers don't need to + # take their input in blocks. So pad it, then ignore the padded output. + + padlen = 16-((len(secret)+len(d)+4) % 16) + if padlen == 16: padlen = 0 + pad = '\x00' * padlen + + slen = struct.pack("!L",len(secret)) + encrypted = e.encrypt("%s%s%s%s" % (slen, secret, d, pad)) + if padlen: + encrypted = encrypted[:-padlen] + return "GKEY1%s%s%s"%(salt, iv, encrypted) + +def decryptSecret(encrypted, password): + """Decrypt a value encrypted with encryptSecret. Raises UnknownFormat + or FormatError if 'encrypted' was not generated with encryptSecret. + Raises BadPassword if the password was not correct. + """ + if encrypted[:5] != "GKEY1": + raise thandy.UnknownFormat() + encrypted = encrypted[5:] + if len(encrypted) < SALTLEN+1+16: + raise thandy.FormatException() + + salt = encrypted[:SALTLEN+1] + iv = encrypted[SALTLEN+1:SALTLEN+1+16] + encrypted = encrypted[SALTLEN+1+16:] + + key = secretToKey(salt, password) + + e = Crypto.Cipher.AES.new(key, Crypto.Cipher.AES.MODE_OFB, iv) + padlen = 16-(len(encrypted) % 16) + if padlen == 16: padlen = 0 + pad = '\x00' * padlen + + decrypted = e.decrypt("%s%s"%(encrypted,pad)) + slen = struct.unpack("!L", decrypted[:4])[0] + secret = decrypted[4:4+slen] + hash = decrypted[4+slen:4+slen+Crypto.Hash.SHA256.digest_size] + + d = Crypto.Hash.SHA256.new() + d.update(salt) + d.update(secret) + d.update(salt) + + if d.digest() != hash: + raise thandy.BadPassword() + + return secret + +class KeyStore(thandy.formats.KeyDB): + def __init__(self, fname, encrypted=True): + thandy.formats.KeyDB.__init__(self) + + self._loaded = None + self._fname = fname + self._passwd = None + self._encrypted = encrypted + + def getpass(self, reprompt=False): + if self._passwd != None: + return self._passwd + while 1: + pwd = getpass.getpass("Password: ", sys.stderr) + if not reprompt: + return pwd + + pwd2 = getpass.getpass("Confirm: ", sys.stderr) + if pwd == pwd2: + return pwd + else: + print "Mismatch; try again." + + def load(self, password=None): + logging.info("Loading private keys from %r...", self._fname) + if not os.path.exists(self._fname): + logging.info("...no such file.") + self._loaded = True + return + + if password is None and self._encrypted: + password = self.getpass() + + contents = open(self._fname, 'rb').read() + if self._encrypted: + contents = decryptSecret(contents, password) + + listOfKeys = simplejson.loads(contents) + self._passwd = password # It worked. + if not listOfKeys.has_key('keys'): + listOfKeys['keys'] = [] + for obj in listOfKeys['keys']: + key = RSAKey.fromJSon(obj) + self.addKey(key) + logging.info("Loaded key %s", key.getKeyID()) + + self._loaded = True + + def setPassword(self, passwd): + self._passwd = passwd + + def clearPassword(self): + self._passwd = None + + def save(self, password=None): + if not self._loaded and self._encrypted: + self.load(password) + + if password is None: + password = self.getpass(True) + + logging.info("Saving private keys into %r...", self._fname) + listOfKeys = { 'keys' : + [ key.format(private=True, includeRoles=True) for key in + self._keys.values() ] + } + contents = simplejson.dumps(listOfKeys) + if self._encrypted: + contents = encryptSecret(contents, password) + thandy.util.replaceFile(self._fname, contents) + self._passwd = password # It worked. + logging.info("Done.") + + diff --git a/lib/thandy/master_keys.py b/lib/thandy/master_keys.py new file mode 100644 index 0000000..0d455d1 --- /dev/null +++ b/lib/thandy/master_keys.py @@ -0,0 +1,5 @@ + + +MASTER_KEYS = [ + +] diff --git a/lib/thandy/repository.py b/lib/thandy/repository.py new file mode 100644 index 0000000..dc13f1b --- /dev/null +++ b/lib/thandy/repository.py @@ -0,0 +1,313 @@ + +import thandy.formats +import thandy.util + +import simplejson +import logging +import os +import threading +import time + +MAX_TIMESTAMP_AGE = 24*60*60 + +class RepositoryFile: + def __init__(self, repository, relativePath, schema, + needRole=None, signedFormat=True, needSigs=1): + self._repository = repository + self._relativePath = relativePath + self._schema = schema + self._needRole = needRole + self._signedFormat = signedFormat + self._needSigs = needSigs + + self._signed_obj = self._main_obj = None + self._sigStatus = None + self._mtime = None + + def getRelativePath(self): + return self._relativePath + + def getPath(self): + return self._repository.getFilename(self._relativePath) + + def _load(self): + fname = self.getPath() + + # Propagate OSError + f = None + fd = os.open(fname, os.O_RDONLY) + try: + f = os.fdopen(fd, 'r') + except: + os.close(fd) + raise + try: + mtime = os.fstat(fd).st_mtime + content = f.read() + finally: + f.close() + + signed_obj,main_obj = self._checkContent(content) + + self._signed_obj = signed_obj + self._main_obj = main_obj + self._mtime = mtime + + def _save(self, content=None): + if content == None: + content = sexpr.encode + + signed_obj,main_obj = self._checkContent(content) + + fname = self.getPath() + thandy.util.replaceFile(fname, contents) + + self._signed_obj = signed_obj + self._main_obj = main_obj + self._mtime = mtime + + def _checkContent(self, content): + + try: + obj = simplejson.loads(content) + except ValueError, e: + raise thandy.FormatException("Couldn't decode content: %s"%e) + + if self._signedFormat: + # This is supposed to be signed. + thandy.formats.SIGNED_SCHEMA.checkMatch(obj) + + main_obj = obj['signed'] + signed_obj = obj + else: + signed_obj = None + main_obj = obj + + if self._schema != None: + self._schema.checkMatch(main_obj) + + return signed_obj, main_obj + + def load(self): + if self._main_obj == None: + self._load() + + def get(self): + return self._main_obj + + def isLoaded(self): + return self._main_obj != None + + def getContent(self): + self.load() + return self._main_obj + + def _checkSignatures(self): + self.load() + sigStatus = thandy.formats.checkSignatures(self._signed_obj, + self._repository._keyDB, + self._needRole, self._relativePath) + self._sigStatus = sigStatus + + def checkSignatures(self): + if self._sigStatus is None: + self._checkSignatures() + return self._sigStatus + +class LocalRepository: + def __init__(self, root): + self._root = root + self._keyDB = thandy.util.getKeylist(None) + + self._keylistFile = RepositoryFile( + self, "/meta/keys.txt", thandy.formats.KEYLIST_SCHEMA, + needRole="master") + self._timestampFile = RepositoryFile( + self, "/meta/timestamp.txt", thandy.formats.TIMESTAMP_SCHEMA, + needRole="timestamp") + self._mirrorlistFile = RepositoryFile( + self, "/meta/mirrors.txt", thandy.formats.MIRRORLIST_SCHEMA, + needRole="mirrors") + self._metaFiles = [ self._keylistFile, + self._timestampFile, + self._mirrorlistFile ] + + self._packageFiles = {} + self._bundleFiles = {} + + def getFilename(self, relativePath): + if relativePath.startswith("/"): + relativePath = relativePath[1:] + return os.path.join(self._root, relativePath) + + def getKeylistFile(self): + return self._keylistFile + + def getTimestampFile(self): + return self._timestampFile + + def getMirrorlistFile(self): + return self._mirrorlistFile + + def getPackageFile(self, relPath): + try: + return self._packageFiles[relPath] + except KeyError: + self._packageFiles[relPath] = pkg = RepositoryFile( + self, relPath, thandy.formats.PACKAGE_SCHEMA, + needRole='package') + return pkg + + def getBundleFile(self, relPath): + try: + return self._bundleFiles[relPath] + except KeyError: + self._bundleFiles[relPath] = pkg = RepositoryFile( + self, relPath, thandy.formats.BUNDLE_SCHEMA, + needRole='bundle') + return pkg + + def getFilesToUpdate(self, now=None, trackingBundles=()): + if now == None: + now = time.time() + + need = set() + + # Fetch missing metafiles. + for f in self._metaFiles: + try: + f.load() + except OSError, e: + print "need", f.getPath() + logging.info("Couldn't load %s: %s. Must fetch it.", + f.getPath(), e) + need.add(f.getRelativePath()) + + # If the timestamp file is out of date, we need to fetch it no + # matter what. (Even if it is isn't signed, it can't possibly + # be good.) + ts = self._timestampFile.get() + if ts: + age = now - thandy.formats.parseTime(ts['at']) + ts = thandy.formats.TimestampFile.fromJSon(ts) + if age > MAX_TIMESTAMP_AGE: + need.add(self._timestampFile.getRelativePath()) + + # If the keylist isn't signed right, we can't check the + # signatures on anything else. + if self._keylistFile.get(): + s = self._keylistFile.checkSignatures() + if not s.isValid(): # For now only require one master key. + need.add(self._keylistFile.getRelativePath()) + + if need: + return need + + # Import the keys from the keylist. + self._keyDB.addFromKeylist(self._keylistFile.get()) + + # If the timestamp isn't signed right, get a new timestamp and a + # new keylist. + s = self._timestampFile.checkSignatures() + if not s.isValid(): + need.add(self._keylistFile.getRelativePath()) + need.add(self._timestampFile.getRelativePath()) + return need + + # FINALLY, we know we have an up-to-date, signed timestamp + # file. Check whether the keys and mirrors file are as + # authenticated. + h_kf = thandy.formats.getDigest(self._keylistFile.get()) + h_expected = ts.getKeylistInfo().getHash() + if h_kf != h_expected: + need.add(self._keylistFile.getRelativePath()) + + if need: + return need + + s = self._mirrorlistFile.checkSignatures() + if not s.isValid(): + need.add(self._mirrorlistFile.getRelativePath()) + + h_mf = thandy.formats.getDigest(self._mirrorlistFile.get()) + h_expected = ts.getMirrorlistInfo().getHash() + if h_mf != h_expected: + need.add(self._mirrorlistFile.getRelativePath()) + + if need: + return need + + # Okay; that's it for the metadata. Do we have the right + # bundles? + bundles = {} + for b in trackingBundles: + try: + binfo = ts.getBundleInfo(b) + except KeyError: + logging.warn("Unrecognized bundle %s"%b) + continue + + rp = binfo.getRelativePath() + bfile = self.getBundleFile(rp) + try: + bfile.load() + except OSError: + need.add(rp) + continue + + h_b = thandy.formats.getDigest(bfile.get()) + h_expected = binfo.getHash() + if h_b != h_expected: + need.add(rp) + continue + + s = bfile.checkSignatures() + if not s.isValid(): + # Can't actually use it. + continue + + bundles[rp] = bfile + + # Okay. So we have some bundles. See if we have their packages. + packages = {} + for bfile in bundles.values(): + bundle = bfile.get() + for pkginfo in bundle['packages']: + rp = pkginfo['path'] + pfile = self.getPackageFile(rp) + try: + pfile.load() + except OSError: + need.add(rp) + continue + + h_p = thandy.formats.getDigest(pfile.get()) + h_expected = thandy.formats.parseHash(pkginfo['hash']) + if h_p != h_expected: + need.add(rp) + continue + + s = pfile.checkSignatures() + if not s.isValid(): + # Can't use it. + continue + packages[rp] = pfile + + # Finally, we have some packages. Do we have their underlying + # files? + for pfile in packages.values(): + package = pfile.get() + for f in package['files']: + rp, h = f[:2] + h_expected = thandy.formats.parseHash(h) + fn = self.getFilename(rp) + try: + h_got = thandy.formats.getFileDigest(fn) + except OSError: + need.add(rp) + continue + if h_got != h_expected: + need.add(rp) + + # Okay; these are the files we need. + return need diff --git a/lib/thandy/tests.py b/lib/thandy/tests.py new file mode 100644 index 0000000..8b967a5 --- /dev/null +++ b/lib/thandy/tests.py @@ -0,0 +1,64 @@ + +import unittest +import doctest +import os +import tempfile + +import thandy.keys +import thandy.formats +import thandy.repository +import thandy.checkJson + +import thandy.tests + +class CanonicalEncodingTest(unittest.TestCase): + def test_encode(self): + enc = thandy.formats.encodeCanonical + self.assertEquals(enc(''), '""') + self.assertEquals(enc('"'), '"\\""') + self.assertEquals(enc('\t\\\n"\r'), + '"\t\\\\\n\\"\r"') + +class CryptoTests(unittest.TestCase): + def test_encrypt(self): + s = "The Secret words are marzipan habidashery zeugma." + password = "the password is swordfish." + encrypted = thandy.keys.encryptSecret(s, password) + self.assertNotEquals(encrypted, s) + self.assert_(encrypted.startswith("GKEY1")) + self.assertEquals(s, thandy.keys.decryptSecret(encrypted, password)) + self.assertRaises(thandy.BadPassword, thandy.keys.decryptSecret, + encrypted, "password") + self.assertRaises(thandy.UnknownFormat, thandy.keys.decryptSecret, + "foobar", password) + + def test_keystore(self): + passwd = "umfitty noonah" + fname = tempfile.mktemp() + ks = thandy.keys.KeyStore(fname) + key1 = thandy.keys.RSAKey.generate(512) + key2 = thandy.keys.RSAKey.generate(512) + ks.addKey(key1) + ks.addKey(key2) + ks.save(passwd) + + ks2 = thandy.keys.KeyStore(fname) + ks2.load(passwd) + self.assertEquals(key1.key.n, ks2.getKey(key1.getKeyID()).key.n) + +def suite(): + suite = unittest.TestSuite() + + suite.addTest(doctest.DocTestSuite(thandy.formats)) + suite.addTest(doctest.DocTestSuite(thandy.keys)) + suite.addTest(doctest.DocTestSuite(thandy.checkJson)) + + loader = unittest.TestLoader() + suite.addTest(loader.loadTestsFromModule(thandy.tests)) + + return suite + + +if __name__ == '__main__': + + unittest.TextTestRunner(verbosity=1).run(suite()) diff --git a/lib/thandy/util.py b/lib/thandy/util.py new file mode 100644 index 0000000..e87ed8b --- /dev/null +++ b/lib/thandy/util.py @@ -0,0 +1,73 @@ + +import os +import sys +import tempfile + +import simplejson + +import thandy.formats +import thandy.keys +import thandy.master_keys + +def moveFile(fromLocation, toLocation): + if sys.platform in ('cygwin', 'win32'): + # Win32 doesn't let rename replace an existing file. + try: + os.unlink(toLocation) + except OSError: + pass + os.rename(fromLocation, toLocation) + + +def replaceFile(fname, contents, textMode=False): + """overwrite the file in 'fname' atomically with the content of 'contents' + """ + dir, prefix = os.path.split(fname) + fd, fname_tmp = tempfile.mkstemp(prefix=prefix, dir=dir, text=textMode) + + try: + os.write(fd, contents) + finally: + os.close(fd) + + moveFile(fname_tmp, fname) + +def userFilename(name): + try: + base = os.environ["THANDY_HOME"] + except KeyError: + base = "~/.thandy" + base = os.path.expanduser(base) + if not os.path.exists(base): + os.makedirs(base, 0700) + return os.path.join(base, name) + +def getKeylist(keys_fname, checkKeys=True): + import thandy.master_keys + + keydb = thandy.formats.Keylist() + + for key in thandy.master_keys.MASTER_KEYS: + keydb.addKey(key) + + user_keys = userFilename("preload_keys") + if os.path.exists(user_keys): + #XXXX somewhat roundabout. + keylist = thandy.formats.makeKeylistObj(user_keys) + keydb.addFromKeylist(keylist, allowMasterKeys=True) + + if keys_fname and os.path.exists(keys_fname): + f = open(keys_fname, 'r') + try: + obj = simplejson.load(f) + finally: + f.close() + ss, role, path = thandy.formats.checkSignedObj(obj, keydb) + if role != 'master': + raise thandy.FormatException("%s wasn't a keylist."%keys_fname) + if checkKeys and not ss.isValid(): + raise thandy.FormatException("%s not signed by enough master keys"% + keys_fname) + keydb.addFromKeylist(obj['signed'], allowMasterKeys=False) + + return keydb -- cgit v1.2.3