summaryrefslogtreecommitdiff
path: root/lib/glider
diff options
context:
space:
mode:
authorNick Mathewson <nickm@torproject.org>2008-10-14 05:10:30 +0000
committerNick Mathewson <nickm@torproject.org>2008-10-14 05:10:30 +0000
commitd90990ee3ecd09a2725b8051759a900ebd488b8c (patch)
treefbe68d0c4b698d45bfdcb6c91a43ea2e60c21329 /lib/glider
parentfb5a6115a6f3ea0216e3ca0645ba1eb31fb02876 (diff)
Rename glider to thandy, based on discussions on #nottor. Please let me know ASAP if there is another program Thandy, or if it means something rude, or whatever.
git-svn-id: file:///home/or/svnrepo/updater/trunk@17085 55e972cd-5a19-0410-ae62-a4d7a52db4cd
Diffstat (limited to 'lib/glider')
-rw-r--r--lib/glider/ClientCLI.py66
-rw-r--r--lib/glider/ServerCLI.py187
-rw-r--r--lib/glider/SignerCLI.py313
-rw-r--r--lib/glider/__init__.py35
-rw-r--r--lib/glider/checkJson.py274
-rw-r--r--lib/glider/download.py127
-rw-r--r--lib/glider/formats.py747
-rw-r--r--lib/glider/keys.py399
-rw-r--r--lib/glider/master_keys.py5
-rw-r--r--lib/glider/repository.py313
-rw-r--r--lib/glider/tests.py64
-rw-r--r--lib/glider/util.py73
12 files changed, 0 insertions, 2603 deletions
diff --git a/lib/glider/ClientCLI.py b/lib/glider/ClientCLI.py
deleted file mode 100644
index c6f13fd..0000000
--- a/lib/glider/ClientCLI.py
+++ /dev/null
@@ -1,66 +0,0 @@
-
-import os
-import sys
-import getopt
-
-import glider.util
-import glider.repository
-import glider.download
-
-def update(args):
- repoRoot = glider.util.userFilename("cache")
- options, args = getopt.getopt(args, "", [ "repo=", "no-download" ])
- download = True
-
- for o, v in options:
- if o == '--repo':
- repoRoot = v
- elif o == "--no-download":
- download = False
-
- repo = glider.repository.LocalRepository(repoRoot)
-
- files = repo.getFilesToUpdate(trackingBundles=args)
-
- if not download:
- return
-
- mirrorlist = repo.getMirrorlistFile().get()
-
- downloader = glider.download.Downloads()
- downloader.start()
-
- for f in files:
- # XXXX Use hash.
- dj = glider.download.DownloadJob(f, repo.getFilename(f),
- mirrorlist)
- downloader.addDownloadJob(dj)
- # XXXX replace file in repository if ok; reload; see what changed.
-
- # Wait for in-progress jobs
-
-# Check my repository
-
-# Tell me what I need to download
-
-# Download stuff
-
-# Tell me what to install.
-
-def usage():
- print "Known commands:"
- print " update [--repo=repository] [--no-download]"
- sys.exit(1)
-
-def main():
- if len(sys.argv) < 2:
- usage()
- cmd = sys.argv[1]
- args = sys.argv[2:]
- if cmd in [ "update" ]:
- globals()[cmd](args)
- else:
- usage()
-
-if __name__ == '__main__':
- main()
diff --git a/lib/glider/ServerCLI.py b/lib/glider/ServerCLI.py
deleted file mode 100644
index c564f5b..0000000
--- a/lib/glider/ServerCLI.py
+++ /dev/null
@@ -1,187 +0,0 @@
-
-import os
-import sys
-import getopt
-import time
-
-import simplejson
-
-import glider.formats
-import glider.util
-import glider.keys
-
-def tstamp():
- return time.strftime("%Y%m%d_%H%M%S", time.localtime())
-
-def snarf(fname):
- f = open(fname, 'rb')
- try:
- return f.read()
- finally:
- f.close()
-
-def snarfObj(fname):
- f = open(fname, 'r')
- try:
- return simplejson.load(f)
- finally:
- f.close()
-
-def insert(args):
- repo = os.environ.get("THANDY_MASTER_REPO")
- backupDir = glider.util.userFilename("old_files")
- checkSigs = True
-
- options, args = getopt.getopt(args, "", ["repo=", "no-check"])
- for o,v in options:
- if o == "--repo":
- repo = v
- elif o == "--no-check":
- checkSigs = False
-
- if not repo:
- print "No repository specified."
- usage()
- if not os.path.exists(repo):
- print "No such repository as %r"%repo
- usage()
-
- if not os.path.exists(backupDir):
- os.makedirs(backupDir, 0700)
-
- if checkSigs:
- keys = glider.util.getKeylist(os.path.join(repo, "meta/keys.txt"))
- else:
- keys = None
-
- n_ok = 0
- for fn in args:
- print "Loading %s..."%fn
- try:
- content = snarf(fn)
- except OSError, e:
- print "Couldn't open %s: %s"%(fn, e)
- continue
-
- try:
- obj = simplejson.loads(content)
- except ValueError, e:
- print "Couldn't decode %s: %s"%(fn, e)
- continue
-
- try:
- ss, r, path = glider.formats.checkSignedObj(obj, keys)
- except glider.FormatException, e:
- print "Bad format on %s: %s"%(fn, e)
- continue
- if checkSigs and not ss.isValid():
- print "Not enough valid signatures on %s"%fn
- continue
-
- print " Looks okay. It goes in %s"%path
- assert path.startswith("/")
- targetPath = os.path.join(repo, path[1:])
- if os.path.exists(targetPath):
- oldContents = snarf(targetPath)
- if oldContents == content:
- print " File unchanged!"
- n_ok += 1
- continue
-
- baseFname = "%s_%s" % (tstamp(), os.path.split(path)[1])
- backupFname = os.path.join(backupDir, baseFname)
- print " Copying old file to %s"%backupFname
- glider.util.replaceFile(backupFname, oldContents)
-
- parentDir = os.path.split(targetPath)[0]
- if not os.path.exists(parentDir):
- print " Making %s"%parentDir
- os.makedirs(parentDir, 0755)
- print " Replacing file..."
- glider.util.replaceFile(targetPath, content)
- print " Done."
- n_ok += 1
- if n_ok != len(args):
- sys.exit(1)
-
-def timestamp(args):
- repo = os.environ.get("THANDY_MASTER_REPO")
- ts_keyfile = glider.util.userFilename("timestamp_key")
-
- options, args = getopt.getopt(args, "", ["repo=", "ts-key="])
- for o,v in options:
- if o == "--repo":
- repo = v
- elif o == "--ts-key":
- ts_keyfile = v
-
- if repo == None:
- print "No repository specified."
- usage()
- if not os.path.exists(repo):
- print "No such repository as %r"%repo
- usage()
-
- tsFname = os.path.join(repo, "meta/timestamp.txt")
-
- try:
- mObj = snarfObj(os.path.join(repo, "meta/mirrors.txt"))
- except OSError:
- print "No mirror list!"
- sys.exit(1)
- try:
- kObj = snarfObj(os.path.join(repo, "meta/keys.txt"))
- except OSError:
- print "No key list!"
- sys.exit(1)
-
- bundles = []
- for dirpath, dirname, fns in os.walk(os.path.join(repo, "bundleinfo")):
- for fn in fns:
- try:
- bObj = snarfObj(fn)
- except (ValueError, OSError), e:
- print "(Couldn't read bundle-like %s)"%fn
- continue
- try:
- _, r, _ = glider.formats.checkSignedObj(bObj)
- except glider.FormatException, e:
- print "Problem reading object from %s"%fn
- continue
- if r != "bundle":
- print "%s was not a good bundle"%fn
- continue
- bundles.append(bObj['signed'])
-
- timestamp = glider.formats.makeTimestampObj(
- mObj['signed'], kObj['signed'], bundles)
- signable = glider.formats.makeSignable(timestamp)
-
- keydb = glider.formats.Keylist()
- #XXXX Still a roundabout way to do this.
- keylist = glider.formats.makeKeylistObj(ts_keyfile, True)
- keydb.addFromKeylist(keylist)
- for k in keydb.iterkeys():
- glider.formats.sign(signable, k)
-
- content = simplejson.dumps(signable, sort_keys=True)
- glider.util.replaceFile(tsFname, content)
-
-def usage():
- print "Known commands:"
- print " insert [--no-check] [--repo=repository] file ..."
- print " timestamp [--repo=repository]"
- sys.exit(1)
-
-def main():
- if len(sys.argv) < 2:
- usage()
- cmd = sys.argv[1]
- args = sys.argv[2:]
- if cmd in [ "insert", "timestamp" ]:
- globals()[cmd](args)
- else:
- usage()
-
-if __name__ == '__main__':
- main()
diff --git a/lib/glider/SignerCLI.py b/lib/glider/SignerCLI.py
deleted file mode 100644
index bf5b73a..0000000
--- a/lib/glider/SignerCLI.py
+++ /dev/null
@@ -1,313 +0,0 @@
-
-import os
-import getopt
-import sys
-import logging
-import simplejson
-
-import glider.keys
-import glider.formats
-
-def getKeyStore():
- return glider.keys.KeyStore(glider.util.userFilename("secret_keys"))
-
-def dumpKey(key, indent=0):
- i = " "*indent
- print "%s%s"%(i, key.getKeyID())
- for r, p in key.getRoles():
- print " %s%s\t%s"%(i, r, p)
-
-def getKey(ks, keyid=None, role=None, path=None):
- if keyid is not None:
- keys = ks.getKeysFuzzy(keyid)
- if None not in (role, path):
- keys = [ k for k in keys if k.hasRole(role, path) ]
- elif None not in (role, path):
- keys = ks.getKeysByRole(role, path)
- else:
- assert False
- if len(keys) < 1:
- print "No such key.\nI wanted",
- if keyid: print "keyid='%s...'"%keyid,
- if None not in (role, path): print "role=%s, path=%s"%(role,path),
- print
- print "I only know about:"
- for k in ks.iterkeys():
- dumpKey(k)
- sys.exit(1)
- elif len(keys) > 1:
- print "Multiple keys match. Possibilities are:"
- for k in keys:
- dumpKey(k)
- sys.exit(1)
- else:
- return keys[0]
-
-# ------------------------------
-
-def makepackage(args):
- options, args = getopt.getopt(args, "", "keyid=")
- keyid = None
- for o,v in options:
- if o == "--keyid":
- keyid = v
-
- if len(args) < 2:
- usage()
-
- configFile = args[0]
- dataFile = args[1]
- print "Generating package."
- package = glider.formats.makePackageObj(configFile, dataFile)
- relpath = package['location']
- print "need a key with role matching [package %s]"%relpath
- ks = getKeyStore()
- ks.load()
- key = getKey(ks, keyid=keyid, role='package', path=relpath)
- signable = glider.formats.makeSignable(package)
- glider.formats.sign(signable, key)
-
- if 1:
- ss, r, p = glider.formats.checkSignedObj(signable, ks)
- assert ss.isValid()
-
- location = os.path.split(package['location'])[-1]
- print "Writing signed package to %s"%location
- f = open(location, 'w')
- simplejson.dump(signable, f, indent=1)
- f.close()
-
-def makebundle(args):
- options, args = getopt.getopt(args, "", "keyid=")
- keyid = None
- for o,v in options:
- if o == "--keyid":
- keyid = v
-
- if len(args) < 2:
- usage()
-
- configFile = args[0]
- packages = {}
- for pkgFile in args[1:]:
- print "Loading", pkgFile
- f = open(pkgFile, 'r')
- p = simplejson.load(f)
- f.close()
- _, r, _ = glider.formats.checkSignedObj(p)
- if r != 'package':
- print pkgFile, "was not a package"
- packages[p['signed']['location']] = p
-
- def getHash(path):
- p = packages[path]
- return glider.formats.getDigest(p['signed'])
-
- bundleObj = glider.formats.makeBundleObj(configFile, getHash)
- signable = glider.formats.makeSignable(bundleObj)
-
- ks = getKeyStore()
- ks.load()
- key = getKey(ks, keyid=keyid, role="bundle", path=bundleObj['location'])
- glider.formats.sign(signable, key)
-
- if 1:
- ss, r, p = glider.formats.checkSignedObj(signable, ks)
- assert ss.isValid()
-
- location = os.path.split(bundleObj['location'])[-1]
- print "Writing signed bundle to %s"%location
- f = open(location, 'w')
- simplejson.dump(signable, f, indent=1)
- f.close()
-
-# ------------------------------
-def makekeylist(args):
- options, args = getopt.getopt(args, "", "keyid=")
- keyid = None
- for o,v in options:
- if o == "--keyid":
- keyid = v
-
- if len(args) < 1:
- usage()
-
- keylist = glider.formats.makeKeylistObj(args[0])
- signable = glider.formats.makeSignable(keylist)
-
- ks = getKeyStore()
- ks.load()
- key = getKey(ks, keyid=keyid, role="master", path="/meta/keys.txt")
- glider.formats.sign(signable, key)
-
- if 1:
- ss, r, p = glider.formats.checkSignedObj(signable, ks)
- assert ss.isValid()
-
- print "writing signed keylist to keys.txt"
- glider.util.replaceFile("keys.txt",
- simplejson.dumps(signable, indent=1, sort_keys=True),
- textMode=True)
-
-def signkeylist(args):
- if len(args) != 1:
- usage()
-
- keylist = simplejson.load(open(args[0], 'r'))
- glider.formats.SIGNED_SCHEMA.checkMatch(keylist)
- glider.formats.KEYLIST_SCHEMA.checkMatch(keylist['signed'])
-
- ks = getKeyStore()
- ks.load()
- keys = ks.getKeysByRole("master", "/meta/keys.txt")
- for k in keys:
- glider.formats.sign(keylist, k)
-
- print "writing signed keylist to keys.txt"
- glider.util.replaceFile("keys.txt",
- simplejson.dumps(keylist, indent=1, sort_keys=True),
- textMode=True)
-
-def makemirrorlist(args):
- options, args = getopt.getopt(args, "", "keyid=")
- keyid = None
- for o,v in options:
- if o == "--keyid":
- keyid = v
-
- if len(args) < 1:
- usage()
-
- mirrorlist = glider.formats.makeMirrorListObj(args[0])
- signable = glider.formats.makeSignable(mirrorlist)
-
- ks = getKeyStore()
- ks.load()
- key = getKey(ks, keyid=keyid, role='mirrors', path="/meta/mirrors.txt")
- glider.formats.sign(signable, key)
-
- if 1:
- ss, r, p = glider.formats.checkSignedObj(signable, ks)
- assert ss.isValid()
-
- print "writing signed mirrorlist to mirrors.txt"
- glider.util.replaceFile("mirrors.txt",
- simplejson.dumps(signable, indent=1, sort_keys=True),
- textMode=True)
-
-# ------------------------------
-
-def keygen(args):
- k = getKeyStore()
- k.load()
- print "Generating key. This will be slow."
- key = glider.keys.RSAKey.generate()
- print "Generated new key: %s" % key.getKeyID()
- k.addKey(key)
- k.save()
-
-def listkeys(args):
- k = getKeyStore()
- k.load()
- for k in k.iterkeys():
- print k.getKeyID()
- for r, p in k.getRoles():
- print " ", r, p
-
-def addrole(args):
- if len(args) < 3:
- usage()
- ks = getKeyStore()
- ks.load()
- k = getKey(ks, args[0])
- r = args[1]
- if r not in glider.formats.ALL_ROLES:
- print "Unrecognized role %r. Known roles are %s"%(
- r,", ".join(glider.format.ALL_ROLES))
- sys.exit(1)
- p = args[2]
- k.addRole(r, p)
- ks.save()
-
-def delrole(args):
- if len(args) < 3:
- usage()
- ks = getKeyStore()
- ks.load()
- k = getKey(ks, args[0])
- r = args[1]
- if r not in glider.formats.ALL_ROLES:
- print "Unrecognized role %r. Known roles are %s"%(
- r,", ".join(glider.format.ALL_ROLES))
- sys.exit(1)
- p = args[2]
-
- #XXXX rep.
- origLen = len(k._roles)
- k._roles = [ (role,path) for role,path in k._roles
- if (role,path) != (r,p) ]
- removed = origLen - len(k._roles)
- print removed, "roles removed"
- if removed:
- ks.save()
-
-def chpass(args):
- ks = getKeyStore()
- print "Old password."
- ks.load()
- print "New password."
- ks.clearPassword()
- ks.save()
-
-def dumpkey(args):
- options, args = getopt.getopt(args, "", ["include-secret", "passwd="])
-
- includeSecret = False
- for o,v in options:
- if o == '--include-secret':
- includeSecret = True
- else:
- print "Unexpected %r"%o
-
- ks = getKeyStore()
- ks.load()
-
- keys = []
- if len(args):
- keys = [ getKey(ks, a) for a in args ]
- else:
- keys = list(ks.iterkeys())
-
- for k in keys:
- data = k.format(private=includeSecret, includeRoles=True)
- print "Key(", simplejson.dumps(data, indent=2), ")"
-
-def usage():
- print "Known commands:"
- print " keygen"
- print " listkeys"
- print " chpass"
- print " addrole keyid role path"
- print " delrole keyid role path"
- print " dumpkey [--include-secret] keyid"
- print " makepackage config datafile"
- print " makebundle config packagefile ..."
- print " signkeylist keylist"
- print " makekeylist keylist"
- print " makemirrorlist config"
- sys.exit(1)
-
-def main():
- if len(sys.argv) < 2:
- usage()
- cmd = sys.argv[1]
- args = sys.argv[2:]
- if cmd in [ "keygen", "listkeys", "addrole", "delrole", "chpass",
- "dumpkey", "makepackage", "makebundle", "signkeylist",
- "makekeylist", "signkeylist", "makemirrorlist", ]:
- globals()[cmd](args)
- else:
- usage()
-
-if __name__ == '__main__':
- main()
diff --git a/lib/glider/__init__.py b/lib/glider/__init__.py
deleted file mode 100644
index 87fd983..0000000
--- a/lib/glider/__init__.py
+++ /dev/null
@@ -1,35 +0,0 @@
-
-__all__ = [ 'formats' ]
-
-_BaseException = Exception
-
-class Exception(_BaseException):
- pass
-
-class FormatException(Exception):
- pass
-
-class UnknownFormat(FormatException):
- pass
-
-class BadSignature(Exception):
- pass
-
-class BadPassword(Exception):
- pass
-
-class InternalError(Exception):
- pass
-
-class RepoError(InternalError):
- pass
-
-class CryptoError(Exception):
- pass
-
-class PubkeyFormatException(FormatException):
- pass
-
-class UnknownMethod(CryptoError):
- pass
-
diff --git a/lib/glider/checkJson.py b/lib/glider/checkJson.py
deleted file mode 100644
index 0c5eed6..0000000
--- a/lib/glider/checkJson.py
+++ /dev/null
@@ -1,274 +0,0 @@
-
-import re
-import sys
-
-import glider
-
-class Schema:
- def matches(self, obj):
- try:
- self.checkMatch(obj)
- except glider.FormatException:
- return False
- else:
- return True
-
- def checkMatch(self, obj):
- raise NotImplemented()
-
-class Any(Schema):
- """
- >>> s = Any()
- >>> s.matches("A String")
- True
- >>> s.matches([1, "list"])
- True
- """
- def checkMatch(self, obj):
- pass
-
-class RE(Schema):
- """
- >>> s = RE("h.*d")
- >>> s.matches("hello world")
- True
- >>> s.matches("Hello World")
- False
- >>> s.matches("hello world!")
- False
- >>> s.matches([33, "Hello"])
- False
- """
- def __init__(self, pat=None, modifiers=0, reObj=None, reName="pattern"):
- if not reObj:
- if not pat.endswith("$"):
- pat += "$"
- reObj = re.compile(pat, modifiers)
- self._re = reObj
- self._reName = reName
- def checkMatch(self, obj):
- if not isinstance(obj, basestring) or not self._re.match(obj):
- raise glider.FormatException("%r did not match %s"
- %(obj,self._reName))
-
-class Str(Schema):
- """
- >>> s = Str("Hi")
- >>> s.matches("Hi")
- True
- >>> s.matches("Not hi")
- False
- """
- def __init__(self, val):
- self._str = val
- def checkMatch(self, obj):
- if self._str != obj:
- raise glider.FormatException("Expected %r; got %r"%(self._str, obj))
-
-class AnyStr(Schema):
- """
- >>> s = AnyStr()
- >>> s.matches("")
- True
- >>> s.matches("a string")
- True
- >>> s.matches(["a"])
- False
- >>> s.matches(3)
- False
- >>> s.matches(u"a unicode string")
- True
- >>> s.matches({})
- False
- """
- def __init__(self):
- pass
- def checkMatch(self, obj):
- if not isinstance(obj, basestring):
- raise glider.FormatException("Expected a string; got %r"%obj)
-
-class ListOf(Schema):
- """
- >>> s = ListOf(RE("(?:..)*"))
- >>> s.matches("hi")
- False
- >>> s.matches([])
- True
- >>> s.matches({})
- False
- >>> s.matches(["Hi", "this", "list", "is", "full", "of", "even", "strs"])
- True
- >>> s.matches(["This", "one", "is not"])
- False
- """
- def __init__(self, schema, minCount=0, maxCount=sys.maxint,listName="list"):
- self._schema = schema
- self._minCount = minCount
- self._maxCount = maxCount
- self._listName = listName
- def checkMatch(self, obj):
- if not isinstance(obj, (list, tuple)):
- raise glider.FormatException("Expected %s; got %r"
- %(self._listName,obj))
- for item in obj:
- try:
- self._schema.checkMatch(item)
- except glider.FormatException, e:
- raise glider.FormatException("%s in %s"%(e, self._listName))
-
- if not (self._minCount <= len(obj) <= self._maxCount):
- raise glider.FormatException("Length of %s out of range"
- %self._listName)
-
-class Struct(Schema):
- """
- >>> s = Struct([ListOf(AnyStr()), AnyStr(), Str("X")])
- >>> s.matches(False)
- False
- >>> s.matches("Foo")
- False
- >>> s.matches([[], "Q", "X"])
- True
- >>> s.matches([[], "Q", "D"])
- False
- >>> s.matches([[3], "Q", "X"])
- False
- >>> s.matches([[], "Q", "X", "Y"])
- False
- """
- def __init__(self, subschemas, allowMore=False, structName="list"):
- self._subschemas = subschemas[:]
- self._allowMore = allowMore
- self._structName = structName
- def checkMatch(self, obj):
- if not isinstance(obj, (list, tuple)):
- raise glider.FormatException("Expected %s; got %r"
- %(self._structName,obj))
- elif len(obj) < len(self._subschemas):
- raise glider.FormatException(
- "Too few fields in %s"%self._structName)
- elif len(obj) > len(self._subschemas) and not self._allowMore:
- raise glider.FormatException(
- "Too many fields in %s"%self._structName)
- for item, schema in zip(obj, self._subschemas):
- schema.checkMatch(item)
-
-class DictOf(Schema):
- """
- >>> s = DictOf(RE(r'[aeiou]+'), Struct([AnyStr(), AnyStr()]))
- >>> s.matches("")
- False
- >>> s.matches({})
- True
- >>> s.matches({"a": ["x", "y"], "e" : ["", ""]})
- True
- >>> s.matches({"a": ["x", 3], "e" : ["", ""]})
- False
- >>> s.matches({"a": ["x", "y"], "e" : ["", ""], "d" : ["a", "b"]})
- False
- """
- def __init__(self, keySchema, valSchema):
- self._keySchema = keySchema
- self._valSchema = valSchema
- def checkMatch(self, obj):
- try:
- iter = obj.iteritems()
- except AttributeError:
- raise glider.FormatException("Expected a dict; got %r"%obj)
-
- for k,v in iter:
- self._keySchema.checkMatch(k)
- self._valSchema.checkMatch(v)
-
-class Opt:
- """Helper; applied to a value in Obj to mark it optional.
-
- >>> s = Obj(k1=Str("X"), k2=Opt(Str("Y")))
- >>> s.matches({'k1': "X", 'k2': "Y"})
- True
- >>> s.matches({'k1': "X", 'k2': "Z"})
- False
- >>> s.matches({'k1': "X"})
- True
- """
- def __init__(self, schema):
- self._schema = schema
- def checkMatch(self, obj):
- self._schema.checkMatch(obj)
-
-class Obj(Schema):
- """
- >>> s = Obj(a=AnyStr(), bc=Struct([Int(), Int()]))
- >>> s.matches({'a':"ZYYY", 'bc':[5,9]})
- True
- >>> s.matches({'a':"ZYYY", 'bc':[5,9], 'xx':5})
- True
- >>> s.matches({'a':"ZYYY", 'bc':[5,9,3]})
- False
- >>> s.matches({'a':"ZYYY"})
- False
-
- """
- def __init__(self, _objname="object", **d):
- self._objname = _objname
- self._required = d.items()
-
-
- def checkMatch(self, obj):
- for k,schema in self._required:
- try:
- item = obj[k]
- except KeyError:
- if not isinstance(schema, Opt):
- raise glider.FormatException("Missing key %s in %s"
- %(k,self._objname))
-
- else:
- try:
- schema.checkMatch(item)
- except glider.FormatException, e:
- raise glider.FormatException("%s in %s.%s"
- %(e,self._objname,k))
-
-
-class Int(Schema):
- """
- >>> s = Int()
- >>> s.matches(99)
- True
- >>> s.matches(False)
- False
- >>> s.matches(0L)
- True
- >>> s.matches("a string")
- False
- >>> Int(lo=10, hi=30).matches(25)
- True
- >>> Int(lo=10, hi=30).matches(5)
- False
- """
- def __init__(self, lo=-sys.maxint, hi=sys.maxint):
- self._lo = lo
- self._hi = hi
- def checkMatch(self, obj):
- if isinstance(obj, bool) or not isinstance(obj, (int, long)):
- # We need to check for bool as a special case, since bool
- # is for historical reasons a subtype of int.
- raise glider.FormatException("Got %r instead of an integer"%obj)
- elif not (self._lo <= obj <= self._hi):
- raise glider.FormatException("%r not in range [%r,%r]"
- %(obj, self._lo, self._hi))
-
-class Bool(Schema):
- """
- >>> s = Bool()
- >>> s.matches(True) and s.matches(False)
- True
- >>> s.matches(11)
- False
- """
- def __init__(self):
- pass
- def checkMatch(self, obj):
- if not isinstance(obj, bool):
- raise glider.FormatException("Got %r instead of a boolean"%obj)
diff --git a/lib/glider/download.py b/lib/glider/download.py
deleted file mode 100644
index 3135e39..0000000
--- a/lib/glider/download.py
+++ /dev/null
@@ -1,127 +0,0 @@
-
-
-import urllib2
-import httplib
-import random
-
-import threading, Queue
-
-import glider.util
-
-class Downloads:
- def __init__(self, n_threads=2):
- self._lock = threading.RLock()
- self.downloads = {}
- self.haveDownloaded = {}
- self.downloadQueue = Queue.Queue()
- self.threads = [ threading.Thread(target=self._thread) ]
- for t in self.threads:
- t.setDaemon(True)
-
- def start(self):
- for t in self.threads:
- t.start()
-
- def isCurrentlyDownloading(self, relPath):
- self._lock.acquire()
- try:
- return self.downloads.has_key(relPath)
- finally:
- self._lock.release()
-
- def isRedundant(self, relPath):
- self._lock.acquire()
- try:
- return (self.downloads.has_key(relPath) or
- self.haveDownloaded.has_key(relPath))
- finally:
- self._lock.release()
-
- def addDownloadJob(self, job):
- rp = job.getRelativePath()
- self._lock.acquire()
- self.downloads[rp] = job
- self._lock.release()
- self.downloadQueue.put(job)
-
- def _thread(self):
- while True:
- job = self.downloadQueue.get()
- job.download()
- rp = job.getRelativePath()
- self._lock.acquire()
- try:
- del self.downloads[rp]
- self.haveDownloaded[rp] = True
- finally:
- self._lock.release()
-
-class DownloadJob:
- def __init__(self, relPath, destPath, mirrorlist=None,
- wantHash=None, canStall=False):
- self._relPath = relPath
- self._wantHash = wantHash
- self._mirrorList = mirrorlist
- self._destPath = destPath
-
- tmppath = glider.util.userFilename("tmp")
- if relPath.startswith("/"):
- relPath = relPath[1:]
- self._tmppath = os.path.join(tmppath, relPath)
-
- d = os.path.dirname(self._tmppath)
- if not os.path.exists(d):
- os.makedirs(d, 0700)
-
- def getRelativePath(self):
- return self._relPath
-
- def haveStalledFile(self):
- return os.path.exists(self._tmppath)
-
- def getURL(self, mirrorlist=None):
- if mirrorlist is None:
- mirrorlist = self._mirrorList
- weightSoFar = 0
- usable = []
-
- for m in mirrorlist['mirrors']:
- for c in m['contents']:
- # CHECK FOR URL SUITABILITY XXXXX
-
- if glider.formats.rolePathMatches(c, self._relPath):
- weightSoFar += m['weight']
- usable.append( (weightSoFar, m) )
- break
-
- wTarget = random.randint(0, weightSoFar)
- mirror = None
- # Could use bisect here instead
- for w, m in mirrorlist:
- if w >= wTarget:
- mirror = m
- break
-
- return m['urlbase'] + self._relPath
-
- def download(self):
- # XXXX RESUME
-
- f_in = urllib2.urlopen(self.getURL())
- f_out = open(self._tmpPath, 'w')
- while True:
- c = f_in.read(1024)
- if not c:
- break
- f_out.write(c)
- f_in.close()
- f_out.close()
- # XXXXX retry on failure
-
- if self._wantHash:
- gotHash = glider.formats.getFileDigest(self._tmpPath)
- if gotHash != self._wantHash:
- # XXXX Corrupt file.
- pass
-
- glider.utils.moveFile(self._tmpPath, self._destPath)
diff --git a/lib/glider/formats.py b/lib/glider/formats.py
deleted file mode 100644
index 9846c50..0000000
--- a/lib/glider/formats.py
+++ /dev/null
@@ -1,747 +0,0 @@
-
-import simplejson
-import time
-import re
-import binascii
-import calendar
-
-import glider.checkJson
-
-import Crypto.Hash.SHA256
-
-class KeyDB:
- """A KeyDB holds public keys, indexed by their key IDs."""
- def __init__(self):
- self._keys = {}
- def addKey(self, k):
- keyid = k.getKeyID()
- try:
- oldkey = self._keys[keyid]
- for r, p in oldkey.getRoles():
- if (r, p) not in k.getRoles():
- k.addRole(r,p)
- except KeyError:
- pass
- self._keys[k.getKeyID()] = k
- def getKey(self, keyid):
- return self._keys[keyid]
- def getKeysByRole(self, role, path):
- results = []
- for key in self._keys.itervalues():
- for r,p in key.getRoles():
- if r == role:
- if rolePathMatches(p, path):
- results.append(key)
- return results
-
- def getKeysFuzzy(self, keyid):
- r = []
- for k,v in self._keys.iteritems():
- if k.startswith(keyid):
- r.append(v)
- return r
- def iterkeys(self):
- return self._keys.itervalues()
-
-_rolePathCache = {}
-def rolePathMatches(rolePath, path):
- """Return true iff the relative path in the filesystem 'path' conforms
- to the pattern 'rolePath': a path that a given key is
- authorized to sign. Patterns are allowed to contain * to
- represent one or more characters in a filename, and ** to
- represent any level of directory structure.
-
- >>> rolePathMatches("a/b/c/", "a/b/c/")
- True
- >>> rolePathMatches("**/c.*", "a/b/c.txt")
- True
- >>> rolePathMatches("**/c.*", "a/b/ctxt")
- False
- >>> rolePathMatches("**/c.*", "a/b/c.txt/foo")
- False
- >>> rolePathMatches("a/*/c", "a/b/c")
- True
- >>> rolePathMatches("a/*/c", "a/b/c.txt")
- False
- >>> rolePathMatches("a/*/c", "a/b/c.txt") #Check cache
- False
- """
- try:
- regex = _rolePathCache[rolePath]
- except KeyError:
- orig = rolePath
- # remove duplicate slashes.
- rolePath = re.sub(r'/+', '/', rolePath)
- # escape, then ** becomes .*
- rolePath = re.escape(rolePath).replace(r'\*\*', r'.*')
- # * becomes [^/]*
- rolePath = rolePath.replace(r'\*', r'[^/]*')
- # and no extra text is allowed.
- rolePath += "$"
- regex = _rolePathCache[orig] = re.compile(rolePath)
- return regex.match(path) != None
-
-class SignatureStatus:
- """Represents the outcome of checking signature(s) on an object."""
- def __init__(self, good, bad, unrecognized, unauthorized):
- # keyids for all the valid signatures
- self._good = good[:]
- # keyids for the invalid signatures (we had the key, and it failed).
- self._bad = bad[:]
- # keyids for signatures where we didn't recognize the key
- self._unrecognized = unrecognized[:]
- # keyids for signatures where we recognized the key, but it doesn't
- # seem to be allowed to sign this kind of document.
- self._unauthorized = unauthorized[:]
-
- def isValid(self, threshold=1):
- """Return true iff we got at least 'threshold' good signatures."""
- return len(self._good) >= threshold
-
- def mayNeedNewKeys(self):
- """Return true iff downloading a new set of keys might tip this
- signature status over to 'valid.'"""
- return len(self._unrecognized) or len(self._unauthorized)
-
-def checkSignatures(signed, keyDB, role=None, path=None):
- """Given an object conformant to SIGNED_SCHEMA and a set of public keys
- in keyDB, verify the signed object in 'signed'."""
-
- SIGNED_SCHEMA.checkMatch(signed)
-
- goodSigs = []
- badSigs = []
- unknownSigs = []
- tangentialSigs = []
-
- signable = signed['signed']
- signatures = signed['signatures']
-
- d_obj = Crypto.Hash.SHA256.new()
- getDigest(signable, d_obj)
- digest = d_obj.digest()
-
- for signature in signatures:
- sig = signature['sig']
- keyid = signature['keyid']
- method = signature['method']
-
- try:
- key = keyDB.getKey(keyid)
- except KeyError:
- unknownSigs.append(keyid)
- continue
-
- try:
- result = key.checkSignature(method, sig, digest=digest)
- except glider.UnknownMethod:
- continue
-
- if result == True:
- if role is not None:
- for r,p in key.getRoles():
- if r == role and rolePathMatches(p, path):
- break
- else:
- tangentialSigs.append(sig)
- continue
-
- goodSigs.append(keyid)
- else:
- badSigs.append(keyid)
-
- return SignatureStatus(goodSigs, badSigs, unknownSigs, tangentialSigs)
-
-def encodeCanonical(obj, outf=None):
- """Encode the object obj in canoncial JSon form, as specified at
- http://wiki.laptop.org/go/Canonical_JSON . It's a restricted
- dialect of json in which keys are always lexically sorted,
- there is no whitespace, floats aren't allowed, and only quote
- and backslash get escaped. The result is encoded in UTF-8,
- and the resulting bits are passed to outf (if provided), or joined
- into a string and returned.
-
- >>> encodeCanonical("")
- '""'
- >>> encodeCanonical([1, 2, 3])
- '[1,2,3]'
- >>> encodeCanonical({"x" : 3, "y" : 2})
- '{"x":3,"y":2}'
- """
- def default(o):
- raise TypeError("Can't encode %r", o)
- def floatstr(o):
- raise TypeError("Floats not allowed.")
- def canonical_str_encoder(s):
- return '"%s"' % re.sub(r'(["\\])', r'\\\1', s)
-
- # XXX This is, alas, a hack. I'll submit a canonical JSon patch to
- # the simplejson folks.
-
- iterator = simplejson.encoder._make_iterencode(
- None, default, canonical_str_encoder, None, floatstr,
- ":", ",", True, False, True)(obj, 0)
-
- result = None
- if outf == None:
- result = [ ]
- outf = result.append
-
- for u in iterator:
- outf(u.encode("utf-8"))
- if result is not None:
- return "".join(result)
-
-def getDigest(obj, digestObj=None):
- """Update 'digestObj' (typically a SHA256 object) with the digest of
- the canonical json encoding of obj. If digestObj is none,
- compute the SHA256 hash and return it.
-
- DOCDOC string equivalence.
- """
- useTempDigestObj = (digestObj == None)
- if useTempDigestObj:
- digestObj = Crypto.Hash.SHA256.new()
-
- if isinstance(obj, str):
- digestObj.update(obj)
- elif isinstance(obj, unicode):
- digestObj.update(obj.encode("utf-8"))
- else:
- encodeCanonical(obj, digestObj.update)
-
- if useTempDigestObj:
- return digestObj.digest()
-
-def getFileDigest(f, digestObj=None):
- """Update 'digestObj' (typically a SHA256 object) with the digest of
- the file object in f. If digestObj is none, compute the SHA256
- hash and return it.
-
- >>> s = "here is a long string"*1000
- >>> import cStringIO, Crypto.Hash.SHA256
- >>> h1 = Crypto.Hash.SHA256.new()
- >>> h2 = Crypto.Hash.SHA256.new()
- >>> getFileDigest(cStringIO.StringIO(s), h1)
- >>> h2.update(s)
- >>> h1.digest() == h2.digest()
- True
- """
- useTempDigestObj = (digestObj == None)
- if useTempDigestObj:
- digestObj = Crypto.Hash.SHA256.new()
-
- while 1:
- s = f.read(4096)
- if not s:
- break
- digestObj.update(s)
-
- if useTempDigestObj:
- return digestObj.digest()
-
-def makeSignable(obj):
- return { 'signed' : obj, 'signatures' : [] }
-
-def sign(signed, key):
- """Add an element to the signatures of 'signed', containing a new signature
- of the "signed" part.
- """
-
- SIGNED_SCHEMA.checkMatch(signed)
-
- signable = signed["signed"]
- signatures = signed['signatures']
-
- keyid = key.getKeyID()
-
- signatures = [ s for s in signatures if s['keyid'] != keyid ]
-
- method, sig = key.sign(signable)
- signatures.append({ 'keyid' : keyid,
- 'method' : method,
- 'sig' : sig })
- signed['signatures'] = signatures
-
-def formatTime(t):
- """Encode the time 't' in YYYY-MM-DD HH:MM:SS format.
-
- >>> formatTime(1221265172)
- '2008-09-13 00:19:32'
- """
- return time.strftime("%Y-%m-%d %H:%M:%S", time.gmtime(t))
-
-def parseTime(s):
- """Parse a time 's' in YYYY-MM-DD HH:MM:SS format."""
- try:
- return calendar.timegm(time.strptime(s, "%Y-%m-%d %H:%M:%S"))
- except ValueError:
- raise glider.FormatError("Malformed time %r", s)
-
-def formatBase64(h):
- """Return the base64 encoding of h with whitespace and = signs omitted."""
- return binascii.b2a_base64(h).rstrip("=\n ")
-
-formatHash = formatBase64
-
-def parseBase64(s):
- """Parse a base64 encoding with whitespace and = signs omitted. """
- extra = len(s) % 4
- if extra:
- padding = "=" * (4 - extra)
- s += padding
- try:
- return binascii.a2b_base64(s)
- except binascii.Error:
- raise glider.FormatError("Invalid base64 encoding")
-
-def parseHash(s):
- h = parseBase64(s)
- if len(h) != Crypto.Hash.SHA256.digest_size:
- raise glider.FormatError("Bad hash length")
- return h
-
-S = glider.checkJson
-
-# A date, in YYYY-MM-DD HH:MM:SS format.
-TIME_SCHEMA = S.RE(r'\d{4}-\d{2}-\d{2} \d{2}:\d{2}:\d{2}')
-# A hash, base64-encoded
-HASH_SCHEMA = S.RE(r'[a-zA-Z0-9\+\/]{43}')
-
-# A hexadecimal value.
-HEX_SCHEMA = S.RE(r'[a-fA-F0-9]+')
-# A base-64 encoded value
-BASE64_SCHEMA = S.RE(r'[a-zA-Z0-9\+\/]+')
-# An RSA key; subtype of PUBKEY_SCHEMA.
-RSAKEY_SCHEMA = S.Obj(
- _keytype=S.Str("rsa"),
- e=BASE64_SCHEMA,
- n=BASE64_SCHEMA)
-# Any public key.
-PUBKEY_SCHEMA = S.Obj(
- _keytype=S.AnyStr())
-
-KEYID_SCHEMA = HASH_SCHEMA
-SIG_METHOD_SCHEMA = S.AnyStr()
-RELPATH_SCHEMA = PATH_PATTERN_SCHEMA = S.AnyStr()
-URL_SCHEMA = S.AnyStr()
-VERSION_SCHEMA = S.ListOf(S.Any()) #XXXX WRONG
-
-# A single signature of an object. Indicates the signature, the id of the
-# signing key, and the signing method.
-SIGNATURE_SCHEMA = S.Obj(
- keyid=KEYID_SCHEMA,
- method=SIG_METHOD_SCHEMA,
- sig=BASE64_SCHEMA)
-
-# A signed object.
-SIGNED_SCHEMA = S.Obj(
- signed=S.Any(),
- signatures=S.ListOf(SIGNATURE_SCHEMA))
-
-ROLENAME_SCHEMA = S.AnyStr()
-
-# A role: indicates that a key is allowed to certify a kind of
-# document at a certain place in the repo.
-ROLE_SCHEMA = S.Struct([ROLENAME_SCHEMA, PATH_PATTERN_SCHEMA])
-
-# A Keylist: indicates a list of live keys and their roles.
-KEYLIST_SCHEMA = S.Obj(
- _type=S.Str("Keylist"),
- ts=TIME_SCHEMA,
- keys=S.ListOf(S.Obj(key=PUBKEY_SCHEMA, roles=S.ListOf(ROLE_SCHEMA))))
-
-# A Mirrorlist: indicates all the live mirrors, and what documents they
-# serve.
-MIRRORLIST_SCHEMA = S.Obj(
- _type=S.Str("Mirrorlist"),
- ts=TIME_SCHEMA,
- mirrors=S.ListOf(S.Obj(name=S.AnyStr(),
- urlbase=URL_SCHEMA,
- contents=S.ListOf(PATH_PATTERN_SCHEMA),
- weight=S.Int(lo=0),
- )))
-
-# A timestamp: indicates the lastest versions of all top-level signed objects.
-TIMESTAMP_SCHEMA = S.Obj(
- _type = S.Str("Timestamp"),
- at = TIME_SCHEMA,
- m = S.Struct([TIME_SCHEMA, HASH_SCHEMA]),
- k = S.Struct([TIME_SCHEMA, HASH_SCHEMA]),
- b = S.DictOf(keySchema=S.AnyStr(),
- valSchema=
- S.Struct([ VERSION_SCHEMA, RELPATH_SCHEMA, TIME_SCHEMA, HASH_SCHEMA ]))
- )
-
-# A Bundle: lists a bunch of packages that should be updated in tandem
-BUNDLE_SCHEMA = S.Obj(
- _type=S.Str("Bundle"),
- at=TIME_SCHEMA,
- name=S.AnyStr(),
- os=S.AnyStr(),
- arch=S.Opt(S.AnyStr()),
- version=VERSION_SCHEMA,
- location=RELPATH_SCHEMA,
- packages=S.ListOf(S.Obj(
- name=S.AnyStr(),
- version=VERSION_SCHEMA,
- path=RELPATH_SCHEMA,
- hash=HASH_SCHEMA,
- order=S.Struct([S.Int(), S.Int(), S.Int()]),
- optional=S.Opt(S.Bool()),
- gloss=S.DictOf(S.AnyStr(), S.AnyStr()),
- longgloss=S.DictOf(S.AnyStr(), S.AnyStr()))))
-
-PACKAGE_SCHEMA = S.Obj(
- _type=S.Str("Package"),
- name=S.AnyStr(),
- location=RELPATH_SCHEMA,
- version=VERSION_SCHEMA,
- format=S.Obj(),
- ts=TIME_SCHEMA,
- files=S.ListOf(S.Struct([RELPATH_SCHEMA, HASH_SCHEMA])),
- shortdesc=S.DictOf(S.AnyStr(), S.AnyStr()),
- longdesc=S.DictOf(S.AnyStr(), S.AnyStr()))
-
-ALL_ROLES = ('timestamp', 'mirrors', 'bundle', 'package', 'master')
-
-class Key:
- #XXXX UNUSED.
- def __init__(self, key, roles=()):
- self.key = key
- self.roles = []
- for r,p in roles:
- self.addRole(r,p)
-
- def addRole(self, role, path):
- assert role in ALL_ROLES
- self.roles.append((role, path))
-
- def getRoles(self):
- return self.roles
-
- @staticmethod
- def fromJSon(obj):
- # must match PUBKEY_SCHEMA
- keytype = obj['_keytype']
- if keytype == 'rsa':
- return Key(glider.keys.RSAKey.fromJSon(obj))
-
- if typeattr == 'rsa':
- key = glider.keys.RSAKey.fromSExpression(sexpr)
- if key is not None:
- return Key(key)
- else:
- return None
-
- def format(self):
- return self.key.format()
-
- def getKeyID(self):
- return self.key.getKeyID()
-
- def sign(self, sexpr=None, digest=None):
- return self.key.sign(sexpr, digest=digest)
-
- def checkSignature(self, method, data, signatute):
- ok = self.key.checkSignature(method, data, signature)
- # XXXX CACHE HERE.
- return ok
-
-class Keylist(KeyDB):
- def __init__(self):
- KeyDB.__init__(self)
-
- def addFromKeylist(self, obj, allowMasterKeys=False):
- for keyitem in obj['keys']:
- key = keyitem['key']
- roles = keyitem['roles']
-
- try:
- key = glider.keys.RSAKey.fromJSon(key)
- except glider.FormatException, e:
- print e
- #LOG skipping key.
- continue
-
- for r,p in roles:
- if r == 'master' and not allowMasterKeys:
- #LOG
- continue
- if r not in ALL_ROLES:
- continue
- key.addRole(r,p)
-
- self.addKey(key)
-
-class StampedInfo:
- def __init__(self, ts, hash, version=None, relpath=None):
- self._ts = ts
- self._hash = hash
- self._version = version
- self._relpath = relpath
-
- @staticmethod
- def fromJSonFields(timeStr, hashStr):
- t = parseTime(timeStr)
- h = parseHash(hashStr)
- return StampedInfo(t, h)
-
- def getHash(self):
- return self._hash
-
- def getRelativePath(self):
- return self._relpath
-
-class TimestampFile:
- def __init__(self, at, mirrorlistinfo, keylistinfo, bundleinfo):
- self._time = at
- self._mirrorListInfo = mirrorlistinfo
- self._keyListInfo = keylistinfo
- self._bundleInfo = bundleinfo
-
- @staticmethod
- def fromJSon(obj):
- # must be validated.
- at = parseTime(obj['at'])
- m = StampedInfo.fromJSonFields(*obj['m'][:2])
- k = StampedInfo.fromJSonFields(*obj['k'][:2])
- b = {}
- for name, bundle in obj['b'].iteritems():
- v = bundle[0]
- rp = bundle[1]
- t = parseTime(bundle[2])
- h = parseHash(bundle[3])
- b[name] = StampedInfo(t, h, v, rp)
-
- return TimestampFile(at, m, k, b)
-
- def getTime(self):
- return self._time
-
- def getMirrorlistInfo(self):
- return self._mirrorListInfo
-
- def getKeylistInfo(self):
- return self._keyListInfo
-
- def getBundleInfo(self, name):
- return self._bundleInfo[name]
-
-def readConfigFile(fname, needKeys=(), optKeys=(), preload={}):
- parsed = preload.copy()
- result = {}
- execfile(fname, parsed)
-
- for k in needKeys:
- try:
- result[k] = parsed[k]
- except KeyError:
- raise glider.FormatError("Missing value for %s in %s"%k,fname)
-
- for k in optKeys:
- try:
- result[k] = parsed[k]
- except KeyError:
- pass
-
- return result
-
-def makePackageObj(config_fname, package_fname):
- preload = {}
- shortDescs = {}
- longDescs = {}
- def ShortDesc(lang, val): shortDescs[lang] = val
- def LongDesc(lang, val): longDescs[lang] = val
- preload = { 'ShortDesc' : ShortDesc, 'LongDesc' : LongDesc }
- r = readConfigFile(config_fname,
- ['name',
- 'version',
- 'format',
- 'location',
- 'relpath',
- ], (), preload)
-
- f = open(package_fname, 'rb')
- digest = getFileDigest(f)
-
- # Check fields!
- result = { '_type' : "Package",
- 'ts' : formatTime(time.time()),
- 'name' : r['name'],
- 'location' : r['location'], #DOCDOC
- 'version' : r['version'],
- 'format' : r['format'],
- 'files' : [ [ r['relpath'], formatHash(digest) ] ],
- 'shortdesc' : shortDescs,
- 'longdesc' : longDescs
- }
-
- PACKAGE_SCHEMA.checkMatch(result)
-
- return result
-
-def makeBundleObj(config_fname, getPackageHash):
- packages = []
- def ShortGloss(lang, val): packages[-1]['gloss'][lang] = val
- def LongGloss(lang, val): packages[-1]['longgloss'][lang] = val
- def Package(name, version, path, order, optional=False):
- packages.append({'name' : name,
- 'version' : version,
- 'path' : path,
- 'order' : order,
- 'optional' : optional,
- 'gloss' : {},
- 'longgloss' : {} })
- preload = { 'ShortGloss' : ShortGloss, 'LongGloss' : LongGloss,
- 'Package' : Package }
- r = readConfigFile(config_fname,
- ['name',
- 'os',
- 'version',
- 'location',
- ], ['arch'], preload)
-
- result = { '_type' : "Bundle",
- 'at' : formatTime(time.time()),
- 'name' : r['name'],
- 'os' : r['os'],
- 'version' : r['version'],
- 'location' : r['location'],
- 'packages' : packages }
- if r.has_key('arch'):
- result['arch'] = r['arch']
-
- for p in packages:
- try:
- p['hash'] = formatHash(getPackageHash(p['path']))
- except KeyError:
- raise glider.FormatException("No such package as %s"%p['path'])
-
- BUNDLE_SCHEMA.checkMatch(result)
- return result
-
-def versionIsNewer(v1, v2):
- return v1 > v2
-
-def makeTimestampObj(mirrorlist_obj, keylist_obj,
- bundle_objs):
- result = { '_type' : 'Timestamp',
- 'at' : formatTime(time.time()) }
- result['m'] = [ mirrorlist_obj['ts'],
- formatHash(getDigest(mirrorlist_obj)) ]
- result['k'] = [ keylist_obj['ts'],
- formatHash(getDigest(keylist_obj)) ]
- result['b'] = bundles = {}
- for bundle in bundle_objs:
- name = bundle['name']
- v = bundle['version']
- entry = [ v, bundle['location'], bundle['at'], formatHash(getDigest(bundle)) ]
- if not bundles.has_key(name) or versionIsNewer(v, bundles[name][0]):
- bundles[name] = entry
-
- TIMESTAMP_SCHEMA.checkMatch(result)
-
- return result
-
-class MirrorInfo:
- def __init__(self, name, urlbase, contents, weight):
- self._name = name
- self._urlbase = urlbase
- self._contents = contents
- self._weight = weight
-
- def canServeFile(self, fname):
- for c in self._contents:
- if rolePathMatches(c, fname):
- return True
- return False
-
- def getFileURL(self, fname):
- if self._urlbase[-1] == '/':
- return self._urlbase+fname
- else:
- return "%s/%s" % (self._urlbase, fname)
-
- def format(self):
- return { 'name' : self._name,
- 'urlbase' : self._urlbase,
- 'contents' : self._contents,
- 'weight' : self._weight }
-
-def makeMirrorListObj(mirror_fname):
- mirrors = []
- def Mirror(*a, **kw): mirrors.append(MirrorInfo(*a, **kw))
- preload = {'Mirror' : Mirror}
- r = readConfigFile(mirror_fname, (), (), preload)
- result = { '_type' : "Mirrorlist",
- 'ts' : formatTime(time.time()),
- 'mirrors' : [ m.format() for m in mirrors ] }
-
- MIRRORLIST_SCHEMA.checkMatch(result)
- return result
-
-def makeKeylistObj(keylist_fname, includePrivate=False):
- keys = []
- def Key(obj): keys.append(obj)
- preload = {'Key': Key}
- r = readConfigFile(keylist_fname, (), (), preload)
-
- klist = []
- for k in keys:
- k = glider.keys.RSAKey.fromJSon(k)
- klist.append({'key': k.format(private=includePrivate), 'roles' : k.getRoles() })
-
- result = { '_type' : "Keylist",
- 'ts' : formatTime(time.time()),
- 'keys' : klist }
-
- KEYLIST_SCHEMA.checkMatch(result)
- return result
-
-SCHEMAS_BY_TYPE = {
- 'Keylist' : KEYLIST_SCHEMA,
- 'Mirrorlist' : MIRRORLIST_SCHEMA,
- 'Timestamp' : TIMESTAMP_SCHEMA,
- 'Bundle' : BUNDLE_SCHEMA,
- 'Package' : PACKAGE_SCHEMA,
- }
-
-def checkSignedObj(obj, keydb=None):
- # Returns signaturestatus, role, path on sucess.
-
- SIGNED_SCHEMA.checkMatch(obj)
- try:
- tp = obj['signed']['_type']
- except KeyError:
- raise glider.FormatException("Untyped object")
- try:
- schema = SCHEMAS_BY_TYPE[tp]
- except KeyError:
- raise glider.FormatException("Unrecognized type %r" % tp)
- schema.checkMatch(obj['signed'])
-
- if tp == 'Keylist':
- role = "master"
- path = "/meta/keys.txt"
- elif tp == 'Mirrorlist':
- role = "mirrors"
- path = "/meta/mirrors.txt"
- elif tp == "Timestamp":
- role = 'timestamp'
- path = "/meta/timestamp.txt"
- elif tp == 'Bundle':
- role = 'bundle'
- path = obj['signed']['location']
- elif tp == 'Package':
- role = 'package'
- path = obj['signed']['location']
- else:
- print tp
- raise "Foo"
-
- ss = None
- if keydb is not None:
- ss = checkSignatures(obj, keydb, role, path)
-
- return ss, role, path
diff --git a/lib/glider/keys.py b/lib/glider/keys.py
deleted file mode 100644
index fefff6c..0000000
--- a/lib/glider/keys.py
+++ /dev/null
@@ -1,399 +0,0 @@
-
-# These require PyCrypto.
-import Crypto.PublicKey.RSA
-import Crypto.Hash.SHA256
-import Crypto.Cipher.AES
-
-import cPickle as pickle
-import binascii
-import logging
-import os
-import struct
-import sys
-import simplejson
-import getpass
-
-import glider.formats
-import glider.util
-
-class PublicKey:
- def __init__(self):
- # Confusingly, these roles are the ones used for a private key to
- # remember what we're willing to do with it.
- self._roles = []
- def format(self):
- raise NotImplemented()
- def sign(self, data):
- # returns a list of method,signature tuples.
- raise NotImplemented()
- def checkSignature(self, method, data, signature):
- # returns True, False, or raises UnknownMethod.
- raise NotImplemented()
- def getKeyID(self):
- raise NotImplemented()
- def getRoles(self):
- return self._roles
- def addRole(self, role, path):
- assert role in glider.formats.ALL_ROLES
- self._roles.append((role, path))
- def clearRoles(self):
- del self._roles[:]
- def hasRole(self, role, path):
- for r, p in self._roles:
- if r == role and glider.formats.rolePathMatches(p, path):
- return True
- return False
-
-if hex(1L).upper() == "0X1L":
- def intToBinary(number):
- """Convert an int or long into a big-endian series of bytes.
- """
- # This "convert-to-hex, then use binascii" approach may look silly,
- # but it's over 10x faster than the Crypto.Util.number approach.
- h = hex(long(number))
- h = h[2:-1]
- if len(h)%2:
- h = "0"+h
- return binascii.a2b_hex(h)
-elif hex(1L).upper() == "0X1":
- def intToBinary(number):
- h = hex(long(number))
- h = h[2:]
- if len(h)%2:
- h = "0"+h
- return binascii.a2b_hex(h)
-else:
- import Crypto.Util.number
- intToBinary = Crypto.Util.number.long_to_bytes
- assert None
-
-def binaryToInt(binary):
- """Convert a big-endian series of bytes into a long.
- """
- return long(binascii.b2a_hex(binary), 16)
-
-def intToBase64(number):
- return glider.formats.formatBase64(intToBinary(number))
-
-def base64ToInt(number):
- return binaryToInt(glider.formats.parseBase64(number))
-
-def _pkcs1_padding(m, size):
- # I'd rather use OAEP+, but apparently PyCrypto barely supports
- # signature verification, and doesn't seem to support signature
- # verification with nondeterministic padding. "argh."
-
- s = [ "\x00\x01", "\xff"* (size-3-len(m)), "\x00", m ]
- r = "".join(s)
- return r
-
-def _xor(a,b):
- if a:
- return not b
- else:
- return b
-
-class RSAKey(PublicKey):
- """
- >>> k = RSAKey.generate(bits=512)
- >>> obj = k.format()
- >>> obj['_keytype']
- 'rsa'
- >>> base64ToInt(obj['e'])
- 65537L
- >>> k1 = RSAKey.fromJSon(obj)
- >>> k1.key.e == k.key.e
- True
- >>> k1.key.n == k.key.n
- True
- >>> k.getKeyID() == k1.getKeyID()
- True
- >>> s = { 'A B C' : "D", "E" : [ "F", "g", 99] }
- >>> method, sig = k.sign(obj=s)
- >>> k.checkSignature(method, sig, obj=s)
- True
- >>> s2 = [ s ]
- >>> k.checkSignature(method, sig, obj=s2)
- False
- """
- def __init__(self, key):
- PublicKey.__init__(self)
- self.key = key
- self.keyid = None
-
- @staticmethod
- def generate(bits=2048):
- key = Crypto.PublicKey.RSA.generate(bits=bits, randfunc=os.urandom)
- return RSAKey(key)
-
- @staticmethod
- def fromJSon(obj):
- # obj must match RSAKEY_SCHEMA
-
- glider.formats.RSAKEY_SCHEMA.checkMatch(obj)
- n = base64ToInt(obj['n'])
- e = base64ToInt(obj['e'])
- if obj.has_key('d'):
- d = base64ToInt(obj['d'])
- p = base64ToInt(obj['p'])
- q = base64ToInt(obj['q'])
- u = base64ToInt(obj['u'])
- key = Crypto.PublicKey.RSA.construct((n, e, d, p, q, u))
- else:
- key = Crypto.PublicKey.RSA.construct((n, e))
-
- result = RSAKey(key)
- if obj.has_key('roles'):
- for r, p in obj['roles']:
- result.addRole(r,p)
-
- return result
-
- def isPrivateKey(self):
- return hasattr(self.key, 'd')
-
- def format(self, private=False, includeRoles=False):
- n = intToBase64(self.key.n)
- e = intToBase64(self.key.e)
- result = { '_keytype' : 'rsa',
- 'e' : e,
- 'n' : n }
- if private:
- result['d'] = intToBase64(self.key.d)
- result['p'] = intToBase64(self.key.p)
- result['q'] = intToBase64(self.key.q)
- result['u'] = intToBase64(self.key.u)
- if includeRoles:
- result['roles'] = self.getRoles()
- return result
-
- def getKeyID(self):
- if self.keyid == None:
- d_obj = Crypto.Hash.SHA256.new()
- glider.formats.getDigest(self.format(), d_obj)
- self.keyid = glider.formats.formatHash(d_obj.digest())
- return self.keyid
-
- def _digest(self, obj, method=None):
- if method in (None, "sha256-pkcs1"):
- d_obj = Crypto.Hash.SHA256.new()
- glider.formats.getDigest(obj, d_obj)
- digest = d_obj.digest()
- return ("sha256-pkcs1", digest)
-
- raise UnknownMethod(method)
-
- def sign(self, obj=None, digest=None):
- assert _xor(obj == None, digest == None)
- if digest == None:
- method, digest = self._digest(obj)
- m = _pkcs1_padding(digest, (self.key.size()+1) // 8)
- sig = intToBase64(self.key.sign(m, "")[0])
- return (method, sig)
-
- def checkSignature(self, method, sig, obj=None, digest=None):
- assert _xor(obj == None, digest == None)
- if method != "sha256-pkcs1":
- raise UnknownMethod("method")
- if digest == None:
- method, digest = self._digest(obj, method)
- sig = base64ToInt(sig)
- m = _pkcs1_padding(digest, (self.key.size()+1) // 8)
- return bool(self.key.verify(m, (sig,)))
-
-SALTLEN=16
-
-def secretToKey(salt, secret):
- """Convert 'secret' to a 32-byte key, using a version of the algorithm
- from RFC2440. The salt must be SALTLEN+1 bytes long, and should
- be random, except for the last byte, which encodes how time-
- consuming the computation should be.
-
- (The goal is to make offline password-guessing attacks harder by
- increasing the time required to convert a password to a key, and to
- make precomputed password tables impossible to generate by )
- """
- assert len(salt) == SALTLEN+1
-
- # The algorithm is basically, 'call the last byte of the salt the
- # "difficulty", and all other bytes of the salt S. Now make
- # an infinite stream of S|secret|S|secret|..., and hash the
- # first N bytes of that, where N is determined by the difficulty.
- #
- # Obviously, this wants a hash algorithm that's tricky to
- # parallelize.
- #
- # Unlike RFC2440, we use a 16-byte salt. Because CPU times
- # have improved, we start at 16 times the previous minimum.
-
- difficulty = ord(salt[-1])
- count = (16L+(difficulty & 15)) << ((difficulty >> 4) + 10)
-
- # Make 'data' nice and long, so that we don't need to call update()
- # a zillion times.
- data = salt[:-1]+secret
- if len(data)<1024:
- data *= (1024 // len(data))+1
-
- d = Crypto.Hash.SHA256.new()
- iters, leftover = divmod(count, len(data))
- for _ in xrange(iters):
- d.update(data)
- #count -= len(data)
- if leftover:
- d.update(data[:leftover])
- #count -= leftover
- #assert count == 0
-
- return d.digest()
-
-def encryptSecret(secret, password, difficulty=0x80):
- """Encrypt the secret 'secret' using the password 'password',
- and return the encrypted result."""
- # The encrypted format is:
- # "GKEY1" -- 5 octets, fixed, denotes data format.
- # SALT -- 17 bytes, used to hash password
- # IV -- 16 bytes; salt for encryption
- # ENCRYPTED IN AES256-OFB, using a key=s2k(password, salt) and IV=IV:
- # SLEN -- 4 bytes; length of secret, big-endian.
- # SECRET -- len(secret) bytes
- # D -- 32 bytes; SHA256 hash of (salt|secret|salt).
- #
- # This format leaks the secret length, obviously.
- assert 0 <= difficulty < 256
- salt = os.urandom(SALTLEN)+chr(difficulty)
- key = secretToKey(salt, password)
-
- d_obj = Crypto.Hash.SHA256.new()
- d_obj.update(salt)
- d_obj.update(secret)
- d_obj.update(salt)
- d = d_obj.digest()
-
- iv = os.urandom(16)
- e = Crypto.Cipher.AES.new(key, Crypto.Cipher.AES.MODE_OFB, iv)
-
- # Stupidly, pycrypto doesn't accept that stream ciphers don't need to
- # take their input in blocks. So pad it, then ignore the padded output.
-
- padlen = 16-((len(secret)+len(d)+4) % 16)
- if padlen == 16: padlen = 0
- pad = '\x00' * padlen
-
- slen = struct.pack("!L",len(secret))
- encrypted = e.encrypt("%s%s%s%s" % (slen, secret, d, pad))
- if padlen:
- encrypted = encrypted[:-padlen]
- return "GKEY1%s%s%s"%(salt, iv, encrypted)
-
-def decryptSecret(encrypted, password):
- """Decrypt a value encrypted with encryptSecret. Raises UnknownFormat
- or FormatError if 'encrypted' was not generated with encryptSecret.
- Raises BadPassword if the password was not correct.
- """
- if encrypted[:5] != "GKEY1":
- raise glider.UnknownFormat()
- encrypted = encrypted[5:]
- if len(encrypted) < SALTLEN+1+16:
- raise glider.FormatException()
-
- salt = encrypted[:SALTLEN+1]
- iv = encrypted[SALTLEN+1:SALTLEN+1+16]
- encrypted = encrypted[SALTLEN+1+16:]
-
- key = secretToKey(salt, password)
-
- e = Crypto.Cipher.AES.new(key, Crypto.Cipher.AES.MODE_OFB, iv)
- padlen = 16-(len(encrypted) % 16)
- if padlen == 16: padlen = 0
- pad = '\x00' * padlen
-
- decrypted = e.decrypt("%s%s"%(encrypted,pad))
- slen = struct.unpack("!L", decrypted[:4])[0]
- secret = decrypted[4:4+slen]
- hash = decrypted[4+slen:4+slen+Crypto.Hash.SHA256.digest_size]
-
- d = Crypto.Hash.SHA256.new()
- d.update(salt)
- d.update(secret)
- d.update(salt)
-
- if d.digest() != hash:
- raise glider.BadPassword()
-
- return secret
-
-class KeyStore(glider.formats.KeyDB):
- def __init__(self, fname, encrypted=True):
- glider.formats.KeyDB.__init__(self)
-
- self._loaded = None
- self._fname = fname
- self._passwd = None
- self._encrypted = encrypted
-
- def getpass(self, reprompt=False):
- if self._passwd != None:
- return self._passwd
- while 1:
- pwd = getpass.getpass("Password: ", sys.stderr)
- if not reprompt:
- return pwd
-
- pwd2 = getpass.getpass("Confirm: ", sys.stderr)
- if pwd == pwd2:
- return pwd
- else:
- print "Mismatch; try again."
-
- def load(self, password=None):
- logging.info("Loading private keys from %r...", self._fname)
- if not os.path.exists(self._fname):
- logging.info("...no such file.")
- self._loaded = True
- return
-
- if password is None and self._encrypted:
- password = self.getpass()
-
- contents = open(self._fname, 'rb').read()
- if self._encrypted:
- contents = decryptSecret(contents, password)
-
- listOfKeys = simplejson.loads(contents)
- self._passwd = password # It worked.
- if not listOfKeys.has_key('keys'):
- listOfKeys['keys'] = []
- for obj in listOfKeys['keys']:
- key = RSAKey.fromJSon(obj)
- self.addKey(key)
- logging.info("Loaded key %s", key.getKeyID())
-
- self._loaded = True
-
- def setPassword(self, passwd):
- self._passwd = passwd
-
- def clearPassword(self):
- self._passwd = None
-
- def save(self, password=None):
- if not self._loaded and self._encrypted:
- self.load(password)
-
- if password is None:
- password = self.getpass(True)
-
- logging.info("Saving private keys into %r...", self._fname)
- listOfKeys = { 'keys' :
- [ key.format(private=True, includeRoles=True) for key in
- self._keys.values() ]
- }
- contents = simplejson.dumps(listOfKeys)
- if self._encrypted:
- contents = encryptSecret(contents, password)
- glider.util.replaceFile(self._fname, contents)
- self._passwd = password # It worked.
- logging.info("Done.")
-
-
diff --git a/lib/glider/master_keys.py b/lib/glider/master_keys.py
deleted file mode 100644
index 0d455d1..0000000
--- a/lib/glider/master_keys.py
+++ /dev/null
@@ -1,5 +0,0 @@
-
-
-MASTER_KEYS = [
-
-]
diff --git a/lib/glider/repository.py b/lib/glider/repository.py
deleted file mode 100644
index b956993..0000000
--- a/lib/glider/repository.py
+++ /dev/null
@@ -1,313 +0,0 @@
-
-import glider.formats
-import glider.util
-
-import simplejson
-import logging
-import os
-import threading
-import time
-
-MAX_TIMESTAMP_AGE = 24*60*60
-
-class RepositoryFile:
- def __init__(self, repository, relativePath, schema,
- needRole=None, signedFormat=True, needSigs=1):
- self._repository = repository
- self._relativePath = relativePath
- self._schema = schema
- self._needRole = needRole
- self._signedFormat = signedFormat
- self._needSigs = needSigs
-
- self._signed_obj = self._main_obj = None
- self._sigStatus = None
- self._mtime = None
-
- def getRelativePath(self):
- return self._relativePath
-
- def getPath(self):
- return self._repository.getFilename(self._relativePath)
-
- def _load(self):
- fname = self.getPath()
-
- # Propagate OSError
- f = None
- fd = os.open(fname, os.O_RDONLY)
- try:
- f = os.fdopen(fd, 'r')
- except:
- os.close(fd)
- raise
- try:
- mtime = os.fstat(fd).st_mtime
- content = f.read()
- finally:
- f.close()
-
- signed_obj,main_obj = self._checkContent(content)
-
- self._signed_obj = signed_obj
- self._main_obj = main_obj
- self._mtime = mtime
-
- def _save(self, content=None):
- if content == None:
- content = sexpr.encode
-
- signed_obj,main_obj = self._checkContent(content)
-
- fname = self.getPath()
- glider.util.replaceFile(fname, contents)
-
- self._signed_obj = signed_obj
- self._main_obj = main_obj
- self._mtime = mtime
-
- def _checkContent(self, content):
-
- try:
- obj = simplejson.loads(content)
- except ValueError, e:
- raise glider.FormatException("Couldn't decode content: %s"%e)
-
- if self._signedFormat:
- # This is supposed to be signed.
- glider.formats.SIGNED_SCHEMA.checkMatch(obj)
-
- main_obj = obj['signed']
- signed_obj = obj
- else:
- signed_obj = None
- main_obj = obj
-
- if self._schema != None:
- self._schema.checkMatch(main_obj)
-
- return signed_obj, main_obj
-
- def load(self):
- if self._main_obj == None:
- self._load()
-
- def get(self):
- return self._main_obj
-
- def isLoaded(self):
- return self._main_obj != None
-
- def getContent(self):
- self.load()
- return self._main_obj
-
- def _checkSignatures(self):
- self.load()
- sigStatus = glider.formats.checkSignatures(self._signed_obj,
- self._repository._keyDB,
- self._needRole, self._relativePath)
- self._sigStatus = sigStatus
-
- def checkSignatures(self):
- if self._sigStatus is None:
- self._checkSignatures()
- return self._sigStatus
-
-class LocalRepository:
- def __init__(self, root):
- self._root = root
- self._keyDB = glider.util.getKeylist(None)
-
- self._keylistFile = RepositoryFile(
- self, "/meta/keys.txt", glider.formats.KEYLIST_SCHEMA,
- needRole="master")
- self._timestampFile = RepositoryFile(
- self, "/meta/timestamp.txt", glider.formats.TIMESTAMP_SCHEMA,
- needRole="timestamp")
- self._mirrorlistFile = RepositoryFile(
- self, "/meta/mirrors.txt", glider.formats.MIRRORLIST_SCHEMA,
- needRole="mirrors")
- self._metaFiles = [ self._keylistFile,
- self._timestampFile,
- self._mirrorlistFile ]
-
- self._packageFiles = {}
- self._bundleFiles = {}
-
- def getFilename(self, relativePath):
- if relativePath.startswith("/"):
- relativePath = relativePath[1:]
- return os.path.join(self._root, relativePath)
-
- def getKeylistFile(self):
- return self._keylistFile
-
- def getTimestampFile(self):
- return self._timestampFile
-
- def getMirrorlistFile(self):
- return self._mirrorlistFile
-
- def getPackageFile(self, relPath):
- try:
- return self._packageFiles[relPath]
- except KeyError:
- self._packageFiles[relPath] = pkg = RepositoryFile(
- self, relPath, glider.formats.PACKAGE_SCHEMA,
- needRole='package')
- return pkg
-
- def getBundleFile(self, relPath):
- try:
- return self._bundleFiles[relPath]
- except KeyError:
- self._bundleFiles[relPath] = pkg = RepositoryFile(
- self, relPath, glider.formats.BUNDLE_SCHEMA,
- needRole='bundle')
- return pkg
-
- def getFilesToUpdate(self, now=None, trackingBundles=()):
- if now == None:
- now = time.time()
-
- need = set()
-
- # Fetch missing metafiles.
- for f in self._metaFiles:
- try:
- f.load()
- except OSError, e:
- print "need", f.getPath()
- logging.info("Couldn't load %s: %s. Must fetch it.",
- f.getPath(), e)
- need.add(f.getRelativePath())
-
- # If the timestamp file is out of date, we need to fetch it no
- # matter what. (Even if it is isn't signed, it can't possibly
- # be good.)
- ts = self._timestampFile.get()
- if ts:
- age = now - glider.formats.parseTime(ts['at'])
- ts = glider.formats.TimestampFile.fromJSon(ts)
- if age > MAX_TIMESTAMP_AGE:
- need.add(self._timestampFile.getRelativePath())
-
- # If the keylist isn't signed right, we can't check the
- # signatures on anything else.
- if self._keylistFile.get():
- s = self._keylistFile.checkSignatures()
- if not s.isValid(): # For now only require one master key.
- need.add(self._keylistFile.getRelativePath())
-
- if need:
- return need
-
- # Import the keys from the keylist.
- self._keyDB.addFromKeylist(self._keylistFile.get())
-
- # If the timestamp isn't signed right, get a new timestamp and a
- # new keylist.
- s = self._timestampFile.checkSignatures()
- if not s.isValid():
- need.add(self._keylistFile.getRelativePath())
- need.add(self._timestampFile.getRelativePath())
- return need
-
- # FINALLY, we know we have an up-to-date, signed timestamp
- # file. Check whether the keys and mirrors file are as
- # authenticated.
- h_kf = glider.formats.getDigest(self._keylistFile.get())
- h_expected = ts.getKeylistInfo().getHash()
- if h_kf != h_expected:
- need.add(self._keylistFile.getRelativePath())
-
- if need:
- return need
-
- s = self._mirrorlistFile.checkSignatures()
- if not s.isValid():
- need.add(self._mirrorlistFile.getRelativePath())
-
- h_mf = glider.formats.getDigest(self._mirrorlistFile.get())
- h_expected = ts.getMirrorlistInfo().getHash()
- if h_mf != h_expected:
- need.add(self._mirrorlistFile.getRelativePath())
-
- if need:
- return need
-
- # Okay; that's it for the metadata. Do we have the right
- # bundles?
- bundles = {}
- for b in trackingBundles:
- try:
- binfo = ts.getBundleInfo(b)
- except KeyError:
- logging.warn("Unrecognized bundle %s"%b)
- continue
-
- rp = binfo.getRelativePath()
- bfile = self.getBundleFile(rp)
- try:
- bfile.load()
- except OSError:
- need.add(rp)
- continue
-
- h_b = glider.formats.getDigest(bfile.get())
- h_expected = binfo.getHash()
- if h_b != h_expected:
- need.add(rp)
- continue
-
- s = bfile.checkSignatures()
- if not s.isValid():
- # Can't actually use it.
- continue
-
- bundles[rp] = bfile
-
- # Okay. So we have some bundles. See if we have their packages.
- packages = {}
- for bfile in bundles.values():
- bundle = bfile.get()
- for pkginfo in bundle['packages']:
- rp = pkginfo['path']
- pfile = self.getPackageFile(rp)
- try:
- pfile.load()
- except OSError:
- need.add(rp)
- continue
-
- h_p = glider.formats.getDigest(pfile.get())
- h_expected = glider.formats.parseHash(pkginfo['hash'])
- if h_p != h_expected:
- need.add(rp)
- continue
-
- s = pfile.checkSignatures()
- if not s.isValid():
- # Can't use it.
- continue
- packages[rp] = pfile
-
- # Finally, we have some packages. Do we have their underlying
- # files?
- for pfile in packages.values():
- package = pfile.get()
- for f in package['files']:
- rp, h = f[:2]
- h_expected = glider.formats.parseHash(h)
- fn = self.getFilename(rp)
- try:
- h_got = glider.formats.getFileDigest(fn)
- except OSError:
- need.add(rp)
- continue
- if h_got != h_expected:
- need.add(rp)
-
- # Okay; these are the files we need.
- return need
diff --git a/lib/glider/tests.py b/lib/glider/tests.py
deleted file mode 100644
index 7aeb4f6..0000000
--- a/lib/glider/tests.py
+++ /dev/null
@@ -1,64 +0,0 @@
-
-import unittest
-import doctest
-import os
-import tempfile
-
-import glider.keys
-import glider.formats
-import glider.repository
-import glider.checkJson
-
-import glider.tests
-
-class CanonicalEncodingTest(unittest.TestCase):
- def test_encode(self):
- enc = glider.formats.encodeCanonical
- self.assertEquals(enc(''), '""')
- self.assertEquals(enc('"'), '"\\""')
- self.assertEquals(enc('\t\\\n"\r'),
- '"\t\\\\\n\\"\r"')
-
-class CryptoTests(unittest.TestCase):
- def test_encrypt(self):
- s = "The Secret words are marzipan habidashery zeugma."
- password = "the password is swordfish."
- encrypted = glider.keys.encryptSecret(s, password)
- self.assertNotEquals(encrypted, s)
- self.assert_(encrypted.startswith("GKEY1"))
- self.assertEquals(s, glider.keys.decryptSecret(encrypted, password))
- self.assertRaises(glider.BadPassword, glider.keys.decryptSecret,
- encrypted, "password")
- self.assertRaises(glider.UnknownFormat, glider.keys.decryptSecret,
- "foobar", password)
-
- def test_keystore(self):
- passwd = "umfitty noonah"
- fname = tempfile.mktemp()
- ks = glider.keys.KeyStore(fname)
- key1 = glider.keys.RSAKey.generate(512)
- key2 = glider.keys.RSAKey.generate(512)
- ks.addKey(key1)
- ks.addKey(key2)
- ks.save(passwd)
-
- ks2 = glider.keys.KeyStore(fname)
- ks2.load(passwd)
- self.assertEquals(key1.key.n, ks2.getKey(key1.getKeyID()).key.n)
-
-def suite():
- suite = unittest.TestSuite()
-
- suite.addTest(doctest.DocTestSuite(glider.formats))
- suite.addTest(doctest.DocTestSuite(glider.keys))
- suite.addTest(doctest.DocTestSuite(glider.checkJson))
-
- loader = unittest.TestLoader()
- suite.addTest(loader.loadTestsFromModule(glider.tests))
-
- return suite
-
-
-if __name__ == '__main__':
-
- unittest.TextTestRunner(verbosity=1).run(suite())
diff --git a/lib/glider/util.py b/lib/glider/util.py
deleted file mode 100644
index e2fb57a..0000000
--- a/lib/glider/util.py
+++ /dev/null
@@ -1,73 +0,0 @@
-
-import os
-import sys
-import tempfile
-
-import simplejson
-
-import glider.formats
-import glider.keys
-import glider.master_keys
-
-def moveFile(fromLocation, toLocation):
- if sys.platform in ('cygwin', 'win32'):
- # Win32 doesn't let rename replace an existing file.
- try:
- os.unlink(toLocation)
- except OSError:
- pass
- os.rename(fromLocation, toLocation)
-
-
-def replaceFile(fname, contents, textMode=False):
- """overwrite the file in 'fname' atomically with the content of 'contents'
- """
- dir, prefix = os.path.split(fname)
- fd, fname_tmp = tempfile.mkstemp(prefix=prefix, dir=dir, text=textMode)
-
- try:
- os.write(fd, contents)
- finally:
- os.close(fd)
-
- moveFile(fname_tmp, fname)
-
-def userFilename(name):
- try:
- base = os.environ["THANDY_HOME"]
- except KeyError:
- base = "~/.thandy"
- base = os.path.expanduser(base)
- if not os.path.exists(base):
- os.makedirs(base, 0700)
- return os.path.join(base, name)
-
-def getKeylist(keys_fname, checkKeys=True):
- import glider.master_keys
-
- keydb = glider.formats.Keylist()
-
- for key in glider.master_keys.MASTER_KEYS:
- keydb.addKey(key)
-
- user_keys = userFilename("preload_keys")
- if os.path.exists(user_keys):
- #XXXX somewhat roundabout.
- keylist = glider.formats.makeKeylistObj(user_keys)
- keydb.addFromKeylist(keylist, allowMasterKeys=True)
-
- if keys_fname and os.path.exists(keys_fname):
- f = open(keys_fname, 'r')
- try:
- obj = simplejson.load(f)
- finally:
- f.close()
- ss, role, path = glider.formats.checkSignedObj(obj, keydb)
- if role != 'master':
- raise glider.FormatException("%s wasn't a keylist."%keys_fname)
- if checkKeys and not ss.isValid():
- raise glider.FormatException("%s not signed by enough master keys"%
- keys_fname)
- keydb.addFromKeylist(obj['signed'], allowMasterKeys=False)
-
- return keydb