Implement all of signing-side and server-side for updater, and the core loops of client-side. More downloader support is needed, and more polishing. See TODO for details. This no longer matches glider-spec.txt exactly. Notably, it uses json instead of sexp.

git-svn-id: file:///home/or/svnrepo/updater/trunk@17084 55e972cd-5a19-0410-ae62-a4d7a52db4cd

19 files changed, 2297 insertions, 288 deletions

diff --git a/Makefile b/Makefile
index 0d58eb4..e904f9a 100644
--- a/Makefile
+++ b/Makefile
@@ -2,5 +2,5 @@
 export PYTHONPATH=./lib
 
 test:
-	python -m sexp.tests
+	#python -m sexp.tests
 	python -m glider.tests
diff --git a/TODO b/TODO
index 7e49aa6..a2140c9 100644
--- a/TODO
+++ b/TODO
@@ -1,39 +1,54 @@
 
-
 o Write spec
 
 . Write server-side code (python)
-  o S-expression lib
-  . Code to manage data formats
-    . Parse
+  X S-expression lib
+  o Code to manage data formats
+    o Parse
     o Validate
-  - Code to wrangle private keys
+  o Code to wrangle private keys
     o Generate
-    . Store, load (password-protected)
-    - Print for posterity
+    o Store, load (password-protected)
+    o Print for posterity
+    o Associate with roles.
+  o Code to generate timestamp files
+  o Code to generate mirror files
+  o Code to generate keylist files
+  o and add new keys to them
+  o and remove keys.
 
-  - Code to generate timestamp files
-  - Code to generate mirror files
-  - Code to generate keylist files, and add new keys to them, and
-    remove keys.
+o Repository code.
+  o make access functions independent of consistency/loading functions
 
 - Write client-side code
-  - Decide early if a python implementation will do for v1.
-    IF SO:
+  . Decide early if a python implementation will do for v1.
      - Adjust httplib, urllib2 to use socks4a.
        - Check SOCKS package for suitability as basis for socks4a support?
      - Look into best packageing practices
-    IF NOT:
-     - Maybe use curllib for downloading, unless there's something
-       better.
-     - Check out Ron's reference code for s-expression handling.
-
-  - Write cacheing code
-  - Write code to pick a mirror
-  - Write code to grab a timestamp file and figure out what to do.
-  - Write code to update other files
-  - Write code to run, telling another process about status,
+
+  o Write cacheing code
+  o Write code to pick a mirror
+  o Write code to grab a timestamp file and figure out what to do.
+  o Write code to update other files
+  . Write code to run, telling another process about status,
     eventually coming up with a list of packages to install or an
     "A-OK" signal.
 
-  - GUI
+  D GUI
+
+  - DL-via-Tor
+  - Install-when-done
+  - Verbose output
+  - Rendezvous-back with Tor when done.
+
+- Wrapping
+  - More unit tests
+  - Setup.py script to install everything
+
+- Documentation
+  - HOWTO guides
+  . full pydoc
+  - revise spec
+
+- Testing
+  - Much bigger unit tests.
diff --git a/lib/glider/ClientCLI.py b/lib/glider/ClientCLI.py
new file mode 100644
index 0000000..c6f13fd
--- /dev/null
+++ b/lib/glider/ClientCLI.py
@@ -0,0 +1,66 @@
+
+import os
+import sys
+import getopt
+
+import glider.util
+import glider.repository
+import glider.download
+
+def update(args):
+    repoRoot = glider.util.userFilename("cache")
+    options, args = getopt.getopt(args, "", [ "repo=", "no-download" ])
+    download = True
+
+    for o, v in options:
+        if o == '--repo':
+            repoRoot = v
+        elif o == "--no-download":
+            download = False
+
+    repo = glider.repository.LocalRepository(repoRoot)
+
+    files = repo.getFilesToUpdate(trackingBundles=args)
+
+    if not download:
+        return
+
+    mirrorlist = repo.getMirrorlistFile().get()
+
+    downloader = glider.download.Downloads()
+    downloader.start()
+
+    for f in files:
+        # XXXX Use hash.
+        dj = glider.download.DownloadJob(f, repo.getFilename(f),
+                                         mirrorlist)
+        downloader.addDownloadJob(dj)
+        # XXXX replace file in repository if ok; reload; see what changed.
+    
+    # Wait for in-progress jobs
+
+# Check my repository
+
+# Tell me what I need to download
+
+# Download stuff
+
+# Tell me what to install.
+
+def usage():
+    print "Known commands:"
+    print "  update [--repo=repository] [--no-download]"
+    sys.exit(1)
+
+def main():
+    if len(sys.argv) < 2:
+        usage()
+    cmd = sys.argv[1]
+    args = sys.argv[2:]
+    if cmd in [ "update" ]:
+        globals()[cmd](args)
+    else:
+        usage()
+
+if __name__ == '__main__':
+    main()
diff --git a/lib/glider/ServerCLI.py b/lib/glider/ServerCLI.py
new file mode 100644
index 0000000..c564f5b
--- /dev/null
+++ b/lib/glider/ServerCLI.py
@@ -0,0 +1,187 @@
+
+import os
+import sys
+import getopt
+import time
+
+import simplejson
+
+import glider.formats
+import glider.util
+import glider.keys
+
+def tstamp():
+    return time.strftime("%Y%m%d_%H%M%S", time.localtime())
+
+def snarf(fname):
+    f = open(fname, 'rb')
+    try:
+        return f.read()
+    finally:
+        f.close()
+
+def snarfObj(fname):
+    f = open(fname, 'r')
+    try:
+        return simplejson.load(f)
+    finally:
+        f.close()
+
+def insert(args):
+    repo = os.environ.get("THANDY_MASTER_REPO")
+    backupDir = glider.util.userFilename("old_files")
+    checkSigs = True
+
+    options, args = getopt.getopt(args, "", ["repo=", "no-check"])
+    for o,v in options:
+        if o == "--repo":
+            repo = v
+        elif o == "--no-check":
+            checkSigs = False
+
+    if not repo:
+        print "No repository specified."
+        usage()
+    if not os.path.exists(repo):
+        print "No such repository as %r"%repo
+        usage()
+
+    if not os.path.exists(backupDir):
+        os.makedirs(backupDir, 0700)
+
+    if checkSigs:
+        keys = glider.util.getKeylist(os.path.join(repo, "meta/keys.txt"))
+    else:
+        keys = None
+
+    n_ok = 0
+    for fn in args:
+        print "Loading %s..."%fn
+        try:
+            content = snarf(fn)
+        except OSError, e:
+            print "Couldn't open %s: %s"%(fn, e)
+            continue
+
+        try:
+            obj = simplejson.loads(content)
+        except ValueError, e:
+            print "Couldn't decode %s: %s"%(fn, e)
+            continue
+
+        try:
+            ss, r, path = glider.formats.checkSignedObj(obj, keys)
+        except glider.FormatException, e:
+            print "Bad format on %s: %s"%(fn, e)
+            continue
+        if checkSigs and not ss.isValid():
+            print "Not enough valid signatures on %s"%fn
+            continue
+
+        print "  Looks okay.  It goes in %s"%path
+        assert path.startswith("/")
+        targetPath = os.path.join(repo, path[1:])
+        if os.path.exists(targetPath):
+            oldContents = snarf(targetPath)
+            if oldContents == content:
+                print "  File unchanged!"
+                n_ok += 1
+                continue
+
+            baseFname = "%s_%s" % (tstamp(), os.path.split(path)[1])
+            backupFname = os.path.join(backupDir, baseFname)
+            print "  Copying old file to %s"%backupFname
+            glider.util.replaceFile(backupFname, oldContents)
+
+        parentDir = os.path.split(targetPath)[0]
+        if not os.path.exists(parentDir):
+            print "  Making %s"%parentDir
+            os.makedirs(parentDir, 0755)
+        print "  Replacing file..."
+        glider.util.replaceFile(targetPath, content)
+        print "  Done."
+        n_ok += 1
+    if n_ok != len(args):
+        sys.exit(1)
+
+def timestamp(args):
+    repo = os.environ.get("THANDY_MASTER_REPO")
+    ts_keyfile = glider.util.userFilename("timestamp_key")
+
+    options, args = getopt.getopt(args, "", ["repo=", "ts-key="])
+    for o,v in options:
+        if o == "--repo":
+            repo = v
+        elif o == "--ts-key":
+            ts_keyfile = v
+
+    if repo == None:
+        print "No repository specified."
+        usage()
+    if not os.path.exists(repo):
+        print "No such repository as %r"%repo
+        usage()
+
+    tsFname = os.path.join(repo, "meta/timestamp.txt")
+
+    try:
+        mObj = snarfObj(os.path.join(repo, "meta/mirrors.txt"))
+    except OSError:
+        print "No mirror list!"
+        sys.exit(1)
+    try:
+        kObj = snarfObj(os.path.join(repo, "meta/keys.txt"))
+    except OSError:
+        print "No key list!"
+        sys.exit(1)
+
+    bundles = []
+    for dirpath, dirname, fns in os.walk(os.path.join(repo, "bundleinfo")):
+        for fn in fns:
+            try:
+                bObj = snarfObj(fn)
+            except (ValueError, OSError), e:
+                print "(Couldn't read bundle-like %s)"%fn
+                continue
+            try:
+                _, r, _ = glider.formats.checkSignedObj(bObj)
+            except glider.FormatException, e:
+                print "Problem reading object from %s"%fn
+                continue
+            if r != "bundle":
+                print "%s was not a good bundle"%fn
+                continue
+            bundles.append(bObj['signed'])
+
+    timestamp = glider.formats.makeTimestampObj(
+        mObj['signed'], kObj['signed'], bundles)
+    signable = glider.formats.makeSignable(timestamp)
+
+    keydb = glider.formats.Keylist()
+    #XXXX Still a roundabout way to do this.
+    keylist = glider.formats.makeKeylistObj(ts_keyfile, True)
+    keydb.addFromKeylist(keylist)
+    for k in keydb.iterkeys():
+        glider.formats.sign(signable, k)
+
+    content = simplejson.dumps(signable, sort_keys=True)
+    glider.util.replaceFile(tsFname, content)
+
+def usage():
+    print "Known commands:"
+    print "  insert [--no-check] [--repo=repository] file ..."
+    print "  timestamp [--repo=repository]"
+    sys.exit(1)
+
+def main():
+    if len(sys.argv) < 2:
+        usage()
+    cmd = sys.argv[1]
+    args = sys.argv[2:]
+    if cmd in [ "insert", "timestamp" ]:
+        globals()[cmd](args)
+    else:
+        usage()
+
+if __name__ == '__main__':
+    main()
diff --git a/lib/glider/SignerCLI.py b/lib/glider/SignerCLI.py
new file mode 100644
index 0000000..bf5b73a
--- /dev/null
+++ b/lib/glider/SignerCLI.py
@@ -0,0 +1,313 @@
+
+import os
+import getopt
+import sys
+import logging
+import simplejson
+
+import glider.keys
+import glider.formats
+
+def getKeyStore():
+    return glider.keys.KeyStore(glider.util.userFilename("secret_keys"))
+
+def dumpKey(key, indent=0):
+    i = " "*indent
+    print "%s%s"%(i, key.getKeyID())
+    for r, p in key.getRoles():
+        print "  %s%s\t%s"%(i, r, p)
+
+def getKey(ks, keyid=None, role=None, path=None):
+    if keyid is not None:
+        keys = ks.getKeysFuzzy(keyid)
+        if None not in (role, path):
+            keys = [ k for k in keys if k.hasRole(role, path) ]
+    elif None not in (role, path):
+        keys = ks.getKeysByRole(role, path)
+    else:
+        assert False
+    if len(keys) < 1:
+        print "No such key.\nI wanted",
+        if keyid: print "keyid='%s...'"%keyid,
+        if None not in (role, path): print "role=%s, path=%s"%(role,path),
+        print
+        print "I only know about:"
+        for k in ks.iterkeys():
+            dumpKey(k)
+        sys.exit(1)
+    elif len(keys) > 1:
+        print "Multiple keys match.  Possibilities are:"
+        for k in keys:
+            dumpKey(k)
+        sys.exit(1)
+    else:
+        return keys[0]
+
+# ------------------------------
+
+def makepackage(args):
+    options, args = getopt.getopt(args, "", "keyid=")
+    keyid = None
+    for o,v in options:
+        if o == "--keyid":
+            keyid = v
+
+    if len(args) < 2:
+        usage()
+
+    configFile = args[0]
+    dataFile = args[1]
+    print "Generating package."
+    package = glider.formats.makePackageObj(configFile, dataFile)
+    relpath = package['location']
+    print "need a key with role matching [package %s]"%relpath
+    ks = getKeyStore()
+    ks.load()
+    key = getKey(ks, keyid=keyid, role='package', path=relpath)
+    signable = glider.formats.makeSignable(package)
+    glider.formats.sign(signable, key)
+
+    if 1:
+        ss, r, p = glider.formats.checkSignedObj(signable, ks)
+        assert ss.isValid()
+
+    location = os.path.split(package['location'])[-1]
+    print "Writing signed package to %s"%location
+    f = open(location, 'w')
+    simplejson.dump(signable, f, indent=1)
+    f.close()
+
+def makebundle(args):
+    options, args = getopt.getopt(args, "", "keyid=")
+    keyid = None
+    for o,v in options:
+        if o == "--keyid":
+            keyid = v
+
+    if len(args) < 2:
+        usage()
+
+    configFile = args[0]
+    packages = {}
+    for pkgFile in args[1:]:
+        print "Loading", pkgFile
+        f = open(pkgFile, 'r')
+        p = simplejson.load(f)
+        f.close()
+        _, r, _ = glider.formats.checkSignedObj(p)
+        if r != 'package':
+            print pkgFile, "was not a package"
+        packages[p['signed']['location']] = p
+
+    def getHash(path):
+        p = packages[path]
+        return glider.formats.getDigest(p['signed'])
+
+    bundleObj = glider.formats.makeBundleObj(configFile, getHash)
+    signable = glider.formats.makeSignable(bundleObj)
+
+    ks = getKeyStore()
+    ks.load()
+    key = getKey(ks, keyid=keyid, role="bundle", path=bundleObj['location'])
+    glider.formats.sign(signable, key)
+
+    if 1:
+        ss, r, p = glider.formats.checkSignedObj(signable, ks)
+        assert ss.isValid()
+
+    location = os.path.split(bundleObj['location'])[-1]
+    print "Writing signed bundle to %s"%location
+    f = open(location, 'w')
+    simplejson.dump(signable, f, indent=1)
+    f.close()
+
+# ------------------------------
+def makekeylist(args):
+    options, args = getopt.getopt(args, "", "keyid=")
+    keyid = None
+    for o,v in options:
+        if o == "--keyid":
+            keyid = v
+
+    if len(args) < 1:
+        usage()
+
+    keylist = glider.formats.makeKeylistObj(args[0])
+    signable = glider.formats.makeSignable(keylist)
+
+    ks = getKeyStore()
+    ks.load()
+    key = getKey(ks, keyid=keyid, role="master", path="/meta/keys.txt")
+    glider.formats.sign(signable, key)
+
+    if 1:
+        ss, r, p = glider.formats.checkSignedObj(signable, ks)
+        assert ss.isValid()
+
+    print "writing signed keylist to keys.txt"
+    glider.util.replaceFile("keys.txt",
+              simplejson.dumps(signable, indent=1, sort_keys=True),
+              textMode=True)
+
+def signkeylist(args):
+    if len(args) != 1:
+        usage()
+
+    keylist = simplejson.load(open(args[0], 'r'))
+    glider.formats.SIGNED_SCHEMA.checkMatch(keylist)
+    glider.formats.KEYLIST_SCHEMA.checkMatch(keylist['signed'])
+
+    ks = getKeyStore()
+    ks.load()
+    keys = ks.getKeysByRole("master", "/meta/keys.txt")
+    for k in keys:
+        glider.formats.sign(keylist, k)
+
+    print "writing signed keylist to keys.txt"
+    glider.util.replaceFile("keys.txt",
+              simplejson.dumps(keylist, indent=1, sort_keys=True),
+              textMode=True)
+
+def makemirrorlist(args):
+    options, args = getopt.getopt(args, "", "keyid=")
+    keyid = None
+    for o,v in options:
+        if o == "--keyid":
+            keyid = v
+
+    if len(args) < 1:
+        usage()
+
+    mirrorlist = glider.formats.makeMirrorListObj(args[0])
+    signable = glider.formats.makeSignable(mirrorlist)
+
+    ks = getKeyStore()
+    ks.load()
+    key = getKey(ks, keyid=keyid, role='mirrors', path="/meta/mirrors.txt")
+    glider.formats.sign(signable, key)
+
+    if 1:
+        ss, r, p = glider.formats.checkSignedObj(signable, ks)
+        assert ss.isValid()
+
+    print "writing signed mirrorlist to mirrors.txt"
+    glider.util.replaceFile("mirrors.txt",
+              simplejson.dumps(signable, indent=1, sort_keys=True),
+              textMode=True)
+
+# ------------------------------
+
+def keygen(args):
+    k = getKeyStore()
+    k.load()
+    print "Generating key. This will be slow."
+    key = glider.keys.RSAKey.generate()
+    print "Generated new key: %s" % key.getKeyID()
+    k.addKey(key)
+    k.save()
+
+def listkeys(args):
+    k = getKeyStore()
+    k.load()
+    for k in k.iterkeys():
+        print k.getKeyID()
+        for r, p in k.getRoles():
+            print " ", r, p
+
+def addrole(args):
+    if len(args) < 3:
+        usage()
+    ks = getKeyStore()
+    ks.load()
+    k = getKey(ks, args[0])
+    r = args[1]
+    if r not in glider.formats.ALL_ROLES:
+        print "Unrecognized role %r.  Known roles are %s"%(
+            r,", ".join(glider.format.ALL_ROLES))
+        sys.exit(1)
+    p = args[2]
+    k.addRole(r, p)
+    ks.save()
+
+def delrole(args):
+    if len(args) < 3:
+        usage()
+    ks = getKeyStore()
+    ks.load()
+    k = getKey(ks, args[0])
+    r = args[1]
+    if r not in glider.formats.ALL_ROLES:
+        print "Unrecognized role %r.  Known roles are %s"%(
+            r,", ".join(glider.format.ALL_ROLES))
+        sys.exit(1)
+    p = args[2]
+
+    #XXXX rep.
+    origLen = len(k._roles)
+    k._roles = [ (role,path) for role,path in k._roles
+                 if (role,path) != (r,p) ]
+    removed = origLen - len(k._roles)
+    print removed, "roles removed"
+    if removed:
+        ks.save()
+
+def chpass(args):
+    ks = getKeyStore()
+    print "Old password."
+    ks.load()
+    print "New password."
+    ks.clearPassword()
+    ks.save()
+
+def dumpkey(args):
+    options, args = getopt.getopt(args, "", ["include-secret", "passwd="])
+
+    includeSecret = False
+    for o,v in options:
+        if o == '--include-secret':
+            includeSecret = True
+        else:
+            print "Unexpected %r"%o
+
+    ks = getKeyStore()
+    ks.load()
+
+    keys = []
+    if len(args):
+        keys = [ getKey(ks, a) for a in args ]
+    else:
+        keys = list(ks.iterkeys())
+
+    for k in keys:
+        data = k.format(private=includeSecret, includeRoles=True)
+        print "Key(", simplejson.dumps(data, indent=2), ")"
+
+def usage():
+    print "Known commands:"
+    print "  keygen"
+    print "  listkeys"
+    print "  chpass"
+    print "  addrole keyid role path"
+    print "  delrole keyid role path"
+    print "  dumpkey [--include-secret] keyid"
+    print "  makepackage config datafile"
+    print "  makebundle config packagefile ..."
+    print "  signkeylist keylist"
+    print "  makekeylist keylist"
+    print "  makemirrorlist config"
+    sys.exit(1)
+
+def main():
+    if len(sys.argv) < 2:
+        usage()
+    cmd = sys.argv[1]
+    args = sys.argv[2:]
+    if cmd in [ "keygen", "listkeys", "addrole", "delrole", "chpass",
+                "dumpkey", "makepackage", "makebundle", "signkeylist",
+                "makekeylist", "signkeylist", "makemirrorlist", ]:
+        globals()[cmd](args)
+    else:
+        usage()
+
+if __name__ == '__main__':
+    main()
diff --git a/lib/glider/__init__.py b/lib/glider/__init__.py
index e05bafe..87fd983 100644
--- a/lib/glider/__init__.py
+++ b/lib/glider/__init__.py
@@ -1,3 +1,35 @@
 
 __all__ = [ 'formats' ]
 
+_BaseException = Exception
+
+class Exception(_BaseException):
+    pass
+
+class FormatException(Exception):
+    pass
+
+class UnknownFormat(FormatException):
+    pass
+
+class BadSignature(Exception):
+    pass
+
+class BadPassword(Exception):
+    pass
+
+class InternalError(Exception):
+    pass
+
+class RepoError(InternalError):
+    pass
+
+class CryptoError(Exception):
+    pass
+
+class PubkeyFormatException(FormatException):
+    pass
+
+class UnknownMethod(CryptoError):
+    pass
+
diff --git a/lib/glider/checkJson.py b/lib/glider/checkJson.py
new file mode 100644
index 0000000..0c5eed6
--- /dev/null
+++ b/lib/glider/checkJson.py
@@ -0,0 +1,274 @@
+
+import re
+import sys
+
+import glider
+
+class Schema:
+    def matches(self, obj):
+        try:
+            self.checkMatch(obj)
+        except glider.FormatException:
+            return False
+        else:
+            return True
+
+    def checkMatch(self, obj):
+        raise NotImplemented()
+
+class Any(Schema):
+    """
+       >>> s = Any()
+       >>> s.matches("A String")
+       True
+       >>> s.matches([1, "list"])
+       True
+    """
+    def checkMatch(self, obj):
+        pass
+
+class RE(Schema):
+    """
+       >>> s = RE("h.*d")
+       >>> s.matches("hello world")
+       True
+       >>> s.matches("Hello World")
+       False
+       >>> s.matches("hello world!")
+       False
+       >>> s.matches([33, "Hello"])
+       False
+    """
+    def __init__(self, pat=None, modifiers=0, reObj=None, reName="pattern"):
+        if not reObj:
+            if not pat.endswith("$"):
+                pat += "$"
+            reObj = re.compile(pat, modifiers)
+        self._re = reObj
+        self._reName = reName
+    def checkMatch(self, obj):
+        if not isinstance(obj, basestring) or not self._re.match(obj):
+            raise glider.FormatException("%r did not match %s"
+                                         %(obj,self._reName))
+
+class Str(Schema):
+    """
+       >>> s = Str("Hi")
+       >>> s.matches("Hi")
+       True
+       >>> s.matches("Not hi")
+       False
+    """
+    def __init__(self, val):
+        self._str = val
+    def checkMatch(self, obj):
+        if self._str != obj:
+            raise glider.FormatException("Expected %r; got %r"%(self._str, obj))
+
+class AnyStr(Schema):
+    """
+       >>> s = AnyStr()
+       >>> s.matches("")
+       True
+       >>> s.matches("a string")
+       True
+       >>> s.matches(["a"])
+       False
+       >>> s.matches(3)
+       False
+       >>> s.matches(u"a unicode string")
+       True
+       >>> s.matches({})
+       False
+    """
+    def __init__(self):
+        pass
+    def checkMatch(self, obj):
+        if not isinstance(obj, basestring):
+            raise glider.FormatException("Expected a string; got %r"%obj)
+
+class ListOf(Schema):
+    """
+       >>> s = ListOf(RE("(?:..)*"))
+       >>> s.matches("hi")
+       False
+       >>> s.matches([])
+       True
+       >>> s.matches({})
+       False
+       >>> s.matches(["Hi", "this", "list", "is", "full", "of", "even", "strs"])
+       True
+       >>> s.matches(["This", "one", "is not"])
+       False
+    """
+    def __init__(self, schema, minCount=0, maxCount=sys.maxint,listName="list"):
+        self._schema = schema
+        self._minCount = minCount
+        self._maxCount = maxCount
+        self._listName = listName
+    def checkMatch(self, obj):
+        if not isinstance(obj, (list, tuple)):
+            raise glider.FormatException("Expected %s; got %r"
+                                         %(self._listName,obj))
+        for item in obj:
+            try:
+                self._schema.checkMatch(item)
+            except glider.FormatException, e:
+                raise glider.FormatException("%s in %s"%(e, self._listName))
+
+        if not (self._minCount <= len(obj) <= self._maxCount):
+            raise glider.FormatException("Length of %s out of range"
+                                         %self._listName)
+
+class Struct(Schema):
+    """
+       >>> s = Struct([ListOf(AnyStr()), AnyStr(), Str("X")])
+       >>> s.matches(False)
+       False
+       >>> s.matches("Foo")
+       False
+       >>> s.matches([[], "Q", "X"])
+       True
+       >>> s.matches([[], "Q", "D"])
+       False
+       >>> s.matches([[3], "Q", "X"])
+       False
+       >>> s.matches([[], "Q", "X", "Y"])
+       False
+    """
+    def __init__(self, subschemas, allowMore=False, structName="list"):
+        self._subschemas = subschemas[:]
+        self._allowMore = allowMore
+        self._structName = structName
+    def checkMatch(self, obj):
+        if not isinstance(obj, (list, tuple)):
+            raise glider.FormatException("Expected %s; got %r"
+                                         %(self._structName,obj))
+        elif len(obj) < len(self._subschemas):
+            raise glider.FormatException(
+                "Too few fields in %s"%self._structName)
+        elif len(obj) > len(self._subschemas) and not self._allowMore:
+            raise glider.FormatException(
+                "Too many fields in %s"%self._structName)
+        for item, schema in zip(obj, self._subschemas):
+            schema.checkMatch(item)
+
+class DictOf(Schema):
+    """
+       >>> s = DictOf(RE(r'[aeiou]+'), Struct([AnyStr(), AnyStr()]))
+       >>> s.matches("")
+       False
+       >>> s.matches({})
+       True
+       >>> s.matches({"a": ["x", "y"], "e" : ["", ""]})
+       True
+       >>> s.matches({"a": ["x", 3], "e" : ["", ""]})
+       False
+       >>> s.matches({"a": ["x", "y"], "e" : ["", ""], "d" : ["a", "b"]})
+       False
+    """
+    def __init__(self, keySchema, valSchema):
+        self._keySchema = keySchema
+        self._valSchema = valSchema
+    def checkMatch(self, obj):
+        try:
+            iter = obj.iteritems()
+        except AttributeError:
+            raise glider.FormatException("Expected a dict; got %r"%obj)
+
+        for k,v in iter:
+            self._keySchema.checkMatch(k)
+            self._valSchema.checkMatch(v)
+
+class Opt:
+    """Helper; applied to a value in Obj to mark it optional.
+
+       >>> s = Obj(k1=Str("X"), k2=Opt(Str("Y")))
+       >>> s.matches({'k1': "X", 'k2': "Y"})
+       True
+       >>> s.matches({'k1': "X", 'k2': "Z"})
+       False
+       >>> s.matches({'k1': "X"})
+       True
+    """
+    def __init__(self, schema):
+        self._schema = schema
+    def checkMatch(self, obj):
+        self._schema.checkMatch(obj)
+
+class Obj(Schema):
+    """
+       >>> s = Obj(a=AnyStr(), bc=Struct([Int(), Int()]))
+       >>> s.matches({'a':"ZYYY", 'bc':[5,9]})
+       True
+       >>> s.matches({'a':"ZYYY", 'bc':[5,9], 'xx':5})
+       True
+       >>> s.matches({'a':"ZYYY", 'bc':[5,9,3]})
+       False
+       >>> s.matches({'a':"ZYYY"})
+       False
+
+    """
+    def __init__(self, _objname="object", **d):
+        self._objname = _objname
+        self._required = d.items()
+
+
+    def checkMatch(self, obj):
+        for k,schema in self._required:
+            try:
+                item = obj[k]
+            except KeyError:
+                if not isinstance(schema, Opt):
+                    raise glider.FormatException("Missing key %s in %s"
+                                                 %(k,self._objname))
+
+            else:
+                try:
+                    schema.checkMatch(item)
+                except glider.FormatException, e:
+                    raise glider.FormatException("%s in %s.%s"
+                                                 %(e,self._objname,k))
+
+
+class Int(Schema):
+    """
+       >>> s = Int()
+       >>> s.matches(99)
+       True
+       >>> s.matches(False)
+       False
+       >>> s.matches(0L)
+       True
+       >>> s.matches("a string")
+       False
+       >>> Int(lo=10, hi=30).matches(25)
+       True
+       >>> Int(lo=10, hi=30).matches(5)
+       False
+    """
+    def __init__(self, lo=-sys.maxint, hi=sys.maxint):
+        self._lo = lo
+        self._hi = hi
+    def checkMatch(self, obj):
+        if isinstance(obj, bool) or not isinstance(obj, (int, long)):
+            # We need to check for bool as a special case, since bool
+            # is for historical reasons a subtype of int.
+            raise glider.FormatException("Got %r instead of an integer"%obj)
+        elif not (self._lo <= obj <= self._hi):
+            raise glider.FormatException("%r not in range [%r,%r]"
+                                         %(obj, self._lo, self._hi))
+
+class Bool(Schema):
+    """
+       >>> s = Bool()
+       >>> s.matches(True) and s.matches(False)
+       True
+       >>> s.matches(11)
+       False
+    """
+    def __init__(self):
+        pass
+    def checkMatch(self, obj):
+        if not isinstance(obj, bool):
+            raise glider.FormatException("Got %r instead of a boolean"%obj)
diff --git a/lib/glider/download.py b/lib/glider/download.py
new file mode 100644
index 0000000..3135e39
--- /dev/null
+++ b/lib/glider/download.py
@@ -0,0 +1,127 @@
+
+
+import urllib2
+import httplib
+import random
+
+import threading, Queue
+
+import glider.util
+
+class Downloads:
+    def __init__(self, n_threads=2):
+        self._lock = threading.RLock()
+        self.downloads = {}
+        self.haveDownloaded = {}
+        self.downloadQueue = Queue.Queue()
+        self.threads = [ threading.Thread(target=self._thread) ]
+        for t in self.threads:
+            t.setDaemon(True)
+
+    def start(self):
+        for t in self.threads:
+            t.start()
+
+    def isCurrentlyDownloading(self, relPath):
+        self._lock.acquire()
+        try:
+            return self.downloads.has_key(relPath)
+        finally:
+            self._lock.release()
+
+    def isRedundant(self, relPath):
+        self._lock.acquire()
+        try:
+            return (self.downloads.has_key(relPath) or
+                    self.haveDownloaded.has_key(relPath))
+        finally:
+            self._lock.release()
+
+    def addDownloadJob(self, job):
+        rp = job.getRelativePath()
+        self._lock.acquire()
+        self.downloads[rp] = job
+        self._lock.release()
+        self.downloadQueue.put(job)
+
+    def _thread(self):
+        while True:
+            job = self.downloadQueue.get()
+            job.download()
+            rp = job.getRelativePath()
+            self._lock.acquire()
+            try:
+                del self.downloads[rp]
+                self.haveDownloaded[rp] = True
+            finally:
+                self._lock.release()
+
+class DownloadJob:
+    def __init__(self, relPath, destPath, mirrorlist=None,
+                 wantHash=None, canStall=False):
+        self._relPath = relPath
+        self._wantHash = wantHash
+        self._mirrorList = mirrorlist
+        self._destPath = destPath
+
+        tmppath = glider.util.userFilename("tmp")
+        if relPath.startswith("/"):
+            relPath = relPath[1:]
+        self._tmppath = os.path.join(tmppath, relPath)
+
+        d = os.path.dirname(self._tmppath)
+        if not os.path.exists(d):
+            os.makedirs(d, 0700)
+
+    def getRelativePath(self):
+        return self._relPath
+
+    def haveStalledFile(self):
+        return os.path.exists(self._tmppath)
+
+    def getURL(self, mirrorlist=None):
+        if mirrorlist is None:
+            mirrorlist = self._mirrorList
+        weightSoFar = 0
+        usable = []
+
+        for m in mirrorlist['mirrors']:
+            for c in m['contents']:
+                # CHECK FOR URL SUITABILITY XXXXX
+
+                if glider.formats.rolePathMatches(c, self._relPath):
+                    weightSoFar += m['weight']
+                    usable.append( (weightSoFar, m) )
+                    break
+
+        wTarget = random.randint(0, weightSoFar)
+        mirror = None
+        # Could use bisect here instead
+        for w, m in mirrorlist:
+            if w >= wTarget:
+                mirror = m
+                break
+
+        return m['urlbase'] + self._relPath
+
+    def download(self):
+        # XXXX RESUME
+
+        f_in = urllib2.urlopen(self.getURL())
+        f_out = open(self._tmpPath, 'w')
+        while True:
+            c = f_in.read(1024)
+            if not c:
+                break
+            f_out.write(c)
+        f_in.close()
+        f_out.close()
+        # XXXXX retry on failure
+
+        if self._wantHash:
+            gotHash = glider.formats.getFileDigest(self._tmpPath)
+            if gotHash != self._wantHash:
+                # XXXX Corrupt file.
+                pass
+
+        glider.utils.moveFile(self._tmpPath, self._destPath)
diff --git a/lib/glider/formats.py b/lib/glider/formats.py
index 34ece7d..9846c50 100644
--- a/lib/glider/formats.py
+++ b/lib/glider/formats.py
@@ -1,28 +1,62 @@
 
-import sexp.access
-import sexp.encode
+import simplejson
 import time
 import re
+import binascii
+import calendar
 
-class FormatException(Exception):
-    pass
+import glider.checkJson
+
+import Crypto.Hash.SHA256
 
 class KeyDB:
+    """A KeyDB holds public keys, indexed by their key IDs."""
     def __init__(self):
-        self.keys = {}
+        self._keys = {}
     def addKey(self, k):
-        self.keys[k.getKeyID()] = k
+        keyid = k.getKeyID()
+        try:
+            oldkey = self._keys[keyid]
+            for r, p in oldkey.getRoles():
+                if (r, p) not in k.getRoles():
+                    k.addRole(r,p)
+        except KeyError:
+            pass
+        self._keys[k.getKeyID()] = k
     def getKey(self, keyid):
-        return self.keys[keyid]
+        return self._keys[keyid]
+    def getKeysByRole(self, role, path):
+        results = []
+        for key in self._keys.itervalues():
+            for r,p in key.getRoles():
+                if r == role:
+                    if rolePathMatches(p, path):
+                        results.append(key)
+        return results
+
+    def getKeysFuzzy(self, keyid):
+        r = []
+        for k,v in self._keys.iteritems():
+            if k.startswith(keyid):
+                r.append(v)
+        return r
+    def iterkeys(self):
+        return self._keys.itervalues()
 
 _rolePathCache = {}
 def rolePathMatches(rolePath, path):
-    """
+    """Return true iff the relative path in the filesystem 'path' conforms
+       to the pattern 'rolePath': a path that a given key is
+       authorized to sign.  Patterns are allowed to contain * to
+       represent one or more characters in a filename, and ** to
+       represent any level of directory structure.
 
     >>> rolePathMatches("a/b/c/", "a/b/c/")
     True
     >>> rolePathMatches("**/c.*", "a/b/c.txt")
     True
+    >>> rolePathMatches("**/c.*", "a/b/ctxt")
+    False
     >>> rolePathMatches("**/c.*", "a/b/c.txt/foo")
     False
     >>> rolePathMatches("a/*/c", "a/b/c")
@@ -35,40 +69,74 @@ def rolePathMatches(rolePath, path):
     try:
         regex = _rolePathCache[rolePath]
     except KeyError:
+        orig = rolePath
+        # remove duplicate slashes.
         rolePath = re.sub(r'/+', '/', rolePath)
+        # escape, then ** becomes .*
         rolePath = re.escape(rolePath).replace(r'\*\*', r'.*')
+        # * becomes [^/]*
         rolePath = rolePath.replace(r'\*', r'[^/]*')
+        # and no extra text is allowed.
         rolePath += "$"
-        regex = _rolePathCache[rolePath] = re.compile(rolePath)
+        regex = _rolePathCache[orig] = re.compile(rolePath)
     return regex.match(path) != None
 
-def checkSignatures(signed, keyDB, role, path):
+class SignatureStatus:
+    """Represents the outcome of checking signature(s) on an object."""
+    def __init__(self, good, bad, unrecognized, unauthorized):
+        # keyids for all the valid signatures
+        self._good = good[:]
+        # keyids for the invalid signatures (we had the key, and it failed).
+        self._bad = bad[:]
+        # keyids for signatures where we didn't recognize the key
+        self._unrecognized = unrecognized[:]
+        # keyids for signatures where we recognized the key, but it doesn't
+        # seem to be allowed to sign this kind of document.
+        self._unauthorized = unauthorized[:]
+
+    def isValid(self, threshold=1):
+        """Return true iff we got at least 'threshold' good signatures."""
+        return len(self._good) >= threshold
+
+    def mayNeedNewKeys(self):
+        """Return true iff downloading a new set of keys might tip this
+           signature status over to 'valid.'"""
+        return len(self._unrecognized) or len(self._unauthorized)
+
+def checkSignatures(signed, keyDB, role=None, path=None):
+    """Given an object conformant to SIGNED_SCHEMA and a set of public keys
+       in keyDB, verify the signed object in 'signed'."""
+
+    SIGNED_SCHEMA.checkMatch(signed)
+
     goodSigs = []
     badSigs = []
     unknownSigs = []
     tangentialSigs = []
 
-    assert signed[0] == "signed"
-    data = signed[1]
+    signable = signed['signed']
+    signatures = signed['signatures']
 
     d_obj = Crypto.Hash.SHA256.new()
-    sexp.encode.hash_canonical(data, d_obj)
+    getDigest(signable, d_obj)
     digest = d_obj.digest()
 
-    for signature in sexp.access.s_children(signed, "signature"):
-        attrs = signature[1]
-        sig = attrs[2]
-        keyid = s_child(attrs, "keyid")[1]
+    for signature in signatures:
+        sig = signature['sig']
+        keyid = signature['keyid']
+        method = signature['method']
+
         try:
             key = keyDB.getKey(keyid)
         except KeyError:
             unknownSigs.append(keyid)
             continue
-        method = s_child(attrs, "method")[1]
+
         try:
             result = key.checkSignature(method, sig, digest=digest)
-        except UnknownMethod:
+        except glider.UnknownMethod:
             continue
+
         if result == True:
             if role is not None:
                 for r,p in key.getRoles():
@@ -82,132 +150,264 @@ def checkSignatures(signed, keyDB, role, path):
         else:
             badSigs.append(keyid)
 
-    return goodSigs, badSigs, unknownSigs, tangentialSigs
+    return SignatureStatus(goodSigs, badSigs, unknownSigs, tangentialSigs)
+
+def encodeCanonical(obj, outf=None):
+    """Encode the object obj in canoncial JSon form, as specified at
+       http://wiki.laptop.org/go/Canonical_JSON .  It's a restricted
+       dialect of json in which keys are always lexically sorted,
+       there is no whitespace, floats aren't allowed, and only quote
+       and backslash get escaped.  The result is encoded in UTF-8,
+       and the resulting bits are passed to outf (if provided), or joined
+       into a string and returned.
+
+       >>> encodeCanonical("")
+       '""'
+       >>> encodeCanonical([1, 2, 3])
+       '[1,2,3]'
+       >>> encodeCanonical({"x" : 3, "y" : 2})
+       '{"x":3,"y":2}'
+    """
+    def default(o):
+        raise TypeError("Can't encode %r", o)
+    def floatstr(o):
+        raise TypeError("Floats not allowed.")
+    def canonical_str_encoder(s):
+        return '"%s"' % re.sub(r'(["\\])', r'\\\1', s)
+
+    # XXX This is, alas, a hack.  I'll submit a canonical JSon patch to
+    # the simplejson folks.
+
+    iterator = simplejson.encoder._make_iterencode(
+        None, default, canonical_str_encoder, None, floatstr,
+        ":", ",", True, False, True)(obj, 0)
+
+    result = None
+    if outf == None:
+        result = [ ]
+        outf = result.append
+
+    for u in iterator:
+        outf(u.encode("utf-8"))
+    if result is not None:
+        return "".join(result)
+
+def getDigest(obj, digestObj=None):
+    """Update 'digestObj' (typically a SHA256 object) with the digest of
+       the canonical json encoding of obj.  If digestObj is none,
+       compute the SHA256 hash and return it.
+
+       DOCDOC string equivalence.
+    """
+    useTempDigestObj = (digestObj == None)
+    if useTempDigestObj:
+        digestObj = Crypto.Hash.SHA256.new()
+
+    if isinstance(obj, str):
+        digestObj.update(obj)
+    elif isinstance(obj, unicode):
+        digestObj.update(obj.encode("utf-8"))
+    else:
+        encodeCanonical(obj, digestObj.update)
+
+    if useTempDigestObj:
+        return digestObj.digest()
+
+def getFileDigest(f, digestObj=None):
+    """Update 'digestObj' (typically a SHA256 object) with the digest of
+       the file object in f.  If digestObj is none, compute the SHA256
+       hash and return it.
+
+       >>> s = "here is a long string"*1000
+       >>> import cStringIO, Crypto.Hash.SHA256
+       >>> h1 = Crypto.Hash.SHA256.new()
+       >>> h2 = Crypto.Hash.SHA256.new()
+       >>> getFileDigest(cStringIO.StringIO(s), h1)
+       >>> h2.update(s)
+       >>> h1.digest() == h2.digest()
+       True
+    """
+    useTempDigestObj = (digestObj == None)
+    if useTempDigestObj:
+        digestObj = Crypto.Hash.SHA256.new()
 
-def sign(signed, key):
-    assert sexp.access.s_tag(signed) == 'signed'
-    s = signed[1]
-    keyid = key.keyID()
+    while 1:
+        s = f.read(4096)
+        if not s:
+            break
+        digestObj.update(s)
 
-    oldsignatures = [ s for s in signed[2:] if s_child(s[1], "keyid") != keyid ]
-    signed[2:] = oldsignatures
+    if useTempDigestObj:
+        return digestObj.digest()
 
-    for method, sig in key.sign(s):
-        signed.append(['signature', [['keyid', keyid], ['method', method]],
-                       sig])
+def makeSignable(obj):
+    return { 'signed' : obj, 'signatures' : [] }
 
-def formatTime(t):
-    """
-    >>> formatTime(1221265172)
-    '2008-09-13 00:19:32'
+def sign(signed, key):
+    """Add an element to the signatures of 'signed', containing a new signature
+       of the "signed" part.
     """
-    return time.strftime("%Y-%m-%d %H:%M:%S", time.gmtime(t))
-
-def parseTime(s):
-    return time.timegm(time.strptime(s, "%Y-%m-%d %H:%M:%S"))
 
-def _parseSchema(s, t=None):
-    sexpr = sexp.parse.parse(s)
-    schema = sexp.access.parseSchema(sexpr, t)
-    return schema
+    SIGNED_SCHEMA.checkMatch(signed)
 
-SCHEMA_TABLE = { }
+    signable = signed["signed"]
+    signatures = signed['signatures']
 
-PUBKEY_TEMPLATE = r"""
-  (=pubkey ((:unordered (=type .) (:anyof (. _)))) _)
-"""
+    keyid = key.getKeyID()
 
-SCHEMA_TABLE['PUBKEY'] = _parseSchema(PUBKEY_TEMPLATE)
+    signatures = [ s for s in signatures if s['keyid'] != keyid ]
 
-TIME_TEMPLATE = r"""/\{d}4-\d{2}-\d{2} \d{2}:\d{2}:\d{2}/"""
+    method, sig = key.sign(signable)
+    signatures.append({ 'keyid' : keyid,
+                        'method' : method,
+                        'sig' : sig })
+    signed['signatures'] = signatures
 
-SCHEMA_TABLE['TIME'] = sexp.access.parseSchema(TIME_TEMPLATE)
-
-ATTRS_TEMPLATE = r"""(:anyof (_ *))"""
+def formatTime(t):
+    """Encode the time 't' in YYYY-MM-DD HH:MM:SS format.
 
-SCHEMA_TABLE['ATTRS'] = _parseSchema(ATTRS_TEMPLATE)
+    >>> formatTime(1221265172)
+    '2008-09-13 00:19:32'
+    """
+    return time.strftime("%Y-%m-%d %H:%M:%S", time.gmtime(t))
 
-SIGNED_TEMPLATE = r"""
- (=signed
-   _
-   (:someof
-     (=signature ((:unordered
-                    (=keyid _) (=method _) .ATTRS)) _)
-   )
- )"""
-
-SIGNED_SCHEMA = _parseSchema(SIGNED_TEMPLATE, SCHEMA_TABLE)
-
-KEYLIST_TEMPLATE = r"""
- (=keylist
-   (=ts .TIME)
-   (=keys
-     (:anyof
-       (=key ((:unordered (=roles (:someof (. .))) .ATTRS)) _)
-     ))
-   *
- )"""
-
-KEYLIST_SCHEMA = _parseSchema(KEYLIST_TEMPLATE, SCHEMA_TABLE)
-
-MIRRORLIST_TEMPLATE = r"""
- (=mirrorlist
-   (=ts .TIME)
-   (=mirrors (:anyof
-     (=mirror ((:unordered (=name .) (=urlbase .) (=contents (:someof .))
-                           .ATTRS)))))
-   *)
-"""
-
-MIRRORLIST_SCHEMA = _parseSchema(MIRRORLIST_TEMPLATE, SCHEMA_TABLE)
-
-TIMESTAMP_TEMPLATE = r"""
- (=ts
-   ((:unordered (=at .TIME) (=m .TIME .) (=k .TIME .)
-           (:anyof (=b . . .TIME . .)) .ATTRS))
- )"""
-
-TIMESTAMP_SCHEMA = _parseSchema(TIMESTAMP_TEMPLATE, SCHEMA_TABLE)
-
-BUNDLE_TEMPLATE = r"""
- (=bundle
-   (=at .TIME)
-   (=os .)
-   (:maybe (=arch .))
-   (=packages
-     (:someof
-      (. . . . ((:unordered
-                  (:maybe (=order . . .))
-                  (:maybe (=optional))
-                  (:anyof (=gloss . .))
-                  (:anyof (=longgloss . .))
-                  .ATTRS)))
-     )
+def parseTime(s):
+    """Parse a time 's' in YYYY-MM-DD HH:MM:SS format."""
+    try:
+        return calendar.timegm(time.strptime(s, "%Y-%m-%d %H:%M:%S"))
+    except ValueError:
+        raise glider.FormatError("Malformed time %r", s)
+
+def formatBase64(h):
+    """Return the base64 encoding of h with whitespace and = signs omitted."""
+    return binascii.b2a_base64(h).rstrip("=\n ")
+
+formatHash = formatBase64
+
+def parseBase64(s):
+    """Parse a base64 encoding with whitespace and = signs omitted. """
+    extra = len(s) % 4
+    if extra:
+        padding = "=" * (4 - extra)
+        s += padding
+    try:
+        return binascii.a2b_base64(s)
+    except binascii.Error:
+        raise glider.FormatError("Invalid base64 encoding")
+
+def parseHash(s):
+    h = parseBase64(s)
+    if len(h) != Crypto.Hash.SHA256.digest_size:
+        raise glider.FormatError("Bad hash length")
+    return h
+
+S = glider.checkJson
+
+# A date, in YYYY-MM-DD HH:MM:SS format.
+TIME_SCHEMA = S.RE(r'\d{4}-\d{2}-\d{2} \d{2}:\d{2}:\d{2}')
+# A hash, base64-encoded
+HASH_SCHEMA = S.RE(r'[a-zA-Z0-9\+\/]{43}')
+
+# A hexadecimal value.
+HEX_SCHEMA = S.RE(r'[a-fA-F0-9]+')
+# A base-64 encoded value
+BASE64_SCHEMA = S.RE(r'[a-zA-Z0-9\+\/]+')
+# An RSA key; subtype of PUBKEY_SCHEMA.
+RSAKEY_SCHEMA = S.Obj(
+    _keytype=S.Str("rsa"),
+    e=BASE64_SCHEMA,
+    n=BASE64_SCHEMA)
+# Any public key.
+PUBKEY_SCHEMA = S.Obj(
+    _keytype=S.AnyStr())
+
+KEYID_SCHEMA = HASH_SCHEMA
+SIG_METHOD_SCHEMA = S.AnyStr()
+RELPATH_SCHEMA = PATH_PATTERN_SCHEMA = S.AnyStr()
+URL_SCHEMA = S.AnyStr()
+VERSION_SCHEMA = S.ListOf(S.Any()) #XXXX WRONG
+
+# A single signature of an object.  Indicates the signature, the id of the
+# signing key, and the signing method.
+SIGNATURE_SCHEMA = S.Obj(
+    keyid=KEYID_SCHEMA,
+    method=SIG_METHOD_SCHEMA,
+    sig=BASE64_SCHEMA)
+
+# A signed object.
+SIGNED_SCHEMA = S.Obj(
+    signed=S.Any(),
+    signatures=S.ListOf(SIGNATURE_SCHEMA))
+
+ROLENAME_SCHEMA = S.AnyStr()
+
+# A role: indicates that a key is allowed to certify a kind of
+# document at a certain place in the repo.
+ROLE_SCHEMA = S.Struct([ROLENAME_SCHEMA, PATH_PATTERN_SCHEMA])
+
+# A Keylist: indicates a list of live keys and their roles.
+KEYLIST_SCHEMA = S.Obj(
+    _type=S.Str("Keylist"),
+    ts=TIME_SCHEMA,
+    keys=S.ListOf(S.Obj(key=PUBKEY_SCHEMA, roles=S.ListOf(ROLE_SCHEMA))))
+
+# A Mirrorlist: indicates all the live mirrors, and what documents they
+# serve.
+MIRRORLIST_SCHEMA = S.Obj(
+    _type=S.Str("Mirrorlist"),
+    ts=TIME_SCHEMA,
+    mirrors=S.ListOf(S.Obj(name=S.AnyStr(),
+                           urlbase=URL_SCHEMA,
+                           contents=S.ListOf(PATH_PATTERN_SCHEMA),
+                           weight=S.Int(lo=0),
+                           )))
+
+# A timestamp: indicates the lastest versions of all top-level signed objects.
+TIMESTAMP_SCHEMA = S.Obj(
+    _type = S.Str("Timestamp"),
+    at = TIME_SCHEMA,
+    m = S.Struct([TIME_SCHEMA, HASH_SCHEMA]),
+    k = S.Struct([TIME_SCHEMA, HASH_SCHEMA]),
+    b = S.DictOf(keySchema=S.AnyStr(),
+            valSchema=
+                 S.Struct([ VERSION_SCHEMA, RELPATH_SCHEMA, TIME_SCHEMA, HASH_SCHEMA ]))
    )
-   *
- )"""
-
-BUNDLE_SCHEMA = _parseSchema(BUNDLE_TEMPLATE, SCHEMA_TABLE)
-
-PACKAGE_TEMPLATE = r"""
- (=package
-  ((:unordered (=name .)
-               (=version .)
-               (=format . (.ATTRS))
-               (=path .)
-               (=ts .TIME)
-               (=digest .)
-               (:anyof (=shortdesc . .))
-               (:anyof (=longdesc . .))
-               .ATTRS)))
-"""
-
-PACKAGE_SCHEMA = _parseSchema(PACKAGE_TEMPLATE, SCHEMA_TABLE)
+
+# A Bundle: lists a bunch of packages that should be updated in tandem
+BUNDLE_SCHEMA = S.Obj(
+   _type=S.Str("Bundle"),
+   at=TIME_SCHEMA,
+   name=S.AnyStr(),
+   os=S.AnyStr(),
+   arch=S.Opt(S.AnyStr()),
+   version=VERSION_SCHEMA,
+   location=RELPATH_SCHEMA,
+   packages=S.ListOf(S.Obj(
+                    name=S.AnyStr(),
+                    version=VERSION_SCHEMA,
+                    path=RELPATH_SCHEMA,
+                    hash=HASH_SCHEMA,
+                    order=S.Struct([S.Int(), S.Int(), S.Int()]),
+                    optional=S.Opt(S.Bool()),
+                    gloss=S.DictOf(S.AnyStr(), S.AnyStr()),
+                    longgloss=S.DictOf(S.AnyStr(), S.AnyStr()))))
+
+PACKAGE_SCHEMA = S.Obj(
+            _type=S.Str("Package"),
+            name=S.AnyStr(),
+            location=RELPATH_SCHEMA,
+            version=VERSION_SCHEMA,
+            format=S.Obj(),
+            ts=TIME_SCHEMA,
+            files=S.ListOf(S.Struct([RELPATH_SCHEMA, HASH_SCHEMA])),
+            shortdesc=S.DictOf(S.AnyStr(), S.AnyStr()),
+            longdesc=S.DictOf(S.AnyStr(), S.AnyStr()))
 
 ALL_ROLES = ('timestamp', 'mirrors', 'bundle', 'package', 'master')
 
 class Key:
-    def __init__(self, key, roles):
+    #XXXX UNUSED.
+    def __init__(self, key, roles=()):
         self.key = key
         self.roles = []
         for r,p in roles:
@@ -215,15 +415,18 @@ class Key:
 
     def addRole(self, role, path):
         assert role in ALL_ROLES
-        self.roles.append(role, path)
+        self.roles.append((role, path))
 
     def getRoles(self):
-        return self.rules
+        return self.roles
 
     @staticmethod
-    def fromSExpression(sexpr):
+    def fromJSon(obj):
         # must match PUBKEY_SCHEMA
-        typeattr = sexp.access.s_attr(sexpr[1], "type")
+        keytype = obj['_keytype']
+        if keytype == 'rsa':
+            return Key(glider.keys.RSAKey.fromJSon(obj))
+
         if typeattr == 'rsa':
             key = glider.keys.RSAKey.fromSExpression(sexpr)
             if key is not None:
@@ -240,29 +443,27 @@ class Key:
     def sign(self, sexpr=None, digest=None):
         return self.key.sign(sexpr, digest=digest)
 
-    def checkSignature(self, method, sexpr=None, digest=None):
-        if digest == None:
-            _, digest = self.key._digest(sexpr, method)
-        ok = self.key.checkSignature(method, digest=digest)
+    def checkSignature(self, method, data, signatute):
+        ok = self.key.checkSignature(method, data, signature)
         # XXXX CACHE HERE.
         return ok
 
-class Keystore(KeyDB):
+class Keylist(KeyDB):
     def __init__(self):
         KeyDB.__init__(self)
 
-    @staticmethod
-    def addFromKeylist(sexpr, allowMasterKeys=False):
-        # Don't do this until we have validated the structure.
-        for ks in sexpr.access.s_lookup_all("keys.key"):
-            attrs = ks[1]
-            key_s = ks[2]
-            roles = s_attr(attrs, "roles")
-            #XXXX Use interface of Key, not RSAKey.
-            key = Key.fromSExpression(key_s)
-            if not key:
+    def addFromKeylist(self, obj, allowMasterKeys=False):
+        for keyitem in obj['keys']:
+            key = keyitem['key']
+            roles = keyitem['roles']
+
+            try:
+                key = glider.keys.RSAKey.fromJSon(key)
+            except glider.FormatException, e:
+                print e
                 #LOG skipping key.
                 continue
+
             for r,p in roles:
                 if r == 'master' and not allowMasterKeys:
                     #LOG
@@ -273,4 +474,274 @@ class Keystore(KeyDB):
 
             self.addKey(key)
 
+class StampedInfo:
+    def __init__(self, ts, hash, version=None, relpath=None):
+        self._ts = ts
+        self._hash = hash
+        self._version = version
+        self._relpath = relpath
+
+    @staticmethod
+    def fromJSonFields(timeStr, hashStr):
+        t = parseTime(timeStr)
+        h = parseHash(hashStr)
+        return StampedInfo(t, h)
+
+    def getHash(self):
+        return self._hash
+
+    def getRelativePath(self):
+        return self._relpath
+
+class TimestampFile:
+    def __init__(self, at, mirrorlistinfo, keylistinfo, bundleinfo):
+        self._time = at
+        self._mirrorListInfo = mirrorlistinfo
+        self._keyListInfo = keylistinfo
+        self._bundleInfo = bundleinfo
+
+    @staticmethod
+    def fromJSon(obj):
+        # must be validated.
+        at = parseTime(obj['at'])
+        m = StampedInfo.fromJSonFields(*obj['m'][:2])
+        k = StampedInfo.fromJSonFields(*obj['k'][:2])
+        b = {}
+        for name, bundle in obj['b'].iteritems():
+            v = bundle[0]
+            rp = bundle[1]
+            t = parseTime(bundle[2])
+            h = parseHash(bundle[3])
+            b[name] = StampedInfo(t, h, v, rp)
+
+        return TimestampFile(at, m, k, b)
+
+    def getTime(self):
+        return self._time
+
+    def getMirrorlistInfo(self):
+        return self._mirrorListInfo
+
+    def getKeylistInfo(self):
+        return self._keyListInfo
+
+    def getBundleInfo(self, name):
+        return self._bundleInfo[name]
+
+def readConfigFile(fname, needKeys=(), optKeys=(), preload={}):
+    parsed = preload.copy()
+    result = {}
+    execfile(fname, parsed)
+
+    for k in needKeys:
+        try:
+            result[k] = parsed[k]
+        except KeyError:
+            raise glider.FormatError("Missing value for %s in %s"%k,fname)
+
+    for k in optKeys:
+        try:
+            result[k] = parsed[k]
+        except KeyError:
+            pass
+
+    return result
+
+def makePackageObj(config_fname, package_fname):
+    preload = {}
+    shortDescs = {}
+    longDescs = {}
+    def ShortDesc(lang, val): shortDescs[lang] = val
+    def LongDesc(lang, val): longDescs[lang] = val
+    preload = { 'ShortDesc' : ShortDesc, 'LongDesc' : LongDesc }
+    r = readConfigFile(config_fname,
+                       ['name',
+                        'version',
+                        'format',
+                        'location',
+                        'relpath',
+                        ], (), preload)
+
+    f = open(package_fname, 'rb')
+    digest = getFileDigest(f)
+
+    # Check fields!
+    result = { '_type' : "Package",
+               'ts' : formatTime(time.time()),
+               'name' : r['name'],
+               'location' : r['location'], #DOCDOC
+               'version' : r['version'],
+               'format' : r['format'],
+               'files' : [ [ r['relpath'], formatHash(digest) ] ],
+               'shortdesc' : shortDescs,
+               'longdesc' : longDescs
+             }
+
+    PACKAGE_SCHEMA.checkMatch(result)
+
+    return result
+
+def makeBundleObj(config_fname, getPackageHash):
+    packages = []
+    def ShortGloss(lang, val): packages[-1]['gloss'][lang] = val
+    def LongGloss(lang, val): packages[-1]['longgloss'][lang] = val
+    def Package(name, version, path, order, optional=False):
+        packages.append({'name' : name,
+                         'version' : version,
+                         'path' : path,
+                         'order' : order,
+                         'optional' : optional,
+                         'gloss' : {},
+                         'longgloss' : {} })
+    preload = { 'ShortGloss' : ShortGloss, 'LongGloss' : LongGloss,
+                'Package' : Package }
+    r = readConfigFile(config_fname,
+                       ['name',
+                        'os',
+                        'version',
+                        'location',
+                        ], ['arch'], preload)
+
+    result = { '_type' : "Bundle",
+               'at' : formatTime(time.time()),
+               'name' : r['name'],
+               'os' : r['os'],
+               'version' : r['version'],
+               'location' : r['location'],
+               'packages' : packages }
+    if r.has_key('arch'):
+        result['arch'] = r['arch']
+
+    for p in packages:
+        try:
+            p['hash'] = formatHash(getPackageHash(p['path']))
+        except KeyError:
+            raise glider.FormatException("No such package as %s"%p['path'])
+
+    BUNDLE_SCHEMA.checkMatch(result)
+    return result
+
+def versionIsNewer(v1, v2):
+    return v1 > v2
+
+def makeTimestampObj(mirrorlist_obj, keylist_obj,
+                     bundle_objs):
+    result = { '_type' : 'Timestamp',
+               'at' : formatTime(time.time()) }
+    result['m'] = [ mirrorlist_obj['ts'],
+                    formatHash(getDigest(mirrorlist_obj)) ]
+    result['k'] = [ keylist_obj['ts'],
+                    formatHash(getDigest(keylist_obj)) ]
+    result['b'] = bundles = {}
+    for bundle in bundle_objs:
+        name = bundle['name']
+        v = bundle['version']
+        entry = [ v, bundle['location'], bundle['at'], formatHash(getDigest(bundle)) ]
+        if not bundles.has_key(name) or versionIsNewer(v, bundles[name][0]):
+            bundles[name] = entry
+
+    TIMESTAMP_SCHEMA.checkMatch(result)
+
+    return result
+
+class MirrorInfo:
+    def __init__(self, name, urlbase, contents, weight):
+        self._name = name
+        self._urlbase = urlbase
+        self._contents = contents
+        self._weight = weight
+
+    def canServeFile(self, fname):
+        for c in self._contents:
+            if rolePathMatches(c, fname):
+                return True
+        return False
+
+    def getFileURL(self, fname):
+        if self._urlbase[-1] == '/':
+            return self._urlbase+fname
+        else:
+            return "%s/%s" % (self._urlbase, fname)
 
+    def format(self):
+        return { 'name' : self._name,
+                 'urlbase' : self._urlbase,
+                 'contents' : self._contents,
+                 'weight' : self._weight }
+
+def makeMirrorListObj(mirror_fname):
+    mirrors = []
+    def Mirror(*a, **kw): mirrors.append(MirrorInfo(*a, **kw))
+    preload = {'Mirror' : Mirror}
+    r = readConfigFile(mirror_fname, (), (), preload)
+    result = { '_type' : "Mirrorlist",
+               'ts' : formatTime(time.time()),
+               'mirrors' : [ m.format() for m in mirrors ] }
+
+    MIRRORLIST_SCHEMA.checkMatch(result)
+    return result
+
+def makeKeylistObj(keylist_fname, includePrivate=False):
+    keys = []
+    def Key(obj): keys.append(obj)
+    preload = {'Key': Key}
+    r = readConfigFile(keylist_fname, (), (), preload)
+
+    klist = []
+    for k in keys:
+        k = glider.keys.RSAKey.fromJSon(k)
+        klist.append({'key': k.format(private=includePrivate), 'roles' : k.getRoles() })
+
+    result = { '_type' : "Keylist",
+               'ts' : formatTime(time.time()),
+               'keys' : klist }
+
+    KEYLIST_SCHEMA.checkMatch(result)
+    return result
+
+SCHEMAS_BY_TYPE = {
+    'Keylist' : KEYLIST_SCHEMA,
+    'Mirrorlist' : MIRRORLIST_SCHEMA,
+    'Timestamp' : TIMESTAMP_SCHEMA,
+    'Bundle' : BUNDLE_SCHEMA,
+    'Package' : PACKAGE_SCHEMA,
+    }
+
+def checkSignedObj(obj, keydb=None):
+    # Returns signaturestatus, role, path on sucess.
+
+    SIGNED_SCHEMA.checkMatch(obj)
+    try:
+        tp = obj['signed']['_type']
+    except KeyError:
+        raise glider.FormatException("Untyped object")
+    try:
+        schema = SCHEMAS_BY_TYPE[tp]
+    except KeyError:
+        raise glider.FormatException("Unrecognized type %r" % tp)
+    schema.checkMatch(obj['signed'])
+
+    if tp == 'Keylist':
+        role = "master"
+        path = "/meta/keys.txt"
+    elif tp == 'Mirrorlist':
+        role = "mirrors"
+        path = "/meta/mirrors.txt"
+    elif tp == "Timestamp":
+        role = 'timestamp'
+        path = "/meta/timestamp.txt"
+    elif tp == 'Bundle':
+        role = 'bundle'
+        path = obj['signed']['location']
+    elif tp == 'Package':
+        role = 'package'
+        path = obj['signed']['location']
+    else:
+        print tp
+        raise "Foo"
+
+    ss = None
+    if keydb is not None:
+        ss = checkSignatures(obj, keydb, role, path)
+
+    return ss, role, path
diff --git a/lib/glider/keys.py b/lib/glider/keys.py
index e2b2736..fefff6c 100644
--- a/lib/glider/keys.py
+++ b/lib/glider/keys.py
@@ -4,25 +4,23 @@ import Crypto.PublicKey.RSA
 import Crypto.Hash.SHA256
 import Crypto.Cipher.AES
 
-import sexp.access
-import sexp.encode
-import sexp.parse
-
 import cPickle as pickle
 import binascii
+import logging
 import os
 import struct
+import sys
+import simplejson
+import getpass
 
-class CryptoError(Exception):
-    pass
-
-class PubkeyFormatException(Exception):
-    pass
-
-class UnknownMethod(Exception):
-    pass
+import glider.formats
+import glider.util
 
 class PublicKey:
+    def __init__(self):
+        # Confusingly, these roles are the ones used for a private key to
+        # remember what we're willing to do with it.
+        self._roles = []
     def format(self):
         raise NotImplemented()
     def sign(self, data):
@@ -34,7 +32,17 @@ class PublicKey:
     def getKeyID(self):
         raise NotImplemented()
     def getRoles(self):
-        raise NotImplemented()
+        return self._roles
+    def addRole(self, role, path):
+        assert role in glider.formats.ALL_ROLES
+        self._roles.append((role, path))
+    def clearRoles(self):
+        del self._roles[:]
+    def hasRole(self, role, path):
+        for r, p in self._roles:
+            if r == role and glider.formats.rolePathMatches(p, path):
+                return True
+        return False
 
 if hex(1L).upper() == "0X1L":
     def intToBinary(number):
@@ -64,8 +72,13 @@ def binaryToInt(binary):
    """
    return long(binascii.b2a_hex(binary), 16)
 
-def _pkcs1_padding(m, size):
+def intToBase64(number):
+    return glider.formats.formatBase64(intToBinary(number))
+
+def base64ToInt(number):
+    return binaryToInt(glider.formats.parseBase64(number))
 
+def _pkcs1_padding(m, size):
     # I'd rather use OAEP+, but apparently PyCrypto barely supports
     # signature verification, and doesn't seem to support signature
     # verification with nondeterministic padding.  "argh."
@@ -83,25 +96,28 @@ def _xor(a,b):
 class RSAKey(PublicKey):
     """
     >>> k = RSAKey.generate(bits=512)
-    >>> sexpr = k.format()
-    >>> sexpr[:2]
-    ('pubkey', [('type', 'rsa')])
-    >>> k1 = RSAKey.fromSExpression(sexpr)
+    >>> obj = k.format()
+    >>> obj['_keytype']
+    'rsa'
+    >>> base64ToInt(obj['e'])
+    65537L
+    >>> k1 = RSAKey.fromJSon(obj)
     >>> k1.key.e == k.key.e
     True
     >>> k1.key.n == k.key.n
     True
     >>> k.getKeyID() == k1.getKeyID()
     True
-    >>> s = ['tag1', ['foobar'], [['foop', 'bar']], 'baz']
-    >>> method, sig = k.sign(sexpr=s)
-    >>> k.checkSignature(method, sig, sexpr=s)
+    >>> s = { 'A B C' : "D", "E" : [ "F", "g", 99] }
+    >>> method, sig = k.sign(obj=s)
+    >>> k.checkSignature(method, sig, obj=s)
     True
     >>> s2 = [ s ]
-    >>> k.checkSignature(method, sig, sexpr=s2)
+    >>> k.checkSignature(method, sig, obj=s2)
     False
     """
     def __init__(self, key):
+        PublicKey.__init__(self)
         self.key = key
         self.keyid = None
 
@@ -111,58 +127,79 @@ class RSAKey(PublicKey):
         return RSAKey(key)
 
     @staticmethod
-    def fromSExpression(sexpr):
-        # sexpr must match PUBKEY_SCHEMA
-        typeattr = sexp.access.s_attr(sexpr[1], "type")
-        if typeattr != "rsa":
-            return None
-        if len(sexpr[2]) != 2:
-            raise PubkeyFormatException("RSA keys must have an e,n pair")
-        e,n = sexpr[2]
-        key = Crypto.PublicKey.RSA.construct((binaryToInt(n), binaryToInt(e)))
-        return RSAKey(key)
-
-    def format(self):
-        n = intToBinary(self.key.n)
-        e = intToBinary(self.key.e)
-        return ("pubkey", [("type", "rsa")], (e, n))
+    def fromJSon(obj):
+        # obj must match RSAKEY_SCHEMA
+
+        glider.formats.RSAKEY_SCHEMA.checkMatch(obj)
+        n = base64ToInt(obj['n'])
+        e = base64ToInt(obj['e'])
+        if obj.has_key('d'):
+            d = base64ToInt(obj['d'])
+            p = base64ToInt(obj['p'])
+            q = base64ToInt(obj['q'])
+            u = base64ToInt(obj['u'])
+            key = Crypto.PublicKey.RSA.construct((n, e, d, p, q, u))
+        else:
+            key = Crypto.PublicKey.RSA.construct((n, e))
+
+        result = RSAKey(key)
+        if obj.has_key('roles'):
+            for r, p in obj['roles']:
+                result.addRole(r,p)
+
+        return result
+
+    def isPrivateKey(self):
+        return hasattr(self.key, 'd')
+
+    def format(self, private=False, includeRoles=False):
+        n = intToBase64(self.key.n)
+        e = intToBase64(self.key.e)
+        result = { '_keytype' : 'rsa',
+                   'e' : e,
+                   'n' : n }
+        if private:
+            result['d'] = intToBase64(self.key.d)
+            result['p'] = intToBase64(self.key.p)
+            result['q'] = intToBase64(self.key.q)
+            result['u'] = intToBase64(self.key.u)
+        if includeRoles:
+            result['roles'] = self.getRoles()
+        return result
 
     def getKeyID(self):
         if self.keyid == None:
-            n = intToBinary(self.key.n)
-            e = intToBinary(self.key.e)
-            keyval = (e,n)
             d_obj = Crypto.Hash.SHA256.new()
-            sexp.encode.hash_canonical(keyval, d_obj)
-            self.keyid = ("rsa", d_obj.digest())
+            glider.formats.getDigest(self.format(), d_obj)
+            self.keyid = glider.formats.formatHash(d_obj.digest())
         return self.keyid
 
-    def _digest(self, sexpr, method=None):
+    def _digest(self, obj, method=None):
         if method in (None, "sha256-pkcs1"):
             d_obj = Crypto.Hash.SHA256.new()
-            sexp.encode.hash_canonical(sexpr, d_obj)
+            glider.formats.getDigest(obj, d_obj)
             digest = d_obj.digest()
             return ("sha256-pkcs1", digest)
 
         raise UnknownMethod(method)
 
-    def sign(self, sexpr=None, digest=None):
-        assert _xor(sexpr == None, digest == None)
+    def sign(self, obj=None, digest=None):
+        assert _xor(obj == None, digest == None)
         if digest == None:
-            method, digest = self._digest(sexpr)
+            method, digest = self._digest(obj)
         m = _pkcs1_padding(digest, (self.key.size()+1) // 8)
-        sig = intToBinary(self.key.sign(m, "")[0])
+        sig = intToBase64(self.key.sign(m, "")[0])
         return (method, sig)
 
-    def checkSignature(self, method, sig, sexpr=None, digest=None):
-        assert _xor(sexpr == None, digest == None)
+    def checkSignature(self, method, sig, obj=None, digest=None):
+        assert _xor(obj == None, digest == None)
         if method != "sha256-pkcs1":
             raise UnknownMethod("method")
         if digest == None:
-            method, digest = self._digest(sexpr, method)
-        sig = binaryToInt(sig)
+            method, digest = self._digest(obj, method)
+        sig = base64ToInt(sig)
         m = _pkcs1_padding(digest, (self.key.size()+1) // 8)
-        return self.key.verify(m, (sig,))
+        return bool(self.key.verify(m, (sig,)))
 
 SALTLEN=16
 
@@ -244,15 +281,21 @@ def encryptSecret(secret, password, difficulty=0x80):
     pad = '\x00' * padlen
 
     slen = struct.pack("!L",len(secret))
-    encrypted = e.encrypt("%s%s%s%s" % (slen, secret, d, pad))[:-padlen]
+    encrypted = e.encrypt("%s%s%s%s" % (slen, secret, d, pad))
+    if padlen:
+        encrypted = encrypted[:-padlen]
     return "GKEY1%s%s%s"%(salt, iv, encrypted)
 
 def decryptSecret(encrypted, password):
+    """Decrypt a value encrypted with encryptSecret.  Raises UnknownFormat
+       or FormatError if 'encrypted' was not generated with encryptSecret.
+       Raises BadPassword if the password was not correct.
+    """
     if encrypted[:5] != "GKEY1":
-        raise UnknownFormat()
+        raise glider.UnknownFormat()
     encrypted = encrypted[5:]
     if len(encrypted) < SALTLEN+1+16:
-        raise FormatError()
+        raise glider.FormatException()
 
     salt = encrypted[:SALTLEN+1]
     iv = encrypted[SALTLEN+1:SALTLEN+1+16]
@@ -276,8 +319,81 @@ def decryptSecret(encrypted, password):
     d.update(salt)
 
     if d.digest() != hash:
-        print repr(decrypted)
-        raise BadPassword()
+        raise glider.BadPassword()
 
     return secret
 
+class KeyStore(glider.formats.KeyDB):
+    def __init__(self, fname, encrypted=True):
+        glider.formats.KeyDB.__init__(self)
+
+        self._loaded = None
+        self._fname = fname
+        self._passwd = None
+        self._encrypted = encrypted
+
+    def getpass(self, reprompt=False):
+        if self._passwd != None:
+            return self._passwd
+        while 1:
+            pwd = getpass.getpass("Password: ", sys.stderr)
+            if not reprompt:
+                return pwd
+
+            pwd2 = getpass.getpass("Confirm: ", sys.stderr)
+            if pwd == pwd2:
+                return pwd
+            else:
+                print "Mismatch; try again."
+
+    def load(self, password=None):
+        logging.info("Loading private keys from %r...", self._fname)
+        if not os.path.exists(self._fname):
+            logging.info("...no such file.")
+            self._loaded = True
+            return
+
+        if password is None and self._encrypted:
+            password = self.getpass()
+
+        contents = open(self._fname, 'rb').read()
+        if self._encrypted:
+            contents = decryptSecret(contents, password)
+
+        listOfKeys = simplejson.loads(contents)
+        self._passwd = password # It worked.
+        if not listOfKeys.has_key('keys'):
+            listOfKeys['keys'] = []
+        for obj in listOfKeys['keys']:
+            key = RSAKey.fromJSon(obj)
+            self.addKey(key)
+            logging.info("Loaded key %s", key.getKeyID())
+
+        self._loaded = True
+
+    def setPassword(self, passwd):
+        self._passwd = passwd
+
+    def clearPassword(self):
+        self._passwd = None
+
+    def save(self, password=None):
+        if not self._loaded and self._encrypted:
+            self.load(password)
+
+        if password is None:
+            password = self.getpass(True)
+
+        logging.info("Saving private keys into %r...", self._fname)
+        listOfKeys = { 'keys' :
+                       [ key.format(private=True, includeRoles=True) for key in
+                         self._keys.values() ]
+                       }
+        contents = simplejson.dumps(listOfKeys)
+        if self._encrypted:
+            contents = encryptSecret(contents, password)
+        glider.util.replaceFile(self._fname, contents)
+        self._passwd = password # It worked.
+        logging.info("Done.")
+
+
diff --git a/lib/glider/master_keys.py b/lib/glider/master_keys.py
new file mode 100644
index 0000000..0d455d1
--- /dev/null
+++ b/lib/glider/master_keys.py
@@ -0,0 +1,5 @@
+
+
+MASTER_KEYS = [
+
+]
diff --git a/lib/glider/repository.py b/lib/glider/repository.py
index 9ec1dc6..b956993 100644
--- a/lib/glider/repository.py
+++ b/lib/glider/repository.py
@@ -1,10 +1,14 @@
 
-import sexp.parse
-import sexp.access
 import glider.formats
+import glider.util
 
+import simplejson
+import logging
 import os
 import threading
+import time
+
+MAX_TIMESTAMP_AGE = 24*60*60
 
 class RepositoryFile:
     def __init__(self, repository, relativePath, schema,
@@ -16,12 +20,15 @@ class RepositoryFile:
         self._signedFormat = signedFormat
         self._needSigs = needSigs
 
-        self._signed_sexpr = None
-        self._main_sexpr = None
+        self._signed_obj = self._main_obj = None
+        self._sigStatus = None
         self._mtime = None
 
+    def getRelativePath(self):
+        return self._relativePath
+
     def getPath(self):
-        return os.path.join(self._repository._root, self._relativePath)
+        return self._repository.getFilename(self._relativePath)
 
     def _load(self):
         fname = self.getPath()
@@ -40,82 +47,267 @@ class RepositoryFile:
         finally:
             f.close()
 
-        signed_sexpr,main_sexpr = self._checkContent(content)
+        signed_obj,main_obj = self._checkContent(content)
 
-        self._signed_sexpr = signed_sexpr
-        self._main_sexpr = main_sexpr
+        self._signed_obj = signed_obj
+        self._main_obj = main_obj
         self._mtime = mtime
 
     def _save(self, content=None):
         if content == None:
             content = sexpr.encode
 
-        signed_sexpr,main_sexpr = self._checkContent(content)
+        signed_obj,main_obj = self._checkContent(content)
 
         fname = self.getPath()
-        fname_tmp = fname+"_tmp"
+        glider.util.replaceFile(fname, contents)
 
-        fd = os.open(fname_tmp, os.WRONLY|os.O_CREAT|os.O_TRUNC, 0644)
-        try:
-            os.write(fd, contents)
-        finally:
-            os.close(fd)
-        if sys.platform in ('cygwin', 'win32'):
-            # Win32 doesn't let rename replace an existing file.
-            try:
-                os.unlink(fname)
-            except OSError:
-                pass
-        os.rename(fname_tmp, fname)
-
-        self._signed_sexpr = signed_sexpr
-        self._main_sexpr = main_sexpr
+        self._signed_obj = signed_obj
+        self._main_obj = main_obj
         self._mtime = mtime
 
     def _checkContent(self, content):
-        sexpr = sexp.parse.parse(content)
-        if not sexpr:
-            raise ParseError()
+
+        try:
+            obj = simplejson.loads(content)
+        except ValueError, e:
+            raise glider.FormatException("Couldn't decode content: %s"%e)
 
         if self._signedFormat:
-            if not glider.formats.SIGNED_SCHEMA.matches(sexpr):
-                raise FormatError()
-
-            sigs = checkSignatures(sexpr, self._repository._keyDB,
-                                   self._needRole, self._relativePath)
-            good = sigs[0]
-            # XXXX If good is too low but unknown is high, we may need
-            # a new key file.
-            if len(good) < 1:
-                raise SignatureError()
-
-            main_sexpr = sexpr[1]
-            signed_sexpr = sexpr
+            # This is supposed to be signed.
+            glider.formats.SIGNED_SCHEMA.checkMatch(obj)
+
+            main_obj = obj['signed']
+            signed_obj = obj
         else:
-            signed_sexpr = None
-            main_sexpr = sexpr
+            signed_obj = None
+            main_obj = obj
 
-        if self._schema != None and not self._schema.matches(main_sexpr):
-            raise FormatError()
+        if self._schema != None:
+            self._schema.checkMatch(main_obj)
 
-        return signed_sexpr, main_sexpr
+        return signed_obj, main_obj
 
     def load(self):
-        if self._main_sexpr == None:
+        if self._main_obj == None:
             self._load()
 
+    def get(self):
+        return self._main_obj
+
+    def isLoaded(self):
+        return self._main_obj != None
+
+    def getContent(self):
+        self.load()
+        return self._main_obj
+
+    def _checkSignatures(self):
+        self.load()
+        sigStatus = glider.formats.checkSignatures(self._signed_obj,
+                                     self._repository._keyDB,
+                                     self._needRole, self._relativePath)
+        self._sigStatus = sigStatus
+
+    def checkSignatures(self):
+        if self._sigStatus is None:
+            self._checkSignatures()
+        return self._sigStatus
+
 class LocalRepository:
     def __init__(self, root):
         self._root = root
-        self._keyDB = None
+        self._keyDB = glider.util.getKeylist(None)
 
         self._keylistFile = RepositoryFile(
-            self, "meta/keys.txt", glider.formats.KEYLIST_SCHEMA,
+            self, "/meta/keys.txt", glider.formats.KEYLIST_SCHEMA,
             needRole="master")
         self._timestampFile = RepositoryFile(
-            self, "meta/timestamp.txt", glider.formats.TIMESTAMP_SCHEMA,
+            self, "/meta/timestamp.txt", glider.formats.TIMESTAMP_SCHEMA,
             needRole="timestamp")
         self._mirrorlistFile = RepositoryFile(
-            self, "meta/mirrors.txt", glider.formats.MIRRORLIST_SCHEMA,
+            self, "/meta/mirrors.txt", glider.formats.MIRRORLIST_SCHEMA,
             needRole="mirrors")
+        self._metaFiles = [ self._keylistFile,
+                            self._timestampFile,
+                            self._mirrorlistFile ]
+
+        self._packageFiles = {}
+        self._bundleFiles = {}
+
+    def getFilename(self, relativePath):
+        if relativePath.startswith("/"):
+            relativePath = relativePath[1:]
+        return os.path.join(self._root, relativePath)
+
+    def getKeylistFile(self):
+        return self._keylistFile
+
+    def getTimestampFile(self):
+        return self._timestampFile
+
+    def getMirrorlistFile(self):
+        return self._mirrorlistFile
+
+    def getPackageFile(self, relPath):
+        try:
+            return self._packageFiles[relPath]
+        except KeyError:
+            self._packageFiles[relPath] = pkg = RepositoryFile(
+                self, relPath, glider.formats.PACKAGE_SCHEMA,
+                needRole='package')
+            return pkg
+
+    def getBundleFile(self, relPath):
+        try:
+            return self._bundleFiles[relPath]
+        except KeyError:
+            self._bundleFiles[relPath] = pkg = RepositoryFile(
+                self, relPath, glider.formats.BUNDLE_SCHEMA,
+                needRole='bundle')
+            return pkg
+
+    def getFilesToUpdate(self, now=None, trackingBundles=()):
+        if now == None:
+            now = time.time()
+
+        need = set()
+
+        # Fetch missing metafiles.
+        for f in self._metaFiles:
+            try:
+                f.load()
+            except OSError, e:
+                print "need", f.getPath()
+                logging.info("Couldn't load %s: %s.  Must fetch it.",
+                             f.getPath(), e)
+                need.add(f.getRelativePath())
+
+        # If the timestamp file is out of date, we need to fetch it no
+        # matter what.  (Even if it is isn't signed, it can't possibly
+        # be good.)
+        ts = self._timestampFile.get()
+        if ts:
+            age = now - glider.formats.parseTime(ts['at'])
+            ts = glider.formats.TimestampFile.fromJSon(ts)
+            if age > MAX_TIMESTAMP_AGE:
+                need.add(self._timestampFile.getRelativePath())
+
+        # If the keylist isn't signed right, we can't check the
+        # signatures on anything else.
+        if self._keylistFile.get():
+            s = self._keylistFile.checkSignatures()
+            if not s.isValid(): # For now only require one master key.
+                need.add(self._keylistFile.getRelativePath())
+
+        if need:
+            return need
+
+        # Import the keys from the keylist.
+        self._keyDB.addFromKeylist(self._keylistFile.get())
+
+        # If the timestamp isn't signed right, get a new timestamp and a
+        # new keylist.
+        s = self._timestampFile.checkSignatures()
+        if not s.isValid():
+            need.add(self._keylistFile.getRelativePath())
+            need.add(self._timestampFile.getRelativePath())
+            return need
+
+        # FINALLY, we know we have an up-to-date, signed timestamp
+        # file.  Check whether the keys and mirrors file are as
+        # authenticated.
+        h_kf = glider.formats.getDigest(self._keylistFile.get())
+        h_expected = ts.getKeylistInfo().getHash()
+        if h_kf != h_expected:
+            need.add(self._keylistFile.getRelativePath())
+
+        if need:
+            return need
+
+        s = self._mirrorlistFile.checkSignatures()
+        if not s.isValid():
+            need.add(self._mirrorlistFile.getRelativePath())
+
+        h_mf = glider.formats.getDigest(self._mirrorlistFile.get())
+        h_expected = ts.getMirrorlistInfo().getHash()
+        if h_mf != h_expected:
+            need.add(self._mirrorlistFile.getRelativePath())
+
+        if need:
+            return need
+
+        # Okay; that's it for the metadata.  Do we have the right
+        # bundles?
+        bundles = {}
+        for b in trackingBundles:
+            try:
+                binfo = ts.getBundleInfo(b)
+            except KeyError:
+                logging.warn("Unrecognized bundle %s"%b)
+                continue
+
+            rp = binfo.getRelativePath()
+            bfile = self.getBundleFile(rp)
+            try:
+                bfile.load()
+            except OSError:
+                need.add(rp)
+                continue
+
+            h_b = glider.formats.getDigest(bfile.get())
+            h_expected = binfo.getHash()
+            if h_b != h_expected:
+                need.add(rp)
+                continue
+
+            s = bfile.checkSignatures()
+            if not s.isValid():
+                # Can't actually use it.
+                continue
+
+            bundles[rp] = bfile
+
+        # Okay.  So we have some bundles.  See if we have their packages.
+        packages = {}
+        for bfile in bundles.values():
+            bundle = bfile.get()
+            for pkginfo in bundle['packages']:
+                rp = pkginfo['path']
+                pfile = self.getPackageFile(rp)
+                try:
+                    pfile.load()
+                except OSError:
+                    need.add(rp)
+                    continue
+
+                h_p = glider.formats.getDigest(pfile.get())
+                h_expected = glider.formats.parseHash(pkginfo['hash'])
+                if h_p != h_expected:
+                    need.add(rp)
+                    continue
+
+                s = pfile.checkSignatures()
+                if not s.isValid():
+                    # Can't use it.
+                    continue
+                packages[rp] = pfile
+
+        # Finally, we have some packages.  Do we have their underlying
+        # files?
+        for pfile in packages.values():
+            package = pfile.get()
+            for f in package['files']:
+                rp, h = f[:2]
+                h_expected = glider.formats.parseHash(h)
+                fn = self.getFilename(rp)
+                try:
+                    h_got = glider.formats.getFileDigest(fn)
+                except OSError:
+                    need.add(rp)
+                    continue
+                if h_got != h_expected:
+                    need.add(rp)
 
+        # Okay; these are the files we need.
+        return need
diff --git a/lib/glider/tests.py b/lib/glider/tests.py
index 1c392e8..7aeb4f6 100644
--- a/lib/glider/tests.py
+++ b/lib/glider/tests.py
@@ -1,21 +1,57 @@
 
 import unittest
 import doctest
+import os
+import tempfile
 
 import glider.keys
 import glider.formats
 import glider.repository
+import glider.checkJson
 
 import glider.tests
 
-class EncryptionTest(unittest.TestCase):
-    pass
+class CanonicalEncodingTest(unittest.TestCase):
+    def test_encode(self):
+        enc = glider.formats.encodeCanonical
+        self.assertEquals(enc(''), '""')
+        self.assertEquals(enc('"'), '"\\""')
+        self.assertEquals(enc('\t\\\n"\r'),
+                          '"\t\\\\\n\\"\r"')
+
+class CryptoTests(unittest.TestCase):
+    def test_encrypt(self):
+        s = "The Secret words are marzipan habidashery zeugma."
+        password = "the password is swordfish."
+        encrypted = glider.keys.encryptSecret(s, password)
+        self.assertNotEquals(encrypted, s)
+        self.assert_(encrypted.startswith("GKEY1"))
+        self.assertEquals(s, glider.keys.decryptSecret(encrypted, password))
+        self.assertRaises(glider.BadPassword, glider.keys.decryptSecret,
+                          encrypted, "password")
+        self.assertRaises(glider.UnknownFormat, glider.keys.decryptSecret,
+                          "foobar", password)
+
+    def test_keystore(self):
+        passwd = "umfitty noonah"
+        fname = tempfile.mktemp()
+        ks = glider.keys.KeyStore(fname)
+        key1 = glider.keys.RSAKey.generate(512)
+        key2 = glider.keys.RSAKey.generate(512)
+        ks.addKey(key1)
+        ks.addKey(key2)
+        ks.save(passwd)
+
+        ks2 = glider.keys.KeyStore(fname)
+        ks2.load(passwd)
+        self.assertEquals(key1.key.n, ks2.getKey(key1.getKeyID()).key.n)
 
 def suite():
     suite = unittest.TestSuite()
 
     suite.addTest(doctest.DocTestSuite(glider.formats))
     suite.addTest(doctest.DocTestSuite(glider.keys))
+    suite.addTest(doctest.DocTestSuite(glider.checkJson))
 
     loader = unittest.TestLoader()
     suite.addTest(loader.loadTestsFromModule(glider.tests))
diff --git a/lib/glider/util.py b/lib/glider/util.py
new file mode 100644
index 0000000..e2fb57a
--- /dev/null
+++ b/lib/glider/util.py
@@ -0,0 +1,73 @@
+
+import os
+import sys
+import tempfile
+
+import simplejson
+
+import glider.formats
+import glider.keys
+import glider.master_keys
+
+def moveFile(fromLocation, toLocation):
+    if sys.platform in ('cygwin', 'win32'):
+        # Win32 doesn't let rename replace an existing file.
+        try:
+            os.unlink(toLocation)
+        except OSError:
+            pass
+    os.rename(fromLocation, toLocation)
+
+
+def replaceFile(fname, contents, textMode=False):
+    """overwrite the file in 'fname' atomically with the content of 'contents'
+    """
+    dir, prefix = os.path.split(fname)
+    fd, fname_tmp = tempfile.mkstemp(prefix=prefix, dir=dir, text=textMode)
+
+    try:
+        os.write(fd, contents)
+    finally:
+        os.close(fd)
+
+    moveFile(fname_tmp, fname)
+
+def userFilename(name):
+    try:
+        base = os.environ["THANDY_HOME"]
+    except KeyError:
+        base = "~/.thandy"
+    base = os.path.expanduser(base)
+    if not os.path.exists(base):
+        os.makedirs(base, 0700)
+    return os.path.join(base, name)
+
+def getKeylist(keys_fname, checkKeys=True):
+    import glider.master_keys
+
+    keydb = glider.formats.Keylist()
+
+    for key in glider.master_keys.MASTER_KEYS:
+        keydb.addKey(key)
+
+    user_keys = userFilename("preload_keys")
+    if os.path.exists(user_keys):
+        #XXXX somewhat roundabout.
+        keylist = glider.formats.makeKeylistObj(user_keys)
+        keydb.addFromKeylist(keylist, allowMasterKeys=True)
+
+    if keys_fname and os.path.exists(keys_fname):
+        f = open(keys_fname, 'r')
+        try:
+            obj = simplejson.load(f)
+        finally:
+            f.close()
+        ss, role, path = glider.formats.checkSignedObj(obj, keydb)
+        if role != 'master':
+            raise glider.FormatException("%s wasn't a keylist."%keys_fname)
+        if checkKeys and not ss.isValid():
+            raise glider.FormatException("%s not signed by enough master keys"%
+                                         keys_fname)
+        keydb.addFromKeylist(obj['signed'], allowMasterKeys=False)
+
+    return keydb
diff --git a/samples/bundle.cfg b/samples/bundle.cfg
new file mode 100644
index 0000000..d2c78b3
--- /dev/null
+++ b/samples/bundle.cfg
@@ -0,0 +1,15 @@
+
+name = "example-bundle"
+version = [0,1,10]
+location = "/bundleinfo/example/os-arch/example-os-arch-0.1.10.txt"
+os = "os"
+arch = "arch"
+
+Package(name="example",
+	version=[0,1,2],
+	path="/pkginfo/example/rpm/example.txt",
+	order=(10,10,10),
+	optional=False)
+ShortGloss("en", "Example package is needed to make the ossifrage squeamish.")
+LongGloss("en", "You wouldn't want an unsqueamish ossifrage, would you?")
+
diff --git a/samples/example-mirrors.txt b/samples/example-mirrors.txt
new file mode 100644
index 0000000..c02583e
--- /dev/null
+++ b/samples/example-mirrors.txt
@@ -0,0 +1,5 @@
+
+Mirror(name="moria",
+       urlbase="http://moria.seul.org/thandy-example/",
+       contents=[ "**" ],
+       weight=1)
diff --git a/samples/example-package.cfg b/samples/example-package.cfg
new file mode 100644
index 0000000..527211b
--- /dev/null
+++ b/samples/example-package.cfg
@@ -0,0 +1,24 @@
+
+
+# This is package is name 'example'
+name = "example"
+
+# Encodes version 0.1.2.
+version = [0, 1, 2]
+
+# What kind of package is this?
+format = "rpm"
+
+# Where in the repository does it go?
+location = "/pkginfo/example/rpm/example.txt"
+
+# Where in the repository does its underlying rpm file go?
+relpath = "/data/example-0.1.2.rpm"
+
+# Decriptions of the package.
+ShortDesc('en',  "Example package")
+LongDesc('en',
+"""This is an example package.
+
+Its description is not quite so long as it might be, but hey.""")
+
diff --git a/samples/testpackage.cfg b/samples/testpackage.cfg
new file mode 100644
index 0000000..8b13789
--- /dev/null
+++ b/samples/testpackage.cfg
@@ -0,0 +1 @@
+
diff --git a/specs/glider-spec.txt b/specs/glider-spec.txt
index cf0bc49..b940e49 100644
--- a/specs/glider-spec.txt
+++ b/specs/glider-spec.txt
@@ -307,6 +307,14 @@
           (signature ({(keyid K) (method M) (ATTR VAL)*}) SIG)+
        )
 
+       { "_type" : "Signed",
+         "signed" : X,
+         "sigatures" : [
+            { "keyid" : K,
+              "method" : M,
+              ...
+              "sig" : S } ]
+
    where: X is a list whose first element describes the signed object.
           K is the identifier of a key signing the document
           M is the method to be used to make the signature
@@ -324,6 +332,11 @@
 
    All keys are of the format:
       (pubkey ({(type TYPE) (ATTR VAL)*}) KEYVAL)
+
+      { "_keytype" : TYPE,
+         ...
+        "keyval" : KEYVAL }
+
    where TYPE is a string describing the type of the key and how it's
    used to sign documents.  The type determines the interpretation of
    KEYVAL.
@@ -336,6 +349,10 @@
    binary format.  [This makes keys 45-60% more compact than using
    decimal integers.]
 
+      {Values given as integers.}
+
+      {'e' : e, 'n' : n, big-endian hex. }
+
    All RSA keys must be at least 2048 bits long.
 
 
@@ -370,6 +387,13 @@
      ...
    )
 
+     { "_type" : "Keylist",
+       "ts" : TIME,
+       "keys" : [
+           { "roles" : [ [ ROLE, PATH ], ... ],
+             ...
+             "key" : KEY }, ... ] }
+
    The "ts" line describes when the keys file was updated.  Clients
    MUST NOT replace a file with an older one, and SHOULD NOT accept a
    file too far in the future.
@@ -392,7 +416,18 @@
        ( (mirror ({(name N) (urlbase U) (contents PATH+) (weight W)
                    (official)?  (ATTR VAL)})) * )
      ...
-  )
+    )
+
+    { "_type" : "Mirrorlist",
+      "mirrors" : [
+         { "name" : N,
+           "urlbase" : U,
+           "contents" : [PATH ... ] ,
+           "weight" : W,
+           "official" : BOOL,
+           ...
+         }, ... ]
+    }
 
   Every mirror is a copy of some or all of the directory hierarchy
   containing at least the /meta, /bundles/, and /pkginfo directories.
@@ -417,12 +452,20 @@
     ({(at TIME)
       (m TIME MIRRORLISTHASH)
       (k TIME KEYLISTHASH)
-      (b NAME VERSION TIME PATH HASH)*})
+      (b NAME VERSION PATH TIME HASH)*})
   )
 
+    { "_type" : Timestamp,
+      "at" : TIME,
+      "m" : [ TIME, HASH ],
+      "k" : [ TIME, HASH ],
+      "b" : { NAME :
+                 [ [ Version, Path, Time, Hash ] ] }
+    }
+
   TIME is when the timestamp was signed.  MIRRORLISTHASH is the digest
   of the mirror-list file; KEYLISTHASH is the digest of the key list
-  file; and the 'b' entries are a list of the latest version of each
+  file; and the 'b' entries are a list of the latest version of all
   bundles and their locations and hashes.
 
 3.6. File formats: bundle files
@@ -440,6 +483,23 @@
                                  (ATTR VAL)*})? )* )
   )
 
+     { "_type" : "Bundle",
+       "name" : NAME,
+       "at" : TIME,
+       "os" : OS,
+       [ "arch" : ARCH, ]
+       "version" : V
+       "packages" :
+          [ { "name" : NAME,
+              "version" : VERSION,
+              "path" : PATH,
+              "hash" : HASH,
+              "order" : [ INST, UPDATE, REMOVE ],
+              [ "optional : BOOL, ]
+              "gloss" : { LANG : TEXT },
+              "longgloss" : { LANG : TEXT },
+              } ] }
+
   Most elements are self-explanatory; the INST, UPDATE, and REMOVE
   elements of the order element are numbers defining the order in
   which the packages are installed, updated, and removed respectively.
@@ -648,6 +708,3 @@ R.2. Integration with existing GPG signatures
   have to be mad to touch it.
 
 
-
-
-