More repository/signing/marshalling/crypto work for glider.

git-svn-id: file:///home/or/svnrepo/updater/trunk@16928 55e972cd-5a19-0410-ae62-a4d7a52db4cd

8 files changed, 372 insertions, 21 deletions

diff --git a/Makefile b/Makefile
new file mode 100644
index 0000000..0d58eb4
--- /dev/null
+++ b/Makefile
@@ -0,0 +1,6 @@
+
+export PYTHONPATH=./lib
+
+test:
+	python -m sexp.tests
+	python -m glider.tests
diff --git a/TODO b/TODO
index 48050c0..7e49aa6 100644
--- a/TODO
+++ b/TODO
@@ -5,9 +5,11 @@ o Write spec
 . Write server-side code (python)
   o S-expression lib
   . Code to manage data formats
+    . Parse
+    o Validate
   - Code to wrangle private keys
-    - Generate
-    - Store, load (password-protected)
+    o Generate
+    . Store, load (password-protected)
     - Print for posterity
 
   - Code to generate timestamp files
diff --git a/lib/glider/formats.py b/lib/glider/formats.py
index 6ccd466..34ece7d 100644
--- a/lib/glider/formats.py
+++ b/lib/glider/formats.py
@@ -48,6 +48,13 @@ def checkSignatures(signed, keyDB, role, path):
     unknownSigs = []
     tangentialSigs = []
 
+    assert signed[0] == "signed"
+    data = signed[1]
+
+    d_obj = Crypto.Hash.SHA256.new()
+    sexp.encode.hash_canonical(data, d_obj)
+    digest = d_obj.digest()
+
     for signature in sexp.access.s_children(signed, "signature"):
         attrs = signature[1]
         sig = attrs[2]
@@ -59,7 +66,7 @@ def checkSignatures(signed, keyDB, role, path):
             continue
         method = s_child(attrs, "method")[1]
         try:
-            result = key.checkSignature(method, data, sig)
+            result = key.checkSignature(method, sig, digest=digest)
         except UnknownMethod:
             continue
         if result == True:
@@ -75,6 +82,8 @@ def checkSignatures(signed, keyDB, role, path):
         else:
             badSigs.append(keyid)
 
+    return goodSigs, badSigs, unknownSigs, tangentialSigs
+
 def sign(signed, key):
     assert sexp.access.s_tag(signed) == 'signed'
     s = signed[1]
@@ -129,7 +138,7 @@ SIGNED_TEMPLATE = r"""
 
 SIGNED_SCHEMA = _parseSchema(SIGNED_TEMPLATE, SCHEMA_TABLE)
 
-KEYFILE_TEMPLATE = r"""
+KEYLIST_TEMPLATE = r"""
  (=keylist
    (=ts .TIME)
    (=keys
@@ -139,7 +148,7 @@ KEYFILE_TEMPLATE = r"""
    *
  )"""
 
-KEYFILE_SCHEMA = _parseSchema(KEYFILE_TEMPLATE, SCHEMA_TABLE)
+KEYLIST_SCHEMA = _parseSchema(KEYLIST_TEMPLATE, SCHEMA_TABLE)
 
 MIRRORLIST_TEMPLATE = r"""
  (=mirrorlist
@@ -194,3 +203,74 @@ PACKAGE_TEMPLATE = r"""
 """
 
 PACKAGE_SCHEMA = _parseSchema(PACKAGE_TEMPLATE, SCHEMA_TABLE)
+
+ALL_ROLES = ('timestamp', 'mirrors', 'bundle', 'package', 'master')
+
+class Key:
+    def __init__(self, key, roles):
+        self.key = key
+        self.roles = []
+        for r,p in roles:
+            self.addRole(r,p)
+
+    def addRole(self, role, path):
+        assert role in ALL_ROLES
+        self.roles.append(role, path)
+
+    def getRoles(self):
+        return self.rules
+
+    @staticmethod
+    def fromSExpression(sexpr):
+        # must match PUBKEY_SCHEMA
+        typeattr = sexp.access.s_attr(sexpr[1], "type")
+        if typeattr == 'rsa':
+            key = glider.keys.RSAKey.fromSExpression(sexpr)
+            if key is not None:
+                return Key(key)
+        else:
+            return None
+
+    def format(self):
+        return self.key.format()
+
+    def getKeyID(self):
+        return self.key.getKeyID()
+
+    def sign(self, sexpr=None, digest=None):
+        return self.key.sign(sexpr, digest=digest)
+
+    def checkSignature(self, method, sexpr=None, digest=None):
+        if digest == None:
+            _, digest = self.key._digest(sexpr, method)
+        ok = self.key.checkSignature(method, digest=digest)
+        # XXXX CACHE HERE.
+        return ok
+
+class Keystore(KeyDB):
+    def __init__(self):
+        KeyDB.__init__(self)
+
+    @staticmethod
+    def addFromKeylist(sexpr, allowMasterKeys=False):
+        # Don't do this until we have validated the structure.
+        for ks in sexpr.access.s_lookup_all("keys.key"):
+            attrs = ks[1]
+            key_s = ks[2]
+            roles = s_attr(attrs, "roles")
+            #XXXX Use interface of Key, not RSAKey.
+            key = Key.fromSExpression(key_s)
+            if not key:
+                #LOG skipping key.
+                continue
+            for r,p in roles:
+                if r == 'master' and not allowMasterKeys:
+                    #LOG
+                    continue
+                if r not in ALL_ROLES:
+                    continue
+                key.addRole(r,p)
+
+            self.addKey(key)
+
+
diff --git a/lib/glider/keys.py b/lib/glider/keys.py
index 9fb73ec..e2b2736 100644
--- a/lib/glider/keys.py
+++ b/lib/glider/keys.py
@@ -2,13 +2,16 @@
 # These require PyCrypto.
 import Crypto.PublicKey.RSA
 import Crypto.Hash.SHA256
+import Crypto.Cipher.AES
 
 import sexp.access
 import sexp.encode
 import sexp.parse
 
+import cPickle as pickle
 import binascii
 import os
+import struct
 
 class CryptoError(Exception):
     pass
@@ -134,26 +137,147 @@ class RSAKey(PublicKey):
             self.keyid = ("rsa", d_obj.digest())
         return self.keyid
 
-    def sign(self, sexpr=None, digest=None):
-        assert _xor(sexpr == None, digest == None)
-        if digest == None:
+    def _digest(self, sexpr, method=None):
+        if method in (None, "sha256-pkcs1"):
             d_obj = Crypto.Hash.SHA256.new()
             sexp.encode.hash_canonical(sexpr, d_obj)
             digest = d_obj.digest()
+            return ("sha256-pkcs1", digest)
+
+        raise UnknownMethod(method)
+
+    def sign(self, sexpr=None, digest=None):
+        assert _xor(sexpr == None, digest == None)
+        if digest == None:
+            method, digest = self._digest(sexpr)
         m = _pkcs1_padding(digest, (self.key.size()+1) // 8)
         sig = intToBinary(self.key.sign(m, "")[0])
-        return ("sha256-pkcs1", sig)
+        return (method, sig)
 
     def checkSignature(self, method, sig, sexpr=None, digest=None):
         assert _xor(sexpr == None, digest == None)
         if method != "sha256-pkcs1":
             raise UnknownMethod("method")
         if digest == None:
-            d_obj = Crypto.Hash.SHA256.new()
-            sexp.encode.hash_canonical(sexpr, d_obj)
-            digest = d_obj.digest()
+            method, digest = self._digest(sexpr, method)
         sig = binaryToInt(sig)
         m = _pkcs1_padding(digest, (self.key.size()+1) // 8)
         return self.key.verify(m, (sig,))
 
+SALTLEN=16
+
+def secretToKey(salt, secret):
+    """Convert 'secret' to a 32-byte key, using a version of the algorithm
+       from RFC2440.  The salt must be SALTLEN+1 bytes long, and should
+       be random, except for the last byte, which encodes how time-
+       consuming the computation should be.
+
+       (The goal is to make offline password-guessing attacks harder by
+       increasing the time required to convert a password to a key, and to
+       make precomputed password tables impossible to generate by )
+    """
+    assert len(salt) == SALTLEN+1
+
+    # The algorithm is basically, 'call the last byte of the salt the
+    # "difficulty", and all other bytes of the salt S.  Now make
+    # an infinite stream of S|secret|S|secret|..., and hash the
+    # first N bytes of that, where N is determined by the difficulty.
+    #
+    # Obviously, this wants a hash algorithm that's tricky to
+    # parallelize.
+    #
+    # Unlike RFC2440, we use a 16-byte salt.  Because CPU times
+    # have improved, we start at 16 times the previous minimum.
+
+    difficulty = ord(salt[-1])
+    count = (16L+(difficulty & 15)) << ((difficulty >> 4) + 10)
+
+    # Make 'data' nice and long, so that we don't need to call update()
+    # a zillion times.
+    data = salt[:-1]+secret
+    if len(data)<1024:
+        data *= (1024 // len(data))+1
+
+    d = Crypto.Hash.SHA256.new()
+    iters, leftover = divmod(count, len(data))
+    for _ in xrange(iters):
+        d.update(data)
+        #count -= len(data)
+    if leftover:
+        d.update(data[:leftover])
+        #count -= leftover
+    #assert count == 0
+
+    return d.digest()
+
+def encryptSecret(secret, password, difficulty=0x80):
+    """Encrypt the secret 'secret' using the password 'password',
+       and return the encrypted result."""
+    # The encrypted format is:
+    #    "GKEY1"  -- 5 octets, fixed, denotes data format.
+    #    SALT     -- 17 bytes, used to hash password
+    #    IV       -- 16 bytes; salt for encryption
+    #    ENCRYPTED IN AES256-OFB, using a key=s2k(password, salt) and IV=IV:
+    #       SLEN   -- 4 bytes; length of secret, big-endian.
+    #       SECRET -- len(secret) bytes
+    #       D      -- 32 bytes; SHA256 hash of (salt|secret|salt).
+    #
+    # This format leaks the secret length, obviously.
+    assert 0 <= difficulty < 256
+    salt = os.urandom(SALTLEN)+chr(difficulty)
+    key = secretToKey(salt, password)
+
+    d_obj = Crypto.Hash.SHA256.new()
+    d_obj.update(salt)
+    d_obj.update(secret)
+    d_obj.update(salt)
+    d = d_obj.digest()
+
+    iv = os.urandom(16)
+    e = Crypto.Cipher.AES.new(key, Crypto.Cipher.AES.MODE_OFB, iv)
+
+    # Stupidly, pycrypto doesn't accept that stream ciphers don't need to
+    # take their input in blocks.  So pad it, then ignore the padded output.
+
+    padlen = 16-((len(secret)+len(d)+4) % 16)
+    if padlen == 16: padlen = 0
+    pad = '\x00' * padlen
+
+    slen = struct.pack("!L",len(secret))
+    encrypted = e.encrypt("%s%s%s%s" % (slen, secret, d, pad))[:-padlen]
+    return "GKEY1%s%s%s"%(salt, iv, encrypted)
+
+def decryptSecret(encrypted, password):
+    if encrypted[:5] != "GKEY1":
+        raise UnknownFormat()
+    encrypted = encrypted[5:]
+    if len(encrypted) < SALTLEN+1+16:
+        raise FormatError()
+
+    salt = encrypted[:SALTLEN+1]
+    iv = encrypted[SALTLEN+1:SALTLEN+1+16]
+    encrypted = encrypted[SALTLEN+1+16:]
+
+    key = secretToKey(salt, password)
+
+    e = Crypto.Cipher.AES.new(key, Crypto.Cipher.AES.MODE_OFB, iv)
+    padlen = 16-(len(encrypted) % 16)
+    if padlen == 16: padlen = 0
+    pad = '\x00' * padlen
+
+    decrypted = e.decrypt("%s%s"%(encrypted,pad))
+    slen = struct.unpack("!L", decrypted[:4])[0]
+    secret = decrypted[4:4+slen]
+    hash = decrypted[4+slen:4+slen+Crypto.Hash.SHA256.digest_size]
+
+    d = Crypto.Hash.SHA256.new()
+    d.update(salt)
+    d.update(secret)
+    d.update(salt)
+
+    if d.digest() != hash:
+        print repr(decrypted)
+        raise BadPassword()
+
+    return secret
 
diff --git a/lib/glider/repository.py b/lib/glider/repository.py
new file mode 100644
index 0000000..9ec1dc6
--- /dev/null
+++ b/lib/glider/repository.py
@@ -0,0 +1,121 @@
+
+import sexp.parse
+import sexp.access
+import glider.formats
+
+import os
+import threading
+
+class RepositoryFile:
+    def __init__(self, repository, relativePath, schema,
+                 needRole=None, signedFormat=True, needSigs=1):
+        self._repository = repository
+        self._relativePath = relativePath
+        self._schema = schema
+        self._needRole = needRole
+        self._signedFormat = signedFormat
+        self._needSigs = needSigs
+
+        self._signed_sexpr = None
+        self._main_sexpr = None
+        self._mtime = None
+
+    def getPath(self):
+        return os.path.join(self._repository._root, self._relativePath)
+
+    def _load(self):
+        fname = self.getPath()
+
+        # Propagate OSError
+        f = None
+        fd = os.open(fname, os.O_RDONLY)
+        try:
+            f = os.fdopen(fd, 'r')
+        except:
+            os.close(fd)
+            raise
+        try:
+            mtime = os.fstat(fd).st_mtime
+            content = f.read()
+        finally:
+            f.close()
+
+        signed_sexpr,main_sexpr = self._checkContent(content)
+
+        self._signed_sexpr = signed_sexpr
+        self._main_sexpr = main_sexpr
+        self._mtime = mtime
+
+    def _save(self, content=None):
+        if content == None:
+            content = sexpr.encode
+
+        signed_sexpr,main_sexpr = self._checkContent(content)
+
+        fname = self.getPath()
+        fname_tmp = fname+"_tmp"
+
+        fd = os.open(fname_tmp, os.WRONLY|os.O_CREAT|os.O_TRUNC, 0644)
+        try:
+            os.write(fd, contents)
+        finally:
+            os.close(fd)
+        if sys.platform in ('cygwin', 'win32'):
+            # Win32 doesn't let rename replace an existing file.
+            try:
+                os.unlink(fname)
+            except OSError:
+                pass
+        os.rename(fname_tmp, fname)
+
+        self._signed_sexpr = signed_sexpr
+        self._main_sexpr = main_sexpr
+        self._mtime = mtime
+
+    def _checkContent(self, content):
+        sexpr = sexp.parse.parse(content)
+        if not sexpr:
+            raise ParseError()
+
+        if self._signedFormat:
+            if not glider.formats.SIGNED_SCHEMA.matches(sexpr):
+                raise FormatError()
+
+            sigs = checkSignatures(sexpr, self._repository._keyDB,
+                                   self._needRole, self._relativePath)
+            good = sigs[0]
+            # XXXX If good is too low but unknown is high, we may need
+            # a new key file.
+            if len(good) < 1:
+                raise SignatureError()
+
+            main_sexpr = sexpr[1]
+            signed_sexpr = sexpr
+        else:
+            signed_sexpr = None
+            main_sexpr = sexpr
+
+        if self._schema != None and not self._schema.matches(main_sexpr):
+            raise FormatError()
+
+        return signed_sexpr, main_sexpr
+
+    def load(self):
+        if self._main_sexpr == None:
+            self._load()
+
+class LocalRepository:
+    def __init__(self, root):
+        self._root = root
+        self._keyDB = None
+
+        self._keylistFile = RepositoryFile(
+            self, "meta/keys.txt", glider.formats.KEYLIST_SCHEMA,
+            needRole="master")
+        self._timestampFile = RepositoryFile(
+            self, "meta/timestamp.txt", glider.formats.TIMESTAMP_SCHEMA,
+            needRole="timestamp")
+        self._mirrorlistFile = RepositoryFile(
+            self, "meta/mirrors.txt", glider.formats.MIRRORLIST_SCHEMA,
+            needRole="mirrors")
+
diff --git a/lib/glider/tests.py b/lib/glider/tests.py
index d149678..1c392e8 100644
--- a/lib/glider/tests.py
+++ b/lib/glider/tests.py
@@ -4,14 +4,14 @@ import doctest
 
 import glider.keys
 import glider.formats
+import glider.repository
+
 import glider.tests
 
-class EncodingTest(unittest.TestCase):
-    def testQuotedString(self):
-        self.assertEquals(1,1)
+class EncryptionTest(unittest.TestCase):
+    pass
 
 def suite():
-    import sexp.tests
     suite = unittest.TestSuite()
 
     suite.addTest(doctest.DocTestSuite(glider.formats))
diff --git a/lib/sexp/access.py b/lib/sexp/access.py
index c5182f0..74f5022 100644
--- a/lib/sexp/access.py
+++ b/lib/sexp/access.py
@@ -97,6 +97,21 @@ def s_descendants(s, tags=()):
         s = s[idx]
         idx = 0
 
+def attrs_to_dict(sexpr):
+    """Return a dictionary mapping keys of the attributes in sexpr to
+       their values.  Only the last element in the attribute list counts.
+
+    >>> s = [ 'given-name',
+    ...      ["Tigra", 'Rachel'], ["Bunny", "Elana"] ]
+    >>> attrs_to_dict(s)
+    {'Tigra': ['Rachel'], 'Bunny': ['Elana']}
+    """
+    result = {}
+    for ch in sexpr:
+        tag = s_tag(ch)
+        if tag is not None:
+            result[tag]=ch[1:]
+    return result
 
 class SExpr(list):
     """Wraps an s-expresion list to return its tagged children as attributes.
@@ -177,9 +192,9 @@ def _s_lookup_all(s, path, callback):
                 if s_tag(ch) == p_item[2:]:
                     _s_lookup_all(ch, path[p_idx+1:], callback)
         else:
-            s = s_child(s, p_item)
-            if s is None:
-                return
+            for ch in s_children(s, p_item):
+                _s_lookup_all(ch, path[p_idx+1:], callback)
+            return
 
     callback(s)
 
@@ -190,7 +205,9 @@ def s_lookup_all(s, path):
     >>> x = ['alice',
     ...           ['father', 'bob', ['mother', 'carol'], ['father', 'dave']],
     ...           ['mother', 'eve', ['mother', 'frances', ['dog', 'spot']],
-    ...                             ['father', 'gill']]]
+    ...                             ['father', 'gill']],
+    ...           ['marmoset', 'tiffany'],
+    ...           ['marmoset', 'gilbert']  ]
     >>> s_lookup_all(x, "father")
     [['father', 'bob', ['mother', 'carol'], ['father', 'dave']]]
     >>> s_lookup_all(x, "father.mother")
@@ -203,6 +220,8 @@ def s_lookup_all(s, path):
     [['dog', 'spot']]
     >>> s_lookup_all(x, "mother.*.dog")
     [['dog', 'spot']]
+    >>> s_lookup_all(x, "marmoset")
+    [['marmoset', 'tiffany'], ['marmoset', 'gilbert']]
     """
     result = []
     _s_lookup_all(s, path, result.append)
diff --git a/lib/sexp/tests.py b/lib/sexp/tests.py
index 7decd03..bfd1053 100644
--- a/lib/sexp/tests.py
+++ b/lib/sexp/tests.py
@@ -11,7 +11,6 @@ class EncodingTest(unittest.TestCase):
         self.assertEquals(1,1)
 
 
-
 def suite():
     import sexp.tests
     suite = unittest.TestSuite()