diff options
author | Nick Mathewson <nickm@torproject.org> | 2008-09-19 19:22:43 +0000 |
---|---|---|
committer | Nick Mathewson <nickm@torproject.org> | 2008-09-19 19:22:43 +0000 |
commit | 047f0117bf3ccf5b7e1a36b760a5c75f988bc72e (patch) | |
tree | e88519e576f0b671a601427d09ebf695dba9fce2 | |
parent | ddc03061218dc00a664aaf10a6a2fec2b604deac (diff) |
More repository/signing/marshalling/crypto work for glider.
git-svn-id: file:///home/or/svnrepo/updater/trunk@16928 55e972cd-5a19-0410-ae62-a4d7a52db4cd
-rw-r--r-- | Makefile | 6 | ||||
-rw-r--r-- | TODO | 6 | ||||
-rw-r--r-- | lib/glider/formats.py | 86 | ||||
-rw-r--r-- | lib/glider/keys.py | 138 | ||||
-rw-r--r-- | lib/glider/repository.py | 121 | ||||
-rw-r--r-- | lib/glider/tests.py | 8 | ||||
-rw-r--r-- | lib/sexp/access.py | 27 | ||||
-rw-r--r-- | lib/sexp/tests.py | 1 |
8 files changed, 372 insertions, 21 deletions
diff --git a/Makefile b/Makefile new file mode 100644 index 0000000..0d58eb4 --- /dev/null +++ b/Makefile @@ -0,0 +1,6 @@ + +export PYTHONPATH=./lib + +test: + python -m sexp.tests + python -m glider.tests @@ -5,9 +5,11 @@ o Write spec . Write server-side code (python) o S-expression lib . Code to manage data formats + . Parse + o Validate - Code to wrangle private keys - - Generate - - Store, load (password-protected) + o Generate + . Store, load (password-protected) - Print for posterity - Code to generate timestamp files diff --git a/lib/glider/formats.py b/lib/glider/formats.py index 6ccd466..34ece7d 100644 --- a/lib/glider/formats.py +++ b/lib/glider/formats.py @@ -48,6 +48,13 @@ def checkSignatures(signed, keyDB, role, path): unknownSigs = [] tangentialSigs = [] + assert signed[0] == "signed" + data = signed[1] + + d_obj = Crypto.Hash.SHA256.new() + sexp.encode.hash_canonical(data, d_obj) + digest = d_obj.digest() + for signature in sexp.access.s_children(signed, "signature"): attrs = signature[1] sig = attrs[2] @@ -59,7 +66,7 @@ def checkSignatures(signed, keyDB, role, path): continue method = s_child(attrs, "method")[1] try: - result = key.checkSignature(method, data, sig) + result = key.checkSignature(method, sig, digest=digest) except UnknownMethod: continue if result == True: @@ -75,6 +82,8 @@ def checkSignatures(signed, keyDB, role, path): else: badSigs.append(keyid) + return goodSigs, badSigs, unknownSigs, tangentialSigs + def sign(signed, key): assert sexp.access.s_tag(signed) == 'signed' s = signed[1] @@ -129,7 +138,7 @@ SIGNED_TEMPLATE = r""" SIGNED_SCHEMA = _parseSchema(SIGNED_TEMPLATE, SCHEMA_TABLE) -KEYFILE_TEMPLATE = r""" +KEYLIST_TEMPLATE = r""" (=keylist (=ts .TIME) (=keys @@ -139,7 +148,7 @@ KEYFILE_TEMPLATE = r""" * )""" -KEYFILE_SCHEMA = _parseSchema(KEYFILE_TEMPLATE, SCHEMA_TABLE) +KEYLIST_SCHEMA = _parseSchema(KEYLIST_TEMPLATE, SCHEMA_TABLE) MIRRORLIST_TEMPLATE = r""" (=mirrorlist @@ -194,3 +203,74 @@ PACKAGE_TEMPLATE = r""" """ PACKAGE_SCHEMA = _parseSchema(PACKAGE_TEMPLATE, SCHEMA_TABLE) + +ALL_ROLES = ('timestamp', 'mirrors', 'bundle', 'package', 'master') + +class Key: + def __init__(self, key, roles): + self.key = key + self.roles = [] + for r,p in roles: + self.addRole(r,p) + + def addRole(self, role, path): + assert role in ALL_ROLES + self.roles.append(role, path) + + def getRoles(self): + return self.rules + + @staticmethod + def fromSExpression(sexpr): + # must match PUBKEY_SCHEMA + typeattr = sexp.access.s_attr(sexpr[1], "type") + if typeattr == 'rsa': + key = glider.keys.RSAKey.fromSExpression(sexpr) + if key is not None: + return Key(key) + else: + return None + + def format(self): + return self.key.format() + + def getKeyID(self): + return self.key.getKeyID() + + def sign(self, sexpr=None, digest=None): + return self.key.sign(sexpr, digest=digest) + + def checkSignature(self, method, sexpr=None, digest=None): + if digest == None: + _, digest = self.key._digest(sexpr, method) + ok = self.key.checkSignature(method, digest=digest) + # XXXX CACHE HERE. + return ok + +class Keystore(KeyDB): + def __init__(self): + KeyDB.__init__(self) + + @staticmethod + def addFromKeylist(sexpr, allowMasterKeys=False): + # Don't do this until we have validated the structure. + for ks in sexpr.access.s_lookup_all("keys.key"): + attrs = ks[1] + key_s = ks[2] + roles = s_attr(attrs, "roles") + #XXXX Use interface of Key, not RSAKey. + key = Key.fromSExpression(key_s) + if not key: + #LOG skipping key. + continue + for r,p in roles: + if r == 'master' and not allowMasterKeys: + #LOG + continue + if r not in ALL_ROLES: + continue + key.addRole(r,p) + + self.addKey(key) + + diff --git a/lib/glider/keys.py b/lib/glider/keys.py index 9fb73ec..e2b2736 100644 --- a/lib/glider/keys.py +++ b/lib/glider/keys.py @@ -2,13 +2,16 @@ # These require PyCrypto. import Crypto.PublicKey.RSA import Crypto.Hash.SHA256 +import Crypto.Cipher.AES import sexp.access import sexp.encode import sexp.parse +import cPickle as pickle import binascii import os +import struct class CryptoError(Exception): pass @@ -134,26 +137,147 @@ class RSAKey(PublicKey): self.keyid = ("rsa", d_obj.digest()) return self.keyid - def sign(self, sexpr=None, digest=None): - assert _xor(sexpr == None, digest == None) - if digest == None: + def _digest(self, sexpr, method=None): + if method in (None, "sha256-pkcs1"): d_obj = Crypto.Hash.SHA256.new() sexp.encode.hash_canonical(sexpr, d_obj) digest = d_obj.digest() + return ("sha256-pkcs1", digest) + + raise UnknownMethod(method) + + def sign(self, sexpr=None, digest=None): + assert _xor(sexpr == None, digest == None) + if digest == None: + method, digest = self._digest(sexpr) m = _pkcs1_padding(digest, (self.key.size()+1) // 8) sig = intToBinary(self.key.sign(m, "")[0]) - return ("sha256-pkcs1", sig) + return (method, sig) def checkSignature(self, method, sig, sexpr=None, digest=None): assert _xor(sexpr == None, digest == None) if method != "sha256-pkcs1": raise UnknownMethod("method") if digest == None: - d_obj = Crypto.Hash.SHA256.new() - sexp.encode.hash_canonical(sexpr, d_obj) - digest = d_obj.digest() + method, digest = self._digest(sexpr, method) sig = binaryToInt(sig) m = _pkcs1_padding(digest, (self.key.size()+1) // 8) return self.key.verify(m, (sig,)) +SALTLEN=16 + +def secretToKey(salt, secret): + """Convert 'secret' to a 32-byte key, using a version of the algorithm + from RFC2440. The salt must be SALTLEN+1 bytes long, and should + be random, except for the last byte, which encodes how time- + consuming the computation should be. + + (The goal is to make offline password-guessing attacks harder by + increasing the time required to convert a password to a key, and to + make precomputed password tables impossible to generate by ) + """ + assert len(salt) == SALTLEN+1 + + # The algorithm is basically, 'call the last byte of the salt the + # "difficulty", and all other bytes of the salt S. Now make + # an infinite stream of S|secret|S|secret|..., and hash the + # first N bytes of that, where N is determined by the difficulty. + # + # Obviously, this wants a hash algorithm that's tricky to + # parallelize. + # + # Unlike RFC2440, we use a 16-byte salt. Because CPU times + # have improved, we start at 16 times the previous minimum. + + difficulty = ord(salt[-1]) + count = (16L+(difficulty & 15)) << ((difficulty >> 4) + 10) + + # Make 'data' nice and long, so that we don't need to call update() + # a zillion times. + data = salt[:-1]+secret + if len(data)<1024: + data *= (1024 // len(data))+1 + + d = Crypto.Hash.SHA256.new() + iters, leftover = divmod(count, len(data)) + for _ in xrange(iters): + d.update(data) + #count -= len(data) + if leftover: + d.update(data[:leftover]) + #count -= leftover + #assert count == 0 + + return d.digest() + +def encryptSecret(secret, password, difficulty=0x80): + """Encrypt the secret 'secret' using the password 'password', + and return the encrypted result.""" + # The encrypted format is: + # "GKEY1" -- 5 octets, fixed, denotes data format. + # SALT -- 17 bytes, used to hash password + # IV -- 16 bytes; salt for encryption + # ENCRYPTED IN AES256-OFB, using a key=s2k(password, salt) and IV=IV: + # SLEN -- 4 bytes; length of secret, big-endian. + # SECRET -- len(secret) bytes + # D -- 32 bytes; SHA256 hash of (salt|secret|salt). + # + # This format leaks the secret length, obviously. + assert 0 <= difficulty < 256 + salt = os.urandom(SALTLEN)+chr(difficulty) + key = secretToKey(salt, password) + + d_obj = Crypto.Hash.SHA256.new() + d_obj.update(salt) + d_obj.update(secret) + d_obj.update(salt) + d = d_obj.digest() + + iv = os.urandom(16) + e = Crypto.Cipher.AES.new(key, Crypto.Cipher.AES.MODE_OFB, iv) + + # Stupidly, pycrypto doesn't accept that stream ciphers don't need to + # take their input in blocks. So pad it, then ignore the padded output. + + padlen = 16-((len(secret)+len(d)+4) % 16) + if padlen == 16: padlen = 0 + pad = '\x00' * padlen + + slen = struct.pack("!L",len(secret)) + encrypted = e.encrypt("%s%s%s%s" % (slen, secret, d, pad))[:-padlen] + return "GKEY1%s%s%s"%(salt, iv, encrypted) + +def decryptSecret(encrypted, password): + if encrypted[:5] != "GKEY1": + raise UnknownFormat() + encrypted = encrypted[5:] + if len(encrypted) < SALTLEN+1+16: + raise FormatError() + + salt = encrypted[:SALTLEN+1] + iv = encrypted[SALTLEN+1:SALTLEN+1+16] + encrypted = encrypted[SALTLEN+1+16:] + + key = secretToKey(salt, password) + + e = Crypto.Cipher.AES.new(key, Crypto.Cipher.AES.MODE_OFB, iv) + padlen = 16-(len(encrypted) % 16) + if padlen == 16: padlen = 0 + pad = '\x00' * padlen + + decrypted = e.decrypt("%s%s"%(encrypted,pad)) + slen = struct.unpack("!L", decrypted[:4])[0] + secret = decrypted[4:4+slen] + hash = decrypted[4+slen:4+slen+Crypto.Hash.SHA256.digest_size] + + d = Crypto.Hash.SHA256.new() + d.update(salt) + d.update(secret) + d.update(salt) + + if d.digest() != hash: + print repr(decrypted) + raise BadPassword() + + return secret diff --git a/lib/glider/repository.py b/lib/glider/repository.py new file mode 100644 index 0000000..9ec1dc6 --- /dev/null +++ b/lib/glider/repository.py @@ -0,0 +1,121 @@ + +import sexp.parse +import sexp.access +import glider.formats + +import os +import threading + +class RepositoryFile: + def __init__(self, repository, relativePath, schema, + needRole=None, signedFormat=True, needSigs=1): + self._repository = repository + self._relativePath = relativePath + self._schema = schema + self._needRole = needRole + self._signedFormat = signedFormat + self._needSigs = needSigs + + self._signed_sexpr = None + self._main_sexpr = None + self._mtime = None + + def getPath(self): + return os.path.join(self._repository._root, self._relativePath) + + def _load(self): + fname = self.getPath() + + # Propagate OSError + f = None + fd = os.open(fname, os.O_RDONLY) + try: + f = os.fdopen(fd, 'r') + except: + os.close(fd) + raise + try: + mtime = os.fstat(fd).st_mtime + content = f.read() + finally: + f.close() + + signed_sexpr,main_sexpr = self._checkContent(content) + + self._signed_sexpr = signed_sexpr + self._main_sexpr = main_sexpr + self._mtime = mtime + + def _save(self, content=None): + if content == None: + content = sexpr.encode + + signed_sexpr,main_sexpr = self._checkContent(content) + + fname = self.getPath() + fname_tmp = fname+"_tmp" + + fd = os.open(fname_tmp, os.WRONLY|os.O_CREAT|os.O_TRUNC, 0644) + try: + os.write(fd, contents) + finally: + os.close(fd) + if sys.platform in ('cygwin', 'win32'): + # Win32 doesn't let rename replace an existing file. + try: + os.unlink(fname) + except OSError: + pass + os.rename(fname_tmp, fname) + + self._signed_sexpr = signed_sexpr + self._main_sexpr = main_sexpr + self._mtime = mtime + + def _checkContent(self, content): + sexpr = sexp.parse.parse(content) + if not sexpr: + raise ParseError() + + if self._signedFormat: + if not glider.formats.SIGNED_SCHEMA.matches(sexpr): + raise FormatError() + + sigs = checkSignatures(sexpr, self._repository._keyDB, + self._needRole, self._relativePath) + good = sigs[0] + # XXXX If good is too low but unknown is high, we may need + # a new key file. + if len(good) < 1: + raise SignatureError() + + main_sexpr = sexpr[1] + signed_sexpr = sexpr + else: + signed_sexpr = None + main_sexpr = sexpr + + if self._schema != None and not self._schema.matches(main_sexpr): + raise FormatError() + + return signed_sexpr, main_sexpr + + def load(self): + if self._main_sexpr == None: + self._load() + +class LocalRepository: + def __init__(self, root): + self._root = root + self._keyDB = None + + self._keylistFile = RepositoryFile( + self, "meta/keys.txt", glider.formats.KEYLIST_SCHEMA, + needRole="master") + self._timestampFile = RepositoryFile( + self, "meta/timestamp.txt", glider.formats.TIMESTAMP_SCHEMA, + needRole="timestamp") + self._mirrorlistFile = RepositoryFile( + self, "meta/mirrors.txt", glider.formats.MIRRORLIST_SCHEMA, + needRole="mirrors") + diff --git a/lib/glider/tests.py b/lib/glider/tests.py index d149678..1c392e8 100644 --- a/lib/glider/tests.py +++ b/lib/glider/tests.py @@ -4,14 +4,14 @@ import doctest import glider.keys import glider.formats +import glider.repository + import glider.tests -class EncodingTest(unittest.TestCase): - def testQuotedString(self): - self.assertEquals(1,1) +class EncryptionTest(unittest.TestCase): + pass def suite(): - import sexp.tests suite = unittest.TestSuite() suite.addTest(doctest.DocTestSuite(glider.formats)) diff --git a/lib/sexp/access.py b/lib/sexp/access.py index c5182f0..74f5022 100644 --- a/lib/sexp/access.py +++ b/lib/sexp/access.py @@ -97,6 +97,21 @@ def s_descendants(s, tags=()): s = s[idx] idx = 0 +def attrs_to_dict(sexpr): + """Return a dictionary mapping keys of the attributes in sexpr to + their values. Only the last element in the attribute list counts. + + >>> s = [ 'given-name', + ... ["Tigra", 'Rachel'], ["Bunny", "Elana"] ] + >>> attrs_to_dict(s) + {'Tigra': ['Rachel'], 'Bunny': ['Elana']} + """ + result = {} + for ch in sexpr: + tag = s_tag(ch) + if tag is not None: + result[tag]=ch[1:] + return result class SExpr(list): """Wraps an s-expresion list to return its tagged children as attributes. @@ -177,9 +192,9 @@ def _s_lookup_all(s, path, callback): if s_tag(ch) == p_item[2:]: _s_lookup_all(ch, path[p_idx+1:], callback) else: - s = s_child(s, p_item) - if s is None: - return + for ch in s_children(s, p_item): + _s_lookup_all(ch, path[p_idx+1:], callback) + return callback(s) @@ -190,7 +205,9 @@ def s_lookup_all(s, path): >>> x = ['alice', ... ['father', 'bob', ['mother', 'carol'], ['father', 'dave']], ... ['mother', 'eve', ['mother', 'frances', ['dog', 'spot']], - ... ['father', 'gill']]] + ... ['father', 'gill']], + ... ['marmoset', 'tiffany'], + ... ['marmoset', 'gilbert'] ] >>> s_lookup_all(x, "father") [['father', 'bob', ['mother', 'carol'], ['father', 'dave']]] >>> s_lookup_all(x, "father.mother") @@ -203,6 +220,8 @@ def s_lookup_all(s, path): [['dog', 'spot']] >>> s_lookup_all(x, "mother.*.dog") [['dog', 'spot']] + >>> s_lookup_all(x, "marmoset") + [['marmoset', 'tiffany'], ['marmoset', 'gilbert']] """ result = [] _s_lookup_all(s, path, result.append) diff --git a/lib/sexp/tests.py b/lib/sexp/tests.py index 7decd03..bfd1053 100644 --- a/lib/sexp/tests.py +++ b/lib/sexp/tests.py @@ -11,7 +11,6 @@ class EncodingTest(unittest.TestCase): self.assertEquals(1,1) - def suite(): import sexp.tests suite = unittest.TestSuite() |