Merge remote-tracking branch 'chiiph/bug/migrate_couch' into develop

14 files changed, 112 insertions, 71 deletions

diff --git a/CHANGELOG b/CHANGELOG
new file mode 100644
index 0000000..0c449c6
--- /dev/null
+++ b/CHANGELOG
@@ -0,0 +1,10 @@
+0.3.0 Aug 9:
+  o Give a return code for bare usernames too. Closes: #3405
+  o Adapt to Soledad 0.2.1 API.
+  o Fix broken pip install
+  o Fix alias lookup return codes. Closes: #3356
+  o Use the new API to import a new ascii armored public key. Fixes
+    #3352.
+  o Add "incoming" index for ease of listing new email.
+  o Add dependency for leap.keymanager.
+  o Adapt to the new keymanager API. Closes #3397.
diff --git a/VERSION.md b/VERSION.md
index 43f6976..f29cb7b 100644
--- a/VERSION.md
+++ b/VERSION.md
@@ -1,7 +1,7 @@
 
-[leap_mx, version 0.2.2]
+[leap_mx, version 0.3.0]
 ------------------------
-Authors: 	Isis Agora Lovecruft, <isis@leap.se> 0x2cdb8b35
+Authors: 	The LEAP Team
 Website: 	https://leap.se
-Github: 	https://github.com/isislovecruft/leap_mx/
+Github: 	https://github.com/leapcode/leap_mx/
 
diff --git a/changes/bug_adapt-to-latest-soledad-api b/changes/bug_adapt-to-latest-soledad-api
deleted file mode 100644
index 95f0deb..0000000
--- a/changes/bug_adapt-to-latest-soledad-api
+++ /dev/null
@@ -1 +0,0 @@
-  o Adapt to Soledad 0.2.1 API.
diff --git a/changes/bug_fix-pip-install b/changes/bug_fix-pip-install
deleted file mode 100644
index fcb5829..0000000
--- a/changes/bug_fix-pip-install
+++ /dev/null
@@ -1 +0,0 @@
-  o Fix broken pip install
diff --git a/changes/bug_fix_return_codes b/changes/bug_fix_return_codes
deleted file mode 100644
index 9debe44..0000000
--- a/changes/bug_fix_return_codes
+++ /dev/null
@@ -1 +0,0 @@
-  o Fix alias lookup return codes. Closes: #3356
diff --git a/changes/bug_importascii b/changes/bug_importascii
deleted file mode 100644
index 902df95..0000000
--- a/changes/bug_importascii
+++ /dev/null
@@ -1,3 +0,0 @@
-  o Use the new API to import a new ascii armored public key. Fixes
-    #3352.
-  o Add "incoming" index for ease of listing new email.
\ No newline at end of file
diff --git a/changes/bug_new_couch_structure b/changes/bug_new_couch_structure
new file mode 100644
index 0000000..eea0ced
--- /dev/null
+++ b/changes/bug_new_couch_structure
@@ -0,0 +1,2 @@
+  o Migrate mx functions to work on the new couchdb structure and
+    views. Fixes #3502.
\ No newline at end of file
diff --git a/changes/feature_add-dependency-for-keymanager b/changes/feature_add-dependency-for-keymanager
deleted file mode 100644
index 0ac1c2a..0000000
--- a/changes/feature_add-dependency-for-keymanager
+++ /dev/null
@@ -1 +0,0 @@
-  o Add dependency for leap.keymanager.
diff --git a/setup.py b/setup.py
index ce7c46b..a0cbbae 100644
--- a/setup.py
+++ b/setup.py
@@ -48,7 +48,7 @@ else:
                   ("/etc/init.d/", ["pkg/leap_mx"])]
 setup(
     name='leap.mx',
-    version="0.2.2",
+    version="0.3.0",
     # bump also src/leap/mx/__init__
     url="http://github.com/leapcode/leap_mx",
     license='AGPLv3+',
diff --git a/src/leap/mx/__init__.py b/src/leap/mx/__init__.py
index 61c9a5c..e63479c 100644
--- a/src/leap/mx/__init__.py
+++ b/src/leap/mx/__init__.py
@@ -17,4 +17,4 @@
 """
 Module initialization file for leap.mx .
 """
-__version__ = "0.2.2"
+__version__ = "0.3.0"
diff --git a/src/leap/mx/alias_resolver.py b/src/leap/mx/alias_resolver.py
index 1d478c7..45a3ed2 100644
--- a/src/leap/mx/alias_resolver.py
+++ b/src/leap/mx/alias_resolver.py
@@ -19,6 +19,8 @@
 """
 Classes for resolving postfix aliases.
 
+Test this with postmap -v -q "foo" tcp:localhost:4242
+
 TODO:
     o Look into using twisted.protocols.postfix.policies classes for
       controlling concurrent connections and throttling resource consumption.
@@ -35,13 +37,27 @@ except ImportError:
     print "for instructions on getting required dependencies."
 
 
+class LEAPPostFixTCPMapserver(postfix.PostfixTCPMapServer):
+    def _cbGot(self, value):
+        if value is None:
+            self.sendCode(500, postfix.quote("NOT FOUND SRY"))
+        else:
+            self.sendCode(200, postfix.quote(value))
+
+
 class AliasResolverFactory(postfix.PostfixTCPMapDeferringDictServerFactory):
+
+    protocol = LEAPPostFixTCPMapserver
+
     def __init__(self, couchdb, *args, **kwargs):
         postfix.PostfixTCPMapDeferringDictServerFactory.__init__(
             self, *args, **kwargs)
         self._cdb = couchdb
 
     def _to_str(self, result):
+        """
+        Properly encodes the result string if any.
+        """
         if isinstance(result, unicode):
             result = result.encode("utf8")
         if result is None:
@@ -49,24 +65,25 @@ class AliasResolverFactory(postfix.PostfixTCPMapDeferringDictServerFactory):
         return result
 
     def spit_result(self, result):
+        """
+        Formats the return codes in a postfix friendly format.
+        """
         if result is None:
-            return defer.succeed("500 NO RESULT")
+            return None
         else:
-            return defer.succeed("200")
+            return defer.succeed(result)
 
     def get(self, key):
+        """
+        Looks up the passed key, but only up to the username id of the key.
+
+        At some point we will have to consider the domain part too.
+        """
         try:
-            log.msg("Processing key: %s" % (key,))
-            if key.find("@") == -1:
-                log.msg("Ignoring key since it's not an email address")
-                return None
-            key = key.split("@")[0]
-            key = key.split("+")[0]
-            log.msg("Final key to query: %s" % (key,))
-            d = self._cdb.queryByLoginOrAlias(key)
-            d.addCallback(self._to_str)
-            d.addErrback(log.err)
+            log.msg("Query key: %s" % (key,))
+            d = self._cdb.queryByAddress(key)
 
+            d.addCallback(self._to_str)
             d.addCallback(self.spit_result)
             d.addErrback(log.err)
             return d
diff --git a/src/leap/mx/check_recipient_access.py b/src/leap/mx/check_recipient_access.py
index b75d1eb..0520c7c 100644
--- a/src/leap/mx/check_recipient_access.py
+++ b/src/leap/mx/check_recipient_access.py
@@ -18,6 +18,8 @@
 
 """
 Classes for resolving postfix recipient access
+
+Test this with postmap -v -q "foo" tcp:localhost:2244
 """
 
 from twisted.protocols import postfix
@@ -25,9 +27,14 @@ from twisted.protocols import postfix
 from leap.mx.alias_resolver import AliasResolverFactory
 
 
-class CheckRecipientAccess(postfix.PostfixTCPMapServer):
-    pass
+class LEAPPostFixTCPMapserverAccess(postfix.PostfixTCPMapServer):
+    def _cbGot(self, value):
+        if value is None:
+            self.sendCode(500, postfix.quote("NOT FOUND SORRY"))
+        else:
+            # We do not send the value in this case
+            self.sendCode(200)
 
 
 class CheckRecipientAccessFactory(AliasResolverFactory):
-    protocol = CheckRecipientAccess
+    protocol = LEAPPostFixTCPMapserverAccess
diff --git a/src/leap/mx/couchdbhelper.py b/src/leap/mx/couchdbhelper.py
index 02ef088..147e6f9 100644
--- a/src/leap/mx/couchdbhelper.py
+++ b/src/leap/mx/couchdbhelper.py
@@ -97,7 +97,7 @@ class ConnectedCouchDB(client.CouchDB):
         """
         pass
 
-    def queryByLoginOrAlias(self, alias):
+    def queryByAddress(self, address):
         """
         Check to see if a particular email or alias exists.
 
@@ -106,47 +106,56 @@ class ConnectedCouchDB(client.CouchDB):
         @return: a deferred for this query
         @rtype twisted.defer.Deferred
         """
-        assert isinstance(alias, str), "Email or alias queries must be string"
+        assert isinstance(address, (str, unicode)), "Email or alias queries must be string"
 
         # TODO: Cache results
 
-        d = self.openView(docId="User",
-                          viewId="by_login_or_alias/",
-                          key=alias,
+        d = self.openView(docId="Identity",
+                          viewId="by_address/",
+                          key=address,
                           reduce=False,
                           include_docs=True)
 
-        d.addCallbacks(partial(self._get_uuid, alias), log.err)
+        d.addCallbacks(partial(self._get_uuid, address), log.err)
 
         return d
 
-    def _get_uuid(self, alias, result):
+    def _get_uuid(self, address, result):
         """
-        Parses the result of the by_login_or_alias query and gets the
-        uuid
+        Parses the result of the by_address query and gets the uuid
 
-        @param alias: alias looked up
-        @type alias: string
+        @param address: alias looked up
+        @type address: string
         @param result: result dictionary
         @type result: dict
         @return: The uuid for alias if available
         @rtype: str
         """
         for row in result["rows"]:
-            if row["key"] == alias:
-                uuid = row["id"]
-                self._cache[uuid] = row["doc"].get("public_key", None)
+            if row["key"] == address:
+                uuid = row["doc"].get("user_id", None)
+                if uuid is None:
+                    log.msg("ERROR: Found doc for %s but there's not user_id!"
+                            % (address,))
                 return uuid
         return None
 
-    def getPubKey(self, uuid):
-        pubkey = None
-        try:
-            pubkey = self._cache[uuid]
-        except:
-            pass
-        return pubkey
+    def getPubKey(self, address):
+        d = self.openView(docId="Identity",
+                          viewId="pgp_key_by_email/",
+                          key=address,
+                          reduce=False,
+                          include_docs=True)
 
+        d.addCallbacks(partial(self._get_pgp_key, address), log.err)
+
+        return d
+
+    def _get_pgp_key(self, address, result):
+        for row in result["rows"]:
+            if row["key"] == address:
+                return row["value"]
+        return None
 
 if __name__ == "__main__":
     from twisted.internet import reactor
diff --git a/src/leap/mx/mail_receiver.py b/src/leap/mx/mail_receiver.py
index 43925df..ff2caa9 100644
--- a/src/leap/mx/mail_receiver.py
+++ b/src/leap/mx/mail_receiver.py
@@ -29,6 +29,7 @@ from email import message_from_string
 
 from twisted.application.service import Service
 from twisted.internet import inotify
+from twisted.internet.defer import DeferredList
 from twisted.python import filepath, log
 
 from leap.soledad.document import SoledadDocument
@@ -76,18 +77,20 @@ class MailReceiver(Service):
                      callbacks=[self._process_incoming_email],
                      recursive=recursive)
 
-    def _get_pubkey(self, uuid):
-        """
-        Given a UUID for a user, retrieve its public key
+    def _gather_uuid_pubkey(self, results):
+        if len(results) < 2:
+            return None, None
 
-        @param uuid: UUID for a user
-        @type uuid: str
+        # DeferredList results are structured like this:
+        # [ (succeeded, pubkey), (succeeded, uuid) ]
+        # succeeded is a bool value that specifies if the
+        # corresponding callback succeeded
+        pubkey_res, uuid_res = results
 
-        @return: uuid, public key
-        @rtype: tuple of (str, str)
-        """
-        log.msg("Fetching pubkey for %s" % (uuid,))
-        return uuid, self._users_cdb.getPubKey(uuid)
+        pubkey = pubkey_res[1] if pubkey_res[0] else None
+        uuid = uuid_res[1] if uuid_res[0] else None
+
+        return uuid, pubkey
 
     def _encrypt_message(self, uuid_pubkey, address, message):
         """
@@ -123,17 +126,19 @@ class MailReceiver(Service):
             return uuid, doc
 
         openpgp_key = None
-        with openpgp.temporary_gpgwrapper() as gpg:
+        with openpgp.TempGPGWrapper(gpgbinary='/usr/bin/gpg') as gpg:
             gpg.import_keys(pubkey)
             key = gpg.list_keys().pop()
             openpgp_key = openpgp._build_key_from_gpg(address, key, pubkey)
 
-        doc.content = {
-            "incoming": True,
-            "_enc_scheme": EncryptionSchemes.PUBKEY,
-            "_enc_json": openpgp.encrypt_asym(json.dumps(data),
-                                              openpgp_key)
-        }
+            doc.content = {
+                "incoming": True,
+                "_enc_scheme": EncryptionSchemes.PUBKEY,
+                "_enc_json": str(gpg.encrypt(
+                    json.dumps(data),
+                    openpgp_key.fingerprint,
+                    symmetric=False))
+            }
 
         return uuid, doc
 
@@ -157,7 +162,6 @@ class MailReceiver(Service):
             uuid = 0
 
         db = CouchDatabase(self._mail_couch_url, "user-%s" % (uuid,))
-        db.put_doc(doc)
 
         log.msg("Done exporting")
 
@@ -206,17 +210,16 @@ class MailReceiver(Service):
                 if owner is None:
                     log.err("Malformed mail, neither To: nor "
                             "Delivered-To: field")
-                owner = owner.split("@")[0]
-                owner = owner.split("+")[0]
                 log.msg("Mail owner: %s" % (owner,))
 
                 log.msg("%s received a new mail" % (owner,))
-                d = self._users_cdb.queryByLoginOrAlias(owner)
-                d.addCallbacks(self._get_pubkey, log.err)
+                dpubk = self._users_cdb.getPubKey(owner)
+                duuid = self._users_cdb.queryByAddress(owner)
+                d = DeferredList([dpubk, duuid])
+                d.addCallbacks(self._gather_uuid_pubkey, log.err)
                 d.addCallbacks(self._encrypt_message, log.err,
                                (owner, mail_data))
                 d.addCallbacks(self._export_message, log.err)
                 d.addCallbacks(self._conditional_remove, log.err,
                                (filepath,))
                 d.addErrback(log.err)
-