diff options
Diffstat (limited to 'service')
21 files changed, 388 insertions, 213 deletions
diff --git a/service/pixelated/adapter/services/mail_service.py b/service/pixelated/adapter/services/mail_service.py index 03889f82..656bb4bc 100644 --- a/service/pixelated/adapter/services/mail_service.py +++ b/service/pixelated/adapter/services/mail_service.py @@ -97,6 +97,11 @@ class MailService(object): trashed_mail = self.mailboxes.move_to_trash(mail_id) self.search_engine.index_mail(trashed_mail) + def recover_mail(self, mail_id): + mail = self.mail(mail_id) + recovered_mail = self.mailboxes.move_to_inbox(mail_id) + self.search_engine.index_mail(recovered_mail) + def delete_permanent(self, mail_id): mail = self.mail(mail_id) self.search_engine.remove_from_index(mail_id) diff --git a/service/pixelated/adapter/services/mailboxes.py b/service/pixelated/adapter/services/mailboxes.py index a7a3a591..e9fe6ce5 100644 --- a/service/pixelated/adapter/services/mailboxes.py +++ b/service/pixelated/adapter/services/mailboxes.py @@ -49,8 +49,14 @@ class Mailboxes(object): return [self._create_or_get(leap_mailbox_name) for leap_mailbox_name in self.account.mailboxes] def move_to_trash(self, mail_id): + return self._move_to(mail_id, self.trash()) + + def move_to_inbox(self, mail_id): + return self._move_to(mail_id, self.inbox()) + + def _move_to(self, mail_id, mailbox): mail = self.querier.mail(mail_id) - mail.set_mailbox(self.trash().mailbox_name) + mail.set_mailbox(mailbox.mailbox_name) mail.save() return mail diff --git a/service/pixelated/certificates/dev.pixelated-project.org.ca.crt b/service/pixelated/certificates/dev.pixelated-project.org.ca.crt index 8f2219a7..22ccb662 100644 --- a/service/pixelated/certificates/dev.pixelated-project.org.ca.crt +++ b/service/pixelated/certificates/dev.pixelated-project.org.ca.crt @@ -1,33 +1,33 @@ -----BEGIN CERTIFICATE----- -MIIFozCCA4ugAwIBAgIBATANBgkqhkiG9w0BAQ0FADBkMRYwFAYDVQQKDA1kZXYu -cGl4ZWxhdGVkMSowKAYDVQQLDCFodHRwczovL2Rldi5waXhlbGF0ZWQtcHJvamVj -dC5vcmcxHjAcBgNVBAMMFWRldi5waXhlbGF0ZWQgUm9vdCBDQTAeFw0xNTAxMjAw -MDAwMDBaFw0yNTAxMjAwMDAwMDBaMGQxFjAUBgNVBAoMDWRldi5waXhlbGF0ZWQx -KjAoBgNVBAsMIWh0dHBzOi8vZGV2LnBpeGVsYXRlZC1wcm9qZWN0Lm9yZzEeMBwG -A1UEAwwVZGV2LnBpeGVsYXRlZCBSb290IENBMIICIjANBgkqhkiG9w0BAQEFAAOC -Ag8AMIICCgKCAgEAnhlrDaixEGkzugWvwp1pPq4UUq8lWSLe173IHRvuElckYmEy -qTG/UpHx7ZAG/6OLNoCavOO4SEZvYGfGKRpqWGAm8O1acFc7S5GSOJcQQ5UO6bj8 -/32x3/wQAm85T1yUYAgJG0EWExheKMkQVXsRrXuYPsJJ6t3R33nQMUouJCY2toUb -Nu2NAhCZSNxL/77akwznYwtOS2aupznH4nzsxI70y9WlUcSIqlISJW9S65CMnIVq -xgtAxDZ+XOoYrS7Ihh87Dge8eWaUaTOTf+TgyLwxO0v3r64/tjnAV+jDzQsQKnhw -HNNOi2X7E85DzdicB+HOz8mP7G4LiLeCUGGiKQ4CcG7LCMBVcaUWpig7TBQT1PgL -JGwqSw98TD5+81mgU4sjC85HRngGRtoeWkjkLE5jXntFd6S7sUUctJ1CIMJN+W2P -D3g0TQp2mX8OY9zhyX87Lbc/YPY3+OHHtlKm/Gey+LplfWG65i7sPhzWYO/8gmML -nZeuwK4nm21aT3ugonnKiMetmFNDlMH2JxLg+pmw2BnwJe8jNYD6i8wDHTb6mYxU -iMJyU7KPIWXOznNTVwn3c2CBijzfmsbtuyswk7RePpXhHlLzgsw0tb0OlnO3EQKQ -7O4FLZO5erpa5ANb9j4/9MYJlfXQHPvhI2dsr3yv893/7fEK0AeuguqLQMMCAwEA -AaNgMF4wHQYDVR0OBBYEFGuiVuLgAlxTj8o7g+Nl8MhODLb1MA4GA1UdDwEB/wQE -AwICBDAMBgNVHRMEBTADAQH/MB8GA1UdIwQYMBaAFGuiVuLgAlxTj8o7g+Nl8MhO -DLb1MA0GCSqGSIb3DQEBDQUAA4ICAQBXd2dgIe4DCKS9BECJoJAanf73cmzgAcUc -dfW2MhppAe3CscL8N4MJSxgJz+HUjmoNNUHpNLxRuigkarYqJxhLh1YvoyfvLJ/l -8+FvHsb3w3S4T1fDhmdnMp04niFUgDvDFgjaeVVg9WVelmAIKsGZZhNMmqqScS+T -uJuSIT/TZmLEcT2QqPVPZEFfT6IsZRgdgKpRw237bhP8TayzSvpsFbwUuVNXbOEc -IPj/rGdzAJWTWC+EVdVBKw3bcNrYoXEkWEtlchsw8uhHuAYOCaf2t5WjZVOwKmQ5 -o9vFP2yWNg1+OxKnHkJupnUbLVCx1qoRJ5M5f7zNOoyr9G6VEb6v+TlFVfSWqXQs -gj/Iy3q4Y9rgdA/4u6V3LdjJgZSFfSHKePeHZuGqKTnY9MxLwARkRfte3/TNFJLe -GdIOaKe236Rf28PYfinBqujhqhlJKWPVJSF1idR0F0jcWXRTPb3bR+0YP0tJXLtS -ICzsgv92LOkUjet1vcJrrLoJMnA27MqZNzClzP+Lrh1JtxA8w8wXTNal6oQbw8sZ -iO1+sca3wifjr4Fh/63ka52dTLDQB++CKkR7vffq8tBUoILxJ0vmiFcVdWWrjJnN -qdAVWM7CTclo/kscqoYDobIusaw/pouyyIdLYHPROlGIObxcr6jBlOvPoPn5gHzi -UBl/Dd7yAQ== +MIIFqzCCA5OgAwIBAgIBATANBgkqhkiG9w0BAQ0FADBoMRowGAYDVQQKDBFQaXhl +bGF0ZWQgUHJvamVjdDEmMCQGA1UECwwdaHR0cHM6Ly9waXhlbGF0ZWQtcHJvamVj +dC5vcmcxIjAgBgNVBAMMGVBpeGVsYXRlZCBQcm9qZWN0IFJvb3QgQ0EwHhcNMTUw +NDAxMDAwMDAwWhcNMjUwNDAxMDAwMDAwWjBoMRowGAYDVQQKDBFQaXhlbGF0ZWQg +UHJvamVjdDEmMCQGA1UECwwdaHR0cHM6Ly9waXhlbGF0ZWQtcHJvamVjdC5vcmcx +IjAgBgNVBAMMGVBpeGVsYXRlZCBQcm9qZWN0IFJvb3QgQ0EwggIiMA0GCSqGSIb3 +DQEBAQUAA4ICDwAwggIKAoICAQDIKlYvQciyw1gq7NgyXxo4OPnlB6cyjLb0vr6V +cnY00F3pwa3cuOASYtZ5p4IMGod1fZoBUlGFbxay0NO/FyOlPVc3FaF1cL5oYurl +a/koAjAihGMGEjsc2gcdODJOHoidgZG4a4WUCVfXP54QFkG6tXve1uWnZAt7W6Z8 +M4DobBqh4HE8aAn1vSIdOltKvuH6Jz9+O2SZdXczEKuHM/Zv8o5qI4+3CKQ/enLS +WUfqZx217zJw4xax4kTsZxaU56XkdXGjAoqKmP3am8sAfSPyt5J6yPvKzUM2hbyQ +v/ZR3AzKX6E5w/kPWfrjr+ABPtMEbcUEQYTR5Ic/46umzseEEUTHaeBhn5XZfan/ +K3+iOXHgukuNX5BF28D10RKrIWutOr6pMWXtFNLsmSvbii6SYg4sr80RhPsox3Lf +QuO36aH/jrkmyeEgTVVyniPUYLeg/IU+lQzOQpoHR7FiS2/39QP9HNbO7kQzi2Fh +aFQW35sRRZuz+Gt38y0DObKOwEZaR2NM2kIHiR6cYWP9fYCekOfJi7rp0hfxcO+J +o+kVoZM53SS0QJGzHnKYo+hHAy01Hj9Kh7N269rruZSjDXsWF/eZp5oYQeItBolq +J3Va8VyQk12Pd+4vcIERWNx8JCBDnYY1+sRBsrsH1mSMRQxOkbOhlx0y64dRNQRa +GV4xxQIDAQABo2AwXjAdBgNVHQ4EFgQUdXb2FIbWofTZ00R/OoQQu2K1mscwDgYD +VR0PAQH/BAQDAgIEMAwGA1UdEwQFMAMBAf8wHwYDVR0jBBgwFoAUdXb2FIbWofTZ +00R/OoQQu2K1mscwDQYJKoZIhvcNAQENBQADggIBAKR6XuNIZGWccQtVxuOKRuNY +5KE5N2Gqi4erN1XkJvOD60an2F6aYjFUN1ZZfq31Oi57GHZ8lwPKZZuAn/pn2vQG +1YpGFXndnoFkKDGb3XVRMHMtFIcV/2u2rX8zfRmZGpHvgSZo+9g4PN2jCipKkuki +pVnLjO7SZi4/AAuzlFPBUHE441xJI6Hl3lLKFezLD4akxrBpFbW+rBOSFupxyBcD +hedH0MyHY6HUQLL4Mm2tobouFStUcBZT0wGEeh/tlaJKAhk/kzryle4IZcqpv01a +6BfsyO9kRABc4b9w26tvd93utgXkpmwni+I31QSFVH/s8oU1pYvrpHO2MQf5rE7X +gyVvUaXSNETE2Nj1/Bp0Bbk/A2IyzOyoIALU+5LBOzgnW8K+Ojg9FQSRuIFZF+fw +VRruRFMziKtFBYaTclWX9mf+tn3MFSe8FcIcpMa0yXu9pDtQxZeeFG/TbseE0VjS +20HHEPnA6YIKbi7vg7LZP+dZoPBc30tJeG/fDCo6EDmFhddtDJpmnFPnr3OVf1AC +B90gGoXQ2RJsP8ZLkYlMn5JYKQbXLQSf806eMYJr3Mo6IBOR+o9dxAyBtEBgAUG6 +jgfKRzGizhHA3LDbxUlJt1k16SU58QBY3p8FHwjbjon7GDEOhSNOtuHAHUc83sHH +SC1pIYpuR4H4lTW5fuyF -----END CERTIFICATE----- diff --git a/service/pixelated/certificates/dev.wazokazi.is.ca.crt b/service/pixelated/certificates/dev.wazokazi.is.ca.crt deleted file mode 100644 index 0fda59da..00000000 --- a/service/pixelated/certificates/dev.wazokazi.is.ca.crt +++ /dev/null @@ -1,33 +0,0 @@ ------BEGIN CERTIFICATE----- -MIIFnzCCA4egAwIBAgIBATANBgkqhkiG9w0BAQ0FADBiMRowGAYDVQQKDBFQaXhl -bGF0ZWRfU3RhZ2luZzEgMB4GA1UECwwXaHR0cHM6Ly9kZXYud2F6b2themkuaXMx -IjAgBgNVBAMMGVBpeGVsYXRlZF9TdGFnaW5nIFJvb3QgQ0EwHhcNMTQxMDIyMDAw -MDAwWhcNMjQxMDIyMDAwMDAwWjBiMRowGAYDVQQKDBFQaXhlbGF0ZWRfU3RhZ2lu -ZzEgMB4GA1UECwwXaHR0cHM6Ly9kZXYud2F6b2themkuaXMxIjAgBgNVBAMMGVBp -eGVsYXRlZF9TdGFnaW5nIFJvb3QgQ0EwggIiMA0GCSqGSIb3DQEBAQUAA4ICDwAw -ggIKAoICAQC541dNmfzVEzLZ0LdV5ASG1STjB9NgoPpCily1bjjWhnglhNHEAfvP -kj2uEGXJip/ndJB/RV4QSqqxgoq8aHfhIspYLZ0IG5pIFnjdIMB5brqpkkf5N2wY -tSFy2gkspx/hk6xnEGyxDQntraY5rqFRLjCAE+cwk9kbqlm8iRrPApMDODizsNKF -8sTW/gC7U/LAkTvzZdnCUg4i72SmBUnZmudXiTiRTnvf2gmdCrPY2tpEguyWh+NB -qeig6wju0NGZl7+y7MnshteEcJ9IMiao88SRvL8B4pK8gA60ix+06QIPmuh8wQac -+rCB1+3sjeOSrBVPXzQkgLWqu/rT551AyKAEQsCm6SKPs/PhcLaZPLfA/onuZMdO -NL8l1DeRyyAXQYZgRMM4P0PUEmVfhV/xmkR62bhuQ8hvXV4lmhRSPqB5Whb0PNJ4 -SuGJTL6hu1b+zNBFqGl4EX/itYLS6anqmR8cIgallfkTOUgunOOUwzoj8hVDrbJF -6cWld0PmtFgUh5vaIpKZ+ygOLEosVqBVZzzN4ObEh+MfIxpzYZJ2ylB2r8DVzDxq -55G0f7LZHL31WcJwVF4lfmz1rx8us83DCWXJcVpxgJpoo/+Z+hyiUFf/XCMNYXn9 -vgZ1+bO7k/zUXGf+rnCgTxQkGTPZ95CCZ1nIMl7RgIJrSirBnlV5hQIDAQABo2Aw -XjAdBgNVHQ4EFgQUCSvehLZHfFbYbEhsNfWin8KNf6MwDgYDVR0PAQH/BAQDAgIE -MAwGA1UdEwQFMAMBAf8wHwYDVR0jBBgwFoAUCSvehLZHfFbYbEhsNfWin8KNf6Mw -DQYJKoZIhvcNAQENBQADggIBAJRD90EJ+zUOFA3Ky7NdNao3JXNLNIVsXTO5FAAM -ZPMeoSKIdKES5kc0vKlswP6RT7tBY5EH6sdjFMGEPK9glgxI1An4ozmgBUndOrql -mab9Ep+9ckfof8Yg6foETkpg6YhdWLbIHs98ajiTY6bozW1jJaRceVf57JLycqcO -FuEJ1fHRMqy2CY96g/Z4bMqkkYkVJKkyv7NIHg//DvRvboB4h+hO0MRB6rPAFC63 -6wxpGYhH4LHWRGS54IKwQxsOxpX38VM67Zj0XSEIx77b2E45unRXoZRwCBEWJlcv -5QWPCDxJgi0hDH1Fer+vQlcjB6/Kzg8AiVD15qGSNMLcjzn2Z334m0ErcYFI/c3A -VAMzCBR40mJx6r2zHUYbo+rR23huRWuFgeIdeeOJnJzpb+GwuGHlCdPl7mb7G8lp -R/U6J+GSUWs55quasE3VimDtX6eqh94HakGuq4n596GsIhYu6YHVSAf/4uTenpDk -ux2qIoSjZlarhoUoayRti4/HcP2a0w604jmyFLPNTi59J/xz/wwc7z8xwAT9DZ6F -4yxCC+u/eePbQY0USTRlrNIF9hi9DeJOxr/lTDZ8iXMxN820QiixH0XRQPrGEMcp -oY0FiYBJHweeaSEqz3AYB6updct7Qi0yS4g1uE8M9m+6SNzmSVpQWmZPr7iM5Cua -e0TX ------END CERTIFICATE----- diff --git a/service/pixelated/certificates/example.wazokazi.is.ca.crt b/service/pixelated/certificates/example.wazokazi.is.ca.crt deleted file mode 100644 index 515a12c4..00000000 --- a/service/pixelated/certificates/example.wazokazi.is.ca.crt +++ /dev/null @@ -1,32 +0,0 @@ ------BEGIN CERTIFICATE----- -MIIFkzCCA3ugAwIBAgIBATANBgkqhkiG9w0BAQ0FADBcMRUwEwYDVQQKDAxMRUFQ -X0V4YW1wbGUxJDAiBgNVBAsMG2h0dHBzOi8vZXhhbXBsZS53YXpva2F6aS5pczEd -MBsGA1UEAwwUTEVBUF9FeGFtcGxlIFJvb3QgQ0EwHhcNMTQxMDMxMDAwMDAwWhcN -MjQxMDMxMDAwMDAwWjBcMRUwEwYDVQQKDAxMRUFQX0V4YW1wbGUxJDAiBgNVBAsM -G2h0dHBzOi8vZXhhbXBsZS53YXpva2F6aS5pczEdMBsGA1UEAwwUTEVBUF9FeGFt -cGxlIFJvb3QgQ0EwggIiMA0GCSqGSIb3DQEBAQUAA4ICDwAwggIKAoICAQCsbVir -UZrbwQ6MLPY8i9KrYpCnFnYPTCZez6kfK0S9+5IVNltZlbedYPN/Mjr3ngdo4tSl -fZK6bk3UX1JhL6GkMMXD8Nk/yckGwaV9IbLdNI5Ihu3ot/SYIV8Bst19+hWUFbtN -Qgwlkyla2ZAqH6pIL7Hc5vED0ouVN5CBErYxsm/+mj5p69BtCZoR0QaKWURF2rTx -COb+xrhMRmqg/JPcsEtK/OqgIplkbjsHfxsknutFE6l/FbFvxc0sSmaMyd6fCVM4 -y9ivZXXh376klYPMuO0fXkGd79gSbWGldcX3MVbFNfwp2SxPXc/zP8/ntg0kqN4R -hASAvsT1bMG9/elXmU5XGouaJ0x5utK9/49qzfKiGyu/hW/ihQsGMx6I05EGxnsL -NFfUooySUqy6bB9BliHgyBaHHToNMJH2LHiyYFANWT6rPWAMH7VKEohNMRxu5IRK -U0E5OwSXu4yI5JU4bviuGSe93V5OOmO6GeIxVupij5i/1XtJ1Z5skuBcq7b7C2/6 -zNKoEu/wE9lbtjPNl7JwIYjaWRLpI+bMDPWjBSDSLCkHXPuxOCWFggGMEPOI4d3J -5hp22UGQxqq7z86ZTxPfx9Ce9oUbl1NOkuNznc0XVjFcglHqz4/75ogNP0dmra+P -GmPoAetOlKOIXxbBOQeSJr4WAgQK3GECb3uvQQIDAQABo2AwXjAdBgNVHQ4EFgQU -ti27+d3zDOJiJeQyx6il6YzZkKowDgYDVR0PAQH/BAQDAgIEMAwGA1UdEwQFMAMB -Af8wHwYDVR0jBBgwFoAUti27+d3zDOJiJeQyx6il6YzZkKowDQYJKoZIhvcNAQEN -BQADggIBABMGDact+sxOf8Yyt+mzckf1uANEw3TAM81ztxuiwYA6/quNvBISdjS6 -PyyZ8S3U+sjJA5FQW1SDd4IbuRFK2Y2Hmeto1uIFAxjQj/jDdW26Wj/FkBmWnCQd -L2/CSU1MXitLuxHE8SaUWrw+W7I/Ga5t+9VRxDI0XNF+OAEPwvp6JequZKIW+OcT -4fU7z4xu2QYSPRVzliJ9fVxb4jllG6kLNLnqkVTQNiOJ8sVbALh1tJSSFGd6ZGNB -8ePrWkPOyYMD6J+zy6uqDTUipaXuVIgA7mPTyu0XKqWO3tZXyakITlxnHgMYqpfE -XNkGArsMNXV3ktp+HCpGVO46+9nMzSPIBxMfpgpeq3ommBov3pANT0S5IrBiyo/T -VjUkWz3D1ymRyK0SWrHZSOq9rUS+tMjKxwmhFTBn1cR7PItWB9pNiVt/Oz0z1ecM -Ij4pMb/jPvV2H6crMjMVSPzbazTE/S6AEl0beZ/o44Frz1aXvuMwgtG6fUFZma4X -6exBWOmqF96qkPH2I0olHIbOTXHz+uBKJnFaSg09RT+AZfFutJ2DkVK7NJgOdyjf -QQcDfkfeRz/A6mI5wjm/UwZRWWytBuUDjU4ZaqxH9AlmzXsJHoYYr0YgTEYlm16o -eGTyR4QlaLlwTg1CvZgHTKxpI/LKn1T4aMHHvd92W2fT7pq4PWlH ------END CERTIFICATE----- diff --git a/service/pixelated/maintenance.py b/service/pixelated/maintenance.py index 8c20e097..b4e616a7 100644 --- a/service/pixelated/maintenance.py +++ b/service/pixelated/maintenance.py @@ -1,4 +1,4 @@ -# + # Copyright (c) 2014 ThoughtWorks, Inc. # # Pixelated is free software: you can redistribute it and/or modify @@ -20,9 +20,10 @@ import sys import json import argparse import email +import re from os.path import join -from mailbox import mboxMessage +from mailbox import mboxMessage, Maildir from pixelated.config.app import App from pixelated.config import app_factory from pixelated.config.args import parser_add_default_arguments @@ -39,6 +40,7 @@ from twisted.internet.threads import deferToThread from leap.mail.imap.memorystore import MemoryStore from leap.mail.imap.soledadstore import SoledadStore +from leap.mail.imap.fields import WithMsgFields from leap.common.events import register, unregister, events_pb2 as proto # monkey patching some specifics @@ -50,51 +52,26 @@ import pixelated.support.ext_keymanager_fetch_key import pixelated.support.ext_requests_urllib3 -def delete_all_mails(args): - leap_session, soledad = args - generation, docs = soledad.get_all_docs() - - for doc in docs: - if doc.content.get('type', None) in ['head', 'cnt', 'flags']: - soledad.delete_doc(doc) - - return args - - -@defer.inlineCallbacks -def load_mails(args, mail_paths): - leap_session, soledad = args - account = leap_session.account - - for path in mail_paths: - print 'Loading mails from %s' % path - for root, dirs, files in os.walk(path): - mbx = account.getMailbox('INBOX') - for f in files: - with open(join(root, f), 'r') as fp: - m = email.message_from_file(fp) - flags = ("\\RECENT",) - r = yield mbx.addMessage(m.as_string(), flags=flags, notify_on_disk=False) - print 'Added message %s' % m.get('subject') - print m.as_string() - - defer.returnValue(args) - return - +def initialize(): + args = parse_args() + app = App() -def dump_soledad(args): - leap_session, soledad = args + init_logging(args) + init_leap_cert(args) - generation, docs = soledad.get_all_docs() + if args.dispatcher or args.dispatcher_stdin: + config_dispatcher(app, args) + else: + config_user_agent(app, args) - for doc in docs: - print doc - print '\n' + init_events_server() + execute_command = create_execute_command(args, app) - return args + reactor.callWhenRunning(execute_command) + reactor.run() -def initialize(): +def parse_args(): parser = argparse.ArgumentParser(description='pixelated maintenance') parser_add_default_arguments(parser) subparsers = parser.add_subparsers(help='commands', dest='command') @@ -105,19 +82,10 @@ def initialize(): subparsers.add_parser('dump-soledad', help='dump the soledad database') subparsers.add_parser('sync', help='sync the soledad database') - args = parser.parse_args() - app = App() - - init_logging(args) - init_leap_cert(args) - - if args.dispatcher or args.dispatcher_stdin: - config_dispatcher(app, args) - else: - config_user_agent(app, args) + return parser.parse_args() - init_events_server() +def create_execute_command(args, app): def execute_command(): def init_soledad(): @@ -135,26 +103,109 @@ def initialize(): return args - d = deferToThread(init_soledad) - d.addCallback(get_soledad_handle) - d.addCallback(soledad_sync) - if args.command == 'reset': - d.addCallback(delete_all_mails) - elif args.command == 'load-mails': - d.addCallback(load_mails, args.file) - elif args.command == 'dump-soledad': - d.addCallback(dump_soledad) - elif args.command == 'sync': - # nothing to do here, sync is already part of the chain - pass + tearDown = defer.Deferred() + + prepare = deferToThread(init_soledad) + prepare.addCallback(get_soledad_handle) + prepare.addCallback(soledad_sync) + add_command_callback(args, prepare, tearDown) + tearDown.addCallback(soledad_sync) + tearDown.addCallback(shutdown) + tearDown.addErrback(shutdown_on_error) + + return execute_command + + +def add_command_callback(args, prepareDeferred, finalizeDeferred): + if args.command == 'reset': + prepareDeferred.addCallback(delete_all_mails) + prepareDeferred.addCallback(flush_to_soledad, finalizeDeferred) + elif args.command == 'load-mails': + prepareDeferred.addCallback(load_mails, args.file) + prepareDeferred.addCallback(flush_to_soledad, finalizeDeferred) + elif args.command == 'dump-soledad': + prepareDeferred.addCallback(dump_soledad) + prepareDeferred.chainDeferred(finalizeDeferred) + elif args.command == 'sync': + # nothing to do here, sync is already part of the chain + prepareDeferred.chainDeferred(finalizeDeferred) + else: + print 'Unsupported command: %s' % args.command + prepareDeferred.chainDeferred(finalizeDeferred) + + return finalizeDeferred + + +def delete_all_mails(args): + leap_session, soledad = args + generation, docs = soledad.get_all_docs() + + for doc in docs: + if doc.content.get('type', None) in ['head', 'cnt', 'flags']: + soledad.delete_doc(doc) + + return args + + +def add_mail_folder(account, maildir, folder_name, deferreds): + mbx = account.getMailbox(folder_name) + for mail in maildir: + flags = (WithMsgFields.RECENT_FLAG,) if mail.get_subdir() == 'new' else () + if 'S' in mail.get_flags(): + flags = (WithMsgFields.SEEN_FLAG,) + flags + if 'R' in mail.get_flags(): + flags = (WithMsgFields.ANSWERED_FLAG,) + flags + + deferreds.append(mbx.addMessage(mail.as_string(), flags=flags, notify_on_disk=False)) + + +@defer.inlineCallbacks +def load_mails(args, mail_paths): + leap_session, soledad = args + account = leap_session.account + + deferreds = [] + + for path in mail_paths: + maildir = Maildir(path, factory=None) + add_mail_folder(account, maildir, 'INBOX', deferreds) + for mail_folder_name in maildir.list_folders(): + mail_folder = maildir.get_folder(mail_folder_name) + add_mail_folder(account, mail_folder, mail_folder_name, deferreds) + + yield defer.DeferredList(deferreds) + defer.returnValue(args) + + +def flush_to_soledad(args, finalize): + leap_session, soledad = args + account = leap_session.account + memstore = account._memstore + permanent_store = memstore._permanent_store + + d = memstore.write_messages(permanent_store) + + def check_flushed(args): + if memstore.is_writing: + reactor.callLater(1, check_flushed, args) else: - print 'Unsupported command: %s' % args.command - d.addCallback(soledad_sync) - d.addCallback(shutdown) - d.addErrback(shutdown_on_error) + finalize.callback((leap_session, soledad)) - reactor.callWhenRunning(execute_command) - reactor.run() + d.addCallback(check_flushed) + + return args + + +def dump_soledad(args): + leap_session, soledad = args + + generation, docs = soledad.get_all_docs() + + for doc in docs: + print doc + print '\n' + + return args def shutdown(args): @@ -163,7 +214,7 @@ def shutdown(args): def shutdown_on_error(error): print error - reactor.stop() + shutdown(None) -if __name__ == 'main': +if __name__ == '__main__': initialize() diff --git a/service/pixelated/resources/mails_resource.py b/service/pixelated/resources/mails_resource.py index c057031a..3822abd3 100644 --- a/service/pixelated/resources/mails_resource.py +++ b/service/pixelated/resources/mails_resource.py @@ -53,6 +53,20 @@ class MailsDeleteResource(Resource): return respond_json(None, request) +class MailsRecoverResource(Resource): + isLeaf = True + + def __init__(self, mail_service): + Resource.__init__(self) + self._mail_service = mail_service + + def render_POST(self, request): + idents = json.loads(request.content.read())['idents'] + for ident in idents: + self._mail_service.recover_mail(ident) + return respond_json(None, request) + + class MailsResource(Resource): def _register_smtp_error_handler(self): @@ -66,6 +80,7 @@ class MailsResource(Resource): def __init__(self, mail_service, draft_service): Resource.__init__(self) self.putChild('delete', MailsDeleteResource(mail_service)) + self.putChild('recover', MailsRecoverResource(mail_service)) self.putChild('read', MailsReadResource(mail_service)) self.putChild('unread', MailsUnreadResource(mail_service)) diff --git a/service/test/mailsets/common/cur/mbox00000002:2,S b/service/test/mailsets/common/cur/mbox00000002:2,S new file mode 100644 index 00000000..2881e352 --- /dev/null +++ b/service/test/mailsets/common/cur/mbox00000002:2,S @@ -0,0 +1,19 @@ +Subject: A read mail +From: bob@try.pixelated-project.org +To: alice@try.pixelated-project.org +X-Pixelated-encryption-status: true +Date: Tue, 08 Apr 2015 18:32:05 +0000 (UTC) + +Q: How many hardware engineers does it take to change a lightbulb? +A: None. We'll fix it in software. + +Q: How many system programmers does it take to change a light bulb? +A: None. The application can work around it. + +Q: How many software engineers does it take to change a lightbulb? +A: None. We'll document it in the manual. + +Q: How many tech writers does it take to change a lightbulb? +A: None. The user can figure it out. + + diff --git a/service/test/mailsets/common/cur/mbox00000003:2,S b/service/test/mailsets/common/cur/mbox00000003:2,S new file mode 100644 index 00000000..36c6706a --- /dev/null +++ b/service/test/mailsets/common/cur/mbox00000003:2,S @@ -0,0 +1,17 @@ +Subject: Scott's Laws +From: scott@try.pixelated-project.org +To: alice@try.pixelated-project.org +X-Pixelated-encryption-status: true +Date: Tue, 09 Apr 2015 23:42:05 +0000 (UTC) + +Scott's First Law: + No matter what goes wrong, it will probably look right. + +Scott's Second Law: + When an error has been detected and corrected, it will be found + to have been wrong in the first place. +Corollary: + After the correction has been found in error, it will be + impossible to fit the original quantity back into the + equation. + diff --git a/service/test/mailsets/common/cur/mbox00000004:2,S b/service/test/mailsets/common/cur/mbox00000004:2,S new file mode 100644 index 00000000..db3be01d --- /dev/null +++ b/service/test/mailsets/common/cur/mbox00000004:2,S @@ -0,0 +1,9 @@ +Subject: Progress is only made on alternate Fridays +From: bob@try.pixelated-project.org +To: alice@try.pixelated-project.org +X-Pixelated-encryption-status: true +Date: Tue, 10 Apr 2015 03:15:38 +0000 (UTC) + +Weinberg's First Law: + Progress is only made on alternate Fridays. + diff --git a/service/test/mailsets/common/cur/mbox00000005:2,S b/service/test/mailsets/common/cur/mbox00000005:2,S new file mode 100644 index 00000000..e02718f1 --- /dev/null +++ b/service/test/mailsets/common/cur/mbox00000005:2,S @@ -0,0 +1,12 @@ +Subject: bypassing trouble +From: oliver@try.pixelated-project.org +To: alice@try.pixelated-project.org +X-Pixelated-encryption-status: true +Date: Tue, 04 Apr 2015 16:41:39 +0000 (UTC) + +If I had a formula for bypassing trouble, I would not pass it around. +Trouble creates a capacity to handle it. I don't say embrace trouble; that's +as bad as treating it as an enemy. But I do say meet it as a friend, for +you'll see a lot of it and you had better be on speaking terms with it. + -- Oliver Wendell Holmes, Jr. + diff --git a/service/test/mailsets/common/new/mbox00000000:2, b/service/test/mailsets/common/new/mbox00000000:2, new file mode 100644 index 00000000..a35e63f0 --- /dev/null +++ b/service/test/mailsets/common/new/mbox00000000:2, @@ -0,0 +1,12 @@ +Subject: Hey Alice +From: bob@try.pixelated-project.org +To: alice@try.pixelated-project.org +X-TW-Pixelated-Tags: cia +X-Pixelated-encryption-status: true +Date: Tue, 01 Apr 2015 08:43:27 +0000 (UTC) + +Yesterday upon the stair +I met a man who wasn't there. +He wasn't there again today -- +I think he's from the CIA. + diff --git a/service/test/mailsets/common/new/mbox00000001 b/service/test/mailsets/common/new/mbox00000001 new file mode 100644 index 00000000..e1c6ba8b --- /dev/null +++ b/service/test/mailsets/common/new/mbox00000001 @@ -0,0 +1,22 @@ +Subject: Garbage collection works +From: bob@try.pixelated-project.org +To: alice@try.pixelated-project.org +X-TW-Pixelated-Tags: garbage +X-Pixelated-encryption-status: true +Date: Tue, 05 Apr 2015 13:52:05 +0000 (UTC) + +=== ALL USERS PLEASE NOTE ======================== + +The garbage collector now works. In addition a new, experimental garbage +collection algorithm has been installed. With SI:%DSK-GC-QLX-BITS set to 17, +(NOT the default) the old garbage collection algorithm remains in force; when +virtual storage is filled, the machine cold boots itself. With SI:%DSK-GC- +QLX-BITS set to 23, the new garbage collector is enabled. Unlike most garbage +collectors, the new gc starts its mark phase from the mind of the user, rather +than from the obarray. This allows the garbage collection of significantly +more Qs. As the garbage collector runs, it may ask you something like "Do you +remember what SI:RDTBL-TRANS does?", and if you can't give a reasonable answer +in thirty seconds, the symbol becomes a candidate for GCing. The variable +SI:%GC-QLX-LUSER-TM governs how long the GC waits before timing out the user. + + diff --git a/service/test/mailsets/common/tmp/.keep b/service/test/mailsets/common/tmp/.keep new file mode 100644 index 00000000..e69de29b --- /dev/null +++ b/service/test/mailsets/common/tmp/.keep diff --git a/service/test/unit/adapter/test_mail_service.py b/service/test/unit/adapter/test_mail_service.py index 34fec708..f4b89f57 100644 --- a/service/test/unit/adapter/test_mail_service.py +++ b/service/test/unit/adapter/test_mail_service.py @@ -101,3 +101,13 @@ class TestMailService(unittest.TestCase): self.mail_service.delete_mail(1) verify(self.mailboxes).move_to_trash(1) + + def test_recover_mail(self): + mail_to_recover = PixelatedMail.from_soledad(*leap_mail(), soledad_querier=None) + when(self.mail_service).mail(1).thenReturn(mail_to_recover) + when(self.mailboxes).move_to_inbox(1).thenReturn(mail_to_recover) + + self.mail_service.recover_mail(1) + + verify(self.mailboxes).move_to_inbox(1) + verify(self.search_engine).index_mail(mail_to_recover) diff --git a/service/test/unit/adapter/test_mailboxes.py b/service/test/unit/adapter/test_mailboxes.py new file mode 100644 index 00000000..5b4548eb --- /dev/null +++ b/service/test/unit/adapter/test_mailboxes.py @@ -0,0 +1,40 @@ +# +# Copyright (c) 2014 ThoughtWorks, Inc. +# +# Pixelated is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# Pixelated is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Affero General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with Pixelated. If not, see <http://www.gnu.org/licenses/>. +import unittest + +from pixelated.adapter.model.mail import PixelatedMail +from pixelated.adapter.services.mailboxes import Mailboxes +from mockito import mock, when, verify +from test.support import test_helper +from mock import MagicMock + + +class PixelatedMailboxesTest(unittest.TestCase): + def setUp(self): + self.querier = mock() + self.search_engine = mock() + self.account = MagicMock() + self.mailboxes = Mailboxes(self.account, self.querier, self.search_engine) + + def test_move_to_inbox(self): + mail = PixelatedMail.from_soledad(*test_helper.leap_mail(), soledad_querier=self.querier) + when(self.querier).mail(1).thenReturn(mail) + when(mail).save().thenReturn(None) + + mail.set_mailbox('TRASH') + recovered_mail = self.mailboxes.move_to_inbox(1) + self.assertEquals('INBOX', recovered_mail.mailbox_name) + verify(mail).save() diff --git a/service/test/unit/fixtures/mailset/cur/.keep b/service/test/unit/fixtures/mailset/cur/.keep new file mode 100644 index 00000000..e69de29b --- /dev/null +++ b/service/test/unit/fixtures/mailset/cur/.keep diff --git a/service/test/unit/fixtures/mailset/mbox00000000 b/service/test/unit/fixtures/mailset/new/mbox00000000 index 3d01c203..3d01c203 100644 --- a/service/test/unit/fixtures/mailset/mbox00000000 +++ b/service/test/unit/fixtures/mailset/new/mbox00000000 diff --git a/service/test/unit/fixtures/mailset/mbox00000001 b/service/test/unit/fixtures/mailset/new/mbox00000001 index fc76bba2..fc76bba2 100644 --- a/service/test/unit/fixtures/mailset/mbox00000001 +++ b/service/test/unit/fixtures/mailset/new/mbox00000001 diff --git a/service/test/unit/fixtures/mailset/tmp/.keep b/service/test/unit/fixtures/mailset/tmp/.keep new file mode 100644 index 00000000..e69de29b --- /dev/null +++ b/service/test/unit/fixtures/mailset/tmp/.keep diff --git a/service/test/unit/maintenance/test_commands.py b/service/test/unit/maintenance/test_commands.py index f23655d8..6f993106 100644 --- a/service/test/unit/maintenance/test_commands.py +++ b/service/test/unit/maintenance/test_commands.py @@ -19,74 +19,96 @@ import email from pixelated.maintenance import delete_all_mails, load_mails from pixelated.bitmask_libraries.session import LeapSession from leap.mail.imap.account import SoledadBackedAccount +from leap.mail.imap.fields import WithMsgFields from leap.soledad.client import Soledad from leap.soledad.common.document import SoledadDocument from mock import MagicMock, ANY from os.path import join, dirname +from twisted.internet import defer, reactor class TestCommands(unittest.TestCase): - def setUp(self): - self.leap_session = MagicMock(spec=LeapSession) - self.soledad = MagicMock(spec=Soledad) - self.account = MagicMock(spec=SoledadBackedAccount) - self.mailbox = MagicMock() - self.leap_session.account = self.account - self.account.getMailbox.return_value = self.mailbox + def setUp(self): + self.leap_session = MagicMock(spec=LeapSession) + self.soledad = MagicMock(spec=Soledad) + self.account = MagicMock(spec=SoledadBackedAccount) + self.mailbox = MagicMock() + self.leap_session.account = self.account + self.account.getMailbox.return_value = self.mailbox - self.args = (self.leap_session, self.soledad) + self.args = (self.leap_session, self.soledad) - def test_delete_all_mails_supports_empty_doclist(self): - self.soledad.get_all_docs.return_value = (1, []) + def test_delete_all_mails_supports_empty_doclist(self): + self.soledad.get_all_docs.return_value = (1, []) - delete_all_mails(self.args) + delete_all_mails(self.args) - self.assertFalse(self.soledad.delete_doc.called) + self.assertFalse(self.soledad.delete_doc.called) - def test_delete_all_mails(self): - doc = MagicMock(spec=SoledadDocument) - doc.content = {'type': 'head'} - self.soledad.get_all_docs.return_value = (1, [doc]) + def test_delete_all_mails(self): + doc = MagicMock(spec=SoledadDocument) + doc.content = {'type': 'head'} + self.soledad.get_all_docs.return_value = (1, [doc]) - delete_all_mails(self.args) + delete_all_mails(self.args) - self.soledad.delete_doc.assert_called_once_with(doc) + self.soledad.delete_doc.assert_called_once_with(doc) - def test_only_mail_documents_are_deleted(self): - docs = self._create_docs_of_type(['head', 'cnt', 'flags', 'mbx', 'foo', None]) - self.soledad.get_all_docs.return_value = (1, docs) + def test_only_mail_documents_are_deleted(self): + docs = self._create_docs_of_type(['head', 'cnt', 'flags', 'mbx', 'foo', None]) + self.soledad.get_all_docs.return_value = (1, docs) - delete_all_mails(self.args) + delete_all_mails(self.args) - for doc in docs: - if doc.content['type'] in ['head', 'cnt', 'flags']: - self.soledad.delete_doc.assert_any_call(doc) - self.assertEqual(3, len(self.soledad.delete_doc.mock_calls)) + for doc in docs: + if doc.content['type'] in ['head', 'cnt', 'flags']: + self.soledad.delete_doc.assert_any_call(doc) + self.assertEqual(3, len(self.soledad.delete_doc.mock_calls)) - def _create_docs_of_type(self, type_list): - return [self._create_doc_type(t) for t in type_list] + def _create_docs_of_type(self, type_list): + return [self._create_doc_type(t) for t in type_list] - def _create_doc_type(self, doc_type): - doc = MagicMock(spec=SoledadDocument) - doc.content = {'type': doc_type} - return doc + def _create_doc_type(self, doc_type): + doc = MagicMock(spec=SoledadDocument) + doc.content = {'type': doc_type} + return doc - def test_load_mails_empty_path_list(self): - load_mails(self.args, []) + def test_load_mails_empty_path_list(self): + load_mails(self.args, []) - self.assertFalse(self.mailbox.called) + self.assertFalse(self.mailbox.called) - def test_load_mails_adds_mails(self): - mail_root = join(dirname(__file__), '..', 'fixtures', 'mailset') + def test_load_mails_adds_mails(self): + # given + mail_root = join(dirname(__file__), '..', 'fixtures', 'mailset') + firstMailDeferred = defer.Deferred() + secondMailDeferred = defer.Deferred() + self.mailbox.addMessage.side_effect = [firstMailDeferred, secondMailDeferred] - foo = load_mails(self.args, [mail_root]) + # when + d = load_mails(self.args, [mail_root]) + # then + def assert_mails_added(_): self.assertTrue(self.mailbox.addMessage.called) - self.mailbox.addMessage.assert_any_call(self._mail_content(join(mail_root, 'mbox00000000')), flags=("\\RECENT",), notify_on_disk=False) - self.mailbox.addMessage.assert_any_call(self._mail_content(join(mail_root, 'mbox00000001')), flags=("\\RECENT",), notify_on_disk=False) + self.mailbox.addMessage.assert_any_call(self._mail_content(join(mail_root, 'new', 'mbox00000000')), flags=(WithMsgFields.RECENT_FLAG,), notify_on_disk=False) + self.mailbox.addMessage.assert_any_call(self._mail_content(join(mail_root, 'new', 'mbox00000001')), flags=(WithMsgFields.RECENT_FLAG,), notify_on_disk=False) - def _mail_content(self, mail_file): - with open(mail_file, 'r') as fp: - m = email.message_from_file(fp) - return m.as_string() + def error_callack(err): + print err + self.assertTrue(False) + + d.addCallback(assert_mails_added) + d.addErrback(error_callack) + + # trigger callbacks for both mails + reactor.callLater(0, firstMailDeferred.callback, None) + reactor.callLater(0, secondMailDeferred.callback, None) + + return d + + def _mail_content(self, mail_file): + with open(mail_file, 'r') as fp: + m = email.message_from_file(fp) + return m.as_string() |