From 5a3a2012bb8982ad0884ed659e61e969345e6fde Mon Sep 17 00:00:00 2001 From: "Kali Kaneko (leap communications)" Date: Mon, 29 Aug 2016 23:10:17 -0400 Subject: [pkg] move mail source to leap.bitmask.mail --- docs/bitmask-mail-index.rst | 96 ++ docs/mail-hacking.rst | 195 +++ docs/mail-journey.rst | 40 + mail/.gitattributes | 1 - mail/.gitignore | 25 - mail/AUTHORS | 8 - mail/CHANGELOG.rst | 37 - mail/HISTORY | 179 -- mail/LICENSE | 619 ------- mail/MANIFEST.in | 6 - mail/README.rst | 37 - mail/changes/VERSION_COMPAT | 11 - mail/changes/next-changelog.rst | 30 - mail/docs/Makefile | 179 -- mail/docs/api/leap.mail.adaptors.rst | 43 - mail/docs/api/leap.mail.adaptors.tests.rst | 28 - mail/docs/api/leap.mail.imap.rst | 52 - mail/docs/api/leap.mail.imap.service.rst | 9 - mail/docs/api/leap.mail.incoming.rst | 20 - mail/docs/api/leap.mail.outgoing.rst | 21 - mail/docs/api/leap.mail.plugins.rst | 20 - mail/docs/api/leap.mail.rst | 105 -- mail/docs/api/leap.mail.smtp.rst | 19 - mail/docs/api/modules.rst | 7 - mail/docs/conf.py | 341 ---- mail/docs/git-commit-message.txt | 8 - mail/docs/hacking.rst | 195 --- mail/docs/index.rst | 96 -- mail/docs/mail_journey.rst | 40 - mail/docs/recreate_apidocs.sh | 4 - mail/docs/tutorial.rst | 3 - mail/pkg/__init__.py | 0 mail/pkg/generate_wheels.sh | 14 - mail/pkg/pip_install_requirements.sh | 86 - mail/pkg/requirements-dev.pip | 14 - mail/pkg/requirements-latest.pip | 9 - mail/pkg/requirements-leap.pip | 3 - mail/pkg/requirements-testing.pip | 7 - mail/pkg/requirements.pip | 4 - mail/pkg/tools/get_authors.sh | 2 - mail/pkg/tools/with_venvwrapper.sh | 16 - mail/pkg/utils.py | 101 -- mail/setup.cfg | 24 - mail/setup.py | 156 -- mail/src/leap/__init__.py | 6 - mail/src/leap/mail/__init__.py | 26 - mail/src/leap/mail/_version.py | 484 ------ mail/src/leap/mail/adaptors/__init__.py | 0 mail/src/leap/mail/adaptors/models.py | 123 -- mail/src/leap/mail/adaptors/soledad.py | 1268 -------------- mail/src/leap/mail/adaptors/soledad_indexes.py | 106 -- mail/src/leap/mail/adaptors/tests/rfc822.message | 1 - mail/src/leap/mail/adaptors/tests/test_models.py | 106 -- .../mail/adaptors/tests/test_soledad_adaptor.py | 529 ------ mail/src/leap/mail/constants.py | 52 - mail/src/leap/mail/cred.py | 80 - mail/src/leap/mail/decorators.py | 149 -- mail/src/leap/mail/errors.py | 27 - mail/src/leap/mail/generator.py | 23 - mail/src/leap/mail/imap/__init__.py | 0 mail/src/leap/mail/imap/account.py | 498 ------ mail/src/leap/mail/imap/mailbox.py | 970 ----------- mail/src/leap/mail/imap/messages.py | 254 --- mail/src/leap/mail/imap/server.py | 693 -------- mail/src/leap/mail/imap/service/README.rst | 39 - mail/src/leap/mail/imap/service/__init__.py | 0 mail/src/leap/mail/imap/service/imap-server.tac | 145 -- mail/src/leap/mail/imap/service/imap.py | 208 --- mail/src/leap/mail/imap/service/manhole.py | 130 -- mail/src/leap/mail/imap/service/notes.txt | 81 - mail/src/leap/mail/imap/service/rfc822.message | 86 - mail/src/leap/mail/imap/tests/.gitignore | 1 - mail/src/leap/mail/imap/tests/getmail | 344 ---- mail/src/leap/mail/imap/tests/imapclient.py | 207 --- .../leap/mail/imap/tests/regressions_mime_struct | 461 ----- mail/src/leap/mail/imap/tests/rfc822.message | 1 - .../mail/imap/tests/rfc822.multi-minimal.message | 1 - .../mail/imap/tests/rfc822.multi-nested.message | 1 - .../mail/imap/tests/rfc822.multi-signed.message | 1 - mail/src/leap/mail/imap/tests/rfc822.multi.message | 1 - mail/src/leap/mail/imap/tests/rfc822.plain.message | 1 - .../src/leap/mail/imap/tests/stress_tests_imap.zsh | 178 -- mail/src/leap/mail/imap/tests/test_imap.py | 1060 ------------ mail/src/leap/mail/imap/tests/walktree.py | 127 -- mail/src/leap/mail/incoming/__init__.py | 0 mail/src/leap/mail/incoming/service.py | 844 ---------- .../tests/rfc822.multi-encrypt-signed.message | 61 - .../leap/mail/incoming/tests/test_incoming_mail.py | 391 ----- mail/src/leap/mail/interfaces.py | 215 --- mail/src/leap/mail/load_tests.py | 29 - mail/src/leap/mail/mail.py | 1070 ------------ mail/src/leap/mail/mailbox_indexer.py | 327 ---- mail/src/leap/mail/outgoing/__init__.py | 0 mail/src/leap/mail/outgoing/service.py | 518 ------ mail/src/leap/mail/outgoing/tests/test_outgoing.py | 263 --- mail/src/leap/mail/plugins/__init__.py | 3 - mail/src/leap/mail/plugins/soledad_sync_hooks.py | 19 - mail/src/leap/mail/rfc3156.py | 390 ----- mail/src/leap/mail/size.py | 57 - mail/src/leap/mail/smtp/README.rst | 44 - mail/src/leap/mail/smtp/__init__.py | 73 - mail/src/leap/mail/smtp/bounces.py | 90 - mail/src/leap/mail/smtp/gateway.py | 413 ----- mail/src/leap/mail/smtp/tests/185CA770.key | 79 - mail/src/leap/mail/smtp/tests/185CA770.pub | 52 - mail/src/leap/mail/smtp/tests/cert/server.crt | 29 - mail/src/leap/mail/smtp/tests/cert/server.key | 51 - mail/src/leap/mail/smtp/tests/mail.txt | 10 - mail/src/leap/mail/smtp/tests/test_gateway.py | 181 -- mail/src/leap/mail/sync_hooks.py | 120 -- mail/src/leap/mail/testing/__init__.py | 353 ---- mail/src/leap/mail/testing/__init__.py~ | 358 ---- mail/src/leap/mail/testing/common.py | 109 -- mail/src/leap/mail/testing/imap.py | 186 -- mail/src/leap/mail/testing/smtp.py | 51 - mail/src/leap/mail/tests/rfc822.bounce.message | 152 -- mail/src/leap/mail/tests/rfc822.message | 86 - .../leap/mail/tests/rfc822.multi-minimal.message | 16 - .../leap/mail/tests/rfc822.multi-nested.message | 619 ------- .../leap/mail/tests/rfc822.multi-signed.message | 238 --- mail/src/leap/mail/tests/rfc822.multi.message | 96 -- mail/src/leap/mail/tests/rfc822.plain.message | 66 - mail/src/leap/mail/tests/test_mail.py | 399 ----- mail/src/leap/mail/tests/test_mailbox_indexer.py | 250 --- mail/src/leap/mail/tests/test_walk.py | 81 - mail/src/leap/mail/utils.py | 375 ----- mail/src/leap/mail/walk.py | 107 -- mail/tox.ini | 18 - mail/versioneer.py | 1774 -------------------- src/leap/bitmask/mail/__init__.py | 26 + src/leap/bitmask/mail/_version.py | 484 ++++++ src/leap/bitmask/mail/adaptors/__init__.py | 0 src/leap/bitmask/mail/adaptors/models.py | 123 ++ src/leap/bitmask/mail/adaptors/soledad.py | 1268 ++++++++++++++ src/leap/bitmask/mail/adaptors/soledad_indexes.py | 106 ++ .../bitmask/mail/adaptors/tests/rfc822.message | 1 + .../bitmask/mail/adaptors/tests/test_models.py | 106 ++ .../mail/adaptors/tests/test_soledad_adaptor.py | 529 ++++++ src/leap/bitmask/mail/constants.py | 52 + src/leap/bitmask/mail/cred.py | 80 + src/leap/bitmask/mail/decorators.py | 149 ++ src/leap/bitmask/mail/errors.py | 27 + src/leap/bitmask/mail/generator.py | 23 + src/leap/bitmask/mail/imap/__init__.py | 0 src/leap/bitmask/mail/imap/account.py | 498 ++++++ src/leap/bitmask/mail/imap/mailbox.py | 970 +++++++++++ src/leap/bitmask/mail/imap/messages.py | 254 +++ src/leap/bitmask/mail/imap/server.py | 693 ++++++++ src/leap/bitmask/mail/imap/service/README.rst | 39 + src/leap/bitmask/mail/imap/service/__init__.py | 0 src/leap/bitmask/mail/imap/service/imap-server.tac | 145 ++ src/leap/bitmask/mail/imap/service/imap.py | 208 +++ src/leap/bitmask/mail/imap/service/manhole.py | 130 ++ src/leap/bitmask/mail/imap/service/notes.txt | 81 + src/leap/bitmask/mail/imap/service/rfc822.message | 86 + src/leap/bitmask/mail/imap/tests/.gitignore | 1 + src/leap/bitmask/mail/imap/tests/getmail | 344 ++++ src/leap/bitmask/mail/imap/tests/imapclient.py | 207 +++ .../mail/imap/tests/regressions_mime_struct | 461 +++++ src/leap/bitmask/mail/imap/tests/rfc822.message | 1 + .../mail/imap/tests/rfc822.multi-minimal.message | 1 + .../mail/imap/tests/rfc822.multi-nested.message | 1 + .../mail/imap/tests/rfc822.multi-signed.message | 1 + .../bitmask/mail/imap/tests/rfc822.multi.message | 1 + .../bitmask/mail/imap/tests/rfc822.plain.message | 1 + .../bitmask/mail/imap/tests/stress_tests_imap.zsh | 178 ++ src/leap/bitmask/mail/imap/tests/test_imap.py | 1060 ++++++++++++ src/leap/bitmask/mail/imap/tests/walktree.py | 127 ++ src/leap/bitmask/mail/incoming/__init__.py | 0 src/leap/bitmask/mail/incoming/service.py | 844 ++++++++++ .../tests/rfc822.multi-encrypt-signed.message | 61 + .../mail/incoming/tests/test_incoming_mail.py | 391 +++++ src/leap/bitmask/mail/interfaces.py | 215 +++ src/leap/bitmask/mail/load_tests.py | 29 + src/leap/bitmask/mail/mail.py | 1070 ++++++++++++ src/leap/bitmask/mail/mailbox_indexer.py | 327 ++++ src/leap/bitmask/mail/outgoing/__init__.py | 0 src/leap/bitmask/mail/outgoing/service.py | 518 ++++++ .../bitmask/mail/outgoing/tests/test_outgoing.py | 263 +++ src/leap/bitmask/mail/plugins/__init__.py | 3 + .../bitmask/mail/plugins/soledad_sync_hooks.py | 19 + src/leap/bitmask/mail/rfc3156.py | 390 +++++ src/leap/bitmask/mail/size.py | 57 + src/leap/bitmask/mail/smtp/README.rst | 44 + src/leap/bitmask/mail/smtp/__init__.py | 73 + src/leap/bitmask/mail/smtp/bounces.py | 90 + src/leap/bitmask/mail/smtp/gateway.py | 413 +++++ src/leap/bitmask/mail/smtp/tests/185CA770.key | 79 + src/leap/bitmask/mail/smtp/tests/185CA770.pub | 52 + src/leap/bitmask/mail/smtp/tests/cert/server.crt | 29 + src/leap/bitmask/mail/smtp/tests/cert/server.key | 51 + src/leap/bitmask/mail/smtp/tests/mail.txt | 10 + src/leap/bitmask/mail/smtp/tests/test_gateway.py | 181 ++ src/leap/bitmask/mail/sync_hooks.py | 120 ++ src/leap/bitmask/mail/testing/__init__.py | 353 ++++ src/leap/bitmask/mail/testing/__init__.py~ | 358 ++++ src/leap/bitmask/mail/testing/common.py | 109 ++ src/leap/bitmask/mail/testing/imap.py | 186 ++ src/leap/bitmask/mail/testing/smtp.py | 51 + src/leap/bitmask/mail/tests/rfc822.bounce.message | 152 ++ src/leap/bitmask/mail/tests/rfc822.message | 86 + .../mail/tests/rfc822.multi-minimal.message | 16 + .../bitmask/mail/tests/rfc822.multi-nested.message | 619 +++++++ .../bitmask/mail/tests/rfc822.multi-signed.message | 238 +++ src/leap/bitmask/mail/tests/rfc822.multi.message | 96 ++ src/leap/bitmask/mail/tests/rfc822.plain.message | 66 + src/leap/bitmask/mail/tests/test_mail.py | 399 +++++ .../bitmask/mail/tests/test_mailbox_indexer.py | 250 +++ src/leap/bitmask/mail/tests/test_walk.py | 81 + src/leap/bitmask/mail/utils.py | 375 +++++ src/leap/bitmask/mail/walk.py | 107 ++ 211 files changed, 17664 insertions(+), 21710 deletions(-) create mode 100644 docs/bitmask-mail-index.rst create mode 100644 docs/mail-hacking.rst create mode 100644 docs/mail-journey.rst delete mode 100644 mail/.gitattributes delete mode 100644 mail/.gitignore delete mode 100644 mail/AUTHORS delete mode 100644 mail/CHANGELOG.rst delete mode 100644 mail/HISTORY delete mode 100644 mail/LICENSE delete mode 100644 mail/MANIFEST.in delete mode 100644 mail/README.rst delete mode 100644 mail/changes/VERSION_COMPAT delete mode 100644 mail/changes/next-changelog.rst delete mode 100644 mail/docs/Makefile delete mode 100644 mail/docs/api/leap.mail.adaptors.rst delete mode 100644 mail/docs/api/leap.mail.adaptors.tests.rst delete mode 100644 mail/docs/api/leap.mail.imap.rst delete mode 100644 mail/docs/api/leap.mail.imap.service.rst delete mode 100644 mail/docs/api/leap.mail.incoming.rst delete mode 100644 mail/docs/api/leap.mail.outgoing.rst delete mode 100644 mail/docs/api/leap.mail.plugins.rst delete mode 100644 mail/docs/api/leap.mail.rst delete mode 100644 mail/docs/api/leap.mail.smtp.rst delete mode 100644 mail/docs/api/modules.rst delete mode 100644 mail/docs/conf.py delete mode 100644 mail/docs/git-commit-message.txt delete mode 100644 mail/docs/hacking.rst delete mode 100644 mail/docs/index.rst delete mode 100644 mail/docs/mail_journey.rst delete mode 100755 mail/docs/recreate_apidocs.sh delete mode 100644 mail/docs/tutorial.rst delete mode 100644 mail/pkg/__init__.py delete mode 100755 mail/pkg/generate_wheels.sh delete mode 100755 mail/pkg/pip_install_requirements.sh delete mode 100644 mail/pkg/requirements-dev.pip delete mode 100644 mail/pkg/requirements-latest.pip delete mode 100644 mail/pkg/requirements-leap.pip delete mode 100644 mail/pkg/requirements-testing.pip delete mode 100644 mail/pkg/requirements.pip delete mode 100755 mail/pkg/tools/get_authors.sh delete mode 100755 mail/pkg/tools/with_venvwrapper.sh delete mode 100644 mail/pkg/utils.py delete mode 100644 mail/setup.cfg delete mode 100644 mail/setup.py delete mode 100644 mail/src/leap/__init__.py delete mode 100644 mail/src/leap/mail/__init__.py delete mode 100644 mail/src/leap/mail/_version.py delete mode 100644 mail/src/leap/mail/adaptors/__init__.py delete mode 100644 mail/src/leap/mail/adaptors/models.py delete mode 100644 mail/src/leap/mail/adaptors/soledad.py delete mode 100644 mail/src/leap/mail/adaptors/soledad_indexes.py delete mode 120000 mail/src/leap/mail/adaptors/tests/rfc822.message delete mode 100644 mail/src/leap/mail/adaptors/tests/test_models.py delete mode 100644 mail/src/leap/mail/adaptors/tests/test_soledad_adaptor.py delete mode 100644 mail/src/leap/mail/constants.py delete mode 100644 mail/src/leap/mail/cred.py delete mode 100644 mail/src/leap/mail/decorators.py delete mode 100644 mail/src/leap/mail/errors.py delete mode 100644 mail/src/leap/mail/generator.py delete mode 100644 mail/src/leap/mail/imap/__init__.py delete mode 100644 mail/src/leap/mail/imap/account.py delete mode 100644 mail/src/leap/mail/imap/mailbox.py delete mode 100644 mail/src/leap/mail/imap/messages.py delete mode 100644 mail/src/leap/mail/imap/server.py delete mode 100644 mail/src/leap/mail/imap/service/README.rst delete mode 100644 mail/src/leap/mail/imap/service/__init__.py delete mode 100644 mail/src/leap/mail/imap/service/imap-server.tac delete mode 100644 mail/src/leap/mail/imap/service/imap.py delete mode 100644 mail/src/leap/mail/imap/service/manhole.py delete mode 100644 mail/src/leap/mail/imap/service/notes.txt delete mode 100644 mail/src/leap/mail/imap/service/rfc822.message delete mode 100644 mail/src/leap/mail/imap/tests/.gitignore delete mode 100755 mail/src/leap/mail/imap/tests/getmail delete mode 100755 mail/src/leap/mail/imap/tests/imapclient.py delete mode 100755 mail/src/leap/mail/imap/tests/regressions_mime_struct delete mode 120000 mail/src/leap/mail/imap/tests/rfc822.message delete mode 120000 mail/src/leap/mail/imap/tests/rfc822.multi-minimal.message delete mode 120000 mail/src/leap/mail/imap/tests/rfc822.multi-nested.message delete mode 120000 mail/src/leap/mail/imap/tests/rfc822.multi-signed.message delete mode 120000 mail/src/leap/mail/imap/tests/rfc822.multi.message delete mode 120000 mail/src/leap/mail/imap/tests/rfc822.plain.message delete mode 100755 mail/src/leap/mail/imap/tests/stress_tests_imap.zsh delete mode 100644 mail/src/leap/mail/imap/tests/test_imap.py delete mode 100644 mail/src/leap/mail/imap/tests/walktree.py delete mode 100644 mail/src/leap/mail/incoming/__init__.py delete mode 100644 mail/src/leap/mail/incoming/service.py delete mode 100644 mail/src/leap/mail/incoming/tests/rfc822.multi-encrypt-signed.message delete mode 100644 mail/src/leap/mail/incoming/tests/test_incoming_mail.py delete mode 100644 mail/src/leap/mail/interfaces.py delete mode 100644 mail/src/leap/mail/load_tests.py delete mode 100644 mail/src/leap/mail/mail.py delete mode 100644 mail/src/leap/mail/mailbox_indexer.py delete mode 100644 mail/src/leap/mail/outgoing/__init__.py delete mode 100644 mail/src/leap/mail/outgoing/service.py delete mode 100644 mail/src/leap/mail/outgoing/tests/test_outgoing.py delete mode 100644 mail/src/leap/mail/plugins/__init__.py delete mode 100644 mail/src/leap/mail/plugins/soledad_sync_hooks.py delete mode 100644 mail/src/leap/mail/rfc3156.py delete mode 100644 mail/src/leap/mail/size.py delete mode 100644 mail/src/leap/mail/smtp/README.rst delete mode 100644 mail/src/leap/mail/smtp/__init__.py delete mode 100644 mail/src/leap/mail/smtp/bounces.py delete mode 100644 mail/src/leap/mail/smtp/gateway.py delete mode 100644 mail/src/leap/mail/smtp/tests/185CA770.key delete mode 100644 mail/src/leap/mail/smtp/tests/185CA770.pub delete mode 100644 mail/src/leap/mail/smtp/tests/cert/server.crt delete mode 100644 mail/src/leap/mail/smtp/tests/cert/server.key delete mode 100644 mail/src/leap/mail/smtp/tests/mail.txt delete mode 100644 mail/src/leap/mail/smtp/tests/test_gateway.py delete mode 100644 mail/src/leap/mail/sync_hooks.py delete mode 100644 mail/src/leap/mail/testing/__init__.py delete mode 100644 mail/src/leap/mail/testing/__init__.py~ delete mode 100644 mail/src/leap/mail/testing/common.py delete mode 100644 mail/src/leap/mail/testing/imap.py delete mode 100644 mail/src/leap/mail/testing/smtp.py delete mode 100644 mail/src/leap/mail/tests/rfc822.bounce.message delete mode 100644 mail/src/leap/mail/tests/rfc822.message delete mode 100644 mail/src/leap/mail/tests/rfc822.multi-minimal.message delete mode 100644 mail/src/leap/mail/tests/rfc822.multi-nested.message delete mode 100644 mail/src/leap/mail/tests/rfc822.multi-signed.message delete mode 100644 mail/src/leap/mail/tests/rfc822.multi.message delete mode 100644 mail/src/leap/mail/tests/rfc822.plain.message delete mode 100644 mail/src/leap/mail/tests/test_mail.py delete mode 100644 mail/src/leap/mail/tests/test_mailbox_indexer.py delete mode 100644 mail/src/leap/mail/tests/test_walk.py delete mode 100644 mail/src/leap/mail/utils.py delete mode 100644 mail/src/leap/mail/walk.py delete mode 100644 mail/tox.ini delete mode 100644 mail/versioneer.py create mode 100644 src/leap/bitmask/mail/__init__.py create mode 100644 src/leap/bitmask/mail/_version.py create mode 100644 src/leap/bitmask/mail/adaptors/__init__.py create mode 100644 src/leap/bitmask/mail/adaptors/models.py create mode 100644 src/leap/bitmask/mail/adaptors/soledad.py create mode 100644 src/leap/bitmask/mail/adaptors/soledad_indexes.py create mode 120000 src/leap/bitmask/mail/adaptors/tests/rfc822.message create mode 100644 src/leap/bitmask/mail/adaptors/tests/test_models.py create mode 100644 src/leap/bitmask/mail/adaptors/tests/test_soledad_adaptor.py create mode 100644 src/leap/bitmask/mail/constants.py create mode 100644 src/leap/bitmask/mail/cred.py create mode 100644 src/leap/bitmask/mail/decorators.py create mode 100644 src/leap/bitmask/mail/errors.py create mode 100644 src/leap/bitmask/mail/generator.py create mode 100644 src/leap/bitmask/mail/imap/__init__.py create mode 100644 src/leap/bitmask/mail/imap/account.py create mode 100644 src/leap/bitmask/mail/imap/mailbox.py create mode 100644 src/leap/bitmask/mail/imap/messages.py create mode 100644 src/leap/bitmask/mail/imap/server.py create mode 100644 src/leap/bitmask/mail/imap/service/README.rst create mode 100644 src/leap/bitmask/mail/imap/service/__init__.py create mode 100644 src/leap/bitmask/mail/imap/service/imap-server.tac create mode 100644 src/leap/bitmask/mail/imap/service/imap.py create mode 100644 src/leap/bitmask/mail/imap/service/manhole.py create mode 100644 src/leap/bitmask/mail/imap/service/notes.txt create mode 100644 src/leap/bitmask/mail/imap/service/rfc822.message create mode 100644 src/leap/bitmask/mail/imap/tests/.gitignore create mode 100755 src/leap/bitmask/mail/imap/tests/getmail create mode 100755 src/leap/bitmask/mail/imap/tests/imapclient.py create mode 100755 src/leap/bitmask/mail/imap/tests/regressions_mime_struct create mode 120000 src/leap/bitmask/mail/imap/tests/rfc822.message create mode 120000 src/leap/bitmask/mail/imap/tests/rfc822.multi-minimal.message create mode 120000 src/leap/bitmask/mail/imap/tests/rfc822.multi-nested.message create mode 120000 src/leap/bitmask/mail/imap/tests/rfc822.multi-signed.message create mode 120000 src/leap/bitmask/mail/imap/tests/rfc822.multi.message create mode 120000 src/leap/bitmask/mail/imap/tests/rfc822.plain.message create mode 100755 src/leap/bitmask/mail/imap/tests/stress_tests_imap.zsh create mode 100644 src/leap/bitmask/mail/imap/tests/test_imap.py create mode 100644 src/leap/bitmask/mail/imap/tests/walktree.py create mode 100644 src/leap/bitmask/mail/incoming/__init__.py create mode 100644 src/leap/bitmask/mail/incoming/service.py create mode 100644 src/leap/bitmask/mail/incoming/tests/rfc822.multi-encrypt-signed.message create mode 100644 src/leap/bitmask/mail/incoming/tests/test_incoming_mail.py create mode 100644 src/leap/bitmask/mail/interfaces.py create mode 100644 src/leap/bitmask/mail/load_tests.py create mode 100644 src/leap/bitmask/mail/mail.py create mode 100644 src/leap/bitmask/mail/mailbox_indexer.py create mode 100644 src/leap/bitmask/mail/outgoing/__init__.py create mode 100644 src/leap/bitmask/mail/outgoing/service.py create mode 100644 src/leap/bitmask/mail/outgoing/tests/test_outgoing.py create mode 100644 src/leap/bitmask/mail/plugins/__init__.py create mode 100644 src/leap/bitmask/mail/plugins/soledad_sync_hooks.py create mode 100644 src/leap/bitmask/mail/rfc3156.py create mode 100644 src/leap/bitmask/mail/size.py create mode 100644 src/leap/bitmask/mail/smtp/README.rst create mode 100644 src/leap/bitmask/mail/smtp/__init__.py create mode 100644 src/leap/bitmask/mail/smtp/bounces.py create mode 100644 src/leap/bitmask/mail/smtp/gateway.py create mode 100644 src/leap/bitmask/mail/smtp/tests/185CA770.key create mode 100644 src/leap/bitmask/mail/smtp/tests/185CA770.pub create mode 100644 src/leap/bitmask/mail/smtp/tests/cert/server.crt create mode 100644 src/leap/bitmask/mail/smtp/tests/cert/server.key create mode 100644 src/leap/bitmask/mail/smtp/tests/mail.txt create mode 100644 src/leap/bitmask/mail/smtp/tests/test_gateway.py create mode 100644 src/leap/bitmask/mail/sync_hooks.py create mode 100644 src/leap/bitmask/mail/testing/__init__.py create mode 100644 src/leap/bitmask/mail/testing/__init__.py~ create mode 100644 src/leap/bitmask/mail/testing/common.py create mode 100644 src/leap/bitmask/mail/testing/imap.py create mode 100644 src/leap/bitmask/mail/testing/smtp.py create mode 100644 src/leap/bitmask/mail/tests/rfc822.bounce.message create mode 100644 src/leap/bitmask/mail/tests/rfc822.message create mode 100644 src/leap/bitmask/mail/tests/rfc822.multi-minimal.message create mode 100644 src/leap/bitmask/mail/tests/rfc822.multi-nested.message create mode 100644 src/leap/bitmask/mail/tests/rfc822.multi-signed.message create mode 100644 src/leap/bitmask/mail/tests/rfc822.multi.message create mode 100644 src/leap/bitmask/mail/tests/rfc822.plain.message create mode 100644 src/leap/bitmask/mail/tests/test_mail.py create mode 100644 src/leap/bitmask/mail/tests/test_mailbox_indexer.py create mode 100644 src/leap/bitmask/mail/tests/test_walk.py create mode 100644 src/leap/bitmask/mail/utils.py create mode 100644 src/leap/bitmask/mail/walk.py diff --git a/docs/bitmask-mail-index.rst b/docs/bitmask-mail-index.rst new file mode 100644 index 0000000..a2133f4 --- /dev/null +++ b/docs/bitmask-mail-index.rst @@ -0,0 +1,96 @@ +.. leap.mail documentation master file, created by + sphinx-quickstart on Mon Aug 25 19:19:48 2014. + You can adapt this file completely to your liking, but it should at least + contain the root `toctree` directive. + +leap.mail +========= + +*decentralized and secure mail delivery and synchronization* + +This is the documentation for the ``leap.mail`` module. It is a `twisted`_ +package that allows to receive, process, send and access existing messages using +the `LEAP`_ platform. + +One way to use this library is to let it launch two standard mail services, +``smtp`` and ``imap``, that run as local proxies and interact with a remote +``LEAP`` provider that offers *a soledad syncronization endpoint* and receives +the outgoing email. This is what `Bitmask`_ client does. + +From the release 0.4.0 on, it's also possible to use a protocol-agnostic email +public API, so that third party mail clients can manipulate the data layer. This +is what the awesome MUA in the `Pixelated`_ project is using. + +.. _`twisted`: https://twistedmatrix.com/trac/ +.. _`LEAP`: https://leap.se/en/docs +.. _`Bitmask`: https://bitmask.net/en/features#email +.. _`Pixelated`: https://pixelated-project.org/ + +How does this all work? +----------------------- + +All the underlying data storage and sync is handled by a library called +`soledad`_, which handles encryption, storage and sync. Based on `u1db`_, +documents are stored locally as local ``sqlcipher`` tables, and syncs against +the soledad sync service in the provider. + +OpenPGP key generation and keyring management are handled by another leap +python library: `keymanager`_. + +See :ref:`the life cycle of a leap email ` for an overview of the life cycle +of an email through ``LEAP`` providers. + +.. _`Soledad`: https://leap.se/en/docs/design/soledad +.. _`u1db`: https://en.wikipedia.org/wiki/U1DB +.. _`keymanager`: https://github.com/leapcode/keymanager/ + + +Data model +---------- +.. TODO clear document types documentation. + +The data model at the present moment consists of several *document types* that split email into +different documents that are stored in ``Soledad``. The idea behind this is to +keep clear the separation between *mutable* and *inmutable* parts, and still being able to +reconstruct arbitrarily nested email structures easily. + +Documentation index +=================== + +.. +.. Contents: +.. toctree:: + :maxdepth: 2 + + hacking + +.. intro +.. tutorial + + +API documentation +----------------- + +If you were looking for the documentation of the ``leap.mail`` module, you will +find it here. + +Of special interest is the `public mail api`_, which should remain relatively +stable across the next few releases. + +.. _`public mail api`: api/mail.html#module-mail + + +.. toctree:: + :maxdepth: 2 + + api/leap.mail + + + +Indices and tables +================== + +* :ref:`genindex` +* :ref:`modindex` +* :ref:`search` + diff --git a/docs/mail-hacking.rst b/docs/mail-hacking.rst new file mode 100644 index 0000000..6c49c21 --- /dev/null +++ b/docs/mail-hacking.rst @@ -0,0 +1,195 @@ +.. _hacking: + +======== +Hacking +======== + +Some hints oriented to `leap.mail` hackers. These notes are mostly related to +the imap server, although they probably will be useful for other pieces too. + +Don't panic! Just manhole into it +================================= + +If you want to inspect the objects living in your application memory, in +realtime, you can manhole into it. + +First of all, check that the modules ``PyCrypto`` and ``pyasn1`` are installed +into your system, they are needed for it to work. + +You just have to pass the ``LEAP_MAIL_MANHOLE=1`` enviroment variable while +launching the client:: + + LEAP_MAIL_MANHOLE=1 bitmask --debug + +And then you can ssh into your application! (password is "leap"):: + + ssh boss@localhost -p 2222 + +Did I mention how *awesome* twisted is?? ``:)`` + + +Profiling +========= +If using ``twistd`` to launch the server, you can use twisted profiling +capabities:: + + LEAP_MAIL_CONFIG=~/.leapmailrc twistd --profile=/tmp/mail-profiling -n -y imap-server.tac + +``--profiler`` option allows you to select different profilers (default is +"hotshot"). + +You can also do profiling when using the ``bitmask`` client. Enable the +``LEAP_PROFILE_IMAPCMD`` environment flag to get profiling of certain IMAP +commands:: + + LEAP_PROFILE_IMAPCMD=1 bitmask --debug + +Offline mode +============ + +The client has an ``--offline`` flag that will make the Mail services (imap, +currently) not try to sync with remote replicas. Very useful during development, +although you need to login with the remote server at least once before being +able to use it. + +Mutt config +=========== + +You cannot live without mutt? You're lucky! Use the following minimal config +with the imap service:: + + set folder="imap://user@provider@localhost:1984" + set spoolfile="imap://user@provider@localhost:1984/INBOX" + set ssl_starttls = no + set ssl_force_tls = no + set imap_pass=MAHSIKRET + + +Running the service with twistd +=============================== + +In order to run the mail service (currently, the imap server only), you will +need a config with this info:: + + [leap_mail] + userid = "user@provider" + uuid = "deadbeefdeadabad" + passwd = "foobar" # Optional + +In the ``LEAP_MAIL_CONFIG`` enviroment variable. If you do not specify a password +parameter, you'll be prompted for it. + +In order to get the user uid (uuid), look into the +``~/.config/leap/leap-backend.conf`` file after you have logged in into your +provider at least once. + +Run the twisted service:: + + LEAP_MAIL_CONFIG=~/.leapmailrc twistd -n -y imap-server.tac + +Now you can telnet into your local IMAP server and read your mail like a real +programmer™:: + + % telnet localhost 1984 + Trying 127.0.0.1... + Connected to localhost. + Escape character is '^]'. + * OK [CAPABILITY IMAP4rev1 LITERAL+ IDLE NAMESPACE] Twisted IMAP4rev1 Ready + tag LOGIN me@myprovider.net mahsikret + tag OK LOGIN succeeded + tag SELECT Inbox + * 2 EXISTS + * 1 RECENT + * FLAGS (\Seen \Answered \Flagged \Deleted \Draft \Recent List) + * OK [UIDVALIDITY 1410453885932] UIDs valid + tag OK [READ-WRITE] SELECT successful + ^] + telnet> Connection closed. + + +Although you probably prefer to use ``offlineimap`` for tests:: + + offlineimap -c LEAPofflineimapRC-tests + + +Minimal offlineimap configuration +--------------------------------- + +You can use this as a sample offlineimap config file:: + + [general] + accounts = leap-local + + [Account leap-local] + localrepository = LocalLeap + remoterepository = RemoteLeap + + [Repository LocalLeap] + type = Maildir + localfolders = ~/LEAPMail/Mail + + [Repository RemoteLeap] + type = IMAP + ssl = no + remotehost = localhost + remoteport = 1984 + remoteuser = user + remotepass = pass + +Testing utilities +----------------- +There are a bunch of utilities to test IMAP delivery in ``imap/tests`` folder. +If looking for a quick way of inspecting mailboxes, have a look at ``getmail``:: + + ./getmail me@testprovider.net mahsikret + 1. Drafts + 2. INBOX + 3. Trash + Which mailbox? [1] 2 + 1 Subject: this is the time of the revolution + 2 Subject: ignore me + + Which message? [1] (Q quits) 1 + 1 X-Leap-Provenance: Thu, 11 Sep 2014 16:52:11 -0000; pubkey="C1F8DE10BD151F99" + Received: from mx1.testprovider.net(mx1.testprovider.net [198.197.196.195]) + (using TLSv1.2 with cipher ECDHE-RSA-AES256-GCM-SHA384 (256/256 bits)) + (Client CN "*.foobar.net", Issuer "Gandi Standard SSL CA" (not verified)) + by blackhole (Postfix) with ESMTPS id DEADBEEF + for ; Thu, 11 Sep 2014 16:52:10 +0000 (UTC) + Delivered-To: 926d4915cfd42b6d96d38660c04613af@testprovider.net + Message-Id: <20140911165205.GB8054@samsara> + From: Kali + + (snip) + +IMAP Message Rendering Regressions +---------------------------------- + +For testing the IMAP server implementation, there is a litte regressions script +that needs some manual work from your side. + +First of all, you need an already initialized account. Which for now basically +means you have created a new account with a provider that offers the Encrypted +Mail Service, using the Bitmask Client wizard. Then you need to log in with that +account, and let it generate the secrets and sync with the remote for a first +time. After this you can run the twistd server locally and offline. + +From the ``leap.mail.imap.tests`` folder, and with an already initialized server +running:: + + ./regressions_mime_struct user@provider pass path_to_samples/ + +You can find several message samples in the ``leap/mail/tests`` folder. + + +Debugging IMAP commands +======================= + +Use ``ngrep`` to obtain logs of the commands:: + + sudo ngrep -d lo -W byline port 1984 + +To get verbose output from thunderbird/icedove, set the following environment +variable:: + + NSPR_LOG_MODULES="imap:5" icedove diff --git a/docs/mail-journey.rst b/docs/mail-journey.rst new file mode 100644 index 0000000..7e64f18 --- /dev/null +++ b/docs/mail-journey.rst @@ -0,0 +1,40 @@ +.. _mail_journey: + +The life cycle of a LEAP Email +============================== +The following are just some notes to facilitate the understanding of the +leap.mail internals to developers and collaborators. + +Server-side: receiving mail from the outside world +-------------------------------------------------- + +1. the mx server receives an email, it gets through all the postfix validation and it's written into disk +2. ``leap_mx`` gets that write in disk notification and encrypts the incoming mail to its intended recipient's pubkey +3. that encrypted blob gets written into couchdb in a way soledad will catch it in the next sync + + +Client-side: fetching and processing incoming email +--------------------------------------------------- +you have an imap and an smtp local server. For IMAP: + +1. soledad syncs +2. **fetch** module sees if there's new encrypted mail for the current user from leap_mx +3. if there is, the mail is decrypted using the user private key, and splitted + into several parts according to the internal mail data model (separating + mutable and inmutable email parts). Those documents it encrypts it properly + like other soledad documents and deletes the pubkey encrypted doc +4. desktop client is notified that there are N new mails +5. when a MUA connects to the **imap** local server, the different documents are glued + together and presented as response to the different imap commands. + + +Client side: sending email +-------------------------- + +1. you write an email to a recipient and hit send +2. the **smtp** local server gets that mail, it checks from whom it is and to whom it is for +3. it signs the mail with the ``From:``'s privkey +4. it retrieves ``To:``'s pubkey with the keymanager and if it finds it encrypts the mail to him/her +5. if it didn't find it and you don't have your client configured to "always encrypt", it goes out with just the signature +6. else, it fails out complaining about this conflict +7. that mail gets relayed to the provider's **smtp** server diff --git a/mail/.gitattributes b/mail/.gitattributes deleted file mode 100644 index 5041731..0000000 --- a/mail/.gitattributes +++ /dev/null @@ -1 +0,0 @@ -src/leap/mail/_version.py export-subst diff --git a/mail/.gitignore b/mail/.gitignore deleted file mode 100644 index aafbdd1..0000000 --- a/mail/.gitignore +++ /dev/null @@ -1,25 +0,0 @@ -*.pyc -dropin.cache -build/ -dist/ -*.egg -*.egg-info -src/_trial_temp -*.swp -*.swo -.* -!.coveragerc -!.tx -!.gitignore -!.gitattributes -bin/ -core -docs/_build -include/ -lib/ -local/ -share/ -MANIFEST -twistd.pid -_trial_temp -_trial_temp.lock diff --git a/mail/AUTHORS b/mail/AUTHORS deleted file mode 100644 index db070f4..0000000 --- a/mail/AUTHORS +++ /dev/null @@ -1,8 +0,0 @@ -Kali Kaneko -Tomás Touceda -drebs -Ivan Alejandro -Ruben Pollan -Bruno Wagner Goncalves -Duda Dornelles -Folker Bernitt diff --git a/mail/CHANGELOG.rst b/mail/CHANGELOG.rst deleted file mode 100644 index 22abf22..0000000 --- a/mail/CHANGELOG.rst +++ /dev/null @@ -1,37 +0,0 @@ -0.4.2 - 13 May, 2016 -+++++++++++++++++++++ - -Bugfixes -~~~~~~~~ -- `#8083 `_: Allow pixelated UA not interfere with Thunderbird operation. -- Cast local identity (version string) to bytes, avoid TLS transport raising exception. - - -0.4.1 - 18 Apr, 2016 -+++++++++++++++++++++ - -Features -~~~~~~~~ -- `#7656 `_: Emit multi-user aware events. -- `#4008 `_: Add token-based authentication to local IMAP/SMTP services. -- `#7889 `_: Use cryptography instead of pycryptopp to reduce dependencies. -- `#7263 `_: Implement local bounces to notify user of SMTP delivery errors. -- Use twisted.cred to authenticate IMAP/SMTP users. -- Verify plain text signed email. -- Validate signature with attachments. -- Use fingerprint instead of key_id to address keys. - - -Bugfixes -~~~~~~~~ -- `#7861 `_: Use the right succeed function for passthrough encrypted email. -- `#7898 `_: Fix IMAP fetch headers -- `#7977 `_: Decode attached keys so they are recognized by keymanager. -- `#7952 `_: Specify openssl backend explicitely. -- Fix the get_body logic for corner-cases in which body is None (yet-to-be synced docs, mainly). -- Let the inbox used in IncomingMail notify any subscribed Mailbox. -- Adds user_id to Account (fixes Pixelated mail leakage). - -Misc -~~~~ -- Change IMAPAccount signature, for consistency with a previous Account change. diff --git a/mail/HISTORY b/mail/HISTORY deleted file mode 100644 index 6ca54e7..0000000 --- a/mail/HISTORY +++ /dev/null @@ -1,179 +0,0 @@ -0.4.0 Oct 28, 2015: - o Expose generic and protocol-agnostic public mail API. - o Make use of the twisted-based, async soledad API. - o Create a OutgoingMail class that has the logic for encrypting, signing and - sending messages. Factors that logic out of EncryptedMessage so it can be - used by other clients. Closes: #6357. - o Refactor email fetching outside IMAP to its own independient IncomingMail - class. Closes: #6361. - o Adapt to new events api on leap.common. Related to #5359. - o Discover public keys via attachment. Closes: #5937. - o Add public key as attachment. Closes: #6617. - o Parse OpenPGP header and import keys from it. Closes: #3879. - o Don't add any footer to the emails. Closes: #4692. - o Add listener for each email added to inbox in IncomingMail. Closes: #6742. - o Ability to reindex local UIDs after a soledad sync. Closes: #6996. - o Feature: add very basic support for message sequence numbers. - o Send a BYE command to all open connections, so that the MUA is notified - when the server is shutted down. - o Fix nested multipart rendering. Closes: #7244 - o Update SMTP gateway docs. Closes #7169. - o Bugfix: fix keyerror when inserting msg on pending_inserts dict. - o Bugfix: Return the first cdoc if no body found - o Lots of style fixes and tests updates. - o If the auth token has expired signal the GUI to request her to log in again - (Closes: #7430) - o don't extract openpgp header if valid attached key (Closes: #7480) - o disable local only tcp bind on docker containers to allow access to IMAP - and SMTP. Related to #7471. - -0.3.10 Sept 26, 2014: - o MessageCollection iterator now creates the LeapMessage with the - collection reference, so setFlags will work properly. - o account#addMailbox can't allow empty mailbox names since it makes - it impossible to create it later (mailbox#__init__ will throw an - error), which makes it impossible to getMailbox or even delete it. - -0.3.9 Apr 4, 2014: - o Footer url shouldn't end in period. Closes #4791. - o Handle non-ascii headers. Closes #5021. - o Soledad writer consumes messages eagerly. Fixes failing - tests. Closes #4715. - o Convert unicode to str when raising exceptions in IMAP server. - Fixes #4830. - o Remove conversion of IMAP folder names to string. This makes the - IMAP server use twisted's transparent 7bit conversion. Fixes - #4830. - o Add a flag to be able to reset the session. Closes #4925. - o Check for none in payload detection. Closes #4933. - o Check for flags doc uniqueness before adding a message. Avoids - duplicates of a single message in the same mailbox while copying - or moving. Closes #4949. - o Correctly process attachments when signing. Fixes #5014. - o Fix bug in which destination folder sometimes was not showing - messages after copy/append. Closes #5167. - o Fix unread notifications to client UI. Only INBOX is - notified. Closes #5177. - o Fix bug in which deleted folder wouldn't show its messages - inside. Closes #5179. - o Keep processing after a decryption error. Closes #5307. - o Enqueue unsetting of recent flag. this was holding the new mails - from being displayed soonish. - o Properly parse emails crafted by Mail.app. Fixes #5013. - o Restrict adding outgoing footer to text/plain messages. - o Sanity check on last_uid setter. Avoids incomplete fetches. - o Stop providing hostname for helo in smtp gateway. Fixes #4335. - o Only try to fetch keys for multipart signed or encrypted emails. - Fixes #4671. - o Add a flag for offline mode in imap. Related to #4943. - o Flush IMAP data to disk when stopping. Closes #5095. - o Signal the client when auth token is invalid for syncing Soledad. - Fixes #5191. - o Ability to support SEARCH Commands, limited to HEADER Message-ID. - This is a quick workaround for avoiding duplicate saves in Drafts - Folder. Closes #4209. - o Use a memory store as write-buffer and read-cache. - o Implement IMAP4 non-synchronizing literals (rfc2088), so APPENDs - can be made in a single round-trip. Closes #5190. - o Defer costly operations to a pool of threads. - o Split the internal representation of messages into three distinct - documents: 1) Flags 2) Headers 3) Content. - o Make use of the Twisted MIME interface. - o Add deduplication ability to the save operation, for body and - attachments. - o Add IMessageCopier interface to mailbox implementation, so bulk - moves are costless. Closes #4654. - o Makes efficient use of indexes and count method. Closes #4616. - o Handle correctly unicode characters in emails. Closes #4838. - -0.3.8 Dec 6, 2013: - o Fail gracefully when failing to decrypt incoming messages. Closes - #4589. - o Fix a bug when adding a message with empty flags. Closes #4496 - o Allow to iterate in an empty mailbox during fetch. Closes #4603 - o Add 'signencrypt' preference to OpenPGP header on outgoing - email. Closes #3878. - o Add a header to incoming emails that reflects if a valid signature - was found when decrypting. Closes #4354. - o Add a footer to outgoing email pointing to the address where - sender keys can be fetched. Closes #4526. - o Serialize Soledad Writes for new messages. Fixes segmentation - fault when sqlcipher was been concurrently accessed from many - threads. Closes #4606 - o Set remote mail polling time to 60 seconds. Closes #4499 - -0.3.7 Nov 15, 2013: - o Uses deferToThread for sendMail. Closes #3937 - o Update pkey to allow multiple accounts. Solves: #4394 - o Change SMTP service name from "relay" to "gateway". Closes #4416. - o Identify ourselves with a fqdn, always. Closes: #4441 - o Remove 'multipart/encrypted' header after decrypting incoming - mail. Closes #4454. - o Fix several bugs with imap mailbox getUIDNext and notifiers that - were breaking the mail indexing after message deletion. This - solves also the perceived mismatch between the number of unread - mails reported by bitmask_client and the number reported by - MUAs. Closes: #4461 - o Check username in authentications. Closes: #4299 - o Reject senders that aren't the user that is currently logged - in. Fixes #3952. - o Prevent already encrypted outgoing messages from being encrypted - again. Closes #4324. - o Correctly handle email headers when gatewaying messages. Also add - OpenPGP header. Closes #4322 and #4447. - -0.3.6 Nov 1, 2013: - o Add support for non-ascii characters in emails. Closes #4000. - o Default to UTF-8 when there is no charset parsed from the mail - contents. - o Refactor get_email_charset to leap.common. - o Return the necessary references (factory, port) from IMAP4 launch - in order to be able to properly stop it. Related to #4199. - o Notify MUA of new mail, using IDLE as advertised. Closes: #3671 - o Use TLS wrapper mode instead of STARTTLS. Closes #3637. - -0.3.5 Oct 18, 2013: - o Do not log mail doc contents. - o Comply with RFC 3156. Closes #4029. - -0.3.4 Oct 4, 2013: - o Improve charset handling when exposing mails to the mail - client. Related to #3660. - o Return Twisted's smtp Port object to be able to stop listening to - it whenever we want. Related to #3873. - -0.3.3 Sep 20, 2013: - o Remove cleartext mail from logs. Closes: #3877. - -0.3.2 Sep 6, 2013: - o Make mail services bind to 127.0.0.1. Closes: #3627. - o Signal unread message to UI when message is saved locally. Closes: - #3654. - o Signal unread to UI when flag in message change. Closes: #3662. - o Use dirspec instead of plain xdg. Closes #3574. - o SMTP service invocation returns factory instance. - -0.3.1 Aug 23, 2013: - o Avoid logging dummy password on imap server. Closes: #3416 - o Do not fail while processing an empty mail, just skip it. Fixes - #3457. - o Notify of unread email explicitly every time the mailbox is - sync'ed. - o Fix signals to emit only string in the contents instead of bool or - int values. - o Improve unseen filter of email. - o Make default imap fetch period 5 minutes. Client can config it via - environment variable for debug. Closes: #3409 - o Refactor imap fetch code for better defer handling. Closes: #3423 - o Emit signals to notify UI for SMTP relay events. Closes #3464. - o Add events for notifications about imap activity. Closes: #3480 - o Update to new soledad package scheme (common, client and - server). Closes #3487. - o Improve packaging: add versioneer, parse_requirements, - classifiers. - -0.3.0 Aug 9, 2013: - o Add dependency for leap.keymanager. - o User 1984 default port for imap. - o Add client certificate authentication. Closes #3376. - o SMTP relay signs outgoing messages. diff --git a/mail/LICENSE b/mail/LICENSE deleted file mode 100644 index bc08fe2..0000000 --- a/mail/LICENSE +++ /dev/null @@ -1,619 +0,0 @@ - GNU GENERAL PUBLIC LICENSE - Version 3, 29 June 2007 - - Copyright (C) 2007 Free Software Foundation, Inc. - Everyone is permitted to copy and distribute verbatim copies - of this license document, but changing it is not allowed. - - Preamble - - The GNU General Public License is a free, copyleft license for -software and other kinds of works. - - The licenses for most software and other practical works are designed -to take away your freedom to share and change the works. By contrast, -the GNU General Public License is intended to guarantee your freedom to -share and change all versions of a program--to make sure it remains free -software for all its users. We, the Free Software Foundation, use the -GNU General Public License for most of our software; it applies also to -any other work released this way by its authors. You can apply it to -your programs, too. - - When we speak of free software, we are referring to freedom, not -price. Our General Public Licenses are designed to make sure that you -have the freedom to distribute copies of free software (and charge for -them if you wish), that you receive source code or can get it if you -want it, that you can change the software or use pieces of it in new -free programs, and that you know you can do these things. - - To protect your rights, we need to prevent others from denying you -these rights or asking you to surrender the rights. Therefore, you have -certain responsibilities if you distribute copies of the software, or if -you modify it: responsibilities to respect the freedom of others. - - For example, if you distribute copies of such a program, whether -gratis or for a fee, you must pass on to the recipients the same -freedoms that you received. You must make sure that they, too, receive -or can get the source code. And you must show them these terms so they -know their rights. - - Developers that use the GNU GPL protect your rights with two steps: -(1) assert copyright on the software, and (2) offer you this License -giving you legal permission to copy, distribute and/or modify it. - - For the developers' and authors' protection, the GPL clearly explains -that there is no warranty for this free software. For both users' and -authors' sake, the GPL requires that modified versions be marked as -changed, so that their problems will not be attributed erroneously to -authors of previous versions. - - Some devices are designed to deny users access to install or run -modified versions of the software inside them, although the manufacturer -can do so. This is fundamentally incompatible with the aim of -protecting users' freedom to change the software. The systematic -pattern of such abuse occurs in the area of products for individuals to -use, which is precisely where it is most unacceptable. Therefore, we -have designed this version of the GPL to prohibit the practice for those -products. If such problems arise substantially in other domains, we -stand ready to extend this provision to those domains in future versions -of the GPL, as needed to protect the freedom of users. - - Finally, every program is threatened constantly by software patents. -States should not allow patents to restrict development and use of -software on general-purpose computers, but in those that do, we wish to -avoid the special danger that patents applied to a free program could -make it effectively proprietary. To prevent this, the GPL assures that -patents cannot be used to render the program non-free. - - The precise terms and conditions for copying, distribution and -modification follow. - - TERMS AND CONDITIONS - - 0. Definitions. - - "This License" refers to version 3 of the GNU General Public License. - - "Copyright" also means copyright-like laws that apply to other kinds of -works, such as semiconductor masks. - - "The Program" refers to any copyrightable work licensed under this -License. Each licensee is addressed as "you". "Licensees" and -"recipients" may be individuals or organizations. - - To "modify" a work means to copy from or adapt all or part of the work -in a fashion requiring copyright permission, other than the making of an -exact copy. The resulting work is called a "modified version" of the -earlier work or a work "based on" the earlier work. - - A "covered work" means either the unmodified Program or a work based -on the Program. - - To "propagate" a work means to do anything with it that, without -permission, would make you directly or secondarily liable for -infringement under applicable copyright law, except executing it on a -computer or modifying a private copy. Propagation includes copying, -distribution (with or without modification), making available to the -public, and in some countries other activities as well. - - To "convey" a work means any kind of propagation that enables other -parties to make or receive copies. Mere interaction with a user through -a computer network, with no transfer of a copy, is not conveying. - - An interactive user interface displays "Appropriate Legal Notices" -to the extent that it includes a convenient and prominently visible -feature that (1) displays an appropriate copyright notice, and (2) -tells the user that there is no warranty for the work (except to the -extent that warranties are provided), that licensees may convey the -work under this License, and how to view a copy of this License. If -the interface presents a list of user commands or options, such as a -menu, a prominent item in the list meets this criterion. - - 1. Source Code. - - The "source code" for a work means the preferred form of the work -for making modifications to it. "Object code" means any non-source -form of a work. - - A "Standard Interface" means an interface that either is an official -standard defined by a recognized standards body, or, in the case of -interfaces specified for a particular programming language, one that -is widely used among developers working in that language. - - The "System Libraries" of an executable work include anything, other -than the work as a whole, that (a) is included in the normal form of -packaging a Major Component, but which is not part of that Major -Component, and (b) serves only to enable use of the work with that -Major Component, or to implement a Standard Interface for which an -implementation is available to the public in source code form. A -"Major Component", in this context, means a major essential component -(kernel, window system, and so on) of the specific operating system -(if any) on which the executable work runs, or a compiler used to -produce the work, or an object code interpreter used to run it. - - The "Corresponding Source" for a work in object code form means all -the source code needed to generate, install, and (for an executable -work) run the object code and to modify the work, including scripts to -control those activities. However, it does not include the work's -System Libraries, or general-purpose tools or generally available free -programs which are used unmodified in performing those activities but -which are not part of the work. For example, Corresponding Source -includes interface definition files associated with source files for -the work, and the source code for shared libraries and dynamically -linked subprograms that the work is specifically designed to require, -such as by intimate data communication or control flow between those -subprograms and other parts of the work. - - The Corresponding Source need not include anything that users -can regenerate automatically from other parts of the Corresponding -Source. - - The Corresponding Source for a work in source code form is that -same work. - - 2. Basic Permissions. - - All rights granted under this License are granted for the term of -copyright on the Program, and are irrevocable provided the stated -conditions are met. This License explicitly affirms your unlimited -permission to run the unmodified Program. The output from running a -covered work is covered by this License only if the output, given its -content, constitutes a covered work. This License acknowledges your -rights of fair use or other equivalent, as provided by copyright law. - - You may make, run and propagate covered works that you do not -convey, without conditions so long as your license otherwise remains -in force. You may convey covered works to others for the sole purpose -of having them make modifications exclusively for you, or provide you -with facilities for running those works, provided that you comply with -the terms of this License in conveying all material for which you do -not control copyright. Those thus making or running the covered works -for you must do so exclusively on your behalf, under your direction -and control, on terms that prohibit them from making any copies of -your copyrighted material outside their relationship with you. - - Conveying under any other circumstances is permitted solely under -the conditions stated below. Sublicensing is not allowed; section 10 -makes it unnecessary. - - 3. Protecting Users' Legal Rights From Anti-Circumvention Law. - - No covered work shall be deemed part of an effective technological -measure under any applicable law fulfilling obligations under article -11 of the WIPO copyright treaty adopted on 20 December 1996, or -similar laws prohibiting or restricting circumvention of such -measures. - - When you convey a covered work, you waive any legal power to forbid -circumvention of technological measures to the extent such circumvention -is effected by exercising rights under this License with respect to -the covered work, and you disclaim any intention to limit operation or -modification of the work as a means of enforcing, against the work's -users, your or third parties' legal rights to forbid circumvention of -technological measures. - - 4. Conveying Verbatim Copies. - - You may convey verbatim copies of the Program's source code as you -receive it, in any medium, provided that you conspicuously and -appropriately publish on each copy an appropriate copyright notice; -keep intact all notices stating that this License and any -non-permissive terms added in accord with section 7 apply to the code; -keep intact all notices of the absence of any warranty; and give all -recipients a copy of this License along with the Program. - - You may charge any price or no price for each copy that you convey, -and you may offer support or warranty protection for a fee. - - 5. Conveying Modified Source Versions. - - You may convey a work based on the Program, or the modifications to -produce it from the Program, in the form of source code under the -terms of section 4, provided that you also meet all of these conditions: - - a) The work must carry prominent notices stating that you modified - it, and giving a relevant date. - - b) The work must carry prominent notices stating that it is - released under this License and any conditions added under section - 7. This requirement modifies the requirement in section 4 to - "keep intact all notices". - - c) You must license the entire work, as a whole, under this - License to anyone who comes into possession of a copy. This - License will therefore apply, along with any applicable section 7 - additional terms, to the whole of the work, and all its parts, - regardless of how they are packaged. This License gives no - permission to license the work in any other way, but it does not - invalidate such permission if you have separately received it. - - d) If the work has interactive user interfaces, each must display - Appropriate Legal Notices; however, if the Program has interactive - interfaces that do not display Appropriate Legal Notices, your - work need not make them do so. - - A compilation of a covered work with other separate and independent -works, which are not by their nature extensions of the covered work, -and which are not combined with it such as to form a larger program, -in or on a volume of a storage or distribution medium, is called an -"aggregate" if the compilation and its resulting copyright are not -used to limit the access or legal rights of the compilation's users -beyond what the individual works permit. Inclusion of a covered work -in an aggregate does not cause this License to apply to the other -parts of the aggregate. - - 6. Conveying Non-Source Forms. - - You may convey a covered work in object code form under the terms -of sections 4 and 5, provided that you also convey the -machine-readable Corresponding Source under the terms of this License, -in one of these ways: - - a) Convey the object code in, or embodied in, a physical product - (including a physical distribution medium), accompanied by the - Corresponding Source fixed on a durable physical medium - customarily used for software interchange. - - b) Convey the object code in, or embodied in, a physical product - (including a physical distribution medium), accompanied by a - written offer, valid for at least three years and valid for as - long as you offer spare parts or customer support for that product - model, to give anyone who possesses the object code either (1) a - copy of the Corresponding Source for all the software in the - product that is covered by this License, on a durable physical - medium customarily used for software interchange, for a price no - more than your reasonable cost of physically performing this - conveying of source, or (2) access to copy the - Corresponding Source from a network server at no charge. - - c) Convey individual copies of the object code with a copy of the - written offer to provide the Corresponding Source. This - alternative is allowed only occasionally and noncommercially, and - only if you received the object code with such an offer, in accord - with subsection 6b. - - d) Convey the object code by offering access from a designated - place (gratis or for a charge), and offer equivalent access to the - Corresponding Source in the same way through the same place at no - further charge. You need not require recipients to copy the - Corresponding Source along with the object code. If the place to - copy the object code is a network server, the Corresponding Source - may be on a different server (operated by you or a third party) - that supports equivalent copying facilities, provided you maintain - clear directions next to the object code saying where to find the - Corresponding Source. Regardless of what server hosts the - Corresponding Source, you remain obligated to ensure that it is - available for as long as needed to satisfy these requirements. - - e) Convey the object code using peer-to-peer transmission, provided - you inform other peers where the object code and Corresponding - Source of the work are being offered to the general public at no - charge under subsection 6d. - - A separable portion of the object code, whose source code is excluded -from the Corresponding Source as a System Library, need not be -included in conveying the object code work. - - A "User Product" is either (1) a "consumer product", which means any -tangible personal property which is normally used for personal, family, -or household purposes, or (2) anything designed or sold for incorporation -into a dwelling. In determining whether a product is a consumer product, -doubtful cases shall be resolved in favor of coverage. For a particular -product received by a particular user, "normally used" refers to a -typical or common use of that class of product, regardless of the status -of the particular user or of the way in which the particular user -actually uses, or expects or is expected to use, the product. A product -is a consumer product regardless of whether the product has substantial -commercial, industrial or non-consumer uses, unless such uses represent -the only significant mode of use of the product. - - "Installation Information" for a User Product means any methods, -procedures, authorization keys, or other information required to install -and execute modified versions of a covered work in that User Product from -a modified version of its Corresponding Source. The information must -suffice to ensure that the continued functioning of the modified object -code is in no case prevented or interfered with solely because -modification has been made. - - If you convey an object code work under this section in, or with, or -specifically for use in, a User Product, and the conveying occurs as -part of a transaction in which the right of possession and use of the -User Product is transferred to the recipient in perpetuity or for a -fixed term (regardless of how the transaction is characterized), the -Corresponding Source conveyed under this section must be accompanied -by the Installation Information. But this requirement does not apply -if neither you nor any third party retains the ability to install -modified object code on the User Product (for example, the work has -been installed in ROM). - - The requirement to provide Installation Information does not include a -requirement to continue to provide support service, warranty, or updates -for a work that has been modified or installed by the recipient, or for -the User Product in which it has been modified or installed. Access to a -network may be denied when the modification itself materially and -adversely affects the operation of the network or violates the rules and -protocols for communication across the network. - - Corresponding Source conveyed, and Installation Information provided, -in accord with this section must be in a format that is publicly -documented (and with an implementation available to the public in -source code form), and must require no special password or key for -unpacking, reading or copying. - - 7. Additional Terms. - - "Additional permissions" are terms that supplement the terms of this -License by making exceptions from one or more of its conditions. -Additional permissions that are applicable to the entire Program shall -be treated as though they were included in this License, to the extent -that they are valid under applicable law. If additional permissions -apply only to part of the Program, that part may be used separately -under those permissions, but the entire Program remains governed by -this License without regard to the additional permissions. - - When you convey a copy of a covered work, you may at your option -remove any additional permissions from that copy, or from any part of -it. (Additional permissions may be written to require their own -removal in certain cases when you modify the work.) You may place -additional permissions on material, added by you to a covered work, -for which you have or can give appropriate copyright permission. - - Notwithstanding any other provision of this License, for material you -add to a covered work, you may (if authorized by the copyright holders of -that material) supplement the terms of this License with terms: - - a) Disclaiming warranty or limiting liability differently from the - terms of sections 15 and 16 of this License; or - - b) Requiring preservation of specified reasonable legal notices or - author attributions in that material or in the Appropriate Legal - Notices displayed by works containing it; or - - c) Prohibiting misrepresentation of the origin of that material, or - requiring that modified versions of such material be marked in - reasonable ways as different from the original version; or - - d) Limiting the use for publicity purposes of names of licensors or - authors of the material; or - - e) Declining to grant rights under trademark law for use of some - trade names, trademarks, or service marks; or - - f) Requiring indemnification of licensors and authors of that - material by anyone who conveys the material (or modified versions of - it) with contractual assumptions of liability to the recipient, for - any liability that these contractual assumptions directly impose on - those licensors and authors. - - All other non-permissive additional terms are considered "further -restrictions" within the meaning of section 10. If the Program as you -received it, or any part of it, contains a notice stating that it is -governed by this License along with a term that is a further -restriction, you may remove that term. If a license document contains -a further restriction but permits relicensing or conveying under this -License, you may add to a covered work material governed by the terms -of that license document, provided that the further restriction does -not survive such relicensing or conveying. - - If you add terms to a covered work in accord with this section, you -must place, in the relevant source files, a statement of the -additional terms that apply to those files, or a notice indicating -where to find the applicable terms. - - Additional terms, permissive or non-permissive, may be stated in the -form of a separately written license, or stated as exceptions; -the above requirements apply either way. - - 8. Termination. - - You may not propagate or modify a covered work except as expressly -provided under this License. Any attempt otherwise to propagate or -modify it is void, and will automatically terminate your rights under -this License (including any patent licenses granted under the third -paragraph of section 11). - - However, if you cease all violation of this License, then your -license from a particular copyright holder is reinstated (a) -provisionally, unless and until the copyright holder explicitly and -finally terminates your license, and (b) permanently, if the copyright -holder fails to notify you of the violation by some reasonable means -prior to 60 days after the cessation. - - Moreover, your license from a particular copyright holder is -reinstated permanently if the copyright holder notifies you of the -violation by some reasonable means, this is the first time you have -received notice of violation of this License (for any work) from that -copyright holder, and you cure the violation prior to 30 days after -your receipt of the notice. - - Termination of your rights under this section does not terminate the -licenses of parties who have received copies or rights from you under -this License. If your rights have been terminated and not permanently -reinstated, you do not qualify to receive new licenses for the same -material under section 10. - - 9. Acceptance Not Required for Having Copies. - - You are not required to accept this License in order to receive or -run a copy of the Program. Ancillary propagation of a covered work -occurring solely as a consequence of using peer-to-peer transmission -to receive a copy likewise does not require acceptance. However, -nothing other than this License grants you permission to propagate or -modify any covered work. These actions infringe copyright if you do -not accept this License. Therefore, by modifying or propagating a -covered work, you indicate your acceptance of this License to do so. - - 10. Automatic Licensing of Downstream Recipients. - - Each time you convey a covered work, the recipient automatically -receives a license from the original licensors, to run, modify and -propagate that work, subject to this License. You are not responsible -for enforcing compliance by third parties with this License. - - An "entity transaction" is a transaction transferring control of an -organization, or substantially all assets of one, or subdividing an -organization, or merging organizations. If propagation of a covered -work results from an entity transaction, each party to that -transaction who receives a copy of the work also receives whatever -licenses to the work the party's predecessor in interest had or could -give under the previous paragraph, plus a right to possession of the -Corresponding Source of the work from the predecessor in interest, if -the predecessor has it or can get it with reasonable efforts. - - You may not impose any further restrictions on the exercise of the -rights granted or affirmed under this License. For example, you may -not impose a license fee, royalty, or other charge for exercise of -rights granted under this License, and you may not initiate litigation -(including a cross-claim or counterclaim in a lawsuit) alleging that -any patent claim is infringed by making, using, selling, offering for -sale, or importing the Program or any portion of it. - - 11. Patents. - - A "contributor" is a copyright holder who authorizes use under this -License of the Program or a work on which the Program is based. The -work thus licensed is called the contributor's "contributor version". - - A contributor's "essential patent claims" are all patent claims -owned or controlled by the contributor, whether already acquired or -hereafter acquired, that would be infringed by some manner, permitted -by this License, of making, using, or selling its contributor version, -but do not include claims that would be infringed only as a -consequence of further modification of the contributor version. For -purposes of this definition, "control" includes the right to grant -patent sublicenses in a manner consistent with the requirements of -this License. - - Each contributor grants you a non-exclusive, worldwide, royalty-free -patent license under the contributor's essential patent claims, to -make, use, sell, offer for sale, import and otherwise run, modify and -propagate the contents of its contributor version. - - In the following three paragraphs, a "patent license" is any express -agreement or commitment, however denominated, not to enforce a patent -(such as an express permission to practice a patent or covenant not to -sue for patent infringement). To "grant" such a patent license to a -party means to make such an agreement or commitment not to enforce a -patent against the party. - - If you convey a covered work, knowingly relying on a patent license, -and the Corresponding Source of the work is not available for anyone -to copy, free of charge and under the terms of this License, through a -publicly available network server or other readily accessible means, -then you must either (1) cause the Corresponding Source to be so -available, or (2) arrange to deprive yourself of the benefit of the -patent license for this particular work, or (3) arrange, in a manner -consistent with the requirements of this License, to extend the patent -license to downstream recipients. "Knowingly relying" means you have -actual knowledge that, but for the patent license, your conveying the -covered work in a country, or your recipient's use of the covered work -in a country, would infringe one or more identifiable patents in that -country that you have reason to believe are valid. - - If, pursuant to or in connection with a single transaction or -arrangement, you convey, or propagate by procuring conveyance of, a -covered work, and grant a patent license to some of the parties -receiving the covered work authorizing them to use, propagate, modify -or convey a specific copy of the covered work, then the patent license -you grant is automatically extended to all recipients of the covered -work and works based on it. - - A patent license is "discriminatory" if it does not include within -the scope of its coverage, prohibits the exercise of, or is -conditioned on the non-exercise of one or more of the rights that are -specifically granted under this License. You may not convey a covered -work if you are a party to an arrangement with a third party that is -in the business of distributing software, under which you make payment -to the third party based on the extent of your activity of conveying -the work, and under which the third party grants, to any of the -parties who would receive the covered work from you, a discriminatory -patent license (a) in connection with copies of the covered work -conveyed by you (or copies made from those copies), or (b) primarily -for and in connection with specific products or compilations that -contain the covered work, unless you entered into that arrangement, -or that patent license was granted, prior to 28 March 2007. - - Nothing in this License shall be construed as excluding or limiting -any implied license or other defenses to infringement that may -otherwise be available to you under applicable patent law. - - 12. No Surrender of Others' Freedom. - - If conditions are imposed on you (whether by court order, agreement or -otherwise) that contradict the conditions of this License, they do not -excuse you from the conditions of this License. If you cannot convey a -covered work so as to satisfy simultaneously your obligations under this -License and any other pertinent obligations, then as a consequence you may -not convey it at all. For example, if you agree to terms that obligate you -to collect a royalty for further conveying from those to whom you convey -the Program, the only way you could satisfy both those terms and this -License would be to refrain entirely from conveying the Program. - - 13. Use with the GNU Affero General Public License. - - Notwithstanding any other provision of this License, you have -permission to link or combine any covered work with a work licensed -under version 3 of the GNU Affero General Public License into a single -combined work, and to convey the resulting work. The terms of this -License will continue to apply to the part which is the covered work, -but the special requirements of the GNU Affero General Public License, -section 13, concerning interaction through a network will apply to the -combination as such. - - 14. Revised Versions of this License. - - The Free Software Foundation may publish revised and/or new versions of -the GNU General Public License from time to time. Such new versions will -be similar in spirit to the present version, but may differ in detail to -address new problems or concerns. - - Each version is given a distinguishing version number. If the -Program specifies that a certain numbered version of the GNU General -Public License "or any later version" applies to it, you have the -option of following the terms and conditions either of that numbered -version or of any later version published by the Free Software -Foundation. If the Program does not specify a version number of the -GNU General Public License, you may choose any version ever published -by the Free Software Foundation. - - If the Program specifies that a proxy can decide which future -versions of the GNU General Public License can be used, that proxy's -public statement of acceptance of a version permanently authorizes you -to choose that version for the Program. - - Later license versions may give you additional or different -permissions. However, no additional obligations are imposed on any -author or copyright holder as a result of your choosing to follow a -later version. - - 15. Disclaimer of Warranty. - - THERE IS NO WARRANTY FOR THE PROGRAM, TO THE EXTENT PERMITTED BY -APPLICABLE LAW. EXCEPT WHEN OTHERWISE STATED IN WRITING THE COPYRIGHT -HOLDERS AND/OR OTHER PARTIES PROVIDE THE PROGRAM "AS IS" WITHOUT WARRANTY -OF ANY KIND, EITHER EXPRESSED OR IMPLIED, INCLUDING, BUT NOT LIMITED TO, -THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR -PURPOSE. THE ENTIRE RISK AS TO THE QUALITY AND PERFORMANCE OF THE PROGRAM -IS WITH YOU. SHOULD THE PROGRAM PROVE DEFECTIVE, YOU ASSUME THE COST OF -ALL NECESSARY SERVICING, REPAIR OR CORRECTION. - - 16. Limitation of Liability. - - IN NO EVENT UNLESS REQUIRED BY APPLICABLE LAW OR AGREED TO IN WRITING -WILL ANY COPYRIGHT HOLDER, OR ANY OTHER PARTY WHO MODIFIES AND/OR CONVEYS -THE PROGRAM AS PERMITTED ABOVE, BE LIABLE TO YOU FOR DAMAGES, INCLUDING ANY -GENERAL, SPECIAL, INCIDENTAL OR CONSEQUENTIAL DAMAGES ARISING OUT OF THE -USE OR INABILITY TO USE THE PROGRAM (INCLUDING BUT NOT LIMITED TO LOSS OF -DATA OR DATA BEING RENDERED INACCURATE OR LOSSES SUSTAINED BY YOU OR THIRD -PARTIES OR A FAILURE OF THE PROGRAM TO OPERATE WITH ANY OTHER PROGRAMS), -EVEN IF SUCH HOLDER OR OTHER PARTY HAS BEEN ADVISED OF THE POSSIBILITY OF -SUCH DAMAGES. - - 17. Interpretation of Sections 15 and 16. - - If the disclaimer of warranty and limitation of liability provided -above cannot be given local legal effect according to their terms, -reviewing courts shall apply local law that most closely approximates -an absolute waiver of all civil liability in connection with the -Program, unless a warranty or assumption of liability accompanies a -copy of the Program in return for a fee. diff --git a/mail/MANIFEST.in b/mail/MANIFEST.in deleted file mode 100644 index cf9cbd3..0000000 --- a/mail/MANIFEST.in +++ /dev/null @@ -1,6 +0,0 @@ -include pkg/* -include versioneer.py -include LICENSE -include CHANGELOG.rst -include README.rst -include src/leap/mail/_version.py diff --git a/mail/README.rst b/mail/README.rst deleted file mode 100644 index f201baa..0000000 --- a/mail/README.rst +++ /dev/null @@ -1,37 +0,0 @@ -leap.mail -========= -Mail services for the LEAP Client. - -.. image:: https://badge.fury.io/py/leap.mail.svg - :target: http://badge.fury.io/py/leap.mail - -.. image:: https://readthedocs.org/projects/leapmail/badge/?version=latest - :target: http://leapmail.readthedocs.org/en/latest/ - :alt: Documentation Status - -More info: https://leap.se - -running tests -------------- - -Use trial to run the test suite:: - - trial leap.mail - -... and all its goodies. To run all imap tests in a loop until some of them -fails:: - - trial -u leap.mail.imap - -Read the *trial* manpage for more options . - -imap regressions ----------------- - -For testing the IMAP server implementation, there are a couple of utilities. -From the ``leap.mail.imap.tests`` folder, and with an already initialized server -running:: - - ./regressions_mime_struct user@provider pass path_to_samples/ - -You can find several message samples in the ``leap/mail/tests`` folder. diff --git a/mail/changes/VERSION_COMPAT b/mail/changes/VERSION_COMPAT deleted file mode 100644 index 77a394f..0000000 --- a/mail/changes/VERSION_COMPAT +++ /dev/null @@ -1,11 +0,0 @@ -################################################# -# This file keeps track of the recent changes -# introduced in internal leap dependencies. -# Add your changes here so we can properly update -# requirements.pip during the release process. -# (leave header when resetting) -################################################# -# -# BEGIN DEPENDENCY LIST ------------------------- -# leap.foo.bar>=x.y.z -leap.keymanager >= 0.5.2 diff --git a/mail/changes/next-changelog.rst b/mail/changes/next-changelog.rst deleted file mode 100644 index 21b1010..0000000 --- a/mail/changes/next-changelog.rst +++ /dev/null @@ -1,30 +0,0 @@ -0.4.3 - xxx -+++++++++++++++++++++++++++++++ - -Please add lines to this file, they will be moved to the CHANGELOG.rst during -the next release. - -There are two template lines for each category, use them as reference. - -I've added a new category `Misc` so we can track doc/style/packaging stuff. - -Features -~~~~~~~~ -- `#8031 `_: Adapt to the new KeyManager API without key types. - -- `#1234 `_: Description of the new feature corresponding with issue #1234. -- New feature without related issue number. - -Bugfixes -~~~~~~~~ -- `#1235 `_: Description for the fixed stuff corresponding with issue #1235. -- Bugfix without related issue number. - -Misc -~~~~ -- `#1236 `_: Description of the new feature corresponding with issue #1236. -- Some change without issue number. - -Known Issues -~~~~~~~~~~~~ -- `#1236 `_: Description of the known issue corresponding with issue #1236. diff --git a/mail/docs/Makefile b/mail/docs/Makefile deleted file mode 100644 index 388b6c6..0000000 --- a/mail/docs/Makefile +++ /dev/null @@ -1,179 +0,0 @@ -# Makefile for Sphinx documentation -# - -# You can set these variables from the command line. -SPHINXOPTS = -SPHINXBUILD = sphinx-build -PAPER = -BUILDDIR = _build - -# User-friendly check for sphinx-build -ifeq ($(shell which $(SPHINXBUILD) >/dev/null 2>&1; echo $$?), 1) -$(error The '$(SPHINXBUILD)' command was not found. Make sure you have Sphinx installed, then set the SPHINXBUILD environment variable to point to the full path of the '$(SPHINXBUILD)' executable. Alternatively you can add the directory with the executable to your PATH. If you don't have Sphinx installed, grab it from http://sphinx-doc.org/) -endif - -# Internal variables. -PAPEROPT_a4 = -D latex_paper_size=a4 -PAPEROPT_letter = -D latex_paper_size=letter -ALLSPHINXOPTS = -d $(BUILDDIR)/doctrees $(PAPEROPT_$(PAPER)) $(SPHINXOPTS) . -# the i18n builder cannot share the environment and doctrees with the others -I18NSPHINXOPTS = $(PAPEROPT_$(PAPER)) $(SPHINXOPTS) . - -.PHONY: help clean html dirhtml singlehtml pickle json htmlhelp qthelp devhelp epub latex latexpdf text man changes linkcheck doctest gettext - -help: - @echo "Please use \`make ' where is one of" - @echo " html to make standalone HTML files" - @echo " dirhtml to make HTML files named index.html in directories" - @echo " singlehtml to make a single large HTML file" - @echo " pickle to make pickle files" - @echo " json to make JSON files" - @echo " htmlhelp to make HTML files and a HTML help project" - @echo " qthelp to make HTML files and a qthelp project" - @echo " devhelp to make HTML files and a Devhelp project" - @echo " epub to make an epub" - @echo " latex to make LaTeX files, you can set PAPER=a4 or PAPER=letter" - @echo " latexpdf to make LaTeX files and run them through pdflatex" - @echo " latexpdfja to make LaTeX files and run them through platex/dvipdfmx" - @echo " text to make text files" - @echo " man to make manual pages" - @echo " texinfo to make Texinfo files" - @echo " info to make Texinfo files and run them through makeinfo" - @echo " gettext to make PO message catalogs" - @echo " changes to make an overview of all changed/added/deprecated items" - @echo " xml to make Docutils-native XML files" - @echo " pseudoxml to make pseudoxml-XML files for display purposes" - @echo " linkcheck to check all external links for integrity" - @echo " doctest to run all doctests embedded in the documentation (if enabled)" - -clean: - rm -rf $(BUILDDIR)/* - -html: - $(SPHINXBUILD) -b html $(ALLSPHINXOPTS) $(BUILDDIR)/html - @echo - @echo "Build finished. The HTML pages are in $(BUILDDIR)/html." - -dirhtml: - $(SPHINXBUILD) -b dirhtml $(ALLSPHINXOPTS) $(BUILDDIR)/dirhtml - @echo - @echo "Build finished. The HTML pages are in $(BUILDDIR)/dirhtml." - -singlehtml: - $(SPHINXBUILD) -b singlehtml $(ALLSPHINXOPTS) $(BUILDDIR)/singlehtml - @echo - @echo "Build finished. The HTML page is in $(BUILDDIR)/singlehtml." - -pickle: - $(SPHINXBUILD) -b pickle $(ALLSPHINXOPTS) $(BUILDDIR)/pickle - @echo - @echo "Build finished; now you can process the pickle files." - -json: - $(SPHINXBUILD) -b json $(ALLSPHINXOPTS) $(BUILDDIR)/json - @echo - @echo "Build finished; now you can process the JSON files." - -htmlhelp: - $(SPHINXBUILD) -b htmlhelp $(ALLSPHINXOPTS) $(BUILDDIR)/htmlhelp - @echo - @echo "Build finished; now you can run HTML Help Workshop with the" \ - ".hhp project file in $(BUILDDIR)/htmlhelp." - -qthelp: - $(SPHINXBUILD) -b qthelp $(ALLSPHINXOPTS) $(BUILDDIR)/qthelp - @echo - @echo "Build finished; now you can run "qcollectiongenerator" with the" \ - ".qhcp project file in $(BUILDDIR)/qthelp, like this:" - @echo "# qcollectiongenerator $(BUILDDIR)/qthelp/leapmail.qhcp" - @echo "To view the help file:" - @echo "# assistant -collectionFile $(BUILDDIR)/qthelp/leapmail.qhc" - -devhelp: - $(SPHINXBUILD) -b devhelp $(ALLSPHINXOPTS) $(BUILDDIR)/devhelp - @echo - @echo "Build finished." - @echo "To view the help file:" - @echo "# mkdir -p $$HOME/.local/share/devhelp/leapmail" - @echo "# ln -s $(BUILDDIR)/devhelp $$HOME/.local/share/devhelp/leapmail" - @echo "# devhelp" - -epub: - $(SPHINXBUILD) -b epub $(ALLSPHINXOPTS) $(BUILDDIR)/epub - @echo - @echo "Build finished. The epub file is in $(BUILDDIR)/epub." - -latex: - $(SPHINXBUILD) -b latex $(ALLSPHINXOPTS) $(BUILDDIR)/latex - @echo - @echo "Build finished; the LaTeX files are in $(BUILDDIR)/latex." - @echo "Run \`make' in that directory to run these through (pdf)latex" \ - "(use \`make latexpdf' here to do that automatically)." - -latexpdf: - $(SPHINXBUILD) -b latex $(ALLSPHINXOPTS) $(BUILDDIR)/latex - @echo "Running LaTeX files through pdflatex..." - $(MAKE) -C $(BUILDDIR)/latex all-pdf - @echo "pdflatex finished; the PDF files are in $(BUILDDIR)/latex." - -latexpdfja: - $(SPHINXBUILD) -b latex $(ALLSPHINXOPTS) $(BUILDDIR)/latex - @echo "Running LaTeX files through platex and dvipdfmx..." - $(MAKE) -C $(BUILDDIR)/latex all-pdf-ja - @echo "pdflatex finished; the PDF files are in $(BUILDDIR)/latex." - -text: - $(SPHINXBUILD) -b text $(ALLSPHINXOPTS) $(BUILDDIR)/text - @echo - @echo "Build finished. The text files are in $(BUILDDIR)/text." - -man: - $(SPHINXBUILD) -b man $(ALLSPHINXOPTS) $(BUILDDIR)/man - @echo - @echo "Build finished. The manual pages are in $(BUILDDIR)/man." - -texinfo: - $(SPHINXBUILD) -b texinfo $(ALLSPHINXOPTS) $(BUILDDIR)/texinfo - @echo - @echo "Build finished. The Texinfo files are in $(BUILDDIR)/texinfo." - @echo "Run \`make' in that directory to run these through makeinfo" \ - "(use \`make info' here to do that automatically)." - -info: - $(SPHINXBUILD) -b texinfo $(ALLSPHINXOPTS) $(BUILDDIR)/texinfo - @echo "Running Texinfo files through makeinfo..." - make -C $(BUILDDIR)/texinfo info - @echo "makeinfo finished; the Info files are in $(BUILDDIR)/texinfo." - -gettext: - $(SPHINXBUILD) -b gettext $(I18NSPHINXOPTS) $(BUILDDIR)/locale - @echo - @echo "Build finished. The message catalogs are in $(BUILDDIR)/locale." - -changes: - $(SPHINXBUILD) -b changes $(ALLSPHINXOPTS) $(BUILDDIR)/changes - @echo - @echo "The overview file is in $(BUILDDIR)/changes." - -linkcheck: - $(SPHINXBUILD) -b linkcheck $(ALLSPHINXOPTS) $(BUILDDIR)/linkcheck - @echo - @echo "Link check complete; look for any errors in the above output " \ - "or in $(BUILDDIR)/linkcheck/output.txt." - -doctest: - $(SPHINXBUILD) -b doctest $(ALLSPHINXOPTS) $(BUILDDIR)/doctest - @echo "Testing of doctests in the sources finished, look at the " \ - "results in $(BUILDDIR)/doctest/output.txt." - -xml: - $(SPHINXBUILD) -b xml $(ALLSPHINXOPTS) $(BUILDDIR)/xml - @echo - @echo "Build finished. The XML files are in $(BUILDDIR)/xml." - -pseudoxml: - $(SPHINXBUILD) -b pseudoxml $(ALLSPHINXOPTS) $(BUILDDIR)/pseudoxml - @echo - @echo "Build finished. The pseudo-XML files are in $(BUILDDIR)/pseudoxml." -build_rtd: - curl -X POST http://readthedocs.org/build/leapmail diff --git a/mail/docs/api/leap.mail.adaptors.rst b/mail/docs/api/leap.mail.adaptors.rst deleted file mode 100644 index 472cade..0000000 --- a/mail/docs/api/leap.mail.adaptors.rst +++ /dev/null @@ -1,43 +0,0 @@ -mail.adaptors package -===================== - -.. automodule:: leap.mail.adaptors - :members: - :undoc-members: - :show-inheritance: - -Subpackages ------------ - -.. toctree:: - - leap.mail.adaptors.tests - -Submodules ----------- - -mail.adaptors.models module ---------------------------- - -.. automodule:: leap.mail.adaptors.models - :members: - :undoc-members: - :show-inheritance: - -mail.adaptors.soledad module ----------------------------- - -.. automodule:: leap.mail.adaptors.soledad - :members: - :undoc-members: - :show-inheritance: - -mail.adaptors.soledad_indexes module ------------------------------------- - -.. automodule:: leap.mail.adaptors.soledad_indexes - :members: - :undoc-members: - :show-inheritance: - - diff --git a/mail/docs/api/leap.mail.adaptors.tests.rst b/mail/docs/api/leap.mail.adaptors.tests.rst deleted file mode 100644 index 2ae76e8..0000000 --- a/mail/docs/api/leap.mail.adaptors.tests.rst +++ /dev/null @@ -1,28 +0,0 @@ -mail.adaptors.tests package -=========================== - -.. automodule:: leap.mail.adaptors.tests - :members: - :undoc-members: - :show-inheritance: - -Submodules ----------- - -mail.adaptors.tests.test_models module --------------------------------------- - -.. automodule:: leap.mail.adaptors.tests.test_models - :members: - :undoc-members: - :show-inheritance: - -mail.adaptors.tests.test_soledad_adaptor module ------------------------------------------------ - -.. automodule:: leap.mail.adaptors.tests.test_soledad_adaptor - :members: - :undoc-members: - :show-inheritance: - - diff --git a/mail/docs/api/leap.mail.imap.rst b/mail/docs/api/leap.mail.imap.rst deleted file mode 100644 index bfaa3fd..0000000 --- a/mail/docs/api/leap.mail.imap.rst +++ /dev/null @@ -1,52 +0,0 @@ -leap.mail.imap package -======================= - -.. automodule:: leap.mail.imap - :members: - :undoc-members: - :show-inheritance: - -Subpackages ------------ - -.. toctree:: - - leap.mail.imap.service - leap.mail.imap.tests - -Submodules ----------- - -leap.mail.imap.account module ------------------------------- - -.. automodule:: leap.mail.imap.account - :members: - :undoc-members: - :show-inheritance: - -leap.mail.imap.mailbox module ------------------------------- - -.. automodule:: leap.mail.imap.mailbox - :members: - :undoc-members: - :show-inheritance: - -leap.mail.imap.messages module ------------------------------- - -.. automodule:: leap.mail.imap.messages - :members: - :undoc-members: - :show-inheritance: - -leap.mail.imap.server module ------------------------------ - -.. automodule:: leap.mail.imap.server - :members: - :undoc-members: - :show-inheritance: - - diff --git a/mail/docs/api/leap.mail.imap.service.rst b/mail/docs/api/leap.mail.imap.service.rst deleted file mode 100644 index 2f3ed4b..0000000 --- a/mail/docs/api/leap.mail.imap.service.rst +++ /dev/null @@ -1,9 +0,0 @@ -leap.mail.imap.service package -=============================== - -.. automodule:: leap.mail.imap.service - :members: - :undoc-members: - :show-inheritance: - - diff --git a/mail/docs/api/leap.mail.incoming.rst b/mail/docs/api/leap.mail.incoming.rst deleted file mode 100644 index 4bd1614..0000000 --- a/mail/docs/api/leap.mail.incoming.rst +++ /dev/null @@ -1,20 +0,0 @@ -leap.mail.incoming package -========================== - -.. automodule:: leap.mail.incoming - :members: - :undoc-members: - :show-inheritance: - -Submodules ----------- - -leap.mail.incoming.service module ----------------------------------- - -.. automodule:: leap.mail.incoming.service - :members: - :undoc-members: - :show-inheritance: - - diff --git a/mail/docs/api/leap.mail.outgoing.rst b/mail/docs/api/leap.mail.outgoing.rst deleted file mode 100644 index af8c173..0000000 --- a/mail/docs/api/leap.mail.outgoing.rst +++ /dev/null @@ -1,21 +0,0 @@ -leap.mail.outgoing package -========================== - -.. automodule:: leap.mail.outgoing - :members: - :undoc-members: - :show-inheritance: - - -Submodules ----------- - -mail.outgoing.service module ----------------------------- - -.. automodule:: leap.mail.outgoing.service - :members: - :undoc-members: - :show-inheritance: - - diff --git a/mail/docs/api/leap.mail.plugins.rst b/mail/docs/api/leap.mail.plugins.rst deleted file mode 100644 index 7a5d6b4..0000000 --- a/mail/docs/api/leap.mail.plugins.rst +++ /dev/null @@ -1,20 +0,0 @@ -leap.mail.plugins package -========================== - -.. automodule:: leap.mail.plugins - :members: - :undoc-members: - :show-inheritance: - -Submodules ----------- - -leap.mail.plugins.soledad_sync_hooks module -------------------------------------------- - -.. automodule:: leap.mail.plugins.soledad_sync_hooks - :members: - :undoc-members: - :show-inheritance: - - diff --git a/mail/docs/api/leap.mail.rst b/mail/docs/api/leap.mail.rst deleted file mode 100644 index 686e648..0000000 --- a/mail/docs/api/leap.mail.rst +++ /dev/null @@ -1,105 +0,0 @@ -leap.mail package -================= - -.. automodule:: leap.mail - :members: - :undoc-members: - :show-inheritance: - -Subpackages ------------ - -.. toctree:: - - leap.mail.adaptors - leap.mail.imap - leap.mail.incoming - leap.mail.outgoing - leap.mail.plugins - leap.mail.smtp - leap.mail.tests - -Submodules ----------- - -leap.mail.constants module ---------------------------- - -.. automodule:: leap.mail.constants - :members: - :undoc-members: - :show-inheritance: - -leap.mail.decorators module ---------------------------- - -.. automodule:: leap.mail.decorators - :members: - :undoc-members: - :show-inheritance: - -leap.mail.interfaces module ----------------------------- - -.. automodule:: leap.mail.interfaces - :members: - :undoc-members: - :show-inheritance: - -leap.mail.load_tests module ----------------------------- - -.. automodule:: leap.mail.load_tests - :members: - :undoc-members: - :show-inheritance: - -leap.mail.mail module ---------------------- - -.. automodule:: leap.mail.mail - :members: - :undoc-members: - :show-inheritance: - -leap.mail.mailbox_indexer module ---------------------------------- - -.. automodule:: leap.mail.mailbox_indexer - :members: - :undoc-members: - :show-inheritance: - -leap.mail.size module ----------------------- - -.. automodule:: leap.mail.size - :members: - :undoc-members: - :show-inheritance: - -leap.mail.sync_hooks module ----------------------------- - -.. automodule:: leap.mail.sync_hooks - :members: - :undoc-members: - :show-inheritance: - -leap.mail.utils module ------------------------ - -.. automodule:: leap.mail.utils - :members: - :undoc-members: - :show-inheritance: - -leap.mail.walk module ----------------------- - -.. automodule:: leap.mail.walk - :members: - :undoc-members: - :show-inheritance: - - diff --git a/mail/docs/api/leap.mail.smtp.rst b/mail/docs/api/leap.mail.smtp.rst deleted file mode 100644 index f35d3f9..0000000 --- a/mail/docs/api/leap.mail.smtp.rst +++ /dev/null @@ -1,19 +0,0 @@ -leap.mail.smtp package -======================= - -.. automodule:: leap.mail.smtp - :members: - :undoc-members: - :show-inheritance: - -Submodules ----------- - -leap.mail.smtp.gateway module ------------------------------ - -.. automodule:: leap.mail.smtp.gateway - :members: - :undoc-members: - :show-inheritance: - diff --git a/mail/docs/api/modules.rst b/mail/docs/api/modules.rst deleted file mode 100644 index 7827779..0000000 --- a/mail/docs/api/modules.rst +++ /dev/null @@ -1,7 +0,0 @@ -mail -==== - -.. toctree:: - :maxdepth: 4 - - mail diff --git a/mail/docs/conf.py b/mail/docs/conf.py deleted file mode 100644 index 746f57c..0000000 --- a/mail/docs/conf.py +++ /dev/null @@ -1,341 +0,0 @@ -# -*- coding: utf-8 -*- -# -# leap.mail documentation build configuration file, created by -# sphinx-quickstart on Mon Aug 25 19:19:48 2014. -# -# This file is execfile()d with the current directory set to its -# containing dir. -# -# Note that not all possible configuration values are present in this -# autogenerated file. -# -# All configuration values have a default; values that are commented out -# serve to show the default. - -import sys -import os - -# If extensions (or modules to document with autodoc) are in another directory, -# add these directories to sys.path here. If the directory is relative to the -# documentation root, use os.path.abspath to make it absolute, like shown here. -#sys.path.insert(0, os.path.abspath('../src')) -#sys.path.insert(0, os.path.abspath('../src/leap')) -#sys.path.insert(0, os.path.abspath('../src/leap/mail')) -#sys.path.insert(0, os.path.abspath('../../leap_common/src/leap/common')) -#sys.path.insert(0, os.path.abspath('../../soledad/client/src/leap/soledad/client')) - -VENV_PATH = os.environ.get('VIRTUAL_ENV') -if VENV_PATH: - sys.path.insert(0, os.path.abspath(VENV_PATH + 'lib/python2.7/site-packages')) - - -# -- General configuration ------------------------------------------------ - -# If your documentation needs a minimal Sphinx version, state it here. -#needs_sphinx = '1.0' - -# Add any Sphinx extension module names here, as strings. They can be -# extensions coming with Sphinx (named 'sphinx.ext.*') or your custom -# ones. -extensions = [ - 'sphinx.ext.autodoc', - 'sphinx.ext.todo', - 'sphinx.ext.coverage', -] - -# Add any paths that contain templates here, relative to this directory. -templates_path = ['_templates'] - -# The suffix of source filenames. -source_suffix = '.rst' - -# The encoding of source files. -#source_encoding = 'utf-8-sig' - -# The master toctree document. -master_doc = 'index' - -# General information about the project. -project = u'leap.mail' -copyright = u'2014-2015, The LEAP Encryption Access Project' - -# The version info for the project you're documenting, acts as replacement for -# |version| and |release|, also used in various other places throughout the -# built documents. -# -# The short X.Y version. -version = '0.4.0rc2' -# The full version, including alpha/beta/rc tags. -release = '0.4.0' - -# The language for content autogenerated by Sphinx. Refer to documentation -# for a list of supported languages. -#language = None - -# There are two options for replacing |today|: either, you set today to some -# non-false value, then it is used: -#today = '' -# Else, today_fmt is used as the format for a strftime call. -#today_fmt = '%B %d, %Y' - -# List of patterns, relative to source directory, that match files and -# directories to ignore when looking for source files. -exclude_patterns = ['_build'] - -# The reST default role (used for this markup: `text`) to use for all -# documents. -#default_role = None - -# If true, '()' will be appended to :func: etc. cross-reference text. -#add_function_parentheses = True - -# If true, the current module name will be prepended to all description -# unit titles (such as .. function::). -#add_module_names = True - -# If true, sectionauthor and moduleauthor directives will be shown in the -# output. They are ignored by default. -#show_authors = False - -# The name of the Pygments (syntax highlighting) style to use. -pygments_style = 'sphinx' - -# A list of ignored prefixes for module index sorting. -#modindex_common_prefix = [] - -# If true, keep warnings as "system message" paragraphs in the built documents. -#keep_warnings = False - - -# -- Options for HTML output ---------------------------------------------- - -# The theme to use for HTML and HTML Help pages. See the documentation for -# a list of builtin themes. -html_theme = 'alabaster' - -# Theme options are theme-specific and customize the look and feel of a theme -# further. For a list of options available for each theme, see the -# documentation. -#html_theme_options = {} - -# Add any paths that contain custom themes here, relative to this directory. -#html_theme_path = [] - -# The name for this set of Sphinx documents. If None, it defaults to -# " v documentation". -#html_title = None - -# A shorter title for the navigation bar. Default is the same as html_title. -#html_short_title = None - -# The name of an image file (relative to this directory) to place at the top -# of the sidebar. -#html_logo = None - -# The name of an image file (within the static path) to use as favicon of the -# docs. This file should be a Windows icon file (.ico) being 16x16 or 32x32 -# pixels large. -#html_favicon = None - -# Add any paths that contain custom static files (such as style sheets) here, -# relative to this directory. They are copied after the builtin static files, -# so a file named "default.css" will overwrite the builtin "default.css". -html_static_path = ['_static'] - -# Add any extra paths that contain custom files (such as robots.txt or -# .htaccess) here, relative to this directory. These files are copied -# directly to the root of the documentation. -#html_extra_path = [] - -# If not '', a 'Last updated on:' timestamp is inserted at every page bottom, -# using the given strftime format. -#html_last_updated_fmt = '%b %d, %Y' - -# If true, SmartyPants will be used to convert quotes and dashes to -# typographically correct entities. -#html_use_smartypants = True - -# Custom sidebar templates, maps document names to template names. -#html_sidebars = {} - -# Additional templates that should be rendered to pages, maps page names to -# template names. -#html_additional_pages = {} - -# If false, no module index is generated. -#html_domain_indices = True - -# If false, no index is generated. -#html_use_index = True - -# If true, the index is split into individual pages for each letter. -#html_split_index = False - -# If true, links to the reST sources are added to the pages. -#html_show_sourcelink = True - -# If true, "Created using Sphinx" is shown in the HTML footer. Default is True. -#html_show_sphinx = True - -# If true, "(C) Copyright ..." is shown in the HTML footer. Default is True. -#html_show_copyright = True - -# If true, an OpenSearch description file will be output, and all pages will -# contain a tag referring to it. The value of this option must be the -# base URL from which the finished HTML is served. -#html_use_opensearch = '' - -# This is the file name suffix for HTML files (e.g. ".xhtml"). -#html_file_suffix = None - -# Output file base name for HTML help builder. -htmlhelp_basename = 'leapmaildoc' - - -# -- Options for LaTeX output --------------------------------------------- - -latex_elements = { -# The paper size ('letterpaper' or 'a4paper'). -#'papersize': 'letterpaper', - -# The font size ('10pt', '11pt' or '12pt'). -#'pointsize': '10pt', - -# Additional stuff for the LaTeX preamble. -#'preamble': '', -} - -# Grouping the document tree into LaTeX files. List of tuples -# (source start file, target name, title, -# author, documentclass [howto, manual, or own class]). -latex_documents = [ - ('index', 'leapmail.tex', u'leap.mail Documentation', - u'Kali Kaneko', 'manual'), -] - -# The name of an image file (relative to this directory) to place at the top of -# the title page. -#latex_logo = None - -# For "manual" documents, if this is true, then toplevel headings are parts, -# not chapters. -#latex_use_parts = False - -# If true, show page references after internal links. -#latex_show_pagerefs = False - -# If true, show URL addresses after external links. -#latex_show_urls = False - -# Documents to append as an appendix to all manuals. -#latex_appendices = [] - -# If false, no module index is generated. -#latex_domain_indices = True - - -# -- Options for manual page output --------------------------------------- - -# One entry per manual page. List of tuples -# (source start file, name, description, authors, manual section). -man_pages = [ - ('index', 'leapmail', u'leap.mail Documentation', - [u'Kali Kaneko'], 1) -] - -# If true, show URL addresses after external links. -#man_show_urls = False - - -# -- Options for Texinfo output ------------------------------------------- - -# Grouping the document tree into Texinfo files. List of tuples -# (source start file, target name, title, author, -# dir menu entry, description, category) -texinfo_documents = [ - ('index', 'leapmail', u'leap.mail Documentation', - u'Kali Kaneko', 'leapmail', 'One line description of project.', - 'Miscellaneous'), -] - -# Documents to append as an appendix to all manuals. -#texinfo_appendices = [] - -# If false, no module index is generated. -#texinfo_domain_indices = True - -# How to display URL addresses: 'footnote', 'no', or 'inline'. -#texinfo_show_urls = 'footnote' - -# If true, do not generate a @detailmenu in the "Top" node's menu. -#texinfo_no_detailmenu = False - - -# -- Options for Epub output ---------------------------------------------- - -# Bibliographic Dublin Core info. -epub_title = u'leap.mail' -epub_author = u'Kali Kaneko' -epub_publisher = u'Kali Kaneko' -epub_copyright = u'2014, Kali Kaneko' - -# The basename for the epub file. It defaults to the project name. -#epub_basename = u'leap.mail' - -# The HTML theme for the epub output. Since the default themes are not optimized -# for small screen space, using the same theme for HTML and epub output is -# usually not wise. This defaults to 'epub', a theme designed to save visual -# space. -#epub_theme = 'epub' - -# The language of the text. It defaults to the language option -# or en if the language is not set. -#epub_language = '' - -# The scheme of the identifier. Typical schemes are ISBN or URL. -#epub_scheme = '' - -# The unique identifier of the text. This can be a ISBN number -# or the project homepage. -#epub_identifier = '' - -# A unique identification for the text. -#epub_uid = '' - -# A tuple containing the cover image and cover page html template filenames. -#epub_cover = () - -# A sequence of (type, uri, title) tuples for the guide element of content.opf. -#epub_guide = () - -# HTML files that should be inserted before the pages created by sphinx. -# The format is a list of tuples containing the path and title. -#epub_pre_files = [] - -# HTML files shat should be inserted after the pages created by sphinx. -# The format is a list of tuples containing the path and title. -#epub_post_files = [] - -# A list of files that should not be packed into the epub file. -epub_exclude_files = ['search.html'] - -# The depth of the table of contents in toc.ncx. -#epub_tocdepth = 3 - -# Allow duplicate toc entries. -#epub_tocdup = True - -# Choose between 'default' and 'includehidden'. -#epub_tocscope = 'default' - -# Fix unsupported image types using the PIL. -#epub_fix_images = False - -# Scale large images. -#epub_max_image_width = 0 - -# How to display URL addresses: 'footnote', 'no', or 'inline'. -#epub_show_urls = 'inline' - -# If false, no index is generated. -#epub_use_index = True diff --git a/mail/docs/git-commit-message.txt b/mail/docs/git-commit-message.txt deleted file mode 100644 index 1b28baf..0000000 --- a/mail/docs/git-commit-message.txt +++ /dev/null @@ -1,8 +0,0 @@ -[bug|feature|docs|pkg] - -... - -Resolves: #XXX -Related: #XXX -Documentation: #XXX -Releases: XXX diff --git a/mail/docs/hacking.rst b/mail/docs/hacking.rst deleted file mode 100644 index 6c49c21..0000000 --- a/mail/docs/hacking.rst +++ /dev/null @@ -1,195 +0,0 @@ -.. _hacking: - -======== -Hacking -======== - -Some hints oriented to `leap.mail` hackers. These notes are mostly related to -the imap server, although they probably will be useful for other pieces too. - -Don't panic! Just manhole into it -================================= - -If you want to inspect the objects living in your application memory, in -realtime, you can manhole into it. - -First of all, check that the modules ``PyCrypto`` and ``pyasn1`` are installed -into your system, they are needed for it to work. - -You just have to pass the ``LEAP_MAIL_MANHOLE=1`` enviroment variable while -launching the client:: - - LEAP_MAIL_MANHOLE=1 bitmask --debug - -And then you can ssh into your application! (password is "leap"):: - - ssh boss@localhost -p 2222 - -Did I mention how *awesome* twisted is?? ``:)`` - - -Profiling -========= -If using ``twistd`` to launch the server, you can use twisted profiling -capabities:: - - LEAP_MAIL_CONFIG=~/.leapmailrc twistd --profile=/tmp/mail-profiling -n -y imap-server.tac - -``--profiler`` option allows you to select different profilers (default is -"hotshot"). - -You can also do profiling when using the ``bitmask`` client. Enable the -``LEAP_PROFILE_IMAPCMD`` environment flag to get profiling of certain IMAP -commands:: - - LEAP_PROFILE_IMAPCMD=1 bitmask --debug - -Offline mode -============ - -The client has an ``--offline`` flag that will make the Mail services (imap, -currently) not try to sync with remote replicas. Very useful during development, -although you need to login with the remote server at least once before being -able to use it. - -Mutt config -=========== - -You cannot live without mutt? You're lucky! Use the following minimal config -with the imap service:: - - set folder="imap://user@provider@localhost:1984" - set spoolfile="imap://user@provider@localhost:1984/INBOX" - set ssl_starttls = no - set ssl_force_tls = no - set imap_pass=MAHSIKRET - - -Running the service with twistd -=============================== - -In order to run the mail service (currently, the imap server only), you will -need a config with this info:: - - [leap_mail] - userid = "user@provider" - uuid = "deadbeefdeadabad" - passwd = "foobar" # Optional - -In the ``LEAP_MAIL_CONFIG`` enviroment variable. If you do not specify a password -parameter, you'll be prompted for it. - -In order to get the user uid (uuid), look into the -``~/.config/leap/leap-backend.conf`` file after you have logged in into your -provider at least once. - -Run the twisted service:: - - LEAP_MAIL_CONFIG=~/.leapmailrc twistd -n -y imap-server.tac - -Now you can telnet into your local IMAP server and read your mail like a real -programmer™:: - - % telnet localhost 1984 - Trying 127.0.0.1... - Connected to localhost. - Escape character is '^]'. - * OK [CAPABILITY IMAP4rev1 LITERAL+ IDLE NAMESPACE] Twisted IMAP4rev1 Ready - tag LOGIN me@myprovider.net mahsikret - tag OK LOGIN succeeded - tag SELECT Inbox - * 2 EXISTS - * 1 RECENT - * FLAGS (\Seen \Answered \Flagged \Deleted \Draft \Recent List) - * OK [UIDVALIDITY 1410453885932] UIDs valid - tag OK [READ-WRITE] SELECT successful - ^] - telnet> Connection closed. - - -Although you probably prefer to use ``offlineimap`` for tests:: - - offlineimap -c LEAPofflineimapRC-tests - - -Minimal offlineimap configuration ---------------------------------- - -You can use this as a sample offlineimap config file:: - - [general] - accounts = leap-local - - [Account leap-local] - localrepository = LocalLeap - remoterepository = RemoteLeap - - [Repository LocalLeap] - type = Maildir - localfolders = ~/LEAPMail/Mail - - [Repository RemoteLeap] - type = IMAP - ssl = no - remotehost = localhost - remoteport = 1984 - remoteuser = user - remotepass = pass - -Testing utilities ------------------ -There are a bunch of utilities to test IMAP delivery in ``imap/tests`` folder. -If looking for a quick way of inspecting mailboxes, have a look at ``getmail``:: - - ./getmail me@testprovider.net mahsikret - 1. Drafts - 2. INBOX - 3. Trash - Which mailbox? [1] 2 - 1 Subject: this is the time of the revolution - 2 Subject: ignore me - - Which message? [1] (Q quits) 1 - 1 X-Leap-Provenance: Thu, 11 Sep 2014 16:52:11 -0000; pubkey="C1F8DE10BD151F99" - Received: from mx1.testprovider.net(mx1.testprovider.net [198.197.196.195]) - (using TLSv1.2 with cipher ECDHE-RSA-AES256-GCM-SHA384 (256/256 bits)) - (Client CN "*.foobar.net", Issuer "Gandi Standard SSL CA" (not verified)) - by blackhole (Postfix) with ESMTPS id DEADBEEF - for ; Thu, 11 Sep 2014 16:52:10 +0000 (UTC) - Delivered-To: 926d4915cfd42b6d96d38660c04613af@testprovider.net - Message-Id: <20140911165205.GB8054@samsara> - From: Kali - - (snip) - -IMAP Message Rendering Regressions ----------------------------------- - -For testing the IMAP server implementation, there is a litte regressions script -that needs some manual work from your side. - -First of all, you need an already initialized account. Which for now basically -means you have created a new account with a provider that offers the Encrypted -Mail Service, using the Bitmask Client wizard. Then you need to log in with that -account, and let it generate the secrets and sync with the remote for a first -time. After this you can run the twistd server locally and offline. - -From the ``leap.mail.imap.tests`` folder, and with an already initialized server -running:: - - ./regressions_mime_struct user@provider pass path_to_samples/ - -You can find several message samples in the ``leap/mail/tests`` folder. - - -Debugging IMAP commands -======================= - -Use ``ngrep`` to obtain logs of the commands:: - - sudo ngrep -d lo -W byline port 1984 - -To get verbose output from thunderbird/icedove, set the following environment -variable:: - - NSPR_LOG_MODULES="imap:5" icedove diff --git a/mail/docs/index.rst b/mail/docs/index.rst deleted file mode 100644 index a2133f4..0000000 --- a/mail/docs/index.rst +++ /dev/null @@ -1,96 +0,0 @@ -.. leap.mail documentation master file, created by - sphinx-quickstart on Mon Aug 25 19:19:48 2014. - You can adapt this file completely to your liking, but it should at least - contain the root `toctree` directive. - -leap.mail -========= - -*decentralized and secure mail delivery and synchronization* - -This is the documentation for the ``leap.mail`` module. It is a `twisted`_ -package that allows to receive, process, send and access existing messages using -the `LEAP`_ platform. - -One way to use this library is to let it launch two standard mail services, -``smtp`` and ``imap``, that run as local proxies and interact with a remote -``LEAP`` provider that offers *a soledad syncronization endpoint* and receives -the outgoing email. This is what `Bitmask`_ client does. - -From the release 0.4.0 on, it's also possible to use a protocol-agnostic email -public API, so that third party mail clients can manipulate the data layer. This -is what the awesome MUA in the `Pixelated`_ project is using. - -.. _`twisted`: https://twistedmatrix.com/trac/ -.. _`LEAP`: https://leap.se/en/docs -.. _`Bitmask`: https://bitmask.net/en/features#email -.. _`Pixelated`: https://pixelated-project.org/ - -How does this all work? ------------------------ - -All the underlying data storage and sync is handled by a library called -`soledad`_, which handles encryption, storage and sync. Based on `u1db`_, -documents are stored locally as local ``sqlcipher`` tables, and syncs against -the soledad sync service in the provider. - -OpenPGP key generation and keyring management are handled by another leap -python library: `keymanager`_. - -See :ref:`the life cycle of a leap email ` for an overview of the life cycle -of an email through ``LEAP`` providers. - -.. _`Soledad`: https://leap.se/en/docs/design/soledad -.. _`u1db`: https://en.wikipedia.org/wiki/U1DB -.. _`keymanager`: https://github.com/leapcode/keymanager/ - - -Data model ----------- -.. TODO clear document types documentation. - -The data model at the present moment consists of several *document types* that split email into -different documents that are stored in ``Soledad``. The idea behind this is to -keep clear the separation between *mutable* and *inmutable* parts, and still being able to -reconstruct arbitrarily nested email structures easily. - -Documentation index -=================== - -.. -.. Contents: -.. toctree:: - :maxdepth: 2 - - hacking - -.. intro -.. tutorial - - -API documentation ------------------ - -If you were looking for the documentation of the ``leap.mail`` module, you will -find it here. - -Of special interest is the `public mail api`_, which should remain relatively -stable across the next few releases. - -.. _`public mail api`: api/mail.html#module-mail - - -.. toctree:: - :maxdepth: 2 - - api/leap.mail - - - -Indices and tables -================== - -* :ref:`genindex` -* :ref:`modindex` -* :ref:`search` - diff --git a/mail/docs/mail_journey.rst b/mail/docs/mail_journey.rst deleted file mode 100644 index 7e64f18..0000000 --- a/mail/docs/mail_journey.rst +++ /dev/null @@ -1,40 +0,0 @@ -.. _mail_journey: - -The life cycle of a LEAP Email -============================== -The following are just some notes to facilitate the understanding of the -leap.mail internals to developers and collaborators. - -Server-side: receiving mail from the outside world --------------------------------------------------- - -1. the mx server receives an email, it gets through all the postfix validation and it's written into disk -2. ``leap_mx`` gets that write in disk notification and encrypts the incoming mail to its intended recipient's pubkey -3. that encrypted blob gets written into couchdb in a way soledad will catch it in the next sync - - -Client-side: fetching and processing incoming email ---------------------------------------------------- -you have an imap and an smtp local server. For IMAP: - -1. soledad syncs -2. **fetch** module sees if there's new encrypted mail for the current user from leap_mx -3. if there is, the mail is decrypted using the user private key, and splitted - into several parts according to the internal mail data model (separating - mutable and inmutable email parts). Those documents it encrypts it properly - like other soledad documents and deletes the pubkey encrypted doc -4. desktop client is notified that there are N new mails -5. when a MUA connects to the **imap** local server, the different documents are glued - together and presented as response to the different imap commands. - - -Client side: sending email --------------------------- - -1. you write an email to a recipient and hit send -2. the **smtp** local server gets that mail, it checks from whom it is and to whom it is for -3. it signs the mail with the ``From:``'s privkey -4. it retrieves ``To:``'s pubkey with the keymanager and if it finds it encrypts the mail to him/her -5. if it didn't find it and you don't have your client configured to "always encrypt", it goes out with just the signature -6. else, it fails out complaining about this conflict -7. that mail gets relayed to the provider's **smtp** server diff --git a/mail/docs/recreate_apidocs.sh b/mail/docs/recreate_apidocs.sh deleted file mode 100755 index 9a79d09..0000000 --- a/mail/docs/recreate_apidocs.sh +++ /dev/null @@ -1,4 +0,0 @@ -#!/bin/sh -# Watchout! this will need much manual touches -# to the generated apidocs. Mainly: s/mail/leap.mail/g -sphinx-apidoc -M -o api ../src/leap/mail diff --git a/mail/docs/tutorial.rst b/mail/docs/tutorial.rst deleted file mode 100644 index 5cf7d02..0000000 --- a/mail/docs/tutorial.rst +++ /dev/null @@ -1,3 +0,0 @@ -Tutorial -======== -`leap.mail` tutorial diff --git a/mail/pkg/__init__.py b/mail/pkg/__init__.py deleted file mode 100644 index e69de29..0000000 diff --git a/mail/pkg/generate_wheels.sh b/mail/pkg/generate_wheels.sh deleted file mode 100755 index 6679d1d..0000000 --- a/mail/pkg/generate_wheels.sh +++ /dev/null @@ -1,14 +0,0 @@ -#!/bin/sh -# Generate wheels for dependencies -# For convenience, u1db and dirspec are allowed with insecure flags enabled. -# Use at your own risk. - -if [ "$WHEELHOUSE" = "" ]; then - WHEELHOUSE=$HOME/wheelhouse -fi - -pip wheel --wheel-dir $WHEELHOUSE pip -pip wheel --wheel-dir $WHEELHOUSE -r pkg/requirements.pip -if [ -f pkg/requirements-testing.pip ]; then - pip wheel --wheel-dir $WHEELHOUSE -r pkg/requirements-testing.pip -fi diff --git a/mail/pkg/pip_install_requirements.sh b/mail/pkg/pip_install_requirements.sh deleted file mode 100755 index 57732e2..0000000 --- a/mail/pkg/pip_install_requirements.sh +++ /dev/null @@ -1,86 +0,0 @@ -#!/bin/bash -# Update pip and install LEAP base/testing requirements. -# For convenience, $insecure_packages are allowed with insecure flags enabled. -# Use at your own risk. -# See $usage for help - -insecure_packages="" -leap_wheelhouse=https://lizard.leap.se/wheels - -show_help() { - usage="Usage: $0 [--testing] [--use-leap-wheels]\n --testing\t\tInstall dependencies from requirements-testing.pip\n -\t\t\tOtherwise, it will install requirements.pip\n ---use-leap-wheels\tUse wheels from leap.se" - echo -e $usage - - exit 1 -} - -process_arguments() { - testing=false - use_leap_wheels=false - - while [ "$#" -gt 0 ]; do - # From http://stackoverflow.com/a/31443098 - case "$1" in - --help) show_help;; - --testing) testing=true; shift 1;; - --use-leap-wheels) use_leap_wheels=true; shift 1;; - - -h) show_help;; - -*) echo "unknown option: $1" >&2; exit 1;; - esac - done -} - -return_wheelhouse() { - if $use_leap_wheels ; then - WHEELHOUSE=$leap_wheelhouse - elif [ "$WHEELHOUSE" = "" ]; then - WHEELHOUSE=$HOME/wheelhouse - fi - - # Tested with bash and zsh - if [[ $WHEELHOUSE != http* && ! -d "$WHEELHOUSE" ]]; then - mkdir $WHEELHOUSE - fi - - echo "$WHEELHOUSE" -} - -return_install_options() { - wheelhouse=`return_wheelhouse` - install_options="-U --find-links=$wheelhouse" - if $use_leap_wheels ; then - install_options="$install_options --trusted-host lizard.leap.se" - fi - - echo $install_options -} - -return_insecure_flags() { - for insecure_package in $insecure_packages; do - flags="$flags --allow-external $insecure_package --allow-unverified $insecure_package" - done - - echo $flags -} - -return_packages() { - if $testing ; then - packages="-r pkg/requirements-testing.pip" - else - packages="-r pkg/requirements.pip" - fi - - echo $packages -} - -process_arguments $@ -install_options=`return_install_options` -insecure_flags=`return_insecure_flags` -packages=`return_packages` - -pip install -U wheel -pip install $install_options pip -pip install $install_options $insecure_flags $packages diff --git a/mail/pkg/requirements-dev.pip b/mail/pkg/requirements-dev.pip deleted file mode 100644 index 4bd76f6..0000000 --- a/mail/pkg/requirements-dev.pip +++ /dev/null @@ -1,14 +0,0 @@ -# --------------------------- -# -- external requirements -- -# -- during development -- -# --------------------------- -# -# For temporary work, you can point this to your developer repo. -# consolidated changes will be pushed to pypi and then added -# to the main requirements.pip -# -# NOTE: you have to run pip install -r pkg/requirements.pip for pip -# to install it. (do it after python setup.py develop and it -# will only install this) - --e git+git://leap.se/soledad@develop#egg=leap.soledad diff --git a/mail/pkg/requirements-latest.pip b/mail/pkg/requirements-latest.pip deleted file mode 100644 index 58347b6..0000000 --- a/mail/pkg/requirements-latest.pip +++ /dev/null @@ -1,9 +0,0 @@ ---index-url https://pypi.python.org/simple/ - -https://launchpad.net/dirspec/stable-13-10/13.10/+download/dirspec-13.10.tar.gz -https://launchpad.net/ubuntu/+archive/primary/+files/u1db_13.09.orig.tar.bz2 --e 'git+https://github.com/leapcode/leap_pycommon.git@develop#egg=leap.common' --e 'git+https://github.com/leapcode/soledad.git@develop#egg=leap.soledad.common&subdirectory=common/' --e 'git+https://github.com/leapcode/soledad.git@develop#egg=leap.soledad.client&subdirectory=client/' --e 'git+https://github.com/leapcode/keymanager.git@develop#egg=leap.keymanager' --e . diff --git a/mail/pkg/requirements-leap.pip b/mail/pkg/requirements-leap.pip deleted file mode 100644 index 134b783..0000000 --- a/mail/pkg/requirements-leap.pip +++ /dev/null @@ -1,3 +0,0 @@ -leap.common>=0.5.1 -leap.soledad.client>=0.8.0 -leap.keymanager>=0.5.0 diff --git a/mail/pkg/requirements-testing.pip b/mail/pkg/requirements-testing.pip deleted file mode 100644 index 41222f7..0000000 --- a/mail/pkg/requirements-testing.pip +++ /dev/null @@ -1,7 +0,0 @@ -setuptools-trial -mock -nose -rednose -nose-progressive -coverage -pep8>=1.1 diff --git a/mail/pkg/requirements.pip b/mail/pkg/requirements.pip deleted file mode 100644 index 0caa66b..0000000 --- a/mail/pkg/requirements.pip +++ /dev/null @@ -1,4 +0,0 @@ -zope.interface -twisted # >= 12.0.3 ?? -zope.proxy -service-identity diff --git a/mail/pkg/tools/get_authors.sh b/mail/pkg/tools/get_authors.sh deleted file mode 100755 index 0169bb1..0000000 --- a/mail/pkg/tools/get_authors.sh +++ /dev/null @@ -1,2 +0,0 @@ -#!/bin/sh -git log --format='%aN <%aE>' | awk '{arr[$0]++} END{for (i in arr){print arr[i], i;}}' | sort -rn | cut -d' ' -f2- diff --git a/mail/pkg/tools/with_venvwrapper.sh b/mail/pkg/tools/with_venvwrapper.sh deleted file mode 100755 index 693c0ac..0000000 --- a/mail/pkg/tools/with_venvwrapper.sh +++ /dev/null @@ -1,16 +0,0 @@ -#!/bin/bash - -#Wraps a command in a virtualenwrapper passed as first argument. -#Example: -#with_virtualenvwrapper.sh leap-bitmask ./run_tests.sh - -wd=`pwd` -alias pyver='python -c "import $1;print $1.__path__[0]; print $1.__version__;"' - -source `which virtualenvwrapper.sh` -echo "Activating virtualenv " $1 -echo "------------------------------------" -workon $1 -cd $wd -echo "running version: " `pyver leap.bitmask` -$2 $3 $4 $5 diff --git a/mail/pkg/utils.py b/mail/pkg/utils.py deleted file mode 100644 index d168010..0000000 --- a/mail/pkg/utils.py +++ /dev/null @@ -1,101 +0,0 @@ -# -*- coding: utf-8 -*- -# utils.py -# Copyright (C) 2013 LEAP -# -# This program is free software: you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation, either version 3 of the License, or -# (at your option) any later version. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with this program. If not, see . -""" -Utils to help in the setup process -""" -import os -import re -import sys - - -def is_develop_mode(): - """ - Returns True if we're calling the setup script using the argument for - setuptools development mode. - - This avoids messing up with dependency pinning and order, the - responsibility of installing the leap dependencies is left to the - developer. - """ - args = sys.argv - devflags = "setup.py", "develop" - if (args[0], args[1]) == devflags: - return True - return False - - -def get_reqs_from_files(reqfiles): - """ - Returns the contents of the top requirement file listed as a - string list with the lines. - - @param reqfiles: requirement files to parse - @type reqfiles: list of str - """ - for reqfile in reqfiles: - if os.path.isfile(reqfile): - return open(reqfile, 'r').read().split('\n') - - -def parse_requirements(reqfiles=['requirements.txt', - 'requirements.pip', - 'pkg/requirements.pip']): - """ - Parses the requirement files provided. - - The passed reqfiles list is a list of possible locations to try, the - function will return the contents of the first path found. - - Checks the value of LEAP_VENV_SKIP_PYSIDE to see if it should - return PySide as a dep or not. Don't set, or set to 0 if you want - to install it through pip. - - @param reqfiles: requirement files to parse - @type reqfiles: list of str - """ - - requirements = [] - skip_pyside = os.getenv("LEAP_VENV_SKIP_PYSIDE", "0") != "0" - for line in get_reqs_from_files(reqfiles): - # -e git://foo.bar/baz/master#egg=foobar - if re.match(r'\s*-e\s+', line): - pass - # do not try to do anything with externals on vcs - # requirements.append(re.sub(r'\s*-e\s+.*#egg=(.*)$', r'\1', - # line)) - # http://foo.bar/baz/foobar/zipball/master#egg=foobar - elif re.match(r'\s*https?:', line): - requirements.append(re.sub(r'\s*https?:.*#egg=(.*)$', r'\1', - line)) - # -f lines are for index locations, and don't get used here - elif re.match(r'\s*-f\s+', line): - pass - - # argparse is part of the standard library starting with 2.7 - # adding it to the requirements list screws distro installs - elif line == 'argparse' and sys.version_info >= (2, 7): - pass - elif line == 'PySide' and skip_pyside: - pass - # do not include comments - elif line.lstrip().startswith('#'): - pass - else: - if line != '': - requirements.append(line) - - return requirements diff --git a/mail/setup.cfg b/mail/setup.cfg deleted file mode 100644 index a49fd35..0000000 --- a/mail/setup.cfg +++ /dev/null @@ -1,24 +0,0 @@ -[aliases] -test = trial - -[pep8] -exclude = versioneer.py,_version.py,*.egg,build,docs -ignore = E731 - -[flake8] -exclude = versioneer.py,_version.py,*.egg,build,docs -ignore = E731 - -[tool:pytest] -pep8ignore = - docs/conf.py ALL - versioneer.py ALL - _version.py ALL - *.egg ALL - -[versioneer] -VCS = git -style = pep440 -versionfile_source = src/leap/mail/_version.py -versionfile_build = leap/mail/_version.py -tag_prefix = diff --git a/mail/setup.py b/mail/setup.py deleted file mode 100644 index ede620d..0000000 --- a/mail/setup.py +++ /dev/null @@ -1,156 +0,0 @@ -# -*- coding: utf-8 -*- -# setup.py -# Copyright (C) 2013 LEAP -# -# This program is free software: you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation, either version 3 of the License, or -# (at your option) any later version. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with this program. If not, see . -""" -Setup file for leap.mail -""" -import re -from setuptools import setup -from setuptools import find_packages -from setuptools import Command - -from pkg import utils - - -import versioneer - -trove_classifiers = [ - 'Development Status :: 4 - Beta', - 'Framework :: Twisted', - 'Intended Audience :: Developers', - 'License :: OSI Approved :: GNU General Public License ' - 'v3 (GPLv3)', - 'Operating System :: OS Independent', - 'Programming Language :: Python', - 'Programming Language :: Python :: 2.6', - 'Programming Language :: Python :: 2.7', - 'Topic :: Communications :: Email', - 'Topic :: Communications :: Email :: Post-Office :: IMAP', - 'Topic :: Communications :: Email :: Post-Office :: POP3', - 'Topic :: Internet', - 'Topic :: Security :: Cryptography', - 'Topic :: Software Development :: Libraries', -] - -DOWNLOAD_BASE = ('https://github.com/leapcode/leap_mail/' - 'archive/%s.tar.gz') -_versions = versioneer.get_versions() -VERSION = _versions['version'] -VERSION_REVISION = _versions['full-revisionid'] -DOWNLOAD_URL = "" - -# get the short version for the download url -_version_short = re.findall('\d+\.\d+\.\d+', VERSION) -if len(_version_short) > 0: - VERSION_SHORT = _version_short[0] - DOWNLOAD_URL = DOWNLOAD_BASE % VERSION_SHORT - -cmdclass = versioneer.get_cmdclass() - - -class freeze_debianver(Command): - - """ - Freezes the version in a debian branch. - To be used after merging the development branch onto the debian one. - """ - user_options = [] - template = r""" -# This file was generated by the `freeze_debianver` command in setup.py -# Using 'versioneer.py' (0.16) from -# revision-control system data, or from the parent directory name of an -# unpacked source archive. Distribution tarballs contain a pre-generated copy -# of this file. - -import json -import sys - -version_json = ''' -{ - "dirty": false, - "error": null, - "full-revisionid": "FULL_REVISIONID", - "version": "VERSION_STRING" -} -''' # END VERSION_JSON - -def get_versions(): - return json.loads(version_json) -""" - - def initialize_options(self): - pass - - def finalize_options(self): - pass - - def run(self): - proceed = str(raw_input( - "This will overwrite the file _version.py. Continue? [y/N] ")) - if proceed != "y": - print("He. You scared. Aborting.") - return - subst_template = self.template.replace( - 'VERSION_STRING', VERSION_SHORT).replace( - 'FULL_REVISIONID', VERSION_REVISION) - versioneer_cfg = versioneer.get_config_from_root('.') - with open(versioneer_cfg.versionfile_source, 'w') as f: - f.write(subst_template) - - -cmdclass["freeze_debianver"] = freeze_debianver - -# XXX add ref to docs - -requirements = utils.parse_requirements() - -if utils.is_develop_mode(): - print - print ("[WARNING] Skipping leap-specific dependencies " - "because development mode is detected.") - print ("[WARNING] You can install " - "the latest published versions with " - "'pip install -r pkg/requirements-leap.pip'") - print ("[WARNING] Or you can instead do 'python setup.py develop' " - "from the parent folder of each one of them.") - print -else: - requirements += utils.parse_requirements( - reqfiles=["pkg/requirements-leap.pip"]) - -setup( - name='leap.mail', - version=VERSION, - cmdclass=cmdclass, - url='https://leap.se/', - download_url=DOWNLOAD_URL, - license='GPLv3+', - author='The LEAP Encryption Access Project', - author_email='info@leap.se', - maintainer='Kali Kaneko', - maintainer_email='kali@leap.se', - description='Mail Services provided by Bitmask, the LEAP Client.', - long_description=open('README.rst').read() + '\n\n\n' + - open('CHANGELOG.rst').read(), - classifiers=trove_classifiers, - namespace_packages=["leap"], - package_dir={'': 'src'}, - packages=find_packages('src'), - test_suite='leap.mail.load_tests.load_tests', - install_requires=requirements, - tests_require=utils.parse_requirements( - reqfiles=['pkg/requirements-testing.pip']), -) diff --git a/mail/src/leap/__init__.py b/mail/src/leap/__init__.py deleted file mode 100644 index f48ad10..0000000 --- a/mail/src/leap/__init__.py +++ /dev/null @@ -1,6 +0,0 @@ -# See http://peak.telecommunity.com/DevCenter/setuptools#namespace-packages -try: - __import__('pkg_resources').declare_namespace(__name__) -except ImportError: - from pkgutil import extend_path - __path__ = extend_path(__path__, __name__) diff --git a/mail/src/leap/mail/__init__.py b/mail/src/leap/mail/__init__.py deleted file mode 100644 index 4b25fe6..0000000 --- a/mail/src/leap/mail/__init__.py +++ /dev/null @@ -1,26 +0,0 @@ -# -*- coding: utf-8 -*- -# __init__.py -# Copyright (C) 2013 LEAP -# -# This program is free software: you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation, either version 3 of the License, or -# (at your option) any later version. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with this program. If not, see . - - -""" -Client mail bits. -""" - - -from ._version import get_versions -__version__ = get_versions()['version'] -del get_versions diff --git a/mail/src/leap/mail/_version.py b/mail/src/leap/mail/_version.py deleted file mode 100644 index 954f488..0000000 --- a/mail/src/leap/mail/_version.py +++ /dev/null @@ -1,484 +0,0 @@ - -# This file helps to compute a version number in source trees obtained from -# git-archive tarball (such as those provided by githubs download-from-tag -# feature). Distribution tarballs (built by setup.py sdist) and build -# directories (produced by setup.py build) will contain a much shorter file -# that just contains the computed version number. - -# This file is released into the public domain. Generated by -# versioneer-0.16 (https://github.com/warner/python-versioneer) - -"""Git implementation of _version.py.""" - -import errno -import os -import re -import subprocess -import sys - - -def get_keywords(): - """Get the keywords needed to look up the version information.""" - # these strings will be replaced by git during git-archive. - # setup.py/versioneer.py will grep for the variable names, so they must - # each be defined on a line of their own. _version.py will just call - # get_keywords(). - git_refnames = "$Format:%d$" - git_full = "$Format:%H$" - keywords = {"refnames": git_refnames, "full": git_full} - return keywords - - -class VersioneerConfig: - """Container for Versioneer configuration parameters.""" - - -def get_config(): - """Create, populate and return the VersioneerConfig() object.""" - # these strings are filled in when 'setup.py versioneer' creates - # _version.py - cfg = VersioneerConfig() - cfg.VCS = "git" - cfg.style = "pep440" - cfg.tag_prefix = "" - cfg.parentdir_prefix = "None" - cfg.versionfile_source = "src/leap/mail/_version.py" - cfg.verbose = False - return cfg - - -class NotThisMethod(Exception): - """Exception raised if a method is not valid for the current scenario.""" - - -LONG_VERSION_PY = {} -HANDLERS = {} - - -def register_vcs_handler(vcs, method): # decorator - """Decorator to mark a method as the handler for a particular VCS.""" - def decorate(f): - """Store f in HANDLERS[vcs][method].""" - if vcs not in HANDLERS: - HANDLERS[vcs] = {} - HANDLERS[vcs][method] = f - return f - return decorate - - -def run_command(commands, args, cwd=None, verbose=False, hide_stderr=False): - """Call the given command(s).""" - assert isinstance(commands, list) - p = None - for c in commands: - try: - dispcmd = str([c] + args) - # remember shell=False, so use git.cmd on windows, not just git - p = subprocess.Popen([c] + args, cwd=cwd, stdout=subprocess.PIPE, - stderr=(subprocess.PIPE if hide_stderr - else None)) - break - except EnvironmentError: - e = sys.exc_info()[1] - if e.errno == errno.ENOENT: - continue - if verbose: - print("unable to run %s" % dispcmd) - print(e) - return None - else: - if verbose: - print("unable to find command, tried %s" % (commands,)) - return None - stdout = p.communicate()[0].strip() - if sys.version_info[0] >= 3: - stdout = stdout.decode() - if p.returncode != 0: - if verbose: - print("unable to run %s (error)" % dispcmd) - return None - return stdout - - -def versions_from_parentdir(parentdir_prefix, root, verbose): - """Try to determine the version from the parent directory name. - - Source tarballs conventionally unpack into a directory that includes - both the project name and a version string. - """ - dirname = os.path.basename(root) - if not dirname.startswith(parentdir_prefix): - if verbose: - print("guessing rootdir is '%s', but '%s' doesn't start with " - "prefix '%s'" % (root, dirname, parentdir_prefix)) - raise NotThisMethod("rootdir doesn't start with parentdir_prefix") - return {"version": dirname[len(parentdir_prefix):], - "full-revisionid": None, - "dirty": False, "error": None} - - -@register_vcs_handler("git", "get_keywords") -def git_get_keywords(versionfile_abs): - """Extract version information from the given file.""" - # the code embedded in _version.py can just fetch the value of these - # keywords. When used from setup.py, we don't want to import _version.py, - # so we do it with a regexp instead. This function is not used from - # _version.py. - keywords = {} - try: - f = open(versionfile_abs, "r") - for line in f.readlines(): - if line.strip().startswith("git_refnames ="): - mo = re.search(r'=\s*"(.*)"', line) - if mo: - keywords["refnames"] = mo.group(1) - if line.strip().startswith("git_full ="): - mo = re.search(r'=\s*"(.*)"', line) - if mo: - keywords["full"] = mo.group(1) - f.close() - except EnvironmentError: - pass - return keywords - - -@register_vcs_handler("git", "keywords") -def git_versions_from_keywords(keywords, tag_prefix, verbose): - """Get version information from git keywords.""" - if not keywords: - raise NotThisMethod("no keywords at all, weird") - refnames = keywords["refnames"].strip() - if refnames.startswith("$Format"): - if verbose: - print("keywords are unexpanded, not using") - raise NotThisMethod("unexpanded keywords, not a git-archive tarball") - refs = set([r.strip() for r in refnames.strip("()").split(",")]) - # starting in git-1.8.3, tags are listed as "tag: foo-1.0" instead of - # just "foo-1.0". If we see a "tag: " prefix, prefer those. - TAG = "tag: " - tags = set([r[len(TAG):] for r in refs if r.startswith(TAG)]) - if not tags: - # Either we're using git < 1.8.3, or there really are no tags. We use - # a heuristic: assume all version tags have a digit. The old git %d - # expansion behaves like git log --decorate=short and strips out the - # refs/heads/ and refs/tags/ prefixes that would let us distinguish - # between branches and tags. By ignoring refnames without digits, we - # filter out many common branch names like "release" and - # "stabilization", as well as "HEAD" and "master". - tags = set([r for r in refs if re.search(r'\d', r)]) - if verbose: - print("discarding '%s', no digits" % ",".join(refs-tags)) - if verbose: - print("likely tags: %s" % ",".join(sorted(tags))) - for ref in sorted(tags): - # sorting will prefer e.g. "2.0" over "2.0rc1" - if ref.startswith(tag_prefix): - r = ref[len(tag_prefix):] - if verbose: - print("picking %s" % r) - return {"version": r, - "full-revisionid": keywords["full"].strip(), - "dirty": False, "error": None - } - # no suitable tags, so version is "0+unknown", but full hex is still there - if verbose: - print("no suitable tags, using unknown + full revision id") - return {"version": "0+unknown", - "full-revisionid": keywords["full"].strip(), - "dirty": False, "error": "no suitable tags"} - - -@register_vcs_handler("git", "pieces_from_vcs") -def git_pieces_from_vcs(tag_prefix, root, verbose, run_command=run_command): - """Get version from 'git describe' in the root of the source tree. - - This only gets called if the git-archive 'subst' keywords were *not* - expanded, and _version.py hasn't already been rewritten with a short - version string, meaning we're inside a checked out source tree. - """ - if not os.path.exists(os.path.join(root, ".git")): - if verbose: - print("no .git in %s" % root) - raise NotThisMethod("no .git directory") - - GITS = ["git"] - if sys.platform == "win32": - GITS = ["git.cmd", "git.exe"] - # if there is a tag matching tag_prefix, this yields TAG-NUM-gHEX[-dirty] - # if there isn't one, this yields HEX[-dirty] (no NUM) - describe_out = run_command(GITS, ["describe", "--tags", "--dirty", - "--always", "--long", - "--match", "%s*" % tag_prefix], - cwd=root) - # --long was added in git-1.5.5 - if describe_out is None: - raise NotThisMethod("'git describe' failed") - describe_out = describe_out.strip() - full_out = run_command(GITS, ["rev-parse", "HEAD"], cwd=root) - if full_out is None: - raise NotThisMethod("'git rev-parse' failed") - full_out = full_out.strip() - - pieces = {} - pieces["long"] = full_out - pieces["short"] = full_out[:7] # maybe improved later - pieces["error"] = None - - # parse describe_out. It will be like TAG-NUM-gHEX[-dirty] or HEX[-dirty] - # TAG might have hyphens. - git_describe = describe_out - - # look for -dirty suffix - dirty = git_describe.endswith("-dirty") - pieces["dirty"] = dirty - if dirty: - git_describe = git_describe[:git_describe.rindex("-dirty")] - - # now we have TAG-NUM-gHEX or HEX - - if "-" in git_describe: - # TAG-NUM-gHEX - mo = re.search(r'^(.+)-(\d+)-g([0-9a-f]+)$', git_describe) - if not mo: - # unparseable. Maybe git-describe is misbehaving? - pieces["error"] = ("unable to parse git-describe output: '%s'" - % describe_out) - return pieces - - # tag - full_tag = mo.group(1) - if not full_tag.startswith(tag_prefix): - if verbose: - fmt = "tag '%s' doesn't start with prefix '%s'" - print(fmt % (full_tag, tag_prefix)) - pieces["error"] = ("tag '%s' doesn't start with prefix '%s'" - % (full_tag, tag_prefix)) - return pieces - pieces["closest-tag"] = full_tag[len(tag_prefix):] - - # distance: number of commits since tag - pieces["distance"] = int(mo.group(2)) - - # commit: short hex revision ID - pieces["short"] = mo.group(3) - - else: - # HEX: no tags - pieces["closest-tag"] = None - count_out = run_command(GITS, ["rev-list", "HEAD", "--count"], - cwd=root) - pieces["distance"] = int(count_out) # total number of commits - - return pieces - - -def plus_or_dot(pieces): - """Return a + if we don't already have one, else return a .""" - if "+" in pieces.get("closest-tag", ""): - return "." - return "+" - - -def render_pep440(pieces): - """Build up version string, with post-release "local version identifier". - - Our goal: TAG[+DISTANCE.gHEX[.dirty]] . Note that if you - get a tagged build and then dirty it, you'll get TAG+0.gHEX.dirty - - Exceptions: - 1: no tags. git_describe was just HEX. 0+untagged.DISTANCE.gHEX[.dirty] - """ - if pieces["closest-tag"]: - rendered = pieces["closest-tag"] - if pieces["distance"] or pieces["dirty"]: - rendered += plus_or_dot(pieces) - rendered += "%d.g%s" % (pieces["distance"], pieces["short"]) - if pieces["dirty"]: - rendered += ".dirty" - else: - # exception #1 - rendered = "0+untagged.%d.g%s" % (pieces["distance"], - pieces["short"]) - if pieces["dirty"]: - rendered += ".dirty" - return rendered - - -def render_pep440_pre(pieces): - """TAG[.post.devDISTANCE] -- No -dirty. - - Exceptions: - 1: no tags. 0.post.devDISTANCE - """ - if pieces["closest-tag"]: - rendered = pieces["closest-tag"] - if pieces["distance"]: - rendered += ".post.dev%d" % pieces["distance"] - else: - # exception #1 - rendered = "0.post.dev%d" % pieces["distance"] - return rendered - - -def render_pep440_post(pieces): - """TAG[.postDISTANCE[.dev0]+gHEX] . - - The ".dev0" means dirty. Note that .dev0 sorts backwards - (a dirty tree will appear "older" than the corresponding clean one), - but you shouldn't be releasing software with -dirty anyways. - - Exceptions: - 1: no tags. 0.postDISTANCE[.dev0] - """ - if pieces["closest-tag"]: - rendered = pieces["closest-tag"] - if pieces["distance"] or pieces["dirty"]: - rendered += ".post%d" % pieces["distance"] - if pieces["dirty"]: - rendered += ".dev0" - rendered += plus_or_dot(pieces) - rendered += "g%s" % pieces["short"] - else: - # exception #1 - rendered = "0.post%d" % pieces["distance"] - if pieces["dirty"]: - rendered += ".dev0" - rendered += "+g%s" % pieces["short"] - return rendered - - -def render_pep440_old(pieces): - """TAG[.postDISTANCE[.dev0]] . - - The ".dev0" means dirty. - - Eexceptions: - 1: no tags. 0.postDISTANCE[.dev0] - """ - if pieces["closest-tag"]: - rendered = pieces["closest-tag"] - if pieces["distance"] or pieces["dirty"]: - rendered += ".post%d" % pieces["distance"] - if pieces["dirty"]: - rendered += ".dev0" - else: - # exception #1 - rendered = "0.post%d" % pieces["distance"] - if pieces["dirty"]: - rendered += ".dev0" - return rendered - - -def render_git_describe(pieces): - """TAG[-DISTANCE-gHEX][-dirty]. - - Like 'git describe --tags --dirty --always'. - - Exceptions: - 1: no tags. HEX[-dirty] (note: no 'g' prefix) - """ - if pieces["closest-tag"]: - rendered = pieces["closest-tag"] - if pieces["distance"]: - rendered += "-%d-g%s" % (pieces["distance"], pieces["short"]) - else: - # exception #1 - rendered = pieces["short"] - if pieces["dirty"]: - rendered += "-dirty" - return rendered - - -def render_git_describe_long(pieces): - """TAG-DISTANCE-gHEX[-dirty]. - - Like 'git describe --tags --dirty --always -long'. - The distance/hash is unconditional. - - Exceptions: - 1: no tags. HEX[-dirty] (note: no 'g' prefix) - """ - if pieces["closest-tag"]: - rendered = pieces["closest-tag"] - rendered += "-%d-g%s" % (pieces["distance"], pieces["short"]) - else: - # exception #1 - rendered = pieces["short"] - if pieces["dirty"]: - rendered += "-dirty" - return rendered - - -def render(pieces, style): - """Render the given version pieces into the requested style.""" - if pieces["error"]: - return {"version": "unknown", - "full-revisionid": pieces.get("long"), - "dirty": None, - "error": pieces["error"]} - - if not style or style == "default": - style = "pep440" # the default - - if style == "pep440": - rendered = render_pep440(pieces) - elif style == "pep440-pre": - rendered = render_pep440_pre(pieces) - elif style == "pep440-post": - rendered = render_pep440_post(pieces) - elif style == "pep440-old": - rendered = render_pep440_old(pieces) - elif style == "git-describe": - rendered = render_git_describe(pieces) - elif style == "git-describe-long": - rendered = render_git_describe_long(pieces) - else: - raise ValueError("unknown style '%s'" % style) - - return {"version": rendered, "full-revisionid": pieces["long"], - "dirty": pieces["dirty"], "error": None} - - -def get_versions(): - """Get version information or return default if unable to do so.""" - # I am in _version.py, which lives at ROOT/VERSIONFILE_SOURCE. If we have - # __file__, we can work backwards from there to the root. Some - # py2exe/bbfreeze/non-CPython implementations don't do __file__, in which - # case we can only use expanded keywords. - - cfg = get_config() - verbose = cfg.verbose - - try: - return git_versions_from_keywords(get_keywords(), cfg.tag_prefix, - verbose) - except NotThisMethod: - pass - - try: - root = os.path.realpath(__file__) - # versionfile_source is the relative path from the top of the source - # tree (where the .git directory might live) to this file. Invert - # this to find the root from __file__. - for i in cfg.versionfile_source.split('/'): - root = os.path.dirname(root) - except NameError: - return {"version": "0+unknown", "full-revisionid": None, - "dirty": None, - "error": "unable to find root of source tree"} - - try: - pieces = git_pieces_from_vcs(cfg.tag_prefix, root, verbose) - return render(pieces, cfg.style) - except NotThisMethod: - pass - - try: - if cfg.parentdir_prefix: - return versions_from_parentdir(cfg.parentdir_prefix, root, verbose) - except NotThisMethod: - pass - - return {"version": "0+unknown", "full-revisionid": None, - "dirty": None, - "error": "unable to compute version"} diff --git a/mail/src/leap/mail/adaptors/__init__.py b/mail/src/leap/mail/adaptors/__init__.py deleted file mode 100644 index e69de29..0000000 diff --git a/mail/src/leap/mail/adaptors/models.py b/mail/src/leap/mail/adaptors/models.py deleted file mode 100644 index 49460f7..0000000 --- a/mail/src/leap/mail/adaptors/models.py +++ /dev/null @@ -1,123 +0,0 @@ -# -*- coding: utf-8 -*- -# models.py -# Copyright (C) 2014 LEAP -# -# This program is free software: you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation, either version 3 of the License, or -# (at your option) any later version. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with this program. If not, see . -""" -Generic Models to be used by the Document Adaptors. -""" -import copy - - -class SerializableModel(object): - """ - A Generic document model, that can be serialized into a dictionary. - - Subclasses of this `SerializableModel` are meant to be added as class - attributes of classes inheriting from DocumentWrapper. - - A subclass __meta__ of this SerializableModel might exist, and contain info - relative to particularities of this model. - - For instance, the use of `__meta__.index` marks the existence of a primary - index in the model, which will be used to do unique queries (in which case - all the other indexed fields in the underlying document will be filled with - the default info contained in the model definition). - """ - - @classmethod - def serialize(klass): - """ - Get a dictionary representation of the public attributes in the model - class. To avoid collisions with builtin functions, any occurrence of an - attribute ended in '_' (like 'type_') will be normalized by removing - the trailing underscore. - - This classmethod is used from within the serialized method of a - DocumentWrapper instance: it provides defaults for the - empty document. - """ - assert isinstance(klass, type) - return _normalize_dict(klass.__dict__) - - -class DocumentWrapper(object): - """ - A Wrapper object that can be manipulated, passed around, and serialized in - a format that the store understands. - It is related to a SerializableModel, which must be specified as the - ``model`` class attribute. The instance of this DocumentWrapper will not - allow any other *public* attributes than those defined in the corresponding - model. - """ - # TODO we could do some very basic type checking here - # TODO set a dirty flag (on __setattr__, whenever the value is != from - # before) - # TODO we could enforce the existence of a correct "model" attribute - # in some other way (other than in the initializer) - - def __init__(self, **kwargs): - if not getattr(self, 'model', None): - raise RuntimeError( - 'DocumentWrapper class needs a model attribute') - - defaults = self.model.serialize() - - if kwargs: - values = copy.deepcopy(defaults) - values.update(_normalize_dict(kwargs)) - else: - values = defaults - - for k, v in values.items(): - k = k.replace('-', '_') - setattr(self, k, v) - - def __setattr__(self, attr, value): - normalized = _normalize_dict(self.model.__dict__) - if not attr.startswith('_') and attr not in normalized: - raise RuntimeError( - "Cannot set attribute because it's not defined " - "in the model %s: %s" % (self.__class__, attr)) - object.__setattr__(self, attr, value) - - def serialize(self): - return _normalize_dict(self.__dict__) - - def create(self): - raise NotImplementedError() - - def update(self): - raise NotImplementedError() - - def delete(self): - raise NotImplementedError() - - @classmethod - def get_or_create(self): - raise NotImplementedError() - - @classmethod - def get_all(self): - raise NotImplementedError() - - -def _normalize_dict(_dict): - items = _dict.items() - items = filter(lambda (k, v): not callable(v), items) - items = filter(lambda (k, v): not k.startswith('_'), items) - items = [(k, v) if not k.endswith('_') else (k[:-1], v) - for (k, v) in items] - items = [(k.replace('-', '_'), v) for (k, v) in items] - return dict(items) diff --git a/mail/src/leap/mail/adaptors/soledad.py b/mail/src/leap/mail/adaptors/soledad.py deleted file mode 100644 index ca8f741..0000000 --- a/mail/src/leap/mail/adaptors/soledad.py +++ /dev/null @@ -1,1268 +0,0 @@ -# soledad.py -# Copyright (C) 2014 LEAP -# -# This program is free software: you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation, either version 3 of the License, or -# (at your option) any later version. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with this program. If not, see . -""" -Soledadad MailAdaptor module. -""" -import logging -import re - -from collections import defaultdict -from email import message_from_string - -from twisted.internet import defer -from twisted.python import log -from zope.interface import implements -from leap.soledad.common import l2db - -from leap.common.check import leap_assert, leap_assert_type - -from leap.mail import constants -from leap.mail import walk -from leap.mail.adaptors import soledad_indexes as indexes -from leap.mail.constants import INBOX_NAME -from leap.mail.adaptors import models -from leap.mail.imap.mailbox import normalize_mailbox -from leap.mail.utils import lowerdict, first -from leap.mail.utils import stringify_parts_map -from leap.mail.interfaces import IMailAdaptor, IMessageWrapper - -from leap.soledad.common.document import SoledadDocument - - -logger = logging.getLogger(__name__) - -# TODO -# [ ] Convenience function to create mail specifying subject, date, etc? - - -_MSGID_PATTERN = r"""<([\w@.]+)>""" -_MSGID_RE = re.compile(_MSGID_PATTERN) - - -class DuplicatedDocumentError(Exception): - """ - Raised when a duplicated document is detected. - """ - pass - - -def cleanup_deferred_locks(): - """ - Need to use this from within trial to cleanup the reactor before - each run. - """ - SoledadDocumentWrapper._k_locks = defaultdict(defer.DeferredLock) - - -class SoledadDocumentWrapper(models.DocumentWrapper): - """ - A Wrapper object that can be manipulated, passed around, and serialized in - a format that the Soledad Store understands. - - It ensures atomicity of the document operations on creation, update and - deletion. - """ - # TODO we could also use a _dirty flag (in models) - # TODO add a get_count() method ??? -- that is extended over l2db. - - # We keep a dictionary with DeferredLocks, that will be - # unique to every subclass of SoledadDocumentWrapper. - _k_locks = defaultdict(defer.DeferredLock) - - @classmethod - def _get_klass_lock(cls): - """ - Get a DeferredLock that is unique for this subclass name. - Used to lock the access to indexes in the `get_or_create` call - for a particular DocumentWrapper. - """ - return cls._k_locks[cls.__name__] - - def __init__(self, doc_id=None, future_doc_id=None, **kwargs): - self._doc_id = doc_id - self._future_doc_id = future_doc_id - self._lock = defer.DeferredLock() - super(SoledadDocumentWrapper, self).__init__(**kwargs) - - @property - def doc_id(self): - return self._doc_id - - @property - def future_doc_id(self): - return self._future_doc_id - - def set_future_doc_id(self, doc_id): - self._future_doc_id = doc_id - - def create(self, store, is_copy=False): - """ - Create the documents for this wrapper. - Since this method will not check for duplication, the - responsibility of avoiding duplicates is left to the caller. - - You might be interested in using `get_or_create` classmethod - instead (that's the preferred way of creating documents from - the wrapper object). - - :return: a deferred that will fire when the underlying - Soledad document has been created. - :rtype: Deferred - """ - leap_assert(self._doc_id is None, - "This document already has a doc_id!") - - def update_doc_id(doc): - self._doc_id = doc.doc_id - self.set_future_doc_id(None) - return doc - - def update_wrapper(failure): - # In the case of some copies (for instance, from one folder to - # another and back to the original folder), the document that we - # want to insert already exists. In this case, putting it - # and overwriting the document with that doc_id is the right thing - # to do. - failure.trap(l2db.errors.RevisionConflict) - self._doc_id = self.future_doc_id - self._future_doc_id = None - return self.update(store) - - if self.future_doc_id is None: - d = store.create_doc(self.serialize()) - else: - d = store.create_doc(self.serialize(), - doc_id=self.future_doc_id) - d.addCallback(update_doc_id) - - if is_copy: - d.addErrback(update_wrapper) - else: - d.addErrback(self._catch_revision_conflict, self.future_doc_id) - return d - - def update(self, store): - """ - Update the documents for this wrapper. - - :return: a deferred that will fire when the underlying - Soledad document has been updated. - :rtype: Deferred - """ - # the deferred lock guards against revision conflicts - return self._lock.run(self._update, store) - - def _update(self, store): - leap_assert(self._doc_id is not None, - "Need to create doc before updating") - - def update_and_put_doc(doc): - doc.content.update(self.serialize()) - d = store.put_doc(doc) - d.addErrback(self._catch_revision_conflict, doc.doc_id) - return d - - d = store.get_doc(self._doc_id) - d.addCallback(update_and_put_doc) - return d - - def _catch_revision_conflict(self, failure, doc_id): - # XXX We can have some RevisionConflicts if we try - # to put the docs that are already there. - # This can happen right now when creating/saving the cdocs - # during a copy. Instead of catching and ignoring this - # error, we should mark them in the copy so there is no attempt to - # create/update them. - failure.trap(l2db.errors.RevisionConflict) - logger.debug("Got conflict while putting %s" % doc_id) - - def delete(self, store): - """ - Delete the documents for this wrapper. - - :return: a deferred that will fire when the underlying - Soledad document has been deleted. - :rtype: Deferred - """ - # the deferred lock guards against conflicts while updating - return self._lock.run(self._delete, store) - - def _delete(self, store): - leap_assert(self._doc_id is not None, - "Need to create doc before deleting") - # XXX might want to flag this DocumentWrapper to avoid - # updating it by mistake. This could go in models.DocumentWrapper - - def delete_doc(doc): - return store.delete_doc(doc) - - d = store.get_doc(self._doc_id) - d.addCallback(delete_doc) - return d - - @classmethod - def get_or_create(cls, store, index, value): - """ - Get a unique DocumentWrapper by index, or create a new one if the - matching query does not exist. - - :param index: the primary index for the model. - :type index: str - :param value: the value to query the primary index. - :type value: str - - :return: a deferred that will be fired with the SoledadDocumentWrapper - matching the index query, either existing or just created. - :rtype: Deferred - """ - return cls._get_klass_lock().run( - cls._get_or_create, store, index, value) - - @classmethod - def _get_or_create(cls, store, index, value): - # TODO shorten this method. - assert store is not None - assert index is not None - assert value is not None - - def get_main_index(): - try: - return cls.model.__meta__.index - except AttributeError: - raise RuntimeError("The model is badly defined") - - # TODO separate into another method? - def try_to_get_doc_from_index(indexes): - values = [] - idx_def = dict(indexes)[index] - if len(idx_def) == 1: - values = [value] - else: - main_index = get_main_index() - fields = cls.model.serialize() - for field in idx_def: - if field == main_index: - values.append(value) - else: - values.append(fields[field]) - d = store.get_from_index(index, *values) - return d - - def get_first_doc_if_any(docs): - if not docs: - return None - if len(docs) > 1: - raise DuplicatedDocumentError - return docs[0] - - def wrap_existing_or_create_new(doc): - if doc: - return cls(doc_id=doc.doc_id, **doc.content) - else: - return create_and_wrap_new_doc() - - def create_and_wrap_new_doc(): - # XXX use closure to store indexes instead of - # querying for them again. - d = store.list_indexes() - d.addCallback(get_wrapper_instance_from_index) - d.addCallback(return_wrapper_when_created) - return d - - def get_wrapper_instance_from_index(indexes): - init_values = {} - idx_def = dict(indexes)[index] - if len(idx_def) == 1: - init_value = {idx_def[0]: value} - return cls(**init_value) - main_index = get_main_index() - fields = cls.model.serialize() - for field in idx_def: - if field == main_index: - init_values[field] = value - else: - init_values[field] = fields[field] - return cls(**init_values) - - def return_wrapper_when_created(wrapper): - d = wrapper.create(store) - d.addCallback(lambda doc: wrapper) - return d - - d = store.list_indexes() - d.addCallback(try_to_get_doc_from_index) - d.addCallback(get_first_doc_if_any) - d.addCallback(wrap_existing_or_create_new) - return d - - @classmethod - def get_all(cls, store): - """ - Get a collection of wrappers around all the documents belonging - to this kind. - - For this to work, the model.__meta__ needs to include a tuple with - the index to be used for listing purposes, and which is the field to be - used to query the index. - - Note that this method only supports indexes of a single field at the - moment. It also might be too expensive to return all the documents - matching the query, so handle with care. - - class __meta__(object): - index = "name" - list_index = ("by-type", "type_") - - :return: a deferred that will be fired with an iterable containing - as many SoledadDocumentWrapper are matching the index defined - in the model as the `list_index`. - :rtype: Deferred - """ - # TODO LIST (get_all) - # [ ] extend support to indexes with n-ples - # [ ] benchmark the cost of querying and returning indexes in a big - # database. This might badly need pagination before being put to - # serious use. - return cls._get_klass_lock().run(cls._get_all, store) - - @classmethod - def _get_all(cls, store): - try: - list_index, list_attr = cls.model.__meta__.list_index - except AttributeError: - raise RuntimeError("The model is badly defined: no list_index") - try: - index_value = getattr(cls.model, list_attr) - except AttributeError: - raise RuntimeError("The model is badly defined: " - "no attribute matching list_index") - - def wrap_docs(docs): - return (cls(doc_id=doc.doc_id, **doc.content) for doc in docs) - - d = store.get_from_index(list_index, index_value) - d.addCallback(wrap_docs) - return d - - def __repr__(self): - try: - idx = getattr(self, self.model.__meta__.index) - except AttributeError: - idx = "" - return "<%s: %s (%s)>" % (self.__class__.__name__, - idx, self._doc_id) - - -# -# Message documents -# - -class FlagsDocWrapper(SoledadDocumentWrapper): - - class model(models.SerializableModel): - type_ = "flags" - chash = "" - - mbox_uuid = "" - seen = False - deleted = False - recent = False - flags = [] - tags = [] - size = 0 - multi = False - - class __meta__(object): - index = "mbox" - - def set_mbox_uuid(self, mbox_uuid): - # XXX raise error if already created, should use copy instead - mbox_uuid = mbox_uuid.replace('-', '_') - new_id = constants.FDOCID.format(mbox_uuid=mbox_uuid, chash=self.chash) - self._future_doc_id = new_id - self.mbox_uuid = mbox_uuid - - def get_flags(self): - """ - Get the flags for this message (as a tuple of strings, not unicode). - """ - return map(str, self.flags) - - -class HeaderDocWrapper(SoledadDocumentWrapper): - - class model(models.SerializableModel): - type_ = "head" - chash = "" - - date = "" - subject = "" - headers = {} - part_map = {} - body = "" # link to phash of body - msgid = "" - multi = False - - class __meta__(object): - index = "chash" - - -class ContentDocWrapper(SoledadDocumentWrapper): - - class model(models.SerializableModel): - type_ = "cnt" - phash = "" - - ctype = "" # XXX index by ctype too? - lkf = [] # XXX not implemented yet! - raw = "" - - content_disposition = "" - content_transfer_encoding = "" - content_type = "" - - class __meta__(object): - index = "phash" - - -class MetaMsgDocWrapper(SoledadDocumentWrapper): - - class model(models.SerializableModel): - type_ = "meta" - fdoc = "" - hdoc = "" - cdocs = [] - - def set_mbox_uuid(self, mbox_uuid): - # XXX raise error if already created, should use copy instead - mbox_uuid = mbox_uuid.replace('-', '_') - chash = re.findall(constants.FDOCID_CHASH_RE, self.fdoc)[0] - new_id = constants.METAMSGID.format(mbox_uuid=mbox_uuid, chash=chash) - new_fdoc_id = constants.FDOCID.format(mbox_uuid=mbox_uuid, chash=chash) - self._future_doc_id = new_id - self.fdoc = new_fdoc_id - - -class MessageWrapper(object): - - # This could benefit of a DeferredLock to create/update all the - # documents at the same time maybe, and defend against concurrent updates? - - implements(IMessageWrapper) - - def __init__(self, mdoc, fdoc, hdoc, cdocs=None, is_copy=False): - """ - Need at least a metamsg-document, a flag-document and a header-document - to instantiate a MessageWrapper. Content-documents can be retrieved - lazily. - - cdocs, if any, should be a dictionary in which the keys are ascending - integers, beginning at one, and the values are dictionaries with the - content of the content-docs. - - is_copy, if set to True, will only attempt to create mdoc and fdoc - (because hdoc and cdocs are supposed to exist already) - """ - self._is_copy = is_copy - - def get_doc_wrapper(doc, cls): - if isinstance(doc, SoledadDocument): - doc_id = doc.doc_id - doc = doc.content - else: - doc_id = None - if not doc: - doc = {} - return cls(doc_id=doc_id, **doc) - - self.mdoc = get_doc_wrapper(mdoc, MetaMsgDocWrapper) - - self.fdoc = get_doc_wrapper(fdoc, FlagsDocWrapper) - self.fdoc.set_future_doc_id(self.mdoc.fdoc) - - self.hdoc = get_doc_wrapper(hdoc, HeaderDocWrapper) - self.hdoc.set_future_doc_id(self.mdoc.hdoc) - - if cdocs is None: - cdocs = {} - cdocs_keys = cdocs.keys() - assert sorted(cdocs_keys) == range(1, len(cdocs_keys) + 1) - self.cdocs = dict([ - (key, get_doc_wrapper(doc, ContentDocWrapper)) - for (key, doc) in cdocs.items()]) - for doc_id, cdoc in zip(self.mdoc.cdocs, self.cdocs.values()): - if cdoc.raw == "": - log.msg("Empty raw field in cdoc %s" % doc_id) - cdoc.set_future_doc_id(doc_id) - - def create(self, store, notify_just_mdoc=False, pending_inserts_dict=None): - """ - Create all the parts for this message in the store. - - :param store: an instance of Soledad - - :param notify_just_mdoc: - if set to True, this method will return *only* the deferred - corresponding to the creation of the meta-message document. - Be warned that in that case there will be no record of failures - when creating the other part-documents. - - Otherwise, this method will return a deferred that will wait for - the creation of all the part documents. - - Setting this flag to True is mostly a convenient workaround for the - fact that massive serial appends will take too much time, and in - most of the cases the MUA will only switch to the mailbox where the - appends have happened after a certain time, which in most of the - times will be enough to have all the queued insert operations - finished. - :type notify_just_mdoc: bool - :param pending_inserts_dict: - a dictionary with the pending inserts ids. - :type pending_inserts_dict: dict - - :return: a deferred whose callback will be called when either all the - part documents have been written, or just the metamsg-doc, - depending on the value of the notify_just_mdoc flag - :rtype: defer.Deferred - """ - if pending_inserts_dict is None: - pending_inserts_dict = {} - - leap_assert(self.cdocs, - "Need non empty cdocs to create the " - "MessageWrapper documents") - leap_assert(self.mdoc.doc_id is None, - "Cannot create: mdoc has a doc_id") - leap_assert(self.fdoc.doc_id is None, - "Cannot create: fdoc has a doc_id") - - def unblock_pending_insert(result): - if pending_inserts_dict: - ci_headers = lowerdict(self.hdoc.headers) - msgid = ci_headers.get('message-id', None) - try: - d = pending_inserts_dict[msgid] - d.callback(msgid) - except KeyError: - pass - return result - - # TODO check that the doc_ids in the mdoc are coherent - self.d = [] - - mdoc_created = self.mdoc.create(store, is_copy=self._is_copy) - fdoc_created = self.fdoc.create(store, is_copy=self._is_copy) - - mdoc_created.addErrback(lambda f: log.err(f)) - fdoc_created.addErrback(lambda f: log.err(f)) - - self.d.append(mdoc_created) - self.d.append(fdoc_created) - - if not self._is_copy: - if self.hdoc.doc_id is None: - self.d.append(self.hdoc.create(store)) - for cdoc in self.cdocs.values(): - if cdoc.doc_id is not None: - # we could be just linking to an existing - # content-doc. - continue - self.d.append(cdoc.create(store)) - - def log_all_inserted(result): - log.msg("All parts inserted for msg!") - return result - - self.all_inserted_d = defer.gatherResults(self.d, consumeErrors=True) - self.all_inserted_d.addCallback(log_all_inserted) - self.all_inserted_d.addCallback(unblock_pending_insert) - self.all_inserted_d.addErrback(lambda failure: log.err(failure)) - - if notify_just_mdoc: - return mdoc_created - else: - return self.all_inserted_d - - def update(self, store): - """ - Update the only mutable parts, which are within the flags document. - """ - return self.fdoc.update(store) - - def delete(self, store): - # TODO - # Eventually this would have to do the duplicate search or send for the - # garbage collector. At least mdoc and t the mdoc and fdoc can be - # unlinked. - d = [] - if self.mdoc.doc_id: - d.append(self.mdoc.delete(store)) - d.append(self.fdoc.delete(store)) - return defer.gatherResults(d) - - def copy(self, store, new_mbox_uuid): - """ - Return a copy of this MessageWrapper in a new mailbox. - - :param store: an instance of Soledad, or anything that behaves alike. - :param new_mbox_uuid: the uuid of the mailbox where we are copying this - message to. - :type new_mbox_uuid: str - :rtype: MessageWrapper - """ - new_mdoc = self.mdoc.serialize() - new_fdoc = self.fdoc.serialize() - - # the future doc_ids is properly set because we modified - # the pointers in mdoc, which has precedence. - new_wrapper = MessageWrapper(new_mdoc, new_fdoc, None, None, - is_copy=True) - new_wrapper.hdoc = self.hdoc - new_wrapper.cdocs = self.cdocs - new_wrapper.set_mbox_uuid(new_mbox_uuid) - - # XXX could flag so that it only creates mdoc/fdoc... - - d = new_wrapper.create(store) - d.addCallback(lambda result: new_wrapper) - d.addErrback(lambda failure: log.err(failure)) - return d - - def set_mbox_uuid(self, mbox_uuid): - """ - Set the mailbox for this wrapper. - This method should only be used before the Documents for the - MessageWrapper have been created, will raise otherwise. - """ - mbox_uuid = mbox_uuid.replace('-', '_') - self.mdoc.set_mbox_uuid(mbox_uuid) - self.fdoc.set_mbox_uuid(mbox_uuid) - - def set_flags(self, flags): - # TODO serialize the get + update - if flags is None: - flags = tuple() - leap_assert_type(flags, tuple) - self.fdoc.flags = list(flags) - self.fdoc.deleted = "\\Deleted" in flags - self.fdoc.seen = "\\Seen" in flags - self.fdoc.recent = "\\Recent" in flags - - def set_tags(self, tags): - # TODO serialize the get + update - if tags is None: - tags = tuple() - leap_assert_type(tags, tuple) - self.fdoc.tags = list(tags) - - def set_date(self, date): - # XXX assert valid date format - self.hdoc.date = date - - def get_subpart_dict(self, index): - """ - :param index: the part to lookup, 1-indexed - :type index: int - :rtype: dict - """ - return self.hdoc.part_map[str(index)] - - def get_subpart_indexes(self): - return self.hdoc.part_map.keys() - - def get_body(self, store): - """ - :rtype: deferred - """ - body_phash = self.hdoc.body - if body_phash: - d = store.get_doc('C-' + body_phash) - d.addCallback(lambda doc: ContentDocWrapper(**doc.content)) - return d - elif self.cdocs: - return self.cdocs[1] - else: - return '' - -# -# Mailboxes -# - - -class MailboxWrapper(SoledadDocumentWrapper): - - class model(models.SerializableModel): - type_ = "mbox" - mbox = INBOX_NAME - uuid = None - flags = [] - recent = [] - created = 1 - closed = False - subscribed = False - - class __meta__(object): - index = "mbox" - list_index = (indexes.TYPE_IDX, 'type_') - - -# -# Soledad Adaptor -# - -class SoledadIndexMixin(object): - """ - This will need a class attribute `indexes`, that is a dictionary containing - the index definitions for the underlying l2db store underlying soledad. - - It needs to be in the following format: - {'index-name': ['field1', 'field2']} - - You can also add a class attribute `wait_for_indexes` to any class - inheriting from this Mixin, that should be a list of strings representing - the methods that need to wait until the indexes have been initialized - before being able to work properly. - """ - # TODO move this mixin to soledad itself - # so that each application can pass a set of indexes for their data model. - - # TODO could have a wrapper class for indexes, supporting introspection - # and __getattr__ - - # TODO make this an interface? - - indexes = {} - wait_for_indexes = [] - store_ready = False - - def initialize_store(self, store): - """ - Initialize the indexes in the database. - - :param store: store - :returns: a Deferred that will fire when the store is correctly - initialized. - :rtype: deferred - """ - # TODO I think we *should* get another deferredLock in here, but - # global to the soledad namespace, to protect from several points - # initializing soledad indexes at the same time. - self._wait_for_indexes() - - d = self._init_indexes(store) - d.addCallback(self._restore_waiting_methods) - return d - - def _init_indexes(self, store): - """ - Initialize the database indexes. - """ - leap_assert(store, "Cannot init indexes with null soledad") - leap_assert_type(self.indexes, dict) - - def _create_index(name, expression): - return store.create_index(name, *expression) - - def init_idexes(indexes): - deferreds = [] - db_indexes = dict(indexes) - # Loop through the indexes we expect to find. - for name, expression in self.indexes.items(): - if name not in db_indexes: - # The index does not yet exist. - d = _create_index(name, expression) - deferreds.append(d) - elif expression != db_indexes[name]: - # The index exists but the definition is not what expected, - # so we delete it and add the proper index expression. - d = store.delete_index(name) - d.addCallback( - lambda _: _create_index(name, *expression)) - deferreds.append(d) - return defer.gatherResults(deferreds, consumeErrors=True) - - def store_ready(whatever): - self.store_ready = True - return whatever - - self.deferred_indexes = store.list_indexes() - self.deferred_indexes.addCallback(init_idexes) - self.deferred_indexes.addCallback(store_ready) - return self.deferred_indexes - - def _wait_for_indexes(self): - """ - Make the marked methods to wait for the indexes to be ready. - Heavily based on - http://blogs.fluidinfo.com/terry/2009/05/11/a-mixin-class-allowing-python-__init__-methods-to-work-with-twisted-deferreds/ - - :param methods: methods that need to wait for the indexes to be ready - :type methods: tuple(str) - """ - leap_assert_type(self.wait_for_indexes, list) - methods = self.wait_for_indexes - - self.waiting = [] - self.stored = {} - - def makeWrapper(method): - def wrapper(*args, **kw): - d = defer.Deferred() - d.addCallback(lambda _: self.stored[method](*args, **kw)) - self.waiting.append(d) - return d - return wrapper - - for method in methods: - self.stored[method] = getattr(self, method) - setattr(self, method, makeWrapper(method)) - - def _restore_waiting_methods(self, _): - for method in self.stored: - setattr(self, method, self.stored[method]) - for d in self.waiting: - d.callback(None) - - -class SoledadMailAdaptor(SoledadIndexMixin): - - implements(IMailAdaptor) - store = None - - indexes = indexes.MAIL_INDEXES - wait_for_indexes = ['get_or_create_mbox', 'update_mbox', 'get_all_mboxes'] - - mboxwrapper_klass = MailboxWrapper - - def __init__(self): - SoledadIndexMixin.__init__(self) - - # Message handling - - def get_msg_from_string(self, MessageClass, raw_msg): - """ - Get an instance of a MessageClass initialized with a MessageWrapper - that contains all the parts obtained from parsing the raw string for - the message. - - :param MessageClass: any Message class that can be initialized passing - an instance of an IMessageWrapper implementor. - :type MessageClass: type - :param raw_msg: a string containing the raw email message. - :type raw_msg: str - :rtype: MessageClass instance. - """ - assert(MessageClass is not None) - mdoc, fdoc, hdoc, cdocs = _split_into_parts(raw_msg) - return self.get_msg_from_docs( - MessageClass, mdoc, fdoc, hdoc, cdocs) - - def get_msg_from_docs(self, MessageClass, mdoc, fdoc, hdoc, cdocs=None, - uid=None): - """ - Get an instance of a MessageClass initialized with a MessageWrapper - that contains the passed part documents. - - This is not the recommended way of obtaining a message, unless you know - how to take care of ensuring the internal consistency between the part - documents, or unless you are glueing together the part documents that - have been previously generated by `get_msg_from_string`. - - :param MessageClass: any Message class that can be initialized passing - an instance of an IMessageWrapper implementor. - :type MessageClass: type - :param fdoc: a dictionary containing values from which a - FlagsDocWrapper can be initialized - :type fdoc: dict - :param hdoc: a dictionary containing values from which a - HeaderDocWrapper can be initialized - :type hdoc: dict - :param cdocs: None, or a dictionary mapping integers (1-indexed) to - dicts from where a ContentDocWrapper can be initialized. - :type cdocs: dict, or None - - :rtype: MessageClass instance. - """ - assert(MessageClass is not None) - return MessageClass(MessageWrapper(mdoc, fdoc, hdoc, cdocs), uid=uid) - - def get_msg_from_mdoc_id(self, MessageClass, store, mdoc_id, - uid=None, get_cdocs=False): - - def wrap_meta_doc(doc): - cls = MetaMsgDocWrapper - return cls(doc_id=doc.doc_id, **doc.content) - - def get_part_docs_from_mdoc_wrapper(wrapper): - d_docs = [] - d_docs.append(store.get_doc(wrapper.fdoc)) - d_docs.append(store.get_doc(wrapper.hdoc)) - for cdoc in wrapper.cdocs: - d_docs.append(store.get_doc(cdoc)) - - def add_mdoc(doc_list): - return [wrapper.serialize()] + doc_list - - d = defer.gatherResults(d_docs) - d.addCallback(add_mdoc) - return d - - def get_parts_doc_from_mdoc_id(): - mbox = re.findall(constants.METAMSGID_MBOX_RE, mdoc_id)[0] - chash = re.findall(constants.METAMSGID_CHASH_RE, mdoc_id)[0] - - def _get_fdoc_id_from_mdoc_id(): - return constants.FDOCID.format(mbox_uuid=mbox, chash=chash) - - def _get_hdoc_id_from_mdoc_id(): - return constants.HDOCID.format(mbox_uuid=mbox, chash=chash) - - d_docs = [] - fdoc_id = _get_fdoc_id_from_mdoc_id() - hdoc_id = _get_hdoc_id_from_mdoc_id() - - d_docs.append(store.get_doc(mdoc_id)) - d_docs.append(store.get_doc(fdoc_id)) - d_docs.append(store.get_doc(hdoc_id)) - - d = defer.gatherResults(d_docs) - return d - - def _err_log_failure_part_docs(failure): - # See https://leap.se/code/issues/7495. - # This avoids blocks, but the real cause still needs to be - # isolated (0.9.0rc3) -- kali - log.msg("BUG ---------------------------------------------------") - log.msg("BUG: Error while retrieving part docs for mdoc id %s" % - mdoc_id) - log.err(failure) - log.msg("BUG (please report above info) ------------------------") - return [] - - def _err_log_cannot_find_msg(failure): - log.msg("BUG: Error while getting msg (uid=%s)" % uid) - return None - - if get_cdocs: - d = store.get_doc(mdoc_id) - d.addCallback(wrap_meta_doc) - d.addCallback(get_part_docs_from_mdoc_wrapper) - d.addErrback(_err_log_failure_part_docs) - - else: - d = get_parts_doc_from_mdoc_id() - - d.addCallback(self._get_msg_from_variable_doc_list, - msg_class=MessageClass, uid=uid) - d.addErrback(_err_log_cannot_find_msg) - return d - - def _get_msg_from_variable_doc_list(self, doc_list, msg_class, uid=None): - if len(doc_list) == 3: - mdoc, fdoc, hdoc = doc_list - cdocs = None - elif len(doc_list) > 3: - # XXX is this case used? - mdoc, fdoc, hdoc = doc_list[:3] - cdocs = dict(enumerate(doc_list[3:], 1)) - return self.get_msg_from_docs( - msg_class, mdoc, fdoc, hdoc, cdocs, uid=uid) - - def get_flags_from_mdoc_id(self, store, mdoc_id): - """ - # XXX stuff here... - """ - mbox = re.findall(constants.METAMSGID_MBOX_RE, mdoc_id)[0] - chash = re.findall(constants.METAMSGID_CHASH_RE, mdoc_id)[0] - - def _get_fdoc_id_from_mdoc_id(): - return constants.FDOCID.format(mbox_uuid=mbox, chash=chash) - - fdoc_id = _get_fdoc_id_from_mdoc_id() - - def wrap_fdoc(doc): - if not doc: - return - cls = FlagsDocWrapper - return cls(doc_id=doc.doc_id, **doc.content) - - def get_flags(fdoc_wrapper): - if not fdoc_wrapper: - return [] - return fdoc_wrapper.get_flags() - - d = store.get_doc(fdoc_id) - d.addCallback(wrap_fdoc) - d.addCallback(get_flags) - return d - - def create_msg(self, store, msg): - """ - :param store: an instance of soledad, or anything that behaves alike - :param msg: a Message object. - - :return: a Deferred that is fired when all the underlying documents - have been created. - :rtype: defer.Deferred - """ - wrapper = msg.get_wrapper() - return wrapper.create(store) - - def update_msg(self, store, msg): - """ - :param msg: a Message object. - :param store: an instance of soledad, or anything that behaves alike - :return: a Deferred that is fired when all the underlying documents - have been updated (actually, it's only the fdoc that's allowed - to update). - :rtype: defer.Deferred - """ - wrapper = msg.get_wrapper() - return wrapper.update(store) - - # batch deletion - - def del_all_flagged_messages(self, store, mbox_uuid): - """ - Delete all messages flagged as deleted. - """ - def err(failure): - log.err(failure) - - def delete_fdoc_and_mdoc_flagged(fdocs): - # low level here, not using the wrappers... - # get meta doc ids from the flag doc ids - fdoc_ids = [doc.doc_id for doc in fdocs] - mdoc_ids = map(lambda s: "M" + s[1:], fdoc_ids) - - def delete_all_docs(mdocs, fdocs): - mdocs = list(mdocs) - doc_ids = [m.doc_id for m in mdocs] - _d = [] - docs = mdocs + fdocs - for doc in docs: - _d.append(store.delete_doc(doc)) - d = defer.gatherResults(_d) - # return the mdocs ids only - d.addCallback(lambda _: doc_ids) - return d - - d = store.get_docs(mdoc_ids) - d.addCallback(delete_all_docs, fdocs) - d.addErrback(err) - return d - - type_ = FlagsDocWrapper.model.type_ - uuid = mbox_uuid.replace('-', '_') - deleted_index = indexes.TYPE_MBOX_DEL_IDX - - d = store.get_from_index(deleted_index, type_, uuid, "1") - d.addCallbacks(delete_fdoc_and_mdoc_flagged, err) - return d - - # count messages - - def get_count_unseen(self, store, mbox_uuid): - """ - Get the number of unseen messages for a given mailbox. - - :param store: instance of Soledad. - :param mbox_uuid: the uuid for this mailbox. - :rtype: int - """ - type_ = FlagsDocWrapper.model.type_ - uuid = mbox_uuid.replace('-', '_') - - unseen_index = indexes.TYPE_MBOX_SEEN_IDX - - d = store.get_count_from_index(unseen_index, type_, uuid, "0") - d.addErrback(self._errback) - return d - - def get_count_recent(self, store, mbox_uuid): - """ - Get the number of recent messages for a given mailbox. - - :param store: instance of Soledad. - :param mbox_uuid: the uuid for this mailbox. - :rtype: int - """ - type_ = FlagsDocWrapper.model.type_ - uuid = mbox_uuid.replace('-', '_') - - recent_index = indexes.TYPE_MBOX_RECENT_IDX - - d = store.get_count_from_index(recent_index, type_, uuid, "1") - d.addErrback(self._errback) - return d - - # search api - - def get_mdoc_id_from_msgid(self, store, mbox_uuid, msgid): - """ - Get the UID for a message with the passed msgid (the one in the headers - msg-id). - This is used by the MUA to retrieve the recently saved draft. - """ - type_ = HeaderDocWrapper.model.type_ - uuid = mbox_uuid.replace('-', '_') - - msgid_index = indexes.TYPE_MSGID_IDX - - def get_mdoc_id(hdoc): - if not hdoc: - log.msg("Could not find a HDOC with MSGID %s" % msgid) - return None - hdoc = hdoc[0] - mdoc_id = hdoc.doc_id.replace("H-", "M-%s-" % uuid) - return mdoc_id - - d = store.get_from_index(msgid_index, type_, msgid) - d.addCallback(get_mdoc_id) - return d - - # Mailbox handling - - def get_or_create_mbox(self, store, name): - """ - Get the mailbox with the given name, or create one if it does not - exist. - - :param store: instance of Soledad - :param name: the name of the mailbox - :type name: str - """ - index = indexes.TYPE_MBOX_IDX - mbox = normalize_mailbox(name) - return MailboxWrapper.get_or_create(store, index, mbox) - - def update_mbox(self, store, mbox_wrapper): - """ - Update the documents for a given mailbox. - :param mbox_wrapper: MailboxWrapper instance - :type mbox_wrapper: MailboxWrapper - :return: a Deferred that will be fired when the mailbox documents - have been updated. - :rtype: defer.Deferred - """ - leap_assert_type(mbox_wrapper, SoledadDocumentWrapper) - return mbox_wrapper.update(store) - - def delete_mbox(self, store, mbox_wrapper): - leap_assert_type(mbox_wrapper, SoledadDocumentWrapper) - return mbox_wrapper.delete(store) - - def get_all_mboxes(self, store): - """ - Retrieve a list with wrappers for all the mailboxes. - - :return: a deferred that will be fired with a list of all the - MailboxWrappers found. - :rtype: defer.Deferred - """ - return MailboxWrapper.get_all(store) - - def _errback(self, failure): - log.err(failure) - - -def _split_into_parts(raw): - # TODO signal that we can delete the original message!----- - # when all the processing is done. - # TODO add the linked-from info ! - # TODO add reference to the original message? - # TODO populate Default FLAGS/TAGS (unseen?) - # TODO seed propely the content_docs with defaults?? - - msg, chash, multi = _parse_msg(raw) - size = len(msg.as_string()) - - parts_map = walk.get_tree(msg) - cdocs_list = list(walk.get_raw_docs(msg)) - cdocs_phashes = [c['phash'] for c in cdocs_list] - body_phash = walk.get_body_phash(msg) - - mdoc = _build_meta_doc(chash, cdocs_phashes) - fdoc = _build_flags_doc(chash, size, multi) - hdoc = _build_headers_doc(msg, chash, body_phash, parts_map) - - # The MessageWrapper expects a dict, one-indexed - cdocs = dict(enumerate(cdocs_list, 1)) - - return mdoc, fdoc, hdoc, cdocs - - -def _parse_msg(raw): - msg = message_from_string(raw) - chash = walk.get_hash(raw) - multi = msg.is_multipart() - return msg, chash, multi - - -def _build_meta_doc(chash, cdocs_phashes): - _mdoc = MetaMsgDocWrapper() - # FIXME passing the inbox name because we don't have the uuid at this - # point. - - _mdoc.fdoc = constants.FDOCID.format(mbox_uuid=INBOX_NAME, chash=chash) - _mdoc.hdoc = constants.HDOCID.format(chash=chash) - _mdoc.cdocs = [constants.CDOCID.format(phash=p) for p in cdocs_phashes] - - return _mdoc.serialize() - - -def _build_flags_doc(chash, size, multi): - _fdoc = FlagsDocWrapper(chash=chash, size=size, multi=multi) - return _fdoc.serialize() - - -def _build_headers_doc(msg, chash, body_phash, parts_map): - """ - Assemble a headers document from the original parsed message, the - content-hash, and the parts map. - - It takes into account possibly repeated headers. - """ - headers = defaultdict(list) - for k, v in msg.items(): - headers[k].append(v) - # "fix" for repeated headers (as in "Received:" - for k, v in headers.items(): - newline = "\n%s: " % (k.lower(),) - headers[k] = newline.join(v) - - lower_headers = lowerdict(dict(headers)) - msgid = first(_MSGID_RE.findall( - lower_headers.get('message-id', ''))) - - _hdoc = HeaderDocWrapper( - chash=chash, headers=headers, body=body_phash, - msgid=msgid) - - def copy_attr(headers, key, doc): - if key in headers: - setattr(doc, key, headers[key]) - - copy_attr(lower_headers, "subject", _hdoc) - copy_attr(lower_headers, "date", _hdoc) - - hdoc = _hdoc.serialize() - # add some of the attr from the parts map to header doc - for key in parts_map: - if key in ('body', 'multi', 'part_map'): - hdoc[key] = parts_map[key] - return stringify_parts_map(hdoc) diff --git a/mail/src/leap/mail/adaptors/soledad_indexes.py b/mail/src/leap/mail/adaptors/soledad_indexes.py deleted file mode 100644 index eec7d28..0000000 --- a/mail/src/leap/mail/adaptors/soledad_indexes.py +++ /dev/null @@ -1,106 +0,0 @@ -# -*- coding: utf-8 -*- -# soledad_indexes.py -# Copyright (C) 2013, 2014 LEAP -# -# This program is free software: you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation, either version 3 of the License, or -# (at your option) any later version. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with this program. If not, see . -""" -Soledad Indexes for Mail Documents. -""" - -# TODO -# [ ] hide most of the constants here - -# Document Type, for indexing - -TYPE = "type" -MBOX = "mbox" -MBOX_UUID = "mbox_uuid" -FLAGS = "flags" -HEADERS = "head" -CONTENT = "cnt" -RECENT = "rct" -HDOCS_SET = "hdocset" - -INCOMING_KEY = "incoming" -ERROR_DECRYPTING_KEY = "errdecr" - -# indexing keys -CONTENT_HASH = "chash" -PAYLOAD_HASH = "phash" -MSGID = "msgid" -UID = "uid" - - -# Index types -# -------------- - -TYPE_IDX = 'by-type' -TYPE_MBOX_IDX = 'by-type-and-mbox' -TYPE_MBOX_UUID_IDX = 'by-type-and-mbox-uuid' -TYPE_SUBS_IDX = 'by-type-and-subscribed' -TYPE_MSGID_IDX = 'by-type-and-message-id' -TYPE_MBOX_SEEN_IDX = 'by-type-and-mbox-and-seen' -TYPE_MBOX_RECENT_IDX = 'by-type-and-mbox-and-recent' -TYPE_MBOX_DEL_IDX = 'by-type-and-mbox-and-deleted' -TYPE_MBOX_C_HASH_IDX = 'by-type-and-mbox-and-contenthash' -TYPE_C_HASH_IDX = 'by-type-and-contenthash' -TYPE_C_HASH_PART_IDX = 'by-type-and-contenthash-and-partnumber' -TYPE_P_HASH_IDX = 'by-type-and-payloadhash' - -# Soledad index for incoming mail, without decrypting errors. -# and the backward-compatible index, will be deprecated at 0.7 -JUST_MAIL_IDX = "just-mail" -JUST_MAIL_COMPAT_IDX = "just-mail-compat" - - -# TODO -# it would be nice to measure the cost of indexing -# by many fields. - -# TODO -# make the indexes dict more readable! - -MAIL_INDEXES = { - # generic - TYPE_IDX: [TYPE], - TYPE_MBOX_IDX: [TYPE, MBOX], - TYPE_MBOX_UUID_IDX: [TYPE, MBOX_UUID], - - # XXX deprecate 0.4.0 - # TYPE_MBOX_UID_IDX: [TYPE, MBOX, UID], - - # mailboxes - TYPE_SUBS_IDX: [TYPE, 'bool(subscribed)'], - - # fdocs uniqueness - TYPE_MBOX_C_HASH_IDX: [TYPE, MBOX, CONTENT_HASH], - - # headers doc - search by msgid. - TYPE_MSGID_IDX: [TYPE, MSGID], - - # content, headers doc - TYPE_C_HASH_IDX: [TYPE, CONTENT_HASH], - - # attachment payload dedup - TYPE_P_HASH_IDX: [TYPE, PAYLOAD_HASH], - - # messages - TYPE_MBOX_SEEN_IDX: [TYPE, MBOX_UUID, 'bool(seen)'], - TYPE_MBOX_RECENT_IDX: [TYPE, MBOX_UUID, 'bool(recent)'], - TYPE_MBOX_DEL_IDX: [TYPE, MBOX_UUID, 'bool(deleted)'], - - # incoming queue - JUST_MAIL_IDX: ["bool(%s)" % (INCOMING_KEY,), - "bool(%s)" % (ERROR_DECRYPTING_KEY,)], -} diff --git a/mail/src/leap/mail/adaptors/tests/rfc822.message b/mail/src/leap/mail/adaptors/tests/rfc822.message deleted file mode 120000 index b19cc28..0000000 --- a/mail/src/leap/mail/adaptors/tests/rfc822.message +++ /dev/null @@ -1 +0,0 @@ -../../tests/rfc822.message \ No newline at end of file diff --git a/mail/src/leap/mail/adaptors/tests/test_models.py b/mail/src/leap/mail/adaptors/tests/test_models.py deleted file mode 100644 index b82cfad..0000000 --- a/mail/src/leap/mail/adaptors/tests/test_models.py +++ /dev/null @@ -1,106 +0,0 @@ -# -*- coding: utf-8 -*- -# test_models.py -# Copyright (C) 2014 LEAP -# -# This program is free software: you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation, either version 3 of the License, or -# (at your option) any later version. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with this program. If not, see . -""" -Tests for the leap.mail.adaptors.models module. -""" -from twisted.trial import unittest - -from leap.mail.adaptors import models - - -class SerializableModelsTestCase(unittest.TestCase): - - def test_good_serialized_model(self): - - class M(models.SerializableModel): - foo = 42 - bar = 33 - baaz_ = None - _nope = 0 - __nope = 0 - - def not_today(self): - pass - - class IgnoreMe(object): - pass - - def killmeplease(x): - return x - - serialized = M.serialize() - expected = {'foo': 42, 'bar': 33, 'baaz': None} - self.assertEqual(serialized, expected) - - -class DocumentWrapperTestCase(unittest.TestCase): - - def test_wrapper_defaults(self): - - class Wrapper(models.DocumentWrapper): - class model(models.SerializableModel): - foo = 42 - bar = 11 - - wrapper = Wrapper() - wrapper._ignored = True - serialized = wrapper.serialize() - expected = {'foo': 42, 'bar': 11} - self.assertEqual(serialized, expected) - - def test_initialized_wrapper(self): - - class Wrapper(models.DocumentWrapper): - class model(models.SerializableModel): - foo = 42 - bar_ = 11 - - wrapper = Wrapper(foo=0, bar=-1) - serialized = wrapper.serialize() - expected = {'foo': 0, 'bar': -1} - self.assertEqual(serialized, expected) - - wrapper.foo = 23 - serialized = wrapper.serialize() - expected = {'foo': 23, 'bar': -1} - self.assertEqual(serialized, expected) - - wrapper = Wrapper(foo=0) - serialized = wrapper.serialize() - expected = {'foo': 0, 'bar': 11} - self.assertEqual(serialized, expected) - - def test_invalid_initialized_wrapper(self): - - class Wrapper(models.DocumentWrapper): - class model(models.SerializableModel): - foo = 42 - - def getwrapper(): - return Wrapper(bar=1) - self.assertRaises(RuntimeError, getwrapper) - - def test_no_model_wrapper(self): - - class Wrapper(models.DocumentWrapper): - pass - - def getwrapper(): - w = Wrapper() - w.foo = None - - self.assertRaises(RuntimeError, getwrapper) diff --git a/mail/src/leap/mail/adaptors/tests/test_soledad_adaptor.py b/mail/src/leap/mail/adaptors/tests/test_soledad_adaptor.py deleted file mode 100644 index 73eaf16..0000000 --- a/mail/src/leap/mail/adaptors/tests/test_soledad_adaptor.py +++ /dev/null @@ -1,529 +0,0 @@ -# -*- coding: utf-8 -*- -# test_soledad_adaptor.py -# Copyright (C) 2014 LEAP -# -# This program is free software: you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation, either version 3 of the License, or -# (at your option) any later version. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with this program. If not, see . -""" -Tests for the Soledad Adaptor module - leap.mail.adaptors.soledad -""" -import os -from functools import partial - -from twisted.internet import defer - -from leap.mail.adaptors import models -from leap.mail.adaptors.soledad import SoledadDocumentWrapper -from leap.mail.adaptors.soledad import SoledadIndexMixin -from leap.mail.adaptors.soledad import SoledadMailAdaptor -from leap.mail.testing.common import SoledadTestMixin - -from email.MIMEMultipart import MIMEMultipart -from email.mime.text import MIMEText - -# DEBUG -# import logging -# logging.basicConfig(level=logging.DEBUG) - - -class CounterWrapper(SoledadDocumentWrapper): - class model(models.SerializableModel): - counter = 0 - flag = None - - -class CharacterWrapper(SoledadDocumentWrapper): - class model(models.SerializableModel): - name = "" - age = 20 - - -class ActorWrapper(SoledadDocumentWrapper): - class model(models.SerializableModel): - type_ = "actor" - name = None - - class __meta__(object): - index = "name" - list_index = ("by-type", "type_") - - -class TestAdaptor(SoledadIndexMixin): - indexes = {'by-name': ['name'], - 'by-type-and-name': ['type', 'name'], - 'by-type': ['type']} - - -class SoledadDocWrapperTestCase(SoledadTestMixin): - """ - Tests for the SoledadDocumentWrapper. - """ - def assert_num_docs(self, num, docs): - self.assertEqual(len(docs[1]), num) - - def test_create_single(self): - - store = self._soledad - wrapper = CounterWrapper() - - def assert_one_doc(docs): - self.assertEqual(docs[0], 1) - - d = wrapper.create(store) - d.addCallback(lambda _: store.get_all_docs()) - d.addCallback(assert_one_doc) - return d - - def test_create_many(self): - - store = self._soledad - w1 = CounterWrapper() - w2 = CounterWrapper(counter=1) - w3 = CounterWrapper(counter=2) - w4 = CounterWrapper(counter=3) - w5 = CounterWrapper(counter=4) - - d1 = [w1.create(store), - w2.create(store), - w3.create(store), - w4.create(store), - w5.create(store)] - - d = defer.gatherResults(d1) - d.addCallback(lambda _: store.get_all_docs()) - d.addCallback(partial(self.assert_num_docs, 5)) - return d - - def test_multiple_updates(self): - - store = self._soledad - wrapper = CounterWrapper(counter=1) - MAX = 100 - - def assert_doc_id(doc): - self.assertTrue(wrapper._doc_id is not None) - return doc - - def assert_counter_initial_ok(doc): - self.assertEqual(wrapper.counter, 1) - - def increment_counter(ignored): - d1 = [] - - def record_revision(revision): - rev = int(revision.split(':')[1]) - self.results.append(rev) - - for i in list(range(MAX)): - wrapper.counter += 1 - wrapper.flag = i % 2 == 0 - d = wrapper.update(store) - d.addCallback(record_revision) - d1.append(d) - - return defer.gatherResults(d1) - - def assert_counter_final_ok(doc): - self.assertEqual(doc.content['counter'], MAX + 1) - self.assertEqual(doc.content['flag'], False) - - def assert_results_ordered_list(ignored): - self.assertEqual(self.results, sorted(range(2, MAX + 2))) - - d = wrapper.create(store) - d.addCallback(assert_doc_id) - d.addCallback(assert_counter_initial_ok) - d.addCallback(increment_counter) - d.addCallback(lambda _: store.get_doc(wrapper._doc_id)) - d.addCallback(assert_counter_final_ok) - d.addCallback(assert_results_ordered_list) - return d - - def test_delete(self): - adaptor = TestAdaptor() - store = self._soledad - - wrapper_list = [] - - def get_or_create_bob(ignored): - def add_to_list(wrapper): - wrapper_list.append(wrapper) - return wrapper - wrapper = CharacterWrapper.get_or_create( - store, 'by-name', 'bob') - wrapper.addCallback(add_to_list) - return wrapper - - def delete_bob(ignored): - wrapper = wrapper_list[0] - return wrapper.delete(store) - - d = adaptor.initialize_store(store) - d.addCallback(lambda _: store.get_all_docs()) - d.addCallback(partial(self.assert_num_docs, 0)) - - # this should create bob document - d.addCallback(get_or_create_bob) - d.addCallback(lambda _: store.get_all_docs()) - d.addCallback(partial(self.assert_num_docs, 1)) - - d.addCallback(delete_bob) - d.addCallback(lambda _: store.get_all_docs()) - d.addCallback(partial(self.assert_num_docs, 0)) - return d - - def test_get_or_create(self): - adaptor = TestAdaptor() - store = self._soledad - - def get_or_create_bob(ignored): - wrapper = CharacterWrapper.get_or_create( - store, 'by-name', 'bob') - return wrapper - - d = adaptor.initialize_store(store) - d.addCallback(lambda _: store.get_all_docs()) - d.addCallback(partial(self.assert_num_docs, 0)) - - # this should create bob document - d.addCallback(get_or_create_bob) - d.addCallback(lambda _: store.get_all_docs()) - d.addCallback(partial(self.assert_num_docs, 1)) - - # this should get us bob document - d.addCallback(get_or_create_bob) - d.addCallback(lambda _: store.get_all_docs()) - d.addCallback(partial(self.assert_num_docs, 1)) - return d - - def test_get_or_create_multi_index(self): - adaptor = TestAdaptor() - store = self._soledad - - def get_or_create_actor_harry(ignored): - wrapper = ActorWrapper.get_or_create( - store, 'by-type-and-name', 'harrison') - return wrapper - - def create_director_harry(ignored): - wrapper = ActorWrapper(name="harrison", type="director") - return wrapper.create(store) - - d = adaptor.initialize_store(store) - d.addCallback(lambda _: store.get_all_docs()) - d.addCallback(partial(self.assert_num_docs, 0)) - - # this should create harrison document - d.addCallback(get_or_create_actor_harry) - d.addCallback(lambda _: store.get_all_docs()) - d.addCallback(partial(self.assert_num_docs, 1)) - - # this should get us harrison document - d.addCallback(get_or_create_actor_harry) - d.addCallback(lambda _: store.get_all_docs()) - d.addCallback(partial(self.assert_num_docs, 1)) - - # create director harry, should create new doc - d.addCallback(create_director_harry) - d.addCallback(lambda _: store.get_all_docs()) - d.addCallback(partial(self.assert_num_docs, 2)) - - # this should get us harrison document, still 2 docs - d.addCallback(get_or_create_actor_harry) - d.addCallback(lambda _: store.get_all_docs()) - d.addCallback(partial(self.assert_num_docs, 2)) - return d - - def test_get_all(self): - adaptor = TestAdaptor() - store = self._soledad - actor_names = ["harry", "carrie", "mark", "david"] - - def create_some_actors(ignored): - deferreds = [] - for name in actor_names: - dw = ActorWrapper.get_or_create( - store, 'by-type-and-name', name) - deferreds.append(dw) - return defer.gatherResults(deferreds) - - d = adaptor.initialize_store(store) - d.addCallback(lambda _: store.get_all_docs()) - d.addCallback(partial(self.assert_num_docs, 0)) - - d.addCallback(create_some_actors) - - d.addCallback(lambda _: store.get_all_docs()) - d.addCallback(partial(self.assert_num_docs, 4)) - - def assert_actor_list_is_expected(res): - got = set([actor.name for actor in res]) - expected = set(actor_names) - self.assertEqual(got, expected) - - d.addCallback(lambda _: ActorWrapper.get_all(store)) - d.addCallback(assert_actor_list_is_expected) - return d - -HERE = os.path.split(os.path.abspath(__file__))[0] - - -class MessageClass(object): - def __init__(self, wrapper, uid): - self.wrapper = wrapper - self.uid = uid - - def get_wrapper(self): - return self.wrapper - - -class SoledadMailAdaptorTestCase(SoledadTestMixin): - """ - Tests for the SoledadMailAdaptor. - """ - - def get_adaptor(self): - adaptor = SoledadMailAdaptor() - adaptor.store = self._soledad - return adaptor - - def assert_num_docs(self, num, docs): - self.assertEqual(len(docs[1]), num) - - def test_mail_adaptor_init(self): - adaptor = self.get_adaptor() - self.assertTrue(isinstance(adaptor.indexes, dict)) - self.assertTrue(len(adaptor.indexes) != 0) - - # Messages - - def test_get_msg_from_string(self): - adaptor = self.get_adaptor() - - with open(os.path.join(HERE, "rfc822.message")) as f: - raw = f.read() - - msg = adaptor.get_msg_from_string(MessageClass, raw) - - chash = ("D27B2771C0DCCDCB468EE65A4540438" - "09DBD11588E87E951545BE0CBC321C308") - phash = ("64934534C1C80E0D4FA04BE1CCBA104" - "F07BCA5F469C86E2C0ABE1D41310B7299") - subject = ("[Twisted-commits] rebuild now works on " - "python versions from 2.2.0 and up.") - self.assertTrue(msg.wrapper.fdoc is not None) - self.assertTrue(msg.wrapper.hdoc is not None) - self.assertTrue(msg.wrapper.cdocs is not None) - self.assertEquals(len(msg.wrapper.cdocs), 1) - self.assertEquals(msg.wrapper.fdoc.chash, chash) - self.assertEquals(msg.wrapper.fdoc.size, 3837) - self.assertEquals(msg.wrapper.hdoc.chash, chash) - self.assertEqual(dict(msg.wrapper.hdoc.headers)['Subject'], - subject) - self.assertEqual(msg.wrapper.hdoc.subject, subject) - self.assertEqual(msg.wrapper.cdocs[1].phash, phash) - - def test_get_msg_from_string_multipart(self): - msg = MIMEMultipart() - msg['Subject'] = 'Test multipart mail' - msg.attach(MIMEText(u'a utf8 message', _charset='utf-8')) - adaptor = self.get_adaptor() - - msg = adaptor.get_msg_from_string(MessageClass, msg.as_string()) - - self.assertEqual( - 'base64', msg.wrapper.cdocs[1].content_transfer_encoding) - self.assertEqual( - 'text/plain', msg.wrapper.cdocs[1].content_type) - self.assertEqual( - 'YSB1dGY4IG1lc3NhZ2U=\n', msg.wrapper.cdocs[1].raw) - - def test_get_msg_from_docs(self): - adaptor = self.get_adaptor() - mdoc = dict( - fdoc="F-Foobox-deadbeef", - hdoc="H-deadbeef", - cdocs=["C-deadabad"]) - fdoc = dict( - mbox_uuid="Foobox", - flags=('\Seen', '\Nice'), - tags=('Personal', 'TODO'), - seen=False, deleted=False, - recent=False, multi=False) - hdoc = dict( - chash="deadbeef", - subject="Test Msg") - cdocs = { - 1: dict( - raw='This is a test message')} - - msg = adaptor.get_msg_from_docs( - MessageClass, mdoc, fdoc, hdoc, cdocs=cdocs) - self.assertEqual(msg.wrapper.fdoc.flags, - ('\Seen', '\Nice')) - self.assertEqual(msg.wrapper.fdoc.tags, - ('Personal', 'TODO')) - self.assertEqual(msg.wrapper.fdoc.mbox_uuid, "Foobox") - self.assertEqual(msg.wrapper.hdoc.multi, False) - self.assertEqual(msg.wrapper.hdoc.subject, - "Test Msg") - self.assertEqual(msg.wrapper.cdocs[1].raw, - "This is a test message") - - def test_get_msg_from_metamsg_doc_id(self): - # TODO complete-me! - pass - - test_get_msg_from_metamsg_doc_id.skip = "Not yet implemented" - - def test_create_msg(self): - adaptor = self.get_adaptor() - - with open(os.path.join(HERE, "rfc822.message")) as f: - raw = f.read() - msg = adaptor.get_msg_from_string(MessageClass, raw) - - def check_create_result(created): - # that's one mdoc, one hdoc, one fdoc, one cdoc - self.assertEqual(len(created), 4) - for doc in created: - self.assertTrue( - doc.__class__.__name__, - "SoledadDocument") - - d = adaptor.create_msg(adaptor.store, msg) - d.addCallback(check_create_result) - return d - - def test_update_msg(self): - adaptor = self.get_adaptor() - with open(os.path.join(HERE, "rfc822.message")) as f: - raw = f.read() - - def assert_msg_has_doc_id(ignored, msg): - wrapper = msg.get_wrapper() - self.assertTrue(wrapper.fdoc.doc_id is not None) - - def assert_msg_has_no_flags(ignored, msg): - wrapper = msg.get_wrapper() - self.assertEqual(wrapper.fdoc.flags, []) - - def update_msg_flags(ignored, msg): - wrapper = msg.get_wrapper() - wrapper.fdoc.flags = ["This", "That"] - return wrapper.update(adaptor.store) - - def assert_msg_has_flags(ignored, msg): - wrapper = msg.get_wrapper() - self.assertEqual(wrapper.fdoc.flags, ["This", "That"]) - - def get_fdoc_and_check_flags(ignored): - def assert_doc_has_flags(doc): - self.assertEqual(doc.content['flags'], - ['This', 'That']) - wrapper = msg.get_wrapper() - d = adaptor.store.get_doc(wrapper.fdoc.doc_id) - d.addCallback(assert_doc_has_flags) - return d - - msg = adaptor.get_msg_from_string(MessageClass, raw) - d = adaptor.create_msg(adaptor.store, msg) - d.addCallback(lambda _: adaptor.store.get_all_docs()) - d.addCallback(partial(self.assert_num_docs, 4)) - d.addCallback(assert_msg_has_doc_id, msg) - d.addCallback(assert_msg_has_no_flags, msg) - - # update it! - d.addCallback(update_msg_flags, msg) - d.addCallback(assert_msg_has_flags, msg) - d.addCallback(get_fdoc_and_check_flags) - return d - - # Mailboxes - - def test_get_or_create_mbox(self): - adaptor = self.get_adaptor() - - def get_or_create_mbox(ignored): - d = adaptor.get_or_create_mbox(adaptor.store, "Trash") - return d - - def assert_good_doc(mbox_wrapper): - self.assertTrue(mbox_wrapper.doc_id is not None) - self.assertEqual(mbox_wrapper.mbox, "Trash") - self.assertEqual(mbox_wrapper.type, "mbox") - self.assertEqual(mbox_wrapper.closed, False) - self.assertEqual(mbox_wrapper.subscribed, False) - - d = adaptor.initialize_store(adaptor.store) - d.addCallback(get_or_create_mbox) - d.addCallback(assert_good_doc) - d.addCallback(lambda _: adaptor.store.get_all_docs()) - d.addCallback(partial(self.assert_num_docs, 1)) - return d - - def test_update_mbox(self): - adaptor = self.get_adaptor() - - wrapper_ref = [] - - def get_or_create_mbox(ignored): - d = adaptor.get_or_create_mbox(adaptor.store, "Trash") - return d - - def update_wrapper(wrapper, wrapper_ref): - wrapper_ref.append(wrapper) - wrapper.subscribed = True - wrapper.closed = True - d = adaptor.update_mbox(adaptor.store, wrapper) - return d - - def get_mbox_doc_and_check_flags(res, wrapper_ref): - wrapper = wrapper_ref[0] - - def assert_doc_has_flags(doc): - self.assertEqual(doc.content['subscribed'], True) - self.assertEqual(doc.content['closed'], True) - d = adaptor.store.get_doc(wrapper.doc_id) - d.addCallback(assert_doc_has_flags) - return d - - d = adaptor.initialize_store(adaptor.store) - d.addCallback(get_or_create_mbox) - d.addCallback(update_wrapper, wrapper_ref) - d.addCallback(get_mbox_doc_and_check_flags, wrapper_ref) - return d - - def test_get_all_mboxes(self): - adaptor = self.get_adaptor() - mboxes = ("Sent", "Trash", "Personal", "ListFoo") - - def get_or_create_mboxes(ignored): - d = [] - for mbox in mboxes: - d.append(adaptor.get_or_create_mbox( - adaptor.store, mbox)) - return defer.gatherResults(d) - - def get_all_mboxes(ignored): - return adaptor.get_all_mboxes(adaptor.store) - - def assert_mboxes_match_expected(wrappers): - names = [m.mbox for m in wrappers] - self.assertEqual(set(names), set(mboxes)) - - d = adaptor.initialize_store(adaptor.store) - d.addCallback(get_or_create_mboxes) - d.addCallback(get_all_mboxes) - d.addCallback(assert_mboxes_match_expected) - return d diff --git a/mail/src/leap/mail/constants.py b/mail/src/leap/mail/constants.py deleted file mode 100644 index 4ef42cb..0000000 --- a/mail/src/leap/mail/constants.py +++ /dev/null @@ -1,52 +0,0 @@ -# *- coding: utf-8 -*- -# constants.py -# Copyright (C) 2014 LEAP -# -# This program is free software: you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation, either version 3 of the License, or -# (at your option) any later version. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with this program. If not, see . -""" -Constants for leap.mail. -""" - -INBOX_NAME = "INBOX" - -# Regular expressions for the identifiers to be used in the Message Data Layer. - -METAMSGID = "M-{mbox_uuid}-{chash}" -METAMSGID_RE = "M\-{mbox_uuid}\-[0-9a-fA-F]+" -METAMSGID_CHASH_RE = "M\-\w+\-([0-9a-fA-F]+)" -METAMSGID_MBOX_RE = "M\-(\w+)\-[0-9a-fA-F]+" - -FDOCID = "F-{mbox_uuid}-{chash}" -FDOCID_RE = "F\-{mbox_uuid}\-[0-9a-fA-F]+" -FDOCID_CHASH_RE = "F\-\w+\-([0-9a-fA-F]+)" - -HDOCID = "H-{chash}" -HDOCID_RE = "H\-[0-9a-fA-F]+" - -CDOCID = "C-{phash}" -CDOCID_RE = "C\-[0-9a-fA-F]+" - - -class MessageFlags(object): - """ - Flags used in Message and Mailbox. - """ - SEEN_FLAG = "\\Seen" - RECENT_FLAG = "\\Recent" - ANSWERED_FLAG = "\\Answered" - FLAGGED_FLAG = "\\Flagged" # yo dawg - DELETED_FLAG = "\\Deleted" - DRAFT_FLAG = "\\Draft" - NOSELECT_FLAG = "\\Noselect" - LIST_FLAG = "List" # is this OK? (no \. ie, no system flag) diff --git a/mail/src/leap/mail/cred.py b/mail/src/leap/mail/cred.py deleted file mode 100644 index 7eab1f0..0000000 --- a/mail/src/leap/mail/cred.py +++ /dev/null @@ -1,80 +0,0 @@ -# -*- coding: utf-8 -*- -# cred.py -# Copyright (C) 2015 LEAP -# -# This program is free software: you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation, either version 3 of the License, or -# (at your option) any later version. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with this program. If not, see . -""" -Credentials handling. -""" - -from zope.interface import implementer -from twisted.cred.checkers import ICredentialsChecker -from twisted.cred.credentials import IUsernamePassword -from twisted.cred.error import UnauthorizedLogin -from twisted.internet import defer - - -@implementer(ICredentialsChecker) -class LocalSoledadTokenChecker(object): - - """ - A Credentials Checker for a LocalSoledad store. - - It checks that: - - 1) The Local SoledadStorage has been correctly unlocked for the given - user. This currently means that the right passphrase has been passed - to the Local SoledadStorage. - - 2) The password passed in the credentials matches whatever token has - been stored in the local encrypted SoledadStorage, associated to the - Protocol that is requesting the authentication. - """ - - credentialInterfaces = (IUsernamePassword,) - service = None - - def __init__(self, soledad_sessions): - """ - :param soledad_sessions: a dict-like object, containing instances - of a Store (soledad instances), indexed by - userid. - """ - self._soledad_sessions = soledad_sessions - - def requestAvatarId(self, credentials): - if self.service is None: - raise NotImplementedError( - "this checker has not defined its service name") - username, password = credentials.username, credentials.password - d = self.checkSoledadToken(username, password, self.service) - d.addErrback(lambda f: defer.fail(UnauthorizedLogin())) - return d - - def checkSoledadToken(self, username, password, service): - soledad = self._soledad_sessions.get(username) - if not soledad: - return defer.fail(Exception("No soledad")) - - def match_token(token): - if token is None: - raise RuntimeError('no token') - if token == password: - return username - else: - raise RuntimeError('bad token') - - d = soledad.get_or_create_service_token(service) - d.addCallback(match_token) - return d diff --git a/mail/src/leap/mail/decorators.py b/mail/src/leap/mail/decorators.py deleted file mode 100644 index 5105de9..0000000 --- a/mail/src/leap/mail/decorators.py +++ /dev/null @@ -1,149 +0,0 @@ -# -*- coding: utf-8 -*- -# decorators.py -# Copyright (C) 2013 LEAP -# -# This program is free software: you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation, either version 3 of the License, or -# (at your option) any later version. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with this program. If not, see . -""" -Useful decorators for mail package. -""" -import logging -import os - -from functools import wraps - -from twisted.internet.threads import deferToThread - - -logger = logging.getLogger(__name__) - - -# TODO -# Should write a helper to be able to pass a timeout argument. -# See this answer: http://stackoverflow.com/a/19019648/1157664 -# And the notes by glyph and jpcalderone - -def deferred_to_thread(f): - """ - Decorator, for deferring methods to Threads. - - It will do a deferToThread of the decorated method - unless the environment variable LEAPMAIL_DEBUG is set. - - It uses a descriptor to delay the definition of the - method wrapper. - """ - class descript(object): - """ - The class to be used as decorator. - - It takes any method as the passed object. - """ - - def __init__(self, f): - """ - Initializes the decorator object. - - :param f: the decorated function - :type f: callable - """ - self.f = f - - def __get__(self, instance, klass): - """ - Descriptor implementation. - - At creation time, the decorated `method` is unbound. - - It will dispatch the make_unbound method if we still do not - have an instance available, and the make_bound method when the - method has already been bound to the instance. - - :param instance: the instance of the class, or None if not exist. - :type instance: instantiated class or None. - """ - if instance is None: - # Class method was requested - return self.make_unbound(klass) - return self.make_bound(instance) - - def _errback(self, failure): - """ - Errorback that logs the exception catched. - - :param failure: a twisted failure - :type failure: Failure - """ - logger.warning('Error in method: %s' % (self.f.__name__)) - logger.exception(failure.getTraceback()) - - def make_unbound(self, klass): - """ - Return a wrapped function with the unbound call, during the - early access to the decortad method. This gets passed - only the class (not the instance since it does not yet exist). - - :param klass: the class to which the still unbound method belongs - :type klass: type - """ - - @wraps(self.f) - def wrapper(*args, **kwargs): - """ - We're temporarily wrapping the decorated method, but this - should not be called, since our application should use - the bound-wrapped method after this decorator class has been - used. - - This documentation will vanish at runtime. - """ - raise TypeError( - 'unbound method {}() must be called with {} instance ' - 'as first argument (got nothing instead)'.format( - self.f.__name__, - klass.__name__) - ) - return wrapper - - def make_bound(self, instance): - """ - Return a function that wraps the bound method call, - after we are able to access the instance object. - - :param instance: an instance of the class the decorated method, - now bound, belongs to. - :type instance: object - """ - - @wraps(self.f) - def wrapper(*args, **kwargs): - """ - Do a proper function wrapper that defers the decorated method - call to a separated thread if the LEAPMAIL_DEBUG - environment variable is set. - - This documentation will vanish at runtime. - """ - if not os.environ.get('LEAPMAIL_DEBUG'): - d = deferToThread(self.f, instance, *args, **kwargs) - d.addErrback(self._errback) - return d - else: - return self.f(instance, *args, **kwargs) - - # This instance does not need the descriptor anymore, - # let it find the wrapper directly next time: - setattr(instance, self.f.__name__, wrapper) - return wrapper - - return descript(f) diff --git a/mail/src/leap/mail/errors.py b/mail/src/leap/mail/errors.py deleted file mode 100644 index 2f18e87..0000000 --- a/mail/src/leap/mail/errors.py +++ /dev/null @@ -1,27 +0,0 @@ -# -*- coding: utf-8 -*- -# errors.py -# Copyright (C) 2015 LEAP -# -# This program is free software: you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation, either version 3 of the License, or -# (at your option) any later version. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with this program. If not, see . -""" -Exceptions for leap.mail -""" - - -class AuthenticationError(Exception): - pass - - -class ConfigurationError(Exception): - pass diff --git a/mail/src/leap/mail/generator.py b/mail/src/leap/mail/generator.py deleted file mode 100644 index bb3f26e..0000000 --- a/mail/src/leap/mail/generator.py +++ /dev/null @@ -1,23 +0,0 @@ -from email.generator import Generator as EmailGenerator - - -class Generator(EmailGenerator): - """ - Generates output from a Message object tree, keeping signatures. - - This code was extracted from Mailman.Generator.Generator, version 2.1.4: - - Most other Generator will be created not setting the foldheader flag, - as we do not overwrite clone(). The original clone() does not - set foldheaders. - - So you need to set foldheaders if you want the toplevel to fold headers - - TODO: Python 3.3 is patched against this problems. See issue 1590744 on - python bug tracker. - """ - def _write_headers(self, msg): - for h, v in msg.items(): - print >> self._fp, '%s:' % h, - print >> self._fp, v - print >> self._fp diff --git a/mail/src/leap/mail/imap/__init__.py b/mail/src/leap/mail/imap/__init__.py deleted file mode 100644 index e69de29..0000000 diff --git a/mail/src/leap/mail/imap/account.py b/mail/src/leap/mail/imap/account.py deleted file mode 100644 index e795c1b..0000000 --- a/mail/src/leap/mail/imap/account.py +++ /dev/null @@ -1,498 +0,0 @@ -# -*- coding: utf-8 -*- -# account.py -# Copyright (C) 2013-2015 LEAP -# -# This program is free software: you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation, either version 3 of the License, or -# (at your option) any later version. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with this program. If not, see . -""" -Soledad Backed IMAP Account. -""" -import logging -import os -import time -from functools import partial - -from twisted.internet import defer -from twisted.mail import imap4 -from twisted.python import log -from zope.interface import implements - -from leap.common.check import leap_assert, leap_assert_type - -from leap.mail.constants import MessageFlags -from leap.mail.mail import Account -from leap.mail.imap.mailbox import IMAPMailbox, normalize_mailbox -from leap.soledad.client import Soledad - -logger = logging.getLogger(__name__) - -PROFILE_CMD = os.environ.get('LEAP_PROFILE_IMAPCMD', False) - -if PROFILE_CMD: - def _debugProfiling(result, cmdname, start): - took = (time.time() - start) * 1000 - log.msg("CMD " + cmdname + " TOOK: " + str(took) + " msec") - return result - - -####################################### -# Soledad IMAP Account -####################################### - - -class IMAPAccount(object): - """ - An implementation of an imap4 Account - that is backed by Soledad Encrypted Documents. - """ - - implements(imap4.IAccount, imap4.INamespacePresenter) - - selected = None - - def __init__(self, store, user_id, d=defer.Deferred()): - """ - Keeps track of the mailboxes and subscriptions handled by this account. - - The account is not ready to be used, since the store needs to be - initialized and we also need to do some initialization routines. - You can either pass a deferred to this constructor, or use - `callWhenReady` method. - - :param store: a Soledad instance. - :type store: Soledad - - :param user_id: The identifier of the user this account belongs to - (user id, in the form user@provider). - :type user_id: str - - - :param d: a deferred that will be fired with this IMAPAccount instance - when the account is ready to be used. - :type d: defer.Deferred - """ - leap_assert(store, "Need a store instance to initialize") - leap_assert_type(store, Soledad) - - # TODO assert too that the name matches the user/uuid with which - # soledad has been initialized. Although afaik soledad doesn't know - # about user_id, only the client backend. - - self.user_id = user_id - self.account = Account( - store, user_id, ready_cb=lambda: d.callback(self)) - - def end_session(self): - """ - Used to mark when the session has closed, and we should not allow any - more commands from the client. - - Right now it's called from the client backend. - """ - # TODO move its use to the service shutdown in leap.mail - self.account.end_session() - - @property - def session_ended(self): - return self.account.session_ended - - def callWhenReady(self, cb, *args, **kw): - """ - Execute callback when the account is ready to be used. - XXX note that this callback will be called with a first ignored - parameter. - """ - # TODO ignore the first parameter and change tests accordingly. - d = self.account.callWhenReady(cb, *args, **kw) - return d - - def getMailbox(self, name): - """ - Return a Mailbox with that name, without selecting it. - - :param name: name of the mailbox - :type name: str - - :returns: an IMAPMailbox instance - :rtype: IMAPMailbox - """ - name = normalize_mailbox(name) - - def check_it_exists(mailboxes): - if name not in mailboxes: - raise imap4.MailboxException("No such mailbox: %r" % name) - return True - - d = self.account.list_all_mailbox_names() - d.addCallback(check_it_exists) - d.addCallback(lambda _: self.account.get_collection_by_mailbox(name)) - d.addCallback(self._return_mailbox_from_collection) - return d - - def _return_mailbox_from_collection(self, collection, readwrite=1): - if collection is None: - return None - mbox = IMAPMailbox(collection, rw=readwrite) - return mbox - - # - # IAccount - # - - def addMailbox(self, name, creation_ts=None): - """ - Add a mailbox to the account. - - :param name: the name of the mailbox - :type name: str - - :param creation_ts: an optional creation timestamp to be used as - mailbox id. A timestamp will be used if no - one is provided. - :type creation_ts: int - - :returns: a Deferred that will contain the document if successful. - :rtype: defer.Deferred - """ - name = normalize_mailbox(name) - - # FIXME --- return failure instead of AssertionError - # See AccountTestCase... - leap_assert(name, "Need a mailbox name to create a mailbox") - - def check_it_does_not_exist(mailboxes): - if name in mailboxes: - raise imap4.MailboxCollision, repr(name) - return mailboxes - - d = self.account.list_all_mailbox_names() - d.addCallback(check_it_does_not_exist) - d.addCallback(lambda _: self.account.add_mailbox( - name, creation_ts=creation_ts)) - d.addCallback(lambda _: self.account.get_collection_by_mailbox(name)) - d.addCallback(self._return_mailbox_from_collection) - return d - - def create(self, pathspec): - """ - Create a new mailbox from the given hierarchical name. - - :param pathspec: - The full hierarchical name of a new mailbox to create. - If any of the inferior hierarchical names to this one - do not exist, they are created as well. - :type pathspec: str - - :return: - A deferred that will fire with a true value if the creation - succeeds. The deferred might fail with a MailboxException - if the mailbox cannot be added. - :rtype: Deferred - - """ - def pass_on_collision(failure): - failure.trap(imap4.MailboxCollision) - return True - - def handle_collision(failure): - failure.trap(imap4.MailboxCollision) - if not pathspec.endswith('/'): - return defer.succeed(False) - else: - return defer.succeed(True) - - def all_good(result): - return all(result) - - paths = filter(None, normalize_mailbox(pathspec).split('/')) - subs = [] - sep = '/' - - for accum in range(1, len(paths)): - partial_path = sep.join(paths[:accum]) - d = self.addMailbox(partial_path) - d.addErrback(pass_on_collision) - subs.append(d) - - df = self.addMailbox(sep.join(paths)) - df.addErrback(handle_collision) - subs.append(df) - - d1 = defer.gatherResults(subs) - d1.addCallback(all_good) - return d1 - - def select(self, name, readwrite=1): - """ - Selects a mailbox. - - :param name: the mailbox to select - :type name: str - - :param readwrite: 1 for readwrite permissions. - :type readwrite: int - - :rtype: IMAPMailbox - """ - name = normalize_mailbox(name) - - def check_it_exists(mailboxes): - if name not in mailboxes: - logger.warning("SELECT: No such mailbox!") - return None - return name - - def set_selected(_): - self.selected = name - - def get_collection(name): - if name is None: - return None - return self.account.get_collection_by_mailbox(name) - - d = self.account.list_all_mailbox_names() - d.addCallback(check_it_exists) - d.addCallback(get_collection) - d.addCallback(partial( - self._return_mailbox_from_collection, readwrite=readwrite)) - return d - - def delete(self, name, force=False): - """ - Deletes a mailbox. - - :param name: the mailbox to be deleted - :type name: str - - :param force: - if True, it will not check for noselect flag or inferior - names. use with care. - :type force: bool - :rtype: Deferred - """ - name = normalize_mailbox(name) - _mboxes = None - - def check_it_exists(mailboxes): - global _mboxes - _mboxes = mailboxes - if name not in mailboxes: - raise imap4.MailboxException("No such mailbox: %r" % name) - - def get_mailbox(_): - return self.getMailbox(name) - - def destroy_mailbox(mbox): - return mbox.destroy() - - def check_can_be_deleted(mbox): - global _mboxes - # See if this box is flagged \Noselect - mbox_flags = mbox.getFlags() - if MessageFlags.NOSELECT_FLAG in mbox_flags: - # Check for hierarchically inferior mailboxes with this one - # as part of their root. - for others in _mboxes: - if others != name and others.startswith(name): - raise imap4.MailboxException( - "Hierarchically inferior mailboxes " - "exist and \\Noselect is set") - return mbox - - d = self.account.list_all_mailbox_names() - d.addCallback(check_it_exists) - d.addCallback(get_mailbox) - if not force: - d.addCallback(check_can_be_deleted) - d.addCallback(destroy_mailbox) - return d - - # FIXME --- not honoring the inferior names... - # if there are no hierarchically inferior names, we will - # delete it from our ken. - # XXX is this right? - # if self._inferiorNames(name) > 1: - # self._index.removeMailbox(name) - - def rename(self, oldname, newname): - """ - Renames a mailbox. - - :param oldname: old name of the mailbox - :type oldname: str - - :param newname: new name of the mailbox - :type newname: str - """ - oldname = normalize_mailbox(oldname) - newname = normalize_mailbox(newname) - - def rename_inferiors((inferiors, mailboxes)): - rename_deferreds = [] - inferiors = [ - (o, o.replace(oldname, newname, 1)) for o in inferiors] - - for (old, new) in inferiors: - if new in mailboxes: - raise imap4.MailboxCollision(repr(new)) - - for (old, new) in inferiors: - d = self.account.rename_mailbox(old, new) - rename_deferreds.append(d) - - d1 = defer.gatherResults(rename_deferreds, consumeErrors=True) - return d1 - - d1 = self._inferiorNames(oldname) - d2 = self.account.list_all_mailbox_names() - - d = defer.gatherResults([d1, d2]) - d.addCallback(rename_inferiors) - return d - - def _inferiorNames(self, name): - """ - Return hierarchically inferior mailboxes. - - :param name: name of the mailbox - :rtype: list - """ - # XXX use wildcard query instead - def filter_inferiors(mailboxes): - inferiors = [] - for infname in mailboxes: - if infname.startswith(name): - inferiors.append(infname) - return inferiors - - d = self.account.list_all_mailbox_names() - d.addCallback(filter_inferiors) - return d - - def listMailboxes(self, ref, wildcard): - """ - List the mailboxes. - - from rfc 3501: - returns a subset of names from the complete set - of all names available to the client. Zero or more untagged LIST - replies are returned, containing the name attributes, hierarchy - delimiter, and name. - - :param ref: reference name - :type ref: str - - :param wildcard: mailbox name with possible wildcards - :type wildcard: str - """ - wildcard = imap4.wildcardToRegexp(wildcard, '/') - - def get_list(mboxes, mboxes_names): - return zip(mboxes_names, mboxes) - - def filter_inferiors(ref): - mboxes = [mbox for mbox in ref if wildcard.match(mbox)] - mbox_d = defer.gatherResults([self.getMailbox(m) for m in mboxes]) - - mbox_d.addCallback(get_list, mboxes) - return mbox_d - - d = self._inferiorNames(normalize_mailbox(ref)) - d.addCallback(filter_inferiors) - return d - - # - # The rest of the methods are specific for leap.mail.imap.account.Account - # - - def isSubscribed(self, name): - """ - Returns True if user is subscribed to this mailbox. - - :param name: the mailbox to be checked. - :type name: str - - :rtype: Deferred (will fire with bool) - """ - name = normalize_mailbox(name) - - def get_subscribed(mbox): - return mbox.collection.get_mbox_attr("subscribed") - - d = self.getMailbox(name) - d.addCallback(get_subscribed) - return d - - def subscribe(self, name): - """ - Subscribe to this mailbox if not already subscribed. - - :param name: name of the mailbox - :type name: str - :rtype: Deferred - """ - name = normalize_mailbox(name) - - def set_subscribed(mbox): - return mbox.collection.set_mbox_attr("subscribed", True) - - d = self.getMailbox(name) - d.addCallback(set_subscribed) - return d - - def unsubscribe(self, name): - """ - Unsubscribe from this mailbox - - :param name: name of the mailbox - :type name: str - :rtype: Deferred - """ - # TODO should raise MailboxException if attempted to unsubscribe - # from a mailbox that is not currently subscribed. - # TODO factor out with subscribe method. - name = normalize_mailbox(name) - - def set_unsubscribed(mbox): - return mbox.collection.set_mbox_attr("subscribed", False) - - d = self.getMailbox(name) - d.addCallback(set_unsubscribed) - return d - - def getSubscriptions(self): - def get_subscribed(mailboxes): - return [x.mbox for x in mailboxes if x.subscribed] - - d = self.account.get_all_mailboxes() - d.addCallback(get_subscribed) - return d - - # - # INamespacePresenter - # - - def getPersonalNamespaces(self): - return [["", "/"]] - - def getSharedNamespaces(self): - return None - - def getOtherNamespaces(self): - return None - - def __repr__(self): - """ - Representation string for this object. - """ - return "" % self.user_id diff --git a/mail/src/leap/mail/imap/mailbox.py b/mail/src/leap/mail/imap/mailbox.py deleted file mode 100644 index e70a1d8..0000000 --- a/mail/src/leap/mail/imap/mailbox.py +++ /dev/null @@ -1,970 +0,0 @@ -# *- coding: utf-8 -*- -# mailbox.py -# Copyright (C) 2013-2015 LEAP -# -# This program is free software: you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation, either version 3 of the License, or -# (at your option) any later version. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with this program. If not, see . -""" -IMAP Mailbox. -""" -import re -import logging -import os -import cStringIO -import StringIO -import time - -from collections import defaultdict -from email.utils import formatdate - -from twisted.internet import defer -from twisted.internet import reactor -from twisted.python import log - -from twisted.mail import imap4 -from zope.interface import implements - -from leap.common.check import leap_assert -from leap.common.check import leap_assert_type -from leap.mail.constants import INBOX_NAME, MessageFlags -from leap.mail.imap.messages import IMAPMessage - -logger = logging.getLogger(__name__) - -# TODO LIST -# [ ] Restore profile_cmd instrumentation -# [ ] finish the implementation of IMailboxListener -# [ ] implement the rest of ISearchableMailbox - - -""" -If the environment variable `LEAP_SKIPNOTIFY` is set, we avoid -notifying clients of new messages. Use during stress tests. -""" -NOTIFY_NEW = not os.environ.get('LEAP_SKIPNOTIFY', False) -PROFILE_CMD = os.environ.get('LEAP_PROFILE_IMAPCMD', False) - -if PROFILE_CMD: - - def _debugProfiling(result, cmdname, start): - took = (time.time() - start) * 1000 - log.msg("CMD " + cmdname + " TOOK: " + str(took) + " msec") - return result - - def do_profile_cmd(d, name): - """ - Add the profiling debug to the passed callback. - :param d: deferred - :param name: name of the command - :type name: str - """ - d.addCallback(_debugProfiling, name, time.time()) - d.addErrback(lambda f: log.msg(f.getTraceback())) - -INIT_FLAGS = (MessageFlags.SEEN_FLAG, MessageFlags.ANSWERED_FLAG, - MessageFlags.FLAGGED_FLAG, MessageFlags.DELETED_FLAG, - MessageFlags.DRAFT_FLAG, MessageFlags.RECENT_FLAG, - MessageFlags.LIST_FLAG) - - -def make_collection_listener(mailbox): - """ - Wrap a mailbox in a class that can be hashed according to the mailbox name. - - This means that dicts or sets will use this new equality rule, so we won't - collect multiple instances of the same mailbox in collections like the - MessageCollection set where we keep track of listeners. - """ - - class HashableMailbox(object): - - def __init__(self, mbox): - self.mbox = mbox - - # See #8083, pixelated adaptor seems to be misusing this class. - self.mailbox_name = self.mbox.mbox_name - - def __hash__(self): - return hash(self.mbox.mbox_name) - - def __eq__(self, other): - return self.mbox.mbox_name == other.mbox.mbox_name - - def notify_new(self): - self.mbox.notify_new() - - return HashableMailbox(mailbox) - - -class IMAPMailbox(object): - """ - A Soledad-backed IMAP mailbox. - - Implements the high-level method needed for the Mailbox interfaces. - The low-level database methods are contained in the generic - MessageCollection class. We receive an instance of it and it is made - accessible in the `collection` attribute. - """ - implements( - imap4.IMailbox, - imap4.IMailboxInfo, - imap4.ISearchableMailbox, - # XXX I think we do not need to implement CloseableMailbox, do we? - # We could remove ourselves from the collectionListener, although I - # think it simply will be garbage collected. - # imap4.ICloseableMailbox - imap4.IMessageCopier) - - init_flags = INIT_FLAGS - - CMD_MSG = "MESSAGES" - CMD_RECENT = "RECENT" - CMD_UIDNEXT = "UIDNEXT" - CMD_UIDVALIDITY = "UIDVALIDITY" - CMD_UNSEEN = "UNSEEN" - - # TODO we should turn this into a datastructure with limited capacity - _listeners = defaultdict(set) - - def __init__(self, collection, rw=1): - """ - :param collection: instance of MessageCollection - :type collection: MessageCollection - - :param rw: read-and-write flag for this mailbox - :type rw: int - """ - self.rw = rw - self._uidvalidity = None - self.collection = collection - self.collection.addListener(make_collection_listener(self)) - - @property - def mbox_name(self): - return self.collection.mbox_name - - @property - def listeners(self): - """ - Returns listeners for this mbox. - - The server itself is a listener to the mailbox. - so we can notify it (and should!) after changes in flags - and number of messages. - - :rtype: set - """ - return self._listeners[self.mbox_name] - - def get_imap_message(self, message): - d = defer.Deferred() - IMAPMessage(message, store=self.collection.store, d=d) - return d - - # FIXME this grows too crazily when many instances are fired, like - # during imaptest stress testing. Should have a queue of limited size - # instead. - - def addListener(self, listener): - """ - Add a listener to the listeners queue. - The server adds itself as a listener when there is a SELECT, - so it can send EXIST commands. - - :param listener: listener to add - :type listener: an object that implements IMailboxListener - """ - if not NOTIFY_NEW: - return - - listeners = self.listeners - logger.debug('adding mailbox listener: %s. Total: %s' % ( - listener, len(listeners))) - listeners.add(listener) - - def removeListener(self, listener): - """ - Remove a listener from the listeners queue. - - :param listener: listener to remove - :type listener: an object that implements IMailboxListener - """ - self.listeners.remove(listener) - - def getFlags(self): - """ - Returns the flags defined for this mailbox. - - :returns: tuple of flags for this mailbox - :rtype: tuple of str - """ - flags = self.collection.mbox_wrapper.flags - if not flags: - flags = self.init_flags - flags_str = map(str, flags) - return flags_str - - def setFlags(self, flags): - """ - Sets flags for this mailbox. - - :param flags: a tuple with the flags - :type flags: tuple of str - """ - # XXX this is setting (overriding) old flags. - # Better pass a mode flag - leap_assert(isinstance(flags, tuple), - "flags expected to be a tuple") - return self.collection.set_mbox_attr("flags", flags) - - def getUIDValidity(self): - """ - Return the unique validity identifier for this mailbox. - - :return: unique validity identifier - :rtype: int - """ - return self.collection.get_mbox_attr("created") - - def getUID(self, message_number): - """ - Return the UID of a message in the mailbox - - .. note:: this implementation does not make much sense RIGHT NOW, - but in the future will be useful to get absolute UIDs from - message sequence numbers. - - - :param message: the message sequence number. - :type message: int - - :rtype: int - :return: the UID of the message. - - """ - # TODO support relative sequences. The (imap) message should - # receive a sequence number attribute: a deferred is not expected - return message_number - - def getUIDNext(self): - """ - Return the likely UID for the next message added to this - mailbox. Currently it returns the higher UID incremented by - one. - - :return: deferred with int - :rtype: Deferred - """ - d = self.collection.get_uid_next() - return d - - def getMessageCount(self): - """ - Returns the total count of messages in this mailbox. - - :return: deferred with int - :rtype: Deferred - """ - return self.collection.count() - - def getUnseenCount(self): - """ - Returns the number of messages with the 'Unseen' flag. - - :return: count of messages flagged `unseen` - :rtype: int - """ - return self.collection.count_unseen() - - def getRecentCount(self): - """ - Returns the number of messages with the 'Recent' flag. - - :return: count of messages flagged `recent` - :rtype: int - """ - return self.collection.count_recent() - - def isWriteable(self): - """ - Get the read/write status of the mailbox. - - :return: 1 if mailbox is read-writeable, 0 otherwise. - :rtype: int - """ - # XXX We don't need to store it in the mbox doc, do we? - # return int(self.collection.get_mbox_attr('rw')) - return self.rw - - def getHierarchicalDelimiter(self): - """ - Returns the character used to delimite hierarchies in mailboxes. - - :rtype: str - """ - return '/' - - def requestStatus(self, names): - """ - Handles a status request by gathering the output of the different - status commands. - - :param names: a list of strings containing the status commands - :type names: iter - """ - r = {} - maybe = defer.maybeDeferred - if self.CMD_MSG in names: - r[self.CMD_MSG] = maybe(self.getMessageCount) - if self.CMD_RECENT in names: - r[self.CMD_RECENT] = maybe(self.getRecentCount) - if self.CMD_UIDNEXT in names: - r[self.CMD_UIDNEXT] = maybe(self.getUIDNext) - if self.CMD_UIDVALIDITY in names: - r[self.CMD_UIDVALIDITY] = maybe(self.getUIDValidity) - if self.CMD_UNSEEN in names: - r[self.CMD_UNSEEN] = maybe(self.getUnseenCount) - - def as_a_dict(values): - return dict(zip(r.keys(), values)) - - d = defer.gatherResults(r.values()) - d.addCallback(as_a_dict) - return d - - def addMessage(self, message, flags, date=None, notify_just_mdoc=True): - """ - Adds a message to this mailbox. - - :param message: the raw message - :type message: str - - :param flags: flag list - :type flags: list of str - - :param date: timestamp - :type date: str, or None - - :param notify_just_mdoc: - boolean passed to the wrapper.create method, to indicate whether - we're insterested in being notified right after the mdoc has been - written (as it's the first doc to be written, and quite small, this - is faster, though potentially unsafe). - Setting it to True improves a *lot* the responsiveness of the - APPENDS: we just need to be notified when the mdoc is saved, and - let's just expect that the other parts are doing just fine. This - will not catch any errors when the inserts of the other parts - fail, but on the other hand allows us to return very quickly, - which seems a good compromise given that we have to serialize the - appends. - However, some operations like the saving of drafts need to wait for - all the parts to be saved, so if some heuristics are met down in - the call chain a Draft message will unconditionally set this flag - to False, and therefore ignoring the setting of this flag here. - :type notify_just_mdoc: bool - - :return: a deferred that will be triggered with the UID of the added - message. - """ - # TODO should raise ReadOnlyMailbox if not rw. - # TODO have a look at the cases for internal date in the rfc - # XXX we could treat the message as an IMessage from here - - # TODO change notify_just_mdoc to something more meaningful, like - # fast_insert_notify? - - # TODO notify_just_mdoc *sometimes* make the append tests fail. - # have to find a better solution for this. A workaround could probably - # be to have a list of the ongoing deferreds related to append, so that - # we queue for later all the requests having to do with these. - - # A better solution will probably involve implementing MULTIAPPEND - # extension or patching imap server to support pipelining. - - if isinstance(message, (cStringIO.OutputType, StringIO.StringIO)): - message = message.getvalue() - - leap_assert_type(message, basestring) - - if flags is None: - flags = tuple() - else: - flags = tuple(str(flag) for flag in flags) - - if date is None: - date = formatdate(time.time()) - - d = self.collection.add_msg(message, flags, date=date, - notify_just_mdoc=notify_just_mdoc) - d.addErrback(lambda failure: log.err(failure)) - return d - - def notify_new(self, *args): - """ - Notify of new messages to all the listeners. - - This will be called indirectly by the underlying collection, that will - notify this IMAPMailbox whenever there are changes in the number of - messages in the collection, since we have added ourselves to the - collection listeners. - - :param args: ignored. - """ - if not NOTIFY_NEW: - return - - def cbNotifyNew(result): - exists, recent = result - for listener in self.listeners: - listener.newMessages(exists, recent) - - d = self._get_notify_count() - d.addCallback(cbNotifyNew) - d.addCallback(self.collection.cb_signal_unread_to_ui) - d.addErrback(lambda failure: log.err(failure)) - - def _get_notify_count(self): - """ - Get message count and recent count for this mailbox. - - :return: a deferred that will fire with a tuple, with number of - messages and number of recent messages. - :rtype: Deferred - """ - # XXX this is way too expensive in cases like multiple APPENDS. - # We should have a way of keep a cache or do a self-increment for that - # kind of calls. - d_exists = defer.maybeDeferred(self.getMessageCount) - d_recent = defer.maybeDeferred(self.getRecentCount) - d_list = [d_exists, d_recent] - - def log_num_msg(result): - exists, recent = tuple(result) - logger.debug("NOTIFY (%r): there are %s messages, %s recent" % ( - self.mbox_name, exists, recent)) - return result - - d = defer.gatherResults(d_list) - d.addCallback(log_num_msg) - return d - - # commands, do not rename methods - - def destroy(self): - """ - Called before this mailbox is permanently deleted. - - Should cleanup resources, and set the \\Noselect flag - on the mailbox. - - """ - # XXX this will overwrite all the existing flags - # should better simply addFlag - self.setFlags((MessageFlags.NOSELECT_FLAG,)) - - def remove_mbox(_): - uuid = self.collection.mbox_uuid - d = self.collection.mbox_wrapper.delete(self.collection.store) - d.addCallback( - lambda _: self.collection.mbox_indexer.delete_table(uuid)) - return d - - d = self.deleteAllDocs() - d.addCallback(remove_mbox) - return d - - def expunge(self): - """ - Remove all messages flagged \\Deleted - """ - if not self.isWriteable(): - raise imap4.ReadOnlyMailbox - return self.collection.delete_all_flagged() - - def _get_message_fun(self, uid): - """ - Return the proper method to get a message for this mailbox, depending - on the passed uid flag. - - :param uid: If true, the IDs specified in the query are UIDs; - otherwise they are message sequence IDs. - :type uid: bool - :rtype: callable - """ - get_message_fun = [ - self.collection.get_message_by_sequence_number, - self.collection.get_message_by_uid][uid] - return get_message_fun - - def _get_messages_range(self, messages_asked, uid=True): - - def get_range(messages_asked): - return self._filter_msg_seq(messages_asked) - - d = self._bound_seq(messages_asked, uid) - if uid: - d.addCallback(get_range) - d.addErrback(lambda f: log.err(f)) - return d - - def _bound_seq(self, messages_asked, uid): - """ - Put an upper bound to a messages sequence if this is open. - - :param messages_asked: IDs of the messages. - :type messages_asked: MessageSet - :return: a Deferred that will fire with a MessageSet - """ - - def set_last_uid(last_uid): - messages_asked.last = last_uid - return messages_asked - - def set_last_seq(all_uid): - messages_asked.last = len(all_uid) - return messages_asked - - if not messages_asked.last: - try: - iter(messages_asked) - except TypeError: - # looks like we cannot iterate - if uid: - d = self.collection.get_last_uid() - d.addCallback(set_last_uid) - else: - d = self.collection.all_uid_iter() - d.addCallback(set_last_seq) - return d - return defer.succeed(messages_asked) - - def _filter_msg_seq(self, messages_asked): - """ - Filter a message sequence returning only the ones that do exist in the - collection. - - :param messages_asked: IDs of the messages. - :type messages_asked: MessageSet - :rtype: set - """ - # TODO we could pass the asked sequence to the indexer - # all_uid_iter, and bound the sql query instead. - def filter_by_asked(all_msg_uid): - set_asked = set(messages_asked) - set_exist = set(all_msg_uid) - return set_asked.intersection(set_exist) - - d = self.collection.all_uid_iter() - d.addCallback(filter_by_asked) - return d - - def fetch(self, messages_asked, uid): - """ - Retrieve one or more messages in this mailbox. - - from rfc 3501: The data items to be fetched can be either a single atom - or a parenthesized list. - - :param messages_asked: IDs of the messages to retrieve information - about - :type messages_asked: MessageSet - - :param uid: If true, the IDs are UIDs. They are message sequence IDs - otherwise. - :type uid: bool - - :rtype: deferred with a generator that yields... - """ - get_msg_fun = self._get_message_fun(uid) - getimapmsg = self.get_imap_message - - def get_imap_messages_for_range(msg_range): - - def _get_imap_msg(messages): - d_imapmsg = [] - # just in case we got bad data in here - for msg in filter(None, messages): - d_imapmsg.append(getimapmsg(msg)) - return defer.gatherResults(d_imapmsg, consumeErrors=True) - - def _zip_msgid(imap_messages): - zipped = zip( - list(msg_range), imap_messages) - return (item for item in zipped) - - # XXX not called?? - def _unset_recent(sequence): - reactor.callLater(0, self.unset_recent_flags, sequence) - return sequence - - d_msg = [] - for msgid in msg_range: - # XXX We want cdocs because we "probably" are asked for the - # body. We should be smarter at do_FETCH and pass a parameter - # to this method in order not to prefetch cdocs if they're not - # going to be used. - d_msg.append(get_msg_fun(msgid, get_cdocs=True)) - - d = defer.gatherResults(d_msg, consumeErrors=True) - d.addCallback(_get_imap_msg) - d.addCallback(_zip_msgid) - d.addErrback(lambda failure: log.err(failure)) - return d - - d = self._get_messages_range(messages_asked, uid) - d.addCallback(get_imap_messages_for_range) - d.addErrback(lambda failure: log.err(failure)) - return d - - def fetch_flags(self, messages_asked, uid): - """ - A fast method to fetch all flags, tricking just the - needed subset of the MIME interface that's needed to satisfy - a generic FLAGS query. - - Given how LEAP Mail is supposed to work without local cache, - this query is going to be quite common, and also we expect - it to be in the form 1:* at the beginning of a session, so - it's not bad to fetch all the FLAGS docs at once. - - :param messages_asked: IDs of the messages to retrieve information - about - :type messages_asked: MessageSet - - :param uid: If 1, the IDs are UIDs. They are message sequence IDs - otherwise. - :type uid: int - - :return: A tuple of two-tuples of message sequence numbers and - flagsPart, which is a only a partial implementation of - MessagePart. - :rtype: tuple - """ - # is_sequence = True if uid == 0 else False - # XXX FIXME ----------------------------------------------------- - # imap/tests, or muas like mutt, it will choke until we implement - # sequence numbers. This is an easy hack meanwhile. - is_sequence = False - # --------------------------------------------------------------- - - if is_sequence: - raise NotImplementedError( - "FETCH FLAGS NOT IMPLEMENTED FOR MESSAGE SEQUENCE NUMBERS YET") - - d = defer.Deferred() - reactor.callLater(0, self._do_fetch_flags, messages_asked, uid, d) - if PROFILE_CMD: - do_profile_cmd(d, "FETCH-ALL-FLAGS") - return d - - def _do_fetch_flags(self, messages_asked, uid, d): - """ - :param messages_asked: IDs of the messages to retrieve information - about - :type messages_asked: MessageSet - - :param uid: If 1, the IDs are UIDs. They are message sequence IDs - otherwise. - :type uid: int - :param d: deferred whose callback will be called with result. - :type d: Deferred - - :rtype: A generator that yields two-tuples of message sequence numbers - and flagsPart - """ - class flagsPart(object): - def __init__(self, uid, flags): - self.uid = uid - self.flags = flags - - def getUID(self): - return self.uid - - def getFlags(self): - return map(str, self.flags) - - def pack_flags(result): - _uid, _flags = result - return _uid, flagsPart(_uid, _flags) - - def get_flags_for_seq(sequence): - d_all_flags = [] - for msgid in sequence: - # TODO implement sequence numbers here too - d_flags_per_uid = self.collection.get_flags_by_uid(msgid) - d_flags_per_uid.addCallback(pack_flags) - d_all_flags.append(d_flags_per_uid) - gotflags = defer.gatherResults(d_all_flags) - gotflags.addCallback(get_uid_flag_generator) - return gotflags - - def get_uid_flag_generator(result): - generator = (item for item in result) - d.callback(generator) - - d_seq = self._get_messages_range(messages_asked, uid) - d_seq.addCallback(get_flags_for_seq) - return d_seq - - @defer.inlineCallbacks - def fetch_headers(self, messages_asked, uid): - """ - A fast method to fetch all headers, tricking just the - needed subset of the MIME interface that's needed to satisfy - a generic HEADERS query. - - Given how LEAP Mail is supposed to work without local cache, - this query is going to be quite common, and also we expect - it to be in the form 1:* at the beginning of a session, so - **MAYBE** it's not too bad to fetch all the HEADERS docs at once. - - :param messages_asked: IDs of the messages to retrieve information - about - :type messages_asked: MessageSet - - :param uid: If true, the IDs are UIDs. They are message sequence IDs - otherwise. - :type uid: bool - - :return: A tuple of two-tuples of message sequence numbers and - headersPart, which is a only a partial implementation of - MessagePart. - :rtype: tuple - """ - # TODO implement sequences - is_sequence = True if uid == 0 else False - if is_sequence: - raise NotImplementedError( - "FETCH HEADERS NOT IMPLEMENTED FOR SEQUENCE NUMBER YET") - - class headersPart(object): - def __init__(self, uid, headers): - self.uid = uid - self.headers = headers - - def getUID(self): - return self.uid - - def getHeaders(self, _): - return dict( - (str(key), str(value)) - for key, value in - self.headers.items()) - - messages_asked = yield self._bound_seq(messages_asked, uid) - seq_messg = yield self._filter_msg_seq(messages_asked) - - result = [] - for msgid in seq_messg: - msg = yield self.collection.get_message_by_uid(msgid) - headers = headersPart(msgid, msg.get_headers()) - result.append((msgid, headers)) - defer.returnValue(iter(result)) - - def store(self, messages_asked, flags, mode, uid): - """ - Sets the flags of one or more messages. - - :param messages: The identifiers of the messages to set the flags - :type messages: A MessageSet object with the list of messages requested - - :param flags: The flags to set, unset, or add. - :type flags: sequence of str - - :param mode: If mode is -1, these flags should be removed from the - specified messages. If mode is 1, these flags should be - added to the specified messages. If mode is 0, all - existing flags should be cleared and these flags should be - added. - :type mode: -1, 0, or 1 - - :param uid: If true, the IDs specified in the query are UIDs; - otherwise they are message sequence IDs. - :type uid: bool - - :return: A deferred, that will be called with a dict mapping message - sequence numbers to sequences of str representing the flags - set on the message after this operation has been performed. - :rtype: deferred - - :raise ReadOnlyMailbox: Raised if this mailbox is not open for - read-write. - """ - if not self.isWriteable(): - log.msg('read only mailbox!') - raise imap4.ReadOnlyMailbox - - d = defer.Deferred() - reactor.callLater(0, self._do_store, messages_asked, flags, - mode, uid, d) - if PROFILE_CMD: - do_profile_cmd(d, "STORE") - - d.addCallback(self.collection.cb_signal_unread_to_ui) - d.addErrback(lambda f: log.err(f)) - return d - - def _do_store(self, messages_asked, flags, mode, uid, observer): - """ - Helper method, invoke set_flags method in the IMAPMessageCollection. - - See the documentation for the `store` method for the parameters. - - :param observer: a deferred that will be called with the dictionary - mapping UIDs to flags after the operation has been - done. - :type observer: deferred - """ - # TODO we should prevent client from setting Recent flag - get_msg_fun = self._get_message_fun(uid) - leap_assert(not isinstance(flags, basestring), - "flags cannot be a string") - flags = tuple(flags) - - def set_flags_for_seq(sequence): - def return_result_dict(list_of_flags): - result = dict(zip(list(sequence), list_of_flags)) - observer.callback(result) - return result - - d_all_set = [] - for msgid in sequence: - d = get_msg_fun(msgid) - d.addCallback(lambda msg: self.collection.update_flags( - msg, flags, mode)) - d_all_set.append(d) - got_flags_setted = defer.gatherResults(d_all_set) - got_flags_setted.addCallback(return_result_dict) - return got_flags_setted - - d_seq = self._get_messages_range(messages_asked, uid) - d_seq.addCallback(set_flags_for_seq) - return d_seq - - # ISearchableMailbox - - def search(self, query, uid): - """ - Search for messages that meet the given query criteria. - - Warning: this is half-baked, and it might give problems since - it offers the SearchableInterface. - We'll be implementing it asap. - - :param query: The search criteria - :type query: list - - :param uid: If true, the IDs specified in the query are UIDs; - otherwise they are message sequence IDs. - :type uid: bool - - :return: A list of message sequence numbers or message UIDs which - match the search criteria or a C{Deferred} whose callback - will be invoked with such a list. - :rtype: C{list} or C{Deferred} - """ - # TODO see if we can raise w/o interrupting flow - # :raise IllegalQueryError: Raised when query is not valid. - # example query: - # ['UNDELETED', 'HEADER', 'Message-ID', - # XXX fixme, does not exist - # '52D44F11.9060107@dev.bitmask.net'] - - # TODO hardcoding for now! -- we'll support generic queries later on - # but doing a quickfix for avoiding duplicate saves in the draft - # folder. # See issue #4209 - - if len(query) > 2: - if query[1] == 'HEADER' and query[2].lower() == "message-id": - msgid = str(query[3]).strip() - logger.debug("Searching for %s" % (msgid,)) - - d = self.collection.get_uid_from_msgid(str(msgid)) - d.addCallback(lambda result: [result]) - return d - - # nothing implemented for any other query - logger.warning("Cannot process query: %s" % (query,)) - return [] - - # IMessageCopier - - def copy(self, message): - """ - Copy the given message object into this mailbox. - - :param message: an IMessage implementor - :type message: LeapMessage - :return: a deferred that will be fired with the message - uid when the copy succeed. - :rtype: Deferred - """ - # if PROFILE_CMD: - # do_profile_cmd(d, "COPY") - - # A better place for this would be the COPY/APPEND dispatcher - # in server.py, but qtreactor hangs when I do that, so this seems - # to work fine for now. - # d.addCallback(lambda r: self.reactor.callLater(0, self.notify_new)) - # deferLater(self.reactor, 0, self._do_copy, message, d) - # return d - - d = self.collection.copy_msg(message.message, - self.collection.mbox_uuid) - return d - - # convenience fun - - def deleteAllDocs(self): - """ - Delete all docs in this mailbox - """ - # FIXME not implemented - return self.collection.delete_all_docs() - - def unset_recent_flags(self, uid_seq): - """ - Unset Recent flag for a sequence of UIDs. - """ - # FIXME not implemented - return self.collection.unset_recent_flags(uid_seq) - - def __repr__(self): - """ - Representation string for this mailbox. - """ - return u"" % ( - self.mbox_name, self.collection.count()) - - -_INBOX_RE = re.compile(INBOX_NAME, re.IGNORECASE) - - -def normalize_mailbox(name): - """ - Return a normalized representation of the mailbox ``name``. - - This method ensures that an eventual initial 'inbox' part of a - mailbox name is made uppercase. - - :param name: the name of the mailbox - :type name: unicode - - :rtype: unicode - """ - # XXX maybe it would make sense to normalize common folders too: - # trash, sent, drafts, etc... - if _INBOX_RE.match(name): - # ensure inital INBOX is uppercase - return INBOX_NAME + name[len(INBOX_NAME):] - return name diff --git a/mail/src/leap/mail/imap/messages.py b/mail/src/leap/mail/imap/messages.py deleted file mode 100644 index d1c7b93..0000000 --- a/mail/src/leap/mail/imap/messages.py +++ /dev/null @@ -1,254 +0,0 @@ -# -*- coding: utf-8 -*- -# imap/messages.py -# Copyright (C) 2013-2015 LEAP -# -# This program is free software: you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation, either version 3 of the License, or -# (at your option) any later version. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with this program. If not, see . -""" -IMAPMessage implementation. -""" -import logging -from twisted.mail import imap4 -from twisted.internet import defer -from zope.interface import implements - -from leap.mail.utils import find_charset, CaseInsensitiveDict - - -logger = logging.getLogger(__name__) - -# TODO -# [ ] Add ref to incoming message during add_msg. - - -class IMAPMessage(object): - """ - The main representation of a message as seen by the IMAP Server. - This class implements the semantics specific to IMAP specification. - """ - implements(imap4.IMessage) - - def __init__(self, message, prefetch_body=True, - store=None, d=defer.Deferred()): - """ - Get an IMAPMessage. A mail.Message is needed, since many of the methods - are proxied to that object. - - - If you do not need to prefetch the body of the message, you can set - `prefetch_body` to False, but the current imap server implementation - expect the getBodyFile method to return inmediately. - - When the prefetch_body option is used, a deferred is also expected as a - parameter, and this will fire when the deferred initialization has - taken place, with this instance of IMAPMessage as a parameter. - - :param message: the abstract message - :type message: mail.Message - :param prefetch_body: Whether to prefetch the content doc for the body. - :type prefetch_body: bool - :param store: an instance of soledad, or anything that behaves like it. - :param d: an optional deferred, that will be fired with the instance of - the IMAPMessage being initialized - :type d: defer.Deferred - """ - # TODO substitute the use of the deferred initialization by a factory - # function, maybe. - - self.message = message - self.__body_fd = None - self.store = store - if prefetch_body: - gotbody = self.__prefetch_body_file() - gotbody.addCallback(lambda _: d.callback(self)) - - # IMessage implementation - - def getUID(self): - """ - Retrieve the unique identifier associated with this Message. - - :return: uid for this message - :rtype: int - """ - return self.message.get_uid() - - def getFlags(self): - """ - Retrieve the flags associated with this Message. - - :return: The flags, represented as strings - :rtype: tuple - """ - return self.message.get_flags() - - def getInternalDate(self): - """ - Retrieve the date internally associated with this message - - According to the spec, this is NOT the date and time in the - RFC-822 header, but rather a date and time that reflects when the - message was received. - - * In SMTP, date and time of final delivery. - * In COPY, internal date/time of the source message. - * In APPEND, date/time specified. - - :return: An RFC822-formatted date string. - :rtype: str - """ - return self.message.get_internal_date() - - # - # IMessagePart - # - - def getBodyFile(self, store=None): - """ - Retrieve a file object containing only the body of this message. - - :return: file-like object opened for reading - :rtype: a deferred that will fire with a StringIO object. - """ - if self.__body_fd is not None: - fd = self.__body_fd - fd.seek(0) - return fd - - if store is None: - store = self.store - return self.message.get_body_file(store) - - def getSize(self): - """ - Return the total size, in octets, of this message. - - :return: size of the message, in octets - :rtype: int - """ - return self.message.get_size() - - def getHeaders(self, negate, *names): - """ - Retrieve a group of message headers. - - :param names: The names of the headers to retrieve or omit. - :type names: tuple of str - - :param negate: If True, indicates that the headers listed in names - should be omitted from the return value, rather - than included. - :type negate: bool - - :return: A mapping of header field names to header field values - :rtype: dict - """ - headers = self.message.get_headers() - return _format_headers(headers, negate, *names) - - def isMultipart(self): - """ - Return True if this message is multipart. - """ - return self.message.is_multipart() - - def getSubPart(self, part): - """ - Retrieve a MIME submessage - - :type part: C{int} - :param part: The number of the part to retrieve, indexed from 0. - :raise IndexError: Raised if the specified part does not exist. - :raise TypeError: Raised if this message is not multipart. - :rtype: Any object implementing C{IMessagePart}. - :return: The specified sub-part. - """ - subpart = self.message.get_subpart(part + 1) - return IMAPMessagePart(subpart) - - def __prefetch_body_file(self): - def assign_body_fd(fd): - self.__body_fd = fd - return fd - d = self.getBodyFile() - d.addCallback(assign_body_fd) - return d - - -class IMAPMessagePart(object): - - def __init__(self, message_part): - self.message_part = message_part - - def getBodyFile(self, store=None): - return self.message_part.get_body_file() - - def getSize(self): - return self.message_part.get_size() - - def getHeaders(self, negate, *names): - headers = self.message_part.get_headers() - return _format_headers(headers, negate, *names) - - def isMultipart(self): - return self.message_part.is_multipart() - - def getSubPart(self, part): - subpart = self.message_part.get_subpart(part + 1) - return IMAPMessagePart(subpart) - - -def _format_headers(headers, negate, *names): - # current server impl. expects content-type to be present, so if for - # some reason we do not have headers, we have to return at least that - # one - if not headers: - logger.warning("No headers found") - return {str('content-type'): str('')} - - names = map(lambda s: s.upper(), names) - - if negate: - def cond(key): - return key.upper() not in names - else: - def cond(key): - return key.upper() in names - - if isinstance(headers, list): - headers = dict(headers) - - # default to most likely standard - charset = find_charset(headers, "utf-8") - - # We will return a copy of the headers dictionary that - # will allow case-insensitive lookups. In some parts of the twisted imap - # server code the keys are expected to be in lower case, and in this way - # we avoid having to convert them. - - _headers = CaseInsensitiveDict() - for key, value in headers.items(): - if not isinstance(key, str): - key = key.encode(charset, 'replace') - if not isinstance(value, str): - value = value.encode(charset, 'replace') - - if value.endswith(";"): - # bastards - value = value[:-1] - - # filter original dict by negate-condition - if cond(key): - _headers[key] = value - - return _headers diff --git a/mail/src/leap/mail/imap/server.py b/mail/src/leap/mail/imap/server.py deleted file mode 100644 index 5a63af0..0000000 --- a/mail/src/leap/mail/imap/server.py +++ /dev/null @@ -1,693 +0,0 @@ -# -*- coding: utf-8 -*- -# server.py -# Copyright (C) 2014 LEAP -# -# This program is free software: you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation, either version 3 of the License, or -# (at your option) any later version. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with this program. If not, see . -""" -LEAP IMAP4 Server Implementation. -""" -import StringIO -from copy import copy - -from twisted.internet.defer import maybeDeferred -from twisted.mail import imap4 -from twisted.python import log - -# imports for LITERAL+ patch -from twisted.internet import defer, interfaces -from twisted.mail.imap4 import IllegalClientResponse -from twisted.mail.imap4 import LiteralString, LiteralFile - -from leap.common.events import emit_async, catalog - - -def _getContentType(msg): - """ - Return a two-tuple of the main and subtype of the given message. - """ - attrs = None - mm = msg.getHeaders(False, 'content-type').get('content-type', None) - if mm: - mm = ''.join(mm.splitlines()) - mimetype = mm.split(';') - if mimetype: - type = mimetype[0].split('/', 1) - if len(type) == 1: - major = type[0] - minor = None - elif len(type) == 2: - major, minor = type - else: - major = minor = None - # XXX patched --------------------------------------------- - attrs = dict(x.strip().split('=', 1) for x in mimetype[1:]) - # XXX patched --------------------------------------------- - else: - major = minor = None - else: - major = minor = None - return major, minor, attrs - -# Monkey-patch _getContentType to avoid bug that passes lower-case boundary in -# BODYSTRUCTURE response. -imap4._getContentType = _getContentType - - -class LEAPIMAPServer(imap4.IMAP4Server): - """ - An IMAP4 Server with a LEAP Storage Backend. - """ - - ############################################################# - # - # Twisted imap4 patch to workaround bad mime rendering in TB. - # See https://leap.se/code/issues/6773 - # and https://bugzilla.mozilla.org/show_bug.cgi?id=149771 - # Still unclear if this is a thunderbird bug. - # TODO send this patch upstream - # - ############################################################# - - def spew_body(self, part, id, msg, _w=None, _f=None): - if _w is None: - _w = self.transport.write - for p in part.part: - if msg.isMultipart(): - msg = msg.getSubPart(p) - elif p > 0: - # Non-multipart messages have an implicit first part but no - # other parts - reject any request for any other part. - raise TypeError("Requested subpart of non-multipart message") - - if part.header: - hdrs = msg.getHeaders(part.header.negate, *part.header.fields) - hdrs = imap4._formatHeaders(hdrs) - # PATCHED ########################################## - _w(str(part) + ' ' + imap4._literal(hdrs + "\r\n")) - # PATCHED ########################################## - elif part.text: - _w(str(part) + ' ') - _f() - return imap4.FileProducer( - msg.getBodyFile() - ).beginProducing(self.transport) - elif part.mime: - hdrs = imap4._formatHeaders(msg.getHeaders(True)) - - # PATCHED ########################################## - _w(str(part) + ' ' + imap4._literal(hdrs + "\r\n")) - # END PATCHED ###################################### - - elif part.empty: - _w(str(part) + ' ') - _f() - if part.part: - # PATCHED ############################################# - # implement partial FETCH - # TODO implement boundary checks - # TODO see if there's a more efficient way, without - # copying the original content into a new buffer. - fd = msg.getBodyFile() - begin = getattr(part, "partialBegin", None) - _len = getattr(part, "partialLength", None) - if begin is not None and _len is not None: - _fd = StringIO.StringIO() - fd.seek(part.partialBegin) - _fd.write(fd.read(part.partialLength)) - _fd.seek(0) - else: - _fd = fd - return imap4.FileProducer( - _fd - # END PATCHED #########################3 - ).beginProducing(self.transport) - else: - mf = imap4.IMessageFile(msg, None) - if mf is not None: - return imap4.FileProducer( - mf.open()).beginProducing(self.transport) - return imap4.MessageProducer( - msg, None, self._scheduler).beginProducing(self.transport) - - else: - _w('BODY ' + - imap4.collapseNestedLists([imap4.getBodyStructure(msg)])) - - ################################################################## - # - # END Twisted imap4 patch to workaround bad mime rendering in TB. - # #6773 - # - ################################################################## - - def lineReceived(self, line): - """ - Attempt to parse a single line from the server. - - :param line: the line from the server, without the line delimiter. - :type line: str - """ - if "login" in line.lower(): - # avoid to log the pass, even though we are using a dummy auth - # by now. - msg = line[:7] + " [...]" - else: - msg = copy(line) - log.msg('rcv (%s): %s' % (self.state, msg)) - imap4.IMAP4Server.lineReceived(self, line) - - def close_server_connection(self): - """ - Send a BYE command so that the MUA at least knows that we're closing - the connection. - """ - self.sendLine( - '* BYE LEAP IMAP Proxy is shutting down; ' - 'so long and thanks for all the fish') - self.transport.loseConnection() - if self.mbox: - self.mbox.removeListener(self) - self.mbox = None - self.state = 'unauth' - - def do_FETCH(self, tag, messages, query, uid=0): - """ - Overwritten fetch dispatcher to use the fast fetch_flags - method - """ - if not query: - self.sendPositiveResponse(tag, 'FETCH complete') - return - - cbFetch = self._IMAP4Server__cbFetch - ebFetch = self._IMAP4Server__ebFetch - - if len(query) == 1 and str(query[0]) == "flags": - self._oldTimeout = self.setTimeout(None) - # no need to call iter, we get a generator - maybeDeferred( - self.mbox.fetch_flags, messages, uid=uid - ).addCallback( - cbFetch, tag, query, uid - ).addErrback(ebFetch, tag) - - elif len(query) == 1 and str(query[0]) == "rfc822.header": - self._oldTimeout = self.setTimeout(None) - # no need to call iter, we get a generator - maybeDeferred( - self.mbox.fetch_headers, messages, uid=uid - ).addCallback( - cbFetch, tag, query, uid - ).addErrback(ebFetch, tag) - else: - self._oldTimeout = self.setTimeout(None) - # no need to call iter, we get a generator - maybeDeferred( - self.mbox.fetch, messages, uid=uid - ).addCallback( - cbFetch, tag, query, uid - ).addErrback( - ebFetch, tag) - - select_FETCH = (do_FETCH, imap4.IMAP4Server.arg_seqset, - imap4.IMAP4Server.arg_fetchatt) - - def _cbSelectWork(self, mbox, cmdName, tag): - """ - Callback for selectWork - - * patched to avoid conformance errors due to incomplete UIDVALIDITY - line. - * patched to accept deferreds for messagecount and recent count - """ - if mbox is None: - self.sendNegativeResponse(tag, 'No such mailbox') - return - if '\\noselect' in [s.lower() for s in mbox.getFlags()]: - self.sendNegativeResponse(tag, 'Mailbox cannot be selected') - return - - d1 = defer.maybeDeferred(mbox.getMessageCount) - d2 = defer.maybeDeferred(mbox.getRecentCount) - return defer.gatherResults([d1, d2]).addCallback( - self.__cbSelectWork, mbox, cmdName, tag) - - def __cbSelectWork(self, ((msg_count, recent_count)), mbox, cmdName, tag): - flags = mbox.getFlags() - self.sendUntaggedResponse('FLAGS (%s)' % ' '.join(flags)) - - # Patched ------------------------------------------------------- - # accept deferreds for the count - self.sendUntaggedResponse(str(msg_count) + ' EXISTS') - self.sendUntaggedResponse(str(recent_count) + ' RECENT') - # ---------------------------------------------------------------- - - # Patched ------------------------------------------------------- - # imaptest was complaining about the incomplete line, we're adding - # "UIDs valid" here. - self.sendPositiveResponse( - None, '[UIDVALIDITY %d] UIDs valid' % mbox.getUIDValidity()) - # ---------------------------------------------------------------- - - s = mbox.isWriteable() and 'READ-WRITE' or 'READ-ONLY' - mbox.addListener(self) - self.sendPositiveResponse(tag, '[%s] %s successful' % (s, cmdName)) - self.state = 'select' - self.mbox = mbox - - def checkpoint(self): - """ - Called when the client issues a CHECK command. - - This should perform any checkpoint operations required by the server. - It may be a long running operation, but may not block. If it returns - a deferred, the client will only be informed of success (or failure) - when the deferred's callback (or errback) is invoked. - """ - # TODO implement a collection of ongoing deferreds? - return None - - ############################################################# - # - # Twisted imap4 patch to support LITERAL+ extension - # TODO send this patch upstream asap! - # - ############################################################# - - def capabilities(self): - cap = {'AUTH': self.challengers.keys()} - if self.ctx and self.canStartTLS: - t = self.transport - ti = interfaces.ISSLTransport - if not self.startedTLS and ti(t, None) is None: - cap['LOGINDISABLED'] = None - cap['STARTTLS'] = None - cap['NAMESPACE'] = None - cap['IDLE'] = None - # patched ############ - cap['LITERAL+'] = None - ###################### - return cap - - def _stringLiteral(self, size, literal_plus=False): - if size > self._literalStringLimit: - raise IllegalClientResponse( - "Literal too long! I accept at most %d octets" % - (self._literalStringLimit,)) - d = defer.Deferred() - self.parseState = 'pending' - self._pendingLiteral = LiteralString(size, d) - # Patched ########################################################### - if not literal_plus: - self.sendContinuationRequest('Ready for %d octets of text' % size) - ##################################################################### - self.setRawMode() - return d - - def _fileLiteral(self, size, literal_plus=False): - d = defer.Deferred() - self.parseState = 'pending' - self._pendingLiteral = LiteralFile(size, d) - if not literal_plus: - self.sendContinuationRequest('Ready for %d octets of data' % size) - self.setRawMode() - return d - - def arg_astring(self, line): - """ - Parse an astring from the line, return (arg, rest), possibly - via a deferred (to handle literals) - """ - line = line.strip() - if not line: - raise IllegalClientResponse("Missing argument") - d = None - arg, rest = None, None - if line[0] == '"': - try: - spam, arg, rest = line.split('"', 2) - rest = rest[1:] # Strip space - except ValueError: - raise IllegalClientResponse("Unmatched quotes") - elif line[0] == '{': - # literal - if line[-1] != '}': - raise IllegalClientResponse("Malformed literal") - - # Patched ################ - if line[-2] == "+": - literalPlus = True - size_end = -2 - else: - literalPlus = False - size_end = -1 - - try: - size = int(line[1:size_end]) - except ValueError: - raise IllegalClientResponse( - "Bad literal size: " + line[1:size_end]) - d = self._stringLiteral(size, literalPlus) - ########################## - else: - arg = line.split(' ', 1) - if len(arg) == 1: - arg.append('') - arg, rest = arg - return d or (arg, rest) - - def arg_literal(self, line): - """ - Parse a literal from the line - """ - if not line: - raise IllegalClientResponse("Missing argument") - - if line[0] != '{': - raise IllegalClientResponse("Missing literal") - - if line[-1] != '}': - raise IllegalClientResponse("Malformed literal") - - # Patched ################## - if line[-2] == "+": - literalPlus = True - size_end = -2 - else: - literalPlus = False - size_end = -1 - - try: - size = int(line[1:size_end]) - except ValueError: - raise IllegalClientResponse( - "Bad literal size: " + line[1:size_end]) - - return self._fileLiteral(size, literalPlus) - ############################# - - # --------------------------------- isSubscribed patch - # TODO -- send patch upstream. - # There is a bug in twisted implementation: - # in cbListWork, it's assumed that account.isSubscribed IS a callable, - # although in the interface documentation it's stated that it can be - # a deferred. - - def _listWork(self, tag, ref, mbox, sub, cmdName): - mbox = self._parseMbox(mbox) - mailboxes = maybeDeferred(self.account.listMailboxes, ref, mbox) - mailboxes.addCallback(self._cbSubscribed) - mailboxes.addCallback( - self._cbListWork, tag, sub, cmdName, - ).addErrback(self._ebListWork, tag) - - def _cbSubscribed(self, mailboxes): - subscribed = [ - maybeDeferred(self.account.isSubscribed, name) - for (name, box) in mailboxes] - - def get_mailboxes_and_subs(result): - subscribed = [i[0] for i, yes in zip(mailboxes, result) if yes] - return mailboxes, subscribed - - d = defer.gatherResults(subscribed) - d.addCallback(get_mailboxes_and_subs) - return d - - def _cbListWork(self, mailboxes_subscribed, tag, sub, cmdName): - mailboxes, subscribed = mailboxes_subscribed - - for (name, box) in mailboxes: - if not sub or name in subscribed: - flags = box.getFlags() - delim = box.getHierarchicalDelimiter() - resp = (imap4.DontQuoteMe(cmdName), - map(imap4.DontQuoteMe, flags), - delim, name.encode('imap4-utf-7')) - self.sendUntaggedResponse( - imap4.collapseNestedLists(resp)) - self.sendPositiveResponse(tag, '%s completed' % (cmdName,)) - # -------------------- end isSubscribed patch ----------- - - # TODO subscribe method had also to be changed to accomodate deferred - def do_SUBSCRIBE(self, tag, name): - name = self._parseMbox(name) - - def _subscribeCb(_): - self.sendPositiveResponse(tag, 'Subscribed') - - def _subscribeEb(failure): - m = failure.value - log.err() - if failure.check(imap4.MailboxException): - self.sendNegativeResponse(tag, str(m)) - else: - self.sendBadResponse( - tag, - "Server error encountered while subscribing to mailbox") - - d = self.account.subscribe(name) - d.addCallbacks(_subscribeCb, _subscribeEb) - return d - - auth_SUBSCRIBE = (do_SUBSCRIBE, arg_astring) - select_SUBSCRIBE = auth_SUBSCRIBE - - def do_UNSUBSCRIBE(self, tag, name): - # unsubscribe method had also to be changed to accomodate - # deferred - name = self._parseMbox(name) - - def _unsubscribeCb(_): - self.sendPositiveResponse(tag, 'Unsubscribed') - - def _unsubscribeEb(failure): - m = failure.value - log.err() - if failure.check(imap4.MailboxException): - self.sendNegativeResponse(tag, str(m)) - else: - self.sendBadResponse( - tag, - "Server error encountered while unsubscribing " - "from mailbox") - - d = self.account.unsubscribe(name) - d.addCallbacks(_unsubscribeCb, _unsubscribeEb) - return d - - auth_UNSUBSCRIBE = (do_UNSUBSCRIBE, arg_astring) - select_UNSUBSCRIBE = auth_UNSUBSCRIBE - - def do_RENAME(self, tag, oldname, newname): - oldname, newname = [self._parseMbox(n) for n in oldname, newname] - if oldname.lower() == 'inbox' or newname.lower() == 'inbox': - self.sendNegativeResponse( - tag, - 'You cannot rename the inbox, or ' - 'rename another mailbox to inbox.') - return - - def _renameCb(_): - self.sendPositiveResponse(tag, 'Mailbox renamed') - - def _renameEb(failure): - m = failure.value - if failure.check(TypeError): - self.sendBadResponse(tag, 'Invalid command syntax') - elif failure.check(imap4.MailboxException): - self.sendNegativeResponse(tag, str(m)) - else: - log.err() - self.sendBadResponse( - tag, - "Server error encountered while " - "renaming mailbox") - - d = self.account.rename(oldname, newname) - d.addCallbacks(_renameCb, _renameEb) - return d - - auth_RENAME = (do_RENAME, arg_astring, arg_astring) - select_RENAME = auth_RENAME - - def do_CREATE(self, tag, name): - name = self._parseMbox(name) - - def _createCb(result): - if result: - self.sendPositiveResponse(tag, 'Mailbox created') - else: - self.sendNegativeResponse(tag, 'Mailbox not created') - - def _createEb(failure): - c = failure.value - if failure.check(imap4.MailboxException): - self.sendNegativeResponse(tag, str(c)) - else: - log.err() - self.sendBadResponse( - tag, "Server error encountered while creating mailbox") - - d = self.account.create(name) - d.addCallbacks(_createCb, _createEb) - return d - - auth_CREATE = (do_CREATE, arg_astring) - select_CREATE = auth_CREATE - - def do_DELETE(self, tag, name): - name = self._parseMbox(name) - if name.lower() == 'inbox': - self.sendNegativeResponse(tag, 'You cannot delete the inbox') - return - - def _deleteCb(result): - self.sendPositiveResponse(tag, 'Mailbox deleted') - - def _deleteEb(failure): - m = failure.value - if failure.check(imap4.MailboxException): - self.sendNegativeResponse(tag, str(m)) - else: - print "SERVER: other error" - log.err() - self.sendBadResponse( - tag, - "Server error encountered while deleting mailbox") - - d = self.account.delete(name) - d.addCallbacks(_deleteCb, _deleteEb) - return d - - auth_DELETE = (do_DELETE, arg_astring) - select_DELETE = auth_DELETE - - # ----------------------------------------------------------------------- - # Patched just to allow __cbAppend to receive a deferred from messageCount - # TODO format and send upstream. - def do_APPEND(self, tag, mailbox, flags, date, message): - mailbox = self._parseMbox(mailbox) - maybeDeferred(self.account.select, mailbox).addCallback( - self._cbAppendGotMailbox, tag, flags, date, message).addErrback( - self._ebAppendGotMailbox, tag) - - def __ebAppend(self, failure, tag): - self.sendBadResponse(tag, 'APPEND failed: ' + str(failure.value)) - - def _cbAppendGotMailbox(self, mbox, tag, flags, date, message): - if not mbox: - self.sendNegativeResponse(tag, '[TRYCREATE] No such mailbox') - return - - d = mbox.addMessage(message, flags, date) - d.addCallback(self.__cbAppend, tag, mbox) - d.addErrback(self.__ebAppend, tag) - - def _ebAppendGotMailbox(self, failure, tag): - self.sendBadResponse( - tag, "Server error encountered while opening mailbox.") - log.err(failure) - - def __cbAppend(self, result, tag, mbox): - - # XXX patched --------------------------------- - def send_response(count): - self.sendUntaggedResponse('%d EXISTS' % count) - self.sendPositiveResponse(tag, 'APPEND complete') - - d = mbox.getMessageCount() - d.addCallback(send_response) - return d - # XXX patched --------------------------------- - # ----------------------------------------------------------------------- - - auth_APPEND = (do_APPEND, arg_astring, imap4.IMAP4Server.opt_plist, - imap4.IMAP4Server.opt_datetime, arg_literal) - select_APPEND = auth_APPEND - - # Need to override the command table after patching - # arg_astring and arg_literal, except on the methods that we are already - # overriding. - - # TODO -------------------------------------------- - # Check if we really need to override these - # methods, or we can monkeypatch. - # do_DELETE = imap4.IMAP4Server.do_DELETE - # do_CREATE = imap4.IMAP4Server.do_CREATE - # do_RENAME = imap4.IMAP4Server.do_RENAME - # do_SUBSCRIBE = imap4.IMAP4Server.do_SUBSCRIBE - # do_UNSUBSCRIBE = imap4.IMAP4Server.do_UNSUBSCRIBE - # do_APPEND = imap4.IMAP4Server.do_APPEND - # ------------------------------------------------- - do_LOGIN = imap4.IMAP4Server.do_LOGIN - do_STATUS = imap4.IMAP4Server.do_STATUS - do_COPY = imap4.IMAP4Server.do_COPY - - _selectWork = imap4.IMAP4Server._selectWork - - arg_plist = imap4.IMAP4Server.arg_plist - arg_seqset = imap4.IMAP4Server.arg_seqset - opt_plist = imap4.IMAP4Server.opt_plist - opt_datetime = imap4.IMAP4Server.opt_datetime - - unauth_LOGIN = (do_LOGIN, arg_astring, arg_astring) - - auth_SELECT = (_selectWork, arg_astring, 1, 'SELECT') - select_SELECT = auth_SELECT - - auth_CREATE = (do_CREATE, arg_astring) - select_CREATE = auth_CREATE - - auth_EXAMINE = (_selectWork, arg_astring, 0, 'EXAMINE') - select_EXAMINE = auth_EXAMINE - - # TODO ----------------------------------------------- - # re-add if we stop overriding DELETE - # auth_DELETE = (do_DELETE, arg_astring) - # select_DELETE = auth_DELETE - # auth_APPEND = (do_APPEND, arg_astring, opt_plist, opt_datetime, - # arg_literal) - # select_APPEND = auth_APPEND - - # ---------------------------------------------------- - - auth_RENAME = (do_RENAME, arg_astring, arg_astring) - select_RENAME = auth_RENAME - - auth_SUBSCRIBE = (do_SUBSCRIBE, arg_astring) - select_SUBSCRIBE = auth_SUBSCRIBE - - auth_UNSUBSCRIBE = (do_UNSUBSCRIBE, arg_astring) - select_UNSUBSCRIBE = auth_UNSUBSCRIBE - - auth_LIST = (_listWork, arg_astring, arg_astring, 0, 'LIST') - select_LIST = auth_LIST - - auth_LSUB = (_listWork, arg_astring, arg_astring, 1, 'LSUB') - select_LSUB = auth_LSUB - - auth_STATUS = (do_STATUS, arg_astring, arg_plist) - select_STATUS = auth_STATUS - - select_COPY = (do_COPY, arg_seqset, arg_astring) - - ############################################################# - # END of Twisted imap4 patch to support LITERAL+ extension - ############################################################# - - def authenticateLogin(self, user, passwd): - result = imap4.IMAP4Server.authenticateLogin(self, user, passwd) - emit_async(catalog.IMAP_CLIENT_LOGIN, str(user)) - return result diff --git a/mail/src/leap/mail/imap/service/README.rst b/mail/src/leap/mail/imap/service/README.rst deleted file mode 100644 index 2cca9b3..0000000 --- a/mail/src/leap/mail/imap/service/README.rst +++ /dev/null @@ -1,39 +0,0 @@ -testing the service -=================== - -Run the twisted service:: - - twistd -n -y imap-server.tac - -And use offlineimap for tests:: - - offlineimap -c LEAPofflineimapRC-tests - -minimal offlineimap configuration ---------------------------------- - -[general] -accounts = leap-local - -[Account leap-local] -localrepository = LocalLeap -remoterepository = RemoteLeap - -[Repository LocalLeap] -type = Maildir -localfolders = ~/LEAPMail/Mail - -[Repository RemoteLeap] -type = IMAP -ssl = no -remotehost = localhost -remoteport = 9930 -remoteuser = user -remotepass = pass - -debugging ---------- - -Use ngrep to obtain logs of the sequences:: - - sudo ngrep -d lo -W byline port 9930 diff --git a/mail/src/leap/mail/imap/service/__init__.py b/mail/src/leap/mail/imap/service/__init__.py deleted file mode 100644 index e69de29..0000000 diff --git a/mail/src/leap/mail/imap/service/imap-server.tac b/mail/src/leap/mail/imap/service/imap-server.tac deleted file mode 100644 index c4d602d..0000000 --- a/mail/src/leap/mail/imap/service/imap-server.tac +++ /dev/null @@ -1,145 +0,0 @@ -#!/usr/bin/env python -# -*- coding: utf-8 -*- -# imap-server.tac -# Copyright (C) 2013,2014 LEAP -# -# This program is free software: you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation, either version 3 of the License, or -# (at your option) any later version. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with this program. If not, see . -""" -TAC file for initialization of the imap service using twistd. - -Use this for debugging and testing the imap server using a native reactor. - -For now, and for debugging/testing purposes, you need -to pass a config file with the following structure: - -[leap_mail] -userid = 'user@provider' -uuid = 'deadbeefdeadabad' -passwd = 'supersecret' # optional, will get prompted if not found. -""" - -# TODO -- this .tac file should be deprecated in favor of bitmask.core.bitmaskd - -import ConfigParser -import getpass -import os -import sys - -from leap.keymanager import KeyManager -from leap.mail.imap.service import imap -from leap.soledad.client import Soledad - -from twisted.application import service, internet - - -# TODO should get this initializers from some authoritative mocked source -# We might want to put them the soledad itself. - -def initialize_soledad(uuid, email, passwd, - secrets, localdb, - gnupg_home, tempdir): - """ - Initializes soledad by hand - - :param email: ID for the user - :param gnupg_home: path to home used by gnupg - :param tempdir: path to temporal dir - :rtype: Soledad instance - """ - server_url = "http://provider" - cert_file = "" - - soledad = Soledad( - uuid, - passwd, - secrets, - localdb, - server_url, - cert_file, - syncable=False) - - return soledad - -###################################################################### -# Remember to set your config files, see module documentation above! -###################################################################### - -print "[+] Running LEAP IMAP Service" - - -bmconf = os.environ.get("LEAP_MAIL_CONFIG", "") -if not bmconf: - print ("[-] Please set LEAP_MAIL_CONFIG environment variable " - "pointing to your config.") - sys.exit(1) - -SECTION = "leap_mail" -cp = ConfigParser.ConfigParser() -cp.read(bmconf) - -userid = cp.get(SECTION, "userid") -uuid = cp.get(SECTION, "uuid") -passwd = unicode(cp.get(SECTION, "passwd")) - -# XXX get this right from the environment variable !!! -port = 1984 - -if not userid or not uuid: - print "[-] Config file missing userid or uuid field" - sys.exit(1) - -if not passwd: - passwd = unicode(getpass.getpass("Soledad passphrase: ")) - - -secrets = os.path.expanduser("~/.config/leap/soledad/%s.secret" % (uuid,)) -localdb = os.path.expanduser("~/.config/leap/soledad/%s.db" % (uuid,)) - -# XXX Is this really used? Should point it to user var dirs defined in xdg? -gnupg_home = "/tmp/" -tempdir = "/tmp/" - -################################################### - -# Ad-hoc soledad/keymanager initialization. - -print "[~] user:", userid -soledad = initialize_soledad(uuid, userid, passwd, secrets, - localdb, gnupg_home, tempdir, userid=userid) -km_args = (userid, "https://localhost", soledad) -km_kwargs = { - "token": "", - "ca_cert_path": "", - "api_uri": "", - "api_version": "", - "uid": uuid, - "gpgbinary": "/usr/bin/gpg" -} -keymanager = KeyManager(*km_args, **km_kwargs) - -################################################## - -# Ok, let's expose the application object for the twistd application -# framework to pick up from here... - - -def getIMAPService(): - soledad_sessions = {userid: soledad} - factory = imap.LeapIMAPFactory(soledad_sessions) - return internet.TCPServer(port, factory, interface="localhost") - - -application = service.Application("LEAP IMAP Application") -service = getIMAPService() -service.setServiceParent(application) diff --git a/mail/src/leap/mail/imap/service/imap.py b/mail/src/leap/mail/imap/service/imap.py deleted file mode 100644 index 4663854..0000000 --- a/mail/src/leap/mail/imap/service/imap.py +++ /dev/null @@ -1,208 +0,0 @@ -# -*- coding: utf-8 -*- -# imap.py -# Copyright (C) 2013-2015 LEAP -# -# This program is free software: you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation, either version 3 of the License, or -# (at your option) any later version. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with this program. If not, see . -""" -IMAP Service Initialization. -""" -import logging -import os - -from collections import defaultdict - -from twisted.cred.portal import Portal, IRealm -from twisted.mail.imap4 import IAccount -from twisted.internet import defer -from twisted.internet import reactor -from twisted.internet.error import CannotListenError -from twisted.internet.protocol import ServerFactory -from twisted.python import log -from zope.interface import implementer - -from leap.common.events import emit_async, catalog -from leap.mail.cred import LocalSoledadTokenChecker -from leap.mail.imap.account import IMAPAccount -from leap.mail.imap.server import LEAPIMAPServer - -# TODO: leave only an implementor of IService in here - -logger = logging.getLogger(__name__) - -DO_MANHOLE = os.environ.get("LEAP_MAIL_MANHOLE", None) -if DO_MANHOLE: - from leap.mail.imap.service import manhole - -# The default port in which imap service will run - -IMAP_PORT = 1984 - -# -# Credentials Handling -# - - -@implementer(IRealm) -class LocalSoledadIMAPRealm(object): - - _encoding = 'utf-8' - - def __init__(self, soledad_sessions): - """ - :param soledad_sessions: a dict-like object, containing instances - of a Store (soledad instances), indexed by - userid. - """ - self._soledad_sessions = soledad_sessions - - def requestAvatar(self, avatarId, mind, *interfaces): - if isinstance(avatarId, str): - avatarId = avatarId.decode(self._encoding) - - def gotSoledad(soledad): - for iface in interfaces: - if iface is IAccount: - avatar = IMAPAccount(soledad, avatarId) - return (IAccount, avatar, - getattr(avatar, 'logout', lambda: None)) - raise NotImplementedError(self, interfaces) - - return self.lookupSoledadInstance(avatarId).addCallback(gotSoledad) - - def lookupSoledadInstance(self, userid): - soledad = self._soledad_sessions[userid] - # XXX this should return the instance after whenReady callback - return defer.succeed(soledad) - - -class IMAPTokenChecker(LocalSoledadTokenChecker): - """A credentials checker that will lookup a token for the IMAP service. - For now it will be using the same identifier than SMTPTokenChecker""" - - service = 'mail_auth' - - -class LocalSoledadIMAPServer(LEAPIMAPServer): - - """ - An IMAP Server that authenticates against a LocalSoledad store. - """ - - def __init__(self, soledad_sessions, *args, **kw): - - LEAPIMAPServer.__init__(self, *args, **kw) - - realm = LocalSoledadIMAPRealm(soledad_sessions) - portal = Portal(realm) - checker = IMAPTokenChecker(soledad_sessions) - self.checker = checker - self.portal = portal - portal.registerChecker(checker) - - -class LeapIMAPFactory(ServerFactory): - - """ - Factory for a IMAP4 server with soledad remote sync and gpg-decryption - capabilities. - """ - - protocol = LocalSoledadIMAPServer - - def __init__(self, soledad_sessions): - """ - Initializes the server factory. - - :param soledad_sessions: a dict-like object, containing instances - of a Store (soledad instances), indexed by - userid. - """ - self._soledad_sessions = soledad_sessions - self._connections = defaultdict() - - def buildProtocol(self, addr): - """ - Return a protocol suitable for the job. - - :param addr: remote ip address - :type addr: str - """ - # TODO should reject anything from addr != localhost, - # just in case. - log.msg("Building protocol for connection %s" % addr) - imapProtocol = self.protocol(self._soledad_sessions) - self._connections[addr] = imapProtocol - return imapProtocol - - def stopFactory(self): - # say bye! - for conn, proto in self._connections.items(): - log.msg("Closing connections for %s" % conn) - proto.close_server_connection() - - def doStop(self): - """ - Stops imap service (fetcher, factory and port). - """ - return ServerFactory.doStop(self) - - -def run_service(soledad_sessions, port=IMAP_PORT): - """ - Main entry point to run the service from the client. - - :param soledad_sessions: a dict-like object, containing instances - of a Store (soledad instances), indexed by userid. - - :returns: the port as returned by the reactor when starts listening, and - the factory for the protocol. - :rtype: tuple - """ - factory = LeapIMAPFactory(soledad_sessions) - - try: - interface = "localhost" - # don't bind just to localhost if we are running on docker since we - # won't be able to access imap from the host - if os.environ.get("LEAP_DOCKERIZED"): - interface = '' - - # TODO use Endpoints !!! - tport = reactor.listenTCP(port, factory, - interface=interface) - except CannotListenError: - logger.error("IMAP Service failed to start: " - "cannot listen in port %s" % (port,)) - except Exception as exc: - logger.error("Error launching IMAP service: %r" % (exc,)) - else: - # all good. - - if DO_MANHOLE: - # TODO get pass from env var.too. - manhole_factory = manhole.getManholeFactory( - {'f': factory, - 'gm': factory.theAccount.getMailbox}, - "boss", "leap") - # TODO use Endpoints !!! - reactor.listenTCP(manhole.MANHOLE_PORT, manhole_factory, - interface="127.0.0.1") - logger.debug("IMAP4 Server is RUNNING in port %s" % (port,)) - emit_async(catalog.IMAP_SERVICE_STARTED, str(port)) - - # FIXME -- change service signature - return tport, factory - - # not ok, signal error. - emit_async(catalog.IMAP_SERVICE_FAILED_TO_START, str(port)) diff --git a/mail/src/leap/mail/imap/service/manhole.py b/mail/src/leap/mail/imap/service/manhole.py deleted file mode 100644 index c83ae89..0000000 --- a/mail/src/leap/mail/imap/service/manhole.py +++ /dev/null @@ -1,130 +0,0 @@ -# -*- coding: utf-8 -*- -# manhole.py -# Copyright (C) 2014 LEAP -# -# This program is free software: you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation, either version 3 of the License, or -# (at your option) any later version. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with this program. If not, see . -""" -Utilities for enabling the manhole administrative interface into the -LEAP Mail application. -""" -MANHOLE_PORT = 2222 - - -def getManholeFactory(namespace, user, secret): - """ - Get an administrative manhole into the application. - - :param namespace: the namespace to show in the manhole - :type namespace: dict - :param user: the user to authenticate into the administrative shell. - :type user: str - :param secret: pass for this manhole - :type secret: str - """ - import string - - from twisted.cred.portal import Portal - from twisted.conch import manhole, manhole_ssh - from twisted.conch.insults import insults - from twisted.cred.checkers import ( - InMemoryUsernamePasswordDatabaseDontUse as MemoryDB) - - from rlcompleter import Completer - - class EnhancedColoredManhole(manhole.ColoredManhole): - """ - A Manhole with some primitive autocomplete support. - """ - # TODO use introspection to make life easier - - def find_common(self, l): - """ - find common parts in thelist items - ex: 'ab' for ['abcd','abce','abf'] - requires an ordered list - """ - if len(l) == 1: - return l[0] - - init = l[0] - for item in l[1:]: - for i, (x, y) in enumerate(zip(init, item)): - if x != y: - init = "".join(init[:i]) - break - - if not init: - return None - return init - - def handle_TAB(self): - """ - Trap the TAB keystroke. - """ - necessarypart = "".join(self.lineBuffer).split(' ')[-1] - completer = Completer(globals()) - if completer.complete(necessarypart, 0): - matches = list(set(completer.matches)) # has multiples - - if len(matches) == 1: - length = len(necessarypart) - self.lineBuffer = self.lineBuffer[:-length] - self.lineBuffer.extend(matches[0]) - self.lineBufferIndex = len(self.lineBuffer) - else: - matches.sort() - commons = self.find_common(matches) - if commons: - length = len(necessarypart) - self.lineBuffer = self.lineBuffer[:-length] - self.lineBuffer.extend(commons) - self.lineBufferIndex = len(self.lineBuffer) - - self.terminal.nextLine() - while matches: - matches, part = matches[4:], matches[:4] - for item in part: - self.terminal.write('%s' % item.ljust(30)) - self.terminal.write('\n') - self.terminal.nextLine() - - self.terminal.eraseLine() - self.terminal.cursorBackward(self.lineBufferIndex + 5) - self.terminal.write("%s %s" % ( - self.ps[self.pn], "".join(self.lineBuffer))) - - def keystrokeReceived(self, keyID, modifier): - """ - Act upon any keystroke received. - """ - self.keyHandlers.update({'\b': self.handle_BACKSPACE}) - m = self.keyHandlers.get(keyID) - if m is not None: - m() - elif keyID in string.printable: - self.characterReceived(keyID, False) - - sshRealm = manhole_ssh.TerminalRealm() - - def chainedProtocolFactory(): - return insults.ServerProtocol(EnhancedColoredManhole, namespace) - - sshRealm = manhole_ssh.TerminalRealm() - sshRealm.chainedProtocolFactory = chainedProtocolFactory - - portal = Portal( - sshRealm, [MemoryDB(**{user: secret})]) - - f = manhole_ssh.ConchFactory(portal) - return f diff --git a/mail/src/leap/mail/imap/service/notes.txt b/mail/src/leap/mail/imap/service/notes.txt deleted file mode 100644 index 623e122..0000000 --- a/mail/src/leap/mail/imap/service/notes.txt +++ /dev/null @@ -1,81 +0,0 @@ -T 127.0.0.1:9930 -> 127.0.0.1:42866 [AP] -* OK [CAPABILITY IMAP4rev1 IDLE NAMESPACE] Twisted IMAP4rev1 Ready. - -## -T 127.0.0.1:42866 -> 127.0.0.1:9930 [AP] -NCLJ1 CAPABILITY. - -## -T 127.0.0.1:9930 -> 127.0.0.1:42866 [AP] -* CAPABILITY IMAP4rev1 IDLE NAMESPACE. -NCLJ1 OK CAPABILITY completed. - -## -T 127.0.0.1:42866 -> 127.0.0.1:9930 [AP] -NCLJ2 LOGIN user "pass". - -# -T 127.0.0.1:9930 -> 127.0.0.1:42866 [AP] -NCLJ2 OK LOGIN succeeded. - -## -T 127.0.0.1:42866 -> 127.0.0.1:9930 [AP] -NCLJ3 CAPABILITY. - -# -T 127.0.0.1:9930 -> 127.0.0.1:42866 [AP] -* CAPABILITY IMAP4rev1 IDLE NAMESPACE. -NCLJ3 OK CAPABILITY completed. - -# -T 127.0.0.1:42866 -> 127.0.0.1:9930 [AP] -NCLJ4 LIST "" "". - -## -T 127.0.0.1:9930 -> 127.0.0.1:42866 [AP] -* LIST (\Seen \Answered \Flagged \Deleted \Draft \Recent List) "/" "INBOX". -NCLJ4 OK LIST completed. - -# -T 127.0.0.1:42866 -> 127.0.0.1:9930 [AP] -NCLJ5 LIST "" "*". - -## -T 127.0.0.1:9930 -> 127.0.0.1:42866 [AP] -* LIST (\Seen \Answered \Flagged \Deleted \Draft \Recent List) "/" "INBOX". -NCLJ5 OK LIST completed. - -# -T 127.0.0.1:42866 -> 127.0.0.1:9930 [AP] -NCLJ6 SELECT INBOX. - -# -T 127.0.0.1:9930 -> 127.0.0.1:42866 [AP] -* 0 EXISTS. -* 3 RECENT. -* FLAGS (\Seen \Answered \Flagged \Deleted \Draft \Recent List). -* OK [UIDVALIDITY 42]. -NCLJ6 OK [READ-WRITE] SELECT successful. - -# -T 127.0.0.1:42866 -> 127.0.0.1:9930 [AP] -NCLJ7 EXAMINE INBOX. - -## -T 127.0.0.1:9930 -> 127.0.0.1:42866 [AP] -* 0 EXISTS. -* 3 RECENT. -* FLAGS (\Seen \Answered \Flagged \Deleted \Draft \Recent List). -* OK [UIDVALIDITY 42]. -NCLJ7 OK [READ-ONLY] EXAMINE successful. - -# -T 127.0.0.1:42866 -> 127.0.0.1:9930 [AP] -NCLJ8 LOGOUT. - -## -T 127.0.0.1:9930 -> 127.0.0.1:42866 [AP] -* BYE Nice talking to you. -NCLJ8 OK LOGOUT successful. - - diff --git a/mail/src/leap/mail/imap/service/rfc822.message b/mail/src/leap/mail/imap/service/rfc822.message deleted file mode 100644 index ee97ab9..0000000 --- a/mail/src/leap/mail/imap/service/rfc822.message +++ /dev/null @@ -1,86 +0,0 @@ -Return-Path: -Delivered-To: exarkun@meson.dyndns.org -Received: from localhost [127.0.0.1] - by localhost with POP3 (fetchmail-6.2.1) - for exarkun@localhost (single-drop); Thu, 20 Mar 2003 14:50:20 -0500 (EST) -Received: from pyramid.twistedmatrix.com (adsl-64-123-27-105.dsl.austtx.swbell.net [64.123.27.105]) - by intarweb.us (Postfix) with ESMTP id 4A4A513EA4 - for ; Thu, 20 Mar 2003 14:49:27 -0500 (EST) -Received: from localhost ([127.0.0.1] helo=pyramid.twistedmatrix.com) - by pyramid.twistedmatrix.com with esmtp (Exim 3.35 #1 (Debian)) - id 18w648-0007Vl-00; Thu, 20 Mar 2003 13:51:04 -0600 -Received: from acapnotic by pyramid.twistedmatrix.com with local (Exim 3.35 #1 (Debian)) - id 18w63j-0007VK-00 - for ; Thu, 20 Mar 2003 13:50:39 -0600 -To: twisted-commits@twistedmatrix.com -From: etrepum CVS -Reply-To: twisted-python@twistedmatrix.com -X-Mailer: CVSToys -Message-Id: -Subject: [Twisted-commits] rebuild now works on python versions from 2.2.0 and up. -Sender: twisted-commits-admin@twistedmatrix.com -Errors-To: twisted-commits-admin@twistedmatrix.com -X-BeenThere: twisted-commits@twistedmatrix.com -X-Mailman-Version: 2.0.11 -Precedence: bulk -List-Help: -List-Post: -List-Subscribe: , - -List-Id: -List-Unsubscribe: , - -List-Archive: -Date: Thu, 20 Mar 2003 13:50:39 -0600 - -Modified files: -Twisted/twisted/python/rebuild.py 1.19 1.20 - -Log message: -rebuild now works on python versions from 2.2.0 and up. - - -ViewCVS links: -http://twistedmatrix.com/users/jh.twistd/viewcvs/cgi/viewcvs.cgi/twisted/python/rebuild.py.diff?r1=text&tr1=1.19&r2=text&tr2=1.20&cvsroot=Twisted - -Index: Twisted/twisted/python/rebuild.py -diff -u Twisted/twisted/python/rebuild.py:1.19 Twisted/twisted/python/rebuild.py:1.20 ---- Twisted/twisted/python/rebuild.py:1.19 Fri Jan 17 13:50:49 2003 -+++ Twisted/twisted/python/rebuild.py Thu Mar 20 11:50:08 2003 -@@ -206,15 +206,27 @@ - clazz.__dict__.clear() - clazz.__getattr__ = __getattr__ - clazz.__module__ = module.__name__ -+ if newclasses: -+ import gc -+ if (2, 2, 0) <= sys.version_info[:3] < (2, 2, 2): -+ hasBrokenRebuild = 1 -+ gc_objects = gc.get_objects() -+ else: -+ hasBrokenRebuild = 0 - for nclass in newclasses: - ga = getattr(module, nclass.__name__) - if ga is nclass: - log.msg("WARNING: new-class %s not replaced by reload!" % reflect.qual(nclass)) - else: -- import gc -- for r in gc.get_referrers(nclass): -- if isinstance(r, nclass): -+ if hasBrokenRebuild: -+ for r in gc_objects: -+ if not getattr(r, '__class__', None) is nclass: -+ continue - r.__class__ = ga -+ else: -+ for r in gc.get_referrers(nclass): -+ if getattr(r, '__class__', None) is nclass: -+ r.__class__ = ga - if doLog: - log.msg('') - log.msg(' (fixing %s): ' % str(module.__name__)) - - -_______________________________________________ -Twisted-commits mailing list -Twisted-commits@twistedmatrix.com -http://twistedmatrix.com/cgi-bin/mailman/listinfo/twisted-commits diff --git a/mail/src/leap/mail/imap/tests/.gitignore b/mail/src/leap/mail/imap/tests/.gitignore deleted file mode 100644 index 60baa9c..0000000 --- a/mail/src/leap/mail/imap/tests/.gitignore +++ /dev/null @@ -1 +0,0 @@ -data/* diff --git a/mail/src/leap/mail/imap/tests/getmail b/mail/src/leap/mail/imap/tests/getmail deleted file mode 100755 index dd3fa0b..0000000 --- a/mail/src/leap/mail/imap/tests/getmail +++ /dev/null @@ -1,344 +0,0 @@ -#!/usr/bin/env python - -# Copyright (c) Twisted Matrix Laboratories. -# See LICENSE in twisted for details. - -# Modifications by LEAP Developers 2014 to fit -# Bitmask configuration settings. -""" -Simple IMAP4 client which displays the subjects of all messages in a -particular mailbox. -""" - -import os -import sys - -from twisted.internet import protocol -from twisted.internet import ssl -from twisted.internet import defer -from twisted.internet import stdio -from twisted.mail import imap4 -from twisted.protocols import basic -from twisted.python import log - -# Global options stored here from main -_opts = {} - - -class TrivialPrompter(basic.LineReceiver): - from os import linesep as delimiter - - promptDeferred = None - - def prompt(self, msg): - assert self.promptDeferred is None - self.display(msg) - self.promptDeferred = defer.Deferred() - return self.promptDeferred - - def display(self, msg): - self.transport.write(msg) - - def lineReceived(self, line): - if self.promptDeferred is None: - return - d, self.promptDeferred = self.promptDeferred, None - d.callback(line) - - -class SimpleIMAP4Client(imap4.IMAP4Client): - """ - A client with callbacks for greeting messages from an IMAP server. - """ - greetDeferred = None - - def serverGreeting(self, caps): - self.serverCapabilities = caps - if self.greetDeferred is not None: - d, self.greetDeferred = self.greetDeferred, None - d.callback(self) - - -class SimpleIMAP4ClientFactory(protocol.ClientFactory): - usedUp = False - - protocol = SimpleIMAP4Client - - def __init__(self, username, onConn): - self.ctx = ssl.ClientContextFactory() - - self.username = username - self.onConn = onConn - - def buildProtocol(self, addr): - """ - Initiate the protocol instance. Since we are building a simple IMAP - client, we don't bother checking what capabilities the server has. We - just add all the authenticators twisted.mail has. - """ - assert not self.usedUp - self.usedUp = True - - p = self.protocol(self.ctx) - p.factory = self - p.greetDeferred = self.onConn - - p.registerAuthenticator(imap4.PLAINAuthenticator(self.username)) - p.registerAuthenticator(imap4.LOGINAuthenticator(self.username)) - p.registerAuthenticator( - imap4.CramMD5ClientAuthenticator(self.username)) - - return p - - def clientConnectionFailed(self, connector, reason): - d, self.onConn = self.onConn, None - d.errback(reason) - - -def cbServerGreeting(proto, username, password): - """ - Initial callback - invoked after the server sends us its greet message. - """ - # Hook up stdio - tp = TrivialPrompter() - stdio.StandardIO(tp) - - # And make it easily accessible - proto.prompt = tp.prompt - proto.display = tp.display - - # Try to authenticate securely - return proto.authenticate( - password).addCallback( - cbAuthentication, - proto).addErrback( - ebAuthentication, proto, username, password - ) - - -def ebConnection(reason): - """ - Fallback error-handler. If anything goes wrong, log it and quit. - """ - log.startLogging(sys.stdout) - log.err(reason) - return reason - - -def cbAuthentication(result, proto): - """ - Callback after authentication has succeeded. - - Lists a bunch of mailboxes. - """ - return proto.list("", "*" - ).addCallback(cbMailboxList, proto - ) - - -def ebAuthentication(failure, proto, username, password): - """ - Errback invoked when authentication fails. - - If it failed because no SASL mechanisms match, offer the user the choice - of logging in insecurely. - - If you are trying to connect to your Gmail account, you will be here! - """ - failure.trap(imap4.NoSupportedAuthentication) - return InsecureLogin(proto, username, password) - - -def InsecureLogin(proto, username, password): - """ - insecure-login. - """ - return proto.login(username, password - ).addCallback(cbAuthentication, proto - ) - - -def cbMailboxList(result, proto): - """ - Callback invoked when a list of mailboxes has been retrieved. - If we have a selected mailbox in the global options, we directly pick it. - Otherwise, we offer a prompt to let user choose one. - """ - all_mbox_list = [e[2] for e in result] - s = '\n'.join(['%d. %s' % (n + 1, m) for (n, m) in zip(range(len(all_mbox_list)), all_mbox_list)]) - if not s: - return defer.fail(Exception("No mailboxes exist on server!")) - - selected_mailbox = _opts.get('mailbox') - - if not selected_mailbox: - return proto.prompt(s + "\nWhich mailbox? [1] " - ).addCallback(cbPickMailbox, proto, all_mbox_list - ) - else: - mboxes_lower = map(lambda s: s.lower(), all_mbox_list) - index = mboxes_lower.index(selected_mailbox.lower()) + 1 - return cbPickMailbox(index, proto, all_mbox_list) - - -def cbPickMailbox(result, proto, mboxes): - """ - When the user selects a mailbox, "examine" it. - """ - mbox = mboxes[int(result or '1') - 1] - return proto.examine(mbox - ).addCallback(cbExamineMbox, proto - ) - - -def cbExamineMbox(result, proto): - """ - Callback invoked when examine command completes. - - Retrieve the subject header of every message in the mailbox. - """ - return proto.fetchSpecific('1:*', - headerType='HEADER.FIELDS', - headerArgs=['SUBJECT'], - ).addCallback(cbFetch, proto, - ) - - -def cbFetch(result, proto): - """ - Display a listing of the messages in the mailbox, based on the collected - headers. - """ - selected_subject = _opts.get('subject', None) - index = None - - if result: - keys = result.keys() - keys.sort() - - if selected_subject: - for k in keys: - # remove 'Subject: ' preffix plus eol - subject = result[k][0][2][9:].rstrip('\r\n') - if subject.lower() == selected_subject.lower(): - index = k - break - else: - for k in keys: - proto.display('%s %s' % (k, result[k][0][2])) - else: - print "Hey, an empty mailbox!" - - if not index: - return proto.prompt("\nWhich message? [1] (Q quits) " - ).addCallback(cbPickMessage, proto) - else: - return cbPickMessage(index, proto) - - -def cbPickMessage(result, proto): - """ - Pick a message. - """ - if result == "Q": - print "Bye!" - return proto.logout() - - return proto.fetchSpecific( - '%s' % result, - headerType='', - headerArgs=['BODY.PEEK[]'], - ).addCallback(cbShowmessage, proto) - - -def cbShowmessage(result, proto): - """ - Display message. - """ - if result: - keys = result.keys() - keys.sort() - for k in keys: - proto.display('%s %s' % (k, result[k][0][2])) - else: - print "Hey, an empty message!" - - return proto.logout() - - -def cbClose(result): - """ - Close the connection when we finish everything. - """ - from twisted.internet import reactor - reactor.stop() - - -def main(): - import argparse - import ConfigParser - import sys - from twisted.internet import reactor - - description = ( - 'Get messages from a LEAP IMAP Proxy.\nThis is a ' - 'debugging tool, do not use this to retrieve any sensitive ' - 'information, or we will send ninjas to your house!') - epilog = ( - 'In case you want to automate the usage of this utility ' - 'you can place your credentials in a file pointed by ' - 'BITMASK_CREDENTIALS. You need to have a [Credentials] ' - 'section, with username= and password fields') - - parser = argparse.ArgumentParser(description=description, epilog=epilog) - credentials = os.environ.get('BITMASK_CREDENTIALS') - - if credentials: - try: - config = ConfigParser.ConfigParser() - config.read(credentials) - username = config.get('Credentials', 'username') - password = config.get('Credentials', 'password') - except Exception, e: - print "Error reading credentials file: {0}".format(e) - sys.exit() - else: - parser.add_argument('username', type=str) - parser.add_argument('password', type=str) - - parser.add_argument('--mailbox', dest='mailbox', default=None, - help='Which mailbox to retrieve. Empty for interactive prompt.') - parser.add_argument('--subject', dest='subject', default=None, - help='A subject for retrieve a mail that matches. Empty for interactive prompt.') - - ns = parser.parse_args() - - if not credentials: - username = ns.username - password = ns.password - - _opts['mailbox'] = ns.mailbox - _opts['subject'] = ns.subject - - hostname = "localhost" - port = "1984" - - onConn = defer.Deferred( - ).addCallback(cbServerGreeting, username, password - ).addErrback(ebConnection - ).addBoth(cbClose) - - factory = SimpleIMAP4ClientFactory(username, onConn) - - if port == '993': - reactor.connectSSL( - hostname, int(port), factory, ssl.ClientContextFactory()) - else: - if not port: - port = 143 - reactor.connectTCP(hostname, int(port), factory) - reactor.run() - - -if __name__ == '__main__': - main() diff --git a/mail/src/leap/mail/imap/tests/imapclient.py b/mail/src/leap/mail/imap/tests/imapclient.py deleted file mode 100755 index c353cee..0000000 --- a/mail/src/leap/mail/imap/tests/imapclient.py +++ /dev/null @@ -1,207 +0,0 @@ -#!/usr/bin/env python - -# Copyright (c) Twisted Matrix Laboratories. -# See LICENSE for details. - - -""" -Simple IMAP4 client which connects to our custome -IMAP4 server: imapserver.py. -""" - -import sys - -from twisted.internet import protocol -from twisted.internet import defer -from twisted.internet import stdio -from twisted.mail import imap4 -from twisted.protocols import basic -from twisted.python import util -from twisted.python import log - - -class TrivialPrompter(basic.LineReceiver): - # from os import linesep as delimiter - - promptDeferred = None - - def prompt(self, msg): - assert self.promptDeferred is None - self.display(msg) - self.promptDeferred = defer.Deferred() - return self.promptDeferred - - def display(self, msg): - self.transport.write(msg) - - def lineReceived(self, line): - if self.promptDeferred is None: - return - d, self.promptDeferred = self.promptDeferred, None - d.callback(line) - - -class SimpleIMAP4Client(imap4.IMAP4Client): - - """ - Add callbacks when the client receives greeting messages from - an IMAP server. - """ - greetDeferred = None - - def serverGreeting(self, caps): - self.serverCapabilities = caps - if self.greetDeferred is not None: - d, self.greetDeferred = self.greetDeferred, None - d.callback(self) - - -class SimpleIMAP4ClientFactory(protocol.ClientFactory): - usedUp = False - protocol = SimpleIMAP4Client - - def __init__(self, username, onConn): - self.username = username - self.onConn = onConn - - def buildProtocol(self, addr): - assert not self.usedUp - self.usedUp = True - - p = self.protocol() - p.factory = self - p.greetDeferred = self.onConn - - p.registerAuthenticator(imap4.PLAINAuthenticator(self.username)) - p.registerAuthenticator(imap4.LOGINAuthenticator(self.username)) - p.registerAuthenticator( - imap4.CramMD5ClientAuthenticator(self.username)) - - return p - - def clientConnectionFailed(self, connector, reason): - d, self.onConn = self.onConn, None - d.errback(reason) - - -def cbServerGreeting(proto, username, password): - """ - Initial callback - invoked after the server sends us its greet message. - """ - # Hook up stdio - tp = TrivialPrompter() - stdio.StandardIO(tp) - - # And make it easily accessible - proto.prompt = tp.prompt - proto.display = tp.display - - # Try to authenticate securely - return proto.authenticate( - password).addCallback( - cbAuthentication, proto).addErrback( - ebAuthentication, proto, username, password) - - -def ebConnection(reason): - """ - Fallback error-handler. If anything goes wrong, log it and quit. - """ - log.startLogging(sys.stdout) - log.err(reason) - return reason - - -def cbAuthentication(result, proto): - """ - Callback after authentication has succeeded. - List a bunch of mailboxes. - """ - return proto.list("", "*" - ).addCallback(cbMailboxList, proto - ) - - -def ebAuthentication(failure, proto, username, password): - """ - Errback invoked when authentication fails. - If it failed because no SASL mechanisms match, offer the user the choice - of logging in insecurely. - If you are trying to connect to your Gmail account, you will be here! - """ - failure.trap(imap4.NoSupportedAuthentication) - return proto.prompt( - "No secure authentication available. Login insecurely? (y/N) " - ).addCallback(cbInsecureLogin, proto, username, password - ) - - -def cbInsecureLogin(result, proto, username, password): - """ - Callback for "insecure-login" prompt. - """ - if result.lower() == "y": - # If they said yes, do it. - return proto.login(username, password - ).addCallback(cbAuthentication, proto - ) - return defer.fail(Exception("Login failed for security reasons.")) - - -def cbMailboxList(result, proto): - """ - Callback invoked when a list of mailboxes has been retrieved. - """ - result = [e[2] for e in result] - s = '\n'.join( - ['%d. %s' % (n + 1, m) for (n, m) in zip(range(len(result)), result)]) - if not s: - return defer.fail(Exception("No mailboxes exist on server!")) - return proto.prompt(s + "\nWhich mailbox? [1] " - ).addCallback(cbPickMailbox, proto, result - ) - - -def cbPickMailbox(result, proto, mboxes): - """ - When the user selects a mailbox, "examine" it. - """ - mbox = mboxes[int(result or '1') - 1] - return proto.status(mbox, 'MESSAGES', 'UNSEEN' - ).addCallback(cbMboxStatus, proto) - - -def cbMboxStatus(result, proto): - print "You have %s messages (%s unseen)!" % ( - result['MESSAGES'], result['UNSEEN']) - return proto.logout() - - -def cbClose(result): - """ - Close the connection when we finish everything. - """ - from twisted.internet import reactor - reactor.stop() - - -def main(): - hostname = raw_input('IMAP4 Server Hostname: ') - port = raw_input('IMAP4 Server Port (the default is 143): ') - username = raw_input('IMAP4 Username: ') - password = util.getPassword('IMAP4 Password: ') - - onConn = defer.Deferred( - ).addCallback(cbServerGreeting, username, password - ).addErrback(ebConnection - ).addBoth(cbClose) - - factory = SimpleIMAP4ClientFactory(username, onConn) - - from twisted.internet import reactor - conn = reactor.connectTCP(hostname, int(port), factory) - reactor.run() - - -if __name__ == '__main__': - main() diff --git a/mail/src/leap/mail/imap/tests/regressions_mime_struct b/mail/src/leap/mail/imap/tests/regressions_mime_struct deleted file mode 100755 index 0332664..0000000 --- a/mail/src/leap/mail/imap/tests/regressions_mime_struct +++ /dev/null @@ -1,461 +0,0 @@ -#!/usr/bin/env python - -# -*- coding: utf-8 -*- -# regression_mime_struct -# Copyright (C) 2014 LEAP -# Copyright (c) Twisted Matrix Laboratories. -# -# This program is free software: you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation, either version 3 of the License, or -# (at your option) any later version. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with this program. If not, see . -""" -Simple Regression Tests for checking MIME struct handling using IMAP4 client. - -Iterates trough all mails under a given folder and tries to APPEND them to -the server being tested. After FETCHING the pushed message, it compares -the received version with the one that was saved, and exits with an error -code if they do not match. -""" -import os -import StringIO -import sys - -from email.parser import Parser - -from twisted.internet import protocol -from twisted.internet import ssl -from twisted.internet import defer -from twisted.internet import stdio -from twisted.mail import imap4 -from twisted.protocols import basic -from twisted.python import log - - -REGRESSIONS_FOLDER = os.environ.get( - "REGRESSIONS_FOLDER", "regressions_test") -print "[+] Using regressions folder:", REGRESSIONS_FOLDER - -parser = Parser() - - -def get_msg_parts(raw): - """ - Return a representation of the parts of a message suitable for - comparison. - - :param raw: string for the message - :type raw: str - """ - m = parser.parsestr(raw) - return [dict(part.items()) - if part.is_multipart() - else part.get_payload() - for part in m.walk()] - - -def compare_msg_parts(a, b): - """ - Compare two sequences of parts of messages. - - :param a: part sequence for message a - :param b: part sequence for message b - - :return: True if both message sequences are equivalent. - :rtype: bool - """ - # XXX This could be smarter and show the differences in the - # different parts when/where they differ. - #import pprint; pprint.pprint(a[0]) - #import pprint; pprint.pprint(b[0]) - - def lowerkey(d): - return dict((k.lower(), v.replace('\r', '')) - for k, v in d.iteritems()) - - def eq(x, y): - # For dicts, we compare a variation with their keys - # in lowercase, and \r removed from their values - if all(map(lambda i: isinstance(i, dict), (x, y))): - x, y = map(lowerkey, (x, y)) - return x == y - - compare_vector = map(lambda tup: eq(tup[0], tup[1]), zip(a, b)) - all_match = all(compare_vector) - - if not all_match: - print "PARTS MISMATCH!" - print "vector: ", compare_vector - index = compare_vector.index(False) - from pprint import pprint - print "Expected:" - pprint(a[index]) - print ("***") - print "Found:" - pprint(b[index]) - print - - return all_match - - -def get_fd(string): - """ - Return a file descriptor with the passed string - as content. - """ - fd = StringIO.StringIO() - fd.write(string) - fd.seek(0) - return fd - - -class TrivialPrompter(basic.LineReceiver): - promptDeferred = None - - def prompt(self, msg): - assert self.promptDeferred is None - self.display(msg) - self.promptDeferred = defer.Deferred() - return self.promptDeferred - - def display(self, msg): - self.transport.write(msg) - - def lineReceived(self, line): - if self.promptDeferred is None: - return - d, self.promptDeferred = self.promptDeferred, None - d.callback(line) - - -class SimpleIMAP4Client(imap4.IMAP4Client): - """ - A client with callbacks for greeting messages from an IMAP server. - """ - greetDeferred = None - - def serverGreeting(self, caps): - self.serverCapabilities = caps - if self.greetDeferred is not None: - d, self.greetDeferred = self.greetDeferred, None - d.callback(self) - - -class SimpleIMAP4ClientFactory(protocol.ClientFactory): - usedUp = False - protocol = SimpleIMAP4Client - - def __init__(self, username, onConn): - self.ctx = ssl.ClientContextFactory() - - self.username = username - self.onConn = onConn - - def buildProtocol(self, addr): - """ - Initiate the protocol instance. Since we are building a simple IMAP - client, we don't bother checking what capabilities the server has. We - just add all the authenticators twisted.mail has. Note: Gmail no - longer uses any of the methods below, it's been using XOAUTH since - 2010. - """ - assert not self.usedUp - self.usedUp = True - - p = self.protocol(self.ctx) - p.factory = self - p.greetDeferred = self.onConn - - p.registerAuthenticator(imap4.PLAINAuthenticator(self.username)) - p.registerAuthenticator(imap4.LOGINAuthenticator(self.username)) - p.registerAuthenticator( - imap4.CramMD5ClientAuthenticator(self.username)) - - return p - - def clientConnectionFailed(self, connector, reason): - d, self.onConn = self.onConn, None - d.errback(reason) - - -def cbServerGreeting(proto, username, password): - """ - Initial callback - invoked after the server sends us its greet message. - """ - # Hook up stdio - tp = TrivialPrompter() - stdio.StandardIO(tp) - - # And make it easily accessible - proto.prompt = tp.prompt - proto.display = tp.display - - # Try to authenticate securely - return proto.authenticate( - password).addCallback( - cbAuthentication, - proto).addErrback( - ebAuthentication, proto, username, password - ) - - -def ebConnection(reason): - """ - Fallback error-handler. If anything goes wrong, log it and quit. - """ - log.startLogging(sys.stdout) - log.err(reason) - return reason - - -def cbAuthentication(result, proto): - """ - Callback after authentication has succeeded. - - Lists a bunch of mailboxes. - """ - return proto.select( - REGRESSIONS_FOLDER - ).addCallback( - cbSelectMbox, proto - ).addErrback( - ebSelectMbox, proto, REGRESSIONS_FOLDER) - - -def ebAuthentication(failure, proto, username, password): - """ - Errback invoked when authentication fails. - - If it failed because no SASL mechanisms match, offer the user the choice - of logging in insecurely. - - If you are trying to connect to your Gmail account, you will be here! - """ - failure.trap(imap4.NoSupportedAuthentication) - return InsecureLogin(proto, username, password) - - -def InsecureLogin(proto, username, password): - """ - Raise insecure-login error. - """ - return proto.login( - username, password - ).addCallback( - cbAuthentication, proto) - - -def cbSelectMbox(result, proto): - """ - Callback invoked when select command finishes successfully. - - If any message is in the test folder, it will flag them as deleted and - expunge. - If no messages found, it will start with the APPEND tests. - """ - print "SELECT: %s EXISTS " % result.get("EXISTS", "??") - - if result["EXISTS"] != 0: - # Flag as deleted, expunge, and do an examine again. - print "There is mail here, will delete..." - return cbDeleteAndExpungeTestFolder(proto) - - else: - return cbAppendNextMessage(proto) - - -def ebSelectMbox(failure, proto, folder): - """ - Errback invoked when the examine command fails. - - Creates the folder. - """ - log.err(failure) - log.msg("Folder %r does not exist. Creating..." % (folder,)) - return proto.create(folder).addCallback(cbAuthentication, proto) - - -def ebExpunge(failure): - log.err(failure) - - -def cbDeleteAndExpungeTestFolder(proto): - """ - Callback invoked fom cbExamineMbox when the number of messages in the - mailbox is not zero. It flags all messages as deleted and expunge the - mailbox. - """ - return proto.setFlags( - "1:*", ("\\Deleted",) - ).addCallback( - lambda r: proto.expunge() - ).addCallback( - cbExpunge, proto - ).addErrback( - ebExpunge) - - -def cbExpunge(result, proto): - return proto.select( - REGRESSIONS_FOLDER - ).addCallback( - cbSelectMbox, proto - ).addErrback(ebSettingDeleted, proto) - - -def ebSettingDeleted(failure, proto): - """ - Report errors during deletion of messages in the mailbox. - """ - print failure.getTraceback() - - -def cbAppendNextMessage(proto): - """ - Appends the next message in the global queue to the test folder. - """ - # 1. Get the next test message from global tuple. - try: - next_sample = SAMPLES.pop() - except IndexError: - # we're done! - return proto.logout() - - print "\nAPPEND %s" % (next_sample,) - raw = open(next_sample).read() - msg = get_fd(raw) - return proto.append( - REGRESSIONS_FOLDER, msg - ).addCallback( - lambda r: proto.select(REGRESSIONS_FOLDER) - ).addCallback( - cbAppend, proto, raw - ).addErrback( - ebAppend, proto, raw) - - -def cbAppend(result, proto, orig_msg): - """ - Fetches the message right after an append. - """ - # XXX keep account of highest UID - uid = "1:*" - - return proto.fetchSpecific( - '%s' % uid, - headerType='', - headerArgs=['BODY.PEEK[]'], - ).addCallback( - cbCompareMessage, proto, orig_msg - ).addErrback(ebAppend, proto, orig_msg) - - -def ebAppend(failure, proto, raw): - """ - Errorback for the append operation - """ - print "ERROR WHILE APPENDING!" - print failure.getTraceback() - - -def cbPickMessage(result, proto): - """ - Pick a message. - """ - return proto.fetchSpecific( - '%s' % result, - headerType='', - headerArgs=['BODY.PEEK[]'], - ).addCallback(cbCompareMessage, proto) - - -def cbCompareMessage(result, proto, raw): - """ - Display message and compare it with the original one. - """ - parts_orig = get_msg_parts(raw) - - if result: - keys = result.keys() - keys.sort() - else: - print "[-] GOT NO RESULT" - return proto.logout() - - latest = max(keys) - - fetched_msg = result[latest][0][2] - parts_fetched = get_msg_parts(fetched_msg) - - equal = compare_msg_parts( - parts_orig, - parts_fetched) - - if equal: - print "[+] MESSAGES MATCH" - return cbAppendNextMessage(proto) - else: - print "[-] ERROR: MESSAGES DO NOT MATCH !!!" - print " ABORTING COMPARISON..." - # FIXME logout and print the subject ... - return proto.logout() - - -def cbClose(result): - """ - Close the connection when we finish everything. - """ - from twisted.internet import reactor - reactor.stop() - - -def main(): - import glob - import sys - - if len(sys.argv) != 4: - print "Usage: regressions " - sys.exit() - - hostname = "localhost" - port = "1984" - username = sys.argv[1] - password = sys.argv[2] - - samplesdir = sys.argv[3] - - if not os.path.isdir(samplesdir): - print ("Could not find samples folder! " - "Make sure of copying mail_breaker contents there.") - sys.exit() - - samples = glob.glob(samplesdir + '/*') - - global SAMPLES - SAMPLES = [] - SAMPLES += samples - - onConn = defer.Deferred( - ).addCallback( - cbServerGreeting, username, password - ).addErrback( - ebConnection - ).addBoth(cbClose) - - factory = SimpleIMAP4ClientFactory(username, onConn) - - from twisted.internet import reactor - reactor.connectTCP(hostname, int(port), factory) - reactor.run() - - -if __name__ == '__main__': - main() diff --git a/mail/src/leap/mail/imap/tests/rfc822.message b/mail/src/leap/mail/imap/tests/rfc822.message deleted file mode 120000 index b19cc28..0000000 --- a/mail/src/leap/mail/imap/tests/rfc822.message +++ /dev/null @@ -1 +0,0 @@ -../../tests/rfc822.message \ No newline at end of file diff --git a/mail/src/leap/mail/imap/tests/rfc822.multi-minimal.message b/mail/src/leap/mail/imap/tests/rfc822.multi-minimal.message deleted file mode 120000 index e0aa678..0000000 --- a/mail/src/leap/mail/imap/tests/rfc822.multi-minimal.message +++ /dev/null @@ -1 +0,0 @@ -../../tests/rfc822.multi-minimal.message \ No newline at end of file diff --git a/mail/src/leap/mail/imap/tests/rfc822.multi-nested.message b/mail/src/leap/mail/imap/tests/rfc822.multi-nested.message deleted file mode 120000 index 306d0de..0000000 --- a/mail/src/leap/mail/imap/tests/rfc822.multi-nested.message +++ /dev/null @@ -1 +0,0 @@ -../../tests/rfc822.multi-nested.message \ No newline at end of file diff --git a/mail/src/leap/mail/imap/tests/rfc822.multi-signed.message b/mail/src/leap/mail/imap/tests/rfc822.multi-signed.message deleted file mode 120000 index 4172244..0000000 --- a/mail/src/leap/mail/imap/tests/rfc822.multi-signed.message +++ /dev/null @@ -1 +0,0 @@ -../../tests/rfc822.multi-signed.message \ No newline at end of file diff --git a/mail/src/leap/mail/imap/tests/rfc822.multi.message b/mail/src/leap/mail/imap/tests/rfc822.multi.message deleted file mode 120000 index 62057d2..0000000 --- a/mail/src/leap/mail/imap/tests/rfc822.multi.message +++ /dev/null @@ -1 +0,0 @@ -../../tests/rfc822.multi.message \ No newline at end of file diff --git a/mail/src/leap/mail/imap/tests/rfc822.plain.message b/mail/src/leap/mail/imap/tests/rfc822.plain.message deleted file mode 120000 index 5bab0e8..0000000 --- a/mail/src/leap/mail/imap/tests/rfc822.plain.message +++ /dev/null @@ -1 +0,0 @@ -../../tests/rfc822.plain.message \ No newline at end of file diff --git a/mail/src/leap/mail/imap/tests/stress_tests_imap.zsh b/mail/src/leap/mail/imap/tests/stress_tests_imap.zsh deleted file mode 100755 index 544faca..0000000 --- a/mail/src/leap/mail/imap/tests/stress_tests_imap.zsh +++ /dev/null @@ -1,178 +0,0 @@ -#!/bin/zsh -# BATCH STRESS TEST FOR IMAP ---------------------- -# http://imgs.xkcd.com/comics/science.jpg -# -# Run imaptest against a LEAP IMAP server -# for a fixed period of time, and collect output. -# -# Author: Kali Kaneko -# Date: 2014 01 26 -# -# To run, you need to have `imaptest` in your path. -# See: -# http://www.imapwiki.org/ImapTest/Installation -# -# For the tests, I'm using a 10MB file sample that -# can be downloaded from: -# http://www.dovecot.org/tmp/dovecot-crlf -# -# Want to contribute to benchmarking? -# -# 1. Create a pristine account in a bitmask provider. -# -# 2. Launch your bitmask client, with different flags -# if you desire. -# -# For example to try the nosync flag in sqlite: -# -# LEAP_SQLITE_NOSYNC=1 bitmask --debug -N --offline -l /tmp/leap.log -# -# 3. Run at several points in time (ie: just after -# launching the bitmask client. one minute after, -# ten minutes after) -# -# mkdir data -# cd data -# ../leap_tests_imap.zsh | tee sqlite_nosync_run2.log -# -# 4. Submit your results to: kali at leap dot se -# together with the logs of the bitmask run. -# -# Please provide also details about your system, and -# the type of hard disk setup you are running against. -# - -# ------------------------------------------------ -# Edit these variables if you are too lazy to pass -# the user and mbox as parameters. Like me. - -USER="test_f14@dev.bitmask.net" -MBOX="~/leap/imaptest/data/dovecot-crlf" - -HOST="localhost" -PORT="1984" - -# in case you have it aliased -GREP="/bin/grep" -IMAPTEST="imaptest" - -# ----------------------------------------------- -# -# These should be kept constant across benchmarking -# runs across different machines, for comparability. - -DURATION=200 -NUM_MSG=200 - - -# TODO add another function, and a cli flag, to be able -# to take several aggretates spaced in time, along a period -# of several minutes. - -imaptest_cmd() { - stdbuf -o0 ${IMAPTEST} user=${USER} pass=1234 host=${HOST} \ - port=${PORT} mbox=${MBOX} clients=1 msgs=${NUM_MSG} \ - no_pipelining 2>/dev/null -} - -stress_imap() { - mkfifo imap_pipe - cat imap_pipe | tee output & - imaptest_cmd >> imap_pipe -} - -wait_and_kill() { - while : - do - sleep $DURATION - pkill -2 imaptest - rm imap_pipe - break - done -} - -print_results() { - sleep 1 - echo - echo - echo "AGGREGATED RESULTS" - echo "----------------------" - echo "\tavg\tstdev" - $GREP "avg" ./output | sed -e 's/^ *//g' -e 's/ *$//g' | \ - gawk ' -function avg(data, count) { - sum=0; - for( x=0; x <= count-1; x++) { - sum += data[x]; - } - return sum/count; -} -function std_dev(data, count) { - sum=0; - for( x=0; x <= count-1; x++) { - sum += data[x]; - } - average = sum/count; - - sumsq=0; - for( x=0; x <= count-1; x++) { - sumsq += (data[x] - average)^2; - } - return sqrt(sumsq/count); -} -BEGIN { - cnt = 0 -} END { - -printf("LOGI:\t%04.2lf\t%04.2f\n", avg(array[1], NR), std_dev(array[1], NR)); -printf("LIST:\t%04.2lf\t%04.2f\n", avg(array[2], NR), std_dev(array[2], NR)); -printf("STAT:\t%04.2lf\t%04.2f\n", avg(array[3], NR), std_dev(array[3], NR)); -printf("SELE:\t%04.2lf\t%04.2f\n", avg(array[4], NR), std_dev(array[4], NR)); -printf("FETC:\t%04.2lf\t%04.2f\n", avg(array[5], NR), std_dev(array[5], NR)); -printf("FET2:\t%04.2lf\t%04.2f\n", avg(array[6], NR), std_dev(array[6], NR)); -printf("STOR:\t%04.2lf\t%04.2f\n", avg(array[7], NR), std_dev(array[7], NR)); -printf("DELE:\t%04.2lf\t%04.2f\n", avg(array[8], NR), std_dev(array[8], NR)); -printf("EXPU:\t%04.2lf\t%04.2f\n", avg(array[9], NR), std_dev(array[9], NR)); -printf("APPE:\t%04.2lf\t%04.2f\n", avg(array[10], NR), std_dev(array[10], NR)); -printf("LOGO:\t%04.2lf\t%04.2f\n", avg(array[11], NR), std_dev(array[11], NR)); - -print "" -print "TOT samples", NR; -} -{ - it = cnt++; - array[1][it] = $1; - array[2][it] = $2; - array[3][it] = $3; - array[4][it] = $4; - array[5][it] = $5; - array[6][it] = $6; - array[7][it] = $7; - array[8][it] = $8; - array[9][it] = $9; - array[10][it] = $10; - array[11][it] = $11; -}' -} - - -{ test $1 = "--help" } && { - echo "Usage: $0 [user@provider] [/path/to/sample.mbox]" - exit 0 -} - -# If the first parameter is passed, take it as the user -{ test $1 } && { - USER=$1 -} - -# If the second parameter is passed, take it as the mbox -{ test $2 } && { - MBOX=$2 -} - -echo "[+] LEAP IMAP TESTS" -echo "[+] Running imaptest for $DURATION seconds with $NUM_MSG messages" -wait_and_kill & -stress_imap -print_results diff --git a/mail/src/leap/mail/imap/tests/test_imap.py b/mail/src/leap/mail/imap/tests/test_imap.py deleted file mode 100644 index 9cca17f..0000000 --- a/mail/src/leap/mail/imap/tests/test_imap.py +++ /dev/null @@ -1,1060 +0,0 @@ -# -*- coding: utf-8 -*- -# test_imap.py -# Copyright (C) 2013 LEAP -# -# This program is free software: you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation, either version 3 of the License, or -# (at your option) any later version. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with this program. If not, see . -""" -Test case for leap.email.imap.server -TestCases taken from twisted tests and modified to make them work -against our implementation of the IMAPAccount. - -@authors: Kali Kaneko, -XXX add authors from the original twisted tests. - -@license: GPLv3, see included LICENSE file -""" -# XXX review license of the original tests!!! -import os -import string -import types - - -from twisted.mail import imap4 -from twisted.internet import defer -from twisted.python import util -from twisted.python import failure - -from twisted import cred - -from leap.mail.imap.mailbox import IMAPMailbox -from leap.mail.imap.messages import CaseInsensitiveDict -from leap.mail.testing.imap import IMAP4HelperMixin - - -TEST_USER = "testuser@leap.se" -TEST_PASSWD = "1234" - - -def strip(f): - return lambda result, f=f: f() - - -def sortNest(l): - l = l[:] - l.sort() - for i in range(len(l)): - if isinstance(l[i], types.ListType): - l[i] = sortNest(l[i]) - elif isinstance(l[i], types.TupleType): - l[i] = tuple(sortNest(list(l[i]))) - return l - - -class TestRealm: - """ - A minimal auth realm for testing purposes only - """ - theAccount = None - - def requestAvatar(self, avatarId, mind, *interfaces): - return imap4.IAccount, self.theAccount, lambda: None - -# -# TestCases -# - -# DEBUG --- -# from twisted.internet.base import DelayedCall -# DelayedCall.debug = True - - -class LEAPIMAP4ServerTestCase(IMAP4HelperMixin): - - """ - Tests for the generic behavior of the LEAPIMAP4Server - which, right now, it's just implemented in this test file as - LEAPIMAPServer. We will move the implementation, together with - authentication bits, to leap.mail.imap.server so it can be instantiated - from the tac file. - - Right now this TestCase tries to mimmick as close as possible the - organization from the twisted.mail.imap tests so we can achieve - a complete implementation. The order in which they appear reflect - the intended order of implementation. - """ - - # - # mailboxes operations - # - - def testCreate(self): - """ - Test whether we can create mailboxes - """ - succeed = ('testbox', 'test/box', 'test/', 'test/box/box', 'foobox') - fail = ('testbox', 'test/box') - acc = self.server.theAccount - - def cb(): - self.result.append(1) - - def eb(failure): - self.result.append(0) - - def login(): - return self.client.login(TEST_USER, TEST_PASSWD) - - def create(): - create_deferreds = [] - for name in succeed + fail: - d = self.client.create(name) - d.addCallback(strip(cb)).addErrback(eb) - create_deferreds.append(d) - dd = defer.gatherResults(create_deferreds) - dd.addCallbacks(self._cbStopClient, self._ebGeneral) - return dd - - self.result = [] - d1 = self.connected.addCallback(strip(login)) - d1.addCallback(strip(create)) - d2 = self.loopback() - d = defer.gatherResults([d1, d2], consumeErrors=True) - d.addCallback(lambda _: acc.account.list_all_mailbox_names()) - return d.addCallback(self._cbTestCreate, succeed, fail) - - def _cbTestCreate(self, mailboxes, succeed, fail): - self.assertEqual(self.result, [1] * len(succeed) + [0] * len(fail)) - - answers = ([u'INBOX', u'testbox', u'test/box', u'test', - u'test/box/box', 'foobox']) - self.assertEqual(sorted(mailboxes), sorted([a for a in answers])) - - def testDelete(self): - """ - Test whether we can delete mailboxes - """ - def add_mailbox(): - return self.server.theAccount.addMailbox('test-delete/me') - - def login(): - return self.client.login(TEST_USER, TEST_PASSWD) - - def delete(): - return self.client.delete('test-delete/me') - - acc = self.server.theAccount.account - - d1 = self.connected.addCallback(add_mailbox) - d1.addCallback(strip(login)) - d1.addCallbacks(strip(delete), self._ebGeneral) - d1.addCallbacks(self._cbStopClient, self._ebGeneral) - d2 = self.loopback() - d = defer.gatherResults([d1, d2]) - d.addCallback(lambda _: acc.list_all_mailbox_names()) - d.addCallback(lambda mboxes: self.assertEqual( - mboxes, ['INBOX'])) - return d - - def testIllegalInboxDelete(self): - """ - Test what happens if we try to delete the user Inbox. - We expect that operation to fail. - """ - self.stashed = None - - def login(): - return self.client.login(TEST_USER, TEST_PASSWD) - - def delete(): - return self.client.delete('inbox') - - def stash(result): - self.stashed = result - - d1 = self.connected.addCallback(strip(login)) - d1.addCallbacks(strip(delete), self._ebGeneral) - d1.addBoth(stash) - d1.addCallbacks(self._cbStopClient, self._ebGeneral) - d2 = self.loopback() - d = defer.gatherResults([d1, d2]) - d.addCallback(lambda _: self.failUnless(isinstance(self.stashed, - failure.Failure))) - return d - - def testNonExistentDelete(self): - """ - Test what happens if we try to delete a non-existent mailbox. - We expect an error raised stating 'No such mailbox' - """ - def login(): - return self.client.login(TEST_USER, TEST_PASSWD) - - def delete(): - return self.client.delete('delete/me') - self.failure = failure - - def deleteFailed(failure): - self.failure = failure - - self.failure = None - d1 = self.connected.addCallback(strip(login)) - d1.addCallback(strip(delete)).addErrback(deleteFailed) - d1.addCallbacks(self._cbStopClient, self._ebGeneral) - d2 = self.loopback() - d = defer.gatherResults([d1, d2]) - d.addCallback(lambda _: self.assertTrue( - str(self.failure.value).startswith('No such mailbox'))) - return d - - def testIllegalDelete(self): - """ - Try deleting a mailbox with sub-folders, and \NoSelect flag set. - An exception is expected. - """ - acc = self.server.theAccount - - def login(): - return self.client.login(TEST_USER, TEST_PASSWD) - - def create_mailboxes(): - d1 = acc.addMailbox('delete') - d2 = acc.addMailbox('delete/me') - d = defer.gatherResults([d1, d2]) - return d - - def get_noselect_mailbox(mboxes): - mbox = mboxes[0] - return mbox.setFlags((r'\Noselect',)) - - def delete_mbox(ignored): - return self.client.delete('delete') - - def deleteFailed(failure): - self.failure = failure - - self.failure = None - - d1 = self.connected.addCallback(strip(login)) - d1.addCallback(strip(create_mailboxes)) - d1.addCallback(get_noselect_mailbox) - - d1.addCallback(delete_mbox).addErrback(deleteFailed) - d1.addCallbacks(self._cbStopClient, self._ebGeneral) - d2 = self.loopback() - d = defer.gatherResults([d1, d2]) - expected = ("Hierarchically inferior mailboxes exist " - "and \\Noselect is set") - d.addCallback(lambda _: - self.assertTrue(self.failure is not None)) - d.addCallback(lambda _: - self.assertEqual(str(self.failure.value), expected)) - return d - - # FIXME --- this test sometimes FAILS (timing issue). - # Some of the deferreds used in the rename op is not waiting for the - # operations properly - def testRename(self): - """ - Test whether we can rename a mailbox - """ - def create_mbox(): - return self.server.theAccount.addMailbox('oldmbox') - - def login(): - return self.client.login(TEST_USER, TEST_PASSWD) - - def rename(): - return self.client.rename('oldmbox', 'newname') - - d1 = self.connected.addCallback(strip(create_mbox)) - d1.addCallback(strip(login)) - d1.addCallbacks(strip(rename), self._ebGeneral) - d1.addCallbacks(self._cbStopClient, self._ebGeneral) - d2 = self.loopback() - d = defer.gatherResults([d1, d2]) - d.addCallback(lambda _: - self.server.theAccount.account.list_all_mailbox_names()) - d.addCallback(lambda mboxes: - self.assertItemsEqual(mboxes, ['INBOX', 'newname'])) - return d - - def testIllegalInboxRename(self): - """ - Try to rename inbox. We expect it to fail. Then it would be not - an inbox anymore, would it? - """ - self.stashed = None - - def login(): - return self.client.login(TEST_USER, TEST_PASSWD) - - def rename(): - return self.client.rename('inbox', 'frotz') - - def stash(stuff): - self.stashed = stuff - - d1 = self.connected.addCallback(strip(login)) - d1.addCallbacks(strip(rename), self._ebGeneral) - d1.addBoth(stash) - d1.addCallbacks(self._cbStopClient, self._ebGeneral) - d2 = self.loopback() - d = defer.gatherResults([d1, d2]) - d.addCallback(lambda _: - self.failUnless(isinstance( - self.stashed, failure.Failure))) - return d - - def testHierarchicalRename(self): - """ - Try to rename hierarchical mailboxes - """ - acc = self.server.theAccount - - def add_mailboxes(): - return defer.gatherResults([ - acc.addMailbox('oldmbox/m1'), - acc.addMailbox('oldmbox/m2')]) - - def login(): - return self.client.login(TEST_USER, TEST_PASSWD) - - def rename(): - return self.client.rename('oldmbox', 'newname') - - d1 = self.connected.addCallback(strip(add_mailboxes)) - d1.addCallback(strip(login)) - d1.addCallbacks(strip(rename), self._ebGeneral) - d1.addCallbacks(self._cbStopClient, self._ebGeneral) - d2 = self.loopback() - d = defer.gatherResults([d1, d2]) - d.addCallback(lambda _: acc.account.list_all_mailbox_names()) - return d.addCallback(self._cbTestHierarchicalRename) - - def _cbTestHierarchicalRename(self, mailboxes): - expected = ['INBOX', 'newname/m1', 'newname/m2'] - self.assertEqual(sorted(mailboxes), sorted([s for s in expected])) - - def testSubscribe(self): - """ - Test whether we can mark a mailbox as subscribed to - """ - acc = self.server.theAccount - - def add_mailbox(): - return acc.addMailbox('this/mbox') - - def login(): - return self.client.login(TEST_USER, TEST_PASSWD) - - def subscribe(): - return self.client.subscribe('this/mbox') - - def get_subscriptions(ignored): - return self.server.theAccount.getSubscriptions() - - d1 = self.connected.addCallback(strip(add_mailbox)) - d1.addCallback(strip(login)) - d1.addCallbacks(strip(subscribe), self._ebGeneral) - d1.addCallbacks(self._cbStopClient, self._ebGeneral) - d2 = self.loopback() - d = defer.gatherResults([d1, d2]) - d.addCallback(get_subscriptions) - d.addCallback(lambda subscriptions: - self.assertEqual(subscriptions, - ['this/mbox'])) - return d - - def testUnsubscribe(self): - """ - Test whether we can unsubscribe from a set of mailboxes - """ - acc = self.server.theAccount - - def add_mailboxes(): - return defer.gatherResults([ - acc.addMailbox('this/mbox'), - acc.addMailbox('that/mbox')]) - - def dc1(): - return acc.subscribe('this/mbox') - - def dc2(): - return acc.subscribe('that/mbox') - - def login(): - return self.client.login(TEST_USER, TEST_PASSWD) - - def unsubscribe(): - return self.client.unsubscribe('this/mbox') - - def get_subscriptions(ignored): - return acc.getSubscriptions() - - d1 = self.connected.addCallback(strip(add_mailboxes)) - d1.addCallback(strip(login)) - d1.addCallback(strip(dc1)) - d1.addCallback(strip(dc2)) - d1.addCallbacks(strip(unsubscribe), self._ebGeneral) - d1.addCallbacks(self._cbStopClient, self._ebGeneral) - d2 = self.loopback() - d = defer.gatherResults([d1, d2]) - d.addCallback(get_subscriptions) - d.addCallback(lambda subscriptions: - self.assertEqual(subscriptions, - ['that/mbox'])) - return d - - def testSelect(self): - """ - Try to select a mailbox - """ - mbox_name = "TESTMAILBOXSELECT" - self.selectedArgs = None - - acc = self.server.theAccount - - def add_mailbox(): - return acc.addMailbox(mbox_name, creation_ts=42) - - def login(): - return self.client.login(TEST_USER, TEST_PASSWD) - - def select(): - def selected(args): - self.selectedArgs = args - self._cbStopClient(None) - d = self.client.select(mbox_name) - d.addCallback(selected) - return d - - d1 = self.connected.addCallback(strip(add_mailbox)) - d1.addCallback(strip(login)) - d1.addCallback(strip(select)) - # d1.addErrback(self._ebGeneral) - - d2 = self.loopback() - - d = defer.gatherResults([d1, d2]) - d.addCallback(self._cbTestSelect) - return d - - def _cbTestSelect(self, ignored): - self.assertTrue(self.selectedArgs is not None) - - self.assertEqual(self.selectedArgs, { - 'EXISTS': 0, 'RECENT': 0, 'UIDVALIDITY': 42, - 'FLAGS': ('\\Seen', '\\Answered', '\\Flagged', - '\\Deleted', '\\Draft', '\\Recent', 'List'), - 'READ-WRITE': True - }) - - # - # capabilities - # - - def testCapability(self): - caps = {} - - def getCaps(): - def gotCaps(c): - caps.update(c) - self.server.transport.loseConnection() - return self.client.getCapabilities().addCallback(gotCaps) - - d1 = self.connected - d1.addCallback( - strip(getCaps)).addErrback(self._ebGeneral) - - d = defer.gatherResults([self.loopback(), d1]) - expected = {'IMAP4rev1': None, 'NAMESPACE': None, 'LITERAL+': None, - 'IDLE': None} - d.addCallback(lambda _: self.assertEqual(expected, caps)) - return d - - def testCapabilityWithAuth(self): - caps = {} - self.server.challengers[ - 'CRAM-MD5'] = cred.credentials.CramMD5Credentials - - def getCaps(): - def gotCaps(c): - caps.update(c) - self.server.transport.loseConnection() - return self.client.getCapabilities().addCallback(gotCaps) - d1 = self.connected.addCallback( - strip(getCaps)).addErrback(self._ebGeneral) - - d = defer.gatherResults([self.loopback(), d1]) - - expCap = {'IMAP4rev1': None, 'NAMESPACE': None, - 'IDLE': None, 'LITERAL+': None, - 'AUTH': ['CRAM-MD5']} - - d.addCallback(lambda _: self.assertEqual(expCap, caps)) - return d - - # - # authentication - # - - def testLogout(self): - """ - Test log out - """ - self.loggedOut = 0 - - def logout(): - def setLoggedOut(): - self.loggedOut = 1 - self.client.logout().addCallback(strip(setLoggedOut)) - self.connected.addCallback(strip(logout)).addErrback(self._ebGeneral) - d = self.loopback() - return d.addCallback(lambda _: self.assertEqual(self.loggedOut, 1)) - - def testNoop(self): - """ - Test noop command - """ - self.responses = None - - def noop(): - def setResponses(responses): - self.responses = responses - self.server.transport.loseConnection() - self.client.noop().addCallback(setResponses) - self.connected.addCallback(strip(noop)).addErrback(self._ebGeneral) - d = self.loopback() - return d.addCallback(lambda _: self.assertEqual(self.responses, [])) - - def testLogin(self): - """ - Test login - """ - def login(): - d = self.client.login(TEST_USER, TEST_PASSWD) - d.addCallback(self._cbStopClient) - d1 = self.connected.addCallback( - strip(login)).addErrback(self._ebGeneral) - d = defer.gatherResults([d1, self.loopback()]) - return d.addCallback(self._cbTestLogin) - - def _cbTestLogin(self, ignored): - self.assertEqual(self.server.state, 'auth') - - def testFailedLogin(self): - """ - Test bad login - """ - def login(): - d = self.client.login("bad_user@leap.se", TEST_PASSWD) - d.addBoth(self._cbStopClient) - - d1 = self.connected.addCallback( - strip(login)).addErrback(self._ebGeneral) - d2 = self.loopback() - d = defer.gatherResults([d1, d2]) - return d.addCallback(self._cbTestFailedLogin) - - def _cbTestFailedLogin(self, ignored): - self.assertEqual(self.server.state, 'unauth') - self.assertEqual(self.server.account, None) - - def testLoginRequiringQuoting(self): - """ - Test login requiring quoting - """ - self.server.checker.userid = '{test}user@leap.se' - self.server.checker.password = '{test}password' - - def login(): - d = self.client.login('{test}user@leap.se', '{test}password') - d.addBoth(self._cbStopClient) - - d1 = self.connected.addCallback( - strip(login)).addErrback(self._ebGeneral) - d = defer.gatherResults([self.loopback(), d1]) - return d.addCallback(self._cbTestLoginRequiringQuoting) - - def _cbTestLoginRequiringQuoting(self, ignored): - self.assertEqual(self.server.state, 'auth') - - # - # Inspection - # - - def testNamespace(self): - """ - Test retrieving namespace - """ - self.namespaceArgs = None - - def login(): - return self.client.login(TEST_USER, TEST_PASSWD) - - def namespace(): - def gotNamespace(args): - self.namespaceArgs = args - self._cbStopClient(None) - return self.client.namespace().addCallback(gotNamespace) - - d1 = self.connected.addCallback(strip(login)) - d1.addCallback(strip(namespace)) - d1.addErrback(self._ebGeneral) - d2 = self.loopback() - d = defer.gatherResults([d1, d2]) - d.addCallback(lambda _: self.assertEqual(self.namespaceArgs, - [[['', '/']], [], []])) - return d - - def testExamine(self): - """ - L{IMAP4Client.examine} issues an I{EXAMINE} command to the server and - returns a L{Deferred} which fires with a C{dict} with as many of the - following keys as the server includes in its response: C{'FLAGS'}, - C{'EXISTS'}, C{'RECENT'}, C{'UNSEEN'}, C{'READ-WRITE'}, C{'READ-ONLY'}, - C{'UIDVALIDITY'}, and C{'PERMANENTFLAGS'}. - - Unfortunately the server doesn't generate all of these so it's hard to - test the client's handling of them here. See - L{IMAP4ClientExamineTests} below. - - See U{RFC 3501}, section 6.3.2, - for details. - """ - # TODO implement the IMAP4ClientExamineTests testcase. - mbox_name = "test_mailbox_e" - acc = self.server.theAccount - self.examinedArgs = None - - def add_mailbox(): - return acc.addMailbox(mbox_name, creation_ts=42) - - def login(): - return self.client.login(TEST_USER, TEST_PASSWD) - - def examine(): - def examined(args): - self.examinedArgs = args - self._cbStopClient(None) - d = self.client.examine(mbox_name) - d.addCallback(examined) - return d - - d1 = self.connected.addCallback(strip(add_mailbox)) - d1.addCallback(strip(login)) - d1.addCallback(strip(examine)) - d1.addErrback(self._ebGeneral) - d2 = self.loopback() - d = defer.gatherResults([d1, d2]) - return d.addCallback(self._cbTestExamine) - - def _cbTestExamine(self, ignored): - self.assertEqual(self.examinedArgs, { - 'EXISTS': 0, 'RECENT': 0, 'UIDVALIDITY': 42, - 'FLAGS': ('\\Seen', '\\Answered', '\\Flagged', - '\\Deleted', '\\Draft', '\\Recent', 'List'), - 'READ-WRITE': False}) - - def _listSetup(self, f, f2=None): - - acc = self.server.theAccount - - def dc1(): - return acc.addMailbox('root_subthing', creation_ts=42) - - def dc2(): - return acc.addMailbox('root_another_thing', creation_ts=42) - - def dc3(): - return acc.addMailbox('non_root_subthing', creation_ts=42) - - def login(): - return self.client.login(TEST_USER, TEST_PASSWD) - - def listed(answers): - self.listed = answers - - self.listed = None - d1 = self.connected.addCallback(strip(login)) - d1.addCallback(strip(dc1)) - d1.addCallback(strip(dc2)) - d1.addCallback(strip(dc3)) - - if f2 is not None: - d1.addCallback(f2) - - d1.addCallbacks(strip(f), self._ebGeneral) - d1.addCallbacks(listed, self._ebGeneral) - d1.addCallbacks(self._cbStopClient, self._ebGeneral) - d2 = self.loopback() - return defer.gatherResults([d1, d2]).addCallback(lambda _: self.listed) - - def testList(self): - """ - Test List command - """ - def list(): - return self.client.list('root', '%') - - d = self._listSetup(list) - d.addCallback(lambda listed: self.assertEqual( - sortNest(listed), - sortNest([ - (IMAPMailbox.init_flags, "/", "root_subthing"), - (IMAPMailbox.init_flags, "/", "root_another_thing") - ]) - )) - return d - - def testLSub(self): - """ - Test LSub command - """ - acc = self.server.theAccount - - def subs_mailbox(): - # why not client.subscribe instead? - return acc.subscribe('root_subthing') - - def lsub(): - return self.client.lsub('root', '%') - - d = self._listSetup(lsub, strip(subs_mailbox)) - d.addCallback(self.assertEqual, - [(IMAPMailbox.init_flags, "/", "root_subthing")]) - return d - - def testStatus(self): - """ - Test Status command - """ - acc = self.server.theAccount - - def add_mailbox(): - return acc.addMailbox('root_subthings') - - # XXX FIXME ---- should populate this a little bit, - # with unseen etc... - - def login(): - return self.client.login(TEST_USER, TEST_PASSWD) - - def status(): - return self.client.status( - 'root_subthings', 'MESSAGES', 'UIDNEXT', 'UNSEEN') - - def statused(result): - self.statused = result - - self.statused = None - - d1 = self.connected.addCallback(strip(add_mailbox)) - d1.addCallback(strip(login)) - d1.addCallbacks(strip(status), self._ebGeneral) - d1.addCallbacks(statused, self._ebGeneral) - d1.addCallbacks(self._cbStopClient, self._ebGeneral) - d2 = self.loopback() - d = defer.gatherResults([d1, d2]) - d.addCallback(lambda _: self.assertEqual( - self.statused, - {'MESSAGES': 0, 'UIDNEXT': '1', 'UNSEEN': 0} - )) - return d - - def testFailedStatus(self): - """ - Test failed status command with a non-existent mailbox - """ - def login(): - return self.client.login(TEST_USER, TEST_PASSWD) - - def status(): - return self.client.status( - 'root/nonexistent', 'MESSAGES', 'UIDNEXT', 'UNSEEN') - - def statused(result): - self.statused = result - - def failed(failure): - self.failure = failure - - self.statused = self.failure = None - d1 = self.connected.addCallback(strip(login)) - d1.addCallbacks(strip(status), self._ebGeneral) - d1.addCallbacks(statused, failed) - d1.addCallbacks(self._cbStopClient, self._ebGeneral) - d2 = self.loopback() - return defer.gatherResults([d1, d2]).addCallback( - self._cbTestFailedStatus) - - def _cbTestFailedStatus(self, ignored): - self.assertEqual( - self.statused, None - ) - self.assertEqual( - self.failure.value.args, - ('Could not open mailbox',) - ) - - # - # messages - # - - def testFullAppend(self): - """ - Test appending a full message to the mailbox - """ - infile = util.sibpath(__file__, 'rfc822.message') - message = open(infile) - acc = self.server.theAccount - mailbox_name = "appendmbox/subthing" - - def add_mailbox(): - return acc.addMailbox(mailbox_name) - - def login(): - return self.client.login(TEST_USER, TEST_PASSWD) - - def append(): - return self.client.append( - mailbox_name, message, - ('\\SEEN', '\\DELETED'), - 'Tue, 17 Jun 2003 11:22:16 -0600 (MDT)', - ) - - d1 = self.connected.addCallback(strip(add_mailbox)) - d1.addCallback(strip(login)) - d1.addCallbacks(strip(append), self._ebGeneral) - d1.addCallbacks(self._cbStopClient, self._ebGeneral) - d2 = self.loopback() - d = defer.gatherResults([d1, d2]) - - d.addCallback(lambda _: acc.getMailbox(mailbox_name)) - d.addCallback(lambda mb: mb.fetch(imap4.MessageSet(start=1), True)) - return d.addCallback(self._cbTestFullAppend, infile) - - def _cbTestFullAppend(self, fetched, infile): - fetched = list(fetched) - self.assertTrue(len(fetched) == 1) - self.assertTrue(len(fetched[0]) == 2) - uid, msg = fetched[0] - parsed = self.parser.parse(open(infile)) - expected_body = parsed.get_payload() - expected_headers = CaseInsensitiveDict(parsed.items()) - - def assert_flags(flags): - self.assertEqual( - set(('\\SEEN', '\\DELETED')), - set(flags)) - - def assert_date(date): - self.assertEqual( - 'Tue, 17 Jun 2003 11:22:16 -0600 (MDT)', - date) - - def assert_body(body): - gotbody = body.read() - self.assertEqual(expected_body, gotbody) - - def assert_headers(headers): - self.assertItemsEqual(map(string.lower, expected_headers), headers) - - d = defer.maybeDeferred(msg.getFlags) - d.addCallback(assert_flags) - - d.addCallback(lambda _: defer.maybeDeferred(msg.getInternalDate)) - d.addCallback(assert_date) - - d.addCallback( - lambda _: defer.maybeDeferred( - msg.getBodyFile, self._soledad)) - d.addCallback(assert_body) - - d.addCallback(lambda _: defer.maybeDeferred(msg.getHeaders, True)) - d.addCallback(assert_headers) - - return d - - def testPartialAppend(self): - """ - Test partially appending a message to the mailbox - """ - # TODO this test sometimes will fail because of the notify_just_mdoc - infile = util.sibpath(__file__, 'rfc822.message') - - acc = self.server.theAccount - - def add_mailbox(): - return acc.addMailbox('PARTIAL/SUBTHING') - - def login(): - return self.client.login(TEST_USER, TEST_PASSWD) - - def append(): - message = file(infile) - return self.client.sendCommand( - imap4.Command( - 'APPEND', - 'PARTIAL/SUBTHING (\\SEEN) "Right now" ' - '{%d}' % os.path.getsize(infile), - (), self.client._IMAP4Client__cbContinueAppend, message - ) - ) - d1 = self.connected.addCallback(strip(add_mailbox)) - d1.addCallback(strip(login)) - d1.addCallbacks(strip(append), self._ebGeneral) - d1.addCallbacks(self._cbStopClient, self._ebGeneral) - d2 = self.loopback() - d = defer.gatherResults([d1, d2]) - - d.addCallback(lambda _: acc.getMailbox("PARTIAL/SUBTHING")) - d.addCallback(lambda mb: mb.fetch(imap4.MessageSet(start=1), True)) - return d.addCallback( - self._cbTestPartialAppend, infile) - - def _cbTestPartialAppend(self, fetched, infile): - fetched = list(fetched) - self.assertTrue(len(fetched) == 1) - self.assertTrue(len(fetched[0]) == 2) - uid, msg = fetched[0] - parsed = self.parser.parse(open(infile)) - expected_body = parsed.get_payload() - - def assert_flags(flags): - self.assertEqual( - set((['\\SEEN'])), set(flags)) - - def assert_body(body): - gotbody = body.read() - self.assertEqual(expected_body, gotbody) - - d = defer.maybeDeferred(msg.getFlags) - d.addCallback(assert_flags) - - d.addCallback(lambda _: defer.maybeDeferred(msg.getBodyFile)) - d.addCallback(assert_body) - return d - - def testCheck(self): - """ - Test check command - """ - def add_mailbox(): - return self.server.theAccount.addMailbox('root/subthing') - - def login(): - return self.client.login(TEST_USER, TEST_PASSWD) - - def select(): - return self.client.select('root/subthing') - - def check(): - return self.client.check() - - d = self.connected.addCallbacks( - strip(add_mailbox), self._ebGeneral) - d.addCallbacks(lambda _: login(), self._ebGeneral) - d.addCallbacks(strip(select), self._ebGeneral) - d.addCallbacks(strip(check), self._ebGeneral) - d.addCallbacks(self._cbStopClient, self._ebGeneral) - d2 = self.loopback() - return defer.gatherResults([d, d2]) - - # Okay, that was much fun indeed - - def testExpunge(self): - """ - Test expunge command - """ - acc = self.server.theAccount - mailbox_name = 'mailboxexpunge' - - def add_mailbox(): - return acc.addMailbox(mailbox_name) - - def login(): - return self.client.login(TEST_USER, TEST_PASSWD) - - def select(): - return self.client.select(mailbox_name) - - def save_mailbox(mailbox): - self.mailbox = mailbox - - def get_mailbox(): - d = acc.getMailbox(mailbox_name) - d.addCallback(save_mailbox) - return d - - def add_messages(): - d = self.mailbox.addMessage( - 'test 1', flags=('\\Deleted', 'AnotherFlag'), - notify_just_mdoc=False) - d.addCallback(lambda _: self.mailbox.addMessage( - 'test 2', flags=('AnotherFlag',), - notify_just_mdoc=False)) - d.addCallback(lambda _: self.mailbox.addMessage( - 'test 3', flags=('\\Deleted',), - notify_just_mdoc=False)) - return d - - def expunge(): - return self.client.expunge() - - def expunged(results): - self.failIf(self.server.mbox is None) - self.results = results - - self.results = None - d1 = self.connected.addCallback(strip(add_mailbox)) - d1.addCallback(strip(login)) - d1.addCallback(strip(get_mailbox)) - d1.addCallbacks(strip(add_messages), self._ebGeneral) - d1.addCallbacks(strip(select), self._ebGeneral) - d1.addCallbacks(strip(expunge), self._ebGeneral) - d1.addCallbacks(expunged, self._ebGeneral) - d1.addCallbacks(self._cbStopClient, self._ebGeneral) - d2 = self.loopback() - d = defer.gatherResults([d1, d2]) - d.addCallback(lambda _: self.mailbox.getMessageCount()) - return d.addCallback(self._cbTestExpunge) - - def _cbTestExpunge(self, count): - # we only left 1 mssage with no deleted flag - self.assertEqual(count, 1) - # the uids of the deleted messages - self.assertItemsEqual(self.results, [1, 3]) - - -class AccountTestCase(IMAP4HelperMixin): - """ - Test the Account. - """ - def _create_empty_mailbox(self): - return self.server.theAccount.addMailbox('') - - def _create_one_mailbox(self): - return self.server.theAccount.addMailbox('one') - - def test_illegalMailboxCreate(self): - self.assertRaises(AssertionError, self._create_empty_mailbox) - - -class IMAP4ServerSearchTestCase(IMAP4HelperMixin): - """ - Tests for the behavior of the search_* functions in L{imap5.IMAP4Server}. - """ - # XXX coming soon to your screens! - pass diff --git a/mail/src/leap/mail/imap/tests/walktree.py b/mail/src/leap/mail/imap/tests/walktree.py deleted file mode 100644 index f259a55..0000000 --- a/mail/src/leap/mail/imap/tests/walktree.py +++ /dev/null @@ -1,127 +0,0 @@ -# -*- coding: utf-8 -*- -# walktree.py -# Copyright (C) 2013 LEAP -# -# This program is free software: you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation, either version 3 of the License, or -# (at your option) any later version. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with this program. If not, see . -""" -Tests for the walktree module. -""" -import os -import sys -import pprint -from email import parser - -from leap.mail import walk as W - -DEBUG = os.environ.get("BITMASK_MAIL_DEBUG") - - -p = parser.Parser() - -# TODO pass an argument of the type of message - -################################################## -# Input from hell - -if len(sys.argv) > 1: - FILENAME = sys.argv[1] -else: - FILENAME = "rfc822.multi-signed.message" - -""" -FILENAME = "rfc822.plain.message" -FILENAME = "rfc822.multi-minimal.message" -""" - -msg = p.parse(open(FILENAME)) -DO_CHECK = False -################################################# - -parts = W.get_parts(msg) - -if DEBUG: - def trim(item): - item = item[:10] - [trim(part["phash"]) for part in parts if part.get('phash', None)] - -raw_docs = list(W.get_raw_docs(msg, parts)) - -body_phash_fun = [W.get_body_phash_simple, - W.get_body_phash_multi][int(msg.is_multipart())] -body_phash = body_phash_fun(W.get_payloads(msg)) -parts_map = W.walk_msg_tree(parts, body_phash=body_phash) - - -# TODO add missing headers! -expected = { - 'body': '1ddfa80485', - 'multi': True, - 'part_map': { - 1: { - 'headers': {'Content-Disposition': 'inline', - 'Content-Type': 'multipart/mixed; ' - 'boundary="z0eOaCaDLjvTGF2l"'}, - 'multi': True, - 'part_map': {1: {'ctype': 'text/plain', - 'headers': [ - ('Content-Type', - 'text/plain; charset=utf-8'), - ('Content-Disposition', - 'inline'), - ('Content-Transfer-Encoding', - 'quoted-printable')], - 'multi': False, - 'parts': 1, - 'phash': '1ddfa80485', - 'size': 206}, - 2: {'ctype': 'text/plain', - 'headers': [('Content-Type', - 'text/plain; charset=us-ascii'), - ('Content-Disposition', - 'attachment; ' - 'filename="attach.txt"')], - 'multi': False, - 'parts': 1, - 'phash': '7a94e4d769', - 'size': 133}, - 3: {'ctype': 'application/octet-stream', - 'headers': [('Content-Type', - 'application/octet-stream'), - ('Content-Disposition', - 'attachment; filename="hack.ico"'), - ('Content-Transfer-Encoding', - 'base64')], - 'multi': False, - 'parts': 1, - 'phash': 'c42cccebbd', - 'size': 12736}}}, - 2: {'ctype': 'application/pgp-signature', - 'headers': [('Content-Type', 'application/pgp-signature')], - 'multi': False, - 'parts': 1, - 'phash': '8f49fbf749', - 'size': 877}}} - -if DEBUG and DO_CHECK: - # TODO turn this into a proper unittest - assert(parts_map == expected) - print "Structure: OK" - - -print -print "RAW DOCS" -pprint.pprint(raw_docs) -print -print "PARTS MAP" -pprint.pprint(parts_map) diff --git a/mail/src/leap/mail/incoming/__init__.py b/mail/src/leap/mail/incoming/__init__.py deleted file mode 100644 index e69de29..0000000 diff --git a/mail/src/leap/mail/incoming/service.py b/mail/src/leap/mail/incoming/service.py deleted file mode 100644 index 1e20862..0000000 --- a/mail/src/leap/mail/incoming/service.py +++ /dev/null @@ -1,844 +0,0 @@ -# -*- coding: utf-8 -*- -# service.py -# Copyright (C) 2015 LEAP -# -# This program is free software: you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation, either version 3 of the License, or -# (at your option) any later version. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with this program. If not, see . -""" -Incoming mail fetcher. -""" -import copy -import logging -import shlex -import time -import warnings - -from email.parser import Parser -from email.utils import parseaddr -from email.utils import formatdate -from StringIO import StringIO -from urlparse import urlparse - -from twisted.application.service import Service -from twisted.python import log -from twisted.python.failure import Failure -from twisted.internet import defer, reactor -from twisted.internet.task import LoopingCall -from twisted.internet.task import deferLater - -from leap.common.events import emit_async, catalog -from leap.common.check import leap_assert, leap_assert_type -from leap.common.mail import get_email_charset -from leap.keymanager import errors as keymanager_errors -from leap.mail.adaptors import soledad_indexes as fields -from leap.mail.generator import Generator -from leap.mail.utils import json_loads, empty -from leap.soledad.client import Soledad -from leap.soledad.common.crypto import ENC_SCHEME_KEY, ENC_JSON_KEY -from leap.soledad.common.errors import InvalidAuthTokenError - - -logger = logging.getLogger(__name__) - -MULTIPART_ENCRYPTED = "multipart/encrypted" -MULTIPART_SIGNED = "multipart/signed" -PGP_BEGIN = "-----BEGIN PGP MESSAGE-----" -PGP_END = "-----END PGP MESSAGE-----" - -# The period between succesive checks of the incoming mail -# queue (in seconds) -INCOMING_CHECK_PERIOD = 60 - - -class MalformedMessage(Exception): - """ - Raised when a given message is not well formed. - """ - pass - - -class IncomingMail(Service): - """ - Fetches and process mail from the incoming pool. - - This object implements IService interface, has public methods - startService and stopService that will actually initiate a - LoopingCall with check_period recurrency. - The LoopingCall itself will invoke the fetch method each time - that the check_period expires. - - This loop will sync the soledad db with the remote server and - process all the documents found tagged as incoming mail. - """ - # TODO implements IService? - - name = "IncomingMail" - - RECENT_FLAG = "\\Recent" - CONTENT_KEY = "content" - - LEAP_SIGNATURE_HEADER = 'X-Leap-Signature' - LEAP_ENCRYPTION_HEADER = 'X-Leap-Encryption' - """ - Header added to messages when they are decrypted by the fetcher, - which states the validity of an eventual signature that might be included - in the encrypted blob. - """ - LEAP_SIGNATURE_VALID = 'valid' - LEAP_SIGNATURE_INVALID = 'invalid' - LEAP_SIGNATURE_COULD_NOT_VERIFY = 'could not verify' - - LEAP_ENCRYPTION_DECRYPTED = 'decrypted' - - def __init__(self, keymanager, soledad, inbox, userid, - check_period=INCOMING_CHECK_PERIOD): - - """ - Initialize IncomingMail.. - - :param keymanager: a keymanager instance - :type keymanager: keymanager.KeyManager - - :param soledad: a soledad instance - :type soledad: Soledad - - :param inbox: the collection for the inbox where the new emails will be - stored - :type inbox: MessageCollection - - :param check_period: the period to fetch new mail, in seconds. - :type check_period: int - """ - leap_assert(keymanager, "need a keymanager to initialize") - leap_assert_type(soledad, Soledad) - leap_assert(check_period, "need a period to check incoming mail") - leap_assert_type(check_period, int) - leap_assert(userid, "need a userid to initialize") - - self._keymanager = keymanager - self._soledad = soledad - self._inbox_collection = inbox - self._userid = userid - - self._listeners = [] - self._loop = None - self._check_period = check_period - - # initialize a mail parser only once - self._parser = Parser() - - def add_listener(self, listener): - """ - Add a listener to inbox insertions. - - This listener function will be called for each message added to the - inbox with its uid as parameter. This function should not be blocking - or it will block the incoming queue. - - :param listener: the listener function - :type listener: callable - """ - self._listeners.append(listener) - - # - # Public API: fetch, start_loop, stop. - # - - def fetch(self): - """ - Fetch incoming mail, to be called periodically. - - Calls a deferred that will execute the fetch callback. - """ - def _sync_errback(failure): - log.err(failure) - - def syncSoledadCallback(_): - # XXX this should be moved to adaptors - d = self._soledad.get_from_index( - fields.JUST_MAIL_IDX, "1", "0") - d.addCallback(self._process_doclist) - d.addErrback(_sync_errback) - return d - - logger.debug("fetching mail for: %s %s" % ( - self._soledad.uuid, self._userid)) - d = self._sync_soledad() - d.addCallbacks(syncSoledadCallback, self._errback) - d.addCallbacks(self._signal_fetch_to_ui, self._errback) - return d - - def startService(self): - """ - Starts a loop to fetch mail. - - :returns: A Deferred whose callback will be invoked with - the LoopingCall instance when loop.stop is called, or - whose errback will be invoked when the function raises an - exception or returned a deferred that has its errback - invoked. - """ - Service.startService(self) - if self._loop is None: - self._loop = LoopingCall(self.fetch) - stop_deferred = self._loop.start(self._check_period) - return stop_deferred - else: - logger.warning("Tried to start an already running fetching loop.") - return defer.fail(Failure('Already running loop.')) - - def stopService(self): - """ - Stops the loop that fetches mail. - """ - if self._loop and self._loop.running is True: - self._loop.stop() - self._loop = None - Service.stopService(self) - - # - # Private methods. - # - - # synchronize incoming mail - - def _errback(self, failure): - log.err(failure) - - def _sync_soledad(self): - """ - Synchronize with remote soledad. - - :returns: a list of LeapDocuments, or None. - :rtype: iterable or None - """ - def _log_synced(result): - log.msg('FETCH soledad SYNCED.') - return result - - def _signal_invalid_auth(failure): - failure.trap(InvalidAuthTokenError) - # if the token is invalid, send an event so the GUI can - # disable mail and show an error message. - emit_async(catalog.SOLEDAD_INVALID_AUTH_TOKEN, self._userid) - - log.msg('FETCH: syncing soledad...') - d = self._soledad.sync() - d.addCallbacks(_log_synced, _signal_invalid_auth) - return d - - def _signal_fetch_to_ui(self, doclist): - """ - Send leap events to ui. - - :param doclist: iterable with msg documents. - :type doclist: iterable. - :returns: doclist - :rtype: iterable - """ - if doclist: - fetched_ts = time.mktime(time.gmtime()) - num_mails = len(doclist) if doclist is not None else 0 - if num_mails != 0: - log.msg("there are %s mails" % (num_mails,)) - emit_async(catalog.MAIL_FETCHED_INCOMING, self._userid, - str(num_mails), str(fetched_ts)) - return doclist - - def _signal_unread_to_ui(self, *args): - """ - Sends unread event to ui. - """ - emit_async(catalog.MAIL_UNREAD_MESSAGES, self._userid, - str(self._inbox_collection.count_unseen())) - - # process incoming mail. - - def _process_doclist(self, doclist): - """ - Iterates through the doclist, checks if each doc - looks like a message, and yields a deferred that will decrypt and - process the message. - - :param doclist: iterable with msg documents. - :type doclist: iterable. - :returns: a list of deferreds for individual messages. - """ - log.msg('processing doclist') - if not doclist: - logger.debug("no docs found") - return - num_mails = len(doclist) - - deferreds = [] - for index, doc in enumerate(doclist): - logger.debug("processing doc %d of %d" % (index + 1, num_mails)) - emit_async(catalog.MAIL_MSG_PROCESSING, self._userid, - str(index), str(num_mails)) - - keys = doc.content.keys() - - # TODO Compatibility check with the index in pre-0.6 mx - # that does not write the ERROR_DECRYPTING_KEY - # This should be removed in 0.7 - - has_errors = doc.content.get(fields.ERROR_DECRYPTING_KEY, None) - - if has_errors is None: - warnings.warn("JUST_MAIL_COMPAT_IDX will be deprecated!", - DeprecationWarning) - - if has_errors: - logger.debug("skipping msg with decrypting errors...") - elif self._is_msg(keys): - # TODO this pipeline is a bit obscure! - d = self._decrypt_doc(doc) - d.addCallback(self._maybe_extract_keys) - d.addCallbacks(self._add_message_locally, self._errback) - deferreds.append(d) - - d = defer.gatherResults(deferreds, consumeErrors=True) - d.addCallback(lambda _: doclist) - return d - - # - # operations on individual messages - # - - def _decrypt_doc(self, doc): - """ - Decrypt the contents of a document. - - :param doc: A document containing an encrypted message. - :type doc: SoledadDocument - - :return: A Deferred that will be fired with the document and the - decrypted message. - :rtype: SoledadDocument, str - """ - log.msg('decrypting msg') - - def process_decrypted(res): - if isinstance(res, tuple): - decrdata, _ = res - success = True - else: - decrdata = "" - success = False - - emit_async(catalog.MAIL_MSG_DECRYPTED, self._userid, - "1" if success else "0") - return self._process_decrypted_doc(doc, decrdata) - - d = self._keymanager.decrypt(doc.content[ENC_JSON_KEY], self._userid) - d.addErrback(self._errback) - d.addCallback(process_decrypted) - d.addCallback(lambda data: (doc, data)) - return d - - def _process_decrypted_doc(self, doc, data): - """ - Process a document containing a succesfully decrypted message. - - :param doc: the incoming message - :type doc: SoledadDocument - :param data: the json-encoded, decrypted content of the incoming - message - :type data: str - - :return: a Deferred that will be fired with an str of the proccessed - data. - :rtype: Deferred - """ - log.msg('processing decrypted doc') - - # XXX turn this into an errBack for each one of - # the deferreds that would process an individual document - try: - msg = json_loads(data) - except UnicodeError as exc: - logger.error("Error while decrypting %s" % (doc.doc_id,)) - logger.exception(exc) - - # we flag the message as "with decrypting errors", - # to avoid further decryption attempts during sync - # cycles until we're prepared to deal with that. - # What is the same, when Ivan deals with it... - # A new decrypting attempt event could be triggered by a - # future a library upgrade, or a cli flag to the client, - # we just `defer` that for now... :) - doc.content[fields.ERROR_DECRYPTING_KEY] = True - deferLater(reactor, 0, self._update_incoming_message, doc) - - # FIXME this is just a dirty hack to delay the proper - # deferred organization here... - # and remember, boys, do not do this at home. - return [] - - if not isinstance(msg, dict): - defer.returnValue(False) - if not msg.get(fields.INCOMING_KEY, False): - defer.returnValue(False) - - # ok, this is an incoming message - rawmsg = msg.get(self.CONTENT_KEY, None) - if rawmsg is None: - return "" - return self._maybe_decrypt_msg(rawmsg) - - def _update_incoming_message(self, doc): - """ - Do a put for a soledad document. This probably has been called only - in the case that we've needed to update the ERROR_DECRYPTING_KEY - flag in an incoming message, to get it out of the decrypting queue. - - :param doc: the SoledadDocument to update - :type doc: SoledadDocument - """ - log.msg("Updating Incoming MSG: SoledadDoc %s" % (doc.doc_id)) - return self._soledad.put_doc(doc) - - def _delete_incoming_message(self, doc): - """ - Delete document. - - :param doc: the SoledadDocument to delete - :type doc: SoledadDocument - """ - log.msg("Deleting Incoming message: %s" % (doc.doc_id,)) - return self._soledad.delete_doc(doc) - - def _maybe_decrypt_msg(self, data): - """ - Tries to decrypt a gpg message if data looks like one. - - :param data: the text to be decrypted. - :type data: str - - :return: a Deferred that will be fired with an str of data, possibly - decrypted. - :rtype: Deferred - """ - leap_assert_type(data, str) - log.msg('maybe decrypting doc') - - # parse the original message - encoding = get_email_charset(data) - msg = self._parser.parsestr(data) - - fromHeader = msg.get('from', None) - senderAddress = None - - if (fromHeader is not None and - (msg.get_content_type() == MULTIPART_ENCRYPTED or - msg.get_content_type() == MULTIPART_SIGNED)): - senderAddress = parseaddr(fromHeader)[1] - - def add_leap_header(ret): - decrmsg, signkey = ret - if (senderAddress is None or signkey is None or - isinstance(signkey, keymanager_errors.KeyNotFound)): - decrmsg.add_header( - self.LEAP_SIGNATURE_HEADER, - self.LEAP_SIGNATURE_COULD_NOT_VERIFY) - elif isinstance(signkey, keymanager_errors.InvalidSignature): - decrmsg.add_header( - self.LEAP_SIGNATURE_HEADER, - self.LEAP_SIGNATURE_INVALID) - else: - self._add_verified_signature_header(decrmsg, - signkey.fingerprint) - return decrmsg.as_string() - - if msg.get_content_type() == MULTIPART_ENCRYPTED: - d = self._decrypt_multipart_encrypted_msg( - msg, encoding, senderAddress) - elif msg.get_content_type() == MULTIPART_SIGNED: - d = self._verify_signature_not_encrypted_msg(msg, senderAddress) - else: - d = self._maybe_decrypt_inline_encrypted_msg( - msg, encoding, senderAddress) - d.addCallback(add_leap_header) - return d - - def _add_verified_signature_header(self, decrmsg, fingerprint): - decrmsg.add_header( - self.LEAP_SIGNATURE_HEADER, - self.LEAP_SIGNATURE_VALID, - pubkey=fingerprint) - - def _add_decrypted_header(self, msg): - msg.add_header(self.LEAP_ENCRYPTION_HEADER, - self.LEAP_ENCRYPTION_DECRYPTED) - - def _decrypt_multipart_encrypted_msg(self, msg, encoding, senderAddress): - """ - Decrypt a message with content-type 'multipart/encrypted'. - - :param msg: The original encrypted message. - :type msg: Message - :param encoding: The encoding of the email message. - :type encoding: str - :param senderAddress: The email address of the sender of the message. - :type senderAddress: str - - :return: A Deferred that will be fired with a tuple containing a - decrypted Message and the signing OpenPGPKey if the signature - is valid or InvalidSignature or KeyNotFound. - :rtype: Deferred - """ - log.msg('decrypting multipart encrypted msg') - msg = copy.deepcopy(msg) - self._msg_multipart_sanity_check(msg) - - # parse message and get encrypted content - pgpencmsg = msg.get_payload()[1] - encdata = pgpencmsg.get_payload() - - # decrypt or fail gracefully - def build_msg(res): - decrdata, signkey = res - - decrmsg = self._parser.parsestr(decrdata) - # remove original message's multipart/encrypted content-type - del(msg['content-type']) - - # replace headers back in original message - for hkey, hval in decrmsg.items(): - try: - # this will raise KeyError if header is not present - msg.replace_header(hkey, hval) - except KeyError: - msg[hkey] = hval - - # all ok, replace payload by unencrypted payload - msg.set_payload(decrmsg.get_payload()) - self._add_decrypted_header(msg) - return (msg, signkey) - - def verify_signature_after_decrypt_an_email(res): - decrdata, signkey = res - if decrdata.get_content_type() == MULTIPART_SIGNED: - res = self._verify_signature_not_encrypted_msg(decrdata, - senderAddress) - return res - - d = self._keymanager.decrypt( - encdata, self._userid, verify=senderAddress) - d.addCallbacks(build_msg, self._decryption_error, errbackArgs=(msg,)) - d.addCallbacks(verify_signature_after_decrypt_an_email) - return d - - def _maybe_decrypt_inline_encrypted_msg(self, origmsg, encoding, - senderAddress): - """ - Possibly decrypt an inline OpenPGP encrypted message. - - :param origmsg: The original, possibly encrypted message. - :type origmsg: Message - :param encoding: The encoding of the email message. - :type encoding: str - :param senderAddress: The email address of the sender of the message. - :type senderAddress: str - - :return: A Deferred that will be fired with a tuple containing a - decrypted Message and the signing OpenPGPKey if the signature - is valid or InvalidSignature or KeyNotFound. - :rtype: Deferred - """ - log.msg('maybe decrypting inline encrypted msg') - - data = self._serialize_msg(origmsg) - - def decrypted_data(res): - decrdata, signkey = res - replaced_data = data.replace(pgp_message, decrdata) - self._add_decrypted_header(origmsg) - return replaced_data, signkey - - def encode_and_return(res): - data, signkey = res - if isinstance(data, unicode): - data = data.encode(encoding, 'replace') - return (self._parser.parsestr(data), signkey) - - # handle exactly one inline PGP message - if PGP_BEGIN in data: - begin = data.find(PGP_BEGIN) - end = data.find(PGP_END) - pgp_message = data[begin:end + len(PGP_END)] - d = self._keymanager.decrypt( - pgp_message, self._userid, verify=senderAddress) - d.addCallbacks(decrypted_data, self._decryption_error, - errbackArgs=(data,)) - else: - d = defer.succeed((data, None)) - d.addCallback(encode_and_return) - return d - - def _verify_signature_not_encrypted_msg(self, origmsg, sender_address): - """ - Possibly decrypt an inline OpenPGP encrypted message. - - :param origmsg: The original, possibly encrypted message. - :type origmsg: Message - :param sender_address: The email address of the sender of the message. - :type sender_address: str - - :return: A Deferred that will be fired with a tuple containing a - signed Message and the signing OpenPGPKey if the signature - is valid or InvalidSignature. - :rtype: Deferred - """ - msg = copy.deepcopy(origmsg) - data = self._serialize_msg(msg.get_payload(0)) - detached_sig = self._extract_signature(msg) - d = self._keymanager.verify(data, sender_address, detached_sig) - - d.addCallback(lambda sign_key: (msg, sign_key)) - d.addErrback(lambda _: (msg, keymanager_errors.InvalidSignature())) - return d - - def _serialize_msg(self, origmsg): - buf = StringIO() - g = Generator(buf) - g.flatten(origmsg) - return buf.getvalue() - - def _extract_signature(self, msg): - body = msg.get_payload(0).get_payload() - - if isinstance(body, str): - body = msg.get_payload(0) - - detached_sig = msg.get_payload(1).get_payload() - msg.set_payload(body) - return detached_sig - - def _decryption_error(self, failure, msg): - """ - Check for known decryption errors - """ - if failure.check(keymanager_errors.DecryptError): - logger.warning('Failed to decrypt encrypted message (%s). ' - 'Storing message without modifications.' - % str(failure.value)) - return (msg, None) - elif failure.check(keymanager_errors.KeyNotFound): - logger.error('Failed to find private key for decryption (%s). ' - 'Storing message without modifications.' - % str(failure.value)) - return (msg, None) - else: - return failure - - @defer.inlineCallbacks - def _maybe_extract_keys(self, msgtuple): - """ - Retrieve attached keys to the mesage and parse message headers for an - *OpenPGP* header as described on the `IETF draft - ` - only urls with https and the same hostname than the email are supported - for security reasons. - - :param msgtuple: a tuple consisting of a SoledadDocument - instance containing the incoming message - and data, the json-encoded, decrypted content of the - incoming message - :type msgtuple: (SoledadDocument, str) - - :return: A Deferred that will be fired with msgtuple when key - extraction finishes - :rtype: Deferred - """ - OpenPGP_HEADER = 'OpenPGP' - doc, data = msgtuple - - # XXX the parsing of the message is done in mailbox.addMessage, maybe - # we should do it in this module so we don't need to parse it again - # here - msg = self._parser.parsestr(data) - _, fromAddress = parseaddr(msg['from']) - - valid_attachment = False - if msg.is_multipart(): - valid_attachment = yield self._maybe_extract_attached_key( - msg.get_payload(), fromAddress) - - if not valid_attachment: - header = msg.get(OpenPGP_HEADER, None) - if header is not None: - yield self._maybe_extract_openpgp_header(header, fromAddress) - - defer.returnValue(msgtuple) - - def _maybe_extract_openpgp_header(self, header, address): - """ - Import keys from the OpenPGP header - - :param header: OpenPGP header string - :type header: str - :param address: email address in the from header - :type address: str - - :return: A Deferred that will be fired when header extraction is done - :rtype: Deferred - """ - d = defer.succeed(None) - fields = dict([f.strip(' ').split('=') for f in header.split(';')]) - if 'url' in fields: - url = shlex.split(fields['url'])[0] # remove quotations - urlparts = urlparse(url) - addressHostname = address.split('@')[1] - if ( - urlparts.scheme == 'https' and - urlparts.hostname == addressHostname - ): - def fetch_error(failure): - if failure.check(keymanager_errors.KeyNotFound): - logger.warning("Url from OpenPGP header %s failed" - % (url,)) - elif failure.check(keymanager_errors.KeyAttributesDiffer): - logger.warning("Key from OpenPGP header url %s didn't " - "match the from address %s" - % (url, address)) - else: - return failure - - d = self._keymanager.fetch_key(address, url) - d.addCallback( - lambda _: - logger.info("Imported key from header %s" % (url,))) - d.addErrback(fetch_error) - else: - logger.debug("No valid url on OpenPGP header %s" % (url,)) - else: - logger.debug("There is no url on the OpenPGP header: %s" - % (header,)) - return d - - def _maybe_extract_attached_key(self, attachments, address): - """ - Import keys from the attachments - - :param attachments: email attachment list - :type attachments: list(email.Message) - :param address: email address in the from header - :type address: str - - :return: A Deferred that will be fired when all the keys are stored - with a boolean: True if there was a valid key attached, or - False otherwise. - :rtype: Deferred - """ - MIME_KEY = "application/pgp-keys" - - def log_key_added(ignored): - logger.debug('Added key found in attachment for %s' % address) - return True - - def failed_put_key(failure): - logger.info("An error has ocurred adding attached key for %s: %s" - % (address, failure.getErrorMessage())) - return False - - deferreds = [] - for attachment in attachments: - if MIME_KEY == attachment.get_content_type(): - d = self._keymanager.put_raw_key( - attachment.get_payload(decode=True), address=address) - d.addCallbacks(log_key_added, failed_put_key) - deferreds.append(d) - d = defer.gatherResults(deferreds) - d.addCallback(lambda result: any(result)) - return d - - def _add_message_locally(self, msgtuple): - """ - Adds a message to local inbox and delete it from the incoming db - in soledad. - - :param msgtuple: a tuple consisting of a SoledadDocument - instance containing the incoming message - and data, the json-encoded, decrypted content of the - incoming message - :type msgtuple: (SoledadDocument, str) - - :return: A Deferred that will be fired when the messages is stored - :rtype: Defferred - """ - doc, raw_data = msgtuple - insertion_date = formatdate(time.time()) - log.msg('adding message %s to local db' % (doc.doc_id,)) - - def msgSavedCallback(result): - if empty(result): - return - - for listener in self._listeners: - listener(result) - - def signal_deleted(doc_id): - emit_async(catalog.MAIL_MSG_DELETED_INCOMING, - self._userid) - return doc_id - - emit_async(catalog.MAIL_MSG_SAVED_LOCALLY, self._userid) - d = self._delete_incoming_message(doc) - d.addCallback(signal_deleted) - return d - - d = self._inbox_collection.add_msg( - raw_data, (self.RECENT_FLAG,), date=insertion_date, - notify_just_mdoc=True) - d.addCallbacks(msgSavedCallback, self._errback) - return d - - # - # helpers - # - - def _msg_multipart_sanity_check(self, msg): - """ - Performs a sanity check against a multipart encrypted msg - - :param msg: The original encrypted message. - :type msg: Message - """ - # sanity check - payload = msg.get_payload() - if len(payload) != 2: - raise MalformedMessage( - 'Multipart/encrypted messages should have exactly 2 body ' - 'parts (instead of %d).' % len(payload)) - if payload[0].get_content_type() != 'application/pgp-encrypted': - raise MalformedMessage( - "Multipart/encrypted messages' first body part should " - "have content type equal to 'application/pgp-encrypted' " - "(instead of %s)." % payload[0].get_content_type()) - if payload[1].get_content_type() != 'application/octet-stream': - raise MalformedMessage( - "Multipart/encrypted messages' second body part should " - "have content type equal to 'octet-stream' (instead of " - "%s)." % payload[1].get_content_type()) - - def _is_msg(self, keys): - """ - Checks if the keys of a dictionary match the signature - of the document type we use for messages. - - :param keys: iterable containing the strings to match. - :type keys: iterable of strings. - :rtype: bool - """ - return ENC_SCHEME_KEY in keys and ENC_JSON_KEY in keys diff --git a/mail/src/leap/mail/incoming/tests/rfc822.multi-encrypt-signed.message b/mail/src/leap/mail/incoming/tests/rfc822.multi-encrypt-signed.message deleted file mode 100644 index 98304f2..0000000 --- a/mail/src/leap/mail/incoming/tests/rfc822.multi-encrypt-signed.message +++ /dev/null @@ -1,61 +0,0 @@ -Content-Type: multipart/encrypted; - boundary="Apple-Mail=_C01A1464-6C43-43BF-8F62-157335B7E25B"; - protocol="application/pgp-encrypted"; -Subject: Enc signed -Mime-Version: 1.0 (Mac OS X Mail 9.3 \(3124\)) -From: Leap Test Key -Date: Tue, 24 May 2016 11:47:24 -0300 -Content-Description: OpenPGP encrypted message -To: leap@leap.se - -This is an OpenPGP/MIME encrypted message (RFC 2440 and 3156) ---Apple-Mail=_C01A1464-6C43-43BF-8F62-157335B7E25B -Content-Type: application/pgp-encrypted -Content-Description: PGP/MIME Versions Identification - ---Apple-Mail=_C01A1464-6C43-43BF-8F62-157335B7E25B -Content-Disposition: inline; - filename=encrypted.asc -Content-Type: application/octet-stream; - name=encrypted.asc -Content-Description: OpenPGP encrypted message - ------BEGIN PGP MESSAGE----- -Version: GnuPG v2 - -hQIMAyj9aG/xtZOwAQ/9Gft0KmOpgzL6z4wmVlLm2aeAvHolXmxWb7N/ByL/dZ4n -YZd/GPRj42X3BwUrDEL5aO3Mcp+rqq8ACh9hsZXiau0Q9cs1K7Gr55Y06qLrIjom -2fLqwLFBxCL2sAX1dvClgStyfsRFk9Y/+5tX+IjWaD8dAoRdxCO8IbUDuYGnaKld -bB9h0NMfKVddCAvuQvX1Zc1Nx0Yb3Hd+ocDD7i9BVgX1BBiGu4/ElS3d32TAVCFs -Na3tjitWB2G472CYu1O6exY7h1F5V4FHfXH6iMRJSYnvV2Jr+oPZENzNdEEA5H/H -fUbpWrpKzPafjho9S5rJBBM/tqtmBQFBIdgFVcBVb+bXO6DJ8SMTLiiGcVUvvm1b -9N2VQIhsxtZ8DpcHHSqFVgT2Gt4UkSrEleSoReg36TzS1s8Uw0oU068PwTe3K0Gx -2pLMdT9NA6X/t7movpXP6tih1l6P5z62dxFl6W12J9OcegISCt0Q7gex1gk/a8zM -rzBJC3mVxRiFlvHPBgD6oUKarnTJPQx5f5dFXg8DXBWR1Eh/aFjPQIzhZBYpmOi8 -HqgjcAA+WhMQ7v5c0enJoJJS+8Xfai/MK2vTUGsfAT6HqHLw1HSIn6XQGEf4sQ/U -NfLeFHHbe9rTk8QhyjrSl2vvek2H4EBQVLF08/FUrAfPELUttOFtysQfC3+M0+PS -6QGyeIlUjKpBJG7HBd4ibuKMQ5vnA+ACsg/TySYeCO6P85xsN+Lmqlr8cAICn/hR -ezFSzlibaIelRgfDEDJdjVyCsa7qBMjhRCvGYBdkyTzIRq53qwD9pkhrQ6nwWQrv -bBzyLrl+NVR8CTEOwbeFLI6qf68kblojk3lwo3Qi3psmeMJdiaV9uevsHrgmEFTH -lZ3rFECPWzmrkMSfVjWu5d8jJqMcqa4lnGzFQKaB76I8BzGhCWrnuvHPB9c9SVhI -AnAwNw3gY5xgsbXMxZhnPgYeBSViPkQkgRCWl8Jz41eiAJ3Gtj8QSSFWGHpX+MgP -ohBaPHz6Fnkhz7Lok97e2AcuRZrDVKV6i28r8mizI3B2Mah6ZV0Yuv0EYNtzBv/v -yV3nu4DWuOOU0301CXBayxJGX0h07z1Ycv7jWD6LNiBXa1vahtbU4WSYNkF0OJaz -nf8O3CZy5twMq5kQYoPacdNNLregAmWquvE1nxqWbtHFMjtXitP7czxzUTU/DE+C -jr+irDoYEregEKg9xov91UCRPZgxL+TML71+tSYOMO3JG6lbGw77PQ8s2So7xore -8+FeDFPaaJqh6uhF5LETRSx8x/haZiXLd+WtO7wF8S3+Vz7AJIFIe8MUadZrYwnH -wfMAktQKbep3iHCeZ5jHYA461AOhnCca2y+GoyHZUDDFwS1pC1RN4lMkafSE1AgH -cmEcjLYsw1gqT0+DfqrvjbXmMjGgkgnkMybJH7df5TKu36Q0Nqvcbc2XLFkalr5V -Vk0SScqKYnKL+cJjabqA8rKkeAh22E2FBCpKPqxSS3te2bRb3XBX26bP0LshkJuy -GPu6LKvwmUn0obPKCnLJvb9ImIGZToXu6Fb/Cd2c3DG1IK5PptQz4f7ZRW98huPO -2w59Bswwt5q4lQqsMEzVRnIDH45MmnhEUeS4NaxqLTO7eJpMpb4VxT2u/Ac3XWKp -o2RE6CbqTyJ+n8tY9OwBRMKzdVd9RFAMqMHTzWTAuU4BgW2vT2sHYZdAsX8sktBr -5mo9P3MqvgdPNpg8+AOB03JlIv0dzrAFWCZxxLLGIIIz0eXsjghHzQ9QjGfr0xFH -Z79AKDjsoRisWyWCnadS2oM9fdAg4T/h1STnfxc44o7N1+ym7u58ODICFi+Kg8IR -JBHIp3CK02JLTLd/WFhUVyWgc6l8gn+oBK+r7Dw+FTWhqX2/ZHCO8qKK1ZK3NIMn -MBcSVvHSnTPtppb+oND5nk38xazVVHnwxNHaIh7g3NxDB4hl5rBhrWsgTNuqDDRU -w7ufvMYr1AOV+8e92cHCEKPM19nFKEgaBFECEptEObesGI3QZPAESlojzQ3cDeBa -=tEyc ------END PGP MESSAGE----- - ---Apple-Mail=_C01A1464-6C43-43BF-8F62-157335B7E25B-- \ No newline at end of file diff --git a/mail/src/leap/mail/incoming/tests/test_incoming_mail.py b/mail/src/leap/mail/incoming/tests/test_incoming_mail.py deleted file mode 100644 index 29422ec..0000000 --- a/mail/src/leap/mail/incoming/tests/test_incoming_mail.py +++ /dev/null @@ -1,391 +0,0 @@ -# -*- coding: utf-8 -*- -# test_incoming_mail.py -# Copyright (C) 2015 LEAP -# -# This program is free software: you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation, either version 3 of the License, or -# (at your option) any later version. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with this program. If not, see . -""" -Test case for leap.mail.incoming.service - -@authors: Ruben Pollan, - -@license: GPLv3, see included LICENSE file -""" - -import json -import os -import tempfile -import uuid - -from email.mime.application import MIMEApplication -from email.mime.multipart import MIMEMultipart -from email.parser import Parser -from mock import Mock - -from twisted.internet import defer -from twisted.python import log - -from leap.keymanager.errors import KeyAddressMismatch -from leap.mail.adaptors import soledad_indexes as fields -from leap.mail.adaptors.soledad import cleanup_deferred_locks -from leap.mail.adaptors.soledad import SoledadMailAdaptor -from leap.mail.mail import MessageCollection -from leap.mail.mailbox_indexer import MailboxIndexer - -from leap.mail.incoming.service import IncomingMail -from leap.mail.rfc3156 import MultipartEncrypted, PGPEncrypted -from leap.mail.testing import KeyManagerWithSoledadTestCase -from leap.mail.testing import ADDRESS, ADDRESS_2 -from leap.soledad.common.document import SoledadDocument -from leap.soledad.common.crypto import ( - EncryptionSchemes, - ENC_JSON_KEY, - ENC_SCHEME_KEY, -) - -HERE = os.path.split(os.path.abspath(__file__))[0] - -# TODO: add some tests for encrypted, unencrypted, signed and unsgined messages - - -class IncomingMailTestCase(KeyManagerWithSoledadTestCase): - """ - Tests for the incoming mail parser - """ - NICKSERVER = "http://domain" - BODY = """ -Governments of the Industrial World, you weary giants of flesh and steel, I -come from Cyberspace, the new home of Mind. On behalf of the future, I ask -you of the past to leave us alone. You are not welcome among us. You have -no sovereignty where we gather. - """ - EMAIL = """from: Test from SomeDomain <%(from)s> -to: %(to)s -subject: independence of cyberspace - -%(body)s - """ % { - "from": ADDRESS_2, - "to": ADDRESS, - "body": BODY - } - - def setUp(self): - cleanup_deferred_locks() - try: - del self._soledad - del self.km - except AttributeError: - pass - - # pytest handles correctly the setupEnv for the class, - # but trial ignores it. - if not getattr(self, 'tempdir', None): - self.tempdir = tempfile.mkdtemp() - - def getCollection(_): - adaptor = SoledadMailAdaptor() - store = self._soledad - adaptor.store = store - mbox_indexer = MailboxIndexer(store) - mbox_name = "INBOX" - mbox_uuid = str(uuid.uuid4()) - - def get_collection_from_mbox_wrapper(wrapper): - wrapper.uuid = mbox_uuid - return MessageCollection( - adaptor, store, - mbox_indexer=mbox_indexer, mbox_wrapper=wrapper) - - d = adaptor.initialize_store(store) - d.addCallback(lambda _: mbox_indexer.create_table(mbox_uuid)) - d.addCallback( - lambda _: adaptor.get_or_create_mbox(store, mbox_name)) - d.addCallback(get_collection_from_mbox_wrapper) - return d - - def setUpFetcher(inbox_collection): - self.fetcher = IncomingMail( - self.km, - self._soledad, - inbox_collection, - ADDRESS) - - # The messages don't exist on soledad will fail on deletion - self.fetcher._delete_incoming_message = Mock( - return_value=defer.succeed(None)) - - d = KeyManagerWithSoledadTestCase.setUp(self) - d.addCallback(getCollection) - d.addCallback(setUpFetcher) - d.addErrback(log.err) - return d - - def tearDown(self): - d = KeyManagerWithSoledadTestCase.tearDown(self) - return d - - def testExtractOpenPGPHeader(self): - """ - Test the OpenPGP header key extraction - """ - KEYURL = "https://leap.se/key.txt" - OpenPGP = "id=12345678; url=\"%s\"; preference=signencrypt" % (KEYURL,) - - message = Parser().parsestr(self.EMAIL) - message.add_header("OpenPGP", OpenPGP) - self.fetcher._keymanager.fetch_key = Mock( - return_value=defer.succeed(None)) - - def fetch_key_called(ret): - self.fetcher._keymanager.fetch_key.assert_called_once_with( - ADDRESS_2, KEYURL) - - d = self._create_incoming_email(message.as_string()) - d.addCallback( - lambda email: - self._mock_soledad_get_from_index(fields.JUST_MAIL_IDX, [email])) - d.addCallback(lambda _: self.fetcher.fetch()) - d.addCallback(fetch_key_called) - return d - - def testExtractOpenPGPHeaderInvalidUrl(self): - """ - Test the OpenPGP header key extraction - """ - KEYURL = "https://someotherdomain.com/key.txt" - OpenPGP = "id=12345678; url=\"%s\"; preference=signencrypt" % (KEYURL,) - - message = Parser().parsestr(self.EMAIL) - message.add_header("OpenPGP", OpenPGP) - self.fetcher._keymanager.fetch_key = Mock() - - def fetch_key_called(ret): - self.assertFalse(self.fetcher._keymanager.fetch_key.called) - - d = self._create_incoming_email(message.as_string()) - d.addCallback( - lambda email: - self._mock_soledad_get_from_index(fields.JUST_MAIL_IDX, [email])) - d.addCallback(lambda _: self.fetcher.fetch()) - d.addCallback(fetch_key_called) - return d - - def testExtractAttachedKey(self): - KEY = "-----BEGIN PGP PUBLIC KEY BLOCK-----\n..." - - message = MIMEMultipart() - message.add_header("from", ADDRESS_2) - key = MIMEApplication("", "pgp-keys") - key.set_payload(KEY) - message.attach(key) - self.fetcher._keymanager.put_raw_key = Mock( - return_value=defer.succeed(None)) - - def put_raw_key_called(_): - self.fetcher._keymanager.put_raw_key.assert_called_once_with( - KEY, address=ADDRESS_2) - - d = self._do_fetch(message.as_string()) - d.addCallback(put_raw_key_called) - return d - - def testExtractInvalidAttachedKey(self): - KEY = "-----BEGIN PGP PUBLIC KEY BLOCK-----\n..." - - message = MIMEMultipart() - message.add_header("from", ADDRESS_2) - key = MIMEApplication("", "pgp-keys") - key.set_payload(KEY) - message.attach(key) - self.fetcher._keymanager.put_raw_key = Mock( - return_value=defer.fail(KeyAddressMismatch())) - - def put_raw_key_called(_): - self.fetcher._keymanager.put_raw_key.assert_called_once_with( - KEY, address=ADDRESS_2) - - d = self._do_fetch(message.as_string()) - d.addCallback(put_raw_key_called) - d.addErrback(log.err) - return d - - def testExtractAttachedKeyAndNotOpenPGPHeader(self): - KEY = "-----BEGIN PGP PUBLIC KEY BLOCK-----\n..." - KEYURL = "https://leap.se/key.txt" - OpenPGP = "id=12345678; url=\"%s\"; preference=signencrypt" % (KEYURL,) - - message = MIMEMultipart() - message.add_header("from", ADDRESS_2) - message.add_header("OpenPGP", OpenPGP) - key = MIMEApplication("", "pgp-keys") - key.set_payload(KEY) - message.attach(key) - - self.fetcher._keymanager.put_raw_key = Mock( - return_value=defer.succeed(None)) - self.fetcher._keymanager.fetch_key = Mock() - - def put_raw_key_called(_): - self.fetcher._keymanager.put_raw_key.assert_called_once_with( - KEY, address=ADDRESS_2) - self.assertFalse(self.fetcher._keymanager.fetch_key.called) - - d = self._do_fetch(message.as_string()) - d.addCallback(put_raw_key_called) - return d - - def testExtractOpenPGPHeaderIfInvalidAttachedKey(self): - KEY = "-----BEGIN PGP PUBLIC KEY BLOCK-----\n..." - KEYURL = "https://leap.se/key.txt" - OpenPGP = "id=12345678; url=\"%s\"; preference=signencrypt" % (KEYURL,) - - message = MIMEMultipart() - message.add_header("from", ADDRESS_2) - message.add_header("OpenPGP", OpenPGP) - key = MIMEApplication("", "pgp-keys") - key.set_payload(KEY) - message.attach(key) - - self.fetcher._keymanager.put_raw_key = Mock( - return_value=defer.fail(KeyAddressMismatch())) - self.fetcher._keymanager.fetch_key = Mock() - - def put_raw_key_called(_): - self.fetcher._keymanager.put_raw_key.assert_called_once_with( - KEY, address=ADDRESS_2) - self.fetcher._keymanager.fetch_key.assert_called_once_with( - ADDRESS_2, KEYURL) - - d = self._do_fetch(message.as_string()) - d.addCallback(put_raw_key_called) - return d - - def testAddDecryptedHeader(self): - class DummyMsg(): - - def __init__(self): - self.headers = {} - - def add_header(self, k, v): - self.headers[k] = v - - msg = DummyMsg() - self.fetcher._add_decrypted_header(msg) - - self.assertEquals(msg.headers['X-Leap-Encryption'], 'decrypted') - - def testDecryptEmail(self): - - self.fetcher._decryption_error = Mock() - self.fetcher._add_decrypted_header = Mock() - - def create_encrypted_message(encstr): - message = Parser().parsestr(self.EMAIL) - newmsg = MultipartEncrypted('application/pgp-encrypted') - for hkey, hval in message.items(): - newmsg.add_header(hkey, hval) - - encmsg = MIMEApplication( - encstr, _subtype='octet-stream', _encoder=lambda x: x) - encmsg.add_header('content-disposition', 'attachment', - filename='msg.asc') - # create meta message - metamsg = PGPEncrypted() - metamsg.add_header('Content-Disposition', 'attachment') - # attach pgp message parts to new message - newmsg.attach(metamsg) - newmsg.attach(encmsg) - return newmsg - - def decryption_error_not_called(_): - self.assertFalse(self.fetcher._decryption_error.called, - "There was some errors with decryption") - - def add_decrypted_header_called(_): - self.assertTrue(self.fetcher._add_decrypted_header.called, - "There was some errors with decryption") - - d = self.km.encrypt(self.EMAIL, ADDRESS, sign=ADDRESS_2) - d.addCallback(create_encrypted_message) - d.addCallback( - lambda message: - self._do_fetch(message.as_string())) - d.addCallback(decryption_error_not_called) - d.addCallback(add_decrypted_header_called) - return d - - def testValidateSignatureFromEncryptedEmailFromAppleMail(self): - enc_signed_file = os.path.join( - HERE, 'rfc822.multi-encrypt-signed.message') - self.fetcher._add_verified_signature_header = Mock() - - def add_verified_signature_header_called(_): - self.assertTrue(self.fetcher._add_verified_signature_header.called, - "There was some errors verifying signature") - - with open(enc_signed_file) as f: - enc_signed_raw = f.read() - - d = self._do_fetch(enc_signed_raw) - d.addCallback(add_verified_signature_header_called) - return d - - def testListener(self): - self.called = False - - def listener(uid): - self.called = True - - def listener_called(_): - self.assertTrue(self.called) - - self.fetcher.add_listener(listener) - d = self._do_fetch(self.EMAIL) - d.addCallback(listener_called) - return d - - def _do_fetch(self, message): - d = self._create_incoming_email(message) - d.addCallback( - lambda email: - self._mock_soledad_get_from_index(fields.JUST_MAIL_IDX, [email])) - d.addCallback(lambda _: self.fetcher.fetch()) - return d - - def _create_incoming_email(self, email_str): - email = SoledadDocument() - data = json.dumps( - {"incoming": True, "content": email_str}, - ensure_ascii=False) - - def set_email_content(encr_data): - email.content = { - fields.INCOMING_KEY: True, - fields.ERROR_DECRYPTING_KEY: False, - ENC_SCHEME_KEY: EncryptionSchemes.PUBKEY, - ENC_JSON_KEY: encr_data - } - return email - d = self.km.encrypt(data, ADDRESS, fetch_remote=False) - d.addCallback(set_email_content) - return d - - def _mock_soledad_get_from_index(self, index_name, value): - get_from_index = self._soledad.get_from_index - - def soledad_mock(idx_name, *key_values): - if index_name == idx_name: - return defer.succeed(value) - return get_from_index(idx_name, *key_values) - self.fetcher._soledad.get_from_index = Mock(side_effect=soledad_mock) diff --git a/mail/src/leap/mail/interfaces.py b/mail/src/leap/mail/interfaces.py deleted file mode 100644 index 10f5123..0000000 --- a/mail/src/leap/mail/interfaces.py +++ /dev/null @@ -1,215 +0,0 @@ -# -*- coding: utf-8 -*- -# interfaces.py -# Copyright (C) 2014,2015 LEAP -# -# This program is free software: you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation, either version 3 of the License, or -# (at your option) any later version. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with this program. If not, see . -""" -Interfaces for the leap.mail module. -""" -from zope.interface import Interface, Attribute - - -class IMessageWrapper(Interface): - """ - I know how to access the different parts into which a given message is - splitted into. - - :ivar fdoc: dict with flag document. - :ivar hdoc: dict with flag document. - :ivar cdocs: dict with content-documents, one-indexed. - """ - - fdoc = Attribute('A dictionaly-like containing the flags document ' - '(mutable)') - hdoc = Attribute('A dictionary-like containing the headers document ' - '(immutable)') - cdocs = Attribute('A dictionary with the content-docs, one-indexed') - - def create(self, store, notify_just_mdoc=False, pending_inserts_dict={}): - """ - Create the underlying wrapper. - """ - - def update(self, store): - """ - Update the only mutable parts, which are within the flags document. - """ - - def delete(self, store): - """ - Delete the parts for this wrapper that are not referenced from anywhere - else. - """ - - def copy(self, store, new_mbox_uuid): - """ - Return a copy of this IMessageWrapper in a new mailbox. - """ - - def set_mbox_uuid(self, mbox_uuid): - """ - Set the mailbox for this wrapper. - """ - - def set_flags(self, flags): - """ - """ - - def set_tags(self, tags): - """ - """ - - def set_date(self, date): - """ - """ - - def get_subpart_dict(self, index): - """ - :param index: the part to lookup, 1-indexed - """ - - def get_subpart_indexes(self): - """ - """ - - def get_body(self, store): - """ - """ - - -# TODO -- split into smaller interfaces? separate mailbox interface at least? - -class IMailAdaptor(Interface): - """ - I know how to store the standard representation for messages and mailboxes, - and how to update the relevant mutable parts when needed. - """ - - def initialize_store(self, store): - """ - Performs whatever initialization is needed before the store can be - used (creating indexes, sanity checks, etc). - - :param store: store - :returns: a Deferred that will fire when the store is correctly - initialized. - :rtype: deferred - """ - - def get_msg_from_string(self, MessageClass, raw_msg): - """ - Get an instance of a MessageClass initialized with a MessageWrapper - that contains all the parts obtained from parsing the raw string for - the message. - - :param MessageClass: an implementor of IMessage - :type raw_msg: str - :rtype: implementor of leap.mail.IMessage - """ - - def get_msg_from_docs(self, MessageClass, mdoc, fdoc, hdoc, cdocs=None, - uid=None): - """ - Get an instance of a MessageClass initialized with a MessageWrapper - that contains the passed part documents. - - This is not the recommended way of obtaining a message, unless you know - how to take care of ensuring the internal consistency between the part - documents, or unless you are glueing together the part documents that - have been previously generated by `get_msg_from_string`. - """ - - def get_flags_from_mdoc_id(self, store, mdoc_id): - """ - """ - - def create_msg(self, store, msg): - """ - :param store: an instance of soledad, or anything that behaves alike - :param msg: a Message object. - - :return: a Deferred that is fired when all the underlying documents - have been created. - :rtype: defer.Deferred - """ - - def update_msg(self, store, msg): - """ - :param msg: a Message object. - :param store: an instance of soledad, or anything that behaves alike - :return: a Deferred that is fired when all the underlying documents - have been updated (actually, it's only the fdoc that's allowed - to update). - :rtype: defer.Deferred - """ - - def get_count_unseen(self, store, mbox_uuid): - """ - Get the number of unseen messages for a given mailbox. - - :param store: instance of Soledad. - :param mbox_uuid: the uuid for this mailbox. - :rtype: int - """ - - def get_count_recent(self, store, mbox_uuid): - """ - Get the number of recent messages for a given mailbox. - - :param store: instance of Soledad. - :param mbox_uuid: the uuid for this mailbox. - :rtype: int - """ - - def get_mdoc_id_from_msgid(self, store, mbox_uuid, msgid): - """ - Get the UID for a message with the passed msgid (the one in the headers - msg-id). - This is used by the MUA to retrieve the recently saved draft. - """ - - # mbox handling - - def get_or_create_mbox(self, store, name): - """ - Get the mailbox with the given name, or create one if it does not - exist. - - :param store: instance of Soledad - :param name: the name of the mailbox - :type name: str - """ - - def update_mbox(self, store, mbox_wrapper): - """ - Update the documents for a given mailbox. - :param mbox_wrapper: MailboxWrapper instance - :type mbox_wrapper: MailboxWrapper - :return: a Deferred that will be fired when the mailbox documents - have been updated. - :rtype: defer.Deferred - """ - - def delete_mbox(self, store, mbox_wrapper): - """ - """ - - def get_all_mboxes(self, store): - """ - Retrieve a list with wrappers for all the mailboxes. - - :return: a deferred that will be fired with a list of all the - MailboxWrappers found. - :rtype: defer.Deferred - """ diff --git a/mail/src/leap/mail/load_tests.py b/mail/src/leap/mail/load_tests.py deleted file mode 100644 index be65b8d..0000000 --- a/mail/src/leap/mail/load_tests.py +++ /dev/null @@ -1,29 +0,0 @@ -# -*- coding: utf-8 -*- -# tests.py -# Copyright (C) 2013 LEAP -# -# This program is free software: you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation, either version 3 of the License, or -# (at your option) any later version. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with this program. If not, see . -""" -Provide a function for loading tests. -""" -import unittest - - -def load_tests(): - suite = unittest.TestSuite() - for test in unittest.defaultTestLoader.discover( - './src/leap/mail/', - top_level_dir='./src/'): - suite.addTest(test) - return suite diff --git a/mail/src/leap/mail/mail.py b/mail/src/leap/mail/mail.py deleted file mode 100644 index 2fde3a1..0000000 --- a/mail/src/leap/mail/mail.py +++ /dev/null @@ -1,1070 +0,0 @@ -# -*- coding: utf-8 -*- -# mail.py -# Copyright (C) 2014,2015 LEAP -# -# This program is free software: you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation, either version 3 of the License, or -# (at your option) any later version. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with this program. If not, see . -""" -Generic Access to Mail objects. - -This module holds the public LEAP Mail API, which should be viewed as the main -entry point for message and account manipulation, in a protocol-agnostic way. - -In the future, pluggable transports will expose this generic API. -""" -import itertools -import uuid -import logging -import StringIO -import time -import weakref - -from collections import defaultdict - -from twisted.internet import defer -from twisted.python import log - -from leap.common.check import leap_assert_type -from leap.common.events import emit_async, catalog - -from leap.mail.adaptors.soledad import SoledadMailAdaptor -from leap.mail.constants import INBOX_NAME -from leap.mail.constants import MessageFlags -from leap.mail.mailbox_indexer import MailboxIndexer -from leap.mail.plugins import soledad_sync_hooks -from leap.mail.utils import find_charset, CaseInsensitiveDict -from leap.mail.utils import lowerdict - -logger = logging.getLogger(name=__name__) - - -# TODO LIST -# [ ] Probably change the name of this module to "api" or "account", mail is -# too generic (there's also IncomingMail, and OutgoingMail -# [ ] Profile add_msg. - -def _get_mdoc_id(mbox, chash): - """ - Get the doc_id for the metamsg document. - """ - return "M+{mbox}+{chash}".format(mbox=mbox, chash=chash) - - -def _write_and_rewind(payload): - fd = StringIO.StringIO() - fd.write(payload) - fd.seek(0) - return fd - - -def _encode_payload(payload, ctype=""): - """ - Properly encode an unicode payload (which can be string or unicode) as a - string. - - :param payload: the payload to encode. currently soledad returns unicode - strings. - :type payload: basestring - :param ctype: optional, the content of the content-type header for this - payload. - :type ctype: str - :rtype: str - """ - # TODO Related, it's proposed that we're able to pass - # the encoding to the soledad documents. Better to store the charset there? - # FIXME ----------------------------------------------- - # this need a dedicated test-suite - charset = find_charset(ctype) - - # XXX get from mail headers if not multipart! - # Beware also that we should pass the proper encoding to - # soledad when it's creating the documents. - # if not charset: - # charset = get_email_charset(payload) - # ----------------------------------------------------- - - if not charset: - charset = "utf-8" - - try: - if isinstance(payload, unicode): - payload = payload.encode(charset) - except UnicodeError as exc: - logger.error( - "Unicode error, using 'replace'. {0!r}".format(exc)) - payload = payload.encode(charset, 'replace') - return payload - - -def _unpack_headers(headers_dict): - """ - Take a "packed" dict containing headers (with repeated keys represented as - line breaks inside each value, preceded by the header key) and return a - list of tuples in which each repeated key has a different tuple. - """ - headers_l = headers_dict.items() - for i, (k, v) in enumerate(headers_l): - splitted = v.split(k.lower() + ": ") - if len(splitted) != 1: - inner = zip( - itertools.cycle([k]), - map(lambda l: l.rstrip('\n'), splitted)) - headers_l = headers_l[:i] + inner + headers_l[i + 1:] - return headers_l - - -class MessagePart(object): - # TODO This class should be better abstracted from the data model. - # TODO support arbitrarily nested multiparts (right now we only support - # the trivial case) - """ - Represents a part of a multipart MIME Message. - """ - - def __init__(self, part_map, cdocs=None, nested=False): - """ - :param part_map: a dictionary mapping the subparts for - this MessagePart (1-indexed). - :type part_map: dict - - The format for the part_map is as follows: - - {u'ctype': u'text/plain', - u'headers': [[u'Content-Type', u'text/plain; charset="utf-8"'], - [u'Content-Transfer-Encoding', u'8bit']], - u'multi': False, - u'parts': 1, - u'phash': u'02D82B29F6BB0C8612D1C', - u'size': 132} - - :param cdocs: optional, a reference to the top-level dict of wrappers - for content-docs (1-indexed). - """ - if cdocs is None: - cdocs = {} - self._pmap = part_map - self._cdocs = cdocs - self._nested = nested - - def get_size(self): - """ - Size of the body, in octets. - """ - total = self._pmap['size'] - _h = self.get_headers() - headers = len( - '\n'.join(["%s: %s" % (k, v) for k, v in dict(_h).items()])) - # have to subtract 2 blank lines - return total - headers - 2 - - def get_body_file(self): - payload = "" - pmap = self._pmap - - multi = pmap.get('multi') - if not multi: - payload = self._get_payload(pmap.get('phash')) - if payload: - payload = _encode_payload(payload) - - return _write_and_rewind(payload) - - def get_headers(self): - return CaseInsensitiveDict(self._pmap.get("headers", [])) - - def is_multipart(self): - return self._pmap.get("multi", False) - - def get_subpart(self, part): - if not self.is_multipart(): - raise TypeError - sub_pmap = self._pmap.get("part_map", {}) - - try: - part_map = sub_pmap[str(part)] - except KeyError: - log.msg("getSubpart for %s: KeyError" % (part,)) - raise IndexError - return MessagePart(part_map, cdocs=self._cdocs, nested=True) - - def _get_payload(self, phash): - for cdocw in self._cdocs.values(): - if cdocw.phash == phash: - return cdocw.raw - return "" - - -class Message(object): - """ - Represents a single message, and gives access to all its attributes. - """ - - def __init__(self, wrapper, uid=None): - """ - :param wrapper: an instance of an implementor of IMessageWrapper - :param uid: - :type uid: int - """ - self._wrapper = wrapper - self._uid = uid - - def get_wrapper(self): - """ - Get the wrapper for this message. - """ - return self._wrapper - - def get_uid(self): - """ - Get the (optional) UID. - """ - return self._uid - - # imap.IMessage methods - - def get_flags(self): - """ - Get flags for this message. - :rtype: tuple - """ - return self._wrapper.fdoc.get_flags() - - def get_internal_date(self): - """ - Retrieve the date internally associated with this message - - According to the spec, this is NOT the date and time in the - RFC-822 header, but rather a date and time that reflects when the - message was received. - - * In SMTP, date and time of final delivery. - * In COPY, internal date/time of the source message. - * In APPEND, date/time specified. - - :return: An RFC822-formatted date string. - :rtype: str - """ - return self._wrapper.hdoc.date - - # imap.IMessageParts - - def get_headers(self): - """ - Get the raw headers document. - """ - return CaseInsensitiveDict(self._wrapper.hdoc.headers) - - def get_body_file(self, store): - """ - Get a file descriptor with the body content. - """ - def write_and_rewind_if_found(cdoc): - payload = cdoc.raw if cdoc else "" - # XXX pass ctype from headers if not multipart? - if payload: - payload = _encode_payload(payload, ctype=cdoc.content_type) - return _write_and_rewind(payload) - - d = defer.maybeDeferred(self._wrapper.get_body, store) - d.addCallback(write_and_rewind_if_found) - return d - - def get_size(self): - """ - Size of the whole message, in octets (including headers). - """ - total = self._wrapper.fdoc.size - return total - - def is_multipart(self): - """ - Return True if this message is multipart. - """ - return self._wrapper.fdoc.multi - - def get_subpart(self, part): - """ - :param part: The number of the part to retrieve, indexed from 1. - :type part: int - :rtype: MessagePart - """ - if not self.is_multipart(): - raise TypeError - try: - subpart_dict = self._wrapper.get_subpart_dict(part) - except KeyError: - raise IndexError - - return MessagePart( - subpart_dict, cdocs=self._wrapper.cdocs) - - # Custom methods. - - def get_tags(self): - """ - Get the tags for this message. - """ - return tuple(self._wrapper.fdoc.tags) - - -class Flagsmode(object): - """ - Modes for setting the flags/tags. - """ - APPEND = 1 - REMOVE = -1 - SET = 0 - - -class MessageCollection(object): - """ - A generic collection of messages. It can be messages sharing the same - mailbox, tag, the result of a given query, or just a bunch of ids for - master documents. - - Since LEAP Mail is primarily oriented to store mail in Soledad, the default - (and, so far, only) implementation of the store is contained in the - Soledad Mail Adaptor, which is passed to every collection on creation by - the root Account object. If you need to use a different adaptor, change the - adaptor class attribute in your Account object. - - Store is a reference to a particular instance of the message store (soledad - instance or proxy, for instance). - """ - - # TODO LIST - # [ ] look at IMessageSet methods - # [ ] make constructor with a per-instance deferredLock to use on - # creation/deletion? - # [ ] instead of a mailbox, we could pass an arbitrary container with - # pointers to different doc_ids (type: foo) - # [ ] To guarantee synchronicity of the documents sent together during a - # sync, we could get hold of a deferredLock that inhibits - # synchronization while we are updating (think more about this!) - # [ ] review the serveral count_ methods. I think it's better to patch - # server to accept deferreds. - # [ ] Use inheritance for the mailbox-collection instead of handling the - # special cases everywhere? - # [ ] or maybe a mailbox_only decorator... - - # Account should provide an adaptor instance when creating this collection. - adaptor = None - store = None - messageklass = Message - - _pending_inserts = dict() - - def __init__(self, adaptor, store, mbox_indexer=None, mbox_wrapper=None): - """ - Constructor for a MessageCollection. - """ - self.adaptor = adaptor - self.store = store - - # XXX think about what to do when there is no mbox passed to - # the initialization. We could still get the MetaMsg by index, instead - # of by doc_id. See get_message_by_content_hash - self.mbox_indexer = mbox_indexer - self.mbox_wrapper = mbox_wrapper - self._listeners = set([]) - - def is_mailbox_collection(self): - """ - Return True if this collection represents a Mailbox. - :rtype: bool - """ - return bool(self.mbox_wrapper) - - @property - def mbox_name(self): - # TODO raise instead? - if self.mbox_wrapper is None: - return None - return self.mbox_wrapper.mbox - - @property - def mbox_uuid(self): - # TODO raise instead? - if self.mbox_wrapper is None: - return None - return self.mbox_wrapper.uuid - - def get_mbox_attr(self, attr): - if self.mbox_wrapper is None: - raise RuntimeError("This is not a mailbox collection") - return getattr(self.mbox_wrapper, attr) - - def set_mbox_attr(self, attr, value): - if self.mbox_wrapper is None: - raise RuntimeError("This is not a mailbox collection") - setattr(self.mbox_wrapper, attr, value) - return self.mbox_wrapper.update(self.store) - - # Get messages - - def get_message_by_content_hash(self, chash, get_cdocs=False): - """ - Retrieve a message by its content hash. - :rtype: Deferred - """ - if not self.is_mailbox_collection(): - # TODO instead of getting the metamsg by chash, in this case we - # should query by (meta) index or use the internal collection of - # pointers-to-docs. - raise NotImplementedError() - - metamsg_id = _get_mdoc_id(self.mbox_name, chash) - - return self.adaptor.get_msg_from_mdoc_id( - self.messageklass, self.store, - metamsg_id, get_cdocs=get_cdocs) - - def get_message_by_sequence_number(self, msn, get_cdocs=False): - """ - Retrieve a message by its Message Sequence Number. - :rtype: Deferred - """ - def get_uid_for_msn(all_uid): - return all_uid[msn - 1] - d = self.all_uid_iter() - d.addCallback(get_uid_for_msn) - d.addCallback( - lambda uid: self.get_message_by_uid( - uid, get_cdocs=get_cdocs)) - d.addErrback(lambda f: log.err(f)) - return d - - def get_message_by_uid(self, uid, absolute=True, get_cdocs=False): - """ - Retrieve a message by its Unique Identifier. - - If this is a Mailbox collection, that is the message UID, unique for a - given mailbox, or a relative sequence number depending on the absolute - flag. For now, only absolute identifiers are supported. - :rtype: Deferred - """ - # TODO deprecate absolute flag, it doesn't make sense UID and - # !absolute. use _by_sequence_number instead. - if not absolute: - raise NotImplementedError("Does not support relative ids yet") - - get_doc_fun = self.mbox_indexer.get_doc_id_from_uid - - def get_msg_from_mdoc_id(doc_id): - if doc_id is None: - return None - return self.adaptor.get_msg_from_mdoc_id( - self.messageklass, self.store, - doc_id, uid=uid, get_cdocs=get_cdocs) - - def cleanup_and_get_doc_after_pending_insert(result): - for key in result: - self._pending_inserts.pop(key, None) - return get_doc_fun(self.mbox_uuid, uid) - - if not self._pending_inserts: - d = get_doc_fun(self.mbox_uuid, uid) - else: - d = defer.gatherResults(self._pending_inserts.values()) - d.addCallback(cleanup_and_get_doc_after_pending_insert) - d.addCallback(get_msg_from_mdoc_id) - return d - - def get_flags_by_uid(self, uid, absolute=True): - # TODO use sequence numbers - if not absolute: - raise NotImplementedError("Does not support relative ids yet") - - def get_flags_from_mdoc_id(doc_id): - if doc_id is None: # XXX needed? or bug? - return None - return self.adaptor.get_flags_from_mdoc_id( - self.store, doc_id) - - def wrap_in_tuple(flags): - return (uid, flags) - - d = self.mbox_indexer.get_doc_id_from_uid(self.mbox_uuid, uid) - d.addCallback(get_flags_from_mdoc_id) - d.addCallback(wrap_in_tuple) - return d - - def count(self): - """ - Count the messages in this collection. - :return: a Deferred that will fire with the integer for the count. - :rtype: Deferred - """ - if not self.is_mailbox_collection(): - raise NotImplementedError() - - d = self.mbox_indexer.count(self.mbox_uuid) - return d - - def count_recent(self): - """ - Count the recent messages in this collection. - :return: a Deferred that will fire with the integer for the count. - :rtype: Deferred - """ - if not self.is_mailbox_collection(): - raise NotImplementedError() - return self.adaptor.get_count_recent(self.store, self.mbox_uuid) - - def count_unseen(self): - """ - Count the unseen messages in this collection. - :return: a Deferred that will fire with the integer for the count. - :rtype: Deferred - """ - if not self.is_mailbox_collection(): - raise NotImplementedError() - return self.adaptor.get_count_unseen(self.store, self.mbox_uuid) - - def get_uid_next(self): - """ - Get the next integer beyond the highest UID count for this mailbox. - - :return: a Deferred that will fire with the integer for the next uid. - :rtype: Deferred - """ - return self.mbox_indexer.get_next_uid(self.mbox_uuid) - - def get_last_uid(self): - """ - Get the last UID for this mailbox. - """ - return self.mbox_indexer.get_last_uid(self.mbox_uuid) - - def all_uid_iter(self): - """ - Iterator through all the uids for this collection. - """ - return self.mbox_indexer.all_uid_iter(self.mbox_uuid) - - def get_uid_from_msgid(self, msgid): - """ - Return the UID(s) of the matching msg-ids for this mailbox collection. - """ - if not self.is_mailbox_collection(): - raise NotImplementedError() - - def get_uid(mdoc_id): - if not mdoc_id: - return None - d = self.mbox_indexer.get_uid_from_doc_id( - self.mbox_uuid, mdoc_id) - return d - - d = self.adaptor.get_mdoc_id_from_msgid( - self.store, self.mbox_uuid, msgid) - d.addCallback(get_uid) - return d - - # Manipulate messages - - def add_msg(self, raw_msg, flags=tuple(), tags=tuple(), date="", - notify_just_mdoc=False): - """ - Add a message to this collection. - - :param raw_msg: the raw message - :param flags: tuple of flags for this message - :param tags: tuple of tags for this message - :param date: - formatted date, it will be used to retrieve the internal - date for this message. According to the spec, this is NOT the date - and time in the RFC-822 header, but rather a date and time that - reflects when the message was received. - :type date: str - :param notify_just_mdoc: - boolean passed to the wrapper.create method, to indicate whether - we're insterested in being notified right after the mdoc has been - written (as it's the first doc to be written, and quite small, this - is faster, though potentially unsafe), or on the contrary we want - to wait untill all the parts have been written. - Used by the imap mailbox implementation to get faster responses. - This will be ignored (and set to False) if a heuristic for a Draft - message is met, which currently is a specific mozilla header. - :type notify_just_mdoc: bool - - :returns: a deferred that will fire with the UID of the inserted - message. - :rtype: deferred - """ - # TODO watch out if the use of this method in IMAP COPY/APPEND is - # passing the right date. - # XXX mdoc ref is a leaky abstraction here. generalize. - leap_assert_type(flags, tuple) - leap_assert_type(date, str) - - msg = self.adaptor.get_msg_from_string(Message, raw_msg) - wrapper = msg.get_wrapper() - - headers = lowerdict(msg.get_headers()) - moz_draft_hdr = "X-Mozilla-Draft-Info" - if moz_draft_hdr.lower() in headers: - log.msg("Setting fast notify to False, Draft detected") - notify_just_mdoc = False - - if notify_just_mdoc: - msgid = headers.get('message-id') - if msgid: - self._pending_inserts[msgid] = defer.Deferred() - - if not self.is_mailbox_collection(): - raise NotImplementedError() - - else: - mbox_id = self.mbox_uuid - wrapper.set_mbox_uuid(mbox_id) - wrapper.set_flags(flags) - wrapper.set_tags(tags) - wrapper.set_date(date) - - def insert_mdoc_id(_, wrapper): - doc_id = wrapper.mdoc.doc_id - if not doc_id: - # --- BUG ----------------------------------------- - # XXX watch out, sometimes mdoc doesn't have doc_id - # but it has future_id. Should be solved already. - logger.error("BUG: (please report) Null doc_id for " - "document %s" % - (wrapper.mdoc.serialize(),)) - return defer.succeed("mdoc_id not inserted") - # XXX BUG ----------------------------------------- - - # XXX BUG sometimes the table is not yet created, - # so workaround is to make sure we always check for it before - # inserting the doc. I should debug into the real cause. - d = self.mbox_indexer.create_table(self.mbox_uuid) - d.addBoth(lambda _: self.mbox_indexer.insert_doc( - self.mbox_uuid, doc_id)) - return d - - d = wrapper.create( - self.store, - notify_just_mdoc=notify_just_mdoc, - pending_inserts_dict=self._pending_inserts) - d.addCallback(insert_mdoc_id, wrapper) - d.addCallback(self.cb_signal_unread_to_ui) - d.addCallback(self.notify_new_to_listeners) - d.addErrback(lambda failure: log.err(failure)) - - return d - - # Listeners - - def addListener(self, listener): - self._listeners.add(listener) - - def removeListener(self, listener): - self._listeners.remove(listener) - - def notify_new_to_listeners(self, result): - for listener in self._listeners: - listener.notify_new() - return result - - def cb_signal_unread_to_ui(self, result): - """ - Sends an unread event to ui, passing *only* the number of unread - messages if *this* is the inbox. This event is catched, for instance, - in the Bitmask client that displays a message with the number of unread - mails in the INBOX. - - Used as a callback in several commands. - - :param result: ignored - """ - # TODO it might make sense to modify the event so that - # it receives both the mailbox name AND the number of unread messages. - if self.mbox_name.lower() == "inbox": - d = defer.maybeDeferred(self.count_unseen) - d.addCallback(self.__cb_signal_unread_to_ui) - return result - - def __cb_signal_unread_to_ui(self, unseen): - """ - Send the unread signal to UI. - :param unseen: number of unseen messages. - :type unseen: int - """ - emit_async(catalog.MAIL_UNREAD_MESSAGES, self.store.uuid, str(unseen)) - - def copy_msg(self, msg, new_mbox_uuid): - """ - Copy the message to another collection. (it only makes sense for - mailbox collections) - """ - # TODO should CHECK first if the mdoc is present in the mailbox - # WITH a Deleted flag... and just simply remove the flag... - # Another option is to delete the previous mdoc if it already exists - # (so we get a new UID) - - if not self.is_mailbox_collection(): - raise NotImplementedError() - - def delete_mdoc_entry_and_insert(failure, mbox_uuid, doc_id): - d = self.mbox_indexer.delete_doc_by_hash(mbox_uuid, doc_id) - d.addCallback(lambda _: self.mbox_indexer.insert_doc( - new_mbox_uuid, doc_id)) - return d - - def insert_copied_mdoc_id(wrapper_new_msg): - # XXX FIXME -- since this is already saved, the future_doc_id - # should be already copied into the doc_id! - # Investigate why we are not receiving the already saved doc_id - doc_id = wrapper_new_msg.mdoc.doc_id - if not doc_id: - doc_id = wrapper_new_msg.mdoc._future_doc_id - - def insert_conditionally(uid, mbox_uuid, doc_id): - indexer = self.mbox_indexer - if uid: - d = indexer.delete_doc_by_hash(mbox_uuid, doc_id) - d.addCallback(lambda _: indexer.insert_doc( - new_mbox_uuid, doc_id)) - return d - else: - d = indexer.insert_doc(mbox_uuid, doc_id) - return d - - def log_result(result): - return result - - def insert_doc(_, mbox_uuid, doc_id): - d = self.mbox_indexer.get_uid_from_doc_id(mbox_uuid, doc_id) - d.addCallback(insert_conditionally, mbox_uuid, doc_id) - d.addErrback(lambda err: log.failure(err)) - d.addCallback(log_result) - return d - - d = self.mbox_indexer.create_table(new_mbox_uuid) - d.addBoth(insert_doc, new_mbox_uuid, doc_id) - return d - - wrapper = msg.get_wrapper() - - d = wrapper.copy(self.store, new_mbox_uuid) - d.addCallback(insert_copied_mdoc_id) - d.addCallback(self.notify_new_to_listeners) - return d - - def delete_msg(self, msg): - """ - Delete this message. - """ - wrapper = msg.get_wrapper() - - def delete_mdoc_id(_, wrapper): - doc_id = wrapper.mdoc.doc_id - return self.mbox_indexer.delete_doc_by_hash( - self.mbox_uuid, doc_id) - d = wrapper.delete(self.store) - d.addCallback(delete_mdoc_id, wrapper) - return d - - def delete_all_flagged(self): - """ - Delete all messages flagged as \\Deleted. - Used from IMAPMailbox.expunge() - """ - def get_uid_list(hashes): - d = [] - for h in hashes: - d.append(self.mbox_indexer.get_uid_from_doc_id( - self.mbox_uuid, h)) - return defer.gatherResults(d), hashes - - def delete_uid_entries((uids, hashes)): - d = [] - for h in hashes: - d.append(self.mbox_indexer.delete_doc_by_hash( - self.mbox_uuid, h)) - - def return_uids_when_deleted(ignored): - return uids - - all_deleted = defer.gatherResults(d).addCallback( - return_uids_when_deleted) - return all_deleted - - mdocs_deleted = self.adaptor.del_all_flagged_messages( - self.store, self.mbox_uuid) - mdocs_deleted.addCallback(get_uid_list) - mdocs_deleted.addCallback(delete_uid_entries) - mdocs_deleted.addErrback(lambda f: log.err(f)) - return mdocs_deleted - - # TODO should add a delete-by-uid to collection? - - def delete_all_docs(self): - def del_all_uid(uid_list): - deferreds = [] - for uid in uid_list: - d = self.get_message_by_uid(uid) - d.addCallback(lambda msg: msg.delete()) - deferreds.append(d) - return defer.gatherResults(deferreds) - - d = self.all_uid_iter() - d.addCallback(del_all_uid) - return d - - def update_flags(self, msg, flags, mode): - """ - Update flags for a given message. - """ - wrapper = msg.get_wrapper() - current = wrapper.fdoc.flags - newflags = map(str, self._update_flags_or_tags(current, flags, mode)) - wrapper.fdoc.flags = newflags - - wrapper.fdoc.seen = MessageFlags.SEEN_FLAG in newflags - wrapper.fdoc.deleted = MessageFlags.DELETED_FLAG in newflags - - d = self.adaptor.update_msg(self.store, msg) - d.addCallback(lambda _: newflags) - return d - - def update_tags(self, msg, tags, mode): - """ - Update tags for a given message. - """ - wrapper = msg.get_wrapper() - current = wrapper.fdoc.tags - newtags = self._update_flags_or_tags(current, tags, mode) - - wrapper.fdoc.tags = newtags - d = self.adaptor.update_msg(self.store, msg) - d.addCallback(newtags) - return d - - def _update_flags_or_tags(self, old, new, mode): - if mode == Flagsmode.APPEND: - final = list((set(tuple(old) + new))) - elif mode == Flagsmode.REMOVE: - final = list(set(old).difference(set(new))) - elif mode == Flagsmode.SET: - final = new - return final - - -class Account(object): - """ - Account is the top level abstraction to access collections of messages - associated with a LEAP Mail Account. - - It primarily handles creation and access of Mailboxes, which will be the - basic collection handled by traditional MUAs, but it can also handle other - types of Collections (tag based, for instance). - - leap.mail.imap.IMAPAccount partially proxies methods in this - class. - """ - - # Adaptor is passed to the returned MessageCollections, so if you want to - # use a different adaptor this is the place to change it, by subclassing - # the Account class. - - adaptor_class = SoledadMailAdaptor - - # this is a defaultdict, indexed by userid, that returns a - # WeakValueDictionary mapping to collection instances so that we always - # return a reference to them instead of creating new ones. however, - # being a dictionary of weakrefs values, they automagically vanish - # from the dict when no hard refs is left to them (so they can be - # garbage collected) this is important because the different wrappers - # rely on several kinds of deferredlocks that are kept as class or - # instance variables. - - # We need it to be a class property because we create more than one Account - # object in the current usage pattern (ie, one in the mail service, and - # another one in the IncomingMailService). When we move to a proper service - # tree we can let it be an instance attribute. - _collection_mapping = defaultdict(weakref.WeakValueDictionary) - - def __init__(self, store, user_id, ready_cb=None): - self.store = store - self.user_id = user_id - self.adaptor = self.adaptor_class() - - self.mbox_indexer = MailboxIndexer(self.store) - - # This flag is only used from the imap service for the moment. - # In the future, we should prevent any public method to continue if - # this is set to True. Also, it would be good to plug to the - # authentication layer. - self.session_ended = False - - self.deferred_initialization = defer.Deferred() - self._ready_cb = ready_cb - - self._init_d = self._initialize_storage() - self._initialize_sync_hooks() - - def _initialize_storage(self): - - def add_mailbox_if_none(mboxes): - if not mboxes: - return self.add_mailbox(INBOX_NAME) - - def finish_initialization(result): - self.deferred_initialization.callback(None) - if self._ready_cb is not None: - self._ready_cb() - - d = self.adaptor.initialize_store(self.store) - d.addCallback(lambda _: self.list_all_mailbox_names()) - d.addCallback(add_mailbox_if_none) - d.addCallback(finish_initialization) - return d - - def callWhenReady(self, cb, *args, **kw): - """ - Execute the callback when the initialization of the Account is ready. - Note that the callback will receive a first meaningless parameter. - """ - # TODO this should ignore the first parameter explicitely - # lambda _: cb(*args, **kw) - self.deferred_initialization.addCallback(cb, *args, **kw) - return self.deferred_initialization - - # Sync hooks - - def _initialize_sync_hooks(self): - soledad_sync_hooks.post_sync_uid_reindexer.set_account(self) - - def _teardown_sync_hooks(self): - soledad_sync_hooks.post_sync_uid_reindexer.set_account(None) - - # - # Public API Starts - # - - def list_all_mailbox_names(self): - - def filter_names(mboxes): - return [m.mbox for m in mboxes] - - d = self.get_all_mailboxes() - d.addCallback(filter_names) - return d - - def get_all_mailboxes(self): - d = self.adaptor.get_all_mboxes(self.store) - return d - - def add_mailbox(self, name, creation_ts=None): - - if creation_ts is None: - # by default, we pass an int value - # taken from the current time - # we make sure to take enough decimals to get a unique - # mailbox-uidvalidity. - creation_ts = int(time.time() * 10E2) - - def set_creation_ts(wrapper): - wrapper.created = creation_ts - d = wrapper.update(self.store) - d.addCallback(lambda _: wrapper) - return d - - def create_uuid(wrapper): - if not wrapper.uuid: - wrapper.uuid = str(uuid.uuid4()) - d = wrapper.update(self.store) - d.addCallback(lambda _: wrapper) - return d - return wrapper - - def create_uid_table_cb(wrapper): - d = self.mbox_indexer.create_table(wrapper.uuid) - d.addCallback(lambda _: wrapper) - return d - - d = self.adaptor.get_or_create_mbox(self.store, name) - d.addCallback(set_creation_ts) - d.addCallback(create_uuid) - d.addCallback(create_uid_table_cb) - return d - - def delete_mailbox(self, name): - - def delete_uid_table_cb(wrapper): - d = self.mbox_indexer.delete_table(wrapper.uuid) - d.addCallback(lambda _: wrapper) - return d - - d = self.adaptor.get_or_create_mbox(self.store, name) - d.addCallback(delete_uid_table_cb) - d.addCallback( - lambda wrapper: self.adaptor.delete_mbox(self.store, wrapper)) - return d - - def rename_mailbox(self, oldname, newname): - - def _rename_mbox(wrapper): - wrapper.mbox = newname - d = wrapper.update(self.store) - d.addCallback(lambda result: wrapper) - return d - - d = self.adaptor.get_or_create_mbox(self.store, oldname) - d.addCallback(_rename_mbox) - return d - - # Get Collections - - def get_collection_by_mailbox(self, name): - """ - :rtype: deferred - :return: a deferred that will fire with a MessageCollection - """ - collection = self._collection_mapping[self.user_id].get( - name, None) - if collection: - return defer.succeed(collection) - - # imap select will use this, passing the collection to SoledadMailbox - def get_collection_for_mailbox(mbox_wrapper): - collection = MessageCollection( - self.adaptor, self.store, self.mbox_indexer, mbox_wrapper) - self._collection_mapping[self.user_id][name] = collection - return collection - - d = self.adaptor.get_or_create_mbox(self.store, name) - d.addCallback(get_collection_for_mailbox) - return d - - def get_collection_by_docs(self, docs): - """ - :rtype: MessageCollection - """ - # get a collection of docs by a list of doc_id - # get.docs(...) --> it should be a generator. does it behave in the - # threadpool? - raise NotImplementedError() - - def get_collection_by_tag(self, tag): - """ - :rtype: MessageCollection - """ - raise NotImplementedError() - - # Session handling - - def end_session(self): - self._teardown_sync_hooks() - self.session_ended = True diff --git a/mail/src/leap/mail/mailbox_indexer.py b/mail/src/leap/mail/mailbox_indexer.py deleted file mode 100644 index c49f808..0000000 --- a/mail/src/leap/mail/mailbox_indexer.py +++ /dev/null @@ -1,327 +0,0 @@ -# -*- coding: utf-8 -*- -# mailbox_indexer.py -# Copyright (C) 2014 LEAP -# -# This program is free software: you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation, either version 3 of the License, or -# (at your option) any later version. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with this program. If not, see . -""" -.. :py:module::mailbox_indexer - -Local tables to store the message Unique Identifiers for a given mailbox. -""" -import re -import uuid - -from leap.mail.constants import METAMSGID_RE - - -def _maybe_first_query_item(thing): - """ - Return the first item the returned query result, or None - if empty. - """ - try: - return thing[0][0] - except (TypeError, IndexError): - return None - - -class WrongMetaDocIDError(Exception): - pass - - -def sanitize(mailbox_uuid): - return mailbox_uuid.replace("-", "_") - - -def check_good_uuid(mailbox_uuid): - """ - Check that the passed mailbox identifier is a valid UUID. - :param mailbox_uuid: the uuid to check - :type mailbox_uuid: str - :return: None - :raises: AssertionError if a wrong uuid was passed. - """ - try: - uuid.UUID(str(mailbox_uuid)) - except (AttributeError, ValueError): - raise AssertionError( - "the mbox_id is not a valid uuid: %s" % mailbox_uuid) - - -class MailboxIndexer(object): - """ - This class contains the commands needed to create, modify and alter the - local-only UID tables for a given mailbox. - - Its purpouse is to keep a local-only index with the messages in each - mailbox, mainly to satisfy the demands of the IMAP specification, but - useful too for any effective listing of the messages in a mailbox. - - Since the incoming mail can be processed at any time in any replica, it's - preferred not to attempt to maintain a global chronological global index. - - These indexes are Message Attributes needed for the IMAP specification (rfc - 3501), although they can be useful for other non-imap store - implementations. - - """ - # The uids are expected to be 32-bits values, but the ROWIDs in sqlite - # are 64-bit values. I *don't* think it really matters for any - # practical use, but it's good to remember we've got that difference going - # on. - - store = None - table_preffix = "leapmail_uid_" - - def __init__(self, store): - self.store = store - - def _query(self, *args, **kw): - assert self.store is not None - return self.store.raw_sqlcipher_query(*args, **kw) - - def _operation(self, *args, **kw): - assert self.store is not None - return self.store.raw_sqlcipher_operation(*args, **kw) - - def create_table(self, mailbox_uuid): - """ - Create the UID table for a given mailbox. - :param mailbox: the mailbox identifier. - :type mailbox: str - :rtype: Deferred - """ - check_good_uuid(mailbox_uuid) - sql = ("CREATE TABLE if not exists {preffix}{name}( " - "uid INTEGER PRIMARY KEY AUTOINCREMENT, " - "hash TEXT UNIQUE NOT NULL)".format( - preffix=self.table_preffix, name=sanitize(mailbox_uuid))) - return self._operation(sql) - - def delete_table(self, mailbox_uuid): - """ - Delete the UID table for a given mailbox. - :param mailbox: the mailbox name - :type mailbox: str - :rtype: Deferred - """ - check_good_uuid(mailbox_uuid) - sql = ("DROP TABLE if exists {preffix}{name}".format( - preffix=self.table_preffix, name=sanitize(mailbox_uuid))) - return self._operation(sql) - - def insert_doc(self, mailbox_uuid, doc_id): - """ - Insert the doc_id for a MetaMsg in the UID table for a given mailbox. - - The doc_id must be in the format: - - M-- - - :param mailbox: the mailbox name - :type mailbox: str - :param doc_id: the doc_id for the MetaMsg - :type doc_id: str - :return: a deferred that will fire with the uid of the newly inserted - document. - :rtype: Deferred - """ - check_good_uuid(mailbox_uuid) - assert doc_id - mailbox_uuid = mailbox_uuid.replace('-', '_') - - if not re.findall(METAMSGID_RE.format(mbox_uuid=mailbox_uuid), doc_id): - raise WrongMetaDocIDError("Wrong format for the MetaMsg doc_id") - - def get_rowid(result): - return _maybe_first_query_item(result) - - sql = ("INSERT INTO {preffix}{name} VALUES (" - "NULL, ?)".format( - preffix=self.table_preffix, name=sanitize(mailbox_uuid))) - values = (doc_id,) - - sql_last = ("SELECT MAX(rowid) FROM {preffix}{name} " - "LIMIT 1;").format( - preffix=self.table_preffix, name=sanitize(mailbox_uuid)) - - d = self._operation(sql, values) - d.addCallback(lambda _: self._query(sql_last)) - d.addCallback(get_rowid) - d.addErrback(lambda f: f.printTraceback()) - return d - - def delete_doc_by_uid(self, mailbox_uuid, uid): - """ - Delete the entry for a MetaMsg in the UID table for a given mailbox. - - :param mailbox_uuid: the mailbox uuid - :type mailbox: str - :param uid: the UID of the message. - :type uid: int - :rtype: Deferred - """ - check_good_uuid(mailbox_uuid) - assert uid - sql = ("DELETE FROM {preffix}{name} " - "WHERE uid=?".format( - preffix=self.table_preffix, name=sanitize(mailbox_uuid))) - values = (uid,) - return self._query(sql, values) - - def delete_doc_by_hash(self, mailbox_uuid, doc_id): - """ - Delete the entry for a MetaMsg in the UID table for a given mailbox. - - The doc_id must be in the format: - - M-- - - :param mailbox_uuid: the mailbox uuid - :type mailbox: str - :param doc_id: the doc_id for the MetaMsg - :type doc_id: str - :return: a deferred that will fire when the deletion has succed. - :rtype: Deferred - """ - check_good_uuid(mailbox_uuid) - assert doc_id - sql = ("DELETE FROM {preffix}{name} " - "WHERE hash=?".format( - preffix=self.table_preffix, name=sanitize(mailbox_uuid))) - values = (doc_id,) - return self._query(sql, values) - - def get_doc_id_from_uid(self, mailbox_uuid, uid): - """ - Get the doc_id for a MetaMsg in the UID table for a given mailbox. - - :param mailbox_uuid: the mailbox uuid - :type mailbox: str - :param uid: the uid for the MetaMsg for this mailbox - :type uid: int - :rtype: Deferred - """ - check_good_uuid(mailbox_uuid) - mailbox_uuid = mailbox_uuid.replace('-', '_') - - def get_hash(result): - return _maybe_first_query_item(result) - - sql = ("SELECT hash from {preffix}{name} " - "WHERE uid=?".format( - preffix=self.table_preffix, name=sanitize(mailbox_uuid))) - values = (uid,) - d = self._query(sql, values) - d.addCallback(get_hash) - return d - - def get_uid_from_doc_id(self, mailbox_uuid, doc_id): - check_good_uuid(mailbox_uuid) - mailbox_uuid = mailbox_uuid.replace('-', '_') - - def get_uid(result): - return _maybe_first_query_item(result) - - sql = ("SELECT uid from {preffix}{name} " - "WHERE hash=?".format( - preffix=self.table_preffix, name=sanitize(mailbox_uuid))) - values = (doc_id,) - d = self._query(sql, values) - d.addCallback(get_uid) - return d - - def get_doc_ids_from_uids(self, mailbox_uuid, uids): - # For IMAP relative numbering /sequences. - # XXX dereference the range (n,*) - raise NotImplementedError() - - def count(self, mailbox_uuid): - """ - Get the number of entries in the UID table for a given mailbox. - - :param mailbox_uuid: the mailbox uuid - :type mailbox_uuid: str - :return: a deferred that will fire with an integer returning the count. - :rtype: Deferred - """ - check_good_uuid(mailbox_uuid) - - def get_count(result): - return _maybe_first_query_item(result) - - sql = ("SELECT Count(*) FROM {preffix}{name};".format( - preffix=self.table_preffix, name=sanitize(mailbox_uuid))) - d = self._query(sql) - d.addCallback(get_count) - d.addErrback(lambda _: 0) - return d - - def get_next_uid(self, mailbox_uuid): - """ - Get the next integer beyond the highest UID count for a given mailbox. - - This is expected by the IMAP implementation. There are no guarantees - that a document to be inserted in the future gets the returned UID: the - only thing that can be assured is that it will be equal or greater than - the value returned. - - :param mailbox_uuid: the mailbox uuid - :type mailbox: str - :return: a deferred that will fire with an integer returning the next - uid. - :rtype: Deferred - """ - check_good_uuid(mailbox_uuid) - d = self.get_last_uid(mailbox_uuid) - d.addCallback(lambda uid: uid + 1) - return d - - def get_last_uid(self, mailbox_uuid): - """ - Get the highest UID for a given mailbox. - """ - check_good_uuid(mailbox_uuid) - sql = ("SELECT MAX(rowid) FROM {preffix}{name} " - "LIMIT 1;").format( - preffix=self.table_preffix, name=sanitize(mailbox_uuid)) - - def getit(result): - rowid = _maybe_first_query_item(result) - if not rowid: - rowid = 0 - return rowid - - d = self._query(sql) - d.addCallback(getit) - return d - - def all_uid_iter(self, mailbox_uuid): - """ - Get a sequence of all the uids in this mailbox. - - :param mailbox_uuid: the mailbox uuid - :type mailbox_uuid: str - """ - check_good_uuid(mailbox_uuid) - - sql = ("SELECT uid from {preffix}{name} ").format( - preffix=self.table_preffix, name=sanitize(mailbox_uuid)) - - def get_results(result): - return [x[0] for x in result] - - d = self._query(sql) - d.addCallback(get_results) - return d diff --git a/mail/src/leap/mail/outgoing/__init__.py b/mail/src/leap/mail/outgoing/__init__.py deleted file mode 100644 index e69de29..0000000 diff --git a/mail/src/leap/mail/outgoing/service.py b/mail/src/leap/mail/outgoing/service.py deleted file mode 100644 index 8b02f2e..0000000 --- a/mail/src/leap/mail/outgoing/service.py +++ /dev/null @@ -1,518 +0,0 @@ -# -*- coding: utf-8 -*- -# outgoing/service.py -# Copyright (C) 2013-2015 LEAP -# -# This program is free software: you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation, either version 3 of the License, or -# (at your option) any later version. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with this program. If not, see . - -""" -OutgoingMail module. - -The OutgoingMail class allows to send mail, and encrypts/signs it if needed. -""" - -import os.path -import re -from StringIO import StringIO -from copy import deepcopy -from email.parser import Parser -from email.mime.application import MIMEApplication -from email.mime.multipart import MIMEMultipart -from email.mime.text import MIMEText - -from OpenSSL import SSL - -from twisted.mail import smtp -from twisted.internet import reactor -from twisted.internet import defer -from twisted.protocols.amp import ssl -from twisted.python import log - -from leap.common.check import leap_assert_type, leap_assert -from leap.common.events import emit_async, catalog -from leap.keymanager.errors import KeyNotFound, KeyAddressMismatch -from leap.mail import __version__ -from leap.mail import errors -from leap.mail.utils import validate_address -from leap.mail.rfc3156 import MultipartEncrypted -from leap.mail.rfc3156 import MultipartSigned -from leap.mail.rfc3156 import encode_base64_rec -from leap.mail.rfc3156 import RFC3156CompliantGenerator -from leap.mail.rfc3156 import PGPSignature -from leap.mail.rfc3156 import PGPEncrypted - -# TODO -# [ ] rename this module to something else, service should be the implementor -# of IService - - -class SSLContextFactory(ssl.ClientContextFactory): - def __init__(self, cert, key): - self.cert = cert - self.key = key - - def getContext(self): - # FIXME -- we should use sslv23 to allow for tlsv1.2 - # and, if possible, explicitely disable sslv3 clientside. - # Servers should avoid sslv3 - self.method = SSL.TLSv1_METHOD # SSLv23_METHOD - ctx = ssl.ClientContextFactory.getContext(self) - ctx.use_certificate_file(self.cert) - ctx.use_privatekey_file(self.key) - return ctx - - -def outgoingFactory(userid, keymanager, opts, check_cert=True, bouncer=None): - - cert = unicode(opts.cert) - key = unicode(opts.key) - hostname = str(opts.hostname) - port = opts.port - - if check_cert: - if not os.path.isfile(cert): - raise errors.ConfigurationError( - 'No valid SMTP certificate could be found for %s!' % userid) - - return OutgoingMail( - str(userid), keymanager, cert, key, hostname, port, - bouncer) - - -class OutgoingMail(object): - """ - Sends Outgoing Mail, encrypting and signing if needed. - """ - - def __init__(self, from_address, keymanager, cert, key, host, port, - bouncer=None): - """ - Initialize the outgoing mail service. - - :param from_address: The sender address. - :type from_address: str - :param keymanager: A KeyManager for retrieving recipient's keys. - :type keymanager: leap.common.keymanager.KeyManager - :param cert: The client certificate for SSL authentication. - :type cert: str - :param key: The client private key for SSL authentication. - :type key: str - :param host: The hostname of the remote SMTP server. - :type host: str - :param port: The port of the remote SMTP server. - :type port: int - """ - - # assert params - leap_assert_type(from_address, str) - leap_assert('@' in from_address) - - # XXX it can be a zope.proxy too - # leap_assert_type(keymanager, KeyManager) - - leap_assert_type(host, str) - leap_assert(host != '') - leap_assert_type(port, int) - leap_assert(port is not 0) - leap_assert_type(cert, unicode) - leap_assert(cert != '') - leap_assert_type(key, unicode) - leap_assert(key != '') - - self._port = port - self._host = host - self._key = key - self._cert = cert - self._from_address = from_address - self._keymanager = keymanager - self._bouncer = bouncer - - def send_message(self, raw, recipient): - """ - Sends a message to a recipient. Maybe encrypts and signs. - - :param raw: The raw message - :type raw: str - :param recipient: The recipient for the message - :type recipient: smtp.User - :return: a deferred which delivers the message when fired - """ - d = self._maybe_encrypt_and_sign(raw, recipient) - d.addCallback(self._route_msg, raw) - d.addErrback(self.sendError, raw) - return d - - def sendSuccess(self, smtp_sender_result): - """ - Callback for a successful send. - - :param smtp_sender_result: The result from the ESMTPSender from - _route_msg - :type smtp_sender_result: tuple(int, list(tuple)) - """ - dest_addrstr = smtp_sender_result[1][0][0] - fromaddr = self._from_address - log.msg('Message sent from %s to %s' % (fromaddr, dest_addrstr)) - emit_async(catalog.SMTP_SEND_MESSAGE_SUCCESS, - fromaddr, dest_addrstr) - - def sendError(self, failure, origmsg): - """ - Callback for an unsuccessfull send. - - :param failure: The result from the last errback. - :type failure: anything - :param origmsg: the original, unencrypted, raw message, to be passed to - the bouncer. - :type origmsg: str - """ - # XXX: need to get the address from the original message to send signal - # emit_async(catalog.SMTP_SEND_MESSAGE_ERROR, self._from_address, - # self._user.dest.addrstr) - - # TODO when we implement outgoing queues/long-term-retries, we could - # examine the error *here* and delay the notification if it's just a - # temporal error. We might want to notify the permanent errors - # differently. - - err = failure.value - log.err(err) - - if self._bouncer: - self._bouncer.bounce_message( - err.message, to=self._from_address, - orig=origmsg) - else: - raise err - - def _route_msg(self, encrypt_and_sign_result, raw): - """ - Sends the msg using the ESMTPSenderFactory. - - :param encrypt_and_sign_result: A tuple containing the 'maybe' - encrypted message and the recipient - :type encrypt_and_sign_result: tuple - """ - message, recipient = encrypt_and_sign_result - log.msg("Connecting to SMTP server %s:%s" % (self._host, self._port)) - msg = message.as_string(False) - - # we construct a defer to pass to the ESMTPSenderFactory - d = defer.Deferred() - d.addCallback(self.sendSuccess) - d.addErrback(self.sendError, raw) - # we don't pass an ssl context factory to the ESMTPSenderFactory - # because ssl will be handled by reactor.connectSSL() below. - factory = smtp.ESMTPSenderFactory( - "", # username is blank, no client auth here - "", # password is blank, no client auth here - self._from_address, - recipient.dest.addrstr, - StringIO(msg), - d, - heloFallback=True, - requireAuthentication=False, - requireTransportSecurity=True) - factory.domain = bytes('leap.mail-' + __version__) - emit_async(catalog.SMTP_SEND_MESSAGE_START, - self._from_address, recipient.dest.addrstr) - reactor.connectSSL( - self._host, self._port, factory, - contextFactory=SSLContextFactory(self._cert, self._key)) - - def _maybe_encrypt_and_sign(self, raw, recipient, fetch_remote=True): - """ - Attempt to encrypt and sign the outgoing message. - - The behaviour of this method depends on: - - 1. the original message's content-type, and - 2. the availability of the recipient's public key. - - If the original message's content-type is "multipart/encrypted", then - the original message is not altered. For any other content-type, the - method attempts to fetch the recipient's public key. If the - recipient's public key is available, the message is encrypted and - signed; otherwise it is only signed. - - Note that, if the C{encrypted_only} configuration is set to True and - the recipient's public key is not available, then the recipient - address would have been rejected in SMTPDelivery.validateTo(). - - The following table summarizes the overall behaviour of the gateway: - - +---------------------------------------------------+----------------+ - | content-type | rcpt pubkey | enforce encr. | action | - +---------------------+-------------+---------------+----------------+ - | multipart/encrypted | any | any | pass | - | other | available | any | encrypt + sign | - | other | unavailable | yes | reject | - | other | unavailable | no | sign | - +---------------------+-------------+---------------+----------------+ - - :param raw: The raw message - :type raw: str - :param recipient: The recipient for the message - :type: recipient: smtp.User - - :return: A Deferred that will be fired with a MIMEMultipart message - and the original recipient Message - :rtype: Deferred - """ - # pass if the original message's content-type is "multipart/encrypted" - origmsg = Parser().parsestr(raw) - - if origmsg.get_content_type() == 'multipart/encrypted': - return defer.succeed((origmsg, recipient)) - - from_address = validate_address(self._from_address) - username, domain = from_address.split('@') - to_address = validate_address(recipient.dest.addrstr) - - def maybe_encrypt_and_sign(message): - d = self._encrypt_and_sign( - message, to_address, from_address, - fetch_remote=fetch_remote) - d.addCallbacks(signal_encrypt_sign, - if_key_not_found_send_unencrypted, - errbackArgs=(message,)) - return d - - def signal_encrypt_sign(newmsg): - emit_async(catalog.SMTP_END_ENCRYPT_AND_SIGN, - self._from_address, - "%s,%s" % (self._from_address, to_address)) - return newmsg, recipient - - def if_key_not_found_send_unencrypted(failure, message): - failure.trap(KeyNotFound, KeyAddressMismatch) - - log.msg('Will send unencrypted message to %s.' % to_address) - emit_async(catalog.SMTP_START_SIGN, self._from_address, to_address) - d = self._sign(message, from_address) - d.addCallback(signal_sign) - return d - - def signal_sign(newmsg): - emit_async(catalog.SMTP_END_SIGN, self._from_address) - return newmsg, recipient - - log.msg("Will encrypt the message with %s and sign with %s." - % (to_address, from_address)) - emit_async(catalog.SMTP_START_ENCRYPT_AND_SIGN, - self._from_address, - "%s,%s" % (self._from_address, to_address)) - d = self._maybe_attach_key(origmsg, from_address, to_address) - d.addCallback(maybe_encrypt_and_sign) - return d - - def _maybe_attach_key(self, origmsg, from_address, to_address): - filename = "%s-email-key.asc" % (from_address,) - - def attach_if_address_hasnt_encrypted(to_key): - # if the sign_used flag is true that means that we got an encrypted - # email from this address, because we conly check signatures on - # encrypted emails. In this case we don't attach. - # XXX: this might not be true some time in the future - if to_key.sign_used: - return origmsg - return get_key_and_attach(None) - - def get_key_and_attach(_): - d = self._keymanager.get_key(from_address, fetch_remote=False) - d.addCallback(attach_key) - return d - - def attach_key(from_key): - msg = origmsg - if not origmsg.is_multipart(): - msg = MIMEMultipart() - for h, v in origmsg.items(): - msg.add_header(h, v) - msg.attach(MIMEText(origmsg.get_payload())) - - keymsg = MIMEApplication(from_key.key_data, _subtype='pgp-keys', - _encoder=lambda x: x) - keymsg.add_header('content-disposition', 'attachment', - filename=filename) - msg.attach(keymsg) - return msg - - d = self._keymanager.get_key(to_address, fetch_remote=False) - d.addCallbacks(attach_if_address_hasnt_encrypted, get_key_and_attach) - d.addErrback(lambda _: origmsg) - return d - - def _encrypt_and_sign(self, origmsg, encrypt_address, sign_address, - fetch_remote=True): - """ - Create an RFC 3156 compliang PGP encrypted and signed message using - C{encrypt_address} to encrypt and C{sign_address} to sign. - - :param origmsg: The original message - :type origmsg: email.message.Message - :param encrypt_address: The address used to encrypt the message. - :type encrypt_address: str - :param sign_address: The address used to sign the message. - :type sign_address: str - - :return: A Deferred with the MultipartEncrypted message - :rtype: Deferred - """ - # create new multipart/encrypted message with 'pgp-encrypted' protocol - - def encrypt(res): - newmsg, origmsg = res - d = self._keymanager.encrypt( - origmsg.as_string(unixfrom=False), - encrypt_address, sign=sign_address, - fetch_remote=fetch_remote) - d.addCallback(lambda encstr: (newmsg, encstr)) - return d - - def create_encrypted_message(res): - newmsg, encstr = res - encmsg = MIMEApplication( - encstr, _subtype='octet-stream', _encoder=lambda x: x) - encmsg.add_header('content-disposition', 'attachment', - filename='msg.asc') - # create meta message - metamsg = PGPEncrypted() - metamsg.add_header('Content-Disposition', 'attachment') - # attach pgp message parts to new message - newmsg.attach(metamsg) - newmsg.attach(encmsg) - return newmsg - - d = self._fix_headers( - origmsg, - MultipartEncrypted('application/pgp-encrypted'), - sign_address) - d.addCallback(encrypt) - d.addCallback(create_encrypted_message) - return d - - def _sign(self, origmsg, sign_address): - """ - Create an RFC 3156 compliant PGP signed MIME message using - C{sign_address}. - - :param origmsg: The original message - :type origmsg: email.message.Message - :param sign_address: The address used to sign the message. - :type sign_address: str - - :return: A Deferred with the MultipartSigned message. - :rtype: Deferred - """ - # apply base64 content-transfer-encoding - encode_base64_rec(origmsg) - # get message text with headers and replace \n for \r\n - fp = StringIO() - g = RFC3156CompliantGenerator( - fp, mangle_from_=False, maxheaderlen=76) - g.flatten(origmsg) - msgtext = re.sub('\r?\n', '\r\n', fp.getvalue()) - # make sure signed message ends with \r\n as per OpenPGP stantard. - if origmsg.is_multipart(): - if not msgtext.endswith("\r\n"): - msgtext += "\r\n" - - def create_signed_message(res): - (msg, _), signature = res - sigmsg = PGPSignature(signature) - # attach original message and signature to new message - msg.attach(origmsg) - msg.attach(sigmsg) - return msg - - dh = self._fix_headers( - origmsg, - MultipartSigned('application/pgp-signature', 'pgp-sha512'), - sign_address) - ds = self._keymanager.sign( - msgtext, sign_address, digest_algo='SHA512', - clearsign=False, detach=True, binary=False) - d = defer.gatherResults([dh, ds]) - d.addCallback(create_signed_message) - return d - - def _fix_headers(self, msg, newmsg, sign_address): - """ - Move some headers from C{origmsg} to C{newmsg}, delete unwanted - headers from C{origmsg} and add new headers to C{newms}. - - Outgoing messages are either encrypted and signed or just signed - before being sent. Because of that, they are packed inside new - messages and some manipulation has to be made on their headers. - - Allowed headers for passing through: - - - From - - Date - - To - - Subject - - Reply-To - - References - - In-Reply-To - - Cc - - Headers to be added: - - - Message-ID (i.e. should not use origmsg's Message-Id) - - Received (this is added automatically by twisted smtp API) - - OpenPGP (see #4447) - - Headers to be deleted: - - - User-Agent - - :param msg: The original message. - :type msg: email.message.Message - :param newmsg: The new message being created. - :type newmsg: email.message.Message - :param sign_address: The address used to sign C{newmsg} - :type sign_address: str - - :return: A Deferred with a touple: - (new Message with the unencrypted headers, - original Message with headers removed) - :rtype: Deferred - """ - origmsg = deepcopy(msg) - # move headers from origmsg to newmsg - headers = origmsg.items() - passthrough = [ - 'from', 'date', 'to', 'subject', 'reply-to', 'references', - 'in-reply-to', 'cc' - ] - headers = filter(lambda x: x[0].lower() in passthrough, headers) - for hkey, hval in headers: - newmsg.add_header(hkey, hval) - del (origmsg[hkey]) - # add a new message-id to newmsg - newmsg.add_header('Message-Id', smtp.messageid()) - # delete user-agent from origmsg - del (origmsg['user-agent']) - - def add_openpgp_header(signkey): - username, domain = sign_address.split('@') - newmsg.add_header( - 'OpenPGP', 'id=%s' % signkey.fingerprint, - url='https://%s/key/%s' % (domain, username), - preference='signencrypt') - return newmsg, origmsg - - d = self._keymanager.get_key(sign_address, private=True) - d.addCallback(add_openpgp_header) - return d diff --git a/mail/src/leap/mail/outgoing/tests/test_outgoing.py b/mail/src/leap/mail/outgoing/tests/test_outgoing.py deleted file mode 100644 index dd053c1..0000000 --- a/mail/src/leap/mail/outgoing/tests/test_outgoing.py +++ /dev/null @@ -1,263 +0,0 @@ -# -*- coding: utf-8 -*- -# test_gateway.py -# Copyright (C) 2013 LEAP -# -# This program is free software: you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation, either version 3 of the License, or -# (at your option) any later version. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with this program. If not, see . - - -""" -SMTP gateway tests. -""" -import re -from copy import deepcopy -from StringIO import StringIO -from email.parser import Parser -from datetime import datetime -from twisted.internet.defer import fail -from twisted.mail.smtp import User -from twisted.python import log - -from mock import Mock - -from leap.mail.rfc3156 import RFC3156CompliantGenerator -from leap.mail.outgoing.service import OutgoingMail -from leap.mail.testing import ADDRESS, ADDRESS_2, PUBLIC_KEY_2 -from leap.mail.testing import KeyManagerWithSoledadTestCase -from leap.mail.testing.smtp import getSMTPFactory -from leap.keymanager import errors - - -BEGIN_PUBLIC_KEY = "-----BEGIN PGP PUBLIC KEY BLOCK-----" - -TEST_USER = u'anotheruser@leap.se' - - -class TestOutgoingMail(KeyManagerWithSoledadTestCase): - EMAIL_DATA = ['HELO gateway.leap.se', - 'MAIL FROM: <%s>' % ADDRESS_2, - 'RCPT TO: <%s>' % ADDRESS, - 'DATA', - 'From: User <%s>' % ADDRESS_2, - 'To: Leap <%s>' % ADDRESS, - 'Date: ' + datetime.now().strftime('%c'), - 'Subject: test message', - '', - 'This is a secret message.', - 'Yours,', - 'A.', - '', - '.', - 'QUIT'] - - def setUp(self): - self.lines = [line for line in self.EMAIL_DATA[4:12]] - self.lines.append('') # add a trailing newline - self.raw = '\r\n'.join(self.lines) - self.expected_body = '\r\n'.join(self.EMAIL_DATA[9:12]) + "\r\n" - self.fromAddr = ADDRESS_2 - - class opts: - cert = u'/tmp/cert' - key = u'/tmp/cert' - hostname = 'remote' - port = 666 - self.opts = opts - - def init_outgoing_and_proto(_): - self.outgoing_mail = OutgoingMail( - self.fromAddr, self.km, opts.cert, - opts.key, opts.hostname, opts.port) - - user = TEST_USER - - # TODO -- this shouldn't need SMTP to be tested!? or does it? - self.proto = getSMTPFactory( - {user: None}, {user: self.km}, {user: None}) - self.dest = User(ADDRESS, 'gateway.leap.se', self.proto, ADDRESS_2) - - d = KeyManagerWithSoledadTestCase.setUp(self) - d.addCallback(init_outgoing_and_proto) - return d - - def test_message_encrypt(self): - """ - Test if message gets encrypted to destination email. - """ - def check_decryption(res): - decrypted, _ = res - self.assertEqual( - '\n' + self.expected_body, - decrypted, - 'Decrypted text differs from plaintext.') - - d = self._set_sign_used(ADDRESS) - d.addCallback( - lambda _: - self.outgoing_mail._maybe_encrypt_and_sign(self.raw, self.dest)) - d.addCallback(self._assert_encrypted) - d.addCallback(lambda message: self.km.decrypt( - message.get_payload(1).get_payload(), ADDRESS)) - d.addCallback(check_decryption) - return d - - def test_message_encrypt_sign(self): - """ - Test if message gets encrypted to destination email and signed with - sender key. - '""" - def check_decryption_and_verify(res): - decrypted, signkey = res - self.assertEqual( - '\n' + self.expected_body, - decrypted, - 'Decrypted text differs from plaintext.') - self.assertTrue(ADDRESS_2 in signkey.address, - "Verification failed") - - d = self._set_sign_used(ADDRESS) - d.addCallback( - lambda _: - self.outgoing_mail._maybe_encrypt_and_sign(self.raw, self.dest)) - d.addCallback(self._assert_encrypted) - d.addCallback(lambda message: self.km.decrypt( - message.get_payload(1).get_payload(), ADDRESS, verify=ADDRESS_2)) - d.addCallback(check_decryption_and_verify) - return d - - def test_message_sign(self): - """ - Test if message is signed with sender key. - """ - # mock the key fetching - self.km._fetch_keys_from_server = Mock( - return_value=fail(errors.KeyNotFound())) - recipient = User('ihavenopubkey@nonleap.se', - 'gateway.leap.se', self.proto, ADDRESS) - self.outgoing_mail = OutgoingMail( - self.fromAddr, self.km, self.opts.cert, self.opts.key, - self.opts.hostname, self.opts.port) - - def check_signed(res): - message, _ = res - self.assertTrue('Content-Type' in message) - self.assertEqual('multipart/signed', message.get_content_type()) - self.assertEqual('application/pgp-signature', - message.get_param('protocol')) - self.assertEqual('pgp-sha512', message.get_param('micalg')) - # assert content of message - body = (message.get_payload(0) - .get_payload(0) - .get_payload(decode=True)) - self.assertEqual(self.expected_body, - body) - # assert content of signature - self.assertTrue( - message.get_payload(1).get_payload().startswith( - '-----BEGIN PGP SIGNATURE-----\n'), - 'Message does not start with signature header.') - self.assertTrue( - message.get_payload(1).get_payload().endswith( - '-----END PGP SIGNATURE-----\n'), - 'Message does not end with signature footer.') - return message - - def verify(message): - # replace EOL before verifying (according to rfc3156) - fp = StringIO() - g = RFC3156CompliantGenerator( - fp, mangle_from_=False, maxheaderlen=76) - g.flatten(message.get_payload(0)) - signed_text = re.sub('\r?\n', '\r\n', - fp.getvalue()) - - def assert_verify(key): - self.assertTrue(ADDRESS_2 in key.address, - 'Signature could not be verified.') - - d = self.km.verify( - signed_text, ADDRESS_2, - detached_sig=message.get_payload(1).get_payload()) - d.addCallback(assert_verify) - return d - - d = self.outgoing_mail._maybe_encrypt_and_sign(self.raw, recipient) - d.addCallback(check_signed) - d.addCallback(verify) - return d - - def test_attach_key(self): - d = self.outgoing_mail._maybe_encrypt_and_sign(self.raw, self.dest) - d.addCallback(self._assert_encrypted) - d.addCallback(self._check_headers, self.lines[:4]) - d.addCallback(lambda message: self.km.decrypt( - message.get_payload(1).get_payload(), ADDRESS)) - d.addCallback(lambda (decrypted, _): - self._check_key_attachment(Parser().parsestr(decrypted))) - return d - - def test_attach_key_not_known(self): - unknown_address = "someunknownaddress@somewhere.com" - lines = deepcopy(self.lines) - lines[1] = "To: <%s>" % (unknown_address,) - raw = '\r\n'.join(lines) - dest = User(unknown_address, 'gateway.leap.se', self.proto, ADDRESS_2) - - d = self.outgoing_mail._maybe_encrypt_and_sign( - raw, dest, fetch_remote=False) - d.addCallback(lambda (message, _): - self._check_headers(message, lines[:4])) - d.addCallback(self._check_key_attachment) - d.addErrback(log.err) - return d - - def _check_headers(self, message, headers): - msgstr = message.as_string(unixfrom=False) - for header in headers: - self.assertTrue(header in msgstr, - "Missing header: %s" % (header,)) - return message - - def _check_key_attachment(self, message): - for payload in message.get_payload(): - if payload.is_multipart(): - return self._check_key_attachment(payload) - if 'application/pgp-keys' == payload.get_content_type(): - keylines = PUBLIC_KEY_2.split('\n') - key = BEGIN_PUBLIC_KEY + '\n\n' + '\n'.join(keylines[4:-1]) - self.assertTrue(key in payload.get_payload(decode=True), - "Key attachment don't match") - return - self.fail("No public key attachment found") - - def _set_sign_used(self, address): - def set_sign(key): - key.sign_used = True - return self.km.put_key(key) - - d = self.km.get_key(address, fetch_remote=False) - d.addCallback(set_sign) - return d - - def _assert_encrypted(self, res): - message, _ = res - self.assertTrue('Content-Type' in message) - self.assertEqual('multipart/encrypted', message.get_content_type()) - self.assertEqual('application/pgp-encrypted', - message.get_param('protocol')) - self.assertEqual(2, len(message.get_payload())) - self.assertEqual('application/pgp-encrypted', - message.get_payload(0).get_content_type()) - self.assertEqual('application/octet-stream', - message.get_payload(1).get_content_type()) - return message diff --git a/mail/src/leap/mail/plugins/__init__.py b/mail/src/leap/mail/plugins/__init__.py deleted file mode 100644 index ddb8691..0000000 --- a/mail/src/leap/mail/plugins/__init__.py +++ /dev/null @@ -1,3 +0,0 @@ -from twisted.plugin import pluginPackagePaths -__path__.extend(pluginPackagePaths(__name__)) -__all__ = [] diff --git a/mail/src/leap/mail/plugins/soledad_sync_hooks.py b/mail/src/leap/mail/plugins/soledad_sync_hooks.py deleted file mode 100644 index 9d48126..0000000 --- a/mail/src/leap/mail/plugins/soledad_sync_hooks.py +++ /dev/null @@ -1,19 +0,0 @@ -# -*- coding: utf-8 -*- -# soledad_sync_hooks.py -# Copyright (C) 2015 LEAP -# -# This program is free software: you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation, either version 3 of the License, or -# (at your option) any later version. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with this program. If not, see . - -from leap.mail.sync_hooks import MailProcessingPostSyncHook -post_sync_uid_reindexer = MailProcessingPostSyncHook() diff --git a/mail/src/leap/mail/rfc3156.py b/mail/src/leap/mail/rfc3156.py deleted file mode 100644 index 7d7bc0f..0000000 --- a/mail/src/leap/mail/rfc3156.py +++ /dev/null @@ -1,390 +0,0 @@ -# -*- coding: utf-8 -*- -# rfc3156.py -# Copyright (C) 2013 LEAP -# -# This program is free software: you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation, either version 3 of the License, or -# (at your option) any later version. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with this program. If not, see . - -""" -Implements RFC 3156: MIME Security with OpenPGP. -""" - -import base64 -from StringIO import StringIO - -from twisted.python import log -from email.mime.application import MIMEApplication -from email.mime.multipart import MIMEMultipart -from email import errors -from email.generator import ( - Generator, - fcre, - NL, - _make_boundary, -) - - -# -# A generator that solves http://bugs.python.org/issue14983 -# - -class RFC3156CompliantGenerator(Generator): - """ - An email generator that addresses Python's issue #14983 for multipart - messages. - - This is just a copy of email.generator.Generator which fixes the following - bug: http://bugs.python.org/issue14983 - """ - - def _handle_multipart(self, msg): - """ - A multipart handling implementation that addresses issue #14983. - - This is just a copy of the parent's method which fixes the following - bug: http://bugs.python.org/issue14983 (see the line marked with - "(***)"). - - :param msg: The multipart message to be handled. - :type msg: email.message.Message - """ - # The trick here is to write out each part separately, merge them all - # together, and then make sure that the boundary we've chosen isn't - # present in the payload. - msgtexts = [] - subparts = msg.get_payload() - if subparts is None: - subparts = [] - elif isinstance(subparts, basestring): - # e.g. a non-strict parse of a message with no starting boundary. - self._fp.write(subparts) - return - elif not isinstance(subparts, list): - # Scalar payload - subparts = [subparts] - for part in subparts: - s = StringIO() - g = self.clone(s) - g.flatten(part, unixfrom=False) - msgtexts.append(s.getvalue()) - # BAW: What about boundaries that are wrapped in double-quotes? - boundary = msg.get_boundary() - if not boundary: - # Create a boundary that doesn't appear in any of the - # message texts. - alltext = NL.join(msgtexts) - boundary = _make_boundary(alltext) - msg.set_boundary(boundary) - # If there's a preamble, write it out, with a trailing CRLF - if msg.preamble is not None: - preamble = msg.preamble - if self._mangle_from_: - preamble = fcre.sub('>From ', msg.preamble) - self._fp.write(preamble + '\n') - # dash-boundary transport-padding CRLF - self._fp.write('--' + boundary + '\n') - # body-part - if msgtexts: - self._fp.write(msgtexts.pop(0)) - # *encapsulation - # --> delimiter transport-padding - # --> CRLF body-part - for body_part in msgtexts: - # delimiter transport-padding CRLF - self._fp.write('\n--' + boundary + '\n') - # body-part - self._fp.write(body_part) - # close-delimiter transport-padding - self._fp.write('\n--' + boundary + '--' + '\n') # (***) Solve #14983 - if msg.epilogue is not None: - self._fp.write('\n') - epilogue = msg.epilogue - if self._mangle_from_: - epilogue = fcre.sub('>From ', msg.epilogue) - self._fp.write(epilogue) - - -# -# Base64 encoding: these are almost the same as python's email.encoder -# solution, but a bit modified. -# - -def _bencode(s): - """ - Encode C{s} in base64. - - :param s: The string to be encoded. - :type s: str - """ - # We can't quite use base64.encodestring() since it tacks on a "courtesy - # newline". Blech! - if not s: - return s - value = base64.encodestring(s) - return value[:-1] - - -def encode_base64(msg): - """ - Encode a non-multipart message's payload in Base64 (in place). - - This method modifies the message contents in place and adds or replaces an - appropriate Content-Transfer-Encoding header. - - :param msg: The non-multipart message to be encoded. - :type msg: email.message.Message - """ - encoding = msg.get('Content-Transfer-Encoding', None) - if encoding is not None: - encoding = encoding.lower() - # XXX Python's email module can only decode quoted-printable, base64 and - # uuencoded data, so we might have to implement other decoding schemes in - # order to support RFC 3156 properly and correctly calculate signatures - # for multipart attachments (eg. 7bit or 8bit encoded attachments). For - # now, if content is already encoded as base64 or if it is encoded with - # some unknown encoding, we just pass. - if encoding in [None, 'quoted-printable', 'x-uuencode', 'uue', 'x-uue']: - orig = msg.get_payload(decode=True) - encdata = _bencode(orig) - msg.set_payload(encdata) - # replace or set the Content-Transfer-Encoding header. - try: - msg.replace_header('Content-Transfer-Encoding', 'base64') - except KeyError: - msg['Content-Transfer-Encoding'] = 'base64' - elif encoding is not 'base64': - log.err('Unknown content-transfer-encoding: %s' % encoding) - - -def encode_base64_rec(msg): - """ - Encode (possibly multipart) messages in base64 (in place). - - This method modifies the message contents in place. - - :param msg: The non-multipart message to be encoded. - :type msg: email.message.Message - """ - if not msg.is_multipart(): - encode_base64(msg) - else: - for sub in msg.get_payload(): - encode_base64_rec(sub) - - -# -# RFC 1847: multipart/signed and multipart/encrypted -# - -class MultipartSigned(MIMEMultipart): - """ - Multipart/Signed MIME message according to RFC 1847. - - 2.1. Definition of Multipart/Signed - - (1) MIME type name: multipart - (2) MIME subtype name: signed - (3) Required parameters: boundary, protocol, and micalg - (4) Optional parameters: none - (5) Security considerations: Must be treated as opaque while in - transit - - The multipart/signed content type contains exactly two body parts. - The first body part is the body part over which the digital signature - was created, including its MIME headers. The second body part - contains the control information necessary to verify the digital - signature. The first body part may contain any valid MIME content - type, labeled accordingly. The second body part is labeled according - to the value of the protocol parameter. - - When the OpenPGP digital signature is generated: - - (1) The data to be signed MUST first be converted to its content- - type specific canonical form. For text/plain, this means - conversion to an appropriate character set and conversion of - line endings to the canonical sequence. - - (2) An appropriate Content-Transfer-Encoding is then applied; see - section 3. In particular, line endings in the encoded data - MUST use the canonical sequence where appropriate - (note that the canonical line ending may or may not be present - on the last line of encoded data and MUST NOT be included in - the signature if absent). - - (3) MIME content headers are then added to the body, each ending - with the canonical sequence. - - (4) As described in section 3 of this document, any trailing - whitespace MUST then be removed from the signed material. - - (5) As described in [2], the digital signature MUST be calculated - over both the data to be signed and its set of content headers. - - (6) The signature MUST be generated detached from the signed data - so that the process does not alter the signed data in any way. - """ - - def __init__(self, protocol, micalg, boundary=None, _subparts=None): - """ - Initialize the multipart/signed message. - - :param boundary: the multipart boundary string. By default it is - calculated as needed. - :type boundary: str - :param _subparts: a sequence of initial subparts for the payload. It - must be an iterable object, such as a list. You can always - attach new subparts to the message by using the attach() method. - :type _subparts: iterable - """ - MIMEMultipart.__init__( - self, _subtype='signed', boundary=boundary, - _subparts=_subparts) - self.set_param('protocol', protocol) - self.set_param('micalg', micalg) - - def attach(self, payload): - """ - Add the C{payload} to the current payload list. - - Also prevent from adding payloads with wrong Content-Type and from - exceeding a maximum of 2 payloads. - - :param payload: The payload to be attached. - :type payload: email.message.Message - """ - # second payload's content type must be equal to the protocol - # parameter given on object creation - if len(self.get_payload()) == 1: - if payload.get_content_type() != self.get_param('protocol'): - raise errors.MultipartConversionError( - 'Wrong content type %s.' % payload.get_content_type) - # prevent from adding more payloads - if len(self._payload) == 2: - raise errors.MultipartConversionError( - 'Cannot have more than two subparts.') - MIMEMultipart.attach(self, payload) - - -class MultipartEncrypted(MIMEMultipart): - """ - Multipart/encrypted MIME message according to RFC 1847. - - 2.2. Definition of Multipart/Encrypted - - (1) MIME type name: multipart - (2) MIME subtype name: encrypted - (3) Required parameters: boundary, protocol - (4) Optional parameters: none - (5) Security considerations: none - - The multipart/encrypted content type contains exactly two body parts. - The first body part contains the control information necessary to - decrypt the data in the second body part and is labeled according to - the value of the protocol parameter. The second body part contains - the data which was encrypted and is always labeled - application/octet-stream. - """ - - def __init__(self, protocol, boundary=None, _subparts=None): - """ - :param protocol: The encryption protocol to be added as a parameter to - the Content-Type header. - :type protocol: str - :param boundary: the multipart boundary string. By default it is - calculated as needed. - :type boundary: str - :param _subparts: a sequence of initial subparts for the payload. It - must be an iterable object, such as a list. You can always - attach new subparts to the message by using the attach() method. - :type _subparts: iterable - """ - MIMEMultipart.__init__( - self, _subtype='encrypted', boundary=boundary, - _subparts=_subparts) - self.set_param('protocol', protocol) - - def attach(self, payload): - """ - Add the C{payload} to the current payload list. - - Also prevent from adding payloads with wrong Content-Type and from - exceeding a maximum of 2 payloads. - - :param payload: The payload to be attached. - :type payload: email.message.Message - """ - # first payload's content type must be equal to the protocol parameter - # given on object creation - if len(self._payload) == 0: - if payload.get_content_type() != self.get_param('protocol'): - raise errors.MultipartConversionError( - 'Wrong content type.') - # second payload is always application/octet-stream - if len(self._payload) == 1: - if payload.get_content_type() != 'application/octet-stream': - raise errors.MultipartConversionError( - 'Wrong content type %s.' % payload.get_content_type) - # prevent from adding more payloads - if len(self._payload) == 2: - raise errors.MultipartConversionError( - 'Cannot have more than two subparts.') - MIMEMultipart.attach(self, payload) - - -# -# RFC 3156: application/pgp-encrypted, application/pgp-signed and -# application-pgp-signature. -# - -class PGPEncrypted(MIMEApplication): - """ - Application/pgp-encrypted MIME media type according to RFC 3156. - - * MIME media type name: application - * MIME subtype name: pgp-encrypted - * Required parameters: none - * Optional parameters: none - """ - - def __init__(self, version=1): - data = "Version: %d" % version - MIMEApplication.__init__(self, data, 'pgp-encrypted') - - -class PGPSignature(MIMEApplication): - """ - Application/pgp-signature MIME media type according to RFC 3156. - - * MIME media type name: application - * MIME subtype name: pgp-signature - * Required parameters: none - * Optional parameters: none - """ - def __init__(self, _data, name='signature.asc'): - MIMEApplication.__init__(self, _data, 'pgp-signature', - _encoder=lambda x: x, name=name) - self.add_header('Content-Description', 'OpenPGP Digital Signature') - - -class PGPKeys(MIMEApplication): - """ - Application/pgp-keys MIME media type according to RFC 3156. - - * MIME media type name: application - * MIME subtype name: pgp-keys - * Required parameters: none - * Optional parameters: none - """ - - def __init__(self, _data): - MIMEApplication.__init__(self, _data, 'pgp-keys') diff --git a/mail/src/leap/mail/size.py b/mail/src/leap/mail/size.py deleted file mode 100644 index c9eaabd..0000000 --- a/mail/src/leap/mail/size.py +++ /dev/null @@ -1,57 +0,0 @@ -# -*- coding: utf-8 -*- -# size.py -# Copyright (C) 2014 LEAP -# -# This program is free software: you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation, either version 3 of the License, or -# (at your option) any later version. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with this program. If not, see . -""" -Recursively get size of objects. -""" -from gc import collect -from itertools import chain -from sys import getsizeof - - -def _get_size(item, seen): - known_types = {dict: lambda d: chain.from_iterable(d.items())} - default_size = getsizeof(0) - - def size_walk(item): - if id(item) in seen: - return 0 - seen.add(id(item)) - s = getsizeof(item, default_size) - for _type, fun in known_types.iteritems(): - if isinstance(item, _type): - s += sum(map(size_walk, fun(item))) - break - return s - - return size_walk(item) - - -def get_size(item): - """ - Return the cumulative size of a given object. - - Currently it supports only dictionaries, and seemingly leaks - some memory, so use with care. - - :param item: the item which size wants to be computed - :rtype: int - """ - seen = set() - size = _get_size(item, seen) - del seen - collect() - return size diff --git a/mail/src/leap/mail/smtp/README.rst b/mail/src/leap/mail/smtp/README.rst deleted file mode 100644 index 1d3a903..0000000 --- a/mail/src/leap/mail/smtp/README.rst +++ /dev/null @@ -1,44 +0,0 @@ -Leap SMTP Gateway -================= - -The Bitmask Client runs a thin SMTP gateway on the user's device, which -intends to encrypt and sign outgoing messages to achieve point to point -encryption. - -The gateway is bound to localhost and the user's MUA should be configured to -send messages to it. After doing its thing, the gateway will relay the -messages to the remote SMTP server. - -Outgoing mail workflow: - - * SMTP gateway receives a message from the MUA. - - * SMTP gateway queries Key Manager for the user's private key. - - * For each recipient (including addresses in "To", "Cc" anc "Bcc" fields), - the following happens: - - - The recipient's address is validated against RFC2822. - - - An attempt is made to fetch the recipient's public PGP key. - - - If key is not found: - - - If the gateway is configured to only send encrypted messages the - recipient is rejected. - - - Otherwise, the message is signed and sent as plain text. - - - If the key is found, the message is encrypted to the recipient and - signed with the sender's private PGP key. - - * Finally, one message for each recipient is gatewayed to provider's SMTP - server. - - -Running tests -------------- - -Tests are run using Twisted's Trial API, like this:: - - python setup.py test -s leap.mail.gateway.tests diff --git a/mail/src/leap/mail/smtp/__init__.py b/mail/src/leap/mail/smtp/__init__.py deleted file mode 100644 index 9fab70a..0000000 --- a/mail/src/leap/mail/smtp/__init__.py +++ /dev/null @@ -1,73 +0,0 @@ -# -*- coding: utf-8 -*- -# __init__.py -# Copyright (C) 2013 LEAP -# -# This program is free software: you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation, either version 3 of the License, or -# (at your option) any later version. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with this program. If not, see . - -""" -SMTP gateway helper function. -""" -import logging -import os - -from twisted.internet import reactor -from twisted.internet.error import CannotListenError - -from leap.common.events import emit_async, catalog - -from leap.mail.smtp.gateway import SMTPFactory - -logger = logging.getLogger(__name__) - - -SMTP_PORT = 2013 - - -def run_service(soledad_sessions, keymanager_sessions, sendmail_opts, - port=SMTP_PORT): - """ - Main entry point to run the service from the client. - - :param soledad_sessions: a dict-like object, containing instances - of a Store (soledad instances), indexed by userid. - :param keymanager_sessions: a dict-like object, containing instances - of Keymanager, indexed by userid. - :param sendmail_opts: a dict-like object of sendmailOptions. - - :returns: the port as returned by the reactor when starts listening, and - the factory for the protocol. - :rtype: tuple - """ - factory = SMTPFactory(soledad_sessions, keymanager_sessions, - sendmail_opts) - - try: - interface = "localhost" - # don't bind just to localhost if we are running on docker since we - # won't be able to access smtp from the host - if os.environ.get("LEAP_DOCKERIZED"): - interface = '' - - # TODO Use Endpoints instead -------------------------------- - tport = reactor.listenTCP(port, factory, interface=interface) - emit_async(catalog.SMTP_SERVICE_STARTED, str(port)) - - return factory, tport - except CannotListenError: - logger.error("STMP Service failed to start: " - "cannot listen in port %s" % port) - emit_async(catalog.SMTP_SERVICE_FAILED_TO_START, str(port)) - except Exception as exc: - logger.error("Unhandled error while launching smtp gateway service") - logger.exception(exc) diff --git a/mail/src/leap/mail/smtp/bounces.py b/mail/src/leap/mail/smtp/bounces.py deleted file mode 100644 index 7a4674b..0000000 --- a/mail/src/leap/mail/smtp/bounces.py +++ /dev/null @@ -1,90 +0,0 @@ -# -*- coding: utf-8 -*- -# bounces.py -# Copyright (C) 2016 LEAP -# -# This program is free software: you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation, either version 3 of the License, or -# (at your option) any later version. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with this program. If not, see . -""" -Deliver bounces to the user Inbox. -""" -import time -from email.message import Message -from email.utils import formatdate - -from leap.mail.constants import INBOX_NAME -from leap.mail.mail import Account - - -# TODO implement localization for this template. - -BOUNCE_TEMPLATE = """This is your local Bitmask Mail Agent running at localhost. - -I'm sorry to have to inform you that your message could not be delivered to one -or more recipients. - -The reasons I got for the error are: - -{raw_error} - -If the problem persists and it's not a network connectivity issue, you might -want to contact your provider ({provider}) with this information (remove any -sensitive data before). - ---- Original message (*before* it was encrypted by bitmask) below ----: - -{orig}""" - - -class Bouncer(object): - """ - Implements a mechanism to deliver bounces to user inbox. - """ - # TODO this should follow RFC 6522, and compose a correct multipart - # attaching the report and the original message. Leaving this for a future - # iteration. - - def __init__(self, inbox_collection): - self._inbox_collection = inbox_collection - - def bounce_message(self, error_data, to, date=None, orig=''): - if not date: - date = formatdate(time.time()) - - raw_data = self._format_msg(error_data, to, date, orig) - d = self._inbox_collection.add_msg( - raw_data, ('\\Recent',), date=date) - return d - - def _format_msg(self, error_data, to, date, orig): - provider = to.split('@')[1] - - msg = Message() - msg.add_header( - 'From', 'bitmask-bouncer@localhost (Bitmask Local Agent)') - msg.add_header('To', to) - msg.add_header('Subject', 'Undelivered Message') - msg.add_header('Date', date) - msg.set_payload(BOUNCE_TEMPLATE.format( - raw_error=error_data, - provider=provider, - orig=orig)) - - return msg.as_string() - - -def bouncerFactory(soledad): - user_id = soledad.uuid - acc = Account(soledad, user_id) - d = acc.callWhenReady(lambda _: acc.get_collection_by_mailbox(INBOX_NAME)) - d.addCallback(lambda inbox: Bouncer(inbox)) - return d diff --git a/mail/src/leap/mail/smtp/gateway.py b/mail/src/leap/mail/smtp/gateway.py deleted file mode 100644 index e49bbe8..0000000 --- a/mail/src/leap/mail/smtp/gateway.py +++ /dev/null @@ -1,413 +0,0 @@ -# -*- coding: utf-8 -*- -# gateway.py -# Copyright (C) 2013 LEAP -# -# This program is free software: you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation, either version 3 of the License, or -# (at your option) any later version. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with this program. If not, see . -""" -LEAP SMTP encrypted gateway. - -The following classes comprise the SMTP gateway service: - - * SMTPFactory - A twisted.internet.protocol.ServerFactory that provides - the SMTPDelivery protocol. - - * SMTPDelivery - A twisted.mail.smtp.IMessageDelivery implementation. It - knows how to validate sender and receiver of messages and it generates - an EncryptedMessage for each recipient. - - * EncryptedMessage - An implementation of twisted.mail.smtp.IMessage that - knows how to encrypt/sign itself before sending. -""" -from email.Header import Header - -from zope.interface import implements -from zope.interface import implementer - -from twisted.cred.portal import Portal, IRealm -from twisted.mail import smtp -from twisted.mail.imap4 import LOGINCredentials, PLAINCredentials -from twisted.internet import defer, protocol -from twisted.python import log - -from leap.common.check import leap_assert_type -from leap.common.events import emit_async, catalog -from leap.mail import errors -from leap.mail.cred import LocalSoledadTokenChecker -from leap.mail.utils import validate_address -from leap.mail.rfc3156 import RFC3156CompliantGenerator -from leap.mail.outgoing.service import outgoingFactory -from leap.mail.smtp.bounces import bouncerFactory -from leap.keymanager.errors import KeyNotFound - -# replace email generator with a RFC 3156 compliant one. -from email import generator - -generator.Generator = RFC3156CompliantGenerator - - -LOCAL_FQDN = "bitmask.local" - - -@implementer(IRealm) -class LocalSMTPRealm(object): - - _encoding = 'utf-8' - - def __init__(self, keymanager_sessions, soledad_sessions, sendmail_opts, - encrypted_only=False): - """ - :param keymanager_sessions: a dict-like object, containing instances - of a Keymanager objects, indexed by - userid. - """ - self._keymanager_sessions = keymanager_sessions - self._soledad_sessions = soledad_sessions - self._sendmail_opts = sendmail_opts - self.encrypted_only = encrypted_only - - def requestAvatar(self, avatarId, mind, *interfaces): - - if isinstance(avatarId, str): - avatarId = avatarId.decode(self._encoding) - - def gotKeymanagerAndSoledad(result): - keymanager, soledad = result - d = bouncerFactory(soledad) - d.addCallback(lambda bouncer: (keymanager, soledad, bouncer)) - return d - - def getMessageDelivery(result): - keymanager, soledad, bouncer = result - # TODO use IMessageDeliveryFactory instead ? - # it could reuse the connections. - if smtp.IMessageDelivery in interfaces: - userid = avatarId - opts = self.getSendingOpts(userid) - - outgoing = outgoingFactory( - userid, keymanager, opts, bouncer=bouncer) - avatar = SMTPDelivery(userid, keymanager, self.encrypted_only, - outgoing) - - return (smtp.IMessageDelivery, avatar, - getattr(avatar, 'logout', lambda: None)) - - raise NotImplementedError(self, interfaces) - - d1 = self.lookupKeymanagerInstance(avatarId) - d2 = self.lookupSoledadInstance(avatarId) - d = defer.gatherResults([d1, d2]) - d.addCallback(gotKeymanagerAndSoledad) - d.addCallback(getMessageDelivery) - return d - - def lookupKeymanagerInstance(self, userid): - try: - keymanager = self._keymanager_sessions[userid] - except: - raise errors.AuthenticationError( - 'No keymanager session found for user %s. Is it authenticated?' - % userid) - # XXX this should return the instance after whenReady callback - return defer.succeed(keymanager) - - def lookupSoledadInstance(self, userid): - try: - soledad = self._soledad_sessions[userid] - except: - raise errors.AuthenticationError( - 'No soledad session found for user %s. Is it authenticated?' - % userid) - # XXX this should return the instance after whenReady callback - return defer.succeed(soledad) - - def getSendingOpts(self, userid): - try: - opts = self._sendmail_opts[userid] - except KeyError: - raise errors.ConfigurationError( - 'No sendingMail options found for user %s' % userid) - return opts - - -class SMTPTokenChecker(LocalSoledadTokenChecker): - """A credentials checker that will lookup a token for the SMTP service. - For now it will be using the same identifier than IMAPTokenChecker""" - - service = 'mail_auth' - - # TODO besides checking for token credential, - # we could also verify the certificate here. - - -class LEAPInitMixin(object): - - """ - A Mixin that takes care of initialization of all the data needed to access - LEAP sessions. - """ - def __init__(self, soledad_sessions, keymanager_sessions, sendmail_opts, - encrypted_only=False): - realm = LocalSMTPRealm( - keymanager_sessions, soledad_sessions, sendmail_opts, - encrypted_only) - portal = Portal(realm) - - checker = SMTPTokenChecker(soledad_sessions) - self.checker = checker - self.portal = portal - portal.registerChecker(checker) - - -class LocalSMTPServer(smtp.ESMTP, LEAPInitMixin): - """ - The Production ESMTP Server: Authentication Needed. - Authenticates against SMTP Token stored in Local Soledad instance. - The Realm will produce a Delivery Object that handles encryption/signing. - """ - - # TODO: implement Queue using twisted.mail.mail.MailService - - def __init__(self, soledads, keyms, sendmailopts, *args, **kw): - encrypted_only = kw.pop('encrypted_only', False) - - LEAPInitMixin.__init__(self, soledads, keyms, sendmailopts, - encrypted_only) - smtp.ESMTP.__init__(self, *args, **kw) - - -# TODO implement retries -- see smtp.SenderMixin -class SMTPFactory(protocol.ServerFactory): - """ - Factory for an SMTP server with encrypted gatewaying capabilities. - """ - - protocol = LocalSMTPServer - domain = LOCAL_FQDN - timeout = 600 - encrypted_only = False - - def __init__(self, soledad_sessions, keymanager_sessions, sendmail_opts, - deferred=None, retries=3): - - self._soledad_sessions = soledad_sessions - self._keymanager_sessions = keymanager_sessions - self._sendmail_opts = sendmail_opts - - def buildProtocol(self, addr): - p = self.protocol( - self._soledad_sessions, self._keymanager_sessions, - self._sendmail_opts, encrypted_only=self.encrypted_only) - p.factory = self - p.host = LOCAL_FQDN - p.challengers = {"LOGIN": LOGINCredentials, "PLAIN": PLAINCredentials} - return p - - -# -# SMTPDelivery -# - -@implementer(smtp.IMessageDelivery) -class SMTPDelivery(object): - """ - Validate email addresses and handle message delivery. - """ - - def __init__(self, userid, keymanager, encrypted_only, outgoing_mail): - """ - Initialize the SMTP delivery object. - - :param userid: The user currently logged in - :type userid: unicode - :param keymanager: A Key Manager from where to get recipients' public - keys. - :param encrypted_only: Whether the SMTP gateway should send unencrypted - mail or not. - :type encrypted_only: bool - :param outgoing_mail: The outgoing mail to send the message - :type outgoing_mail: leap.mail.outgoing.service.OutgoingMail - """ - self._userid = userid - self._outgoing_mail = outgoing_mail - self._km = keymanager - self._encrypted_only = encrypted_only - self._origin = None - - def receivedHeader(self, helo, origin, recipients): - """ - Generate the 'Received:' header for a message. - - :param helo: The argument to the HELO command and the client's IP - address. - :type helo: (str, str) - :param origin: The address the message is from. - :type origin: twisted.mail.smtp.Address - :param recipients: A list of the addresses for which this message is - bound. - :type: list of twisted.mail.smtp.User - - @return: The full "Received" header string. - :type: str - """ - myHostname, clientIP = helo - headerValue = "by bitmask.local from %s with ESMTP ; %s" % ( - clientIP, smtp.rfc822date()) - # email.Header.Header used for automatic wrapping of long lines - return "Received: %s" % Header(s=headerValue, header_name='Received') - - def validateTo(self, user): - """ - Validate the address of a recipient of the message, possibly - rejecting it if the recipient key is not available. - - This method is called once for each recipient, i.e. for each SMTP - protocol line beginning with "RCPT TO:", which includes all addresses - in "To", "Cc" and "Bcc" MUA fields. - - The recipient's address is validated against the RFC 2822 definition. - If self._encrypted_only is True and no key is found for a recipient, - then that recipient is rejected. - - The method returns an encrypted message object that is able to send - itself to the user's address. - - :param user: The user whose address we wish to validate. - :type: twisted.mail.smtp.User - - @return: A callable which takes no arguments and returns an - encryptedMessage. - @rtype: no-argument callable - - @raise SMTPBadRcpt: Raised if messages to the address are not to be - accepted. - """ - # try to find recipient's public key - address = validate_address(user.dest.addrstr) - - # verify if recipient key is available in keyring - def found(_): - log.msg("Accepting mail for %s..." % user.dest.addrstr) - emit_async(catalog.SMTP_RECIPIENT_ACCEPTED_ENCRYPTED, - self._userid, user.dest.addrstr) - - def not_found(failure): - failure.trap(KeyNotFound) - - # if key was not found, check config to see if will send anyway - if self._encrypted_only: - emit_async(catalog.SMTP_RECIPIENT_REJECTED, self._userid, - user.dest.addrstr) - raise smtp.SMTPBadRcpt(user.dest.addrstr) - log.msg("Warning: will send an unencrypted message (because " - "encrypted_only' is set to False).") - emit_async( - catalog.SMTP_RECIPIENT_ACCEPTED_UNENCRYPTED, - self._userid, user.dest.addrstr) - - def encrypt_func(_): - return lambda: EncryptedMessage(user, self._outgoing_mail) - - d = self._km.get_key(address) - d.addCallbacks(found, not_found) - d.addCallback(encrypt_func) - return d - - def validateFrom(self, helo, origin): - """ - Validate the address from which the message originates. - - :param helo: The argument to the HELO command and the client's IP - address. - :type: (str, str) - :param origin: The address the message is from. - :type origin: twisted.mail.smtp.Address - - @return: origin or a Deferred whose callback will be passed origin. - @rtype: Deferred or Address - - @raise twisted.mail.smtp.SMTPBadSender: Raised if messages from this - address are not to be accepted. - """ - # accept mail from anywhere. To reject an address, raise - # smtp.SMTPBadSender here. - if str(origin) != str(self._userid): - log.msg("Rejecting sender {0}, expected {1}".format(origin, - self._userid)) - raise smtp.SMTPBadSender(origin) - self._origin = origin - return origin - - -# -# EncryptedMessage -# - -class EncryptedMessage(object): - """ - Receive plaintext from client, encrypt it and send message to a - recipient. - """ - implements(smtp.IMessage) - - def __init__(self, user, outgoing_mail): - """ - Initialize the encrypted message. - - :param user: The recipient of this message. - :type user: twisted.mail.smtp.User - :param outgoing_mail: The outgoing mail to send the message - :type outgoing_mail: leap.mail.outgoing.service.OutgoingMail - """ - # assert params - leap_assert_type(user, smtp.User) - - self._user = user - self._lines = [] - self._outgoing_mail = outgoing_mail - - def lineReceived(self, line): - """ - Handle another line. - - :param line: The received line. - :type line: str - """ - self._lines.append(line) - - def eomReceived(self): - """ - Handle end of message. - - This method will encrypt and send the message. - - :returns: a deferred - """ - log.msg("Message data complete.") - self._lines.append('') # add a trailing newline - raw_mail = '\r\n'.join(self._lines) - - return self._outgoing_mail.send_message(raw_mail, self._user) - - def connectionLost(self): - """ - Log an error when the connection is lost. - """ - log.msg("Connection lost unexpectedly!") - log.err() - emit_async(catalog.SMTP_CONNECTION_LOST, self._userid, - self._user.dest.addrstr) - # unexpected loss of connection; don't save - - self._lines = [] diff --git a/mail/src/leap/mail/smtp/tests/185CA770.key b/mail/src/leap/mail/smtp/tests/185CA770.key deleted file mode 100644 index 587b416..0000000 --- a/mail/src/leap/mail/smtp/tests/185CA770.key +++ /dev/null @@ -1,79 +0,0 @@ ------BEGIN PGP PRIVATE KEY BLOCK----- -Version: GnuPG v1.4.10 (GNU/Linux) - -lQIVBFCJNL4BEADFsI1TCD4yq7ZqL7VhdVviTuX6JUps8/mVEhRVOZhojLcTYaqQ -gs6T6WabRxcK7ymOnf4K8NhYdz6HFoJN46BT87etokx7J/Sl2OhpiqBQEY+jW8Rp -+3MSGrGmvFw0s1lGrz/cXzM7UNgWSTOnYZ5nJS1veMhy0jseZOUK7ekp2oEDjGZh -pzgd3zICCR2SvlpLIXB2Nr/CUcuRWTcc5LlKmbjMybu0E/uuY14st3JL+7qI6QX0 -atFm0VhFVpagOl0vWKxakUx4hC7j1wH2ADlCvSZPG0StSLUyHkJx3UPsmYxOZFao -ATED3Okjwga6E7PJEbzyqAkvzw/M973kaZCUSH75ZV0cQnpdgXV3DK1gSa3d3gug -W1lE0V7pwnN2NTOYfBMi+WloCs/bp4iZSr4QP1duZ3IqKraeBDCk7MoFo4A9Wk07 -kvqPwF9IBgatu62WVEZIzwyViN+asFUGfgp+8D7gtnlWAw0V6y/lSTzyl+dnLP98 -Hfr2eLBylFs+Kl3Pivpg2uHw09LLCrjeLEN3dj9SfBbA9jDIo9Zhs1voiIK/7Shx -E0BRJaBgG3C4QaytYEu7RFFOKuvBai9w2Y5OfsKFo8rA7v4dxFFDvzKGujCtNnwf -oyaGlZmMBU5MUmHUNiG8ON21COZBtK5oMScuY1VC9CQonj3OClg3IbU9SQARAQAB -/gNlAkdOVQG0JGRyZWJzIChncGcgdGVzdCBrZXkpIDxkcmVic0BsZWFwLnNlPokC -OAQTAQIAIgUCUIk0vgIbAwYLCQgHAwIGFQgCCQoLBBYCAwECHgECF4AACgkQty9e -xhhcp3Bdhw//bdPUNbp6rgIjRRuwYvGJ6IuiFuFWJQ0m3iAuuAoZo5GHAPqZAuGk -dMVYu0dtCtZ68MJ/QpjBCT9RRL+mgIgfLfUSj2ZknP4nb6baiG5u28l0KId/e5IC -iQKBnIsjxKxhLBVHSzRaS1P+vZeF2C2R9XyNy0eCnAwyCMcD0R8TVROGQ7i4ZQsM -bMj1LPpOwhV/EGp23nD+upWOVbn/wQHOYV2kMiA/8fizmWRIWsV4/68uMA+WDP4L -40AnJ0fcs04f9deM9P6pjlm00VD7qklYEGw6Mpr2g/M73kGh1nlAv+ImQBGlLMle -RXyzHY3WAhzmRKWO4koFuKeR9Q0EMzk2R4/kuagdWEpM+bhwE4xPV1tPZhn9qFTz -pQD4p/VT4qNQKOD0+aTFWre65Rt2cFFMLI7UmEHNLi0NB9JCIAi4+l+b9WQNlmaO -C8EhOGwRzmehUyHmXM3BNW28MnyKFJ7bBFMd7uJz+vAPOrr6OzuNvVCv2I2ICkTs -ihIj/zw5GXxkPO7YbMu9rKG0nKF1N3JB1gUJ78DHmhbjeaGSvHw85sPD0/1dPZK4 -8Gig8i62aCxf8OlJPlt8ZhBBolzs6ITUNa75Rw9fJsj3UWuv2VFaIuR57bFWmY3s -A9KPgdf7jVQlAZKlVyli7IkyaZmxDZNFQoTdIC9uo0aggIDP8zKv0n2dBz4EUIk0 -vgEQAOO8BAR7sBdqj2RRMRNeWSA4S9GuHfV3YQARnqYsbITs1jRgAo7jx9Z5C80c -ZOxOUVK7CJjtTqU0JB9QP/zwV9hk5i6y6aQTysclQyTNN10aXu/3zJla5Duhz+Cs -+5UcVAmNJX9FgTMVvhKDEIY/LNmb9MoBLMut1CkDx+WPCV45WOIBCDdj2HpIjie4 -phs0/65SWjPiVg3WsFZljVxpJCGXP48Eet2bf8afYH1lx3sQMcNbyJACIPtz+YKz -c7jIKwKSWzg1VyYikbk9eWCxcz6VKNJKi94YH9c7U8X3TdZ8G0kGYUldjYDvesyl -nuQlcGCtSGKOAhrN/Bu2R0gpFgYl247u79CmjotefMdv8BGUDW6u9/Sep9xN3dW8 -S87h6M/tvs0ChlkDDpJedzCd7ThdikGvFRJfW/8sT/+qoTKskySQaDIeNJnxZuyK -wELLMBvCZGpamwmnkEGhvuZWq0h/DwyTs4QAE8OVHXJSM3UN7hM4lJIUh+sRKJ1F -AXXTdSY4cUNaS+OKtj2LJ85zFqhfAZ4pFwLCgYbJtU5hej2LnMJNbYcSkjxbk+c5 -IjkoZRF+ExjZlc0VLYNT57ZriwZ/pX42ofjOyMR/dkHQuFik/4K7v1ZemfaTdm07 -SEMBknR6OZsy/5+viEtXiih3ptTMaT9row+g+cFoxdXkisKvABEBAAH+AwMCIlVK -Xs3x0Slgwx03cTNIoWXmishkPCJlEEdcjldz2VyQF9hjdp1VIe+npI26chKwCZqm -U8yYbJh4UBrugUUzKKd4EfnmKfu+/BsJciFRVKwBtiolIiUImzcHPWktYLwo9yzX -W42teShXXVgWmsJN1/6FqJdsLg8dxWesXMKoaNF4n1P7zx6vKBmDHTRz7PToaI/d -5/nKrjED7ZT1h+qR5i9UUgbvF0ySp8mlqk/KNqHUSLDB9kf/JDg4XVtPHGGd9Ik/ -60UJ7aDfohi4Z0VgwWmfLBwcQ3It+ENtnPFufH3WHW8c1UA4wVku9tOTqyrRG6tP -TZGiRfuwsv7Hq3pWT6rntbDkTiVgESM4C1fiZblc98iWUKGXSHqm+te1TwXOUCci -J/gryXcjQFM8A0rwA/m+EvsoWuzoqIl3x++p3/3/mGux6UD4O7OhJNRVRz+8Mhq1 -ksrR9XkQzpq3Yv3ulTHz7l+WCRRXxw5+XWAkRHHF47Vf/na38NJQHcsCBbRIuLYR -wBzS48cYzYkF6VejKThdQmdYJ0/fUrlUBCAJWgrfqCihFLDa1s4jJ16/fqi8a97Y -4raVy2hrF2vFc/wet13hsaddVn4rPRAMDEGdgEmJX7MmU1emT/yaIG9lvjMpI2c5 -ADXGF2yYYa7H8zPIFyHU1RSavlT0S/K9yzIZvv+jA5KbNeGp+WWFT8MLZs0IhoCZ -d1EgLUYAt7LPUSm2lBy1w/IL+VtYuyn/UVFo2xWiHd1ABiNWl1ji3X9Ki5613QqH -bvn4z46voCzdZ02rYkAwrdqDr92fiBR8ctwA0AudaG6nf2ztmFKtM3E/RPMkPgKF -8NHYc7QxS2jruJxXBtjRBMtoIaZ0+AXUO6WuEJrDLDHWaM08WKByQMm808xNCbRr -CpiK8qyR3SwkfaOMCp22mqViirQ2KfuVvBpBT2pBYlgDKs50nE+stDjUMv+FDKAo -5NtiyPfNtaBOYnXAEQb/hjjW5bKq7JxHSxIWAYKbNKIWgftJ3ACZAsBMHfaOCFNH -+XLojAoxOI+0zbN6FtjN+YMU1XrLd6K49v7GEiJQZVQSfLCecVDhDU9paNROA/Xq -/3nDCTKhd3stTPnc8ymLAwhTP0bSoFh/KtU96D9ZMC2cu9XZ+UcSQYES/ncZWcLw -wTKrt+VwBG1z3DbV2O0ruUiXTLcZMsrwbUSDx1RVhmKZ0i42AttMdauFQ9JaX2CS -2ddqFBS1b4X6+VCy44KkpdXsmp0NWMgm/PM3PTisCxrha7bI5/LqfXG0b+GuIFb4 -h/lEA0Ae0gMgkzm3ePAPPVlRj7kFl5Osjxm3YVRW23WWGDRF5ywIROlBjbdozA0a -MyMgXlG9hhJseIpFveoiwqenNE5Wxg0yQbnhMUTKeCQ0xskG82P+c9bvDsevAQUR -uv1JAGGxDd1/4nk0M5m9/Gf4Bn0uLAz29LdMg0FFUvAm2ol3U3uChm7OISU8dqFy -JdCFACKBMzAREiXfgH2TrTxAhpy5uVcUSQV8x5J8qJ/mUoTF1WE3meXEm9CIvIAF -Mz49KKebLS3zGFixMcKLAOKA+s/tUWO7ZZoJyQjvQVerLyDo6UixVb11LQUJQOXb -ZIuSKV7deCgBDQ26C42SpF3rHfEQa7XH7j7tl1IIW/9DfYJYVQHaz1NTq6zcjWS2 -e+cUexBPhxbadGn0zelXr6DLJqQT7kaVeYOHlkYUHkZXdHE4CWoHqOboeB02uM/A -e7nge1rDi57ySrsF4AVl59QJYBPR43AOVbCJAh8EGAECAAkFAlCJNL4CGwwACgkQ -ty9exhhcp3DetA/8D/IscSBlWY3TjCD2P7t3+X34USK8EFD3QJse9dnCWOLcskFQ -IoIfhRM752evFu2W9owEvxSQdG+otQAOqL72k1EH2g7LsADuV8I4LOYOnLyeIE9I -b+CFPBkmzTEzrdYp6ITUU7qqgkhcgnltKGHoektIjxE8gtxCKEdyxkzazum6nCQQ -kSBZOXVU3ezm+A2QHHP6XT1GEbdKbJ0tIuJR8ADu08pBx2c/LDBBreVStrrt1Dbz -uR+U8MJsfLVcYX/Rw3V+KA24oLRzg91y3cfi3sNU/kmd5Cw42Tj00B+FXQny51Mq -s4KyqHobj62II68eL5HRB2pcGsoaedQyxu2cYSeVyarBOiUPNYkoGDJoKdDyZRIB -NNK0W+ASTf0zeHhrY/okt1ybTVtvbt6wkTEbKVePUaYmNmhre1cAj4uNwFzYjkzJ -cm+8XWftD+TV8cE5DyVdnF00SPDuPzodRAPXaGpQUMLkE4RPr1TAwcuoPH9aFHZ/ -se6rw6TQHLd0vMk0U/DocikXpSJ1N6caE3lRwI/+nGfXNiCr8MIdofgkBeO86+G7 -k0UXS4v5FKk1nwTyt4PkFJDvAJX6rZPxIZ9NmtA5ao5vyu1DT5IhoXgDzwurAe8+ -R+y6gtA324hXIweFNt7SzYPfI4SAjunlmm8PIBf3owBrk3j+w6EQoaCreK4= -=6HcJ ------END PGP PRIVATE KEY BLOCK----- diff --git a/mail/src/leap/mail/smtp/tests/185CA770.pub b/mail/src/leap/mail/smtp/tests/185CA770.pub deleted file mode 100644 index 38af19f..0000000 --- a/mail/src/leap/mail/smtp/tests/185CA770.pub +++ /dev/null @@ -1,52 +0,0 @@ ------BEGIN PGP PUBLIC KEY BLOCK----- -Version: GnuPG v1.4.10 (GNU/Linux) - -mQINBFCJNL4BEADFsI1TCD4yq7ZqL7VhdVviTuX6JUps8/mVEhRVOZhojLcTYaqQ -gs6T6WabRxcK7ymOnf4K8NhYdz6HFoJN46BT87etokx7J/Sl2OhpiqBQEY+jW8Rp -+3MSGrGmvFw0s1lGrz/cXzM7UNgWSTOnYZ5nJS1veMhy0jseZOUK7ekp2oEDjGZh -pzgd3zICCR2SvlpLIXB2Nr/CUcuRWTcc5LlKmbjMybu0E/uuY14st3JL+7qI6QX0 -atFm0VhFVpagOl0vWKxakUx4hC7j1wH2ADlCvSZPG0StSLUyHkJx3UPsmYxOZFao -ATED3Okjwga6E7PJEbzyqAkvzw/M973kaZCUSH75ZV0cQnpdgXV3DK1gSa3d3gug -W1lE0V7pwnN2NTOYfBMi+WloCs/bp4iZSr4QP1duZ3IqKraeBDCk7MoFo4A9Wk07 -kvqPwF9IBgatu62WVEZIzwyViN+asFUGfgp+8D7gtnlWAw0V6y/lSTzyl+dnLP98 -Hfr2eLBylFs+Kl3Pivpg2uHw09LLCrjeLEN3dj9SfBbA9jDIo9Zhs1voiIK/7Shx -E0BRJaBgG3C4QaytYEu7RFFOKuvBai9w2Y5OfsKFo8rA7v4dxFFDvzKGujCtNnwf -oyaGlZmMBU5MUmHUNiG8ON21COZBtK5oMScuY1VC9CQonj3OClg3IbU9SQARAQAB -tCRkcmVicyAoZ3BnIHRlc3Qga2V5KSA8ZHJlYnNAbGVhcC5zZT6JAjgEEwECACIF -AlCJNL4CGwMGCwkIBwMCBhUIAgkKCwQWAgMBAh4BAheAAAoJELcvXsYYXKdwXYcP -/23T1DW6eq4CI0UbsGLxieiLohbhViUNJt4gLrgKGaORhwD6mQLhpHTFWLtHbQrW -evDCf0KYwQk/UUS/poCIHy31Eo9mZJz+J2+m2ohubtvJdCiHf3uSAokCgZyLI8Ss -YSwVR0s0WktT/r2XhdgtkfV8jctHgpwMMgjHA9EfE1UThkO4uGULDGzI9Sz6TsIV -fxBqdt5w/rqVjlW5/8EBzmFdpDIgP/H4s5lkSFrFeP+vLjAPlgz+C+NAJydH3LNO -H/XXjPT+qY5ZtNFQ+6pJWBBsOjKa9oPzO95BodZ5QL/iJkARpSzJXkV8sx2N1gIc -5kSljuJKBbinkfUNBDM5NkeP5LmoHVhKTPm4cBOMT1dbT2YZ/ahU86UA+Kf1U+Kj -UCjg9PmkxVq3uuUbdnBRTCyO1JhBzS4tDQfSQiAIuPpfm/VkDZZmjgvBIThsEc5n -oVMh5lzNwTVtvDJ8ihSe2wRTHe7ic/rwDzq6+js7jb1Qr9iNiApE7IoSI/88ORl8 -ZDzu2GzLvayhtJyhdTdyQdYFCe/Ax5oW43mhkrx8PObDw9P9XT2SuPBooPIutmgs -X/DpST5bfGYQQaJc7OiE1DWu+UcPXybI91Frr9lRWiLkee2xVpmN7APSj4HX+41U -JQGSpVcpYuyJMmmZsQ2TRUKE3SAvbqNGoICAz/Myr9J9uQINBFCJNL4BEADjvAQE -e7AXao9kUTETXlkgOEvRrh31d2EAEZ6mLGyE7NY0YAKO48fWeQvNHGTsTlFSuwiY -7U6lNCQfUD/88FfYZOYusumkE8rHJUMkzTddGl7v98yZWuQ7oc/grPuVHFQJjSV/ -RYEzFb4SgxCGPyzZm/TKASzLrdQpA8fljwleOVjiAQg3Y9h6SI4nuKYbNP+uUloz -4lYN1rBWZY1caSQhlz+PBHrdm3/Gn2B9Zcd7EDHDW8iQAiD7c/mCs3O4yCsCkls4 -NVcmIpG5PXlgsXM+lSjSSoveGB/XO1PF903WfBtJBmFJXY2A73rMpZ7kJXBgrUhi -jgIazfwbtkdIKRYGJduO7u/Qpo6LXnzHb/ARlA1urvf0nqfcTd3VvEvO4ejP7b7N -AoZZAw6SXncwne04XYpBrxUSX1v/LE//qqEyrJMkkGgyHjSZ8WbsisBCyzAbwmRq -WpsJp5BBob7mVqtIfw8Mk7OEABPDlR1yUjN1De4TOJSSFIfrESidRQF103UmOHFD -WkvjirY9iyfOcxaoXwGeKRcCwoGGybVOYXo9i5zCTW2HEpI8W5PnOSI5KGURfhMY -2ZXNFS2DU+e2a4sGf6V+NqH4zsjEf3ZB0LhYpP+Cu79WXpn2k3ZtO0hDAZJ0ejmb -Mv+fr4hLV4ood6bUzGk/a6MPoPnBaMXV5IrCrwARAQABiQIfBBgBAgAJBQJQiTS+ -AhsMAAoJELcvXsYYXKdw3rQP/A/yLHEgZVmN04wg9j+7d/l9+FEivBBQ90CbHvXZ -wlji3LJBUCKCH4UTO+dnrxbtlvaMBL8UkHRvqLUADqi+9pNRB9oOy7AA7lfCOCzm -Dpy8niBPSG/ghTwZJs0xM63WKeiE1FO6qoJIXIJ5bShh6HpLSI8RPILcQihHcsZM -2s7pupwkEJEgWTl1VN3s5vgNkBxz+l09RhG3SmydLSLiUfAA7tPKQcdnPywwQa3l -Ura67dQ287kflPDCbHy1XGF/0cN1figNuKC0c4Pdct3H4t7DVP5JneQsONk49NAf -hV0J8udTKrOCsqh6G4+tiCOvHi+R0QdqXBrKGnnUMsbtnGEnlcmqwTolDzWJKBgy -aCnQ8mUSATTStFvgEk39M3h4a2P6JLdcm01bb27esJExGylXj1GmJjZoa3tXAI+L -jcBc2I5MyXJvvF1n7Q/k1fHBOQ8lXZxdNEjw7j86HUQD12hqUFDC5BOET69UwMHL -qDx/WhR2f7Huq8Ok0By3dLzJNFPw6HIpF6UidTenGhN5UcCP/pxn1zYgq/DCHaH4 -JAXjvOvhu5NFF0uL+RSpNZ8E8reD5BSQ7wCV+q2T8SGfTZrQOWqOb8rtQ0+SIaF4 -A88LqwHvPkfsuoLQN9uIVyMHhTbe0s2D3yOEgI7p5ZpvDyAX96MAa5N4/sOhEKGg -q3iu -=RChS ------END PGP PUBLIC KEY BLOCK----- diff --git a/mail/src/leap/mail/smtp/tests/cert/server.crt b/mail/src/leap/mail/smtp/tests/cert/server.crt deleted file mode 100644 index a27391c..0000000 --- a/mail/src/leap/mail/smtp/tests/cert/server.crt +++ /dev/null @@ -1,29 +0,0 @@ ------BEGIN CERTIFICATE----- -MIIFBjCCAu4CCQCWn3oMoQrDJTANBgkqhkiG9w0BAQUFADBFMQswCQYDVQQGEwJV -UzETMBEGA1UECAwKU29tZS1TdGF0ZTEhMB8GA1UECgwYSW50ZXJuZXQgV2lkZ2l0 -cyBQdHkgTHRkMB4XDTEzMTAyMzE0NDUwNFoXDTE2MDcxOTE0NDUwNFowRTELMAkG -A1UEBhMCVVMxEzARBgNVBAgMClNvbWUtU3RhdGUxITAfBgNVBAoMGEludGVybmV0 -IFdpZGdpdHMgUHR5IEx0ZDCCAiIwDQYJKoZIhvcNAQEBBQADggIPADCCAgoCggIB -APexTvEvG7cSmZdAERHt9TB11cSor54Y/F7NmYMdSOJNi4Y0kwkSslpdfipi+mt/ -NFg/uGKi1mcgvuXdVbVPZ9rCgVpIzMncO8RAP7a5+I2zKUzqMCCbLH16sYpo/rDk -VQ5V15TwLsTzOFGG8Cgp68TR8zHuZ4Edf2zMGC1IaiJ6W38LTnJgsowYOCFDAF3z -L36kxMO5gNGEUYV6tjltx+rAcXka3po+xiAgvW6q65UUgDHcIdEGG2dc9bkxxPl7 -RkprF2RwwADNzYS7Tn+Hpmjy06pfYZHNME+Iw515bCRF3GQFUU4BpGnY7EO+h4P9 -Kb1h948gUT9/oswXG+q2Kwk8AoggMJkUOWDFiCa5UjW1GBoxxb7VtZ+QTJXxlFWc -M2VzT7M/HX+P4b05vY4MXJjxPAFKrAGS7J8DKW8WJNUnXa9XSDBHg5qijDzZ/zGm -HTdG6iADnJLmOHBQgFQ12a/n9mYV2GPVC6FlgDzG9f0/SUPBUCafyWYz1LwKY4VM -2NLx/iwYMQsNIMSZQfNmufNDBr70+BShe3ZpbmKB/J33d87AuJd2HjnsThTEAAr+ -6CejyYmwFutoDUCF8IaKGJEp7OGP2//ub4nt5WwW8DYLRi8EqtzEnxPo5ZiayHMY -GHR1jpX1O5JVJFUE79bZCFFHKmtJc4kVZS4m4rTLsk83AgMBAAEwDQYJKoZIhvcN -AQEFBQADggIBAEt4PIRqVuALQSdgZ+GiZYuvEVjxoDVtMSc/ym93Gi8R7DDivFH9 -4suQc5QUiuEF8lpEtkmh+PZ+oFdQkjhBH80h7p4BUSyBy5Yi6dy7ATTlBAqwzCYZ -4wzHeJzu1SI6FinZLksoULbcw04n410aGHkLa6I9O3vCC4kXSnBlwU1sUsJphxM2 -3pkHBpvv79XYf5kFqZPzF16aO7rxFuVvqgXLyzwuyP9kH5zMA21Kioxs/pNyg1lm -5h0VinpHLPse+4tYih1L1WLMpEZiSwZgFhoRtlcdIVXokZPaX4G2EkdrMmSQruWg -Uz8Av6LEYHmRfbYwYM2kEX/+AF8thpTQDbvxjqYk5oyGX4wpKGpih1ac/jYu3O8B -VLhbxZlBYcLxCqqNsGJrWaiHj2Jf4GhUB0O9hXfaZDMqEGXT9GzOz0yF6b3pDQVy -H0lKIBb+kQzB/jhZKu4vrTAowXtt/av5d7D+rpAU1SxfUhBOPNSRoJUI5NSBbokp -a7u4azdB2IQETX3d2rhDk09EbG1XmMi5Vg1oa8nxfMOWXZnDMusJoZClKjrthmwd -rtR5et44XYhX6p217RBkYMDOVFT7aZpu4SaFeqZIuarVYodSmgXToOFXPsrLppRQ -adOT0FpU64RPNrQz5NF1bSIjqrHSaRVacf8yr7qqxNnpMsrtkDJzsMBz ------END CERTIFICATE----- diff --git a/mail/src/leap/mail/smtp/tests/cert/server.key b/mail/src/leap/mail/smtp/tests/cert/server.key deleted file mode 100644 index 197a449..0000000 --- a/mail/src/leap/mail/smtp/tests/cert/server.key +++ /dev/null @@ -1,51 +0,0 @@ ------BEGIN RSA PRIVATE KEY----- -MIIJKQIBAAKCAgEA97FO8S8btxKZl0AREe31MHXVxKivnhj8Xs2Zgx1I4k2LhjST -CRKyWl1+KmL6a380WD+4YqLWZyC+5d1VtU9n2sKBWkjMydw7xEA/trn4jbMpTOow -IJssfXqximj+sORVDlXXlPAuxPM4UYbwKCnrxNHzMe5ngR1/bMwYLUhqInpbfwtO -cmCyjBg4IUMAXfMvfqTEw7mA0YRRhXq2OW3H6sBxeRremj7GICC9bqrrlRSAMdwh -0QYbZ1z1uTHE+XtGSmsXZHDAAM3NhLtOf4emaPLTql9hkc0wT4jDnXlsJEXcZAVR -TgGkadjsQ76Hg/0pvWH3jyBRP3+izBcb6rYrCTwCiCAwmRQ5YMWIJrlSNbUYGjHF -vtW1n5BMlfGUVZwzZXNPsz8df4/hvTm9jgxcmPE8AUqsAZLsnwMpbxYk1Sddr1dI -MEeDmqKMPNn/MaYdN0bqIAOckuY4cFCAVDXZr+f2ZhXYY9ULoWWAPMb1/T9JQ8FQ -Jp/JZjPUvApjhUzY0vH+LBgxCw0gxJlB82a580MGvvT4FKF7dmluYoH8nfd3zsC4 -l3YeOexOFMQACv7oJ6PJibAW62gNQIXwhooYkSns4Y/b/+5vie3lbBbwNgtGLwSq -3MSfE+jlmJrIcxgYdHWOlfU7klUkVQTv1tkIUUcqa0lziRVlLibitMuyTzcCAwEA -AQKCAgAFQdcqGVTeQt/NrQdvuPw+RhH+dZIcqe0ZWgXLGaEFZJ30gEMqqyHr9xYJ -ckZcZ7vFr7yLI2enkrNaj6MVERVkOEKkluz5z9nY5YA0safL4iSbRFE3L/P2ydbg -2C+ns4D2p+3GdH6ZoYvtdw6723/skoQ16Bh8ThL5TS+qLmJKTwyIGsZUeSbxAEaY -tiJY3COC7Z5bhSFt0QAl9B/QAjt/CQyfhGl7Hp/36Jn8slYDuQariD+TfyyvufJh -NuQ2Y15vj+xULmx01+lnys30uP1YNuc1M4cPoCpJVd7JBd28u1rdKJu8Kx7BPGBv -Y6jerU3ofh7SA96VmXDsIgVuquUo51Oklspe6a9VaDmzLvjYqJsBKQ7BH3J2f07x -NiOob56CGXykX51Ig3WBK1wKn+pA69FL62DbkEa6SykGCqdZPdgBF/kiMc0TESsl -867Em63Yx/2hq+mG3Dknnq8jWXf+Es/zZSSak6N4154IxPOD3m1hzuUq73PP7Ptt -KFe6NfU0DmAuTJL3FqNli8F8lFfvJfuwmW2qk5iTMfwPxybSd8FPbGxi7aRgoZdh -7fIbTFJ0X2f83/SO+9rCzV+B091+d7TM8AaOJ4dEoS74rlRZg53EgmAU0phVnE+l -taMNKGHy2kpJrv9IHX3w5Gm6CjNJj5t4ccS0J18NFFJ+j077eQKCAQEA/RJNRUBS -mI5l0eirl78Q9uDPh1usChZpQiLsvscIJITWQ1vtXSRCvP0hVQRRv8+4CtrZr2rX -v0afkzg/3HNFaNsjYT6aHjgnombFqfpyS/NZN/p3gOzi2h+1Sujzz5fBUGhNLVgZ -F2GLnJbiIHnM1BmKA6597pHpXcRMh1E3DSjDMQAEEsBgF6MyS+MT9WfNwHvJukii -k028tNzR4wRq3Xo3WTfvXZRjbX54Ew9Zy3+TFiu19j2FmuOoqyj+ZvMic4EYmTaY -BWm7viDff4dW34dR9sYCuTWWehLtMJGroA38e7lTLfNOHNDGaUZWkfxs4uJCsxvP -0fPp3xlbU3NUGwKCAQEA+o8SeHwEN+VN2dZvC3wFvbnRvWLc1aLnNcndRE9QLVcC -B4LMRuQMpxaNYRiSQPppoPTNq6zWbo6FEjUO5Md7R8I8dbg1vHo4PzuHOu2wXNcm -DEicocCpSKShSS27NCK6uoSsTqTIlG4u+1x9/R2gJEjlTqjeIkOQkPv7PbWhrUyt -XqvzPy4bewOz9Brmd6ryi8ZLtNbUSNwMyd64s9b1V4A6JRlYZrMDOQ6kXEZo+mbL -ynet0vuj7lYxsAZvxoPIq+Gi5i0CrDYtze6JCg+kGahjMX0zXRjXrYh/YID8NWYT -0GXr2+a0V5pXg86YCDp/jpr3lq75HJJ+vIvm2VHLFQKCAQATEm0GWgmfe6PKxPkh -j4GsyVZ6gfseK4A1PsKOwhsn/WbUXrotuczZx03axV+P0AyzrLiZErk9rgnao3OU -no9Njq5E5t3ghyTdhVdCLyCr/qPrpxGYgsG55IfaJGIzc+FauPGQCEKj03MdEvXp -sqQwG9id3GmbMB3hNij6TbGTaU4EhFbKPvs+7Mqek3dumCsWZX3Xbx/pcANXsgiT -TkLrfAltzNxaNhOkLdLIxPBkeLHSCutEqnBGMwAEHivGAG7JO6Jp8YZVahl/A6U0 -TDPM1rrjmRqdcJ9thb2gWmoPvt4XSOku3lY1r7o0NtvRVq+yDZEvRFpOHU6zxIpw -aJGfAoIBAQDiTvvF62379pc8nJwr6VdeKEozHuqL49mmEbBTFLg8W4wvsIpFtZFg -EdSc0I65NfTWNobV+wSrUvsKmPXc2fiVtfDZ+wo+NL49Ds10Al/7WzC4g5VF3DiK -rngnGrEtw/iYo2Dmn5uzxVmWG9KIHowYeeb0Bz6sAA7BhXdGI5nmZ41oJzNL659S -muOdJfboO3Vbnj2fFzMio+7BHvQBK7Tp1Z2vCJd6G1Jb5Me7uLT1BognVbWhDTzh -9uRmM0oeKcXEycZS1HDHjyAMEtmgRsRXkGoXtxf/jIKx8MnsJlSm/o4C+yvvsQ9O -2M8W9DEJrZys93eNmHjUv9TNBCf8Pg6JAoIBAQDDItnQPLntCUgd7dy0dDjQYBGN -4wVRJNINpgjqwJj0hVjB/dmvrcxkXcOG4VAH+iNH8A25qLU+RTDcNipuL3uEFKbF -O4DSjFih3qL1Y8otTXSrPeqZOMvYpY8dXS5uyI7DSWQQZyZ9bMpeWbxgx4LHqPPH -rdcVJy9Egw1ZIOA7JBFM02uGn9TVwFzNUJk0G/3xwVHzDxYNbJ98vDfflc2vD4CH -OAN6un0pOuol2h200F6zFgc5mbETWHCPIom+ZMXIX3bq7g341c/cgqIELPTk8DLS -s+AgrZ4qYmskrFaD0PHakWsQNHGC8yOh80lgE3Gl4nxSGAvkcR7dkSmsIQFL ------END RSA PRIVATE KEY----- diff --git a/mail/src/leap/mail/smtp/tests/mail.txt b/mail/src/leap/mail/smtp/tests/mail.txt deleted file mode 100644 index 9542047..0000000 --- a/mail/src/leap/mail/smtp/tests/mail.txt +++ /dev/null @@ -1,10 +0,0 @@ -HELO drebs@riseup.net -MAIL FROM: drebs@riseup.net -RCPT TO: drebs@riseup.net -RCPT TO: drebs@leap.se -DATA -Subject: leap test - -Hello world! -. -QUIT diff --git a/mail/src/leap/mail/smtp/tests/test_gateway.py b/mail/src/leap/mail/smtp/tests/test_gateway.py deleted file mode 100644 index 9d88afb..0000000 --- a/mail/src/leap/mail/smtp/tests/test_gateway.py +++ /dev/null @@ -1,181 +0,0 @@ -# -*- coding: utf-8 -*- -# test_gateway.py -# Copyright (C) 2013 LEAP -# -# This program is free software: you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation, either version 3 of the License, or -# (at your option) any later version. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with this program. If not, see . - -""" -SMTP gateway tests. -""" -import re -import tempfile -from datetime import datetime - -from twisted.internet import reactor -from twisted.internet.defer import inlineCallbacks, fail, succeed, Deferred -from twisted.test import proto_helpers - -from mock import Mock - -from leap.keymanager import openpgp, errors -from leap.mail.testing import KeyManagerWithSoledadTestCase -from leap.mail.testing import ADDRESS, ADDRESS_2 -from leap.mail.testing.smtp import getSMTPFactory, TEST_USER - - -# some regexps -IP_REGEX = "(([0-9]|[1-9][0-9]|1[0-9]{2}|2[0-4][0-9]|25[0-5])\.){3}" + \ - "([0-9]|[1-9][0-9]|1[0-9]{2}|2[0-4][0-9]|25[0-5])" -HOSTNAME_REGEX = "(([a-zA-Z0-9]|[a-zA-Z0-9][a-zA-Z0-9\-]*[a-zA-Z0-9])\.)*" + \ - "([A-Za-z0-9]|[A-Za-z0-9][A-Za-z0-9\-]*[A-Za-z0-9])" -IP_OR_HOST_REGEX = '(' + IP_REGEX + '|' + HOSTNAME_REGEX + ')' - - -class TestSmtpGateway(KeyManagerWithSoledadTestCase): - - EMAIL_DATA = ['HELO gateway.leap.se', - 'MAIL FROM: <%s>' % ADDRESS_2, - 'RCPT TO: <%s>' % ADDRESS, - 'DATA', - 'From: User <%s>' % ADDRESS_2, - 'To: Leap <%s>' % ADDRESS, - 'Date: ' + datetime.now().strftime('%c'), - 'Subject: test message', - '', - 'This is a secret message.', - 'Yours,', - 'A.', - '', - '.', - 'QUIT'] - - def setUp(self): - # pytest handles correctly the setupEnv for the class, - # but trial ignores it. - if not getattr(self, 'tempdir', None): - self.tempdir = tempfile.mkdtemp() - return KeyManagerWithSoledadTestCase.setUp(self) - - def tearDown(self): - return KeyManagerWithSoledadTestCase.tearDown(self) - - def assertMatch(self, string, pattern, msg=None): - if not re.match(pattern, string): - msg = self._formatMessage(msg, '"%s" does not match pattern "%s".' - % (string, pattern)) - raise self.failureException(msg) - - @inlineCallbacks - def test_gateway_accepts_valid_email(self): - """ - Test if SMTP server responds correctly for valid interaction. - """ - - SMTP_ANSWERS = ['220 ' + IP_OR_HOST_REGEX + - ' NO UCE NO UBE NO RELAY PROBES', - '250 ' + IP_OR_HOST_REGEX + ' Hello ' + - IP_OR_HOST_REGEX + ', nice to meet you', - '250 Sender address accepted', - '250 Recipient address accepted', - '354 Continue'] - - user = TEST_USER - proto = getSMTPFactory({user: None}, {user: self.km}, {user: None}) - transport = proto_helpers.StringTransport() - proto.makeConnection(transport) - reply = "" - for i, line in enumerate(self.EMAIL_DATA): - reply += yield self.getReply(line + '\r\n', proto, transport) - self.assertMatch(reply, '\r\n'.join(SMTP_ANSWERS), - 'Did not get expected answer from gateway.') - proto.setTimeout(None) - - @inlineCallbacks - def test_missing_key_rejects_address(self): - """ - Test if server rejects to send unencrypted when 'encrypted_only' is - True. - """ - # remove key from key manager - pubkey = yield self.km.get_key(ADDRESS) - pgp = openpgp.OpenPGPScheme( - self._soledad, gpgbinary=self.gpg_binary_path) - yield pgp.delete_key(pubkey) - # mock the key fetching - self.km._fetch_keys_from_server = Mock( - return_value=fail(errors.KeyNotFound())) - user = TEST_USER - proto = getSMTPFactory( - {user: None}, {user: self.km}, {user: None}, - encrypted_only=True) - transport = proto_helpers.StringTransport() - proto.makeConnection(transport) - yield self.getReply(self.EMAIL_DATA[0] + '\r\n', proto, transport) - yield self.getReply(self.EMAIL_DATA[1] + '\r\n', proto, transport) - reply = yield self.getReply(self.EMAIL_DATA[2] + '\r\n', - proto, transport) - # ensure the address was rejected - self.assertEqual( - '550 Cannot receive for specified address\r\n', - reply, - 'Address should have been rejected with appropriate message.') - proto.setTimeout(None) - - @inlineCallbacks - def test_missing_key_accepts_address(self): - """ - Test if server accepts to send unencrypted when 'encrypted_only' is - False. - """ - # remove key from key manager - pubkey = yield self.km.get_key(ADDRESS) - pgp = openpgp.OpenPGPScheme( - self._soledad, gpgbinary=self.gpg_binary_path) - yield pgp.delete_key(pubkey) - # mock the key fetching - self.km._fetch_keys_from_server = Mock( - return_value=fail(errors.KeyNotFound())) - user = TEST_USER - proto = getSMTPFactory({user: None}, {user: self.km}, {user: None}) - transport = proto_helpers.StringTransport() - proto.makeConnection(transport) - yield self.getReply(self.EMAIL_DATA[0] + '\r\n', proto, transport) - yield self.getReply(self.EMAIL_DATA[1] + '\r\n', proto, transport) - reply = yield self.getReply(self.EMAIL_DATA[2] + '\r\n', - proto, transport) - # ensure the address was accepted - self.assertEqual( - '250 Recipient address accepted\r\n', - reply, - 'Address should have been accepted with appropriate message.') - proto.setTimeout(None) - - def getReply(self, line, proto, transport): - proto.lineReceived(line) - - if line[:4] not in ['HELO', 'MAIL', 'RCPT', 'DATA']: - return succeed("") - - def check_transport(_): - reply = transport.value() - if reply: - transport.clear() - return succeed(reply) - - d = Deferred() - d.addCallback(check_transport) - reactor.callLater(0, lambda: d.callback(None)) - return d - - return check_transport(None) diff --git a/mail/src/leap/mail/sync_hooks.py b/mail/src/leap/mail/sync_hooks.py deleted file mode 100644 index 8efbb7c..0000000 --- a/mail/src/leap/mail/sync_hooks.py +++ /dev/null @@ -1,120 +0,0 @@ -# -*- coding: utf-8 -*- -# sync_hooks.py -# Copyright (C) 2015 LEAP -# -# This program is free software: you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation, either version 3 of the License, or -# (at your option) any later version. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with this program. If not, see . -""" -Soledad PostSync Hooks. - -Process every new document of interest after every soledad synchronization, -using the hooks that soledad exposes via plugins. -""" -import logging - -from re import compile as regex_compile - -from zope.interface import implements -from twisted.internet import defer -from twisted.plugin import IPlugin -from twisted.python import log - -from leap.soledad.client.interfaces import ISoledadPostSyncPlugin -from leap.mail import constants - -logger = logging.getLogger(__name__) - - -def _get_doc_type_preffix(s): - return s[:2] - - -class MailProcessingPostSyncHook(object): - implements(IPlugin, ISoledadPostSyncPlugin) - - META_DOC_PREFFIX = _get_doc_type_preffix(constants.METAMSGID) - watched_doc_types = (META_DOC_PREFFIX, ) - - _account = None - _pending_docs = [] - _processing_deferreds = [] - - def process_received_docs(self, doc_id_list): - if self._has_configured_account(): - process_fun = self._make_uid_index - else: - self._processing_deferreds = [] - process_fun = self._queue_doc_id - - for doc_id in doc_id_list: - if _get_doc_type_preffix(doc_id) in self.watched_doc_types: - log.msg("Mail post-sync hook: processing %s" % doc_id) - process_fun(doc_id) - - return defer.gatherResults(self._processing_deferreds) - - def set_account(self, account): - self._account = account - if account: - self._process_queued_docs() - - def _has_configured_account(self): - return self._account is not None - - def _queue_doc_id(self, doc_id): - self._pending_docs.append(doc_id) - - def _make_uid_index(self, mdoc_id): - indexer = self._account.mbox_indexer - mbox_uuid = _get_mbox_uuid(mdoc_id) - if mbox_uuid: - chash = _get_chash_from_mdoc(mdoc_id) - logger.debug("Making index table for %s:%s" % (mbox_uuid, chash)) - index_docid = constants.METAMSGID.format( - mbox_uuid=mbox_uuid.replace('-', '_'), - chash=chash) - # XXX could avoid creating table if I track which ones I already - # have seen -- but make sure *it's already created* before - # inserting the index entry!. - d = indexer.create_table(mbox_uuid) - d.addBoth(lambda _: indexer.insert_doc(mbox_uuid, index_docid)) - self._processing_deferreds.append(d) - - def _process_queued_docs(self): - assert(self._has_configured_account()) - pending = self._pending_docs - log.msg("Mail post-sync hook: processing queued docs") - - def remove_pending_docs(res): - self._pending_docs = [] - return res - - d = self.process_received_docs(pending) - d.addCallback(remove_pending_docs) - return d - - -_mbox_uuid_regex = regex_compile(constants.METAMSGID_MBOX_RE) -_mdoc_chash_regex = regex_compile(constants.METAMSGID_CHASH_RE) - - -def _get_mbox_uuid(doc_id): - matches = _mbox_uuid_regex.findall(doc_id) - if matches: - return matches[0].replace('_', '-') - - -def _get_chash_from_mdoc(doc_id): - matches = _mdoc_chash_regex.findall(doc_id) - if matches: - return matches[0] diff --git a/mail/src/leap/mail/testing/__init__.py b/mail/src/leap/mail/testing/__init__.py deleted file mode 100644 index 8822a5c..0000000 --- a/mail/src/leap/mail/testing/__init__.py +++ /dev/null @@ -1,353 +0,0 @@ -# -*- coding: utf-8 -*- -# __init__.py -# Copyright (C) 2013 LEAP -# -# This program is free software: you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation, either version 3 of the License, or -# (at your option) any later version. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with this program. If not, see . -""" -Base classes and keys for leap.mail tests. -""" -import os -import distutils.spawn -from mock import Mock -from twisted.internet.defer import gatherResults -from twisted.trial import unittest -from twisted.internet import defer - - -from leap.soledad.client import Soledad -from leap.keymanager import KeyManager - - -from leap.common.testing.basetest import BaseLeapTest - -ADDRESS = 'leap@leap.se' -ADDRESS_2 = 'anotheruser@leap.se' - - -class defaultMockSharedDB(object): - get_doc = Mock(return_value=None) - put_doc = Mock(side_effect=None) - open = Mock(return_value=None) - close = Mock(return_value=None) - syncable = True - - def __call__(self): - return self - - -class KeyManagerWithSoledadTestCase(unittest.TestCase, BaseLeapTest): - - def setUp(self): - self.gpg_binary_path = self._find_gpg() - - self._soledad = Soledad( - u"leap@leap.se", - u"123456", - secrets_path=self.tempdir + "/secret.gpg", - local_db_path=self.tempdir + "/soledad.u1db", - server_url='', - cert_file=None, - auth_token=None, - shared_db=defaultMockSharedDB(), - syncable=False) - - self.km = self._key_manager() - - class Response(object): - code = 200 - phrase = '' - - def deliverBody(self, x): - return '' - - self.km._async_client_pinned.request = Mock( - return_value=defer.succeed(Response())) - - d1 = self.km.put_raw_key(PRIVATE_KEY, ADDRESS) - d2 = self.km.put_raw_key(PRIVATE_KEY_2, ADDRESS_2) - return gatherResults([d1, d2]) - - def tearDown(self): - km = self._key_manager() - # wait for the indexes to be ready for the tear down - d = km._openpgp.deferred_init - d.addCallback(lambda _: self.delete_all_keys(km)) - d.addCallback(lambda _: self._soledad.close()) - return d - - def delete_all_keys(self, km): - def delete_keys(keys): - deferreds = [] - for key in keys: - d = km._openpgp.delete_key(key) - deferreds.append(d) - return gatherResults(deferreds) - - def check_deleted(_, private): - d = km.get_all_keys(private=private) - d.addCallback(lambda keys: self.assertEqual(keys, [])) - return d - - deferreds = [] - for private in [True, False]: - d = km.get_all_keys(private=private) - d.addCallback(delete_keys) - d.addCallback(check_deleted, private) - deferreds.append(d) - return gatherResults(deferreds) - - def _key_manager(self, user=ADDRESS, url='', token=None, - ca_cert_path=None): - return KeyManager(user, url, self._soledad, token=token, - gpgbinary=self.gpg_binary_path, - ca_cert_path=ca_cert_path) - - def _find_gpg(self): - gpg_path = distutils.spawn.find_executable('gpg') - if gpg_path is not None: - return os.path.realpath(gpg_path) - else: - return "/usr/bin/gpg" - - def get_public_binary_key(self): - with open(PATH + '/fixtures/public_key.bin', 'r') as binary_public_key: - return binary_public_key.read() - - def get_private_binary_key(self): - with open( - PATH + '/fixtures/private_key.bin', 'r') as binary_private_key: - return binary_private_key.read() - - -# key 24D18DDF: public key "Leap Test Key " -KEY_FINGERPRINT = "E36E738D69173C13D709E44F2F455E2824D18DDF" -PUBLIC_KEY = """ ------BEGIN PGP PUBLIC KEY BLOCK----- -Version: GnuPG v1.4.10 (GNU/Linux) - -mQINBFC9+dkBEADNRfwV23TWEoGc/x0wWH1P7PlXt8MnC2Z1kKaKKmfnglVrpOiz -iLWoiU58sfZ0L5vHkzXHXCBf6Eiy/EtUIvdiWAn+yASJ1mk5jZTBKO/WMAHD8wTO -zpMsFmWyg3xc4DkmFa9KQ5EVU0o/nqPeyQxNMQN7px5pPwrJtJFmPxnxm+aDkPYx -irDmz/4DeDNqXliazGJKw7efqBdlwTHkl9Akw2gwy178pmsKwHHEMOBOFFvX61AT -huKqHYmlCGSliwbrJppTG7jc1/ls3itrK+CWTg4txREkSpEVmfcASvw/ZqLbjgfs -d/INMwXnR9U81O8+7LT6yw/ca4ppcFoJD7/XJbkRiML6+bJ4Dakiy6i727BzV17g -wI1zqNvm5rAhtALKfACha6YO43aJzairO4II1wxVHvRDHZn2IuKDDephQ3Ii7/vb -hUOf6XCSmchkAcpKXUOvbxm1yfB1LRa64mMc2RcZxf4mW7KQkulBsdV5QG2276lv -U2UUy2IutXcGP5nXC+f6sJJGJeEToKJ57yiO/VWJFjKN8SvP+7AYsQSqINUuEf6H -T5gCPCraGMkTUTPXrREvu7NOohU78q6zZNaL3GW8ai7eSeANSuQ8Vzffx7Wd8Y7i -Pw9sYj0SMFs1UgjbuL6pO5ueHh+qyumbtAq2K0Bci0kqOcU4E9fNtdiovQARAQAB -tBxMZWFwIFRlc3QgS2V5IDxsZWFwQGxlYXAuc2U+iQI3BBMBCAAhBQJQvfnZAhsD -BQsJCAcDBRUKCQgLBRYCAwEAAh4BAheAAAoJEC9FXigk0Y3fT7EQAKH3IuRniOpb -T/DDIgwwjz3oxB/W0DDMyPXowlhSOuM0rgGfntBpBb3boezEXwL86NPQxNGGruF5 -hkmecSiuPSvOmQlqlS95NGQp6hNG0YaKColh+Q5NTspFXCAkFch9oqUje0LdxfSP -QfV9UpeEvGyPmk1I9EJV/YDmZ4+Djge1d7qhVZInz4Rx1NrSyF/Tc2EC0VpjQFsU -Y9Kb2YBBR7ivG6DBc8ty0jJXi7B4WjkFcUEJviQpMF2dCLdonCehYs1PqsN1N7j+ -eFjQd+hqVMJgYuSGKjvuAEfClM6MQw7+FmFwMyLgK/Ew/DttHEDCri77SPSkOGSI -txCzhTg6798f6mJr7WcXmHX1w1Vcib5FfZ8vTDFVhz/XgAgArdhPo9V6/1dgSSiB -KPQ/spsco6u5imdOhckERE0lnAYvVT6KE81TKuhF/b23u7x+Wdew6kK0EQhYA7wy -7LmlaNXc7rMBQJ9Z60CJ4JDtatBWZ0kNrt2VfdDHVdqBTOpl0CraNUjWE5YMDasr -K2dF5IX8D3uuYtpZnxqg0KzyLg0tzL0tvOL1C2iudgZUISZNPKbS0z0v+afuAAnx -2pTC3uezbh2Jt8SWTLhll4i0P4Ps5kZ6HQUO56O+/Z1cWovX+mQekYFmERySDR9n -3k1uAwLilJmRmepGmvYbB8HloV8HqwgguQINBFC9+dkBEAC0I/xn1uborMgDvBtf -H0sEhwnXBC849/32zic6udB6/3Efk9nzbSpL3FSOuXITZsZgCHPkKarnoQ2ztMcS -sh1ke1C5gQGms75UVmM/nS+2YI4vY8OX/GC/on2vUyncqdH+bR6xH5hx4NbWpfTs -iQHmz5C6zzS/kuabGdZyKRaZHt23WQ7JX/4zpjqbC99DjHcP9BSk7tJ8wI4bkMYD -uFVQdT9O6HwyKGYwUU4sAQRAj7XCTGvVbT0dpgJwH4RmrEtJoHAx4Whg8mJ710E0 -GCmzf2jqkNuOw76ivgk27Kge+Hw00jmJjQhHY0yVbiaoJwcRrPKzaSjEVNgrpgP3 -lXPRGQArgESsIOTeVVHQ8fhK2YtTeCY9rIiO+L0OX2xo9HK7hfHZZWL6rqymXdyS -fhzh/f6IPyHFWnvj7Brl7DR8heMikygcJqv+ed2yx7iLyCUJ10g12I48+aEj1aLe -dP7lna32iY8/Z0SHQLNH6PXO9SlPcq2aFUgKqE75A/0FMk7CunzU1OWr2ZtTLNO1 -WT/13LfOhhuEq9jTyTosn0WxBjJKq18lnhzCXlaw6EAtbA7CUwsD3CTPR56aAXFK -3I7KXOVAqggrvMe5Tpdg5drfYpI8hZovL5aAgb+7Y5ta10TcJdUhS5K3kFAWe/td -U0cmWUMDP1UMSQ5Jg6JIQVWhSwARAQABiQIfBBgBCAAJBQJQvfnZAhsMAAoJEC9F -Xigk0Y3fRwsP/i0ElYCyxeLpWJTwo1iCLkMKz2yX1lFVa9nT1BVTPOQwr/IAc5OX -NdtbJ14fUsKL5pWgW8OmrXtwZm1y4euI1RPWWubG01ouzwnGzv26UcuHeqC5orZj -cOnKtL40y8VGMm8LoicVkRJH8blPORCnaLjdOtmA3rx/v2EXrJpSa3AhOy0ZSRXk -ZSrK68AVNwamHRoBSYyo0AtaXnkPX4+tmO8X8BPfj125IljubvwZPIW9VWR9UqCE -VPfDR1XKegVb6VStIywF7kmrknM1C5qUY28rdZYWgKorw01hBGV4jTW0cqde3N51 -XT1jnIAa+NoXUM9uQoGYMiwrL7vNsLlyyiW5ayDyV92H/rIuiqhFgbJsHTlsm7I8 -oGheR784BagAA1NIKD1qEO9T6Kz9lzlDaeWS5AUKeXrb7ZJLI1TTCIZx5/DxjLqM -Tt/RFBpVo9geZQrvLUqLAMwdaUvDXC2c6DaCPXTh65oCZj/hqzlJHH+RoTWWzKI+ -BjXxgUWF9EmZUBrg68DSmI+9wuDFsjZ51BcqvJwxyfxtTaWhdoYqH/UQS+D1FP3/ -diZHHlzwVwPICzM9ooNTgbrcDzyxRkIVqsVwBq7EtzcvgYUyX53yG25Giy6YQaQ2 -ZtQ/VymwFL3XdUWV6B/hU4PVAFvO3qlOtdJ6TpE+nEWgcWjCv5g7RjXX -=MuOY ------END PGP PUBLIC KEY BLOCK----- -""" -PRIVATE_KEY = """ ------BEGIN PGP PRIVATE KEY BLOCK----- -Version: GnuPG v1.4.10 (GNU/Linux) - -lQcYBFC9+dkBEADNRfwV23TWEoGc/x0wWH1P7PlXt8MnC2Z1kKaKKmfnglVrpOiz -iLWoiU58sfZ0L5vHkzXHXCBf6Eiy/EtUIvdiWAn+yASJ1mk5jZTBKO/WMAHD8wTO -zpMsFmWyg3xc4DkmFa9KQ5EVU0o/nqPeyQxNMQN7px5pPwrJtJFmPxnxm+aDkPYx -irDmz/4DeDNqXliazGJKw7efqBdlwTHkl9Akw2gwy178pmsKwHHEMOBOFFvX61AT -huKqHYmlCGSliwbrJppTG7jc1/ls3itrK+CWTg4txREkSpEVmfcASvw/ZqLbjgfs -d/INMwXnR9U81O8+7LT6yw/ca4ppcFoJD7/XJbkRiML6+bJ4Dakiy6i727BzV17g -wI1zqNvm5rAhtALKfACha6YO43aJzairO4II1wxVHvRDHZn2IuKDDephQ3Ii7/vb -hUOf6XCSmchkAcpKXUOvbxm1yfB1LRa64mMc2RcZxf4mW7KQkulBsdV5QG2276lv -U2UUy2IutXcGP5nXC+f6sJJGJeEToKJ57yiO/VWJFjKN8SvP+7AYsQSqINUuEf6H -T5gCPCraGMkTUTPXrREvu7NOohU78q6zZNaL3GW8ai7eSeANSuQ8Vzffx7Wd8Y7i -Pw9sYj0SMFs1UgjbuL6pO5ueHh+qyumbtAq2K0Bci0kqOcU4E9fNtdiovQARAQAB -AA/+JHtlL39G1wsH9R6UEfUQJGXR9MiIiwZoKcnRB2o8+DS+OLjg0JOh8XehtuCs -E/8oGQKtQqa5bEIstX7IZoYmYFiUQi9LOzIblmp2vxOm+HKkxa4JszWci2/ZmC3t -KtaA4adl9XVnshoQ7pijuCMUKB3naBEOAxd8s9d/JeReGIYkJErdrnVfNk5N71Ds -FmH5Ll3XtEDvgBUQP3nkA6QFjpsaB94FHjL3gDwum/cxzj6pCglcvHOzEhfY0Ddb -J967FozQTaf2JW3O+w3LOqtcKWpq87B7+O61tVidQPSSuzPjCtFF0D2LC9R/Hpky -KTMQ6CaKja4MPhjwywd4QPcHGYSqjMpflvJqi+kYIt8psUK/YswWjnr3r4fbuqVY -VhtiHvnBHQjz135lUqWvEz4hM3Xpnxydx7aRlv5NlevK8+YIO5oFbWbGNTWsPZI5 -jpoFBpSsnR1Q5tnvtNHauvoWV+XN2qAOBTG+/nEbDYH6Ak3aaE9jrpTdYh0CotYF -q7csANsDy3JvkAzeU6WnYpsHHaAjqOGyiZGsLej1UcXPFMosE/aUo4WQhiS8Zx2c -zOVKOi/X5vQ2GdNT9Qolz8AriwzsvFR+bxPzyd8V6ALwDsoXvwEYinYBKK8j0OPv -OOihSR6HVsuP9NUZNU9ewiGzte/+/r6pNXHvR7wTQ8EWLcEIAN6Zyrb0bHZTIlxt -VWur/Ht2mIZrBaO50qmM5RD3T5oXzWXi/pjLrIpBMfeZR9DWfwQwjYzwqi7pxtYx -nJvbMuY505rfnMoYxb4J+cpRXV8MS7Dr1vjjLVUC9KiwSbM3gg6emfd2yuA93ihv -Pe3mffzLIiQa4mRE3wtGcioC43nWuV2K2e1KjxeFg07JhrezA/1Cak505ab/tmvP -4YmjR5c44+yL/YcQ3HdFgs4mV+nVbptRXvRcPpolJsgxPccGNdvHhsoR4gwXMS3F -RRPD2z6x8xeN73Q4KH3bm01swQdwFBZbWVfmUGLxvN7leCdfs9+iFJyqHiCIB6Iv -mQfp8F0IAOwSo8JhWN+V1dwML4EkIrM8wUb4yecNLkyR6TpPH/qXx4PxVMC+vy6x -sCtjeHIwKE+9vqnlhd5zOYh7qYXEJtYwdeDDmDbL8oks1LFfd+FyAuZXY33DLwn0 -cRYsr2OEZmaajqUB3NVmj3H4uJBN9+paFHyFSXrH68K1Fk2o3n+RSf2EiX+eICwI -L6rqoF5sSVUghBWdNegV7qfy4anwTQwrIMGjgU5S6PKW0Dr/3iO5z3qQpGPAj5OW -ATqPWkDICLbObPxD5cJlyyNE2wCA9VVc6/1d6w4EVwSq9h3/WTpATEreXXxTGptd -LNiTA1nmakBYNO2Iyo3djhaqBdWjk+EIAKtVEnJH9FAVwWOvaj1RoZMA5DnDMo7e -SnhrCXl8AL7Z1WInEaybasTJXn1uQ8xY52Ua4b8cbuEKRKzw/70NesFRoMLYoHTO -dyeszvhoDHberpGRTciVmpMu7Hyi33rM31K9epA4ib6QbbCHnxkWOZB+Bhgj1hJ8 -xb4RBYWiWpAYcg0+DAC3w9gfxQhtUlZPIbmbrBmrVkO2GVGUj8kH6k4UV6kUHEGY -HQWQR0HcbKcXW81ZXCCD0l7ROuEWQtTe5Jw7dJ4/QFuqZnPutXVRNOZqpl6eRShw -7X2/a29VXBpmHA95a88rSQsL+qm7Fb3prqRmuMCtrUZgFz7HLSTuUMR867QcTGVh -cCBUZXN0IEtleSA8bGVhcEBsZWFwLnNlPokCNwQTAQgAIQUCUL352QIbAwULCQgH -AwUVCgkICwUWAgMBAAIeAQIXgAAKCRAvRV4oJNGN30+xEACh9yLkZ4jqW0/wwyIM -MI896MQf1tAwzMj16MJYUjrjNK4Bn57QaQW926HsxF8C/OjT0MTRhq7heYZJnnEo -rj0rzpkJapUveTRkKeoTRtGGigqJYfkOTU7KRVwgJBXIfaKlI3tC3cX0j0H1fVKX -hLxsj5pNSPRCVf2A5mePg44HtXe6oVWSJ8+EcdTa0shf03NhAtFaY0BbFGPSm9mA -QUe4rxugwXPLctIyV4uweFo5BXFBCb4kKTBdnQi3aJwnoWLNT6rDdTe4/nhY0Hfo -alTCYGLkhio77gBHwpTOjEMO/hZhcDMi4CvxMPw7bRxAwq4u+0j0pDhkiLcQs4U4 -Ou/fH+pia+1nF5h19cNVXIm+RX2fL0wxVYc/14AIAK3YT6PVev9XYEkogSj0P7Kb -HKOruYpnToXJBERNJZwGL1U+ihPNUyroRf29t7u8flnXsOpCtBEIWAO8Muy5pWjV -3O6zAUCfWetAieCQ7WrQVmdJDa7dlX3Qx1XagUzqZdAq2jVI1hOWDA2rKytnReSF -/A97rmLaWZ8aoNCs8i4NLcy9Lbzi9QtornYGVCEmTTym0tM9L/mn7gAJ8dqUwt7n -s24dibfElky4ZZeItD+D7OZGeh0FDuejvv2dXFqL1/pkHpGBZhEckg0fZ95NbgMC -4pSZkZnqRpr2GwfB5aFfB6sIIJ0HGARQvfnZARAAtCP8Z9bm6KzIA7wbXx9LBIcJ -1wQvOPf99s4nOrnQev9xH5PZ820qS9xUjrlyE2bGYAhz5Cmq56ENs7THErIdZHtQ -uYEBprO+VFZjP50vtmCOL2PDl/xgv6J9r1Mp3KnR/m0esR+YceDW1qX07IkB5s+Q -us80v5LmmxnWcikWmR7dt1kOyV/+M6Y6mwvfQ4x3D/QUpO7SfMCOG5DGA7hVUHU/ -Tuh8MihmMFFOLAEEQI+1wkxr1W09HaYCcB+EZqxLSaBwMeFoYPJie9dBNBgps39o -6pDbjsO+or4JNuyoHvh8NNI5iY0IR2NMlW4mqCcHEazys2koxFTYK6YD95Vz0RkA -K4BErCDk3lVR0PH4StmLU3gmPayIjvi9Dl9saPRyu4Xx2WVi+q6spl3ckn4c4f3+ -iD8hxVp74+wa5ew0fIXjIpMoHCar/nndsse4i8glCddINdiOPPmhI9Wi3nT+5Z2t -9omPP2dEh0CzR+j1zvUpT3KtmhVICqhO+QP9BTJOwrp81NTlq9mbUyzTtVk/9dy3 -zoYbhKvY08k6LJ9FsQYySqtfJZ4cwl5WsOhALWwOwlMLA9wkz0eemgFxStyOylzl -QKoIK7zHuU6XYOXa32KSPIWaLy+WgIG/u2ObWtdE3CXVIUuSt5BQFnv7XVNHJllD -Az9VDEkOSYOiSEFVoUsAEQEAAQAP/1AagnZQZyzHDEgw4QELAspYHCWLXE5aZInX -wTUJhK31IgIXNn9bJ0hFiSpQR2xeMs9oYtRuPOu0P8oOFMn4/z374fkjZy8QVY3e -PlL+3EUeqYtkMwlGNmVw5a/NbNuNfm5Darb7pEfbYd1gPcni4MAYw7R2SG/57GbC -9gucvspHIfOSfBNLBthDzmK8xEKe1yD2eimfc2T7IRYb6hmkYfeds5GsqvGI6mwI -85h4uUHWRc5JOlhVM6yX8hSWx0L60Z3DZLChmc8maWnFXd7C8eQ6P1azJJbW71Ih -7CoK0XW4LE82vlQurSRFgTwfl7wFYszW2bOzCuhHDDtYnwH86Nsu0DC78ZVRnvxn -E8Ke/AJgrdhIOo4UAyR+aZD2+2mKd7/waOUTUrUtTzc7i8N3YXGi/EIaNReBXaq+ -ZNOp24BlFzRp+FCF/pptDW9HjPdiV09x0DgICmeZS4Gq/4vFFIahWctg52NGebT0 -Idxngjj+xDtLaZlLQoOz0n5ByjO/Wi0ANmMv1sMKCHhGvdaSws2/PbMR2r4caj8m -KXpIgdinM/wUzHJ5pZyF2U/qejsRj8Kw8KH/tfX4JCLhiaP/mgeTuWGDHeZQERAT -xPmRFHaLP9/ZhvGNh6okIYtrKjWTLGoXvKLHcrKNisBLSq+P2WeFrlme1vjvJMo/ -jPwLT5o9CADQmcbKZ+QQ1ZM9v99iDZol7SAMZX43JC019sx6GK0u6xouJBcLfeB4 -OXacTgmSYdTa9RM9fbfVpti01tJ84LV2SyL/VJq/enJF4XQPSynT/tFTn1PAor6o -tEAAd8fjKdJ6LnD5wb92SPHfQfXqI84rFEO8rUNIE/1ErT6DYifDzVCbfD2KZdoF -cOSp7TpD77sY1bs74ocBX5ejKtd+aH99D78bJSMM4pSDZsIEwnomkBHTziubPwJb -OwnATy0LmSMAWOw5rKbsh5nfwCiUTM20xp0t5JeXd+wPVWbpWqI2EnkCEN+RJr9i -7dp/ymDQ+Yt5wrsN3NwoyiexPOG91WQVCADdErHsnglVZZq9Z8Wx7KwecGCUurJ2 -H6lKudv5YOxPnAzqZS5HbpZd/nRTMZh2rdXCr5m2YOuewyYjvM757AkmUpM09zJX -MQ1S67/UX2y8/74TcRF97Ncx9HeELs92innBRXoFitnNguvcO6Esx4BTe1OdU6qR -ER3zAmVf22Le9ciXbu24DN4mleOH+OmBx7X2PqJSYW9GAMTsRB081R6EWKH7romQ -waxFrZ4DJzZ9ltyosEJn5F32StyLrFxpcrdLUoEaclZCv2qka7sZvi0EvovDVEBU -e10jOx9AOwf8Gj2ufhquQ6qgVYCzbP+YrodtkFrXRS3IsljIchj1M2ffB/0bfoUs -rtER9pLvYzCjBPg8IfGLw0o754Qbhh/ReplCRTusP/fQMybvCvfxreS3oyEriu/G -GufRomjewZ8EMHDIgUsLcYo2UHZsfF7tcazgxMGmMvazp4r8vpgrvW/8fIN/6Adu -tF+WjWDTvJLFJCe6O+BFJOWrssNrrra1zGtLC1s8s+Wfpe+bGPL5zpHeebGTwH1U -22eqgJArlEKxrfarz7W5+uHZJHSjF/K9ZvunLGD0n9GOPMpji3UO3zeM8IYoWn7E -/EWK1XbjnssNemeeTZ+sDh+qrD7BOi+vCX1IyBxbfqnQfJZvmcPWpruy1UsO+aIC -0GY8Jr3OL69dDQ21jueJAh8EGAEIAAkFAlC9+dkCGwwACgkQL0VeKCTRjd9HCw/+ -LQSVgLLF4ulYlPCjWIIuQwrPbJfWUVVr2dPUFVM85DCv8gBzk5c121snXh9Swovm -laBbw6ate3BmbXLh64jVE9Za5sbTWi7PCcbO/bpRy4d6oLmitmNw6cq0vjTLxUYy -bwuiJxWREkfxuU85EKdouN062YDevH+/YResmlJrcCE7LRlJFeRlKsrrwBU3BqYd -GgFJjKjQC1peeQ9fj62Y7xfwE9+PXbkiWO5u/Bk8hb1VZH1SoIRU98NHVcp6BVvp -VK0jLAXuSauSczULmpRjbyt1lhaAqivDTWEEZXiNNbRyp17c3nVdPWOcgBr42hdQ -z25CgZgyLCsvu82wuXLKJblrIPJX3Yf+si6KqEWBsmwdOWybsjygaF5HvzgFqAAD -U0goPWoQ71PorP2XOUNp5ZLkBQp5etvtkksjVNMIhnHn8PGMuoxO39EUGlWj2B5l -Cu8tSosAzB1pS8NcLZzoNoI9dOHrmgJmP+GrOUkcf5GhNZbMoj4GNfGBRYX0SZlQ -GuDrwNKYj73C4MWyNnnUFyq8nDHJ/G1NpaF2hiof9RBL4PUU/f92JkceXPBXA8gL -Mz2ig1OButwPPLFGQhWqxXAGrsS3Ny+BhTJfnfIbbkaLLphBpDZm1D9XKbAUvdd1 -RZXoH+FTg9UAW87eqU610npOkT6cRaBxaMK/mDtGNdc= -=JTFu ------END PGP PRIVATE KEY BLOCK----- -""" - -# key 7FEE575A: public key "anotheruser " -PUBLIC_KEY_2 = """ ------BEGIN PGP PUBLIC KEY BLOCK----- -Version: GnuPG v1.4.10 (GNU/Linux) - -mI0EUYwJXgEEAMbTKHuPJ5/Gk34l9Z06f+0WCXTDXdte1UBoDtZ1erAbudgC4MOR -gquKqoj3Hhw0/ILqJ88GcOJmKK/bEoIAuKaqlzDF7UAYpOsPZZYmtRfPC2pTCnXq -Z1vdeqLwTbUspqXflkCkFtfhGKMq5rH8GV5a3tXZkRWZhdNwhVXZagC3ABEBAAG0 -IWFub3RoZXJ1c2VyIDxhbm90aGVydXNlckBsZWFwLnNlPoi4BBMBAgAiBQJRjAle -AhsDBgsJCAcDAgYVCAIJCgsEFgIDAQIeAQIXgAAKCRB/nfpof+5XWotuA/4tLN4E -gUr7IfLy2HkHAxzw7A4rqfMN92DIM9mZrDGaWRrOn3aVF7VU1UG7MDkHfPvp/cFw -ezoCw4s4IoHVc/pVlOkcHSyt4/Rfh248tYEJmFCJXGHpkK83VIKYJAithNccJ6Q4 -JE/o06Mtf4uh/cA1HUL4a4ceqUhtpLJULLeKo7iNBFGMCV4BBADsyQI7GR0wSAxz -VayLjuPzgT+bjbFeymIhjuxKIEwnIKwYkovztW+4bbOcQs785k3Lp6RzvigTpQQt -Z/hwcLOqZbZw8t/24+D+Pq9mMP2uUvCFFqLlVvA6D3vKSQ/XNN+YB919WQ04jh63 -yuRe94WenT1RJd6xU1aaUff4rKizuQARAQABiJ8EGAECAAkFAlGMCV4CGwwACgkQ -f536aH/uV1rPZQQAqCzRysOlu8ez7PuiBD4SebgRqWlxa1TF1ujzfLmuPivROZ2X -Kw5aQstxgGSjoB7tac49s0huh4X8XK+BtJBfU84JS8Jc2satlfwoyZ35LH6sDZck -I+RS/3we6zpMfHs3vvp9xgca6ZupQxivGtxlJs294TpJorx+mFFqbV17AzQ= -=Thdu ------END PGP PUBLIC KEY BLOCK----- -""" - -PRIVATE_KEY_2 = """ ------BEGIN PGP PRIVATE KEY BLOCK----- -Version: GnuPG v1.4.10 (GNU/Linux) - -lQHYBFGMCV4BBADG0yh7jyefxpN+JfWdOn/tFgl0w13bXtVAaA7WdXqwG7nYAuDD -kYKriqqI9x4cNPyC6ifPBnDiZiiv2xKCALimqpcwxe1AGKTrD2WWJrUXzwtqUwp1 -6mdb3Xqi8E21LKal35ZApBbX4RijKuax/BleWt7V2ZEVmYXTcIVV2WoAtwARAQAB -AAP7BLuSAx7tOohnimEs74ks8l/L6dOcsFQZj2bqs4AoY3jFe7bV0tHr4llypb/8 -H3/DYvpf6DWnCjyUS1tTnXSW8JXtx01BUKaAufSmMNg9blKV6GGHlT/Whe9uVyks -7XHk/+9mebVMNJ/kNlqq2k+uWqJohzC8WWLRK+d1tBeqDsECANZmzltPaqUsGV5X -C3zszE3tUBgptV/mKnBtopKi+VH+t7K6fudGcG+bAcZDUoH/QVde52mIIjjIdLje -uajJuHUCAO1mqh+vPoGv4eBLV7iBo3XrunyGXiys4a39eomhxTy3YktQanjjx+ty -GltAGCs5PbWGO6/IRjjvd46wh53kzvsCAO0J97gsWhzLuFnkxFAJSPk7RRlyl7lI -1XS/x0Og6j9XHCyY1OYkfBm0to3UlCfkgirzCYlTYObCofzdKFIPDmSqHbQhYW5v -dGhlcnVzZXIgPGFub3RoZXJ1c2VyQGxlYXAuc2U+iLgEEwECACIFAlGMCV4CGwMG -CwkIBwMCBhUIAgkKCwQWAgMBAh4BAheAAAoJEH+d+mh/7ldai24D/i0s3gSBSvsh -8vLYeQcDHPDsDiup8w33YMgz2ZmsMZpZGs6fdpUXtVTVQbswOQd8++n9wXB7OgLD -izgigdVz+lWU6RwdLK3j9F+Hbjy1gQmYUIlcYemQrzdUgpgkCK2E1xwnpDgkT+jT -oy1/i6H9wDUdQvhrhx6pSG2kslQst4qjnQHYBFGMCV4BBADsyQI7GR0wSAxzVayL -juPzgT+bjbFeymIhjuxKIEwnIKwYkovztW+4bbOcQs785k3Lp6RzvigTpQQtZ/hw -cLOqZbZw8t/24+D+Pq9mMP2uUvCFFqLlVvA6D3vKSQ/XNN+YB919WQ04jh63yuRe -94WenT1RJd6xU1aaUff4rKizuQARAQABAAP9EyElqJ3dq3EErXwwT4mMnbd1SrVC -rUJrNWQZL59mm5oigS00uIyR0SvusOr+UzTtd8ysRuwHy5d/LAZsbjQStaOMBILx -77TJveOel0a1QK0YSMF2ywZMCKvquvjli4hAtWYz/EwfuzQN3t23jc5ny+GqmqD2 -3FUxLJosFUfLNmECAO9KhVmJi+L9dswIs+2Dkjd1eiRQzNOEVffvYkGYZyKxNiXF -UA5kvyZcB4iAN9sWCybE4WHZ9jd4myGB0MPDGxkCAP1RsXJbbuD6zS7BXe5gwunO -2q4q7ptdSl/sJYQuTe1KNP5d/uGsvlcFfsYjpsopasPjFBIncc/2QThMKlhoEaEB -/0mVAxpT6SrEvUbJ18z7kna24SgMPr3OnPMxPGfvNLJY/Xv/A17YfoqjmByCvsKE -JCDjopXtmbcrZyoEZbEht9mko4ifBBgBAgAJBQJRjAleAhsMAAoJEH+d+mh/7lda -z2UEAKgs0crDpbvHs+z7ogQ+Enm4EalpcWtUxdbo83y5rj4r0TmdlysOWkLLcYBk -o6Ae7WnOPbNIboeF/FyvgbSQX1POCUvCXNrGrZX8KMmd+Sx+rA2XJCPkUv98Hus6 -THx7N776fcYHGumbqUMYrxrcZSbNveE6SaK8fphRam1dewM0 -=a5gs ------END PGP PRIVATE KEY BLOCK----- -""" diff --git a/mail/src/leap/mail/testing/__init__.py~ b/mail/src/leap/mail/testing/__init__.py~ deleted file mode 100644 index ffaadd8..0000000 --- a/mail/src/leap/mail/testing/__init__.py~ +++ /dev/null @@ -1,358 +0,0 @@ -# -*- coding: utf-8 -*- -# __init__.py -# Copyright (C) 2013 LEAP -# -# This program is free software: you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation, either version 3 of the License, or -# (at your option) any later version. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with this program. If not, see . -""" -Base classes and keys for leap.mail tests. -""" -import os -import distutils.spawn -from mock import Mock -from twisted.internet.defer import gatherResults, succeed -from twisted.trial import unittest -from twisted.web.client import Response -from twisted.internet import defer -from twisted.python import log - - -from leap.soledad.client import Soledad -from leap.keymanager import KeyManager - - -from leap.common.testing.basetest import BaseLeapTest - -ADDRESS = 'leap@leap.se' -ADDRESS_2 = 'anotheruser@leap.se' - -class defaultMockSharedDB(object): - get_doc = Mock(return_value=None) - put_doc = Mock(side_effect=None) - open = Mock(return_value=None) - close = Mock(return_value=None) - syncable = True - - def __call__(self): - return self - - -class KeyManagerWithSoledadTestCase(unittest.TestCase, BaseLeapTest): - - def setUp(self): - self.gpg_binary_path = self._find_gpg() - - self._soledad = Soledad( - u"leap@leap.se", - u"123456", - secrets_path=self.tempdir + "/secret.gpg", - local_db_path=self.tempdir + "/soledad.u1db", - server_url='', - cert_file=None, - auth_token=None, - shared_db=defaultMockSharedDB(), - syncable=False) - - self.km = self._key_manager() - - class Response(object): - code = 200 - phrase = '' - def deliverBody(self, x): - return - - # XXX why the fuck is this needed? ------------------------ - self.km._async_client_pinned.request = Mock( - return_value=defer.succeed(Response())) - #self.km._async_client.request = Mock(return_value='') - #self.km._async_client_pinned.request = Mock( - #return_value='') - # ------------------------------------------------------- - - d1 = self.km.put_raw_key(PRIVATE_KEY, ADDRESS) - d2 = self.km.put_raw_key(PRIVATE_KEY_2, ADDRESS_2) - return gatherResults([d1, d2]) - - def tearDown(self): - km = self._key_manager() - # wait for the indexes to be ready for the tear down - d = km._openpgp.deferred_init - d.addCallback(lambda _: self.delete_all_keys(km)) - d.addCallback(lambda _: self._soledad.close()) - return d - - def delete_all_keys(self, km): - def delete_keys(keys): - deferreds = [] - for key in keys: - d = km._openpgp.delete_key(key) - deferreds.append(d) - return gatherResults(deferreds) - - def check_deleted(_, private): - d = km.get_all_keys(private=private) - d.addCallback(lambda keys: self.assertEqual(keys, [])) - return d - - deferreds = [] - for private in [True, False]: - d = km.get_all_keys(private=private) - d.addCallback(delete_keys) - d.addCallback(check_deleted, private) - deferreds.append(d) - return gatherResults(deferreds) - - def _key_manager(self, user=ADDRESS, url='', token=None, - ca_cert_path=None): - return KeyManager(user, url, self._soledad, token=token, - gpgbinary=self.gpg_binary_path, - ca_cert_path=ca_cert_path) - - def _find_gpg(self): - gpg_path = distutils.spawn.find_executable('gpg') - if gpg_path is not None: - return os.path.realpath(gpg_path) - else: - return "/usr/bin/gpg" - - def get_public_binary_key(self): - with open(PATH + '/fixtures/public_key.bin', 'r') as binary_public_key: - return binary_public_key.read() - - def get_private_binary_key(self): - with open( - PATH + '/fixtures/private_key.bin', 'r') as binary_private_key: - return binary_private_key.read() - - -# key 24D18DDF: public key "Leap Test Key " -KEY_FINGERPRINT = "E36E738D69173C13D709E44F2F455E2824D18DDF" -PUBLIC_KEY = """ ------BEGIN PGP PUBLIC KEY BLOCK----- -Version: GnuPG v1.4.10 (GNU/Linux) - -mQINBFC9+dkBEADNRfwV23TWEoGc/x0wWH1P7PlXt8MnC2Z1kKaKKmfnglVrpOiz -iLWoiU58sfZ0L5vHkzXHXCBf6Eiy/EtUIvdiWAn+yASJ1mk5jZTBKO/WMAHD8wTO -zpMsFmWyg3xc4DkmFa9KQ5EVU0o/nqPeyQxNMQN7px5pPwrJtJFmPxnxm+aDkPYx -irDmz/4DeDNqXliazGJKw7efqBdlwTHkl9Akw2gwy178pmsKwHHEMOBOFFvX61AT -huKqHYmlCGSliwbrJppTG7jc1/ls3itrK+CWTg4txREkSpEVmfcASvw/ZqLbjgfs -d/INMwXnR9U81O8+7LT6yw/ca4ppcFoJD7/XJbkRiML6+bJ4Dakiy6i727BzV17g -wI1zqNvm5rAhtALKfACha6YO43aJzairO4II1wxVHvRDHZn2IuKDDephQ3Ii7/vb -hUOf6XCSmchkAcpKXUOvbxm1yfB1LRa64mMc2RcZxf4mW7KQkulBsdV5QG2276lv -U2UUy2IutXcGP5nXC+f6sJJGJeEToKJ57yiO/VWJFjKN8SvP+7AYsQSqINUuEf6H -T5gCPCraGMkTUTPXrREvu7NOohU78q6zZNaL3GW8ai7eSeANSuQ8Vzffx7Wd8Y7i -Pw9sYj0SMFs1UgjbuL6pO5ueHh+qyumbtAq2K0Bci0kqOcU4E9fNtdiovQARAQAB -tBxMZWFwIFRlc3QgS2V5IDxsZWFwQGxlYXAuc2U+iQI3BBMBCAAhBQJQvfnZAhsD -BQsJCAcDBRUKCQgLBRYCAwEAAh4BAheAAAoJEC9FXigk0Y3fT7EQAKH3IuRniOpb -T/DDIgwwjz3oxB/W0DDMyPXowlhSOuM0rgGfntBpBb3boezEXwL86NPQxNGGruF5 -hkmecSiuPSvOmQlqlS95NGQp6hNG0YaKColh+Q5NTspFXCAkFch9oqUje0LdxfSP -QfV9UpeEvGyPmk1I9EJV/YDmZ4+Djge1d7qhVZInz4Rx1NrSyF/Tc2EC0VpjQFsU -Y9Kb2YBBR7ivG6DBc8ty0jJXi7B4WjkFcUEJviQpMF2dCLdonCehYs1PqsN1N7j+ -eFjQd+hqVMJgYuSGKjvuAEfClM6MQw7+FmFwMyLgK/Ew/DttHEDCri77SPSkOGSI -txCzhTg6798f6mJr7WcXmHX1w1Vcib5FfZ8vTDFVhz/XgAgArdhPo9V6/1dgSSiB -KPQ/spsco6u5imdOhckERE0lnAYvVT6KE81TKuhF/b23u7x+Wdew6kK0EQhYA7wy -7LmlaNXc7rMBQJ9Z60CJ4JDtatBWZ0kNrt2VfdDHVdqBTOpl0CraNUjWE5YMDasr -K2dF5IX8D3uuYtpZnxqg0KzyLg0tzL0tvOL1C2iudgZUISZNPKbS0z0v+afuAAnx -2pTC3uezbh2Jt8SWTLhll4i0P4Ps5kZ6HQUO56O+/Z1cWovX+mQekYFmERySDR9n -3k1uAwLilJmRmepGmvYbB8HloV8HqwgguQINBFC9+dkBEAC0I/xn1uborMgDvBtf -H0sEhwnXBC849/32zic6udB6/3Efk9nzbSpL3FSOuXITZsZgCHPkKarnoQ2ztMcS -sh1ke1C5gQGms75UVmM/nS+2YI4vY8OX/GC/on2vUyncqdH+bR6xH5hx4NbWpfTs -iQHmz5C6zzS/kuabGdZyKRaZHt23WQ7JX/4zpjqbC99DjHcP9BSk7tJ8wI4bkMYD -uFVQdT9O6HwyKGYwUU4sAQRAj7XCTGvVbT0dpgJwH4RmrEtJoHAx4Whg8mJ710E0 -GCmzf2jqkNuOw76ivgk27Kge+Hw00jmJjQhHY0yVbiaoJwcRrPKzaSjEVNgrpgP3 -lXPRGQArgESsIOTeVVHQ8fhK2YtTeCY9rIiO+L0OX2xo9HK7hfHZZWL6rqymXdyS -fhzh/f6IPyHFWnvj7Brl7DR8heMikygcJqv+ed2yx7iLyCUJ10g12I48+aEj1aLe -dP7lna32iY8/Z0SHQLNH6PXO9SlPcq2aFUgKqE75A/0FMk7CunzU1OWr2ZtTLNO1 -WT/13LfOhhuEq9jTyTosn0WxBjJKq18lnhzCXlaw6EAtbA7CUwsD3CTPR56aAXFK -3I7KXOVAqggrvMe5Tpdg5drfYpI8hZovL5aAgb+7Y5ta10TcJdUhS5K3kFAWe/td -U0cmWUMDP1UMSQ5Jg6JIQVWhSwARAQABiQIfBBgBCAAJBQJQvfnZAhsMAAoJEC9F -Xigk0Y3fRwsP/i0ElYCyxeLpWJTwo1iCLkMKz2yX1lFVa9nT1BVTPOQwr/IAc5OX -NdtbJ14fUsKL5pWgW8OmrXtwZm1y4euI1RPWWubG01ouzwnGzv26UcuHeqC5orZj -cOnKtL40y8VGMm8LoicVkRJH8blPORCnaLjdOtmA3rx/v2EXrJpSa3AhOy0ZSRXk -ZSrK68AVNwamHRoBSYyo0AtaXnkPX4+tmO8X8BPfj125IljubvwZPIW9VWR9UqCE -VPfDR1XKegVb6VStIywF7kmrknM1C5qUY28rdZYWgKorw01hBGV4jTW0cqde3N51 -XT1jnIAa+NoXUM9uQoGYMiwrL7vNsLlyyiW5ayDyV92H/rIuiqhFgbJsHTlsm7I8 -oGheR784BagAA1NIKD1qEO9T6Kz9lzlDaeWS5AUKeXrb7ZJLI1TTCIZx5/DxjLqM -Tt/RFBpVo9geZQrvLUqLAMwdaUvDXC2c6DaCPXTh65oCZj/hqzlJHH+RoTWWzKI+ -BjXxgUWF9EmZUBrg68DSmI+9wuDFsjZ51BcqvJwxyfxtTaWhdoYqH/UQS+D1FP3/ -diZHHlzwVwPICzM9ooNTgbrcDzyxRkIVqsVwBq7EtzcvgYUyX53yG25Giy6YQaQ2 -ZtQ/VymwFL3XdUWV6B/hU4PVAFvO3qlOtdJ6TpE+nEWgcWjCv5g7RjXX -=MuOY ------END PGP PUBLIC KEY BLOCK----- -""" -PRIVATE_KEY = """ ------BEGIN PGP PRIVATE KEY BLOCK----- -Version: GnuPG v1.4.10 (GNU/Linux) - -lQcYBFC9+dkBEADNRfwV23TWEoGc/x0wWH1P7PlXt8MnC2Z1kKaKKmfnglVrpOiz -iLWoiU58sfZ0L5vHkzXHXCBf6Eiy/EtUIvdiWAn+yASJ1mk5jZTBKO/WMAHD8wTO -zpMsFmWyg3xc4DkmFa9KQ5EVU0o/nqPeyQxNMQN7px5pPwrJtJFmPxnxm+aDkPYx -irDmz/4DeDNqXliazGJKw7efqBdlwTHkl9Akw2gwy178pmsKwHHEMOBOFFvX61AT -huKqHYmlCGSliwbrJppTG7jc1/ls3itrK+CWTg4txREkSpEVmfcASvw/ZqLbjgfs -d/INMwXnR9U81O8+7LT6yw/ca4ppcFoJD7/XJbkRiML6+bJ4Dakiy6i727BzV17g -wI1zqNvm5rAhtALKfACha6YO43aJzairO4II1wxVHvRDHZn2IuKDDephQ3Ii7/vb -hUOf6XCSmchkAcpKXUOvbxm1yfB1LRa64mMc2RcZxf4mW7KQkulBsdV5QG2276lv -U2UUy2IutXcGP5nXC+f6sJJGJeEToKJ57yiO/VWJFjKN8SvP+7AYsQSqINUuEf6H -T5gCPCraGMkTUTPXrREvu7NOohU78q6zZNaL3GW8ai7eSeANSuQ8Vzffx7Wd8Y7i -Pw9sYj0SMFs1UgjbuL6pO5ueHh+qyumbtAq2K0Bci0kqOcU4E9fNtdiovQARAQAB -AA/+JHtlL39G1wsH9R6UEfUQJGXR9MiIiwZoKcnRB2o8+DS+OLjg0JOh8XehtuCs -E/8oGQKtQqa5bEIstX7IZoYmYFiUQi9LOzIblmp2vxOm+HKkxa4JszWci2/ZmC3t -KtaA4adl9XVnshoQ7pijuCMUKB3naBEOAxd8s9d/JeReGIYkJErdrnVfNk5N71Ds -FmH5Ll3XtEDvgBUQP3nkA6QFjpsaB94FHjL3gDwum/cxzj6pCglcvHOzEhfY0Ddb -J967FozQTaf2JW3O+w3LOqtcKWpq87B7+O61tVidQPSSuzPjCtFF0D2LC9R/Hpky -KTMQ6CaKja4MPhjwywd4QPcHGYSqjMpflvJqi+kYIt8psUK/YswWjnr3r4fbuqVY -VhtiHvnBHQjz135lUqWvEz4hM3Xpnxydx7aRlv5NlevK8+YIO5oFbWbGNTWsPZI5 -jpoFBpSsnR1Q5tnvtNHauvoWV+XN2qAOBTG+/nEbDYH6Ak3aaE9jrpTdYh0CotYF -q7csANsDy3JvkAzeU6WnYpsHHaAjqOGyiZGsLej1UcXPFMosE/aUo4WQhiS8Zx2c -zOVKOi/X5vQ2GdNT9Qolz8AriwzsvFR+bxPzyd8V6ALwDsoXvwEYinYBKK8j0OPv -OOihSR6HVsuP9NUZNU9ewiGzte/+/r6pNXHvR7wTQ8EWLcEIAN6Zyrb0bHZTIlxt -VWur/Ht2mIZrBaO50qmM5RD3T5oXzWXi/pjLrIpBMfeZR9DWfwQwjYzwqi7pxtYx -nJvbMuY505rfnMoYxb4J+cpRXV8MS7Dr1vjjLVUC9KiwSbM3gg6emfd2yuA93ihv -Pe3mffzLIiQa4mRE3wtGcioC43nWuV2K2e1KjxeFg07JhrezA/1Cak505ab/tmvP -4YmjR5c44+yL/YcQ3HdFgs4mV+nVbptRXvRcPpolJsgxPccGNdvHhsoR4gwXMS3F -RRPD2z6x8xeN73Q4KH3bm01swQdwFBZbWVfmUGLxvN7leCdfs9+iFJyqHiCIB6Iv -mQfp8F0IAOwSo8JhWN+V1dwML4EkIrM8wUb4yecNLkyR6TpPH/qXx4PxVMC+vy6x -sCtjeHIwKE+9vqnlhd5zOYh7qYXEJtYwdeDDmDbL8oks1LFfd+FyAuZXY33DLwn0 -cRYsr2OEZmaajqUB3NVmj3H4uJBN9+paFHyFSXrH68K1Fk2o3n+RSf2EiX+eICwI -L6rqoF5sSVUghBWdNegV7qfy4anwTQwrIMGjgU5S6PKW0Dr/3iO5z3qQpGPAj5OW -ATqPWkDICLbObPxD5cJlyyNE2wCA9VVc6/1d6w4EVwSq9h3/WTpATEreXXxTGptd -LNiTA1nmakBYNO2Iyo3djhaqBdWjk+EIAKtVEnJH9FAVwWOvaj1RoZMA5DnDMo7e -SnhrCXl8AL7Z1WInEaybasTJXn1uQ8xY52Ua4b8cbuEKRKzw/70NesFRoMLYoHTO -dyeszvhoDHberpGRTciVmpMu7Hyi33rM31K9epA4ib6QbbCHnxkWOZB+Bhgj1hJ8 -xb4RBYWiWpAYcg0+DAC3w9gfxQhtUlZPIbmbrBmrVkO2GVGUj8kH6k4UV6kUHEGY -HQWQR0HcbKcXW81ZXCCD0l7ROuEWQtTe5Jw7dJ4/QFuqZnPutXVRNOZqpl6eRShw -7X2/a29VXBpmHA95a88rSQsL+qm7Fb3prqRmuMCtrUZgFz7HLSTuUMR867QcTGVh -cCBUZXN0IEtleSA8bGVhcEBsZWFwLnNlPokCNwQTAQgAIQUCUL352QIbAwULCQgH -AwUVCgkICwUWAgMBAAIeAQIXgAAKCRAvRV4oJNGN30+xEACh9yLkZ4jqW0/wwyIM -MI896MQf1tAwzMj16MJYUjrjNK4Bn57QaQW926HsxF8C/OjT0MTRhq7heYZJnnEo -rj0rzpkJapUveTRkKeoTRtGGigqJYfkOTU7KRVwgJBXIfaKlI3tC3cX0j0H1fVKX -hLxsj5pNSPRCVf2A5mePg44HtXe6oVWSJ8+EcdTa0shf03NhAtFaY0BbFGPSm9mA -QUe4rxugwXPLctIyV4uweFo5BXFBCb4kKTBdnQi3aJwnoWLNT6rDdTe4/nhY0Hfo -alTCYGLkhio77gBHwpTOjEMO/hZhcDMi4CvxMPw7bRxAwq4u+0j0pDhkiLcQs4U4 -Ou/fH+pia+1nF5h19cNVXIm+RX2fL0wxVYc/14AIAK3YT6PVev9XYEkogSj0P7Kb -HKOruYpnToXJBERNJZwGL1U+ihPNUyroRf29t7u8flnXsOpCtBEIWAO8Muy5pWjV -3O6zAUCfWetAieCQ7WrQVmdJDa7dlX3Qx1XagUzqZdAq2jVI1hOWDA2rKytnReSF -/A97rmLaWZ8aoNCs8i4NLcy9Lbzi9QtornYGVCEmTTym0tM9L/mn7gAJ8dqUwt7n -s24dibfElky4ZZeItD+D7OZGeh0FDuejvv2dXFqL1/pkHpGBZhEckg0fZ95NbgMC -4pSZkZnqRpr2GwfB5aFfB6sIIJ0HGARQvfnZARAAtCP8Z9bm6KzIA7wbXx9LBIcJ -1wQvOPf99s4nOrnQev9xH5PZ820qS9xUjrlyE2bGYAhz5Cmq56ENs7THErIdZHtQ -uYEBprO+VFZjP50vtmCOL2PDl/xgv6J9r1Mp3KnR/m0esR+YceDW1qX07IkB5s+Q -us80v5LmmxnWcikWmR7dt1kOyV/+M6Y6mwvfQ4x3D/QUpO7SfMCOG5DGA7hVUHU/ -Tuh8MihmMFFOLAEEQI+1wkxr1W09HaYCcB+EZqxLSaBwMeFoYPJie9dBNBgps39o -6pDbjsO+or4JNuyoHvh8NNI5iY0IR2NMlW4mqCcHEazys2koxFTYK6YD95Vz0RkA -K4BErCDk3lVR0PH4StmLU3gmPayIjvi9Dl9saPRyu4Xx2WVi+q6spl3ckn4c4f3+ -iD8hxVp74+wa5ew0fIXjIpMoHCar/nndsse4i8glCddINdiOPPmhI9Wi3nT+5Z2t -9omPP2dEh0CzR+j1zvUpT3KtmhVICqhO+QP9BTJOwrp81NTlq9mbUyzTtVk/9dy3 -zoYbhKvY08k6LJ9FsQYySqtfJZ4cwl5WsOhALWwOwlMLA9wkz0eemgFxStyOylzl -QKoIK7zHuU6XYOXa32KSPIWaLy+WgIG/u2ObWtdE3CXVIUuSt5BQFnv7XVNHJllD -Az9VDEkOSYOiSEFVoUsAEQEAAQAP/1AagnZQZyzHDEgw4QELAspYHCWLXE5aZInX -wTUJhK31IgIXNn9bJ0hFiSpQR2xeMs9oYtRuPOu0P8oOFMn4/z374fkjZy8QVY3e -PlL+3EUeqYtkMwlGNmVw5a/NbNuNfm5Darb7pEfbYd1gPcni4MAYw7R2SG/57GbC -9gucvspHIfOSfBNLBthDzmK8xEKe1yD2eimfc2T7IRYb6hmkYfeds5GsqvGI6mwI -85h4uUHWRc5JOlhVM6yX8hSWx0L60Z3DZLChmc8maWnFXd7C8eQ6P1azJJbW71Ih -7CoK0XW4LE82vlQurSRFgTwfl7wFYszW2bOzCuhHDDtYnwH86Nsu0DC78ZVRnvxn -E8Ke/AJgrdhIOo4UAyR+aZD2+2mKd7/waOUTUrUtTzc7i8N3YXGi/EIaNReBXaq+ -ZNOp24BlFzRp+FCF/pptDW9HjPdiV09x0DgICmeZS4Gq/4vFFIahWctg52NGebT0 -Idxngjj+xDtLaZlLQoOz0n5ByjO/Wi0ANmMv1sMKCHhGvdaSws2/PbMR2r4caj8m -KXpIgdinM/wUzHJ5pZyF2U/qejsRj8Kw8KH/tfX4JCLhiaP/mgeTuWGDHeZQERAT -xPmRFHaLP9/ZhvGNh6okIYtrKjWTLGoXvKLHcrKNisBLSq+P2WeFrlme1vjvJMo/ -jPwLT5o9CADQmcbKZ+QQ1ZM9v99iDZol7SAMZX43JC019sx6GK0u6xouJBcLfeB4 -OXacTgmSYdTa9RM9fbfVpti01tJ84LV2SyL/VJq/enJF4XQPSynT/tFTn1PAor6o -tEAAd8fjKdJ6LnD5wb92SPHfQfXqI84rFEO8rUNIE/1ErT6DYifDzVCbfD2KZdoF -cOSp7TpD77sY1bs74ocBX5ejKtd+aH99D78bJSMM4pSDZsIEwnomkBHTziubPwJb -OwnATy0LmSMAWOw5rKbsh5nfwCiUTM20xp0t5JeXd+wPVWbpWqI2EnkCEN+RJr9i -7dp/ymDQ+Yt5wrsN3NwoyiexPOG91WQVCADdErHsnglVZZq9Z8Wx7KwecGCUurJ2 -H6lKudv5YOxPnAzqZS5HbpZd/nRTMZh2rdXCr5m2YOuewyYjvM757AkmUpM09zJX -MQ1S67/UX2y8/74TcRF97Ncx9HeELs92innBRXoFitnNguvcO6Esx4BTe1OdU6qR -ER3zAmVf22Le9ciXbu24DN4mleOH+OmBx7X2PqJSYW9GAMTsRB081R6EWKH7romQ -waxFrZ4DJzZ9ltyosEJn5F32StyLrFxpcrdLUoEaclZCv2qka7sZvi0EvovDVEBU -e10jOx9AOwf8Gj2ufhquQ6qgVYCzbP+YrodtkFrXRS3IsljIchj1M2ffB/0bfoUs -rtER9pLvYzCjBPg8IfGLw0o754Qbhh/ReplCRTusP/fQMybvCvfxreS3oyEriu/G -GufRomjewZ8EMHDIgUsLcYo2UHZsfF7tcazgxMGmMvazp4r8vpgrvW/8fIN/6Adu -tF+WjWDTvJLFJCe6O+BFJOWrssNrrra1zGtLC1s8s+Wfpe+bGPL5zpHeebGTwH1U -22eqgJArlEKxrfarz7W5+uHZJHSjF/K9ZvunLGD0n9GOPMpji3UO3zeM8IYoWn7E -/EWK1XbjnssNemeeTZ+sDh+qrD7BOi+vCX1IyBxbfqnQfJZvmcPWpruy1UsO+aIC -0GY8Jr3OL69dDQ21jueJAh8EGAEIAAkFAlC9+dkCGwwACgkQL0VeKCTRjd9HCw/+ -LQSVgLLF4ulYlPCjWIIuQwrPbJfWUVVr2dPUFVM85DCv8gBzk5c121snXh9Swovm -laBbw6ate3BmbXLh64jVE9Za5sbTWi7PCcbO/bpRy4d6oLmitmNw6cq0vjTLxUYy -bwuiJxWREkfxuU85EKdouN062YDevH+/YResmlJrcCE7LRlJFeRlKsrrwBU3BqYd -GgFJjKjQC1peeQ9fj62Y7xfwE9+PXbkiWO5u/Bk8hb1VZH1SoIRU98NHVcp6BVvp -VK0jLAXuSauSczULmpRjbyt1lhaAqivDTWEEZXiNNbRyp17c3nVdPWOcgBr42hdQ -z25CgZgyLCsvu82wuXLKJblrIPJX3Yf+si6KqEWBsmwdOWybsjygaF5HvzgFqAAD -U0goPWoQ71PorP2XOUNp5ZLkBQp5etvtkksjVNMIhnHn8PGMuoxO39EUGlWj2B5l -Cu8tSosAzB1pS8NcLZzoNoI9dOHrmgJmP+GrOUkcf5GhNZbMoj4GNfGBRYX0SZlQ -GuDrwNKYj73C4MWyNnnUFyq8nDHJ/G1NpaF2hiof9RBL4PUU/f92JkceXPBXA8gL -Mz2ig1OButwPPLFGQhWqxXAGrsS3Ny+BhTJfnfIbbkaLLphBpDZm1D9XKbAUvdd1 -RZXoH+FTg9UAW87eqU610npOkT6cRaBxaMK/mDtGNdc= -=JTFu ------END PGP PRIVATE KEY BLOCK----- -""" - -# key 7FEE575A: public key "anotheruser " -PUBLIC_KEY_2 = """ ------BEGIN PGP PUBLIC KEY BLOCK----- -Version: GnuPG v1.4.10 (GNU/Linux) - -mI0EUYwJXgEEAMbTKHuPJ5/Gk34l9Z06f+0WCXTDXdte1UBoDtZ1erAbudgC4MOR -gquKqoj3Hhw0/ILqJ88GcOJmKK/bEoIAuKaqlzDF7UAYpOsPZZYmtRfPC2pTCnXq -Z1vdeqLwTbUspqXflkCkFtfhGKMq5rH8GV5a3tXZkRWZhdNwhVXZagC3ABEBAAG0 -IWFub3RoZXJ1c2VyIDxhbm90aGVydXNlckBsZWFwLnNlPoi4BBMBAgAiBQJRjAle -AhsDBgsJCAcDAgYVCAIJCgsEFgIDAQIeAQIXgAAKCRB/nfpof+5XWotuA/4tLN4E -gUr7IfLy2HkHAxzw7A4rqfMN92DIM9mZrDGaWRrOn3aVF7VU1UG7MDkHfPvp/cFw -ezoCw4s4IoHVc/pVlOkcHSyt4/Rfh248tYEJmFCJXGHpkK83VIKYJAithNccJ6Q4 -JE/o06Mtf4uh/cA1HUL4a4ceqUhtpLJULLeKo7iNBFGMCV4BBADsyQI7GR0wSAxz -VayLjuPzgT+bjbFeymIhjuxKIEwnIKwYkovztW+4bbOcQs785k3Lp6RzvigTpQQt -Z/hwcLOqZbZw8t/24+D+Pq9mMP2uUvCFFqLlVvA6D3vKSQ/XNN+YB919WQ04jh63 -yuRe94WenT1RJd6xU1aaUff4rKizuQARAQABiJ8EGAECAAkFAlGMCV4CGwwACgkQ -f536aH/uV1rPZQQAqCzRysOlu8ez7PuiBD4SebgRqWlxa1TF1ujzfLmuPivROZ2X -Kw5aQstxgGSjoB7tac49s0huh4X8XK+BtJBfU84JS8Jc2satlfwoyZ35LH6sDZck -I+RS/3we6zpMfHs3vvp9xgca6ZupQxivGtxlJs294TpJorx+mFFqbV17AzQ= -=Thdu ------END PGP PUBLIC KEY BLOCK----- -""" - -PRIVATE_KEY_2 = """ ------BEGIN PGP PRIVATE KEY BLOCK----- -Version: GnuPG v1.4.10 (GNU/Linux) - -lQHYBFGMCV4BBADG0yh7jyefxpN+JfWdOn/tFgl0w13bXtVAaA7WdXqwG7nYAuDD -kYKriqqI9x4cNPyC6ifPBnDiZiiv2xKCALimqpcwxe1AGKTrD2WWJrUXzwtqUwp1 -6mdb3Xqi8E21LKal35ZApBbX4RijKuax/BleWt7V2ZEVmYXTcIVV2WoAtwARAQAB -AAP7BLuSAx7tOohnimEs74ks8l/L6dOcsFQZj2bqs4AoY3jFe7bV0tHr4llypb/8 -H3/DYvpf6DWnCjyUS1tTnXSW8JXtx01BUKaAufSmMNg9blKV6GGHlT/Whe9uVyks -7XHk/+9mebVMNJ/kNlqq2k+uWqJohzC8WWLRK+d1tBeqDsECANZmzltPaqUsGV5X -C3zszE3tUBgptV/mKnBtopKi+VH+t7K6fudGcG+bAcZDUoH/QVde52mIIjjIdLje -uajJuHUCAO1mqh+vPoGv4eBLV7iBo3XrunyGXiys4a39eomhxTy3YktQanjjx+ty -GltAGCs5PbWGO6/IRjjvd46wh53kzvsCAO0J97gsWhzLuFnkxFAJSPk7RRlyl7lI -1XS/x0Og6j9XHCyY1OYkfBm0to3UlCfkgirzCYlTYObCofzdKFIPDmSqHbQhYW5v -dGhlcnVzZXIgPGFub3RoZXJ1c2VyQGxlYXAuc2U+iLgEEwECACIFAlGMCV4CGwMG -CwkIBwMCBhUIAgkKCwQWAgMBAh4BAheAAAoJEH+d+mh/7ldai24D/i0s3gSBSvsh -8vLYeQcDHPDsDiup8w33YMgz2ZmsMZpZGs6fdpUXtVTVQbswOQd8++n9wXB7OgLD -izgigdVz+lWU6RwdLK3j9F+Hbjy1gQmYUIlcYemQrzdUgpgkCK2E1xwnpDgkT+jT -oy1/i6H9wDUdQvhrhx6pSG2kslQst4qjnQHYBFGMCV4BBADsyQI7GR0wSAxzVayL -juPzgT+bjbFeymIhjuxKIEwnIKwYkovztW+4bbOcQs785k3Lp6RzvigTpQQtZ/hw -cLOqZbZw8t/24+D+Pq9mMP2uUvCFFqLlVvA6D3vKSQ/XNN+YB919WQ04jh63yuRe -94WenT1RJd6xU1aaUff4rKizuQARAQABAAP9EyElqJ3dq3EErXwwT4mMnbd1SrVC -rUJrNWQZL59mm5oigS00uIyR0SvusOr+UzTtd8ysRuwHy5d/LAZsbjQStaOMBILx -77TJveOel0a1QK0YSMF2ywZMCKvquvjli4hAtWYz/EwfuzQN3t23jc5ny+GqmqD2 -3FUxLJosFUfLNmECAO9KhVmJi+L9dswIs+2Dkjd1eiRQzNOEVffvYkGYZyKxNiXF -UA5kvyZcB4iAN9sWCybE4WHZ9jd4myGB0MPDGxkCAP1RsXJbbuD6zS7BXe5gwunO -2q4q7ptdSl/sJYQuTe1KNP5d/uGsvlcFfsYjpsopasPjFBIncc/2QThMKlhoEaEB -/0mVAxpT6SrEvUbJ18z7kna24SgMPr3OnPMxPGfvNLJY/Xv/A17YfoqjmByCvsKE -JCDjopXtmbcrZyoEZbEht9mko4ifBBgBAgAJBQJRjAleAhsMAAoJEH+d+mh/7lda -z2UEAKgs0crDpbvHs+z7ogQ+Enm4EalpcWtUxdbo83y5rj4r0TmdlysOWkLLcYBk -o6Ae7WnOPbNIboeF/FyvgbSQX1POCUvCXNrGrZX8KMmd+Sx+rA2XJCPkUv98Hus6 -THx7N776fcYHGumbqUMYrxrcZSbNveE6SaK8fphRam1dewM0 -=a5gs ------END PGP PRIVATE KEY BLOCK----- -""" diff --git a/mail/src/leap/mail/testing/common.py b/mail/src/leap/mail/testing/common.py deleted file mode 100644 index 1bf1de2..0000000 --- a/mail/src/leap/mail/testing/common.py +++ /dev/null @@ -1,109 +0,0 @@ -# -*- coding: utf-8 -*- -# common.py -# Copyright (C) 2014 LEAP -# -# This program is free software: you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation, either version 3 of the License, or -# (at your option) any later version. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with this program. If not, see . -""" -Common utilities for testing Soledad. -""" -import os -import tempfile - -from twisted.internet import defer -from twisted.trial import unittest - -from leap.common.testing.basetest import BaseLeapTest -from leap.soledad.client import Soledad - -# TODO move to common module, or Soledad itself -# XXX remove duplication - -TEST_USER = "testuser@leap.se" -TEST_PASSWD = "1234" - - -def _initialize_soledad(email, gnupg_home, tempdir): - """ - Initializes soledad by hand - - :param email: ID for the user - :param gnupg_home: path to home used by gnupg - :param tempdir: path to temporal dir - :rtype: Soledad instance - """ - - uuid = "foobar-uuid" - passphrase = u"verysecretpassphrase" - secret_path = os.path.join(tempdir, "secret.gpg") - local_db_path = os.path.join(tempdir, "soledad.u1db") - server_url = "https://provider" - cert_file = "" - - soledad = Soledad( - uuid, - passphrase, - secret_path, - local_db_path, - server_url, - cert_file, - syncable=False) - - return soledad - - -class SoledadTestMixin(unittest.TestCase, BaseLeapTest): - """ - It is **VERY** important that this base is added *AFTER* unittest.TestCase - """ - - def setUp(self): - self.results = [] - self.setUpEnv() - - # pytest handles correctly the setupEnv for the class, - # but trial ignores it. - if not getattr(self, 'tempdir', None): - self.tempdir = tempfile.gettempdir() - - # Soledad: config info - self.gnupg_home = "%s/gnupg" % self.tempdir - self.email = 'leap@leap.se' - - # initialize soledad by hand so we can control keys - self._soledad = _initialize_soledad( - self.email, - self.gnupg_home, - self.tempdir) - - return defer.succeed(True) - - def tearDown(self): - """ - tearDown method called after each test. - """ - self.results = [] - try: - self._soledad.close() - except Exception: - print "ERROR WHILE CLOSING SOLEDAD" - # logging.exception(exc) - self.tearDownEnv() - - @classmethod - def setUpClass(self): - pass - - @classmethod - def tearDownClass(self): - pass diff --git a/mail/src/leap/mail/testing/imap.py b/mail/src/leap/mail/testing/imap.py deleted file mode 100644 index 72acbf2..0000000 --- a/mail/src/leap/mail/testing/imap.py +++ /dev/null @@ -1,186 +0,0 @@ -# -*- coding: utf-8 -*- -# utils.py -# Copyright (C) 2014, 2015 LEAP -# -# This program is free software: you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation, either version 3 of the License, or -# (at your option) any later version. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with this program. If not, see . -""" -Common utilities for testing Soledad IMAP Server. -""" -from email import parser - -from mock import Mock -from twisted.cred.checkers import ICredentialsChecker -from twisted.cred.credentials import IUsernamePassword -from twisted.cred.error import UnauthorizedLogin -from twisted.cred.portal import Portal, IRealm -from twisted.mail import imap4 -from twisted.internet import defer -from twisted.protocols import loopback -from twisted.python import log -from zope.interface import implementer - -from leap.mail.adaptors import soledad as soledad_adaptor -from leap.mail.imap.account import IMAPAccount -from leap.mail.imap.server import LEAPIMAPServer -from leap.mail.testing.common import SoledadTestMixin - -TEST_USER = "testuser@leap.se" -TEST_PASSWD = "1234" - - -# -# Simple IMAP4 Client for testing -# - -class SimpleClient(imap4.IMAP4Client): - """ - A Simple IMAP4 Client to test our - Soledad-LEAPServer - """ - - def __init__(self, deferred, contextFactory=None): - imap4.IMAP4Client.__init__(self, contextFactory) - self.deferred = deferred - self.events = [] - - def serverGreeting(self, caps): - self.deferred.callback(None) - - def modeChanged(self, writeable): - self.events.append(['modeChanged', writeable]) - self.transport.loseConnection() - - def flagsChanged(self, newFlags): - self.events.append(['flagsChanged', newFlags]) - self.transport.loseConnection() - - def newMessages(self, exists, recent): - self.events.append(['newMessages', exists, recent]) - self.transport.loseConnection() - -# -# Dummy credentials for tests -# - - -@implementer(IRealm) -class TestRealm(object): - - def __init__(self, account): - self._account = account - - def requestAvatar(self, avatarId, mind, *interfaces): - avatar = self._account - return (imap4.IAccount, avatar, - getattr(avatar, 'logout', lambda: None)) - - -@implementer(ICredentialsChecker) -class TestCredentialsChecker(object): - - credentialInterfaces = (IUsernamePassword,) - - userid = TEST_USER - password = TEST_PASSWD - - def requestAvatarId(self, credentials): - username, password = credentials.username, credentials.password - d = self.checkTestCredentials(username, password) - d.addErrback(lambda f: defer.fail(UnauthorizedLogin())) - return d - - def checkTestCredentials(self, username, password): - if username == self.userid and password == self.password: - return defer.succeed(username) - else: - return defer.fail(Exception("Wrong credentials")) - - -class TestSoledadIMAPServer(LEAPIMAPServer): - - def __init__(self, account, *args, **kw): - - LEAPIMAPServer.__init__(self, *args, **kw) - - realm = TestRealm(account) - portal = Portal(realm) - checker = TestCredentialsChecker() - self.checker = checker - self.portal = portal - portal.registerChecker(checker) - - -class IMAP4HelperMixin(SoledadTestMixin): - """ - MixIn containing several utilities to be shared across - different TestCases - """ - serverCTX = None - clientCTX = None - - def setUp(self): - - soledad_adaptor.cleanup_deferred_locks() - - USERID = TEST_USER - - def setup_server(account): - self.server = TestSoledadIMAPServer( - account=account, - contextFactory=self.serverCTX) - self.server.theAccount = account - - d_server_ready = defer.Deferred() - self.client = SimpleClient( - d_server_ready, contextFactory=self.clientCTX) - self.connected = d_server_ready - - def setup_account(_): - self.parser = parser.Parser() - - # XXX this should be fixed in soledad. - # Soledad sync makes trial block forever. The sync it's mocked to - # fix this problem. _mock_soledad_get_from_index can be used from - # the tests to provide documents. - # TODO see here, possibly related? - # -- http://www.pythoneye.com/83_20424875/ - self._soledad.sync = Mock() - - d = defer.Deferred() - self.acc = IMAPAccount(self._soledad, USERID, d=d) - return d - - d = super(IMAP4HelperMixin, self).setUp() - d.addCallback(setup_account) - d.addCallback(setup_server) - return d - - def tearDown(self): - SoledadTestMixin.tearDown(self) - del self._soledad - del self.client - del self.server - del self.connected - - def _cbStopClient(self, ignore): - self.client.transport.loseConnection() - - def _ebGeneral(self, failure): - self.client.transport.loseConnection() - self.server.transport.loseConnection() - if hasattr(self, 'function'): - log.err(failure, "Problem with %r" % (self.function,)) - - def loopback(self): - return loopback.loopbackAsync(self.server, self.client) diff --git a/mail/src/leap/mail/testing/smtp.py b/mail/src/leap/mail/testing/smtp.py deleted file mode 100644 index d8690f1..0000000 --- a/mail/src/leap/mail/testing/smtp.py +++ /dev/null @@ -1,51 +0,0 @@ -from twisted.mail import smtp - -from leap.mail.smtp.gateway import SMTPFactory, LOCAL_FQDN -from leap.mail.smtp.gateway import SMTPDelivery -from leap.mail.outgoing.service import outgoingFactory - -TEST_USER = u'anotheruser@leap.se' - - -class UnauthenticatedSMTPServer(smtp.SMTP): - - encrypted_only = False - - def __init__(self, soledads, keyms, opts, encrypted_only=False): - smtp.SMTP.__init__(self) - - userid = TEST_USER - keym = keyms[userid] - - class Opts: - cert = '/tmp/cert' - key = '/tmp/cert' - hostname = 'remote' - port = 666 - - outgoing = outgoingFactory( - userid, keym, Opts, check_cert=False) - avatar = SMTPDelivery(userid, keym, encrypted_only, outgoing) - self.delivery = avatar - - def validateFrom(self, helo, origin): - return origin - - -class UnauthenticatedSMTPFactory(SMTPFactory): - """ - A Factory that produces a SMTP server that does not authenticate user. - Only for tests! - """ - protocol = UnauthenticatedSMTPServer - domain = LOCAL_FQDN - encrypted_only = False - - -def getSMTPFactory(soledad_s, keymanager_s, sendmail_opts, - encrypted_only=False): - factory = UnauthenticatedSMTPFactory - factory.encrypted_only = encrypted_only - proto = factory( - soledad_s, keymanager_s, sendmail_opts).buildProtocol(('127.0.0.1', 0)) - return proto diff --git a/mail/src/leap/mail/tests/rfc822.bounce.message b/mail/src/leap/mail/tests/rfc822.bounce.message deleted file mode 100644 index 7a51ac0..0000000 --- a/mail/src/leap/mail/tests/rfc822.bounce.message +++ /dev/null @@ -1,152 +0,0 @@ -Return-Path: <> -X-Original-To: yoyo@dev.pixelated-project.org -Delivered-To: a6973ec1af0a6d1e2a1e4db4ff85f6c2@deliver.local -Received: by dev1.dev.pixelated-project.org (Postfix) - id 92CEA83164; Thu, 16 Jun 2016 14:53:34 +0200 (CEST) -Date: Thu, 16 Jun 2016 14:53:34 +0200 (CEST) -From: MAILER-DAEMON@dev1.dev.pixelated-project.org (Mail Delivery System) -Subject: Undelivered Mail Returned to Sender -To: yoyo@dev.pixelated-project.org -Auto-Submitted: auto-replied -MIME-Version: 1.0 -Content-Type: multipart/report; report-type=delivery-status; - boundary="8F60183010.1466081614/dev1.dev.pixelated-project.org" -Message-Id: <20160616125334.92CEA83164@dev1.dev.pixelated-project.org> - -This is a MIME-encapsulated message. - ---8F60183010.1466081614/dev1.dev.pixelated-project.org -Content-Description: Notification -Content-Type: text/plain; charset=us-ascii - -This is the mail system at host dev1.dev.pixelated-project.org. - -I'm sorry to have to inform you that your message could not -be delivered to one or more recipients. It's attached below. - -For further assistance, please send mail to postmaster. - -If you do so, please include this problem report. You can -delete your own text from the attached returned message. - - The mail system - -: host caribou.leap.se[176.53.69.122] said: 550 5.1.1 - : Recipient address rejected: User unknown in virtual alias - table (in reply to RCPT TO command) - ---8F60183010.1466081614/dev1.dev.pixelated-project.org -Content-Description: Delivery report -Content-Type: message/delivery-status - -Reporting-MTA: dns; dev1.dev.pixelated-project.org -X-Postfix-Queue-ID: 8F60183010 -X-Postfix-Sender: rfc822; yoyo@dev.pixelated-project.org -Arrival-Date: Thu, 16 Jun 2016 14:53:33 +0200 (CEST) - -Final-Recipient: rfc822; nobody@leap.se -Original-Recipient: rfc822;nobody@leap.se -Action: failed -Status: 5.1.1 -Remote-MTA: dns; caribou.leap.se -Diagnostic-Code: smtp; 550 5.1.1 : Recipient address rejected: - User unknown in virtual alias table - ---8F60183010.1466081614/dev1.dev.pixelated-project.org -Content-Description: Undelivered Message -Content-Type: message/rfc822 - -Return-Path: -Received: from leap.mail-0.4.0rc1+111.g736ea86 (localhost [127.0.0.1]) - (using TLSv1 with cipher ECDHE-RSA-AES128-SHA (128/128 bits)) - (Client CN "yoyo@dev.pixelated-project.org", Issuer "Pixelated Project Root CA (client certificates only!)" (verified OK)) - by dev1.dev.pixelated-project.org (Postfix) with ESMTPS id 8F60183010 - for ; Thu, 16 Jun 2016 14:53:33 +0200 (CEST) -MIME-Version: 1.0 -Content-Type: multipart/signed; protocol="application/pgp-signature"; - micalg="pgp-sha512"; boundary="===============7598747164910592838==" -To: nobody@leap.se -Subject: vrgg -From: yoyo@dev.pixelated-project.org -Date: Thu, 16 Jun 2016 13:53:32 -0000 -Message-Id: <20160616125332.16961.677041909.5@dev1.dev.pixelated-project.org> -OpenPGP: id=CB546109E857BC34DFF2BCB3288870B39C400C24; - url="https://dev.pixelated-project.org/key/yoyo"; preference="signencrypt" - ---===============7598747164910592838== -Content-Type: multipart/mixed; boundary="===============3737055506052708210==" -MIME-Version: 1.0 -To: nobody@leap.se -Subject: vrgg -From: yoyo@dev.pixelated-project.org -Date: Thu, 16 Jun 2016 13:53:32 -0000 - ---===============3737055506052708210== -Content-Type: text/plain; charset="utf-8" -MIME-Version: 1.0 -Content-Transfer-Encoding: base64 - - ---===============3737055506052708210== -Content-Type: application/pgp-keys -MIME-Version: 1.0 -content-disposition: attachment; filename="yoyo@dev.pixelated-project.org-email-key.asc" -Content-Transfer-Encoding: base64 - -LS0tLS1CRUdJTiBQR1AgUFVCTElDIEtFWSBCTE9DSy0tLS0tCgptUUlOQkZkZ01BZ0JFQURIWWpU -T20wcTdOT0lYVUpoTmlHVXg2S05OZ1M0Q0I2VlMvbGtab2UvYjZuRjdCSENmCkFnRVkxeFlxMkIv -MzA3YzBtNTZWMEZvOWt2ZmZCUWhQckU5WG9rckI5blRlN1RsSDZUNTdiV09LSWMyMHhNSy8KSlVU -djZ3UEpybjdLN0VyNEdxbzdrUmpWcFVBcWlBbGFxMkhVYllGd2NEMnBIb0VENmU2L01CZDBVUTFX -b2s4QQpPNURDc2ZmeWhBZ0NFU1poK2w2VHlsVEJXYTJDTmJvUTl0SWtPZ0ZWTk9kTW9uWkxoTk1N -Y0tIeU54dmF5bUdCCjhjQlRISVE2UWhGRThvR2JDRTdvczdZWWhyTmNmcUsyMzJJQllzTHNXN3Vk -QmdwRTA0YkpwQWlvbW1zTHBCYmwKV0pCSjdqeEhwWmhJR3JGL1ltejNsSXpkbm9Mb3BSSWJyS0pC -MmxaVDhIUHBlTVVJdVE2eHErd3RhQXFJVzlPTgo5U29uZWYyVU5BL3VseW1LeDRkOFhxbEwxY3hE -aDFQU1E5YVlPcVg0RDlrMklmOXZmR2hET0xVMzR2Y2VFOC8vCnM1WGdTY2ZFbHg2SWlEVWZHdGx2 -aE5zQUM4TmhhUU1sOHJjUXVoRDA2RFdvSUowMVhkeFJVM2JSVVZkc0I1NWMKcXRWSHJMbVBVb256 -NU13MGFURzlTZzZudUlQcU1QOVNKRlBzbVpzR3ZYVnZWbCtSNzl1SFBlc25yWkoyTjZqOQpNaUth -S045NFBhL1dJUnRoYWdzVnpHeHNtd2orTVZCRkZKRmh0TUtnNlFzYUsvbzRLNGJFR1ZLdWNXQk1i -MnNxCldmd0o0SndTcHcrOHgyS3p6aXhWTllTZXhRdm9oMkc3RDRmRXdISDJzazNST3k3dTlldjhs -bEVqUFFBUkFRQUIKdEQ5NWIzbHZRR1JsZGk1d2FYaGxiR0YwWldRdGNISnZhbVZqZEM1dmNtY2dQ -SGx2ZVc5QVpHVjJMbkJwZUdWcwpZWFJsWkMxd2NtOXFaV04wTG05eVp6NkpBajRFRXdFQ0FDZ0ZB -bGRnTUFnQ0d5OEZDUUhnTUZnR0N3a0lCd01DCkJoVUlBZ2tLQ3dRV0FnTUJBaDRCQWhlQUFBb0pF -Q2lJY0xPY1FBd2s4djBQL2o2MmNyNjRUMlZPMVNKdHp1RlEKWjVpeVJsVFVHSGN2NW5hQjlUSDdI -VVB3cTVwekZiTkg5SnhNRjVFRWtvZjdvV0hWeldWVTFBM1NDdzVNZ2FFbwppWTk5ZFBGNzdHazJ4 -ZEczNXZlWmIwWkg2WkVLdks1S042VXBucG5IeStxaVZVc1FLcE9DdUZKNkF0UlVEOTRJClJ2YnUv -S1hsMHdORDlzVXFlYkJZN1BBSlRNY1RjLzVEdWpIT1Erd3VlSkFtaFZZbEozVnpZK1lBS2t5U05B -QVoKZ3VVenNyUm5xQWU5SmU5TGgrcERpcVpHT2tEK1Z3b2kvRlVPQXJwbWFnNzZONTVjR3hiK2VG -QUlzRHYrM1NNOQpjUDFyQkFON2lEaGgvdkdJeHgzMFlrYUlpMmpmcXg3VXUydnNwSXh6K0NsWWdi -dm1wZm1CWmFqVzYzR0FsK3YvCngrby92eFZmVTMraTZ3alFjRS8vRTBTR2pvY3lQdUw0ZTZLNERy -S3k2SHQycjBQckdHVFZ0dUZPaWU2dnVzbVcKL09sdVB1dGszU3o1S1BmRDFpRXBobmpPQ0pNRkZx -Z2xRM1pPa3MweG00WGdwWW1ycnpQcXc1WWlzK1NEVjhobwp6anlrSzRWUlcrcC9IcUVzU29GQm5a -MG5XSmg2Q1pZOExIeVNiMVJwaFlMRFpWd21JRXd1OW12Vm1ISVIyWUZVCllNZEx4UExiOFZNei9t -QWpMb2Q0OGNSSzdSTzBSZ1RoMTUyK0VieXRGR3k5Y2tiS3VzRmJzVTFCQjN2MFJyUlUKenozTTcx -T3hjcFhVQ0tpWlI0MEVYZnErSnVtZVFudm1wSWdZdUNaQkh5MzJwQUJuOHNDdUlrMStyQnp4bXdt -bgp0WGh0K0RvNlExYXYyVjZYR00xV2xoKzEKPU8zaHEKLS0tLS1FTkQgUEdQIFBVQkxJQyBLRVkg -QkxPQ0stLS0tLQo= ---===============3737055506052708210==-- - ---===============7598747164910592838== -Content-Type: application/pgp-signature; name="signature.asc" -MIME-Version: 1.0 -Content-Description: OpenPGP Digital Signature - ------BEGIN PGP SIGNATURE----- - -iQIcBAABCgAGBQJXYqFNAAoJECiIcLOcQAwkDEIQAL67/XJXDv+lusoy18jr7Ony -WQEP0pIRLp4GywGpH3dAITFAkuamO4VX3QEdVGjOHNoaT8VkSVWf9mnsYLl+Mh2v -1OIwMv0u8WyVtrcxyXijIznnJv8X1RgyCzpUJcmOh04VZcDyxKbnFHWSDMfJ4Jtq -qnXDONcfEeT8pwrGjP5qzTgcF/irG3w5svyQjEtj6kycddYtqUc9Hx3cMaRIzsHg -kuUzznSzU/6P0Z345q/kXyYvU9rlcsP9vogrsqL2ueLwYSipxUJQUrRWG82FYoCo -PAKNdGIt0xl2gEW+xWZkJqFarPiUFCx//+bVBelKrqj6rjwbj+E7mHJW318JYVHQ -en3Smv7pEWlT4hZHXnoe8ng6TAvKzQjf7/bUxq2JpKSycp2hDO3Qz3Tv+kc+jC/r -5UDWe/flR+syq8lAQTRSn6057g3BgDG2RtAwsjedg1aTFSrljSxbKlK4vsj5Muek -Olq9+MUdMFSE3Jj/JC2COcS3rlt/Qt+JLDYXKahU3CodaSgF2dobikDe1bW0/QNS -7O4Ng2PK0pA416RCFRUgPXerUnMGiWAiq7BoRHeym9y7fkHYhIYGpPVKXJ6t67y5 -JjvuzwfwG8SZTp4Wy2pg1Mr6znm6uVBxUDxTHyP3BjciI1zpEigOIg9UwJ9nCDxL -uUGz4VqipNKbkpRkjLLW -=3IaF ------END PGP SIGNATURE----- - ---===============7598747164910592838==-- - ---8F60183010.1466081614/dev1.dev.pixelated-project.org-- diff --git a/mail/src/leap/mail/tests/rfc822.message b/mail/src/leap/mail/tests/rfc822.message deleted file mode 100644 index ee97ab9..0000000 --- a/mail/src/leap/mail/tests/rfc822.message +++ /dev/null @@ -1,86 +0,0 @@ -Return-Path: -Delivered-To: exarkun@meson.dyndns.org -Received: from localhost [127.0.0.1] - by localhost with POP3 (fetchmail-6.2.1) - for exarkun@localhost (single-drop); Thu, 20 Mar 2003 14:50:20 -0500 (EST) -Received: from pyramid.twistedmatrix.com (adsl-64-123-27-105.dsl.austtx.swbell.net [64.123.27.105]) - by intarweb.us (Postfix) with ESMTP id 4A4A513EA4 - for ; Thu, 20 Mar 2003 14:49:27 -0500 (EST) -Received: from localhost ([127.0.0.1] helo=pyramid.twistedmatrix.com) - by pyramid.twistedmatrix.com with esmtp (Exim 3.35 #1 (Debian)) - id 18w648-0007Vl-00; Thu, 20 Mar 2003 13:51:04 -0600 -Received: from acapnotic by pyramid.twistedmatrix.com with local (Exim 3.35 #1 (Debian)) - id 18w63j-0007VK-00 - for ; Thu, 20 Mar 2003 13:50:39 -0600 -To: twisted-commits@twistedmatrix.com -From: etrepum CVS -Reply-To: twisted-python@twistedmatrix.com -X-Mailer: CVSToys -Message-Id: -Subject: [Twisted-commits] rebuild now works on python versions from 2.2.0 and up. -Sender: twisted-commits-admin@twistedmatrix.com -Errors-To: twisted-commits-admin@twistedmatrix.com -X-BeenThere: twisted-commits@twistedmatrix.com -X-Mailman-Version: 2.0.11 -Precedence: bulk -List-Help: -List-Post: -List-Subscribe: , - -List-Id: -List-Unsubscribe: , - -List-Archive: -Date: Thu, 20 Mar 2003 13:50:39 -0600 - -Modified files: -Twisted/twisted/python/rebuild.py 1.19 1.20 - -Log message: -rebuild now works on python versions from 2.2.0 and up. - - -ViewCVS links: -http://twistedmatrix.com/users/jh.twistd/viewcvs/cgi/viewcvs.cgi/twisted/python/rebuild.py.diff?r1=text&tr1=1.19&r2=text&tr2=1.20&cvsroot=Twisted - -Index: Twisted/twisted/python/rebuild.py -diff -u Twisted/twisted/python/rebuild.py:1.19 Twisted/twisted/python/rebuild.py:1.20 ---- Twisted/twisted/python/rebuild.py:1.19 Fri Jan 17 13:50:49 2003 -+++ Twisted/twisted/python/rebuild.py Thu Mar 20 11:50:08 2003 -@@ -206,15 +206,27 @@ - clazz.__dict__.clear() - clazz.__getattr__ = __getattr__ - clazz.__module__ = module.__name__ -+ if newclasses: -+ import gc -+ if (2, 2, 0) <= sys.version_info[:3] < (2, 2, 2): -+ hasBrokenRebuild = 1 -+ gc_objects = gc.get_objects() -+ else: -+ hasBrokenRebuild = 0 - for nclass in newclasses: - ga = getattr(module, nclass.__name__) - if ga is nclass: - log.msg("WARNING: new-class %s not replaced by reload!" % reflect.qual(nclass)) - else: -- import gc -- for r in gc.get_referrers(nclass): -- if isinstance(r, nclass): -+ if hasBrokenRebuild: -+ for r in gc_objects: -+ if not getattr(r, '__class__', None) is nclass: -+ continue - r.__class__ = ga -+ else: -+ for r in gc.get_referrers(nclass): -+ if getattr(r, '__class__', None) is nclass: -+ r.__class__ = ga - if doLog: - log.msg('') - log.msg(' (fixing %s): ' % str(module.__name__)) - - -_______________________________________________ -Twisted-commits mailing list -Twisted-commits@twistedmatrix.com -http://twistedmatrix.com/cgi-bin/mailman/listinfo/twisted-commits diff --git a/mail/src/leap/mail/tests/rfc822.multi-minimal.message b/mail/src/leap/mail/tests/rfc822.multi-minimal.message deleted file mode 100644 index 582297c..0000000 --- a/mail/src/leap/mail/tests/rfc822.multi-minimal.message +++ /dev/null @@ -1,16 +0,0 @@ -Content-Type: multipart/mixed; boundary="===============6203542367371144092==" -MIME-Version: 1.0 -Subject: [TEST] 010 - Inceptos cum lorem risus congue -From: testmailbitmaskspam@gmail.com -To: test_c5@dev.bitmask.net - ---===============6203542367371144092== -Content-Type: text/plain; charset="us-ascii" -MIME-Version: 1.0 -Content-Transfer-Encoding: 7bit - -Howdy from python! -The subject: [TEST] 010 - Inceptos cum lorem risus congue -Current date & time: Wed Jan 8 16:36:21 2014 -Trying to attach: [] ---===============6203542367371144092==-- diff --git a/mail/src/leap/mail/tests/rfc822.multi-nested.message b/mail/src/leap/mail/tests/rfc822.multi-nested.message deleted file mode 100644 index 694bef5..0000000 --- a/mail/src/leap/mail/tests/rfc822.multi-nested.message +++ /dev/null @@ -1,619 +0,0 @@ -From: TEST -Content-Type: multipart/alternative; - boundary="Apple-Mail=_F4EF9C8E-2E66-4FC6-8840-F435ADBED5C8" -X-Smtp-Server: smtp.example.com:test.bitmask -Subject: test simple attachment -X-Universally-Unique-Identifier: 0ea1b4b2-cdb8-43c3-b54c-dc88a19c6e0a -Date: Wed, 8 Jul 2015 04:25:56 +0900 -Message-Id: <47278179-628A-43F5-95C9-BC7E1753C521@example.com> -To: test_alpha14_001@dev.bitmask.net -Mime-Version: 1.0 (Apple Message framework v1251.1) - - ---Apple-Mail=_F4EF9C8E-2E66-4FC6-8840-F435ADBED5C8 -Content-Transfer-Encoding: 7bit -Content-Type: text/plain; - charset=us-ascii - -this is a simple attachment ---Apple-Mail=_F4EF9C8E-2E66-4FC6-8840-F435ADBED5C8 -Content-Type: multipart/related; - type="text/html"; - boundary="Apple-Mail=_C7D5288F-B043-4A7F-AF3F-1EDF1A78438B" - - ---Apple-Mail=_C7D5288F-B043-4A7F-AF3F-1EDF1A78438B -Content-Transfer-Encoding: 7bit -Content-Type: text/html; - charset=us-ascii - -this is a simple attachment ---Apple-Mail=_C7D5288F-B043-4A7F-AF3F-1EDF1A78438B -Content-Transfer-Encoding: base64 -Content-Disposition: inline; - filename="saing_ergol.jpg" -Content-Type: image/jpg; - x-mac-hide-extension=yes; - x-unix-mode=0600; - name="saint_ergol.jpg" -Content-Id: <163B7957-4342-485F-8FD6-D46A4A53A2C1> - -/9j/4AAQSkZJRgABAQEAYABgAAD/4QCURXhpZgAASUkqAAgAAAACADEBAgALAAAAJgAAAGmHBAAB -AAAAMgAAAAAAAABQaWNhc2EgMy4wAAAEAAKgBAABAAAALAEAAAOgBAABAAAAHgEAAACQBwAEAAAA -MDIxMAWgBAABAAAAaAAAAAAAAAACAAEAAgAFAAAAhgAAAAIABwAEAAAAMDEwMAAAAAAgICAgAAD/ -7QAcUGhvdG9zaG9wIDMuMAA4QklNBAQAAAAAAAD//gAmRmlsZSB3cml0dGVuIGJ5IEFkb2JlIFBo -b3Rvc2hvcKggNS4y/9sAQwAFAwQEBAMFBAQEBQUFBgcMCAcHBwcPCwsJDBEPEhIRDxERExYcFxMU -GhURERghGBodHR8fHxMXIiQiHiQcHh8e/9sAQwEFBQUHBgcOCAgOHhQRFB4eHh4eHh4eHh4eHh4e -Hh4eHh4eHh4eHh4eHh4eHh4eHh4eHh4eHh4eHh4eHh4eHh4e/8AAEQgBHgEsAwEiAAIRAQMRAf/E -AB0AAAEFAQEBAQAAAAAAAAAAAAUCAwQGBwgAAQn/xABQEAABAwIEAwUFBQUGBAQFAgcCAwQFARIA -BhETFCEiBzEyQVEVI0JSYSQzYnGBCBZDcqElNIKRkqJTscHRRGOy0jVUc4PhF2SkwsPT4vDy/8QA -FAEBAAAAAAAAAAAAAAAAAAAAAP/EABQRAQAAAAAAAAAAAAAAAAAAAAD/2gAMAwEAAhEDEQA/ANW2 -3RMhJzJOiUK4up+4Lv8A4JUt/rgjBZfmN1JRBsaDchEklXfSaQ15GkdfFX6YteR4tNtDiuuCC7gh -6iHpt89O+vLFg+ARv8WAQ0bizSSaD1CkNo4fuL58J2/BdhXSJ+PAex64sL+PCDTEjK47vw4D1qnz -49b1+Pw4bMrf+IOFWj/NgPdV5Wn/AIceSUuMus/5sKMRw2Y+Lo6i8OAV4kiETtL5sMGpaACofUXh -/F+WPN1vvei0R/1YV7tRLpC4fxYBvcT2hJMztIenxXD/AP76YVvCICQnd+G/COpMNwvu/iMvh/Om -Bkgs63QXYtk+IIv4h9FuAJm4+0XCdpdN4+Lp+v8A3xGBZYXBJ3mVt1nzW/T1w6qmperb0koNoeG8 -f+44Emo6QBJN9YoSnVv+FJUvr5p4CVxBKXEmsdxdNvwCPpT5S+mIvGSBN1d4zbOkytuLqEgH1p/1 -phDtZwg1VIt+0LbwEKXjr6j/ABKfWmGGjhQmSvE/CVqVp3Xaf8Mq+Gv0rgLEyUW4JJVT4vx3D+mv -w4dN04IVyRv6Oq6y4fr3YQ0IXMeLkQMRtutIOr/LAqY4VNJqMgYJo2/iG3/TgGJDMyyCS5JuQEh6 -UhctlQH9a24EuJbjnFzGShyISuV3HiqRidPpUfBhSpEoBptjfKIp+7AUXgFdp8WhjiM3FYd1y5fr -kj/+7igI0h/FZXATmSz7dIkW0omSfUYIyQGCo18x6sNHMSjFVVZ28mEESK1tcwFURL5a2692BgLM -RBJMloRe4iIbQWQJL8Q99448kzjyVVJNtFqXCJf/ABJUAX+nOnTgGpiWeO0l0F5K5NTqVFRFZsqg -VO46Wj4cRop8pxC7FN4+EhSEr05gDScj+C8qFStPyxMWEuldBaUUWTIhDZeJKkkVP4daENLh9MMG -Kj5IWOy6ZFaSi7ZaNAyIfmoQlTAS28lME0/vkpvUuFuuTluYpDX/AIlL+/DDSQmkFVV1JibJqmRE -ZKMwIkD866hWuqeGHCjdNp7SF4xJO0UUC9lGIL+VqlKH5YdSKNjY8XJLNdkfERM1ht15W1pd4cA0 -bjMSkO1TXkl7R98u54ZUCS6unwjzGuGnr6U2lVBklEBUEb7XLojSLyU0t8OBTh8oSSCaaLVNqur0 -ELlZK3q0tKlRLpri6/um4FoRIGgIiQ2JEe6QpU/g+XTrgKe79rPrU1JWR4xMbiJs8dW20LkuOo86 -euLXkyPnEGntKSknwrOfAgm5JUEh7q8lNO/lXBiJg4tsDYkGa+8gO4Jl/wCmnV3YK+8USVLZ6vh6 -PX9cA3/aCaVvtJdRa3wiil1f7sKbuH3FluP1yH+QLS8vmx401ib7lnw22eHCVk1t0RFY+rqO0BL/ -AK+eAc4p0nu778y+XoDp/rhYLOCMS4ldNNPqUuttLUfzxBj94jESbGSY9VpIj1f7sPpKOBcEPiR8 -I22l3eeAfCSJNv8AfXeK0yWG78PnhJksropxT1O6mtorDp/zwhuKyiRD4VBL8FuHiJ0VdRqadPl6 -cBToqUKPcIFwx2q9JG5tAiL5SKnQWL5GPEXzIXYgpaXzYzA0U2J7DZYLrSIjK9iRad415bZV/wAW -Lbld4oLtBMUdtuokV1yJCV/0INRL/PAWsxHCTG48KuIvD1Y8ZF04BIF1+D/Vh3wn4MNH81/+rHkl -FLyuDAKtuAui7DG5aYiQdP8A6cP9VmEqjcA3dJCXTbgHbv8Adho07rh/04i++Td9R/Z/iH4h/LEq -4f5R8WAauIlbbOr5sNJIrClcXVb8v/TEhZMd3cI/F0/LiPcKYffGPwl8PV8v0r9cB5VToLr/AAl/ -3w2bckw93Z7zxB4bvy+WuEGPujITu6rb7P8AbXDpimolaQWl8v4cBFSWTLpUW3xTK0yILSSLy19M -J4cve9ZkXSN/iK38VPOmJjtPwiSxipb0mPUQ/wDfDTdmSYbdgW9V3i/1U/PADDTWEFSQC7cK4Osv -Ll7s/g/LFbnSeEqKaaNyYn9qVIPCXounT/1ji7pR6nUQncRdRJWdKunnWnrhjg7URVH3dtxXl4kv -yKvfT6VwDGWnlzUWy5nb4UNzquGnfYfxYmSo7bQiT8SfUIpmIjZX+bWlcDJXejW5Wsw2SIulO4ki -9Knp1BX60w5BOJJTpK9dmRWmC3jQ/wDudyg4AYlH+9XJyz30yt6lGHr8VydKcsNfYU2/EpmDZQSF -Mi49VLp9DpUsWl6mok4FZAA92Nto3D0V/Lz9MQ9kSbiShuhu6SEljLqr9K0wFb2RJXiWjl7t/Akm -5SPq86p1KlS/TDSqJCkkx3nztFUSWICYIkJD8vIeRUxMerJppCg5O1MS6QWATEvxDU9LcDHCm84J -sotHEiRD0u0UhEiqPK6oFUqVwC3DFumqlILhHDtESaS7mNJIh8tK1DTDjRSPdpW3tUFhuG4X5pEk -XzUpUudK4S9jfY2X9xcDQapl9pVbPFRJIfmr088KbpuvZiqyDx0u3QtUQJN4KpD+KnPUh591cAwH -Ctm5Nr0C3SuMBmCucl8wal34YVki4dqgUk6+0+AikgK4e+xTX4sSjGYUZIERrkiJEoR3pWkdS5VD -UuRU+XEVVGSkHpFs76atyZbkakQql5jqBVtL64B9wimQCuKxjtD1A5lfuh8k66d419cXBJwptIKc -GaC20IkIgJj4eY618WBOX4V42PcI1+DBINseGAC8VemulcWAy6/dhaJXdI2Db/XAI6VQLYDbLwkJ -AIkWvnzwwkTwXApkshaVvQNtwjTyxISIdolFPDbdaVn/AErjLc5zXs3tryVEoOTLjUFxVSTBK0Q6 -a3F1YDVne8QCQrAh8XgHqH64jKl96oRgIj1FcAlb+euHHAjYkmR3DddulZ4qfrjAP2jc/SBOEsoZ -QACcO1RRVGwS4nXltAdK8vxYAjCdpEl2g9qbWCyS5XbQMWRLSL4Ww3LlTltalTQRrz6sbU3RtutC -34hGwBIv9uM77Gsh/uLk9nHqAHHEJOHRpoo+Ov47uqlMXreWvJRS8fitLa9/08q63eeAmfcJWqBc -RfDYHh/yw8KgFTVR0FS8/By/pga7HfAVCcroDd0j7rw/L4sRyXSGtqbghEeVKJrJW/8AqwAqHb9D -wRBBNFUusGjwXIJLXaXCmoOtLq9/PEHedRbhrw14iToU3IIgQKiXz7ZXUrSv81MTNtuILryiICok -vtkT1htEvr8eoUpX/Fgrl8dj3fGLrt0uoy4wHNw1/PUsBZEpBO9dN2sCbhLqMRO60fLyxNPwfNis -N3DUnBKNt9QkyK4rFi6q/StcSmT5RNJK5suSZdV4omQ3fStfLAGrbvEGEqjaZW9JW3Ya4gbBu3BI -urqAsKbum6hlaYEVvhE7iup5aUwDqRDtdXi+LCjttx4BEgust+a3Dd2302GWAcMRsu+L/wBOGNxM -XBCXT8mJXwXYQqnuWlZ1D4S+IcBD3hTPpDcu6enqtH6YdNNEjJT4bbbvFcP5YYVuG5AQuIuoPhu9 -dPr9MeScEKQWmChEPSRBbd6/kXrgPJKfZ/uTEfiEerp/7Y8BCQDaBl8tvi/w4TasQEW9bcQiN3hH -/tXD7RNTiOo9serpEOkvzwDrdEhD357iheIvw4UZWmPXaNuFH8OB0hxCjhL7xAfDfvANuAlGp8Im -Al8JYGPVhbRi4qLIcQp8qxCJfrpX/liY7uF2P2k0xttHrH/rgE4UFRJUk3LohTIRsTchd/lgCPHN -xSVInIbJJCVhH0W07x8OJzQkRbjsBanaJdJ9PPu0xXjUUct0k0wXXJUesSRAwVG3zqI9OFRjxxGp -L3AuLceoS4AhtKnLnpTSuAKO+OT30yNclFLRSuciJF+tB6cQnpPBb2+/Hbt97xI3fkXThYSm4xIn -exxCdqgincHTXnd1/wBcQwkmq6tzaYBe64TMTStH/wAutNP64CO4cOC3RHilRTESEBcpF0111tuH -nXEF66TFUSdn94NoGswEwIbddCqBU0KmIOY2/wDYSqbYFyRQIVC4ZEF10Du8VtB1KnPvwwkxWsJy -ViZKiKaqAmq2FyFP4yfVpgCDRwjcP2xqoip4hK9Akh/lOh0KnpiU4FYj8Cgo23CAtgXD86VAg/yx -GbxrjhLWz90797uAuRi5EUqFrs1Guv6Urh9uiixVEk2xoN1BJR4AomgSRU7ipTy78AObrM7+GUbM -dsi3PdrEBEVC5EmBjpQ6fniL7NeNJAlF5VdMiPbQ3rhSXHv+8Ctaa0/lwb2SFwkmJm7akO8oCljw -efcVKVpUqaYRw6KIDcFpeIQbLKobpV53CmdbaF9MBKhJoU24i5WXXT+Ix97bp6+Gv+3BoH0W52iY -uQUEhu6ekuX0rirTCKZHuF7sRG5VdRtZ/wDxCendiGqo3kFUl00eGFoQ3P8AeErht15Kj02187sB -fEU00EiTU6iH8HixgvbLJIse1iFmbwIotdoN42ltCtuUK6nTrroPnjaoeWTctBtPc6RHqWSEhL8v -+WOY/wBo1qnJZgmpCPA3b5SRZxjVIQEw3QuKpnpy+LAbJ2wdozHKGWnjRMFymFUi4YSRG0ip6dVe -eMr/AGX8llO5gddpE62kRTTVL2ULvrAhLxndypr+mM37SE3Ehn1DK+aJv3iSqZTrps2E0m2gD9zQ -afJprSvcXPHQX/6pdnOR4GHyvl8150U0uGQZsAWclcI8x8VeeA15X3jgSTR27vFuIjaWn+L+mBzg -nC73qZqKIh4fcjbb6eLA7JMo+norjpbKr2H1+6auESuKlfi/pixNExELtlBsiXhSURtL/wDGAU4J -QgEU23u+nw22jiNSig1KgpbdNfD0f+3D52iGwIAJfONvhr9MOJN0doOIVQUU05lshz/pgKvk8lFz -VUFmCagpEo2BQ1hEdf4ZXjTSmJwC+GPXJjv9VvUpduj1cxpy50wJdSDx63SU3uJJUhECUDqJGnMh -U0KminLBSPTeNGQoILe5V6QcE2L3WpcwL3uAfVFwrH+Bds8EiEDELhtr8NfUa4fjGJF9rXRQFS0h -JIWxDcXxFrTCQZkTRW6xsI3dSiJ9I0/I+6uByKg2L75tbum0RWVSIS+bnrgH3AppgK6CKaZJ+7Ax -WVQO30PpxKSdPOLSUQDdTtLdEdpX8+dCuL/LEe5w2cWpuXyjhRK4myblJcSD56ajTnhi4SbjuGgT -dQvdEo2JIhKnKo3AXTWn9cBa4yQRcpGRGA+ERHqD+h0piZ8H8uK9GLcc3VUFYE9vwCR7/VTz0rQS -wdaERNxu6VLbiEfDgFbnXb4RHqLouwoyHpK/xYYcbZJXXn09XThO4NgkXhU8VuAfcJpqBtl1CXVg -c4TtcESnURl/qKnd+VfriVdtq2j4bekvl/8A8cIaKIuTVtv3E+kx/wCX+H64BIbJN+pbcISJMiIL -bvw1/wC+GnBOlHbXYM00RLqIfEJW/HTzH64mXeL8PUY/DhaRJkrb0XD8vit9MA4Y9YXdXy/zeuIb -jb4hLrtHq6Cs/wAu/E8y6Pw4iqkmVwqI7lxeGzAQTFwKvUZkn/wyAOn/AHYrr1PfkEF1D21Ej3Ac -popWkXop1f0xaXu8JpJoLbZXeKwekfm78AnCaKDhXrAhckN4rGAi5DuqQ/iwEG14mqSbYNxQUh4p -mTYRBXXvIK0r54UDdmpHtXKCIL7dyYlsmNpV+A7aV7sLcTBNIx8pe64dLpEhRG9IaFpb4ueKVlLO -0LmLMBRbaSapzyAkso2EDteI15Xjofl50wF1cOkWjRIWPFfMYC5VAhGneVNR8OuIAOnTlVBBiG4S -hWqr3pGC4V7+8qdWBj3bF17QjX/92LZJVS4dg689sqVu7/LD/tR0xkHTYTNNuyS60hWASI+/3fuu -dNO/ALVUYk0JymsCaKDrhh3GYXCXmFbCr04cBTgXDoUwPbQEVthNYwND0LQhp04Y4ckHaBDYo82F -FhDZExV10r16EPOmEx7MY+PSUXcroKLqk4VeEBCSQV52c7u/y9MAo3SjaQFpH3uZJ21EnI+6MHI9 -1fi13KUx5u+dNgERcgTe4UwdKXttoqeJI7ht/rgK92ZSP33Ky/Bqq7zoLBI0BEukkzGwvFjzdq+9 -sJJrtgFq594qwJyQk5DyWoJUPAGHr4V27qZURNOPRIiEyRFU0iEtPEFa1tuxFip5u+BUXZuouQbf -AS3uhKo6X0opbdTzxYjhWq6RIIHc12OHFIvAqN1dddK06sAnDUk5Uk2KwXDcmVywmIhQvuq0IfP+ -bATm7N8pGKrw7BBR4ncLpSOW2CVV7+jToLX88VtkNsgKd5xcgr4mrkCZkRVHUxu+7V/K7ErMUoSG -TYpPLrnhm6zpcXBX7H2ig60Cteq3XyxnPY/2mDmjKUxF9qMkbtvHKkK/EIjutgqdaCpRYdNa0r0+ -H4cBfXs5H5Fy4q9/u0ekVtt5ILqq17kaWUIVDrXlTSuMjnc0PMkuFc3yDMHeYph4LlJqp9+x1HSq -NCDWu5pt8jtwf7WG62TnCGYM2vAkW7RcigOH6mz4qhXkuPz2/wAQe6uAfYpl192n57LtEzeCCka0 -K1mwcrEHWOltdaD1269/ngCGTOynOHaQD7NGf5J1Cx8ouKhxgomKq4jyEj0Hl3Y3jKnZ/k3LcU1a -QmXmrZG7cMyRI1SK3kWteeDEYs3UcERGC/iESTvLp9MSDWETSQFsdviG5Eur6eLAIdtWqloi29yn -aXUF135YlJC3La2D6bbQ9zb03eHDUg3UXO0um23q2S6S/wBWFJDa48B3CN1+zaN3n8WAYVaqL+Hq -IS8JMwL/AA8ywpOrlMKDukNfOgtktNf9WPo7yO+oXSSnh9z1D/uwtFOhJ0qoNCL1FHl/zwFO9k7j -Rdjw32pdUiExRRISGnKleQ+PTExu1TbGQuUTURFIRO6NMBL/AEfH+WJziFRcu1UyYIdXUJcMBjdT -vLmV2EybNmRigIG2IhG4RBUbtC+YRrTAQ0nEgs0SYNHMcSbsis23JpKiPkHVXvpg+CaijcUxcuvl -L3wL2/QvXXENgK3tAVOMkRuK1UU3IKj/AJV6ta1w6qXRwyjZdQSVtAl2Y2l5W9+v64CL7LT6uJZg -O3aKSpMLbTu8N4UpqGPJM2qavwJ7he9Fs5VSC6heLSpYdVcC2aJJp8K2RHpvT3Q7tenSg1078Smj -wrBUI/fWimKAvBu8XfoemAkt2ai0vxithDZaIltKXfioVPDgoBEoFyVlvV1YQyIiAeu64viMSIfp -ywsFveql4huEREeq71wDlvw9H83iwwbVQTAkztG7q/CWF2rCJEO2RfD0fDhpJwN6qafwlcfuS/7Y -BXD7Ye7C3DAD767wlaQl8wj5a/hxOBQSDx4S4RIvuz27vDb8vy4BhJQiNXcC23wl4ht+anqP0wwk -jaZEhfb/AD+H8q+mJwC3TtH/AIf4+kdcJuTEBEQtTLp6Qu/5YBzq2hEvF8VuEqkXUJeH/EX/ACwO -OQtVFAjQtUO26+20qfniUagqAuh19Nt1wWjbXz1wA6QTH3q5AHu/CWzcVv5V78cO/tF5mkn3aLIx -t5sGrQxRsTAgAituvT07u/yx3ce2SVrYNzb8I+L+uMy7YOzGBzxDnxKJoPkklCQFFYrhOvxDbSv6 -0rgOaA7ZMwNsqOstyyISSyrNNuMin0iQcrhP1L8eAHYllOSzb2kRLaJM40RLiFV0zMSQAS1v1pXz -7sCc8ZHlss5lGJcrA9HwpKiZCJaDqVNC0rTBPsizxIZCzGM3HxpySl3CGkodo7NedRGlO8qaYDtF -2PAzo8MiabpISUVQUNUUnw28iHnbWtMQZNRvMmkPEgW6uK1yh2mx0LXqofw64g5S7RIHOzRV9ls9 -xqyt3+g+JYlUdbtKU8OJztZRo9Vkhc2oqJCNxbtr4bfS3kWAYVUTkGk0SgbbpRcW6pJgkREAc6KJ -20uw5Jk4kIol48zUtS2WwiCw8SlQeevPSheuILJaQTj1Xz4EE1I+5RmgIB039wqXFStcDknDpCPQ -UFtt2tVFJHoIeDMir1UqF3Td3YCHNvHSdqjFYF3X90YCpYXeXUioFerlXBaCJYdpdofDJqK7Krlb -dI0CHxDSuultcQ527LcU1cvgaqEggTl0k5edSqp9G8G5b82AEnIZils1rwmVI105FNJNpvpgCSQg -Y31PfCtev1wF3kM8ZbYvUkJZ5HNm5CooIEsiJidDLQuXPAKMzZE5tMUI+YQHiVxbGxTvG0R67yvr -pSug4qmcOyOaQdrzcyftJwo1TRSjE1ituoWlDVLTrGmObZhHMmX5NWQ3jjnC6pJhwhiJX09RGtef -0wHZMK+Yy3tDJahoC+kEifxxJgFhKp1tr4KaXW4wrKLHJ8tHv3c+/Xi3kI+X4tspq3SeJEqVqJqj -omZUPXvxXYftokGhtXCDBBo+ZvE3aZprHtlaNqgCnSmlx0xdf2pURm8nwXaDle9zluYK1dIQtJo4 -Itba0p6nry+bAEOzDN2X88LSPZZma9eHWIih3ghyaHXwhUx8yH0xdVe0px2ayC+Uhio4W8Paja/m -EkDVRt8YBWt35YwXKSOXYl6xEWexMKrp2MFrVV17kipRbcqVBS6vEHiHG3ZwJrm3ITybUh8uzWds -otxbvOJuVbJJFzopS2lbyHTADFf2hnhNFZlTIy4sbiEFxniSBUaFp0jUuf1wYaftCErk8syM8mOj -jWx7bhdaVtJL1rTq1PT6Y5omHHD5cXbOcvQ7aWc+8J4udV3TwVCvpVNGn3I2lrz8sPqsYtOPZsYJ -zl17JCQtjBPdJVyZBWhFQ1BEQGn/AKsB1UH7RWU49ukhmCEn4VZVIVrVmxq9Fe4/5cSm/b92XruN -v95F7i6R/s1a0tO7TXGB5oymcW+y3HlA5eczCqVq8EymCNddGo8ycOK0tAR05DrimRXGRsxPxooy -jBNcdkmcY5ScpEN1dBWXu0Cn1wHbDTPGVZtuxdxs3HKIyglwt3SatvfpSv8AXBU3xt6AndbS2laU -2fLHBEfGtyBiIg1zHFtECWkQjjJsbYSLXQlDtuL8sEY6NynJJG6mO0CQg16qVoDAk1XBIp/CNVB5 -FXTzwHfLgkWjvcXNBMi94S5ImIEXy60LH33K6pKILXCPVamsQld81taYHAoiVqHGAmoQ7hEm8MRI -bvQywg953aKDyR2WxkIriiC5CVPhry10rrgJircrxUUsLxdZIiYjr9QqOE3Ji3LrNdr0ioCZkYjp -3cq1oVP88BxcOFAJBNsxSTH3hoCCzUyG7xJlXQcE2ixEsQrhvolbwypbS+6NvrTrp+WAD5gtTNJB -8bHqVJS29Ud0KDyGvf7z0w/Dvickk5E91MhEWwkYGqOg8xr7umLAYpkySacS6QvK3pMw+vnzw+yj -yJ6q7XM/utkTvuEgu+SvxU+bADuIkuNSQbAZIiO4vaA+6O3y00xMZJuCtc8MumpaX3iJDb/u88Fg -FuhbaiFw/wCrCjcCNokZ/wCgsBW5XjEw3E0UFOq0bgMSEfTShYU3U2zFSwLbeohckI8h76UrSuLE -bpMQIusbvwFiCq8TUZELkwHc8XwiOndzr64Ac3dPHIAKJmV5WiqVq4/49KDg+BEKQ7h+8+IhC0eQ -/rgY3dMyblw2wmsXjJMwEh/PElJ8iqAknYSalw2isFpFTv0/LAJSj1hcKqqLApu23iSJF/W7DUgJ -IBxIo9Q9NxLEI/0w+ahDcuJmQlb4QEsOt3gk4Ftfa4EbiEukrfXTABVpJqmaSDkHS6aolaQgBj/W -lK64aSISMUxNBQh6htvSIR/FrUqYLTEaMk32uhD8RIiZf1pgA7FRNVJB3Zw4+7ESvQ3fw6F0lXAH -G6xEyEhO0RPrAguL+bkXdj0rJM4mHXdvjBNMEiUIhC278sVlw8GEaOncksxaIppXWFYNofmGla4w -ftI7YGso1eSEei6QZsEPdEsaoiurUtEqbZV08XxYDMO0gX072tlGtHi7twuuLQFyMeo1CtU8uXfi -29tfYf8Auy4/eDI58SzSFNNdgiZqqpHQaX6d3u6+eGP2QstupntT/eBf7SnG3EuQheIrGPLn3c+/ -HXOaBTFwTlN+u2WttIk3Ipd/cVR88Bz7+y/2fuo/L6+ZnYBFupJ1cglsqkIpCX3ZUup01r640aYb -uIsCtA1FFnQqGxTckIthp3Kp1IS11xfmjxFCPtF4gXT1qk8G0jp5VpSvLFWNw8KbfO1DBRNNJNMQ -RkrDH8SZGXd64BjbUdtHRXoLqSFok5JyBJKhTnbzGnOnngBl9jISUfLLsWdzVR4mmBuQNIeFEaCp -Sp39Y8vTAztleKZd7MppQTTXFJmVwEaSlxqFyup81PWmMo7D83ZwhJOAybNrIO4GSSEUHKhkQpDX -4aV88A7+0ApJZdmIwpts1dkquo4SapuSVQ2e8RKtaa/pjqPscnonNeQY6Zi0UGyKo2mgj0ikdOVR -rypzw1nXIcPmnK/AkCBN10veKiFxEGt1LP8AFTXAWHZw/Zrld409sAgN3ELmodypDXl+dacsAW7T -WakzCSMM2kgbFaKjxfwkklXlqNaefpjjjtDheEy5ttg2BTeJqM2ajkbyCpW7ltRvIirXW6qn6Yum -YO1bicvyKDGedEirIkntCwVAHI1PSiZrH4Nfp4cV6dzQ6YuDXm8vQ79FdcU3C8iZqAkr82zSvvCp -T1wGPzYqcaF3uyEtnYTRt2redvfX19caP2F9o0XFoyOQ88Gf7nTvTVVPxR61e5UK110/6Yoz2aUX -lXT5tY2RUXEngCijYJ9VKbIUHSg24BSCwru1VEkQTTU8IfKNOXOn1wGvZ17OEsmZ9QUmlgex7tcS -iiYPNpWSRrTQVRUMSpuXVG/F3/Z4lEYvtVQj1DfPWsw1KHkzWAOHFal9QBNQdL6UpXTw4pnY92kw -q8Gl2adosX7VywfUwclaS8atQdakNa/DjXshZByu2ze17SkM+Qj/ACwxVTcoCmAJEJAJDQbB00Pn -gMwzaz9jTs1lJMPZKjZ0omIRkUTlyulQtRGq6p17vSnw4qTduK/Zk8gpCbmxeRZE4OKbQgECVvxE -4pW6nV3/AFxrL2Jku0bND6ZUWQfw71cmDAE5Uw4NZQ613KgBeQ1540fIWT4WJhyaZLeLuU5Z+LAT -dnckg1bnQlST/CVuAweT7Jc9LysO0fxuVoseA41BVBZUeJCtLy3CqV1a8vw+LFn7MuyEnmeGK+b4 -eOUhZ2HN+q2ZOTQQZhTW3purUtdO+tca6xcOs2vppKQZoMlJmTGFaq9RAqxQG6ulfmrrin5tfOGs -rcxm2rkZaWHLisWwAiXYx6HKtEbeq+lOoq4AZlfsty64exzR8xdTjqSQcroA593wzIPBubdt1alb -1Y+9k+R8szGQo54tm4YdX3iZs0yREUqioVNNFBIu6lO+uCrJ44TcTXsebeu2+ZHiUPliTICEWyQc -1EK+VtLfF8WM17Ycms8ydoEh7IynGsvZdsc7obsU91wnSlTUpQOWlb6c68+WA6bRfOmwqsf7bQIx -EjQcMBck0qXylSvUOHYq7qTj0Ytyool71dRFVrv6fCVLeWKeqs6THbTMF1F3hNxXcrOGptv/ACi6 -i6dfPGgx4vBjxXXNdk4JAhVaovANJUacrqEY19cApwTxNuPArGKdvUkm8EiSKny6+VcCnEgiLjfc -tnW2V3VwAqiqXdcFUyraWBLuQlhbuicg+bR6CRJ8U/ZgqIl+aZDy0x5kUW5cPHaxxaCLZAViFNZZ -C3T4/FUcBeoov7PSTEwIfhtMgMf8J0pzwRVU21RJQzES8IiZdReflilQT58o3auRcunYqD0E2eIr -3fWuoULzwa450SpIyRmKIW2kszIBE/xFQ68/0wDUxnyFi8wDCSRmm66Ss2VS3Rr4Oqg201+tcFHc -oJASiBh0/ESN4/y10xmmeGL6NnWeeMttuPUYCXtEOPvBVGg89AIfHTvxYsnrZfncuK5kykYOWrtX -cVSUM7R08dnV3YC0gXFt99dHY6rvd33Fp5jW3COOInFwrIWq9NxLGQ+mmhDTA5u6auWQin/cfF1X -gaQ/NTqrityEw6d2pwjlBR9cRAYvBFJUKFzHqTr7yuAtjhZZFVVS8BTSStEiMCLn5U58+7EdJ069 -n7gguTciFQSEEiG36UvxBh3Cz4NwQfcOkXUKjMCJA6fDrrzHnhTh43TVdXbnvEulsTAhFUafxB0L -ASqqLKEIkTJdYri3SbKpDbQta30pSumPJSTxA9tA7vmBFyKto9/TQ7f6YAKuBT4VpxgCsqraDlNY -wt0/grUK7nicZEo3ErHTZNoqQ3+6MxV/QaahgLTCSRPmiBR5oLpkBEBiBiH+LpwnMbgULXK6xijt -FYArWkRelOWmAkYTxNwKhM0EFPESXUhvj849RDg/GCpIe8Jb7OmIiKBAJdX1rp5YAAlDpzKTOSkr -yRH7pJRG01R8r/yxzN+0b2d5mj5hVzEw67mJekSxigAiKStPSl2vdjtAExEOoLrR6R8Q/pgBnMm4 -tEt899S0iSQ2b7it9KVp/wA8ByX2BZ4cdnMezYzsI6Qg5R4XErleKu7UtKFSlR6gpT0x1bMPk0AF -2ma5AraILtgAejyqQHWmuOUs6tXGf+0OMy/lmKek6YPEyVMXO77q64zKlelIraaUpTHU1roWl3vy -RQHbHbZ9XL1oZVwCm6inDoJke24ckRWkYbRDTnzqNa86jirTcojFgRPeOTZgqo7XSJZI7RpzpVJS -71+HFkBR8lH742JkRdR8MApfnXn008sZF+0BnDKsW7jsuyjldRRJVNZVsmAEKQ99C6fFTz0wGV5t -lJbtYzB/Z8lKxuX36pCqCzYEiV+QaUu67dCrX0xecqdk8S0ewqEXNryjGJSInK5GRJb1eeqdKd/P -/LFGzX+7ck9SnSmDUg2F1qVhWrq9OvMCGzv/AFxfMpdsWUVAVhFwNo3SakRk2RtSSSoOpDTn0cu/ -xYC7uMxPISPKSQzICEKhamSFhHb1adWlOWvlijdrbeazXkyRmcszzGShU0hLYK1QV7br+fwlTXGI -NO1DM0FNSycO5aqRMo6JTg3CO5tbnOhCNa89MDgzpPSkYWX2zlrGxqxC2EUQ4YCGpaqFWvO3u5+m -Ahw6cauaRKImg3JISSIXIiVwDfUqHWvRWpVwSz5KrLxJi9Bom6cEmk5QTtqu5NOnPW2taAkJfXqr -gN9jbOjbQz81yXXFDg+GEwVSE61oW4X/AD0prgY72U4z3bb7Urt2rl1EI0HU7K00+L1wEN6s4ILn -bnfWuuG4yIh+tNKaYYVFGwRTO4h8Zjdb+mEGI2EQmA+HovuLnh1u3cOd3hkbhRSJQ7flp8WAmZcj -XktNNYuLZqPZBc+hvYJbpU56aVrTWnLHRvYR2Px9PZ55ohJh0+cy224aqGKTBAgG60wqVx9/y4uP -7PuScsw+So6WGNXQlm1sobpw2LfVK21MNdem6teQ4uj2cLLpyDZ2ALy0XGKSjm463KOnJWpiH5UH -AO5XynFxMPc0ZsY1x7RkH5N0UbRIRIgDWvcGlBxFhk28fmjJrRg/BdbL0Au5etRutVSLpvp611rg -J+/yJRAkig6e5fb5d2J5cQ0SbOz16OfVUr/Fz5YrQR/7vvc0sZDM5/vZGwiDbL6qP3pN1g5IV+E9 -DtHXAWPJiR25IqJIL5eeuXz1q5QO3gVfx6+nriN2eos5jMTB9Hnt5uj2cgslxN1r65wqNSv0tPp8 -6d2Bzh1IQTKamYTLfARLCJbR00xerW2rLFosSI87CprStfmw48FpFu5WJeLOXsfHtm0Pld41C0UH -CydK1Cp0+MrvFgG27zJ8S4YxCiK7iHlIBzahfVUo18Neszr/AA9a9xYsuRs8ZZylkuHh84ZffNpm -jWii+raq9VKVKtp1UGlaFWo0pz1wEcNXjYcxSy8UyZOsrw4wr6OI7ReAuH3255/TpxSs3yfbBD0h -o9nmJoTdOKR2qptrumtSrSla611rSmlPLu7sBvbt8+cuECEHRFdtoLtJVJUFQt8KiZlUe/Dhk8JI -k3ca+TbkVrls5ihMufxBtjz7sDAUWlgJyTYyUEi2gUYAok5C3ysOtL8SmTpSNbk2QBD3hEiYuWyo -GzuHWzpErqcu/AOgUeobVsoiDS4x2leDcJA5G3S2tKcrsDpNNiWTJaPmcyNYlEhuamKytyHVqGoK -V5h64hw75mu7dMXbbhk0FxUPZfqggudvIk7g78DO2BqtLZXVzAxlTFZpc4bKjJFYI0GtaCsnb318 -sBRcuZuyeLtqnm0GsTNOUuIbzsUAqtXmhW9QV1prXTGzwTN49iUH0bmaONS4C3VDMe8uXh8q4xMH -2RZmCVhs2g+y88FJO102MTYuSMdR7vXSvfpg/lLsxybGuI6Wgu1ddMVCTWXbk5EgeCP3gn1eHywG -pTWR1loCWDMkws9QfhYItEbditfQ/FX9cRf2Ymfs/Ji7NS/cbPlUwuAQ6KaacqcrsaQ9WJONFZsC -bkSEbB+EhqXi/LzxWcnt1ISdXaKAvsu7VErrdsjqVddPxYAFNsXmWc1ul2xmqzfiSiW4ssZpKkVd -wk9K+HTyxBBFqMZ7PXW3ESElHKQuQsXGni27+odMXDPEgm5BWJFyug4X+ztl0ekxPzK7T4fPHKXb -xmDPmUJ5rCN8yLv492gL1AXaIkZABeKulfiwHQ0Y6byTIXaDMxjR6W7rgw9+VOVito8tMOpN9twL -tzY2WuEgbE5cIKpH3ahQi58vLHJeR+2iciZVV7NxsdmGHdl9sakiQFb8JUrTkOnljoLJXadl/MWw -5aPzTdbu2k248hNmVvLVMw5j+VeWA0GScPlNpBj9pkF+o9t+BJLjTxaUUr4qUx6VYltJLoLL7KA2 -++jRMf5S0p1054CK5wYtJAWhTDFzIF7x0kmsiaRacq2VMxtKnfpgjl99EzqROWhoOxUIrz4MhK6n -rQa1pWuAlMkxFLcFHY2BJQGye8kQ+pJ0+WvlgrCPm8fxUlIPNtqQ9a6jkiAdPz+L1xmHaB2jR8I0 -XGC2J2SESFBg2MhVZlXoutrS4hrWnOmKKGX8zdoOWkpbP8qom1IRFqzvIWbnq5ipUepuf5jgNjnu -37s1i+JTGVXeqIEQ/ZESMVSpbyvp0/FjIe2Dt2azbjhMshIpkfu26qjbbAXHwa106ufr04Mfui3h -FUigoEPbEb769yF75AKjz5U1Byly9dcYZnXPk1JNF3cgzjmUW9ebiDFozAAfLInaRLalcn3c+WA2 -f9kKHY/uo6za+eIO5JdUkxBNYjVEaFpW6lOd1fTF67UO17KOR/7NlpV0TxVDcBq2bGSpfz7nhpXH -Ks3mqcyFmh0hlSSdRryQa/2ogQBtidedBCytR6acqYpntK5w6cyRunssSorA/JzcSRfNpXkWA1yT -z52idqvthpDOYHLkftCmux4xJInNLuVOqvj/AJeWMsk4V4KpLiEiusIiiLn70COnKo0UT5VpTFp2 -ZCZOJzFLZk9pLO91RfeZkqDFEen3tQ5ipWlOXLBaCy/ltSVdS2zl1eLuFFmKjxaMNIg5VqI2V6/M -q1wGWu3jxoyViRld9mv7xVIQtuOnffSvPliG0UIkiTIAVRFJQhSUOwRGo+Kn5Y3HOGR4UocY8Tn4 -VwLVNZq1ethkGdlSOu6LhMq1oPfrdTFNmMhtUuMXbSUc7GPFInIRRmuNhANd7W3/AP5wFbh414Nz -xSNB2KAisYCAmJXDoI1rT8+dPLD6pNU4f7YCiaaaBWJJ3g2XVqXhPX4x89Pw4GMotQnrxEXOwLIr -lTLUkr/DQLw1pTXXTDTtq3QBVcVkLSG5AbyIyKvLly0rpgFhMELjdFEy6RtVvK4Sp3fQvpriZmCU -ZKSRC2hWsOqKRJqVTvExr53p16afp04rpiJK2+IvDh9JqSgJbBm5WXK1JBMCI1dO/XAJSTUduEEU -kffEIpjthbd389Mat2FZRy7mTtaistzbAHLUUFBXJot0kdOoiMq+n4MGOx/9n/MmavZUtOouo6De -qkiJDaS6Zc9CINaVGmOlezzJbHsnyk6ThIcHLrj9gVbxVUckQ0ChVrXTb+o4CzTcKtINI6ICSuj3 -MiLlxsrEJi2R60xCvfpcI0xW+0KPWfNYxy9Uatvt6s0u4W16EkBtBHT0rUsW2RB86kpMmtlVl004 -9sSev2bUdVO/TFGmG6a6XCRLw5ZrJFwBLisQBHx6ZarK93jvwAyCasZDK8i3m2a6DMUlJybjhvHd -MyIkqDr/AA7benFYzXKFsZNicxN4tpLOf7TQnWhiRJNkAqYifxeVnpgwjF5oLMDObjX+1JPZEhbe -+/v0S15UQ/mK27FSexKcsD7NCiKjaJzvIkJ8Mjc5jGrfrM9NenWzq0wBNlNZmkHpOXbM085ZgXT9 -pxdg8OvFJalRdPX0pT/dh/8AeaDcxr6LYsJVCBzfOpezJMj2towEAU7uoLSHzxGSdSGa3sYxkIqR -TzJmMTRh5RsskkaESjpStdLuV/xD54hqrM4admFN6HdumSow8dBJgXDJAQCBuhGlLjL5vTAOx7VH -MGeJoUZVf2lJZiQQYKuLwGTatOag1qPir04NS6mR835kmXzp08iF2j0mSjZJZK0STEedPpzwOzA3 -9l5ShYqAePZBjDvCSy/NsrRVKQMap0Rt+USLmVfhw8lm7KGXboOeyvVWdZ125VROy1RzpSplzrr5 -0wF93G+0kmobFoR3CbMkVkFROvKpp2lbSuFw6nCSqQsTi3LghJvxSMqqBl3aCompfzxYYyBFKMSE -pV8wFAyWbf2qKto+mt3ViUrEpqPVXKDlBRwukIqmQNy3dPOtMBXQUWdyYx6iMoTdNcidCLlurtad -xDTb18WDBIqOyXeoIqJoubhtIEVUFdS+MaDStP8APHm+XxXMehiKyZWmqLNvfb3+VMTEoFu0t3Gy -GyXSYjGjYJU8+kfPAYfmPs/GNz7wmXXMim4ciT1WMJmBtbrdOYkXxelMBIpx2hRMgxhk8kwD9R28 -EbhZ2mkHmRj8AjjXs4ZNg52P2FzQTkBVuYP2TZUV0NC8NCGndTASVynnZKPXaKdp0qmiqI/afZR8 -SOheEdBwFszB2qZTyo9Z5Zm8yRzKSFr78bCsEre7kX+WKjm3t4yfHxSrmLmAeqJEO+QtiP6Uspd3 -fXBPLnZTktSMtfM46aklPeOZNbeJUir8+vg/lxXnH7PvZm5kN9BHhritJDiXCSRF52lWuAfjO1SN -mYwVCB1tkKYkkQCJFryBKnmF5eIsYZ+0tLN5bPqUPlt+g76dl4qif8YyoO3rXwpjrpTSuBna7leW -7O84Euxm+LR3SJqSLm5XaEdLSrrqWg8sVbs5j3WYsykXsEJFmQkifRtpCZgVA+l2tfPAezQKKb2O -TEweikgmjJs4zwCYEVKdfdzuwhxl2ej99ZaBXbLXEInxNpoBQdyvPypVMhxenr5EjSj3MlHQTNyS -aJtmACRiksJDXSvdVShtx10+fEyQRZzcqzhG0VIuyJBJy+VcrEkIk36V6VofxVSEBwGTTcMoi0Xe -iKI7a4p1ETIlSM6X86/Fbpprg12aZyzNBSDptDTxsE3KRe9WvMUir00KmhUsrz01xq+aHzFskWX3 -LZi9JOHGJJVkYkZGoO4ianyaENA5/NjHcrxu7mhncBshUMbEBDirQp86ddTNO5Pn6YDcOzRNjDO3 -xJooKOhJJRVq/WudLkI0uVZOaeIty/UK40Z7m4W0Yk+bLG/uVJuq+FsIgqVDsseofBS6um5TGTSq -y0M9YxLZm1i/aCqb8mbtYBSfXnrvtFf/AA31HpxufZ+xh42Q9koLA/lhako8NZEQkVUT53KDXk4p -+LqwFbeymfvaboU8sAnHtvdpNifikbYrdaG1cW9YFr4SxzvnXL+ZI126zM59lcUkqIvEiMLxSUHT -3iPOlda95jXG+dsuSZ7NbJ0ojmpjCw6g/wDzJhHLkPcCievu1PzxzhP5fy+0Vaw0PmF1IzYobNjo -w4MdeeqKlOXKvngKjsuLCT3gUtEVCK/15aa151LBXK+XXkzvu7DQh2FvtN+IXi2AuVK1p54smX+y -uWdsvaksDpNmLVJ+SrSxURaVKmpfnTGjZHy3FyjKMF2TLbRVOOkWTJYox+Il1N11R6dz9cBJ7H8k -wPBCmq2ay3GkqwIhWVbLktdUm6hBdb1jUNNaYub/ACyyWn2LsmLpszIVHLKPk2YEgnICFRVQUUrT -uUpdgnGR/wDZls3x0pIKEhGOR6UHzZUC0buK1pp8NupYNSTpxKNVY1u8jicSlrd7Ey5+8QkAG4Kh -9K24DJ89rReRTdT2yvBOJJLiIl02WNdiuPKizJVK7QfpjOZicarsmMfHsGMOiW6/XVYLbAuWVRvJ -Dc5jUrqlTmOLX2kSxLzSUAhJOsqx7ISJzHSLO9ik+AeoAOtNOuleWMHcCK6S8goANk1CUJqkmBEF -1S1qI1+HT64B+VdKIGTRia7SPJIREbLCco18BKUp01LT6YgmoS6pKEdpfD8o6fTyx5K5D3lgESvu -QBQLum3S4a+tMWfsvyS+z1mVrl2JWQTeK9RG4uEREe8dfWtMBEyFk+Yz1m1jleEBEnTu4h3j0AQp -3kX5Y7M7GuyNnkuRy8+JmnGzqrddJ6isjRcOXdtH8PPqw32UdmMHkn2dmRFgg7e0kybqul7gNsld -bSgU/mxrPGf3xBis6TUiVd9QVEamSiNR1tCnf+WAQxW/shjKe4lnTZUhVVZdA866EWnxF4cNyDNw -LSTJP3CzR1xIiytM1xt8VRr8WI6ScW9YrwSPu0ZZInbBJFEkLRoVKkN3z3Yr2Zc3IoO2eZLHQoxZ -cPOt2R3Ls9O5Q6U5kl64AxlREVz2FJUHMogahDcZCSorc7lA8unAfPDyJj5VjDMXhsnUoJQ7VBEA -2hAupVxX8tNNcVHtrzY6guz95M5dkoQnQvCYEq5RsV2lPB1U7ra89ccoXN1pOVLMWZJFzKMmYptV -WixOTcuCLUhEi15UH5eWA6NzpmrIeW8iTSbSadFwy4xOX0kD3HKAB94oPnpU7tcNuM4QMzl11nRs -8h2QzTwYdVqssQWsg++UEKF0GQ0xzetGx7kJCbgeEg04ZqgtsOn9CdOVa0pzRqNddfit+HDrLJst -JBKyTZG5jCtUnr5eVAhItwh6qJ+Ihrry+mA6BdosSyFO5oGeapov0k2WXWvEikUQ1qVaVWurS+6u -3gV2VZ6iZSS9sNm7V3N5SjnKEYigjti+bWaGssZa2mXixgjd4pPCvJLM3eYZZBdN65cXltC2DkQE -Ff8ABg2bNnKRM7MNHy45sdr7iELCI3ICy8St9Q5VGg4DpKEgY8v3Zy+pKoKwbRgvmtd8hcKSCxDX -poQlTpuK+mtPhxKyoz7O3eWmEp2hMWimY5BMnLpRzXQ1KVMqAX5VARxyzJZmzMmq8KWmHTIphJsg -5jhbVTJWP0upbpTQE6DTw0w52lTMzmzNKkzlaKzA3hTRSSZhUSPoAaDrr+dK4D9ApCWJiyLaYOnq -hdSQJswC4fUNS0rpiuhmBRR0cWg/fETAvtS/sfrQ6dbS1rS6ldfhwFcIpuZB45TCHdsUepVqUqtc -JULkaVdeQ/TFdZTDOJZcSQLisuvw7V40myJQdda2qApStMBdWWbCd/aX1jISVJFsKkOZJPPTQh1I -f8sSm8wzT+1oItd63qFQFmyo6d9eQldSlcU521kBSHL/ALKn2zh2qLm1OVAkHOo8ttTTo/TuwWXW -nRcKtiRznwrUwHifcmq0ER500s1MK4C6qyDdy0QIuolREhtcpEJF5UG4hLX9MPgoo7aAgu2Xeoiu -PSVwqj9dNNP64zSkhHrv15Jw+uV2hG+RhCSQIOeg1qNelX8sfcvyDcW68g2coKOLiEUE3izZygNC -58lCOh0p/wAsBa8xqFDuFU09gXRKiQkW7db+KohWluH1U1CbquU1uGLxEk2kiEVfOpDcHf8ATDTR -wT5uKC7ldRZyIqCScqBDbT01D/OmDG2m7DYvejcXSSeyuIlTurXppgMK/a6h2st2coTblF9vMVd5 -i6TciokQmPvEz7rK4yPs3i4Fj2eJTLk5h64epL77NtcIbyJXgOunSW2JV1xrn7UBFIJZdyTIOV4l -u5kUydOiAUklUSOy7l08tNe7GV51ecXxOWxeTD9wgWyDZojsAubfpqXLuubbutcBSMvqAUw2YkjH -QqTtW0V1A3XHWV4FStNddCT0/wAWLRHqSzTtIdZiXigcyyAlIuuPeCkKoh/eAER18VbqaYp+Wk48 -p15KO3nCNU1U0RQ+/dJDXWoKBbpQbTAf9WFuH3t2VkXyhxyDwRUckq5c3kSwD7wenT7wufy4A/Hv -Nr2jHtph1JRMgJNBZsg9+KtfeJjQq/CKop0rprgJ2SIqFnJdBjxzZ8kNzVdMBJdJahiFp6102yuO -hYH5aWJpxX9pOkEbVEySaBcrbQrxKlfl3KBrUbcK7N3gx+ZUnLnikyXVUTMiuFBfUhrVI606uf5+ -K3AbBmBYWPaEu8QRdRqkA63HTUmwqoRBVAQuOl1dxuoVPLw4tv7OrhTM2aprMzaN2ExXJRy2HxNt -OSazMu+o28iHFEzFmx9GlxUWunuHus2b9yFoKpU6jYOhrrU1OfKuNA/Z49m5bhH0zvPmEeO0JKrg -QlFLEOpAQ+betcBG/aVzYo2cEhGrINBXSJQ3hdTCVAdKVTrSn8al+KB2aN27Ro6Fszjm0kyQF7w0 -rdYuSB1WBIFCGmlwVHp88Fs+PNJ/jXzYHca7XEn8ZZ0lvXgLhkPwhWtBxZM0Q7okozJqEkg2cEIk -8GcRvJJ23SHh6XjbbQxoA4CoquHDRw1YyEIu2br7sm6dZfWuAopyNtR56fdEY91MXfs4bt5dornS -YRa5pRjx9mLk3bbTwUaF9ncc/FpjM4SQKC35RMJSHlFXigsSaBxjYmv/AIlqVK8h07+WNpaM3CLS -MaOVvaaKTUl20xCmIE8ji+83EqfEkWAkPXQiAuZtZeWRXL2TLNlLUpBsdVSq2W1u59NRxTMwZsWY -wjopQ/a0optxzmOeo8LIC48Ld6Ned5YBSeYo+Seus3yxxeZYWHEYU3LTdSeKjX+7uqhd7whrXTux -nmaFn0obGSlJv94czSCXALtnNwPmKqJe75fXAO9pco6bQ6GW1ZibXkl7VJ2MmkRJVs6HWgkB+hCX -PFFNwomqJCz2G4qipwal2wqQ9J6curu54Ovvbk3uqLzwSUo7AnEik7MRXSNDptur9K8vXEFJvJKR -6DGQW2BTtKOZufEqKneSf4fPvwEGQInyqTZMzQUVX22bMjHYSAy+f0u79cdr9kmXW/Zdk9dRys64 -qNeJrvBRWFcVxMRpdTlcKfP/AG45O7KomPlM4MfaUachDpltyZKARglr4CrQK07yx2Tm6Sh04JnP -Ciu7y3LsxinhRAEa/LUU+XoPPAEc2yEXCCUI9mzGPn19yMdJ9RtnNSvp/h53UxVp7tBl2GRoLNAt -XUhLtnCLBddvaSbsqlQVRVTpXWnTqeAvahFtXfZ6/aSje5jlkmhQuYm95GkFo6b4UrQqVDzLAU8v -i+zxNRb7NQNnDtdKYi5ODAuFuT6bFh50Cta1wF0zlmRGLOai5J++lYtyz41g4ikfexi1fhM6V6aa -088AJWJmJdWMc5sc8W4ncuiizmIY6bguEhqr9oSpXQtRIe6pYlzEg0lX2Xczt1nuUnk4t7HeOxQB -WOkkqULmr8I6lTliM4y2xyFlxBo5eKQE5HyhLDNtDM2LYFz01KhVrZ0aUtwEbMCeYMwZMSzt/Zyj -p/lhRo8S4Yb1VrqAIop61tP1+XGJw8C+Ty+8mYsGLB5lUuLfSah7rpUlx29nQ7R93p/vxsTLNGYB -ZDFwkOcopkiT2ym2RgQKokXLdCtPFXvrjPc1SUPlXtNGJ7Tma7tEnjmYdNmH3BGuAcONRpzOmoc+ -eAo80WRYn2EtEwq889YoEU05WWIW6rg/uu6lcMrs4mUCFfPsyS87NS6SqktHNek0ACmoCVSrSnLT -E6cdZjdrA9zB7Ljsv5hoMsuzRAU/ctjqFltKUK/o8OEPVE5ZaThcrZGWi/az8ZFs+vO9CPoNaV59 -9vO6tcBEn2843gIzMDSUQbp5qQUYKRsZpv7KZU0Egp6+eD2SW2didx9MloxeXRzKkWXyoJ9R7I+9 -VurTp566liI3DKGVo6dpAS9ZPNNJFJpCl4bUbfen+HW7TAIo9mWT3j9bMjptKRcmLCLiU1qmaomV -yxCdNPMsBbcrzUSxkJ+GY5Y9pSU1Gex2JktxXDOKdBERlbSgkPUOnhwf7LU89SOTmwU7Vo/L6DEi -Zt2pElStQTrpd1VpWutalT9MZ+oxdJuJvNGR33sGHi3gNEkH61AWBRYbS6dK92terHyQkMt5Wl30 -MyZ5WnWyS1yb18moZqaiNS0rSult12mA7DNN5HtEsuj7bXTQuWXFaHFddtqWoEnURqJjSvpgPmBm -3lJVBd2jCKN0iTUGRe5eVDfGnKqahCGlOde/BBW5o99qWIOUzuJs5aT1y7ET9aENOnCTmJCLV4Rd -nMP5J2kVzZvMImk5C74btOfPAAHZQqcg8kpuEgItq2XHhWqm8kSpAFK7iJ/T5aYkwjNP2Oq9YzDX -7SqNkxHTxJLoalcKawKFTXniwcGsukqJHnMmLa29JZZFVdidPi7/AE7sfFW71eTWkicSiZCIi0X9 -ipCguJDySOgnW7TzrgIgN5InfsYQmBcOUBXcoNJ5IgXG7QySKpa3euJKQvnzhBym5zEMfH7ggus2 -SeGloOm2ppqZDy8WAcemom3XjVNgnjkRJditCGkqzCpFqaZUKtCHl3a4NOPZ6IthjVoQniZdbwQc -IJK6d4LDbWndgHeMkkDVc8HuLLq9LUoEiSc6fxk6iOtC/PFhb8PxYkojHDtDcuqpFKpXF/NSnI8V -ZwnEuXvu20cm1SH+6+0lkhJXvGqNahp+uuJDJ0QsiUiXi/FOfdgqUr1JFT/jCYUr+umAxDtozEjJ -dpEnIJv0EGrRrwy4E2NURSIdFNsa06NB6q/NjK2TrMk+9VHem3LwldlquTmwCWt5VrrW2nuNylLc -aD2sSikhnV9BSG+yJVUbVeJCwnVtEwUqpTuDp0trTnjNcryTFtmiRTlDAWLv7ETp2ZEbYiL77p77 -aYATGL1bTSbdoxRXBYT3UB0I7CLQx3S7q0s8WJSojHZ0S99FIikKayRJt6rp6UGhpjSg0rW6vdcO -JKrVEnHtBi/aqMRdWr3e6bX07+impbZ0AefrgZm4kyV4hoZizVMVG6QtiSFIi6zS1r5BrywHm6hI -OF10DkXqKqRCBCj0Kj3q9/OlKV54GR6zy+1s5feLcEEzLxVIaUrWtPX1xYnyLcR4lRm63thJFwo5 -eD0u1OtRYaU52EFD8sV+0mL0hQcmg4Iegkwt7++mtfywGsnAySm++Js6fouUFU3gSqxIARgiPQI1 -57g/Cf642Psqdey4p+io8azDN2km2YvnVojIDZpwLsK/dq0+b4sc3qziLmPXkBB9LS3UJryPS3sF -uFKGNNdKKhp3fFTGrZKzV+7qpMc5Ngko2WYIO3XAWkC6VdKcYGuhbwVr5DgLRIMY9lmhq8LLAPYt -sRsF2r1baeR5kI6CN3eI1rqGBWcPbhOUsly0q19sTqqbB+3mg90SVOTVwmqPiIvPFuSnC9poCJx2 -bCbMFVHL7wjJxVSGlNSry309e/GaZ9nEcr8Y2Xee2m67UWiDCaO41WKhVNFVJYdbKpVPTATGUG6T -zQ1IY11khNgqJbROQOOKVQHw6a9AKjQufnhrMEx7GhUF20bw00/JWRy2/wAvgRilcVjhsaXy+fpi -hOMzoRPDskMxyUS8oahzKVnFIKukepuv1acjrQaVwGLNUtLO5HNrlm+bzSio+y3EZ0oIOP4nR+Mf -TAJza42XCEWo8ayLWLX3OPadLwt3romXrtlX9K4qzh06dqqvnyxvXSokQqEt1pFTuxMbuikm6SCs -Ugu8VdKkR+A1yW5ac+XIu6uGFkXDSNFpwaI75itvuQtVuDpMB58g19cARkBg+I/styuqzMUCQ40N -pVU6iV4606dKV88QpBRQjHccm5ERtDr3REaF4Rr8uLY3kvZKAIRbUBZlHJXk6bA5tCp13FKlSvRT -ny88V0eFbTcj7Jcg/R3VU25KI2gqlXuKnp9MBpX7N6hITGZE05hdg4SYC9bG2WEhVNEqHQSGvjHl -jeLJRGCmBWkv3cTlzTmI542MQbLq23KN6VpzDWzz/FjmHsceN4ftCh5CQeMWylyZMzco7qCtxUEk -ldK9PKv+rG/59mE4KTfISBuk0Y+fQdoZbegRpPEVAqFKNFadPVUi6a4A1O5kWzQ4gnbZFTKo5mYq -tnKoo8S1eOiDRMDMdR0qXLXxYEQgyWX5PIexsZRlnoqxM1J7KRtnJgNSpSpV+OpiPixAcC4HImZk -xZh+7sPmJJwOXn5iLpijS2qg6ULw693PDqrhQssZ1yc0bIv2MS+SlIyJkUSSdEjUqKKbR879K8qY -CBmuPfQ3ZnP5eeNnXtKDmE13oogarGZGp3cw/g9+Ck6pmb94HMIKzXKMHmyEJZIXv25qurbzAj51 -R0HFhZTyjTtA/d/Ksr7DFeCJ4ULNh7pVxXTp3C5Dp81LsDoThUHGSJIkTym8FVdF+T1EVY5yag21 -06rdCLuPXAAIx06j8ykxUikMoxecWqTYXzY0nMe5cIh7ylacxompSleunVgZnvslmJvMTV3lvY3H -48RIvCMV0A2PuxbnX1592Dcgj+6mZZFynMBDx6UiJTCorA8jBEuVABCuhBXnz0xVu2ftOfZOWLL+ -T34Nm7Yk1EnEc/FdmSRcxGg94FzU19MBTc0Ry6E4zRztFryjuj1N4s4aHurizMddqiQdA3V/33YI -TD6ed5Ris7DmAIKJeoFltmyaGRuSYpczI/P4fLniY3F1m+elY3LL+Oyr7QgN6VQTWE2zkadd4HWu -t3Pn8pYocetAxuWpbjo2RZZiUJsUOQnupII/xSqfdrXAWTPcejKGxzBknJgQuW5BgMcz9yO+5JMr -lnFNPBWlKeIsR0pDJLGZnmuQ8vTWYTfQ4NIp4sFSUFyQ6qr2fMOvl54EKzTeSaRkNJP5TM4x5KIx -0S0MgSED51PcpS4i9enGqZSedpTRxlYWmQzjW8ag5btn1gG5JqQ8ysu8XPxYCi5Fjch/vBIscySr -rMyyrBMWrBgiQC5kFBtoHT4qhr48VrMyT5F2gxcFGRq7JCjdVAUTcHQhIvvCpStKHpWlK08tMaOB -M8s/u9JZL9lu57LMiqnJuXrbbN46cFYnTbrWlS8XfTw4nNx7JYWUmG3bQo6d5xVkFF3SzQ67ZCYj -UdLa6cudP0wGuKqChNryintHeepE3bLrZeAmznpuHWzTQtKaYVKy0a0AicsIddRTbI2ZRSqTlnr0 -kQaF5fTEFWPdOXqUbBAaiLYSKTim2aritoNdCT1PlWpYfaFMCDWQcts4D0kjFPhnkVdoq/w1dD0L -n5lgGnsxEoSrWEYv4FZqVpFKbzpK4qcqAp161/zxKCeatlXke0Wjk49ICJdqnNmW6NedNute7XHy -MTzZDuV2BLZi42UAVXDNyi3XQVqPItsufV8VuPswM9Ht9tocw/jUVRJUnOXgI2fT4k+nw09MBLSz -ADY2uYCRevUySFNur7bAjZiWtLVKWeHWmFqyE4g7SiSZzacw7SJwaCazdVBcPirrZyLTuwFcOhey -ass59iEi2ajtG9y2qCDnUtLS6PFr6YFm4btAdSC+W8tMnDkdvg1GbgVxO7TcR1HX/LAXBxCyD7g4 -ti2zEUS2IiXFQG97Y6lr0UIK6j64nyrgSMpbhnwuCLgmrrg0bS/CtSo8qcsZzIJsWyQxZOYTe3QT -VmE+ItEhLWgLcvp59+CMgMWM3aoGT2BIKpkK5IuCZvLh8VSoNl1f88BVO2XsXzBMoFmiFZqLTF32 -xi/NIAIBLWlUraUvprz54wfMEwjJH7Pkoq2aL7O5VTtARVDpT0HT079fFjrmKeRqkZIoKMMrSQq3 -DwDdsqqrf57dbeY6YfZQMfKSaCacCxU4JL3qrSBEDEq+FMtwacsBwsKaiZrjvdKfjHwj4vT9MWmM -i56ZS4ZSBkZLc8KiIGZIFUtaFprbppjrbL/ZjklebSevssQ6klvq8UKiwqhcXKmqSetpaeXdjbMu -Q8XCMhj4Zgg0appDdw1oiRUHTupgPzwkMn5mUh0F2mT3TZ0kuqJqkiRXCXLnUq1w7lTshzxmRVqK -cDIpt1PdkuQFakXrrXH6ImLMVeqwiIbSut8PrzwFdvBbKkV/DJ/GIgj1D5FgPz87QMt5iydmBLLu -YpUEiFcXAfZvddIUDd5/y24C5XfQLFwTl88kRkGzW6OVQ6gQdAXKlde9M8dS/tYKZTd5KJOWcoFM -L/3EhBIlyOncPRztr544/epuG32ZREB96XWPVu6eh/FpgLFIZ0mJaH9kqbDZvvquQBqsSQiR2XjX -8FbfDiLOzDPi0PZvFLs21nDIPwuFKlQpen81vlTnit/BcOJyRLPnou11kFFE+ohWP4RHkPVy+mmA -eScLGyXUbImInci/dKdfiOlac64fVlH14rtlgYNytUBJsZWoaDZXQfIipiKrIJ7Qi0RNp0+8AViI -FSEumtRr6YigoRKkSl5EoJdXxXYAjKyTqQcM0yWNZuxSFBsSiIgQpUK6mttKeuHQFaQcWoXuXjlV -NMWwgV6pl3cyrXTA5kLh2rsJ76rhQSTSAQvIit8OLXlqHFSMeO5JHaTZLtnLqTJYd1BIh+6Ee+4v -pgBSRNWaRfaXrZbhyFdLZ92q4A6e50+WnmXdh+6UzA3eSSbACFV0ncCIW9Zcuke7EVuIyQLlGooE -4SarkrvWiWzQx0PWvTfzwYbtYeNy+845hNoShPGiiSpBYLZvaNaqV0+f4cAFJ83hHsY+i0VxfNhL -juJREhJW7mOleVeWOqnqjfMTeTbNv7UazWVknqCEi5sEVUb/AO6lXuqNfixyS4TubpOxeAv94RId -RbAVLTXn82N67GnEpmjskKClIr2s3glSdteJbFdwVeSopqVppWlO/TAFuzpixksuvGwt32cBzJl0 -Xj5LwPkHCI2VoB/xPB4a9+CeXJaNzJnDJ6c68DNyhZbUbjwVyD9odCGhjUKF1HSnljzSNFePkSyL -MLuYPLLpB7HKkCoyKDUwAlE076aKDXXSuuD8bkWPXjJqZerLryEbKJSscbREGb4gr4xPXTTv5lTA -AuzprOSErCy04f72Novi4VzDPURSfM0SrcJqXa1ryHywWzbnDsvjcqHE+23T+BQfisvlh3aJilQu -Yp6jdaJfDriozXadNPms6xhMsLy0jv8AEnJulhB4KNO/bpTn+WmMwtZ5kz3HfY805jTlECHbfvwS -dEdvIQOlfDb83iwE/tDksvvpqRyyhnBCRy6RcawkWsbU1xVLvTUoNaXUEddSrgBk94WW2SuYIRtx -Lxirw66i8UK7HaLXqLXwF3Yiw7GacxMq+bMFxdQlopq8eKSrEKd47d1x0t86YPum76GyvGJMniDR -bMKW88eJzBKtnwU1rtLBr7uuAhw8HMSVkWTORaSUy+J22YiwERVRMfeKJqU500+X0xbofsb7VM1p -IZfs4CFZJFsG/tAUgvpqNdOZFrjeuwfIMDl/KrHOM62XXkE0AdM1yuV2hMeQI0r+HlSmLa4zMzUA -UYmEdPZK3ebwSYWigZeFRwfhDXvrrgAHZ12R9n/ZvDoZqYm6UkBSFMnjm0jVKvIhANOm4saek3ap -gMo5A2zzhR+MbmKVvhDlypjOWWallXq6Aj+9WZWyopvHg+7i4oq940KvLpp3+uCj3MzqfdIM8uma -yKCoDIyyaIiTmtC+5R9da95YCH2gdmMHnhJdyoi1bSnGEXEpo3CIiWtDPTQqVqGONu1ns+Xgc8vm -yKEo9ZrVos0Xq1JSppV5Urrr60L9NMfoI3XEW679K9Nsklc+e7Ni65o/h+LliUlFNn1KvFxRMV9F -ERJtzBOtOQ9351/XAc4KvGKEUggo/hFuLfbiqS0Csg6Yifg8OvL6YNRieX3b5JoLbJNW6DYhVETc -bTvSvK2tR92VOeuJTdOYFIZltmSY2XLoRjnnEt3JFdztPVPX9a4RBNZxoy9ksUZsX3FEtJoOY1uY -9WupI9HdXANmOWXbRV65hMrOWaH3V0qsKrYu4j8OttPpgg3j30Tlwtixyo7K0XyeYbiEO8KKVqGv -08OJUhHvHMm2T490KLZIrn37tpCCRDzqKnT50xOVUkrFXYo2qCltpAnAgKDz0KuvxeWArIfvI0ky -hkwP2SkhvOmaOYQVVV1+JO4KD/XA5xIE+SdTMstnMYOPIeFdFJN+JbFQdK0qFNbqa4LN2cgLImy4 -bjp+JEbN3l4RXQC6nhIK0rbTHnELJC4axK4MXe0qKhOW0CYqCdC1DcGp6FSlMBHis1QLFw+kpLMM -3x0jtJtSIESQc/L0UrprX64ek3m1lr2WutmVRR2vuJMHINSP/wC2d+mnpiTKotUHCssu24Zumqm2 -SQTYNwbKn4daiYkQ1wYgOzdZ26Xkm0ko33xK1s7Z3igRd9lfl/LAAIwVE3DVsL/MUkiySIiSUNFq -5E7uQ7lCrfph2PGWeZt4ko18om/Q2Wy8m8V3UtPhrYnbbjSIrJsShFey5BsDsU7bjURtK6n4/FSn -pixshUTZC3HpTQLpAjIitwETLUe4Yx+w5Bim6ErjVZdN3prrTXBVuoV/u/eD8REf/wCMeNNbiPdg -FvxdHw4C59zdA5Jy/wC1Jl+1ZCRbaQqHbcfpywBN3am3VUc2J9f8Y7REfm/7Ywft77dMt5UinkFl -8wfzSokmBtDEhQP8Rf8AbGNdrfbpmbPCLlWCWOJjQZi2ftVFh96VVR0IK1pz1+mMphGbds3XknzA -1xds1eBBPq97Qg6i86efVgDUw3fOeGls1LBLPljbEKDsyBVdIiurRLXw059RYgoFFCs8vP2SzVB8 -idzbfa601NNFLnrdrTv9cWY0XE2ynXb6eZSyiUSzJAVkS3VxHuRR8wGnd+LA9pl+YUzR7ETZrxb5 -R4V4uzE2LNJYCpXWtaeLngIGc4FjHhwblgEXLHwzlO1YlEEm5BXWqqlKeKpUp5YoppkPUQeLqEv8 -WmNVPJok0SUdrOmzM4xTbXJapjLum6vhEfIba4gTuVXQtBdiwXF4kgD8WyZiSAtFOZWU8Vw93fgK -GbNZOMGQJseyuqSaCvTaWneNfQqYYBMt3bs6h+H5S+uDRpyGW3aqCgfZVCFbach0rh5KDTEOQ4N2 -quszPgk9q4m6nivqXhCtO/ASsqPvZc2lLKIggmJKJgQhdadvlz5fzYdm5BZ8bVGbBASFq2TA23SC -QXU6lPmU078QFfajaQJfxOEEvEICQiNAp5d3Lzw1vfZ9hC9sQkmRJEF9x0Hxa+VPpgLVGuIeHNmU -lCBmGPTSckyTTebZKGVvvlNBrWg0qPIcCfbkwpHrrqTy6lzxJbaU6iVOgjShV5fBgOCniU+7U6iE -hDqIq/DyxaOyrIcpnrMCUXG/ZkSuFd4QXiloPdpy54B3svyXIZ6zGQqAunHkW4+eJtrtrq1rp9cd -rNI2DhMvw4sWz1BNgQpgkiz2isqNtSPnXWldcTsg5Dj8r5cSho+NBNNMbVSEFR3T8y8Xng29TTE/ -cHcNvSF5kQjXl1dXdgMj7bZLL+VJOMkk4tkLiLFRsmXH7S6iRDckfSNaaUKuMbb5+zJn2Pmoly2a -yxKx1zM1HgoOUjEqU8ddKH/LjSO3rs0eyT2MkYiHZKVaIW+8BUkypdeSapVPwVuxhKrMouP2pRaO -FGJV4lJiszK1cy5ElRelaVLTXlgI2SVGse6Z5km4prMRLBXhnzEnmw6VMrhpTlXqHXDsJDqzaM62 -iYqIYrR6nugUf7D5OlSqVoqV5KaeHy6cSwJvCNEkF4Fk/LNDW09yNVA4y4unZO7qLl54Hyb7jp2H -uABbwtqLp8jFWroJUOoe+GuoqcvpgLHOw8lG5fytmhRbKzlvbsk+bPLzVEipW50jTqP5K4v/AGP9 -n/HTbXPso5jkItgqSjX2UwMhkyPlUNs/CI6emKbkLIcfnTOEwSh7EHHvE1imGzYWYiZF/wAM6F5d -w46Imm5Rskll85J6yjbLQbNgEpGXoI/wRGlNoK69ReeAUWYM0T+bXgxsWCbOFMUWJJrDwLEqhS41 -y7jUp5CPSPrhcwxfPY8WSaMiMKo6tLhFhSfTzmveVS+BCmGcsCDU0GkkzNZ6venTLrJaxlGpVKtb -3KnmVvfdj7K5iTkJ1WSQcGhGtEuCKWEOi6v/AIdgn/Er6n5YATIZZnHMOllltEx1rZUVCjG7kgjG -ZXV63a3iWOnyYvWS4OLy62XzNMZgB8T8dk3BaoJICHKiLVH6lit8Uqmm1QUy86KTL3kTlNNb3hF5 -Onp/154cjmLpzmBjISrxjPvodUl5KRdnawjTr/CQTpoJGNPPAXqBRmpucOdlHBobqRDEw5XACaWv -NZTlzMv6YsbOOQvcGUu/MjWqRUFwIiFdKUtpT05YqXtQZlkC3tVeJy+ouHDPCP7U+K7XopXwp17u -7FoZLSJUV32LFjopWgJ1UEiqOlNCKvrXAY81hyeSrqUdsItYRLbBq4gSFquN3QVwjh2dFq0ZE5Jm -xJ4oW2ZDGuAXZiXnrQLraaU0xbmmT5BtFJM1odld1EXCPzQAi8iIKiWnP64aOBzQ2VSubSKjdsO5 -sJzAkV1O4R1T6+/zwFKjLVEmKe9DruFCIVVUXjhAlypypuh82nzYOyDxi2aIRbT92kEdpNZRs5kj -FIhoVKdOnT3+uDUfH5gUk0lXLbMSYgO8ZXo+KvwkPnh1JrKIKkmUVNrtV7uhZsir3/W6lmArLJrF -rSZPiZwim1btF7buXEq/8Ot3hwVBu3eq8W7YRyjpPcTQBY1lXP16qUrdyxKDLc4+ZcIoEi0R/FtJ -GI/Qqa4Iw+VyX2l5s3xOGCv2beciqRB566UHvwEDK+V49y3SUUbAmIqkQkLDaMht0tKhUofd54vk -ZFsY1kLZszaotx8ACFo3fNhYEJOBuMC2/hs8P9cSXCyKDcl3KwIopjcRKdIiP64BsGqI22+7EfDb -04CZwzhlfKrdJXMkq1ZCors3KdQ3/Wvl+uMP7bf2gCi3CENkB+1XcLlauvZeaWpWXJ07j7vXHOeb -fbE7Ie0p2YMnkhxKxGpcW6QiVBCiX8MqeHAbr2n/ALSBOUfZ+Q2N9FW5bij3pMeu2m2P8TX6YwqT -zVOPpvjp+SueJCuoJubrBKgWW0T8QFyx5WLko8yTFyDSUTdNNpmXU8G0CrQkuWnnzw1KyCbF2l7Q -bNXMhIEqLxUjuVLUq6mY91K4CrsmrVeSSduXMdw5Ok0yS6hEurxfgH1wfgWbVs8QbvtlNFzxjUhY -LbpuR6SEbKfw/rg7GZJzZmtJ5+6jNr7PX94e5YBkKfeI869NK4fylBvhmGcpEgBKKLk0YbgAIKnQ -feCfV0/T5sAdaTUe5jF49oaDuWd5fSZXqAIixMfgS+ZSvzYGLTzt8/22MPwUcLhBw3YqBfVy+Qro -e8VfWl3PBplkFu2cbjQAbJyjwmvtXZtOKc+SNKXcta4s2X+zNGQSJzKNuGar7kU6atnJBwL2pa0X -rqOtplTAQ4x9JL5rGZTZxyEwqkT+FZtFr2ot6jY5Tsryv88QTZtWjRr7lrJCwLi4VsSI3yqJnWpp -qUDyArun0xO/cPgpj7cC6cpvpsnD5Y7eDdpiRUUpp8JjbTXEbKgxvtNrLQz/AG5Rd4rtIEHTGSNC -qNe+v3a1KcqeHAVHtgyaOXYJi7UeLu30gkLtjt3ihwneTbq77O//AA4yllBvnaW8j8TVR2l0F1BQ -qUqNMbj2zuCmYVrleJWUXYOTN2wF17so94A+/QEvTv6cUd7CxKcYl7LeLprPYziYzbPoScCWi6P+ -L0wFGjykHYKimG41ZDvKpdNgjbpdZXl5YgKrEoBXABLKdRFf4vp6UwYZOIlSFdIqImMom6FZILOg -kf4iZUwHIRILrwtUIrh+L9aeWA1LsJ7I1s9SyTmQ308vp/emmsKSpfldWmOx4Ls/g8utEm0TDhti -kmIri2b320HlWpeeAH7N7MlOw+AklLxJVAi6Tuu0LTutxp+yRJCShmoXhH4em3lgBMeooQKioYEp -4blASErfXDCTVuglamG2REVpfZ+r/diUyRZiqruHt7YkJeL/ANuFAioNxJublPh6y8/8GAGO2qKa -SqCh9KgkJCoCJCQenf3YBPct5ZdtxbO42LdtytIgUbN7breVdK4siorJ3Fx7obRtG4yL/wDp4fbv -FkwFteBCI9RqGV3h/wDp4DDO1XsxlJSHVGGzCC8agO41jHLkEmzMR8k6J186+WMuyZ2eyUpMR0yM -r7YRkFfZ0mLB+YvGPwkoY16yHHWM7KLR7QXMe2XfqFamkzZHcapV/nClMZTKs5RDMq8s7W/dxw7X -QEkhfibldKpaWLogGlOX4sBcm7f3oxOXZVDjGyWzNZtdoiRFp3UD4VFfrTw4nTEe3jWDh+pmF1E+ -0rWwSNm/KPq/8NOlfAPpbgTCzAk+blMIoZjzImkS0XARCJA3Zh3UJQq0pS6vzFiaxlHrycSJq2az -mbDoXGPCPWOgk6d9KFWmhFT6YAItDrNpZNqUPxpF/cctIreI7uTqQV7v01w7QZAsxKsvaUc9lIdC -59IpgKUZlwa+KidK+NbT/wDOLTl5ipJUk0YSXdA1clbJz3/iXytOVjf0GndriBG5Ti0zGCXhduBa -WrNYfeI3L5xXvXdf/nAU9FwLZBzKISL5CHfq7DP45bMqtfkrXrTRxNeJOXqjPKqjBjIzTfbXVjiO -nszLqPfQ3H/FV+nfi0N2b5jJlEi/QTneFNxKTBJ0MIxvTkCSNO4a6d2IMSxGG9mNIlguUK5dXJC4 -6nU47IruIWLv2x1uwEl0+i4J6rmTMDx09sVSQZyaiP36tf8Aw7VtT17rtMfMy9oDJtKmMtnqDyg5 -KlD9mORE1gGvcSnylXzp5aYmzb5rmJ4q1i6pruI8th5NlaLWP1p7wUa15bn1piBl/KkO+jRdQOSc -vSTAyLbfTRVq5d1pXSqta176VrrpX6YDXwEbLS+XxEeEq2l1D1KeEcO3dFw//wA2Gj3PFYHi/FgE -AJCqQkd13gIrbhH/ACwkFFB3RK8tzq8Y4dVJbduTC78N/ThwfD1B4huL+bANgO4A3I2/zW4UaY+G -z+Yit/7Y9cVnTjztwLRkq5UPpSSJQrvoOAizchGwUUvLyCiCCKAXEfTdbT4afnjj/tY7UMydoO+c -a/OHy+oquixFYOle0R1oWnxVu8OC3aln5x2oybZoPHQ+X2zxsnvj1GkqZa0qQ08X0xT27OUQyuxi -WiKAtXa7695s3hICJB0080y5d+AqPCx8M9SaLsF01kFWZElfc6Z6JAdTTpT4da64am5xRtGcCpwI -iuKqgPy6l1xu1tU+QvTBLMqkXDZoEXMa+iU1CbOQVW63iWgD90pX4fTngc4h2Mw9JVOeais53CJy -7uA0FS50FXTzL8WAcAph89Zk0Dgk3rpJxGLufeurwHnRNWml3f4cWmPa5dTzw6cyFijctwvaKkad -yUhXuQWCtbaUr+WJmR41wnl9BWSilFY1svsNUhf+8inVPCuA1r93XXvxdY/Lco7jJYXz90TpchRz -OzIAO5Kpe6dpV/64Co5HiWbmMllF544laQdJovB2SD2eY/dq8u5Mq4ubjLcWo0XdqMGrIk9uOlTT -/wDDOqc0nQ0p30LzLFyy4izjY9djIPPanBCKD5WwbnzQ/u1/xKD54aOLdNJsmy7YHLdJAWD4iMbX -LI+aS9KfgrgIcOzbpwjxSZRNgL0uAndsyIUFqfcvE9ddNaW1rXFgaLSTmPMpY2q6g2x06SICJX0/ -u7sdfTENoxR4dWPmQXUKNH2TLGJ3AqxMdUXP6a484fC0VSaO1gtu9kzu4HXtGOjdz+etvPAQ8xrM -ZBwqpOgumJF7JlgLq2lbfszmmnrz54ocnDzCfHbiLGJREW0C/LhupdWhe6c0L4aW1HqwYm5R02zG -+bSxtSWSIYN+YnYJBUa1QdemtO7ATOE5INGW5KOd/wCx+xZYSO4L6fcrjX0+uAxjPDhOUzGSD43X -tbqTctm11qr6hWgVfS7D+bszNV4J+LbLy8W1XVTUY/CTOQSGgrf6vp+uEvk55zJ8FLP2UWm9dcI6 -kekPfD1pl6iNbcCc4IqJxQu5A13pOVRJR4oY3cSF1DEtPmGod+ATldrLIO2slHooPbhVeiFlxKkA -+8E9fyuwCklouxsMWC+4Ilvmp/F1LlXT1xZmmXXybRBzEzCDYlIdV6QLdKgiJFRZL866d2KbduJJ -IJtrrbi6QuIvzwHf37NXDl2KZbubOl0xQIkzTAwHxV/F/wAsaM4TFc/44ppj4CA7i05992M1/ZtE -WnYrllMg3CTQ8Sm0J21IuWla3UxfnCJO3Y7DY7R6bx2THn69WAUrxgpWkBlb4rWxl8XL4v8APEF2 -T5NkXCNjXIS6REFfF9OrE5JqimoTUQREfAoRbOhV8vPHm7cW1qCbbcT8IkIIj1V9OrAAOOmuHLiU -V7i6rU0Vv/fgnxT7h0hFgvcVolcBj3fWpYHvWMb1DwwJl1e9JFv0+XLUsUrtdz84yhkfcI0BcKCK -LYljbkQiRcyENdTpQeelMAMzXOReZMxlH/urMOU0FdkZVOSVj0t2pa7Y9XUVbcXlkms5fICKOWoA -nKRNyal9seKlbpT3tPTGLdh80nmIlynQi5SS4pQhKefkG6rSz3gt+9Pp+mN/yoo1afZoRyyeorqq -LPHTQBJBmVB16Cry78AJyu1a5SW9jqNuARcpWqmosSr+QK3S0R8VBp64hybOSXSVjVofeRXG5llh -qtRsCAUr986Wpzur8uBaVUWs8+kMpyXFunStkjmd/XiFbA6qos0ud3d4R6cSHD54WWnSctFO2jV8 -vti2Jzuy0qVfD3fdAX+2mAJtX0+vGkjl32VIyLRUkwep+4jIilRoJCH/ABa0wHZS0fCZYeKx8w+Q -Ypq2yeaX4Fvyat2m22pXx615Ut6Rw5KQryPjmLaWigfrIICMblSIMkmaF3eTlSug10/F+mBqKcxO -5iZu2z+HIou4eJRbVKOgQoNfubh0WXr9e7ASlt5y4atHeXl4mDVtcDE8SRScu6/h73O4E6aa1uw5 -MOHjl5JySz+2XaIbLldH+4wLb+IIF3EqQ4XBQ8e2buZBT2w5GSVJFWWX6X0qsXcknp90l04OSbXL -cW1jovMjsItND7WhCMlqlugFNffW+Pn54CgS66KjWIqpFGqyUSJPLGUbDSVXr/8AOOufSHxVu8se -aZeknqW+5j5rOrita0VkWL7hWol5ooAI6bYd1K+ddcWdim4nMyqi5jVGj2SSFebfkdvBtO5JkHnQ -i9MWFyhKt1OFr2iMcqgjTbTjWzZJWiYU8NSIud9ad+A0S7o6f92PGJCHT4vhwsOrqLHumy7ANXKX -9QBh0LvF1/y4909JeL8OIsrJRsM04uUfoNG9wp7qx2jfXuHAPqrN2jddd2tsopBcqqXTaP1xyd28 -dpUhmuaKPgni/sVk6UT2mi1irnQBPeEudCHn4cQO3DtWzFnGUk8ops30XBpAuK7ZoFzxWqdn31PJ -Hq15YpeT/wB3Y2PZsXKzV6m0dXLoIn7q1RELTFbv5VwEuM9sJpKyCCyHGNl4/hpEfumwW8hcBX4K -V7ywluxnotk6Xj3JucxNl11pMr72pN1NOpGmJ7eDnJ1psSDZAnTlqm2iXLBawSNDuSXoWtD105XY -tsJAymWYLLPDHvsydKEPEgIL8R8bRTX+GXlgK2yiZJ20jkHkU6kpJsKai43iYLtO8FU/xjixR+X1 -pR2qWyu/WU95MJEzFMXKNC+9T/8ANDzpg7lSBFtIIPmKK8a1dvCWauiMrox3TxM1PLbKvKmLarvL -yCD4nPsclXRJpJEdvBvh7hrp/CUwAVWDZsTXknwb7xghbZ8ErH106qUp8YYkg6WJug7iVkFFmCW4 -1VLxScZX7xOvzEHP8sRjavH1qaiJoOifEo1uMvsb2njQ5fw1R7sCXrV05kEJCEZrobW64Ztr+lm4 -Aq8S109DpgDsS4KNkEnLFmo7YtPtbE7x+0x6w+8RL+StcO5iUaxMggu7M3Ps9Ir/AISXjl//AO2W -AWYpD2EkzGL3yaikMtGAPgVRPku3H8NNcGDnE/Zi6ikbxIxoj0F1krHr6a86/KWAlQ7cRaf2g5BR -u0ujpW7+OxU5orV/LXA7OdwskmLta0Ykhjp0B61V2Rfdq0L6d+HTT9lhdIOTKNQSJg8EQuEminNB -YvPp1wEe5mFNk1kE4037xJJSDnUr7Ssr0pql6/8AtwFIzatGzOVxfKIrk+hyKOmBUMhNVqoXuF66 -U6qU08WKlmJ019scDNyQRaIoJxMml4iVtG5F1X0pzxa3ZFDO9vMD9BAd32LLeEt1oQ3pkNKd9Brj -KMyyCJSAtBhEHakfuMF3ThYrXIGdaIKV9NBt0wDEISMlNi0bLHLOH6oo77joFBa+m2pTv1qWJXaL -Le1M0JFGw4JyAkIrtkw6Rdj0qDSndXURphrL7N5JSr4ZQ+CZiPBKumwWg2dAOqJV09a0w12XrOCz -AkxXRBckiVepHeQqbolS8qV118NC/PAEcwQaL4J9SEeAm3YIJvT3ukiKoU32wV9Br5UxRW7wmwKq -JommRNdki3v0qXd3YueblmMpDz60XvoRaboHMWneJFtLlot+XPXFAMiFIrunpK7owH6F9ggvG3Y/ -lklzBRRRn4yW+G6v4MXlkRNktxy52/is3iMbv9FMVbs0ZjE9lWW2xAuntsExIRMrefPnp+eLYkQp -pCK5r2l1ERXXf564BSq3UShIoEPwnvEJFr/gww7cLJnsCFpEXwrFcI/6MRXr5HdVJB509IhcZdJe -fnhbclF26qm8Cl3hK9W39a0LuwENInBXFfcJDcFrkrrfyqljC/2k5hqpJsYld41TcJtTUFV6iSrY -RqFeWlBoVFKeWN7abiCVxH74riG01vDXy8WOeP2hXzdt2ipLprXOkWe4OyiRkzGvKp1E61osPPQw -8sBmGTM0I5fVffukaDZm5QEXLmTbCb4deSotT1tP6a2+LG65dTcZhyoxi4394l0XbUU2sY90bhaJ -dSq6g08Gn+rHO7Ru1ThxdxcJxqLR0oL6Vd//AA/r+Krf+F5Y6c7AlE22V44bZV28XIRbILPN3fSp -zKz0RGv9MBdDkIlhJpREAim9ogCiiT0QAWcMNB0rqp5lX6YGKz2V4dAW8XIKXL+/eTJBU1V/LRHl -1FX6eHE+byOnKNI6LFxttWC6qx8MG0xsv02lBHxFQcGsj5TiY9v7SFb2kS9ynHOQEdoe6gop06QH -8sAHcJvJ8UOJbLtotIRIIdM/tj7XkNXBfAniZDwck9Fq+mWbVJREi4CHa/3ZidvKq1afeH+mLO4T -Yxrfi0TQZboihxxdRkVS0Eda864oWcpKU4h4CbNeFy+SpdTc7X004qPgQD4KfiwBGWnFI9RVPLyL -aazOqkImO9QWbOo69VS0rQKU154pq3s32euUlnY2CJXDIyzQPtMg5IetJvqNS2h+lMQMvwLpiqMM -5hDkZd2qLn9245ztM4xLnYTtenUodfOhV6sX7LUGaL5WZfBHyuYU1RaJJNAHh4YflH09a4DNQWYx -7eJkI2NmxFddRbL8E5O59Juy8Ltf0SHy1xbzn5ZgXCQ/Zw6zTt/3ySMtdxzXmoNK6c6DWumv0xKk -6lAZxUXYM/3gnHSFs2/UPrQC3QG6FPgIq/DTy6sD3uXidEkpMzWZYtxt0oLGDoVW7UNa1onWuvUf -PWtfrTAbKAiPhDHjtEPBh0cJ54BCRDZ8v+3HIn7X+eJJ9m9vHwj/AG42NQVRXVbnRcCcW3+AdbCH -TlWuOoe0CY/d/IsxLCdqiDVTbH5jqPTSlPPH55cGIq8c7cuiuJDiVWx2GruJFeFKV1offpdgFhNL -RJiS5rpPFCUHaRW+03LAHXv/ABjy501xfsiZX4FoSko23EyY2yO2Hul0ajaRp/8AmI6eKmKj2Pps -SzR7ScogvHtEhTcoKASpIJEZUorr5EJbeNGOQcZZnXjknJi4UdEmLYgtSSdW6VoeuugLBbX/ABYC -5wWWYdQxfSD/AH3iAppya49IkNf7q/T07q911aYuQRayce+kJ1mu9UVIUZZIfB/5b5Gnd8uumK63 -mm8f7H9nw7XhVWpC1QUP79uX94Yl+MPEP8mLFFPkRkGrSPeLuSZNVHUYfVZIR9fvGta1+NOtMAAz -Q4UbOHSEzKmmmQi2mCT6Rsr/AHd8nr6fHiBmCQUIFXMle9UbCnHTZtguEh/gPaUph3tAkEyaFLII -oO24sxJIVg/vkeZe9RrT50ueA/taNy+qzj2J7jNNgLeTVL7pePXKgp6V56qBd+mAsTTNBNGSskoz -MXSiqbKRJQ7iSMeaLmn83dXFPk86EvmNedbb7Rwp0qpdIglIJ+L/AAqDhjNCgwj1JjJb66jYSaPC -E/vWp9SC3Lx26YxrNeYiJukuu/4lwJbLwU/wF7lan5jb1f0wGmZ1zNIcQKmX2G3FtBJ+ld1ESS3Q -sKf0pf8Ay4Hg6zBlJwqNhvWLBnauPzNHPhLX4rS8sCXc08Uyo6fQiJiMIqRBudRKslx0P/TWuGof -MWZlAdLyFijVkw9mOTI7t1FYtUlO74dPF/TAbPl8XBZMZtJkAXcdMLLFvWmLVQbm6nPnpS7vxVJV -ZZSPVUlpLbRSX9iyzMeki2x90ty/FQerzxKyEsop7MY5k4UhmGqkK+citdaYf3danp5YVmt0oSTU -iZsWjp+gpCzB32kg6H7tauvzad+AyTNuaFk5OMF2w2pJkko0kFVgtuKv3alafNQfXFRVcM5J2zUk -FnTl5ukm5JG7qSHkJU/Fj02soU2q5mX/ALSkt9Ru5SHp8OggV2PQTV8nxiHBgSyoi0uWO0kDIuRU -wBuCJOLyEqTZzxac6KjV01K33TgCuSP+mFdnMaQpSc2Jmo6gkk3CDZQ7CVC73g8++nP8sO9ocgmh -FRgkHDSRCSb8bB6XCHSBUp5aj/niz5S3I3syatpBFihIEqMixfEdxLslCqCwV9a0rpgKVnNq1aC6 -9pGbSUcr8RtJ2kgk1UAVUxoNO4+vFbj2sgvbGsWy6ijshEEk/ER+WlMWHtAGFjZD932hnJCwFQUn -wncTkSG5On+DXEFks8dyApxZoNk+P4hql1AQqiOtREtPDgP0PyOm+QyVGIPjuecGnviRq/CNKacx -xOcKLIbRCB/CJW7vn+mIMOs+KHY76IIqEgmoqJIjddZTWv3mH5AR4clLLiHwkmHi/wB+A89WWFUh -URX2x+IuI8X6Dhpo6dIOCHZ8NtvvlhHn60IaYSyTRdgP2YBISttUAh/xfe1xM4URMkyR2PiExDw/ -n11wDr1ZYUkiEF1BIrSFO8/r6Yxz9p3LLybyuxzEga6AxKvEFts1SXFKheVbf89cbKkis23U11kC -HpISJEh/y6sCcxwrObhHkavYui5SK3oIhIajz/iUwHJLtwhOspydJJbMq9jZyMwij9lZgNw2vGvn -Wmvi0xrv7IScehDzDSNmGr94Dol1XDICJszRr1WDfSnfjnzMrdbLshLQSDORJG3h0AFyKBiND199 -WmoKUGvdSunjxrH7Ms9MFnV004N6uioSRSL5EAbEkNB5I1R+Lq+KneOA6rboorAguPS13dxqCNwd -487vXHgWcJ1QTd7ZSRCptCjdtW0+avliSipaqQie6oRCQoeHYCuAc81nFEiicuufZokW44k1g3bQ -r3inTl1f8sBHzFmJvDAlx6KktMKpgSUSyDdLdpzoX4efxFgFlaJzNNSCWZMyOWSEskqSfAo+8CMb -lSvRSvmtXzLFnh4FrHtVUYu8U10ivkSW+0qHd3c6d2IuYJRuhHruXL4IWJtEvaN9qqytK06aBWnn -TAMQuVYeGZPo6CNRkzXIl3stxNy+7QqV0I689MB5iaWcpKqZbftcvQZFacmLa9d8t3e5D4/5sU9X -NTrPk8xi2gcBCgqSoQSlyDiQAf4y5/w0fw18WA/aHna12upk1YJaatJgxVbo3JCr/wAFmn4en41M -A7mDNEbEGuXttCAcJiQpM1LyXvU6auXNaU5KWVKwPXuwmAkO015FIuGGf4TIUeVPscZIMAJ0ol5L -q386Edda6emmEdmXZqUb/a2dFo6Smo10Lt+g7P3DMqjeSypV++UpTu+EcFs3Z8yCi8bKTEPGzyq7 -aiyT1+6FJVRKpFQenypyrp9MBvvx+PCvFj4Ajf3Y+15a6eXdgOef21c1DE5XjMvqMzXbyhksRitt -EFnPpr82OSMzPh3UOEeLuUWQ8MAEY2pWleOnzjpTHQ37Y7mQLPgtmpN1ExiVK2OQuFIfnT5clPrj -niPTkpmBbMGz6gIryQAILDr11Gtda179K178BrHZkKcRldisq5ZOU3KSj10aNu4qxUKgLJ10+JMq -AXLCpVmom7k4l3MOnLe1NGVc2CuRMq9TR0H8o1pTEuDi3ySqpRps29VGwv0E6JWp0EfdronSneJ0 -pStC76YrmYiFmo4BZRfaibQUogdRq6jVOfDnX5g1roXPXzwB/J8g8sdRcoZthTVFTitkruKoXuXQ -6+GivhP+fFuj5SWKCeC2cgyJdUlIwyusZvh+8b//AE1B54xxw+UXhXHBSkkiq1dIxyaihUKtY5au -qadfUxrSnfyp5YvPZ1OSUoJxrrbMFqVaODLqqLlKlyTkOXKunIqf1wBSVzA8kG8SMbGmLfj+JEiO -4Gbug/aGqn/ln8OKHnucYje0y2AezySUWSbEdxIJGNaKtS1+IddaY88zlKZlaydFUko9OQUFNxwZ -VCou0g04gPSpUpTWnLXz1xWmsWwWjk0UKrIqOo83Jr/GDgB5lSmulRLzp/TAWOQJ1Fx4zruVCWeQ -opCaSJiO+xUGtKcq/LjPOIasZtBRyw9z7wjKwT3UTLpLTw8sG7G45faS3CpaRZJgonSmm+gdNSCt -fLn5c6Yqjt0g+lG7ZFExZm6Mm4qncVAIuQl66YDR27daJj3S7ZY3rFgW2qF5CRMl/ARUr36VrTAV -V1IIN3jZewk0Ek2D74QVSMtUVOXkOuLhInXJCLriCJ8ghRSMdDTkSqRjWo6etaVpTWtef1xnjxN8 -3kF2ajlMkkwSarjZdQklK6hSmvnSvr+mAs2Qmb5tJycfKSX2hRUY4x6fcEI02luXw/XF57VWbeWj -Gs2+bLuXDlLgpNsn4kpABtTUqP1xnmX8ssOMRZLLOarlMnErGBaCoppqB189KV8+/F6gZprIQ1ZJ -03VApAVI2QBMqVEl0qag4CleV2vnXngMMlUVGxl9j4Ikvsy4F4t2nOpYNQ8SUkySlnkkajcnQsnl -txEgdvuir6jgVmhJYaJyLxyaq7xRSqtfFrUK6U7/AKYumSE2yOUgUqlU208ktHvEa+RhzSVGvrSv -f3fngKzm6SWlpMnLttwyxCKLnc6iVWEbalWlPDrX1xa80Ok2zTK0XHtvbUD1P2LYfvRG2lFkSLxV -0t10xRYtwqEok4TbtlXJGK4KLUuqKqZUMq93dXu07vWmLmbqSWziM3Bk3bK0aDKKJLhqFCK4STHS -ngrTy0wAKbavF5D3fAsmuwLlnVQxDfRqdSTEa+RU8PLvxFhE5Sbm2DHe6XbwVitttEqnShF0+HDe -amjhGdfbrmqttU6lry0pULqCP0p5YtHYvEm/zxEhHPFUttRBZxRbnQq7g00HT6euA7n2xQh2aZLI -ESaSfSRpCPIdPMce4xum3SQJsxIVCuG1yl/1HywaogKBkSKCQKiVhaEVtPoP0xJSZvCVvMkan8VL -yt/TlgBDckb7d5raPyrI+L5uQ+eHTdEmqNphvEVxANl1tPXpxPJo/TQNusadEKd20udC8X5YivnH -AK1SJV2e5TSlarkVtP1wCjeEuZKEBqD8IkiJdXp3YfBMRuTFHqHwe5G23/viIaTUgEbVKlTuqVcR -hdtA02xWpeNbeVOnXv8APAczftC5VTY9oTqQbNkH60ta7bRg3CkSoDWil4dxdNdenFB7NZ5qnmZJ -9LM15tZBAukZIkgbABVITTMa0qoIj01Avlx1J2y5ZaZ5yO9hUXKzV8nQKtHJDTRM7uWunPT1xyqG -XvbMs0bOHCQzbt3wIUogPB1KgiJGY9+te+6lNcB3BkWW9t5fYzKjYGzqQapuJECuE0BqHSP+WCqo -qKKhILouk02R2t0kTuFca6UuKmOUexfPS2Ss8o5RclJvSqqSEuuo8qsDkxPQSAD0oNKUx0a+zgqW -X46Rat9l5JOeDaUrXVNGl2lxU8+XlgDrgniDhmgKPFut9Qb07tpsFfn/AE7sZznXNULxaSe9FT2Y -I5JTiXqxiTOPIfGVAryJT8NOeLJPrPqSq8G2XpHgtRKi7lr0rKLHTS7XlbSlPTHOud8tqJyuVhlp -g0Ip46dxyKUYzTTWbGJXblD5VIiKlK1KvPAXTPce1aZCnXzGSN63XFstNkK21KvBIuYqKV+5S08v -lxj7+YmMoTb7NeX5iIQ9hPmzZiggtvpNmiojoIadFtteo9NSrimzWa4R07NzKUnnaziKqksqbqh7 -6gqHQLwrXSyg0py9cQ6ZkZSUll1lBZajIWWZtiaruRHdTdFdX3pDWnIvrzr9cAfzrmyYlM2lDz2f -+JbkZIqP2QUJrw6/OtKhpcVvpX8sQYma7I6sE0815azFNSiWqRvkX1oLCNdBIRrzGmmnLywh5x/a -T2oUbxMbE5bfE1rRbhiOiKhohdVTSlNbq9/59+KPmSUSeTTlZywboL3UFQWw2p1KlKUrWlPLXTX8 -64D/2Q== - ---Apple-Mail=_C7D5288F-B043-4A7F-AF3F-1EDF1A78438B-- - ---Apple-Mail=_F4EF9C8E-2E66-4FC6-8840-F435ADBED5C8-- diff --git a/mail/src/leap/mail/tests/rfc822.multi-signed.message b/mail/src/leap/mail/tests/rfc822.multi-signed.message deleted file mode 100644 index 9907c2d..0000000 --- a/mail/src/leap/mail/tests/rfc822.multi-signed.message +++ /dev/null @@ -1,238 +0,0 @@ -Date: Mon, 6 Jan 2014 04:40:47 -0400 -From: Kali Kaneko -To: penguin@example.com -Subject: signed message -Message-ID: <20140106084047.GA21317@samsara.lan> -MIME-Version: 1.0 -Content-Type: multipart/signed; micalg=pgp-sha1; - protocol="application/pgp-signature"; boundary="z9ECzHErBrwFF8sy" -Content-Disposition: inline -User-Agent: Mutt/1.5.21 (2012-12-30) - - ---z9ECzHErBrwFF8sy -Content-Type: multipart/mixed; boundary="z0eOaCaDLjvTGF2l" -Content-Disposition: inline - - ---z0eOaCaDLjvTGF2l -Content-Type: text/plain; charset=utf-8 -Content-Disposition: inline -Content-Transfer-Encoding: quoted-printable - -This is an example of a signed message, -with attachments. - - ---=20 -Nihil sine chao! =E2=88=B4 - ---z0eOaCaDLjvTGF2l -Content-Type: text/plain; charset=us-ascii -Content-Disposition: attachment; filename="attach.txt" - -this is attachment in plain text. - ---z0eOaCaDLjvTGF2l -Content-Type: application/octet-stream -Content-Disposition: attachment; filename="hack.ico" -Content-Transfer-Encoding: base64 - -AAABAAMAEBAAAAAAAABoBQAANgAAACAgAAAAAAAAqAgAAJ4FAABAQAAAAAAAACgWAABGDgAA -KAAAABAAAAAgAAAAAQAIAAAAAABAAQAAAAAAAAAAAAAAAAAAAAAAAAAAAAD///8Ai4uLAEZG -RgDDw8MAJCQkAGVlZQDh4eEApqamADQ0NADw8PAADw8PAFVVVQDT09MAtLS0AJmZmQAaGhoA -PT09AMvLywAsLCwA+Pj4AAgICADp6ekA2traALy8vABeXl4An5+fAJOTkwAfHx8A9PT0AOXl -5QA4ODgAuLi4ALCwsACPj48ABQUFAPv7+wDt7e0AJycnADExMQDe3t4A0NDQAL+/vwCcnJwA -/f39ACkpKQDy8vIA6+vrADY2NgDn5+cAOjo6AOPj4wDc3NwASEhIANjY2ADV1dUAU1NTAMnJ -yQC6uroApKSkAAEBAQAGBgYAICAgAP7+/gD6+voA+fn5AC0tLQD19fUA8/PzAPHx8QDv7+8A -Pj4+AO7u7gDs7OwA6urqAOjo6ADk5OQAVFRUAODg4ADf398A3d3dANvb2wBfX18A2dnZAMrK -ygDCwsIAu7u7ALm5uQC3t7cAs7OzAKWlpQCdnZ0AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA -AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA -AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA -AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA -AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA -AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA -AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA -AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA -AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA -AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA -AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA -AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA -AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAABKRC5ESDRELi4uNEUhIhcK -LgEBAUEeAQEBAUYCAAATNC4BPwEUMwE/PwFOQgAAACsuAQEBQUwBAQEBSk0AABVWSCwBP0RP -QEFBFDNTUkdbLk4eOg0xEh5MTEw5RlEqLgdKTQAcGEYBAQEBJQ4QPBklWwAAAANKAT8/AUwy -AAAAOxoAAAA1LwE/PwEeEQAAAFpJGT0mVUgBAQE/SVYFFQZIKEtVNjFUJR4eSTlIKARET0gs -AT8dS1kJH1dINzgnGy5EAQEBASk+AAAtUAwAACNYLgE/AQEYFQAAC1UwAAAAW0QBAQEkMRkA -AAZDGwAAME8WRC5EJU4lOwhIT0UgD08KAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA -AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAACgAAAAgAAAAQAAAAAEACAAAAAAA -gAQAAAAAAAAAAAAAAAAAAAAAAAAAAAAA////AH9/fwC/v78APz8/AN/f3wBfX18An5+fAB0d -HQAuLi4A7+/vAM/PzwCvr68Ab29vAE5OTgAPDw8AkZGRAPf39wDn5+cAJiYmANfX1wA3NzcA -x8fHAFdXVwC3t7cAh4eHAAcHBwAWFhYAaGhoAEhISAClpaUAmZmZAHl5eQCMjIwAdHR0APv7 -+wALCwsA8/PzAOvr6wDj4+MAKioqANvb2wDT09MAy8vLAMPDwwBTU1MAu7u7AFtbWwBjY2MA -AwMDABkZGQAjIyMANDQ0ADw8PABCQkIAtLS0AEtLSwCioqIAnJycAGxsbAD9/f0ABQUFAPn5 -+QAJCQkA9fX1AA0NDQDx8fEAERERAO3t7QDp6ekA5eXlAOHh4QAsLCwA3d3dADAwMADZ2dkA -OTk5ANHR0QDNzc0AycnJAMXFxQDBwcEAUVFRAL29vQBZWVkAXV1dALKysgBycnIAk5OTAIqK -igABAQEABgYGAAwMDAD+/v4A/Pz8APr6+gAXFxcA+Pj4APb29gD09PQA8vLyACQkJADw8PAA -JycnAOzs7AApKSkA6urqAOjo6AAvLy8A5ubmAOTk5ADi4uIAODg4AODg4ADe3t4A3NzcANra -2gDY2NgA1tbWANTU1ABNTU0A0tLSANDQ0ABUVFQAzs7OAMzMzABYWFgAysrKAMjIyABcXFwA -xsbGAF5eXgDExMQAYGBgAMDAwABkZGQAuLi4AG1tbQC2trYAtbW1ALCwsACurq4Aenp6AKOj -owChoaEAoKCgAJ6engCdnZ0AmpqaAI2NjQCSkpIAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA -AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA -AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA -AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA -AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA -AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA -AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA -AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA -AAAAAAAAAAAAAAAAAAAAAAAAFHFvR3Fvb0dHJ1F0R0dHR29HR0YLf28nJkVraGtHBXMnAQEB -AQEBAQEBCxEBAQEBAQEBASdzASOMHHsZSQEBcnEBAV1dXV1dXQFOJQEBXV1dXV0BR0kBOwAA -AAAIUAFyJwFdXV1dXV1dAU4lAV1dXV1dXQFHbVgAAAAAAAAoaG5xAV1dXV1dXV0BfSUBXV1d -XV1dASd2HQAAAAAAAFoMEkcBXV1dXV1dXQFOZAEBXV1dXV0BbU8TAAAAAAAAAFkmcQFdXV1d -XV1dAU4lAV1dXV1dXQEnSzgAAAAAAABaN2tHAV1dXV1dXV0BTiUBXV1dXV1dAUdtHwAAAAAA -AEpEJycBXV1dXV1dAQFOJQFdAV1dAV0BRykBIgAAAABlfAFzJwEBAQEBAQEBAQtAAQEBAQEB -AQFuSQE8iFeBEG8BXUeGTn0LdnR3fH0LOYR8Tk5OTnxOeouNTQspJ0YFd30rgCljIwpTlCxm -X2KERWMlJSUlJSURFE1hPEYMBysRYSV0RwF3NT0AGjYpAQtjAQEBAQEBAQFvKQGKMzEAP4dC -AXESEmcAAAAAAEpEKiUBXV1dXV1dAUduLEEAAAAAAIFdcUSWAAAAAAAAADp1ZAFdXV1dXV0B -bwVVAAAAAAAAW4Jta34AAAAAAAAAhRQlAV1dXV1dAQFtK0gAAAAAAAAAEGtFhwAAAAAAAACJ -S2QBXV1dXV1dAW5NFQAAAAAAAACTa2geAAAAAAAAAAx0ZAFdXV1dXV0BR0YNAAAAAAAADxRu -J14tAAAAAAAvXQslAV1dXV1dXQFHcW4JAAAAAAAhAXFuAWMgbBsJAhEBTWIBAQEBAQEBAW5y -AW+DZWBwkQEBcQtHbWh2hnZEbm6LFG9HR21uR3FGgFFGa2oqFgVob3FNf0t0dAUncnR0SY1N -KW5xK01ucUlRLklyRksqR250S3pGAQEBAQEBAQEBeWIBUFRINA1uAUYFAQqOTGlSiAEBb0cB -XV1dAQFdAQF9I4pcAAAAABNHEnIKBAAAAAA9kAFJJwFdXV1dXV1dAXptZwAAAAAAAAZqbY4A -AAAAAAAbcm5HAV1dXV1dXV0BFFZbAAAAAAAAZ3pLNQAAAAAAAACPa0cBXV1dXV1dXQEpkgAA -AAAAAAAygHppAAAAAAAAAJVrcQFdXV1dXV1dAXl9QwAAAAAAADZxcRcAAAAAAAA9UW1vAV1d -XV1dXV0BC2EwAAAAAAAAkmhGGD0AAAAAAHg+cW8BAV1dAV1dAQFOESWBAAAAJJUBJykBkEMA -AAAOJgFzRwE8AV1dXV1dAX0lAV8WEDp1AQFxSwEBBTkhAxEBPHJzSXEFcnJJcnFyFnRycRJr -RW5ycXl8cXJuRSYScQVJcQAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA -AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA -AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAKAAAAEAAAACAAAAAAQAIAAAA -AAAAEgAAAAAAAAAAAAAAAAAAAAAAAAAAAAD///8Af39/AL+/vwA/Pz8A39/fAF9fXwCfn58A -Hx8fAO/v7wAvLy8Ab29vAI+PjwAPDw8A0NDQALCwsABQUFAA9/f3ABcXFwDn5+cAJycnAMjI -yABHR0cAqKioAGdnZwCXl5cAd3d3AIeHhwAHBwcA2NjYALi4uABXV1cANTU1ADo6OgD7+/sA -CwsLAPPz8wATExMA6+vrABsbGwDj4+MAIyMjANTU1AArKysAzMzMAMTExABLS0sAtLS0AKys -rABbW1sApKSkAGNjYwCbm5sAa2trAJOTkwBzc3MAi4uLAHt7ewCDg4MAAwMDANzc3AAyMjIA -vLy8AFNTUwD9/f0ABQUFAPn5+QAJCQkADQ0NAPHx8QDt7e0AFRUVAOnp6QAZGRkA5eXlAB0d -HQDh4eEAISEhACUlJQDa2toAKSkpANbW1gDS0tIAysrKADw8PADGxsYAwsLCAEVFRQBJSUkA -urq6ALa2tgCysrIArq6uAFlZWQCqqqoAXV1dAKampgBlZWUAoqKiAJ2dnQBtbW0AmZmZAHFx -cQCVlZUAeXl5AH19fQCJiYkAhYWFAAEBAQACAgIABAQEAP7+/gAGBgYA/Pz8AAgICAD6+voA -CgoKAPj4+AAMDAwA9vb2APT09AASEhIA8vLyABQUFADu7u4AFhYWAOzs7AAYGBgA6urqAOjo -6AAeHh4AICAgAOTk5AAiIiIA4uLiACQkJADg4OAAJiYmAN7e3gDd3d0AKCgoANvb2wAqKioA -2dnZACwsLADX19cALi4uANXV1QAxMTEA09PTADMzMwDR0dEANDQ0AM3NzQA5OTkAy8vLADs7 -OwDJyckAPT09AMfHxwBAQEAAxcXFAMPDwwDBwcEAwMDAAL6+vgBKSkoAvb29ALu7uwC5ubkA -UVFRALe3twBSUlIAtbW1AFRUVACzs7MAVlZWAFhYWABaWloAra2tAFxcXACrq6sAXl5eAKmp -qQCnp6cAZGRkAKOjowChoaEAaGhoAKCgoACenp4AnJycAG5ubgCampoAcHBwAJiYmABycnIA -lpaWAJSUlAB2dnYAkpKSAHh4eACQkJAAenp6AI6OjgB8fHwAjIyMAIiIiACCgoIAhISEAAAA -AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA -AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA -AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA -AAAAAAAAAAAAAAAAAAAAAAAAAAC1WlpaWlpaWlpaWlpaWlpaWlpaHjAHr6taWlpaWlpaWlpa -WlpaWlpaq68HMB5aWlpap6KlWzBaA6KoWlpaWlq1WgEBAQEBAQEBAQEBAQEBAQEBAQGXNkUB -AQEBAQEBAQEBAQEBAQEBAQFFNpcBAQEBASg4EI6HPa5lfgEBAQEBWloBAQEBAQEBAQEBAQEB -AQEBAQEBlzZFAQEBAQEBAQEBAQEBAQEBAQEBRTaXAQEBETpEAAAAAAAAAH/FbwEBAVpaAQEB -AQEBAQEBAQEBAQEBAQEBAZc2RQEBAQEBAQEBAQEBAQEBAQEBAUU2lwEBhFQAAAAAAAAAAAAA -ALJCAQFaWgEBAQEBAQEBAQEBAQEBAQEBAQGXNkUBAQEBAQEBAQEBAQEBAQEBAQFFNpcBeJoA -AAAAAAAAAAAAAAAAMQEBWloBAQEBAQEBAQEBAQEBAQEBAQEBlzZFAQEBAQEBAQEBAQEBAQEB -AQEBRTZSATUAAAAAAAAAAAAAAAAAAABnAVpaAQEBAQEBAQEBAQEBAQEBAQEBAZc2RQEBAQEB -AQEBAQEBAQEBAQEBAUU2Tx1wAAAAAAAAAAAAAAAAAAAAgkaoWgEBAQEBAQEBAQEBAQEBAQEB -AQGXNkUBAQEBAQEBAQEBAQEBAQEBAQFFNgVrAAAAAAAAAAAAAAAAAAAAAABioloBAQEBAQEB -AQEBAQEBAQEBAQEBlzZFAQEBAQEBAQEBAQEBAQEBAQEBRWcqngAAAAAAAAAAAAAAAAAAAAAA -tANaAQEBAQEBAQEBAQEBAQEBAQEBAZc2RQEBAQEBAQEBAQEBAQEBAQEBAUXDpIcAAAAAAAAA -AAAAAAAAAAAAAJRaWgEBAQEBAQEBAQEBAQEBAQEBAQGXNkUBAQEBAQEBAQEBAQEBAQEBAQFF -wa9HAAAAAAAAAAAAAAAAAAAAAABOMFoBAQEBAQEBAQEBAQEBAQEBAQEBlzZFAQEBAQEBAQEB -AQEBAQEBAQEBRWVZggAAAAAAAAAAAAAAAAAAAAAAjltaAQEBAQEBAQEBAQEBAQEBAQEBAZc2 -RQEBAQEBAQEBAQEBAQEBAQEBAUXFmZYAAAAAAAAAAAAAAAAAAAAAAKqlWgEBAQEBAQEBAQEB -AQEBAQEBAQGXNkUBAQEBAQEBAQEBAQEBAQEBAQFFNorHAAAAAAAAAAAAAAAAAAAAAABloloB -AQEBAQEBAQEBAQEBAQEBAQEBlzZFAQEBAQEBAQEBAQEBAQEBAQEBRTY8UwAAAAAAAAAAAAAA -AAAAAAASEz5aAQEBAQEBAQEBAQEBAQEBAQEBAZc2RQEBAQEBAQEBAQEBAQEBAQEBAUU2lQFd -AAAAAAAAAAAAAAAAAAAA0AFaWgEBAQEBAQEBAQEBAQEBAQEBAQGXNkUBAQEBAQEBAQEBAQEB -AQEBAQFFNpcBhoUAAAAAAAAAAAAAAAAAVxEBWloBAQEBAQEBAQEBAQEBAQEBAQEBlzZFAQEB -AQEBAQEBAQEBAQEBAQEBRTaXAQGXTQAAAAAAAAAAAAAAnCgBAVpaAQEBAQEBAQEBAQEBAQEB -AQEBAZc2RQEBAQEBAQEBAQEBAQEBAQEBAUU2lwEBASiwAAAAAAAAAAAcwncBAQFaWgEBAQEB -AQEBAQEBAQEBAQEBAQGXNkUBAQEBAQEBAQEBAQEBAQEBAQFFNpcBAQEBASy8khINgiFojQEB -AQEBWjCVl5eXl5eXl5dSUpeXl5eXl5eTHsWdlZeXl5eXl5eXl5eXl5eXl5eVncUek5eXl1I8 -ipsvs6iVBU9Sl5eXlTAHNjY2NjY2Zb1ivbtiY2c2NjY2NsVlxjY2NjY2NjY2NjY2NjY2NjY2 -NsZlxTY2NjY2xr8yFxcXusHGNjY2NjYHW3hFRUURAY8HC7Jh0ahFb3pFRRGdxkp4RUVFRUVF -RUVFRUVFRUVFRXhKxp0RRUVFIkKhDLkxwMiXInNFRUV4W1oBAQEBCcclAAAAAAAAnK0BAQEB -lzZFAQEBAQEBAQEBAQEBAQEBAQEBRTaXAQEBAQ4ucAAAAAAAdAaNAQEBAVpaAQEBpYMAAAAA -AAAAAAAAGHUBAZc2RQEBAQEBAQEBAQEBAQEBAQEBAUU2lwEBAWtwAAAAAAAAAAAADboBAQFa -WgEBHnIAAAAAAAAAAAAAAACxcwGXNkUBAQEBAQEBAQEBAQEBAQEBAQFFNpcBAcQAAAAAAAAA -AAAAAABtwQEBWloBiCcAAAAAAAAAAAAAAAAAAM0BUjZFAQEBAQEBAQEBAQEBAQEBAQEBRTaX -AbsAAAAAAAAAAAAAAAAAAHCiAVpaAQYAAAAAAAAAAAAAAAAAAAAck082RQEBAQEBAQEBAQEB -AQEBAQEBAUU2UUVLAAAAAAAAAAAAAAAAAAAAIQEePkoNAAAAAAAAAAAAAAAAAAAAAMCLxkUB -AQEBAQEBAQEBAQEBAQEBAQFFNgViAAAAAAAAAAAAAAAAAAAAAACppKK9AAAAAAAAAAAAAAAA -AAAAAACQnxlFAQEBAQEBAQEBAQEBAQEBAQEBRcZPrAAAAAAAAAAAAAAAAAAAAAAAZqOjCwAA -AAAAAAAAAAAAAAAAAAAAQ7i/RQEBAQEBAQEBAQEBAQEBAQEBAUUZVSsAAAAAAAAAAAAAAAAA -AAAAAFRZpT8AAAAAAAAAAAAAAAAAAAAAAADKvkUBAQEBAQEBAQEBAQEBAQEBAQFFZVpJAAAA -AAAAAAAAAAAAAAAAAAAUXKU/AAAAAAAAAAAAAAAAAAAAAAAAyr5FAQEBAQEBAQEBAQEBAQEB -AQEBRWVaSQAAAAAAAAAAAAAAAAAAAAAAFFyjCwAAAAAAAAAAAAAAAAAAAAAAdl40RQEBAQEB -AQEBAQEBAQEBAQEBAUUZVSsAAAAAAAAAAAAAAAAAAAAAAKCoVrcAAAAAAAAAAAAAAAAAAAAA -ACCZxUUBAQEBAQEBAQEBAQEBAQEBAQFFxo1fAAAAAAAAAAAAAAAAAAAAAABpVqh+fQAAAAAA -AAAAAAAAAAAAAADRijZFAQEBAQEBAQEBAQEBAQEBAQEBRTaKXAAAAAAAAAAAAAAAAAAAAAA7 -LANaAWgAAAAAAAAAAAAAAAAAAABJSJE2RQEBAQEBAQEBAQEBAQEBAQEBAUU2KgEKAAAAAAAA -AAAAAAAAAAAAHwGrWgF8kAAAAAAAAAAAAAAAAAAAZQGXNkUBAQEBAQEBAQEBAQEBAQEBAQFF -NpcBHm0AAAAAAAAAAAAAAAAAEk8BWloBAZVLAAAAAAAAAAAAAAAANwEBlzZFAQEBAQEBAQEB -AQEBAQEBAQEBRTaXAQHFAAAAAAAAAAAAAAAAQx4BAVpaAQEBj1QAAAAAAAAAAAByGQEBAZc2 -RQEBAQEBAQEBAQEBAQEBAQEBAUU2lwEBARcSAAAAAAAAAAAAjJkBAQFaWgEBAQFxuphuAAAA -ABK8jwEBAQGXNkUBAQEBAQEBAQEBAQEBAQEBAQFFNpcBAQEBSMlLAAAAAG0rDEUBAQEBWlt4 -RUVFeAFFLWU6DC8FcXNFRUURncZKeEVFRUVFRUVFRUVFRUVFRUV4SsadEUVFRXUBhC8MOmWi -JgF3RUVFeFsHNjY2NjY2Z7+9Yru+wzY2NjY2NsVlxjY2NjY2NsU0vr6/wzY2NjY2NsZlxTY2 -NjY2NmUytbO3Yhk2NjY2NjYHMJWXl5eXl5eXl5eXl5eXl5eXl5MexZ2Vl5eXHQWdXgwMYKKK -T5eXl5WdxR6Tl5eXKgWVrWfOvquPipWXl5eVMFoBAQEBAQEBAQEBAQEBAQEBAQEBlzZFAQEB -AYE5kHYAAEMpvJEBAQEBRTaXAQEBAXFiBEcAAG4Spi8BAQEBAVpaAQEBAQEBAQEBAQEBAQEB -AQEBAZc2RQEBAcF7AAAAAAAAAABBaUIBAUU2lwEBAZsgAAAAAAAAAAAAFooBAQFaWgEBAQEB -AQEBAQEBAQEBAQEBAQGXNkUBAQsAAAAAAAAAAAAAAACxcwFFNpcBAQ92AAAAAAAAAAAAAABN -UQEBWloBAQEBAQEBAQEBAQEBAQEBAQEBlzZFAcwAAAAAAAAAAAAAAAAAABgBejaXAZd5AAAA -AAAAAAAAAAAAAImAAVpaAQEBAQEBAQEBAQEBAQEBAQEBAZc2c1JDAAAAAAAAAAAAAAAAAAAA -W3E2KgGeAAAAAAAAAAAAAAAAAAAAMwGrWgEBAQEBAQEBAQEBAQEBAQEBAQGXNm9kAAAAAAAA -AAAAAAAAAAAAAAQJZ4ukAAAAAAAAAAAAAAAAAAAAAHKVpVoBAQEBAQEBAQEBAQEBAQEBAQEB -l8OGKQAAAAAAAAAAAAAAAAAAAAAcor+LNQAAAAAAAAAAAAAAAAAAAAAAaqJaAQEBAQEBAQEB -AQEBAQEBAQEBAZdjHmwAAAAAAAAAAAAAAAAAAAAAAM8ymT0AAAAAAAAAAAAAAAAAAAAAAFg+ -WgEBAQEBAQEBAQEBAQEBAQEBAQGXvWUAAAAAAAAAAAAAAAAAAAAAAABhuFmCAAAAAAAAAAAA -AAAAAAAAAACOW1oBAQEBAQEBAQEBAQEBAQEBAQEBl7vOAAAAAAAAAAAAAAAAAAAAAAAAtGCv -RwAAAAAAAAAAAAAAAAAAAAAATjBaAQEBAQEBAQEBAQEBAQEBAQEBAZcHYgAAAAAAAAAAAAAA -AAAAAAAAAAu4pIcAAAAAAAAAAAAAAAAAAAAAAD1aWgEBAQEBAQEBAQEBAQEBAQEBAQGXNBUj -AAAAAAAAAAAAAAAAAAAAAAAyvSpXAAAAAAAAAAAAAAAAAAAAAAAYpFoBAQEBAQEBAQEBAQEB -AQEBAQEBl2ckVAAAAAAAAAAAAAAAAAAAAACDiMMFzAAAAAAAAAAAAAAAAAAAAAAAr6NaAQEB -AQEBAQEBAQEBAQEBAQEBAZc2b7sAAAAAAAAAAAAAAAAAAAAAaW82HRMlAAAAAAAAAAAAAAAA -AAAAlECpWgEBAQEBAQEBAQEBAQEBAQEBAQGXNngBBAAAAAAAAAAAAAAAAAAAKUZ3NpcBzwAA -AAAAAAAAAAAAAAAAAA8BWloBAQEBAQEBAQEBAQEBAQEBAQEBlzZFAZGCAAAAAAAAAAAAAAAA -dC0BRTaXAXGwAAAAAAAAAAAAAAAAAAIBAVpaAQEBAQEBAQEBAQEBAQEBAQEBAZc2RQEBlY4A -AAAAAAAAAAAACD4BAUU2lwEBd7YAAAAAAAAAAAAAbmtvAQFaWgEBAQEBAQEBAQEBAQEBAQEB -AQGXNkUBAQEJyw0AAAAAAAB0M0wBAQFFNpcBAQEBF1AAAAAAAAAAVD4BAQEBWloBAQEBAQEB -AQEBAQEBAQEBAQEBlzZFAQEBAQETB7ymprxliwEBAQEBRTaXAQEBAQF1qxqsV7QbVXEBAQEB -AVq1WlpaWlpaWlpaWlpaWlpaWlpaHjAHr6taWlpaPqKkPj6kLadaWlpaq68HMB5aWlpaqaNW -pz4DLaQeWlpaWlq1AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA -AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA -AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA -AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA -AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA -AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA -AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA -AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA -AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA -AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA= - ---z0eOaCaDLjvTGF2l-- - ---z9ECzHErBrwFF8sy -Content-Type: application/pgp-signature - ------BEGIN PGP SIGNATURE----- -Version: GnuPG v1.4.15 (GNU/Linux) - -iQIcBAEBAgAGBQJSymwPAAoJECNji/csWTvBhtcP/2AKF0uk6ljrfMWhNBSFwDqv -kYng3slREnF/pxnIGOpR2GAxPBPjRipZOuUU8QL+pXBwk5kWzb9RYpr26xMYWRtl -vXdVbob5NolNEYrqTkkQ1kejERQGFyescsUJDcEDXJl024czKWbxHTYYN4vlYJMK -PZ5mPSdADFn970PnVXfNix3Rjvv7SFQGammDBGjQzyROkoiDKPZcomp6dzm6zEXC -w8i42WfHU8GkyVVNvXZI52Xw3LUXiXsJ58B1V1O5U42facepG6S+S0DC/PWptqPw -sAM9/YGkvBNWrsJA/BavXPRLE1gVpu+hZZEsOqRvs244k7JTrVo54xDbdeOT2nTr -BDk4e88vmCVKGgE9MZjDbjgOHDZhmsxNQm4DBGRH2huF0noUc/8Sm4KhSO49S2mN -QjIT5QrPerQNiP5QtShHZRJX7ElXYZWX1SG/c9jQjfd0W1XK/cGtwClICe+lpprt -mLC2607yalbRhCxV9bQlVUnd2tY3NY4UgIKgCEiEwb1hf/k9jQDvpk16VuNWSZQJ -jFeg9F2WdNjQMp79cyvnayyhjS9o/K2LbSIgJi7KdlQcVZ/2DQfbMjCwByR7P9g8 -gcAKh8V7E6IpAu1mnvs4FDagipppK6hOTRj2s/I3xZzneprSK1WaVro/8LAWZe9X -sSdfcAhT7Tno7PB/Acoh -=+okv ------END PGP SIGNATURE----- - ---z9ECzHErBrwFF8sy-- diff --git a/mail/src/leap/mail/tests/rfc822.multi.message b/mail/src/leap/mail/tests/rfc822.multi.message deleted file mode 100644 index 30f74e5..0000000 --- a/mail/src/leap/mail/tests/rfc822.multi.message +++ /dev/null @@ -1,96 +0,0 @@ -Date: Fri, 19 May 2000 09:55:48 -0400 (EDT) -From: Doug Sauder -To: Joe Blow -Subject: Test message from PINE -Message-ID: -MIME-Version: 1.0 -Content-Type: MULTIPART/MIXED; BOUNDARY="-1463757054-952513540-958744548=:8452" - - This message is in MIME format. The first part should be readable text, - while the remaining parts are likely unreadable without MIME-aware tools. - Send mail to mime@docserver.cac.washington.edu for more info. - ----1463757054-952513540-958744548=:8452 -Content-Type: TEXT/PLAIN; charset=US-ASCII - -This is a test message from PINE MUA. - - ----1463757054-952513540-958744548=:8452 -Content-Type: APPLICATION/octet-stream; name="redball.png" -Content-Transfer-Encoding: BASE64 -Content-ID: -Content-Description: A PNG graphic file -Content-Disposition: attachment; filename="redball.png" - -iVBORw0KGgoAAAANSUhEUgAAABsAAAAbCAMAAAC6CgRnAAADAFBMVEX///8A -AAABAAALAAAVAAAaAAAXAAARAAAKAAADAAAcAAAyAABEAABNAABIAAA9AAAj -AAAWAAAmAABhAAB7AACGAACHAAB9AAB0AABgAAA5AAAUAAAGAAAnAABLAABv -AACQAAClAAC7AAC/AACrAAChAACMAABzAABbAAAuAAAIAABMAAB3AACZAAC0 -GRnKODjVPT3bKSndBQW4AACoAAB5AAAxAAAYAAAEAABFAACaAAC7JCTRYWHf -hITmf3/mVlbqHx/SAAC5AACjAABdAABCAAAoAAAJAABnAAC6Dw/QVFTek5Pl -rKzpmZntZWXvJSXXAADBAACxAACcAABtAABTAAA2AAAbAAAFAABKAACBAADL -ICDdZ2fonJzrpqbtiorvUVHvFBTRAADDAAC2AAB4AABeAABAAAAiAABXAACS -AADCAADaGxvoVVXseHjveHjvV1fvJibhAADOAAC3AACnAACVAABHAAArAAAP -AACdAADFAADhBQXrKCjvPDzvNTXvGxvjAADQAADJAAC1AACXAACEAABsAABP -AAASAAACAABiAADpAADvAgLnAADYAADLAAC6AACwAABwAAATAAAkAABYAADI -AADTAADNAACzAACDAABuAAAeAAB+AADAAACkAACNAAB/AABpAABQAAAwAACR -AACpAAC8AACqAACbAABlAABJAAAqAAAOAAA0AACsAACvAACtAACmAACJAAB6 -AABrAABaAAA+AAApAABqAACCAACfAACeAACWAACPAAB8AAAZAAAHAABVAACO -AACKAAA4AAAQAAA/AAByAACAAABcAAA3AAAsAABmAABDAABWAAAgAAAzAAA8 -AAA6AAAfAAAMAAAdAAANAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA -AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA -AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAD8 -LtlFAAAAAXRSTlMAQObYZgAAABZ0RVh0U29mdHdhcmUAZ2lmMnBuZyAyLjAu -MT1evmgAAAIISURBVHicY2CAg/8QwIABmJhZWFnZ2Dk4MaU5uLh5eHn5+LkF -BDlQJf8zC/EIi4iKiUtI8koJScsgyf5nlpWTV1BUUlZRVVPX4NFk1UJIyghp -6+jq6RsYGhmbKJgK85mZW8Dk/rNaSlhZ29ja2Ts4Ojkr6Li4urFDNf53N/Ow -8vTy9vH18w8IDAoWDQkNC4+ASP5ni4wKio6JjYtPSExKTnFWSE1LF4A69n9G -ZlZ2Tm5efkFhUXFySWlZlEd5RSVY7j+TkGRVdU1tXX1DY1Ozcktpa1t7h2Yn -OAj+d7l1tyo79vT29SdNSJ44SbFVdHIo9xSIHNPUaWqTpifNSJrZnK00S0U1 -a/acUG5piNz/uXLzVJ2qm6dXz584S2WB1cJFi5cshZr539xVftnyFKUVTi2T -VjqvyhJLXb1m7TqoHPt6F/HW0g0bN63crGqVtWXrtu07BJihcsw71+zanRW8 -Z89eq337RQ/Ip60xO3gIElX/LbikDm8T36KwbNmRo7O3zpHkPSZwHBqL//8f -lz1x2OOkyKJTi7aqbzutfUZI2gIuF8F2lr/D5dw2+fZdwpl8YVOlI+CJ4/9/ -joOyYed5QzMvhGqnm2V0WiClm///D0lfXHtJ6vLlK9w7rx7vQk5SQJbFtSms -1y9evXid7QZacgOxmSxktNzdtSwwU+J/VICaCPFIYU3XAJhIOtjf5sfyAAAA -JXRFWHRDb21tZW50AGNsaXAyZ2lmIHYuMC42IGJ5IFl2ZXMgUGlndWV0NnM7 -vAAAAABJRU5ErkJggg== ----1463757054-952513540-958744548=:8452 -Content-Type: APPLICATION/octet-stream; name="blueball.png" -Content-Transfer-Encoding: BASE64 -Content-ID: -Content-Description: A PNG graphic file -Content-Disposition: attachment; filename="blueball.png" - -iVBORw0KGgoAAAANSUhEUgAAABsAAAAbCAMAAAC6CgRnAAADAFBMVEX///8A -AAgAABAAABgAAAAACCkAEEIAEEoACDEAEFIIIXMIKXsIKYQIIWsAGFoACDkI -IWMQOZwYQqUYQq0YQrUQOaUQMZQAGFIQMYwpUrU5Y8Y5Y84pWs4YSs4YQs4Y -Qr1Ca8Z7nNacvd6Mtd5jlOcxa94hUt4YStYYQsYQMaUAACHO5+/n7++cxu9S -hO8pWucQOa1Ke86tzt6lzu9ajO8QMZxahNat1ufO7++Mve9Ke+8YOaUYSsaM -vee15++Uve8AAClajOdzpe9rnO8IKYwxY+8pWu8IIXsAAAAAAAAAAAAAAAAA -AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA -AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA -AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA -AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA -AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA -AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA -AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA -AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA -AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA -AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA -AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA -AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAADB -Mg1VAAAAAXRSTlMAQObYZgAAABZ0RVh0U29mdHdhcmUAZ2lmMnBuZyAyLjAu -MT1evmgAAAGISURBVHicddJtV5swGAbgEk6AJhBSk4bMCUynBSLaqovbrG/b -fPn/vyh70lbsscebL5xznTsh5BmNhgQoRChwo50EOIohUYLDj4zHhKYQkrEo -Qdvock4ne0IKMVUpKZLQDeqSTIsv+18PyqqWUw2IBsRM7307PPp+fDJrWtnp -LDJvewYxnewfnvanZ+fzpmwXijC8KbqEa3Fx2ff91Y95U9XCUpaDeQwiMpHX -P/v+1++bWVPWQoGFawtjury9vru/f/C1Vi7ezT0WWpQHf/7+u/G71aLThK/M -jRxmT6KdzZ9fGk9yatMsTgZLl3XVgFRAC6spj/13enssqJVtWVa3NdBSacL8 -+VZmYqKmdd1CSYoOiMOSGwtzlqqlFFIuOqv0a1ZEZrUkWICLLFW266y1KvWE -1zV/iDAH1EopnVLCiygZCIomH3NCKX0lnI+B1iuuzCGTxwXjnDO4d7NpbX42 -YJJHkBwmAm2TxwAZg40J3+Xtbv1rgOAZwG0NxW62p+lT+Yi747sD/wEUVMzY -mWkOvwAAACV0RVh0Q29tbWVudABjbGlwMmdpZiB2LjAuNiBieSBZdmVzIFBp -Z3VldDZzO7wAAAAASUVORK5CYII= ----1463757054-952513540-958744548=:8452-- diff --git a/mail/src/leap/mail/tests/rfc822.plain.message b/mail/src/leap/mail/tests/rfc822.plain.message deleted file mode 100644 index fc627c3..0000000 --- a/mail/src/leap/mail/tests/rfc822.plain.message +++ /dev/null @@ -1,66 +0,0 @@ -From pyar-bounces@python.org.ar Wed Jan 8 14:46:02 2014 -Return-Path: -X-Spam-Checker-Version: SpamAssassin 3.3.2 (2011-06-06) on spamd2.riseup.net -X-Spam-Level: ** -X-Spam-Pyzor: Reported 0 times. -X-Spam-Status: No, score=2.1 required=8.0 tests=AM_TRUNCATED,CK_419SIZE, - CK_NAIVER_NO_DNS,CK_NAIVE_NO_DNS,ENV_FROM_DIFF0,HAS_REPLY_TO,LINK_NR_TOP, - NO_REAL_NAME,RDNS_NONE,RISEUP_SPEAR_C shortcircuit=no autolearn=disabled - version=3.3.2 -Delivered-To: kali@leap.se -Received: from mx1.riseup.net (mx1-pn.riseup.net [10.0.1.33]) - (using TLSv1 with cipher DHE-RSA-AES256-SHA (256/256 bits)) - (Client CN "*.riseup.net", Issuer "Gandi Standard SSL CA" (not verified)) - by vireo.riseup.net (Postfix) with ESMTPS id 6C39A8F - for ; Wed, 8 Jan 2014 18:46:02 +0000 (UTC) -Received: from pyar.usla.org.ar (unknown [190.228.30.157]) - by mx1.riseup.net (Postfix) with ESMTP id F244C533F4 - for ; Wed, 8 Jan 2014 10:46:01 -0800 (PST) -Received: from [127.0.0.1] (localhost [127.0.0.1]) - by pyar.usla.org.ar (Postfix) with ESMTP id CC51D26A4F - for ; Wed, 8 Jan 2014 15:46:00 -0300 (ART) -MIME-Version: 1.0 -Content-Type: text/plain; charset="iso-8859-1" -Content-Transfer-Encoding: quoted-printable -From: pyar-request@python.org.ar -To: kali@leap.se -Subject: confirm 0e47e4342e4d42508e8c283175b05b3377148ac2 -Reply-To: pyar-request@python.org.ar -Auto-Submitted: auto-replied -Message-ID: -Date: Wed, 08 Jan 2014 15:45:59 -0300 -Precedence: bulk -X-BeenThere: pyar@python.org.ar -X-Mailman-Version: 2.1.15 -List-Id: Python Argentina -X-List-Administrivia: yes -Errors-To: pyar-bounces@python.org.ar -Sender: "pyar" -X-Virus-Scanned: clamav-milter 0.97.8 at mx1 -X-Virus-Status: Clean - -Mailing list subscription confirmation notice for mailing list pyar - -We have received a request de kaliyuga@riseup.net for subscription of -your email address, "kaliyuga@riseup.net", to the pyar@python.org.ar -mailing list. To confirm that you want to be added to this mailing -list, simply reply to this message, keeping the Subject: header -intact. Or visit this web page: - - http://listas.python.org.ar/confirm/pyar/0e47e4342e4d42508e8c283175b05b= -3377148ac2 - - -Or include the following line -- and only the following line -- in a -message to pyar-request@python.org.ar: - - confirm 0e47e4342e4d42508e8c283175b05b3377148ac2 - -Note that simply sending a `reply' to this message should work from -most mail readers, since that usually leaves the Subject: line in the -right form (additional "Re:" text in the Subject: is okay). - -If you do not wish to be subscribed to this list, please simply -disregard this message. If you think you are being maliciously -subscribed to the list, or have any other questions, send them to -pyar-owner@python.org.ar. diff --git a/mail/src/leap/mail/tests/test_mail.py b/mail/src/leap/mail/tests/test_mail.py deleted file mode 100644 index f9cded2..0000000 --- a/mail/src/leap/mail/tests/test_mail.py +++ /dev/null @@ -1,399 +0,0 @@ -# -*- coding: utf-8 -*- -# test_mail.py -# Copyright (C) 2014 LEAP -# -# This program is free software: you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation, either version 3 of the License, or -# (at your option) any later version. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with this program. If not, see . -""" -Tests for the mail module. -""" -import os -import time -import uuid - -from functools import partial -from email.parser import Parser -from email.Utils import formatdate - -from leap.mail.adaptors.soledad import SoledadMailAdaptor -from leap.mail.mail import MessageCollection, Account, _unpack_headers -from leap.mail.mailbox_indexer import MailboxIndexer -from leap.mail.testing.common import SoledadTestMixin - -HERE = os.path.split(os.path.abspath(__file__))[0] - - -def _get_raw_msg(multi=False): - if multi: - sample = "rfc822.multi.message" - else: - sample = "rfc822.message" - with open(os.path.join(HERE, sample)) as f: - raw = f.read() - return raw - - -def _get_parsed_msg(multi=False): - mail_parser = Parser() - raw = _get_raw_msg(multi=multi) - return mail_parser.parsestr(raw) - - -def _get_msg_time(): - timestamp = time.mktime((2010, 12, 12, 1, 1, 1, 1, 1, 1)) - return formatdate(timestamp) - - -class CollectionMixin(object): - - def get_collection(self, mbox_collection=True, mbox_name=None, - mbox_uuid=None): - """ - Get a collection for tests. - """ - adaptor = SoledadMailAdaptor() - store = self._soledad - adaptor.store = store - - if mbox_collection: - mbox_indexer = MailboxIndexer(store) - mbox_name = mbox_name or "TestMbox" - mbox_uuid = mbox_uuid or str(uuid.uuid4()) - else: - mbox_indexer = mbox_name = None - - def get_collection_from_mbox_wrapper(wrapper): - wrapper.uuid = mbox_uuid - return MessageCollection( - adaptor, store, - mbox_indexer=mbox_indexer, mbox_wrapper=wrapper) - - d = adaptor.initialize_store(store) - if mbox_collection: - d.addCallback(lambda _: mbox_indexer.create_table(mbox_uuid)) - d.addCallback(lambda _: adaptor.get_or_create_mbox(store, mbox_name)) - d.addCallback(get_collection_from_mbox_wrapper) - return d - - -# TODO profile add_msg. Why are these tests so SLOW??! -class MessageTestCase(SoledadTestMixin, CollectionMixin): - """ - Tests for the Message class. - """ - msg_flags = ('\Recent', '\Unseen', '\TestFlag') - msg_tags = ('important', 'todo', 'wonderful') - internal_date = "19-Mar-2015 19:22:21 -0500" - - maxDiff = None - - def _do_insert_msg(self, multi=False): - """ - Inserts and return a regular message, for tests. - """ - def insert_message(collection): - self._mbox_uuid = collection.mbox_uuid - return collection.add_msg( - raw, flags=self.msg_flags, tags=self.msg_tags, - date=self.internal_date) - - raw = _get_raw_msg(multi=multi) - - d = self.get_collection() - d.addCallback(insert_message) - return d - - def get_inserted_msg(self, multi=False): - d = self._do_insert_msg(multi=multi) - d.addCallback(lambda _: self.get_collection(mbox_uuid=self._mbox_uuid)) - d.addCallback(lambda col: col.get_message_by_uid(1)) - return d - - def test_get_flags(self): - d = self.get_inserted_msg() - d.addCallback(self._test_get_flags_cb) - return d - - def _test_get_flags_cb(self, msg): - self.assertTrue(msg is not None) - self.assertEquals(tuple(msg.get_flags()), self.msg_flags) - - def test_get_internal_date(self): - d = self.get_inserted_msg() - d.addCallback(self._test_get_internal_date_cb) - - def _test_get_internal_date_cb(self, msg): - self.assertTrue(msg is not None) - self.assertDictEqual(msg.get_internal_date(), - self.internal_date) - - def test_get_headers(self): - d = self.get_inserted_msg() - d.addCallback(self._test_get_headers_cb) - return d - - def _test_get_headers_cb(self, msg): - self.assertTrue(msg is not None) - expected = [ - (str(key.lower()), str(value)) - for (key, value) in _get_parsed_msg().items()] - self.assertItemsEqual(_unpack_headers(msg.get_headers()), expected) - - def test_get_body_file(self): - d = self.get_inserted_msg(multi=True) - d.addCallback(self._test_get_body_file_cb) - return d - - def _test_get_body_file_cb(self, msg): - self.assertTrue(msg is not None) - orig = _get_parsed_msg(multi=True) - expected = orig.get_payload()[0].get_payload() - d = msg.get_body_file(self._soledad) - - def assert_body(fd): - self.assertTrue(fd is not None) - self.assertEqual(fd.read(), expected) - d.addCallback(assert_body) - return d - - def test_get_size(self): - d = self.get_inserted_msg() - d.addCallback(self._test_get_size_cb) - return d - - def _test_get_size_cb(self, msg): - self.assertTrue(msg is not None) - expected = len(_get_parsed_msg().as_string()) - self.assertEqual(msg.get_size(), expected) - - def test_is_multipart_no(self): - d = self.get_inserted_msg() - d.addCallback(self._test_is_multipart_no_cb) - return d - - def _test_is_multipart_no_cb(self, msg): - self.assertTrue(msg is not None) - expected = _get_parsed_msg().is_multipart() - self.assertEqual(msg.is_multipart(), expected) - - def test_is_multipart_yes(self): - d = self.get_inserted_msg(multi=True) - d.addCallback(self._test_is_multipart_yes_cb) - return d - - def _test_is_multipart_yes_cb(self, msg): - self.assertTrue(msg is not None) - expected = _get_parsed_msg(multi=True).is_multipart() - self.assertEqual(msg.is_multipart(), expected) - - def test_get_subpart(self): - d = self.get_inserted_msg(multi=True) - d.addCallback(self._test_get_subpart_cb) - return d - - def _test_get_subpart_cb(self, msg): - self.assertTrue(msg is not None) - - def test_get_tags(self): - d = self.get_inserted_msg() - d.addCallback(self._test_get_tags_cb) - return d - - def _test_get_tags_cb(self, msg): - self.assertTrue(msg is not None) - self.assertEquals(msg.get_tags(), self.msg_tags) - - -class MessageCollectionTestCase(SoledadTestMixin, CollectionMixin): - """ - Tests for the MessageCollection class. - """ - _mbox_uuid = None - - def assert_collection_count(self, _, expected): - def _assert_count(count): - self.assertEqual(count, expected) - - d = self.get_collection() - d.addCallback(lambda col: col.count()) - d.addCallback(_assert_count) - return d - - def add_msg_to_collection(self): - raw = _get_raw_msg() - - def add_msg_to_collection(collection): - # We keep the uuid in case we need to instantiate the same - # collection afterwards. - self._mbox_uuid = collection.mbox_uuid - d = collection.add_msg(raw, date=_get_msg_time()) - return d - - d = self.get_collection() - d.addCallback(add_msg_to_collection) - return d - - def test_is_mailbox_collection(self): - d = self.get_collection() - d.addCallback(self._test_is_mailbox_collection_cb) - return d - - def _test_is_mailbox_collection_cb(self, collection): - self.assertTrue(collection.is_mailbox_collection()) - - def test_get_uid_next(self): - d = self.add_msg_to_collection() - d.addCallback(lambda _: self.get_collection()) - d.addCallback(lambda col: col.get_uid_next()) - d.addCallback(self._test_get_uid_next_cb) - - def _test_get_uid_next_cb(self, next_uid): - self.assertEqual(next_uid, 2) - - def test_add_and_count_msg(self): - d = self.add_msg_to_collection() - d.addCallback(self._test_add_and_count_msg_cb) - return d - - def _test_add_and_count_msg_cb(self, _): - return partial(self.assert_collection_count, expected=1) - - def test_copy_msg(self): - # TODO ---- update when implementing messagecopier - # interface - pass - test_copy_msg.skip = "Not yet implemented" - - def test_delete_msg(self): - d = self.add_msg_to_collection() - - def del_msg(collection): - def _delete_it(msg): - self.assertTrue(msg is not None) - return collection.delete_msg(msg) - - d = collection.get_message_by_uid(1) - d.addCallback(_delete_it) - return d - - # We need to instantiate an mbox collection with the same uuid that - # the one in which we inserted the doc. - d.addCallback(lambda _: self.get_collection(mbox_uuid=self._mbox_uuid)) - d.addCallback(del_msg) - d.addCallback(self._test_delete_msg_cb) - return d - - def _test_delete_msg_cb(self, _): - return partial(self.assert_collection_count, expected=0) - - def test_update_flags(self): - d = self.add_msg_to_collection() - d.addCallback(self._test_update_flags_cb) - return d - - def _test_update_flags_cb(self, msg): - pass - - def test_update_tags(self): - d = self.add_msg_to_collection() - d.addCallback(self._test_update_tags_cb) - return d - - def _test_update_tags_cb(self, msg): - pass - - -class AccountTestCase(SoledadTestMixin): - """ - Tests for the Account class. - """ - def get_account(self, user_id): - store = self._soledad - return Account(store, user_id) - - def test_add_mailbox(self): - acc = self.get_account('some_user_id') - d = acc.callWhenReady(lambda _: acc.add_mailbox("TestMailbox")) - d.addCallback(lambda _: acc.list_all_mailbox_names()) - d.addCallback(self._test_add_mailbox_cb) - return d - - def _test_add_mailbox_cb(self, mboxes): - expected = ['INBOX', 'TestMailbox'] - self.assertItemsEqual(mboxes, expected) - - def test_delete_mailbox(self): - acc = self.get_account('some_user_id') - d = acc.callWhenReady(lambda _: acc.delete_mailbox("Inbox")) - d.addCallback(lambda _: acc.list_all_mailbox_names()) - d.addCallback(self._test_delete_mailbox_cb) - return d - - def _test_delete_mailbox_cb(self, mboxes): - expected = [] - self.assertItemsEqual(mboxes, expected) - - def test_rename_mailbox(self): - acc = self.get_account('some_user_id') - d = acc.callWhenReady(lambda _: acc.add_mailbox("OriginalMailbox")) - d.addCallback(lambda _: acc.rename_mailbox( - "OriginalMailbox", "RenamedMailbox")) - d.addCallback(lambda _: acc.list_all_mailbox_names()) - d.addCallback(self._test_rename_mailbox_cb) - return d - - def _test_rename_mailbox_cb(self, mboxes): - expected = ['INBOX', 'RenamedMailbox'] - self.assertItemsEqual(mboxes, expected) - - def test_get_all_mailboxes(self): - acc = self.get_account('some_user_id') - d = acc.callWhenReady(lambda _: acc.add_mailbox("OneMailbox")) - d.addCallback(lambda _: acc.add_mailbox("TwoMailbox")) - d.addCallback(lambda _: acc.add_mailbox("ThreeMailbox")) - d.addCallback(lambda _: acc.add_mailbox("anotherthing")) - d.addCallback(lambda _: acc.add_mailbox("anotherthing2")) - d.addCallback(lambda _: acc.get_all_mailboxes()) - d.addCallback(self._test_get_all_mailboxes_cb) - return d - - def _test_get_all_mailboxes_cb(self, mailboxes): - expected = ["INBOX", "OneMailbox", "TwoMailbox", "ThreeMailbox", - "anotherthing", "anotherthing2"] - names = [m.mbox for m in mailboxes] - self.assertItemsEqual(names, expected) - - def test_get_collection_by_mailbox(self): - acc = self.get_account('some_user_id') - d = acc.callWhenReady(lambda _: acc.get_collection_by_mailbox("INBOX")) - d.addCallback(self._test_get_collection_by_mailbox_cb) - return d - - def _test_get_collection_by_mailbox_cb(self, collection): - self.assertTrue(collection.is_mailbox_collection()) - - def assert_uid_next_empty_collection(uid): - self.assertEqual(uid, 1) - d = collection.get_uid_next() - d.addCallback(assert_uid_next_empty_collection) - return d - - def test_get_collection_by_docs(self): - pass - - test_get_collection_by_docs.skip = "Not yet implemented" - - def test_get_collection_by_tag(self): - pass - - test_get_collection_by_tag.skip = "Not yet implemented" diff --git a/mail/src/leap/mail/tests/test_mailbox_indexer.py b/mail/src/leap/mail/tests/test_mailbox_indexer.py deleted file mode 100644 index 5c1891d..0000000 --- a/mail/src/leap/mail/tests/test_mailbox_indexer.py +++ /dev/null @@ -1,250 +0,0 @@ -# -*- coding: utf-8 -*- -# test_mailbox_indexer.py -# Copyright (C) 2014 LEAP -# -# This program is free software: you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation, either version 3 of the License, or -# (at your option) any later version. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with this program. If not, see . -""" -Tests for the mailbox_indexer module. -""" -import uuid -from functools import partial - -from leap.mail import mailbox_indexer as mi -from leap.mail.testing.common import SoledadTestMixin - -hash_test0 = '590c9f8430c7435807df8ba9a476e3f1295d46ef210f6efae2043a4c085a569e' -hash_test1 = '1b4f0e9851971998e732078544c96b36c3d01cedf7caa332359d6f1d83567014' -hash_test2 = '60303ae22b998861bce3b28f33eec1be758a213c86c93c076dbe9f558c11c752' -hash_test3 = 'fd61a03af4f77d870fc21e05e7e80678095c92d808cfb3b5c279ee04c74aca13' -hash_test4 = 'a4e624d686e03ed2767c0abd85c14426b0b1157d2ce81d27bb4fe4f6f01d688a' - - -def fmt_hash(mailbox_uuid, hash): - return "M-" + mailbox_uuid.replace('-', '_') + "-" + hash - -mbox_id = str(uuid.uuid4()) - - -class MailboxIndexerTestCase(SoledadTestMixin): - """ - Tests for the MailboxUID class. - """ - def get_mbox_uid(self): - m_uid = mi.MailboxIndexer(self._soledad) - return m_uid - - def list_mail_tables_cb(self, ignored): - def filter_mailuid_tables(tables): - filtered = [ - table[0] for table in tables if - table[0].startswith(mi.MailboxIndexer.table_preffix)] - return filtered - - sql = "SELECT name FROM sqlite_master WHERE type='table';" - d = self._soledad.raw_sqlcipher_query(sql) - d.addCallback(filter_mailuid_tables) - return d - - def select_uid_rows(self, mailbox): - sql = "SELECT * FROM %s%s;" % ( - mi.MailboxIndexer.table_preffix, mailbox.replace('-', '_')) - d = self._soledad.raw_sqlcipher_query(sql) - return d - - def test_create_table(self): - def assert_table_created(tables): - self.assertEqual( - tables, ["leapmail_uid_" + mbox_id.replace('-', '_')]) - - m_uid = self.get_mbox_uid() - d = m_uid.create_table(mbox_id) - d.addCallback(self.list_mail_tables_cb) - d.addCallback(assert_table_created) - return d - - def test_create_and_delete_table(self): - def assert_table_deleted(tables): - self.assertEqual(tables, []) - - m_uid = self.get_mbox_uid() - d = m_uid.create_table(mbox_id) - d.addCallback(lambda _: m_uid.delete_table(mbox_id)) - d.addCallback(self.list_mail_tables_cb) - d.addCallback(assert_table_deleted) - return d - - def test_insert_doc(self): - m_uid = self.get_mbox_uid() - - h1 = fmt_hash(mbox_id, hash_test0) - h2 = fmt_hash(mbox_id, hash_test1) - h3 = fmt_hash(mbox_id, hash_test2) - h4 = fmt_hash(mbox_id, hash_test3) - h5 = fmt_hash(mbox_id, hash_test4) - - def assert_uid_rows(rows): - expected = [(1, h1), (2, h2), (3, h3), (4, h4), (5, h5)] - self.assertEquals(rows, expected) - - d = m_uid.create_table(mbox_id) - d.addCallback(lambda _: m_uid.insert_doc(mbox_id, h1)) - d.addCallback(lambda _: m_uid.insert_doc(mbox_id, h2)) - d.addCallback(lambda _: m_uid.insert_doc(mbox_id, h3)) - d.addCallback(lambda _: m_uid.insert_doc(mbox_id, h4)) - d.addCallback(lambda _: m_uid.insert_doc(mbox_id, h5)) - d.addCallback(lambda _: self.select_uid_rows(mbox_id)) - d.addCallback(assert_uid_rows) - return d - - def test_insert_doc_return(self): - m_uid = self.get_mbox_uid() - - def assert_rowid(rowid, expected=None): - self.assertEqual(rowid, expected) - - h1 = fmt_hash(mbox_id, hash_test0) - h2 = fmt_hash(mbox_id, hash_test1) - h3 = fmt_hash(mbox_id, hash_test2) - - d = m_uid.create_table(mbox_id) - d.addCallback(lambda _: m_uid.insert_doc(mbox_id, h1)) - d.addCallback(partial(assert_rowid, expected=1)) - d.addCallback(lambda _: m_uid.insert_doc(mbox_id, h2)) - d.addCallback(partial(assert_rowid, expected=2)) - d.addCallback(lambda _: m_uid.insert_doc(mbox_id, h3)) - d.addCallback(partial(assert_rowid, expected=3)) - return d - - def test_delete_doc(self): - m_uid = self.get_mbox_uid() - - h1 = fmt_hash(mbox_id, hash_test0) - h2 = fmt_hash(mbox_id, hash_test1) - h3 = fmt_hash(mbox_id, hash_test2) - h4 = fmt_hash(mbox_id, hash_test3) - h5 = fmt_hash(mbox_id, hash_test4) - - def assert_uid_rows(rows): - expected = [(4, h4), (5, h5)] - self.assertEquals(rows, expected) - - d = m_uid.create_table(mbox_id) - d.addCallback(lambda _: m_uid.insert_doc(mbox_id, h1)) - d.addCallback(lambda _: m_uid.insert_doc(mbox_id, h2)) - d.addCallback(lambda _: m_uid.insert_doc(mbox_id, h3)) - d.addCallback(lambda _: m_uid.insert_doc(mbox_id, h4)) - d.addCallback(lambda _: m_uid.insert_doc(mbox_id, h5)) - - d.addCallbacks(lambda _: m_uid.delete_doc_by_uid(mbox_id, 1)) - d.addCallbacks(lambda _: m_uid.delete_doc_by_uid(mbox_id, 2)) - d.addCallbacks(lambda _: m_uid.delete_doc_by_hash(mbox_id, h3)) - - d.addCallback(lambda _: self.select_uid_rows(mbox_id)) - d.addCallback(assert_uid_rows) - return d - - def test_get_doc_id_from_uid(self): - m_uid = self.get_mbox_uid() - - h1 = fmt_hash(mbox_id, hash_test0) - - def assert_doc_hash(res): - self.assertEqual(res, h1) - - d = m_uid.create_table(mbox_id) - d.addCallback(lambda _: m_uid.insert_doc(mbox_id, h1)) - d.addCallback(lambda _: m_uid.get_doc_id_from_uid(mbox_id, 1)) - d.addCallback(assert_doc_hash) - return d - - def test_count(self): - m_uid = self.get_mbox_uid() - - h1 = fmt_hash(mbox_id, hash_test0) - h2 = fmt_hash(mbox_id, hash_test1) - h3 = fmt_hash(mbox_id, hash_test2) - h4 = fmt_hash(mbox_id, hash_test3) - h5 = fmt_hash(mbox_id, hash_test4) - - d = m_uid.create_table(mbox_id) - d.addCallback(lambda _: m_uid.insert_doc(mbox_id, h1)) - d.addCallback(lambda _: m_uid.insert_doc(mbox_id, h2)) - d.addCallback(lambda _: m_uid.insert_doc(mbox_id, h3)) - d.addCallback(lambda _: m_uid.insert_doc(mbox_id, h4)) - d.addCallback(lambda _: m_uid.insert_doc(mbox_id, h5)) - - def assert_count_after_inserts(count): - self.assertEquals(count, 5) - - d.addCallback(lambda _: m_uid.count(mbox_id)) - d.addCallback(assert_count_after_inserts) - - d.addCallbacks(lambda _: m_uid.delete_doc_by_uid(mbox_id, 1)) - d.addCallbacks(lambda _: m_uid.delete_doc_by_uid(mbox_id, 2)) - - def assert_count_after_deletions(count): - self.assertEquals(count, 3) - - d.addCallback(lambda _: m_uid.count(mbox_id)) - d.addCallback(assert_count_after_deletions) - return d - - def test_get_next_uid(self): - m_uid = self.get_mbox_uid() - - h1 = fmt_hash(mbox_id, hash_test0) - h2 = fmt_hash(mbox_id, hash_test1) - h3 = fmt_hash(mbox_id, hash_test2) - h4 = fmt_hash(mbox_id, hash_test3) - h5 = fmt_hash(mbox_id, hash_test4) - - d = m_uid.create_table(mbox_id) - d.addCallback(lambda _: m_uid.insert_doc(mbox_id, h1)) - d.addCallback(lambda _: m_uid.insert_doc(mbox_id, h2)) - d.addCallback(lambda _: m_uid.insert_doc(mbox_id, h3)) - d.addCallback(lambda _: m_uid.insert_doc(mbox_id, h4)) - d.addCallback(lambda _: m_uid.insert_doc(mbox_id, h5)) - - def assert_next_uid(result, expected=1): - self.assertEquals(result, expected) - - d.addCallback(lambda _: m_uid.get_next_uid(mbox_id)) - d.addCallback(partial(assert_next_uid, expected=6)) - return d - - def test_all_uid_iter(self): - - m_uid = self.get_mbox_uid() - - h1 = fmt_hash(mbox_id, hash_test0) - h2 = fmt_hash(mbox_id, hash_test1) - h3 = fmt_hash(mbox_id, hash_test2) - h4 = fmt_hash(mbox_id, hash_test3) - h5 = fmt_hash(mbox_id, hash_test4) - - d = m_uid.create_table(mbox_id) - d.addCallback(lambda _: m_uid.insert_doc(mbox_id, h1)) - d.addCallback(lambda _: m_uid.insert_doc(mbox_id, h2)) - d.addCallback(lambda _: m_uid.insert_doc(mbox_id, h3)) - d.addCallback(lambda _: m_uid.insert_doc(mbox_id, h4)) - d.addCallback(lambda _: m_uid.insert_doc(mbox_id, h5)) - d.addCallback(lambda _: m_uid.delete_doc_by_uid(mbox_id, 1)) - d.addCallback(lambda _: m_uid.delete_doc_by_uid(mbox_id, 4)) - - def assert_all_uid(result, expected=[2, 3, 5]): - self.assertEquals(result, expected) - - d.addCallback(lambda _: m_uid.all_uid_iter(mbox_id)) - d.addCallback(partial(assert_all_uid)) - return d diff --git a/mail/src/leap/mail/tests/test_walk.py b/mail/src/leap/mail/tests/test_walk.py deleted file mode 100644 index 826ec10..0000000 --- a/mail/src/leap/mail/tests/test_walk.py +++ /dev/null @@ -1,81 +0,0 @@ -""" -Tests for leap.mail.walk module -""" -import os.path -from email.parser import Parser - -from leap.mail import walk - -CORPUS = { - 'simple': 'rfc822.message', - 'multimin': 'rfc822.multi-minimal.message', - 'multisigned': 'rfc822.multi-signed.message', - 'bounced': 'rfc822.bounce.message', -} - -_here = os.path.dirname(__file__) -_parser = Parser() - - -# tests - - -def test_simple_mail(): - msg = _parse('simple') - tree = walk.get_tree(msg) - assert len(tree['part_map']) == 0 - assert tree['ctype'] == 'text/plain' - assert tree['multi'] is False - - -def test_multipart_minimal(): - msg = _parse('multimin') - tree = walk.get_tree(msg) - - assert tree['multi'] is True - assert len(tree['part_map']) == 1 - first = tree['part_map'][1] - assert first['multi'] is False - assert first['ctype'] == 'text/plain' - - -def test_multi_signed(): - msg = _parse('multisigned') - tree = walk.get_tree(msg) - assert tree['multi'] is True - assert len(tree['part_map']) == 2 - - _first = tree['part_map'][1] - _second = tree['part_map'][2] - assert len(_first['part_map']) == 3 - assert(_second['multi'] is False) - - -def test_bounce_mime(): - msg = _parse('bounced') - tree = walk.get_tree(msg) - - ctypes = [tree['part_map'][index]['ctype'] - for index in sorted(tree['part_map'].keys())] - third = tree['part_map'][3] - three_one_ctype = third['part_map'][1]['ctype'] - assert three_one_ctype == 'multipart/signed' - - assert ctypes == [ - 'text/plain', - 'message/delivery-status', - 'message/rfc822'] - - -# utils - -def _parse(name): - _str = _get_string_for_message(name) - return _parser.parsestr(_str) - - -def _get_string_for_message(name): - filename = os.path.join(_here, CORPUS[name]) - with open(filename) as f: - msgstr = f.read() - return msgstr diff --git a/mail/src/leap/mail/utils.py b/mail/src/leap/mail/utils.py deleted file mode 100644 index 64fca98..0000000 --- a/mail/src/leap/mail/utils.py +++ /dev/null @@ -1,375 +0,0 @@ -# -*- coding: utf-8 -*- -# utils.py -# Copyright (C) 2013 LEAP -# -# This program is free software: you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation, either version 3 of the License, or -# (at your option) any later version. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with this program. If not, see . -""" -Mail utilities. -""" -from email.utils import parseaddr -import json -import re -import traceback -import Queue - -from leap.soledad.common.document import SoledadDocument -from leap.common.check import leap_assert_type -from twisted.mail import smtp - - -CHARSET_PATTERN = r"""charset=([\w-]+)""" -CHARSET_RE = re.compile(CHARSET_PATTERN, re.IGNORECASE) - - -def first(things): - """ - Return the head of a collection. - """ - try: - return things[0] - except (IndexError, TypeError): - return None - - -def empty(thing): - """ - Return True if a thing is None or its length is zero. - If thing is a number (int, float, long), return False. - """ - if thing is None: - return True - if isinstance(thing, (int, float, long)): - return False - if isinstance(thing, SoledadDocument): - thing = thing.content - try: - return len(thing) == 0 - except (ReferenceError, TypeError): - return True - - -def maybe_call(thing): - """ - Return the same thing, or the result of its invocation if it is a - callable. - """ - return thing() if callable(thing) else thing - - -def find_charset(thing, default=None): - """ - Looks into the object 'thing' for a charset specification. - It searchs into the object's `repr`. - - :param thing: the object to look into. - :type thing: object - :param default: the dafault charset to return if no charset is found. - :type default: str - - :return: the charset or 'default' - :rtype: str or None - """ - charset = first(CHARSET_RE.findall(repr(thing))) - if charset is None: - charset = default - return charset - - -def lowerdict(_dict): - """ - Return a dict with the keys in lowercase. - - :param _dict: the dict to convert - :rtype: dict - """ - # TODO should properly implement a CaseInsensitive dict. - # Look into requests code. - return dict((key.lower(), value) - for key, value in _dict.items()) - - -PART_MAP = "part_map" -PHASH = "phash" - - -def _str_dict(d, k): - """ - Convert the dictionary key to string if it was a string. - - :param d: the dict - :type d: dict - :param k: the key - :type k: object - """ - if isinstance(k, int): - val = d[k] - d[str(k)] = val - del(d[k]) - - -def stringify_parts_map(d): - """ - Modify a dictionary making all the nested dicts under "part_map" keys - having strings as keys. - - :param d: the dictionary to modify - :type d: dictionary - :rtype: dictionary - """ - for k in d: - if k == PART_MAP: - pmap = d[k] - for kk in pmap.keys(): - _str_dict(d[k], kk) - for kk in pmap.keys(): - stringify_parts_map(d[k][str(kk)]) - return d - - -def phash_iter(d): - """ - A recursive generator that extracts all the payload-hashes - from an arbitrary nested parts-map dictionary. - - :param d: the dictionary to walk - :type d: dictionary - :return: a list of all the phashes found - :rtype: list - """ - if PHASH in d: - yield d[PHASH] - if PART_MAP in d: - for key in d[PART_MAP]: - for phash in phash_iter(d[PART_MAP][key]): - yield phash - - -def accumulator(fun, lim): - """ - A simple accumulator that uses a closure and a mutable - object to collect items. - When the count of items is greater than `lim`, the - collection is flushed after invoking a map of the function `fun` - over it. - - The returned accumulator can also be flushed at any moment - by passing a boolean as a second parameter. - - :param fun: the function to call over the collection - when its size is greater than `lim` - :type fun: callable - :param lim: the turning point for the collection - :type lim: int - :rtype: function - - >>> from pprint import pprint - >>> acc = accumulator(pprint, 2) - >>> acc(1) - >>> acc(2) - [1, 2] - >>> acc(3) - >>> acc(4) - [3, 4] - >>> acc = accumulator(pprint, 5) - >>> acc(1) - >>> acc(2) - >>> acc(3) - >>> acc(None, flush=True) - [1,2,3] - """ - KEY = "items" - _o = {KEY: []} - - def _accumulator(item, flush=False): - collection = _o[KEY] - collection.append(item) - if len(collection) >= lim or flush: - map(fun, filter(None, collection)) - _o[KEY] = [] - - return _accumulator - - -def accumulator_queue(fun, lim): - """ - A version of the accumulator that uses a queue. - - When the count of items is greater than `lim`, the - queue is flushed after invoking the function `fun` - over its items. - - The returned accumulator can also be flushed at any moment - by passing a boolean as a second parameter. - - :param fun: the function to call over the collection - when its size is greater than `lim` - :type fun: callable - :param lim: the turning point for the collection - :type lim: int - :rtype: function - """ - _q = Queue.Queue() - - def _accumulator(item, flush=False): - _q.put(item) - if _q.qsize() >= lim or flush: - collection = [_q.get() for i in range(_q.qsize())] - map(fun, filter(None, collection)) - - return _accumulator - - -def validate_address(address): - """ - Validate C{address} as defined in RFC 2822. - - :param address: The address to be validated. - :type address: str - - @return: A valid address. - @rtype: str - - @raise smtp.SMTPBadRcpt: Raised if C{address} is invalid. - """ - leap_assert_type(address, str) - # in the following, the address is parsed as described in RFC 2822 and - # ('', '') is returned if the parse fails. - _, address = parseaddr(address) - if address == '': - raise smtp.SMTPBadRcpt(address) - return address - -# -# String manipulation -# - - -class CustomJsonScanner(object): - """ - This class is a context manager definition used to monkey patch the default - json string parsing behavior. - The emails can have more than one encoding, so the `str` objects have more - than one encoding and json does not support direct work with `str` - (only `unicode`). - """ - - def _parse_string_str(self, s, idx, *args, **kwargs): - """ - Parses the string "s" starting at the point idx and returns an `str` - object. Which basically means it works exactly the same as the regular - JSON string parsing, except that it doesn't try to decode utf8. - We need this because mail raw strings might have bytes in multiple - encodings. - - :param s: the string we want to parse - :type s: str - :param idx: the starting point for parsing - :type idx: int - - :returns: the parsed string and the index where the - string ends. - :rtype: tuple (str, int) - """ - # NOTE: we just want to use this monkey patched version if we are - # calling the loads from our custom method. Otherwise, we use the - # json's default parser. - monkey_patched = False - for i in traceback.extract_stack(): - # look for json_loads method in the call stack - if i[2] == json_loads.__name__: - monkey_patched = True - break - - if not monkey_patched: - return self._orig_scanstring(s, idx, *args, **kwargs) - - # TODO profile to see if a compiled regex can get us some - # benefit here. - found = False - end = s.find("\"", idx) - while not found: - try: - if s[end - 1] != "\\": - found = True - else: - end = s.find("\"", end + 1) - except Exception: - found = True - return s[idx:end].decode("string-escape"), end + 1 - - def __enter__(self): - """ - Replace the json methods with the needed ones. - Also make a backup to restore them later. - """ - # backup original values - self._orig_make_scanner = json.scanner.make_scanner - self._orig_scanstring = json.decoder.scanstring - - # We need the make_scanner function to be the python one so we can - # monkey_patch the json string parsing - json.scanner.make_scanner = json.scanner.py_make_scanner - - # And now we monkey patch the money method - json.decoder.scanstring = self._parse_string_str - - def __exit__(self, exc_type, exc_value, traceback): - """ - Restores the backuped methods. - """ - # restore original values - json.scanner.make_scanner = self._orig_make_scanner - json.decoder.scanstring = self._orig_scanstring - - -def json_loads(data): - """ - It works as json.loads but supporting multiple encodings in the same - string and accepting an `str` parameter that won't be converted to unicode. - - :param data: the string to load the objects from - :type data: str - - :returns: the corresponding python object result of parsing 'data', this - behaves similarly as json.loads, with the exception of that - returns always `str` instead of `unicode`. - """ - obj = None - with CustomJsonScanner(): - # We need to use the cls parameter in order to trigger the code - # that will let us control the string parsing method. - obj = json.loads(data, cls=json.JSONDecoder) - - return obj - - -class CaseInsensitiveDict(dict): - """ - A dictionary subclass that will allow case-insenstive key lookups. - """ - def __init__(self, d=None): - if d is None: - d = [] - if isinstance(d, dict): - for key, value in d.items(): - self[key] = value - else: - for key, value in d: - self[key] = value - - def __setitem__(self, key, value): - super(CaseInsensitiveDict, self).__setitem__(key.lower(), value) - - def __getitem__(self, key): - return super(CaseInsensitiveDict, self).__getitem__(key.lower()) diff --git a/mail/src/leap/mail/walk.py b/mail/src/leap/mail/walk.py deleted file mode 100644 index d143d61..0000000 --- a/mail/src/leap/mail/walk.py +++ /dev/null @@ -1,107 +0,0 @@ -# -*- coding: utf-8 -*- -# walk.py -# Copyright (C) 2013-2015 LEAP -# -# This program is free software: you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation, either version 3 of the License, or -# (at your option) any later version. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with this program. If not, see . -""" -Walk a message tree and generate documents that can be inserted in the backend -store. -""" -from email.parser import Parser - -from cryptography.hazmat.backends.multibackend import MultiBackend -from cryptography.hazmat.backends.openssl.backend import ( - Backend as OpenSSLBackend) -from cryptography.hazmat.primitives import hashes - -from leap.mail.utils import first - -crypto_backend = MultiBackend([OpenSSLBackend()]) - -_parser = Parser() - - -def get_tree(msg): - p = {} - p['ctype'] = msg.get_content_type() - p['headers'] = msg.items() - - payload = msg.get_payload() - is_multi = msg.is_multipart() - if is_multi: - p['part_map'] = dict( - [(idx, get_tree(part)) for idx, part in enumerate(payload, 1)]) - p['parts'] = len(payload) - p['phash'] = None - else: - p['parts'] = 0 - p['size'] = len(payload) - p['phash'] = get_hash(payload) - p['part_map'] = {} - p['multi'] = is_multi - return p - - -def get_tree_from_string(messagestr): - return get_tree(_parser.parsestr(messagestr)) - - -def get_body_phash(msg): - """ - Find the body payload-hash for this message. - """ - for part in msg.walk(): - # XXX what other ctypes should be considered body? - if part.get_content_type() in ("text/plain", "text/html"): - # XXX avoid hashing again - return get_hash(part.get_payload()) - - -def get_raw_docs(msg): - """ - We get also some of the headers to be able to - index the content. Here we remove any mutable part, as the the filename - in the content disposition. - """ - return ( - {'type': 'cnt', - 'raw': part.get_payload(), - 'phash': get_hash(part.get_payload()), - 'content-type': part.get_content_type(), - 'content-disposition': first(part.get( - 'content-disposition', '').split(';')), - 'content-transfer-encoding': part.get( - 'content-transfer-encoding', '') - } for part in msg.walk() if not isinstance(part.get_payload(), list)) - - -def get_hash(s): - digest = hashes.Hash(hashes.SHA256(), crypto_backend) - digest.update(s) - return digest.finalize().encode("hex").upper() - - -""" -Groucho Marx: Now pay particular attention to this first clause, because it's - most important. There's the party of the first part shall be - known in this contract as the party of the first part. How do you - like that, that's pretty neat eh? - -Chico Marx: No, that's no good. -Groucho Marx: What's the matter with it? - -Chico Marx: I don't know, let's hear it again. -Groucho Marx: So the party of the first part shall be known in this contract as - the party of the first part. -""" diff --git a/mail/tox.ini b/mail/tox.ini deleted file mode 100644 index e2393f7..0000000 --- a/mail/tox.ini +++ /dev/null @@ -1,18 +0,0 @@ -[tox] -envlist = py27 - -[testenv] -commands = py.test --pep8 {posargs} -deps = - mock - pep8 - pytest - pytest-pep8 - pdbpp - setuptools-trial - -e../soledad/common - -e../soledad/client - -e../keymanager - -e. -setenv = - HOME=/tmp diff --git a/mail/versioneer.py b/mail/versioneer.py deleted file mode 100644 index 7ed2a21..0000000 --- a/mail/versioneer.py +++ /dev/null @@ -1,1774 +0,0 @@ - -# Version: 0.16 - -"""The Versioneer - like a rocketeer, but for versions. - -The Versioneer -============== - -* like a rocketeer, but for versions! -* https://github.com/warner/python-versioneer -* Brian Warner -* License: Public Domain -* Compatible With: python2.6, 2.7, 3.3, 3.4, 3.5, and pypy -* [![Latest Version] -(https://pypip.in/version/versioneer/badge.svg?style=flat) -](https://pypi.python.org/pypi/versioneer/) -* [![Build Status] -(https://travis-ci.org/warner/python-versioneer.png?branch=master) -](https://travis-ci.org/warner/python-versioneer) - -This is a tool for managing a recorded version number in distutils-based -python projects. The goal is to remove the tedious and error-prone "update -the embedded version string" step from your release process. Making a new -release should be as easy as recording a new tag in your version-control -system, and maybe making new tarballs. - - -## Quick Install - -* `pip install versioneer` to somewhere to your $PATH -* add a `[versioneer]` section to your setup.cfg (see below) -* run `versioneer install` in your source tree, commit the results - -## Version Identifiers - -Source trees come from a variety of places: - -* a version-control system checkout (mostly used by developers) -* a nightly tarball, produced by build automation -* a snapshot tarball, produced by a web-based VCS browser, like github's - "tarball from tag" feature -* a release tarball, produced by "setup.py sdist", distributed through PyPI - -Within each source tree, the version identifier (either a string or a number, -this tool is format-agnostic) can come from a variety of places: - -* ask the VCS tool itself, e.g. "git describe" (for checkouts), which knows - about recent "tags" and an absolute revision-id -* the name of the directory into which the tarball was unpacked -* an expanded VCS keyword ($Id$, etc) -* a `_version.py` created by some earlier build step - -For released software, the version identifier is closely related to a VCS -tag. Some projects use tag names that include more than just the version -string (e.g. "myproject-1.2" instead of just "1.2"), in which case the tool -needs to strip the tag prefix to extract the version identifier. For -unreleased software (between tags), the version identifier should provide -enough information to help developers recreate the same tree, while also -giving them an idea of roughly how old the tree is (after version 1.2, before -version 1.3). Many VCS systems can report a description that captures this, -for example `git describe --tags --dirty --always` reports things like -"0.7-1-g574ab98-dirty" to indicate that the checkout is one revision past the -0.7 tag, has a unique revision id of "574ab98", and is "dirty" (it has -uncommitted changes. - -The version identifier is used for multiple purposes: - -* to allow the module to self-identify its version: `myproject.__version__` -* to choose a name and prefix for a 'setup.py sdist' tarball - -## Theory of Operation - -Versioneer works by adding a special `_version.py` file into your source -tree, where your `__init__.py` can import it. This `_version.py` knows how to -dynamically ask the VCS tool for version information at import time. - -`_version.py` also contains `$Revision$` markers, and the installation -process marks `_version.py` to have this marker rewritten with a tag name -during the `git archive` command. As a result, generated tarballs will -contain enough information to get the proper version. - -To allow `setup.py` to compute a version too, a `versioneer.py` is added to -the top level of your source tree, next to `setup.py` and the `setup.cfg` -that configures it. This overrides several distutils/setuptools commands to -compute the version when invoked, and changes `setup.py build` and `setup.py -sdist` to replace `_version.py` with a small static file that contains just -the generated version data. - -## Installation - -First, decide on values for the following configuration variables: - -* `VCS`: the version control system you use. Currently accepts "git". - -* `style`: the style of version string to be produced. See "Styles" below for - details. Defaults to "pep440", which looks like - `TAG[+DISTANCE.gSHORTHASH[.dirty]]`. - -* `versionfile_source`: - - A project-relative pathname into which the generated version strings should - be written. This is usually a `_version.py` next to your project's main - `__init__.py` file, so it can be imported at runtime. If your project uses - `src/myproject/__init__.py`, this should be `src/myproject/_version.py`. - This file should be checked in to your VCS as usual: the copy created below - by `setup.py setup_versioneer` will include code that parses expanded VCS - keywords in generated tarballs. The 'build' and 'sdist' commands will - replace it with a copy that has just the calculated version string. - - This must be set even if your project does not have any modules (and will - therefore never import `_version.py`), since "setup.py sdist" -based trees - still need somewhere to record the pre-calculated version strings. Anywhere - in the source tree should do. If there is a `__init__.py` next to your - `_version.py`, the `setup.py setup_versioneer` command (described below) - will append some `__version__`-setting assignments, if they aren't already - present. - -* `versionfile_build`: - - Like `versionfile_source`, but relative to the build directory instead of - the source directory. These will differ when your setup.py uses - 'package_dir='. If you have `package_dir={'myproject': 'src/myproject'}`, - then you will probably have `versionfile_build='myproject/_version.py'` and - `versionfile_source='src/myproject/_version.py'`. - - If this is set to None, then `setup.py build` will not attempt to rewrite - any `_version.py` in the built tree. If your project does not have any - libraries (e.g. if it only builds a script), then you should use - `versionfile_build = None`. To actually use the computed version string, - your `setup.py` will need to override `distutils.command.build_scripts` - with a subclass that explicitly inserts a copy of - `versioneer.get_version()` into your script file. See - `test/demoapp-script-only/setup.py` for an example. - -* `tag_prefix`: - - a string, like 'PROJECTNAME-', which appears at the start of all VCS tags. - If your tags look like 'myproject-1.2.0', then you should use - tag_prefix='myproject-'. If you use unprefixed tags like '1.2.0', this - should be an empty string, using either `tag_prefix=` or `tag_prefix=''`. - -* `parentdir_prefix`: - - a optional string, frequently the same as tag_prefix, which appears at the - start of all unpacked tarball filenames. If your tarball unpacks into - 'myproject-1.2.0', this should be 'myproject-'. To disable this feature, - just omit the field from your `setup.cfg`. - -This tool provides one script, named `versioneer`. That script has one mode, -"install", which writes a copy of `versioneer.py` into the current directory -and runs `versioneer.py setup` to finish the installation. - -To versioneer-enable your project: - -* 1: Modify your `setup.cfg`, adding a section named `[versioneer]` and - populating it with the configuration values you decided earlier (note that - the option names are not case-sensitive): - - ```` - [versioneer] - VCS = git - style = pep440 - versionfile_source = src/myproject/_version.py - versionfile_build = myproject/_version.py - tag_prefix = - parentdir_prefix = myproject- - ```` - -* 2: Run `versioneer install`. This will do the following: - - * copy `versioneer.py` into the top of your source tree - * create `_version.py` in the right place (`versionfile_source`) - * modify your `__init__.py` (if one exists next to `_version.py`) to define - `__version__` (by calling a function from `_version.py`) - * modify your `MANIFEST.in` to include both `versioneer.py` and the - generated `_version.py` in sdist tarballs - - `versioneer install` will complain about any problems it finds with your - `setup.py` or `setup.cfg`. Run it multiple times until you have fixed all - the problems. - -* 3: add a `import versioneer` to your setup.py, and add the following - arguments to the setup() call: - - version=versioneer.get_version(), - cmdclass=versioneer.get_cmdclass(), - -* 4: commit these changes to your VCS. To make sure you won't forget, - `versioneer install` will mark everything it touched for addition using - `git add`. Don't forget to add `setup.py` and `setup.cfg` too. - -## Post-Installation Usage - -Once established, all uses of your tree from a VCS checkout should get the -current version string. All generated tarballs should include an embedded -version string (so users who unpack them will not need a VCS tool installed). - -If you distribute your project through PyPI, then the release process should -boil down to two steps: - -* 1: git tag 1.0 -* 2: python setup.py register sdist upload - -If you distribute it through github (i.e. users use github to generate -tarballs with `git archive`), the process is: - -* 1: git tag 1.0 -* 2: git push; git push --tags - -Versioneer will report "0+untagged.NUMCOMMITS.gHASH" until your tree has at -least one tag in its history. - -## Version-String Flavors - -Code which uses Versioneer can learn about its version string at runtime by -importing `_version` from your main `__init__.py` file and running the -`get_versions()` function. From the "outside" (e.g. in `setup.py`), you can -import the top-level `versioneer.py` and run `get_versions()`. - -Both functions return a dictionary with different flavors of version -information: - -* `['version']`: A condensed version string, rendered using the selected - style. This is the most commonly used value for the project's version - string. The default "pep440" style yields strings like `0.11`, - `0.11+2.g1076c97`, or `0.11+2.g1076c97.dirty`. See the "Styles" section - below for alternative styles. - -* `['full-revisionid']`: detailed revision identifier. For Git, this is the - full SHA1 commit id, e.g. "1076c978a8d3cfc70f408fe5974aa6c092c949ac". - -* `['dirty']`: a boolean, True if the tree has uncommitted changes. Note that - this is only accurate if run in a VCS checkout, otherwise it is likely to - be False or None - -* `['error']`: if the version string could not be computed, this will be set - to a string describing the problem, otherwise it will be None. It may be - useful to throw an exception in setup.py if this is set, to avoid e.g. - creating tarballs with a version string of "unknown". - -Some variants are more useful than others. Including `full-revisionid` in a -bug report should allow developers to reconstruct the exact code being tested -(or indicate the presence of local changes that should be shared with the -developers). `version` is suitable for display in an "about" box or a CLI -`--version` output: it can be easily compared against release notes and lists -of bugs fixed in various releases. - -The installer adds the following text to your `__init__.py` to place a basic -version in `YOURPROJECT.__version__`: - - from ._version import get_versions - __version__ = get_versions()['version'] - del get_versions - -## Styles - -The setup.cfg `style=` configuration controls how the VCS information is -rendered into a version string. - -The default style, "pep440", produces a PEP440-compliant string, equal to the -un-prefixed tag name for actual releases, and containing an additional "local -version" section with more detail for in-between builds. For Git, this is -TAG[+DISTANCE.gHEX[.dirty]] , using information from `git describe --tags ---dirty --always`. For example "0.11+2.g1076c97.dirty" indicates that the -tree is like the "1076c97" commit but has uncommitted changes (".dirty"), and -that this commit is two revisions ("+2") beyond the "0.11" tag. For released -software (exactly equal to a known tag), the identifier will only contain the -stripped tag, e.g. "0.11". - -Other styles are available. See details.md in the Versioneer source tree for -descriptions. - -## Debugging - -Versioneer tries to avoid fatal errors: if something goes wrong, it will tend -to return a version of "0+unknown". To investigate the problem, run `setup.py -version`, which will run the version-lookup code in a verbose mode, and will -display the full contents of `get_versions()` (including the `error` string, -which may help identify what went wrong). - -## Updating Versioneer - -To upgrade your project to a new release of Versioneer, do the following: - -* install the new Versioneer (`pip install -U versioneer` or equivalent) -* edit `setup.cfg`, if necessary, to include any new configuration settings - indicated by the release notes -* re-run `versioneer install` in your source tree, to replace - `SRC/_version.py` -* commit any changed files - -### Upgrading to 0.16 - -Nothing special. - -### Upgrading to 0.15 - -Starting with this version, Versioneer is configured with a `[versioneer]` -section in your `setup.cfg` file. Earlier versions required the `setup.py` to -set attributes on the `versioneer` module immediately after import. The new -version will refuse to run (raising an exception during import) until you -have provided the necessary `setup.cfg` section. - -In addition, the Versioneer package provides an executable named -`versioneer`, and the installation process is driven by running `versioneer -install`. In 0.14 and earlier, the executable was named -`versioneer-installer` and was run without an argument. - -### Upgrading to 0.14 - -0.14 changes the format of the version string. 0.13 and earlier used -hyphen-separated strings like "0.11-2-g1076c97-dirty". 0.14 and beyond use a -plus-separated "local version" section strings, with dot-separated -components, like "0.11+2.g1076c97". PEP440-strict tools did not like the old -format, but should be ok with the new one. - -### Upgrading from 0.11 to 0.12 - -Nothing special. - -### Upgrading from 0.10 to 0.11 - -You must add a `versioneer.VCS = "git"` to your `setup.py` before re-running -`setup.py setup_versioneer`. This will enable the use of additional -version-control systems (SVN, etc) in the future. - -## Future Directions - -This tool is designed to make it easily extended to other version-control -systems: all VCS-specific components are in separate directories like -src/git/ . The top-level `versioneer.py` script is assembled from these -components by running make-versioneer.py . In the future, make-versioneer.py -will take a VCS name as an argument, and will construct a version of -`versioneer.py` that is specific to the given VCS. It might also take the -configuration arguments that are currently provided manually during -installation by editing setup.py . Alternatively, it might go the other -direction and include code from all supported VCS systems, reducing the -number of intermediate scripts. - - -## License - -To make Versioneer easier to embed, all its code is dedicated to the public -domain. The `_version.py` that it creates is also in the public domain. -Specifically, both are released under the Creative Commons "Public Domain -Dedication" license (CC0-1.0), as described in -https://creativecommons.org/publicdomain/zero/1.0/ . - -""" - -from __future__ import print_function -try: - import configparser -except ImportError: - import ConfigParser as configparser -import errno -import json -import os -import re -import subprocess -import sys - - -class VersioneerConfig: - """Container for Versioneer configuration parameters.""" - - -def get_root(): - """Get the project root directory. - - We require that all commands are run from the project root, i.e. the - directory that contains setup.py, setup.cfg, and versioneer.py . - """ - root = os.path.realpath(os.path.abspath(os.getcwd())) - setup_py = os.path.join(root, "setup.py") - versioneer_py = os.path.join(root, "versioneer.py") - if not (os.path.exists(setup_py) or os.path.exists(versioneer_py)): - # allow 'python path/to/setup.py COMMAND' - root = os.path.dirname(os.path.realpath(os.path.abspath(sys.argv[0]))) - setup_py = os.path.join(root, "setup.py") - versioneer_py = os.path.join(root, "versioneer.py") - if not (os.path.exists(setup_py) or os.path.exists(versioneer_py)): - err = ("Versioneer was unable to run the project root directory. " - "Versioneer requires setup.py to be executed from " - "its immediate directory (like 'python setup.py COMMAND'), " - "or in a way that lets it use sys.argv[0] to find the root " - "(like 'python path/to/setup.py COMMAND').") - raise VersioneerBadRootError(err) - try: - # Certain runtime workflows (setup.py install/develop in a setuptools - # tree) execute all dependencies in a single python process, so - # "versioneer" may be imported multiple times, and python's shared - # module-import table will cache the first one. So we can't use - # os.path.dirname(__file__), as that will find whichever - # versioneer.py was first imported, even in later projects. - me = os.path.realpath(os.path.abspath(__file__)) - if os.path.splitext(me)[0] != os.path.splitext(versioneer_py)[0]: - print("Warning: build in %s is using versioneer.py from %s" - % (os.path.dirname(me), versioneer_py)) - except NameError: - pass - return root - - -def get_config_from_root(root): - """Read the project setup.cfg file to determine Versioneer config.""" - # This might raise EnvironmentError (if setup.cfg is missing), or - # configparser.NoSectionError (if it lacks a [versioneer] section), or - # configparser.NoOptionError (if it lacks "VCS="). See the docstring at - # the top of versioneer.py for instructions on writing your setup.cfg . - setup_cfg = os.path.join(root, "setup.cfg") - parser = configparser.SafeConfigParser() - with open(setup_cfg, "r") as f: - parser.readfp(f) - VCS = parser.get("versioneer", "VCS") # mandatory - - def get(parser, name): - if parser.has_option("versioneer", name): - return parser.get("versioneer", name) - return None - cfg = VersioneerConfig() - cfg.VCS = VCS - cfg.style = get(parser, "style") or "" - cfg.versionfile_source = get(parser, "versionfile_source") - cfg.versionfile_build = get(parser, "versionfile_build") - cfg.tag_prefix = get(parser, "tag_prefix") - if cfg.tag_prefix in ("''", '""'): - cfg.tag_prefix = "" - cfg.parentdir_prefix = get(parser, "parentdir_prefix") - cfg.verbose = get(parser, "verbose") - return cfg - - -class NotThisMethod(Exception): - """Exception raised if a method is not valid for the current scenario.""" - -# these dictionaries contain VCS-specific tools -LONG_VERSION_PY = {} -HANDLERS = {} - - -def register_vcs_handler(vcs, method): # decorator - """Decorator to mark a method as the handler for a particular VCS.""" - def decorate(f): - """Store f in HANDLERS[vcs][method].""" - if vcs not in HANDLERS: - HANDLERS[vcs] = {} - HANDLERS[vcs][method] = f - return f - return decorate - - -def run_command(commands, args, cwd=None, verbose=False, hide_stderr=False): - """Call the given command(s).""" - assert isinstance(commands, list) - p = None - for c in commands: - try: - dispcmd = str([c] + args) - # remember shell=False, so use git.cmd on windows, not just git - p = subprocess.Popen([c] + args, cwd=cwd, stdout=subprocess.PIPE, - stderr=(subprocess.PIPE if hide_stderr - else None)) - break - except EnvironmentError: - e = sys.exc_info()[1] - if e.errno == errno.ENOENT: - continue - if verbose: - print("unable to run %s" % dispcmd) - print(e) - return None - else: - if verbose: - print("unable to find command, tried %s" % (commands,)) - return None - stdout = p.communicate()[0].strip() - if sys.version_info[0] >= 3: - stdout = stdout.decode() - if p.returncode != 0: - if verbose: - print("unable to run %s (error)" % dispcmd) - return None - return stdout -LONG_VERSION_PY['git'] = ''' -# This file helps to compute a version number in source trees obtained from -# git-archive tarball (such as those provided by githubs download-from-tag -# feature). Distribution tarballs (built by setup.py sdist) and build -# directories (produced by setup.py build) will contain a much shorter file -# that just contains the computed version number. - -# This file is released into the public domain. Generated by -# versioneer-0.16 (https://github.com/warner/python-versioneer) - -"""Git implementation of _version.py.""" - -import errno -import os -import re -import subprocess -import sys - - -def get_keywords(): - """Get the keywords needed to look up the version information.""" - # these strings will be replaced by git during git-archive. - # setup.py/versioneer.py will grep for the variable names, so they must - # each be defined on a line of their own. _version.py will just call - # get_keywords(). - git_refnames = "%(DOLLAR)sFormat:%%d%(DOLLAR)s" - git_full = "%(DOLLAR)sFormat:%%H%(DOLLAR)s" - keywords = {"refnames": git_refnames, "full": git_full} - return keywords - - -class VersioneerConfig: - """Container for Versioneer configuration parameters.""" - - -def get_config(): - """Create, populate and return the VersioneerConfig() object.""" - # these strings are filled in when 'setup.py versioneer' creates - # _version.py - cfg = VersioneerConfig() - cfg.VCS = "git" - cfg.style = "%(STYLE)s" - cfg.tag_prefix = "%(TAG_PREFIX)s" - cfg.parentdir_prefix = "%(PARENTDIR_PREFIX)s" - cfg.versionfile_source = "%(VERSIONFILE_SOURCE)s" - cfg.verbose = False - return cfg - - -class NotThisMethod(Exception): - """Exception raised if a method is not valid for the current scenario.""" - - -LONG_VERSION_PY = {} -HANDLERS = {} - - -def register_vcs_handler(vcs, method): # decorator - """Decorator to mark a method as the handler for a particular VCS.""" - def decorate(f): - """Store f in HANDLERS[vcs][method].""" - if vcs not in HANDLERS: - HANDLERS[vcs] = {} - HANDLERS[vcs][method] = f - return f - return decorate - - -def run_command(commands, args, cwd=None, verbose=False, hide_stderr=False): - """Call the given command(s).""" - assert isinstance(commands, list) - p = None - for c in commands: - try: - dispcmd = str([c] + args) - # remember shell=False, so use git.cmd on windows, not just git - p = subprocess.Popen([c] + args, cwd=cwd, stdout=subprocess.PIPE, - stderr=(subprocess.PIPE if hide_stderr - else None)) - break - except EnvironmentError: - e = sys.exc_info()[1] - if e.errno == errno.ENOENT: - continue - if verbose: - print("unable to run %%s" %% dispcmd) - print(e) - return None - else: - if verbose: - print("unable to find command, tried %%s" %% (commands,)) - return None - stdout = p.communicate()[0].strip() - if sys.version_info[0] >= 3: - stdout = stdout.decode() - if p.returncode != 0: - if verbose: - print("unable to run %%s (error)" %% dispcmd) - return None - return stdout - - -def versions_from_parentdir(parentdir_prefix, root, verbose): - """Try to determine the version from the parent directory name. - - Source tarballs conventionally unpack into a directory that includes - both the project name and a version string. - """ - dirname = os.path.basename(root) - if not dirname.startswith(parentdir_prefix): - if verbose: - print("guessing rootdir is '%%s', but '%%s' doesn't start with " - "prefix '%%s'" %% (root, dirname, parentdir_prefix)) - raise NotThisMethod("rootdir doesn't start with parentdir_prefix") - return {"version": dirname[len(parentdir_prefix):], - "full-revisionid": None, - "dirty": False, "error": None} - - -@register_vcs_handler("git", "get_keywords") -def git_get_keywords(versionfile_abs): - """Extract version information from the given file.""" - # the code embedded in _version.py can just fetch the value of these - # keywords. When used from setup.py, we don't want to import _version.py, - # so we do it with a regexp instead. This function is not used from - # _version.py. - keywords = {} - try: - f = open(versionfile_abs, "r") - for line in f.readlines(): - if line.strip().startswith("git_refnames ="): - mo = re.search(r'=\s*"(.*)"', line) - if mo: - keywords["refnames"] = mo.group(1) - if line.strip().startswith("git_full ="): - mo = re.search(r'=\s*"(.*)"', line) - if mo: - keywords["full"] = mo.group(1) - f.close() - except EnvironmentError: - pass - return keywords - - -@register_vcs_handler("git", "keywords") -def git_versions_from_keywords(keywords, tag_prefix, verbose): - """Get version information from git keywords.""" - if not keywords: - raise NotThisMethod("no keywords at all, weird") - refnames = keywords["refnames"].strip() - if refnames.startswith("$Format"): - if verbose: - print("keywords are unexpanded, not using") - raise NotThisMethod("unexpanded keywords, not a git-archive tarball") - refs = set([r.strip() for r in refnames.strip("()").split(",")]) - # starting in git-1.8.3, tags are listed as "tag: foo-1.0" instead of - # just "foo-1.0". If we see a "tag: " prefix, prefer those. - TAG = "tag: " - tags = set([r[len(TAG):] for r in refs if r.startswith(TAG)]) - if not tags: - # Either we're using git < 1.8.3, or there really are no tags. We use - # a heuristic: assume all version tags have a digit. The old git %%d - # expansion behaves like git log --decorate=short and strips out the - # refs/heads/ and refs/tags/ prefixes that would let us distinguish - # between branches and tags. By ignoring refnames without digits, we - # filter out many common branch names like "release" and - # "stabilization", as well as "HEAD" and "master". - tags = set([r for r in refs if re.search(r'\d', r)]) - if verbose: - print("discarding '%%s', no digits" %% ",".join(refs-tags)) - if verbose: - print("likely tags: %%s" %% ",".join(sorted(tags))) - for ref in sorted(tags): - # sorting will prefer e.g. "2.0" over "2.0rc1" - if ref.startswith(tag_prefix): - r = ref[len(tag_prefix):] - if verbose: - print("picking %%s" %% r) - return {"version": r, - "full-revisionid": keywords["full"].strip(), - "dirty": False, "error": None - } - # no suitable tags, so version is "0+unknown", but full hex is still there - if verbose: - print("no suitable tags, using unknown + full revision id") - return {"version": "0+unknown", - "full-revisionid": keywords["full"].strip(), - "dirty": False, "error": "no suitable tags"} - - -@register_vcs_handler("git", "pieces_from_vcs") -def git_pieces_from_vcs(tag_prefix, root, verbose, run_command=run_command): - """Get version from 'git describe' in the root of the source tree. - - This only gets called if the git-archive 'subst' keywords were *not* - expanded, and _version.py hasn't already been rewritten with a short - version string, meaning we're inside a checked out source tree. - """ - if not os.path.exists(os.path.join(root, ".git")): - if verbose: - print("no .git in %%s" %% root) - raise NotThisMethod("no .git directory") - - GITS = ["git"] - if sys.platform == "win32": - GITS = ["git.cmd", "git.exe"] - # if there is a tag matching tag_prefix, this yields TAG-NUM-gHEX[-dirty] - # if there isn't one, this yields HEX[-dirty] (no NUM) - describe_out = run_command(GITS, ["describe", "--tags", "--dirty", - "--always", "--long", - "--match", "%%s*" %% tag_prefix], - cwd=root) - # --long was added in git-1.5.5 - if describe_out is None: - raise NotThisMethod("'git describe' failed") - describe_out = describe_out.strip() - full_out = run_command(GITS, ["rev-parse", "HEAD"], cwd=root) - if full_out is None: - raise NotThisMethod("'git rev-parse' failed") - full_out = full_out.strip() - - pieces = {} - pieces["long"] = full_out - pieces["short"] = full_out[:7] # maybe improved later - pieces["error"] = None - - # parse describe_out. It will be like TAG-NUM-gHEX[-dirty] or HEX[-dirty] - # TAG might have hyphens. - git_describe = describe_out - - # look for -dirty suffix - dirty = git_describe.endswith("-dirty") - pieces["dirty"] = dirty - if dirty: - git_describe = git_describe[:git_describe.rindex("-dirty")] - - # now we have TAG-NUM-gHEX or HEX - - if "-" in git_describe: - # TAG-NUM-gHEX - mo = re.search(r'^(.+)-(\d+)-g([0-9a-f]+)$', git_describe) - if not mo: - # unparseable. Maybe git-describe is misbehaving? - pieces["error"] = ("unable to parse git-describe output: '%%s'" - %% describe_out) - return pieces - - # tag - full_tag = mo.group(1) - if not full_tag.startswith(tag_prefix): - if verbose: - fmt = "tag '%%s' doesn't start with prefix '%%s'" - print(fmt %% (full_tag, tag_prefix)) - pieces["error"] = ("tag '%%s' doesn't start with prefix '%%s'" - %% (full_tag, tag_prefix)) - return pieces - pieces["closest-tag"] = full_tag[len(tag_prefix):] - - # distance: number of commits since tag - pieces["distance"] = int(mo.group(2)) - - # commit: short hex revision ID - pieces["short"] = mo.group(3) - - else: - # HEX: no tags - pieces["closest-tag"] = None - count_out = run_command(GITS, ["rev-list", "HEAD", "--count"], - cwd=root) - pieces["distance"] = int(count_out) # total number of commits - - return pieces - - -def plus_or_dot(pieces): - """Return a + if we don't already have one, else return a .""" - if "+" in pieces.get("closest-tag", ""): - return "." - return "+" - - -def render_pep440(pieces): - """Build up version string, with post-release "local version identifier". - - Our goal: TAG[+DISTANCE.gHEX[.dirty]] . Note that if you - get a tagged build and then dirty it, you'll get TAG+0.gHEX.dirty - - Exceptions: - 1: no tags. git_describe was just HEX. 0+untagged.DISTANCE.gHEX[.dirty] - """ - if pieces["closest-tag"]: - rendered = pieces["closest-tag"] - if pieces["distance"] or pieces["dirty"]: - rendered += plus_or_dot(pieces) - rendered += "%%d.g%%s" %% (pieces["distance"], pieces["short"]) - if pieces["dirty"]: - rendered += ".dirty" - else: - # exception #1 - rendered = "0+untagged.%%d.g%%s" %% (pieces["distance"], - pieces["short"]) - if pieces["dirty"]: - rendered += ".dirty" - return rendered - - -def render_pep440_pre(pieces): - """TAG[.post.devDISTANCE] -- No -dirty. - - Exceptions: - 1: no tags. 0.post.devDISTANCE - """ - if pieces["closest-tag"]: - rendered = pieces["closest-tag"] - if pieces["distance"]: - rendered += ".post.dev%%d" %% pieces["distance"] - else: - # exception #1 - rendered = "0.post.dev%%d" %% pieces["distance"] - return rendered - - -def render_pep440_post(pieces): - """TAG[.postDISTANCE[.dev0]+gHEX] . - - The ".dev0" means dirty. Note that .dev0 sorts backwards - (a dirty tree will appear "older" than the corresponding clean one), - but you shouldn't be releasing software with -dirty anyways. - - Exceptions: - 1: no tags. 0.postDISTANCE[.dev0] - """ - if pieces["closest-tag"]: - rendered = pieces["closest-tag"] - if pieces["distance"] or pieces["dirty"]: - rendered += ".post%%d" %% pieces["distance"] - if pieces["dirty"]: - rendered += ".dev0" - rendered += plus_or_dot(pieces) - rendered += "g%%s" %% pieces["short"] - else: - # exception #1 - rendered = "0.post%%d" %% pieces["distance"] - if pieces["dirty"]: - rendered += ".dev0" - rendered += "+g%%s" %% pieces["short"] - return rendered - - -def render_pep440_old(pieces): - """TAG[.postDISTANCE[.dev0]] . - - The ".dev0" means dirty. - - Eexceptions: - 1: no tags. 0.postDISTANCE[.dev0] - """ - if pieces["closest-tag"]: - rendered = pieces["closest-tag"] - if pieces["distance"] or pieces["dirty"]: - rendered += ".post%%d" %% pieces["distance"] - if pieces["dirty"]: - rendered += ".dev0" - else: - # exception #1 - rendered = "0.post%%d" %% pieces["distance"] - if pieces["dirty"]: - rendered += ".dev0" - return rendered - - -def render_git_describe(pieces): - """TAG[-DISTANCE-gHEX][-dirty]. - - Like 'git describe --tags --dirty --always'. - - Exceptions: - 1: no tags. HEX[-dirty] (note: no 'g' prefix) - """ - if pieces["closest-tag"]: - rendered = pieces["closest-tag"] - if pieces["distance"]: - rendered += "-%%d-g%%s" %% (pieces["distance"], pieces["short"]) - else: - # exception #1 - rendered = pieces["short"] - if pieces["dirty"]: - rendered += "-dirty" - return rendered - - -def render_git_describe_long(pieces): - """TAG-DISTANCE-gHEX[-dirty]. - - Like 'git describe --tags --dirty --always -long'. - The distance/hash is unconditional. - - Exceptions: - 1: no tags. HEX[-dirty] (note: no 'g' prefix) - """ - if pieces["closest-tag"]: - rendered = pieces["closest-tag"] - rendered += "-%%d-g%%s" %% (pieces["distance"], pieces["short"]) - else: - # exception #1 - rendered = pieces["short"] - if pieces["dirty"]: - rendered += "-dirty" - return rendered - - -def render(pieces, style): - """Render the given version pieces into the requested style.""" - if pieces["error"]: - return {"version": "unknown", - "full-revisionid": pieces.get("long"), - "dirty": None, - "error": pieces["error"]} - - if not style or style == "default": - style = "pep440" # the default - - if style == "pep440": - rendered = render_pep440(pieces) - elif style == "pep440-pre": - rendered = render_pep440_pre(pieces) - elif style == "pep440-post": - rendered = render_pep440_post(pieces) - elif style == "pep440-old": - rendered = render_pep440_old(pieces) - elif style == "git-describe": - rendered = render_git_describe(pieces) - elif style == "git-describe-long": - rendered = render_git_describe_long(pieces) - else: - raise ValueError("unknown style '%%s'" %% style) - - return {"version": rendered, "full-revisionid": pieces["long"], - "dirty": pieces["dirty"], "error": None} - - -def get_versions(): - """Get version information or return default if unable to do so.""" - # I am in _version.py, which lives at ROOT/VERSIONFILE_SOURCE. If we have - # __file__, we can work backwards from there to the root. Some - # py2exe/bbfreeze/non-CPython implementations don't do __file__, in which - # case we can only use expanded keywords. - - cfg = get_config() - verbose = cfg.verbose - - try: - return git_versions_from_keywords(get_keywords(), cfg.tag_prefix, - verbose) - except NotThisMethod: - pass - - try: - root = os.path.realpath(__file__) - # versionfile_source is the relative path from the top of the source - # tree (where the .git directory might live) to this file. Invert - # this to find the root from __file__. - for i in cfg.versionfile_source.split('/'): - root = os.path.dirname(root) - except NameError: - return {"version": "0+unknown", "full-revisionid": None, - "dirty": None, - "error": "unable to find root of source tree"} - - try: - pieces = git_pieces_from_vcs(cfg.tag_prefix, root, verbose) - return render(pieces, cfg.style) - except NotThisMethod: - pass - - try: - if cfg.parentdir_prefix: - return versions_from_parentdir(cfg.parentdir_prefix, root, verbose) - except NotThisMethod: - pass - - return {"version": "0+unknown", "full-revisionid": None, - "dirty": None, - "error": "unable to compute version"} -''' - - -@register_vcs_handler("git", "get_keywords") -def git_get_keywords(versionfile_abs): - """Extract version information from the given file.""" - # the code embedded in _version.py can just fetch the value of these - # keywords. When used from setup.py, we don't want to import _version.py, - # so we do it with a regexp instead. This function is not used from - # _version.py. - keywords = {} - try: - f = open(versionfile_abs, "r") - for line in f.readlines(): - if line.strip().startswith("git_refnames ="): - mo = re.search(r'=\s*"(.*)"', line) - if mo: - keywords["refnames"] = mo.group(1) - if line.strip().startswith("git_full ="): - mo = re.search(r'=\s*"(.*)"', line) - if mo: - keywords["full"] = mo.group(1) - f.close() - except EnvironmentError: - pass - return keywords - - -@register_vcs_handler("git", "keywords") -def git_versions_from_keywords(keywords, tag_prefix, verbose): - """Get version information from git keywords.""" - if not keywords: - raise NotThisMethod("no keywords at all, weird") - refnames = keywords["refnames"].strip() - if refnames.startswith("$Format"): - if verbose: - print("keywords are unexpanded, not using") - raise NotThisMethod("unexpanded keywords, not a git-archive tarball") - refs = set([r.strip() for r in refnames.strip("()").split(",")]) - # starting in git-1.8.3, tags are listed as "tag: foo-1.0" instead of - # just "foo-1.0". If we see a "tag: " prefix, prefer those. - TAG = "tag: " - tags = set([r[len(TAG):] for r in refs if r.startswith(TAG)]) - if not tags: - # Either we're using git < 1.8.3, or there really are no tags. We use - # a heuristic: assume all version tags have a digit. The old git %d - # expansion behaves like git log --decorate=short and strips out the - # refs/heads/ and refs/tags/ prefixes that would let us distinguish - # between branches and tags. By ignoring refnames without digits, we - # filter out many common branch names like "release" and - # "stabilization", as well as "HEAD" and "master". - tags = set([r for r in refs if re.search(r'\d', r)]) - if verbose: - print("discarding '%s', no digits" % ",".join(refs-tags)) - if verbose: - print("likely tags: %s" % ",".join(sorted(tags))) - for ref in sorted(tags): - # sorting will prefer e.g. "2.0" over "2.0rc1" - if ref.startswith(tag_prefix): - r = ref[len(tag_prefix):] - if verbose: - print("picking %s" % r) - return {"version": r, - "full-revisionid": keywords["full"].strip(), - "dirty": False, "error": None - } - # no suitable tags, so version is "0+unknown", but full hex is still there - if verbose: - print("no suitable tags, using unknown + full revision id") - return {"version": "0+unknown", - "full-revisionid": keywords["full"].strip(), - "dirty": False, "error": "no suitable tags"} - - -@register_vcs_handler("git", "pieces_from_vcs") -def git_pieces_from_vcs(tag_prefix, root, verbose, run_command=run_command): - """Get version from 'git describe' in the root of the source tree. - - This only gets called if the git-archive 'subst' keywords were *not* - expanded, and _version.py hasn't already been rewritten with a short - version string, meaning we're inside a checked out source tree. - """ - if not os.path.exists(os.path.join(root, ".git")): - if verbose: - print("no .git in %s" % root) - raise NotThisMethod("no .git directory") - - GITS = ["git"] - if sys.platform == "win32": - GITS = ["git.cmd", "git.exe"] - # if there is a tag matching tag_prefix, this yields TAG-NUM-gHEX[-dirty] - # if there isn't one, this yields HEX[-dirty] (no NUM) - describe_out = run_command(GITS, ["describe", "--tags", "--dirty", - "--always", "--long", - "--match", "%s*" % tag_prefix], - cwd=root) - # --long was added in git-1.5.5 - if describe_out is None: - raise NotThisMethod("'git describe' failed") - describe_out = describe_out.strip() - full_out = run_command(GITS, ["rev-parse", "HEAD"], cwd=root) - if full_out is None: - raise NotThisMethod("'git rev-parse' failed") - full_out = full_out.strip() - - pieces = {} - pieces["long"] = full_out - pieces["short"] = full_out[:7] # maybe improved later - pieces["error"] = None - - # parse describe_out. It will be like TAG-NUM-gHEX[-dirty] or HEX[-dirty] - # TAG might have hyphens. - git_describe = describe_out - - # look for -dirty suffix - dirty = git_describe.endswith("-dirty") - pieces["dirty"] = dirty - if dirty: - git_describe = git_describe[:git_describe.rindex("-dirty")] - - # now we have TAG-NUM-gHEX or HEX - - if "-" in git_describe: - # TAG-NUM-gHEX - mo = re.search(r'^(.+)-(\d+)-g([0-9a-f]+)$', git_describe) - if not mo: - # unparseable. Maybe git-describe is misbehaving? - pieces["error"] = ("unable to parse git-describe output: '%s'" - % describe_out) - return pieces - - # tag - full_tag = mo.group(1) - if not full_tag.startswith(tag_prefix): - if verbose: - fmt = "tag '%s' doesn't start with prefix '%s'" - print(fmt % (full_tag, tag_prefix)) - pieces["error"] = ("tag '%s' doesn't start with prefix '%s'" - % (full_tag, tag_prefix)) - return pieces - pieces["closest-tag"] = full_tag[len(tag_prefix):] - - # distance: number of commits since tag - pieces["distance"] = int(mo.group(2)) - - # commit: short hex revision ID - pieces["short"] = mo.group(3) - - else: - # HEX: no tags - pieces["closest-tag"] = None - count_out = run_command(GITS, ["rev-list", "HEAD", "--count"], - cwd=root) - pieces["distance"] = int(count_out) # total number of commits - - return pieces - - -def do_vcs_install(manifest_in, versionfile_source, ipy): - """Git-specific installation logic for Versioneer. - - For Git, this means creating/changing .gitattributes to mark _version.py - for export-time keyword substitution. - """ - GITS = ["git"] - if sys.platform == "win32": - GITS = ["git.cmd", "git.exe"] - files = [manifest_in, versionfile_source] - if ipy: - files.append(ipy) - try: - me = __file__ - if me.endswith(".pyc") or me.endswith(".pyo"): - me = os.path.splitext(me)[0] + ".py" - versioneer_file = os.path.relpath(me) - except NameError: - versioneer_file = "versioneer.py" - files.append(versioneer_file) - present = False - try: - f = open(".gitattributes", "r") - for line in f.readlines(): - if line.strip().startswith(versionfile_source): - if "export-subst" in line.strip().split()[1:]: - present = True - f.close() - except EnvironmentError: - pass - if not present: - f = open(".gitattributes", "a+") - f.write("%s export-subst\n" % versionfile_source) - f.close() - files.append(".gitattributes") - run_command(GITS, ["add", "--"] + files) - - -def versions_from_parentdir(parentdir_prefix, root, verbose): - """Try to determine the version from the parent directory name. - - Source tarballs conventionally unpack into a directory that includes - both the project name and a version string. - """ - dirname = os.path.basename(root) - if not dirname.startswith(parentdir_prefix): - if verbose: - print("guessing rootdir is '%s', but '%s' doesn't start with " - "prefix '%s'" % (root, dirname, parentdir_prefix)) - raise NotThisMethod("rootdir doesn't start with parentdir_prefix") - return {"version": dirname[len(parentdir_prefix):], - "full-revisionid": None, - "dirty": False, "error": None} - -SHORT_VERSION_PY = """ -# This file was generated by 'versioneer.py' (0.16) from -# revision-control system data, or from the parent directory name of an -# unpacked source archive. Distribution tarballs contain a pre-generated copy -# of this file. - -import json -import sys - -version_json = ''' -%s -''' # END VERSION_JSON - - -def get_versions(): - return json.loads(version_json) -""" - - -def versions_from_file(filename): - """Try to determine the version from _version.py if present.""" - try: - with open(filename) as f: - contents = f.read() - except EnvironmentError: - raise NotThisMethod("unable to read _version.py") - mo = re.search(r"version_json = '''\n(.*)''' # END VERSION_JSON", - contents, re.M | re.S) - if not mo: - raise NotThisMethod("no version_json in _version.py") - return json.loads(mo.group(1)) - - -def write_to_version_file(filename, versions): - """Write the given version number to the given _version.py file.""" - os.unlink(filename) - contents = json.dumps(versions, sort_keys=True, - indent=1, separators=(",", ": ")) - with open(filename, "w") as f: - f.write(SHORT_VERSION_PY % contents) - - print("set %s to '%s'" % (filename, versions["version"])) - - -def plus_or_dot(pieces): - """Return a + if we don't already have one, else return a .""" - if "+" in pieces.get("closest-tag", ""): - return "." - return "+" - - -def render_pep440(pieces): - """Build up version string, with post-release "local version identifier". - - Our goal: TAG[+DISTANCE.gHEX[.dirty]] . Note that if you - get a tagged build and then dirty it, you'll get TAG+0.gHEX.dirty - - Exceptions: - 1: no tags. git_describe was just HEX. 0+untagged.DISTANCE.gHEX[.dirty] - """ - if pieces["closest-tag"]: - rendered = pieces["closest-tag"] - if pieces["distance"] or pieces["dirty"]: - rendered += plus_or_dot(pieces) - rendered += "%d.g%s" % (pieces["distance"], pieces["short"]) - if pieces["dirty"]: - rendered += ".dirty" - else: - # exception #1 - rendered = "0+untagged.%d.g%s" % (pieces["distance"], - pieces["short"]) - if pieces["dirty"]: - rendered += ".dirty" - return rendered - - -def render_pep440_pre(pieces): - """TAG[.post.devDISTANCE] -- No -dirty. - - Exceptions: - 1: no tags. 0.post.devDISTANCE - """ - if pieces["closest-tag"]: - rendered = pieces["closest-tag"] - if pieces["distance"]: - rendered += ".post.dev%d" % pieces["distance"] - else: - # exception #1 - rendered = "0.post.dev%d" % pieces["distance"] - return rendered - - -def render_pep440_post(pieces): - """TAG[.postDISTANCE[.dev0]+gHEX] . - - The ".dev0" means dirty. Note that .dev0 sorts backwards - (a dirty tree will appear "older" than the corresponding clean one), - but you shouldn't be releasing software with -dirty anyways. - - Exceptions: - 1: no tags. 0.postDISTANCE[.dev0] - """ - if pieces["closest-tag"]: - rendered = pieces["closest-tag"] - if pieces["distance"] or pieces["dirty"]: - rendered += ".post%d" % pieces["distance"] - if pieces["dirty"]: - rendered += ".dev0" - rendered += plus_or_dot(pieces) - rendered += "g%s" % pieces["short"] - else: - # exception #1 - rendered = "0.post%d" % pieces["distance"] - if pieces["dirty"]: - rendered += ".dev0" - rendered += "+g%s" % pieces["short"] - return rendered - - -def render_pep440_old(pieces): - """TAG[.postDISTANCE[.dev0]] . - - The ".dev0" means dirty. - - Eexceptions: - 1: no tags. 0.postDISTANCE[.dev0] - """ - if pieces["closest-tag"]: - rendered = pieces["closest-tag"] - if pieces["distance"] or pieces["dirty"]: - rendered += ".post%d" % pieces["distance"] - if pieces["dirty"]: - rendered += ".dev0" - else: - # exception #1 - rendered = "0.post%d" % pieces["distance"] - if pieces["dirty"]: - rendered += ".dev0" - return rendered - - -def render_git_describe(pieces): - """TAG[-DISTANCE-gHEX][-dirty]. - - Like 'git describe --tags --dirty --always'. - - Exceptions: - 1: no tags. HEX[-dirty] (note: no 'g' prefix) - """ - if pieces["closest-tag"]: - rendered = pieces["closest-tag"] - if pieces["distance"]: - rendered += "-%d-g%s" % (pieces["distance"], pieces["short"]) - else: - # exception #1 - rendered = pieces["short"] - if pieces["dirty"]: - rendered += "-dirty" - return rendered - - -def render_git_describe_long(pieces): - """TAG-DISTANCE-gHEX[-dirty]. - - Like 'git describe --tags --dirty --always -long'. - The distance/hash is unconditional. - - Exceptions: - 1: no tags. HEX[-dirty] (note: no 'g' prefix) - """ - if pieces["closest-tag"]: - rendered = pieces["closest-tag"] - rendered += "-%d-g%s" % (pieces["distance"], pieces["short"]) - else: - # exception #1 - rendered = pieces["short"] - if pieces["dirty"]: - rendered += "-dirty" - return rendered - - -def render(pieces, style): - """Render the given version pieces into the requested style.""" - if pieces["error"]: - return {"version": "unknown", - "full-revisionid": pieces.get("long"), - "dirty": None, - "error": pieces["error"]} - - if not style or style == "default": - style = "pep440" # the default - - if style == "pep440": - rendered = render_pep440(pieces) - elif style == "pep440-pre": - rendered = render_pep440_pre(pieces) - elif style == "pep440-post": - rendered = render_pep440_post(pieces) - elif style == "pep440-old": - rendered = render_pep440_old(pieces) - elif style == "git-describe": - rendered = render_git_describe(pieces) - elif style == "git-describe-long": - rendered = render_git_describe_long(pieces) - else: - raise ValueError("unknown style '%s'" % style) - - return {"version": rendered, "full-revisionid": pieces["long"], - "dirty": pieces["dirty"], "error": None} - - -class VersioneerBadRootError(Exception): - """The project root directory is unknown or missing key files.""" - - -def get_versions(verbose=False): - """Get the project version from whatever source is available. - - Returns dict with two keys: 'version' and 'full'. - """ - if "versioneer" in sys.modules: - # see the discussion in cmdclass.py:get_cmdclass() - del sys.modules["versioneer"] - - root = get_root() - cfg = get_config_from_root(root) - - assert cfg.VCS is not None, "please set [versioneer]VCS= in setup.cfg" - handlers = HANDLERS.get(cfg.VCS) - assert handlers, "unrecognized VCS '%s'" % cfg.VCS - verbose = verbose or cfg.verbose - assert cfg.versionfile_source is not None, \ - "please set versioneer.versionfile_source" - assert cfg.tag_prefix is not None, "please set versioneer.tag_prefix" - - versionfile_abs = os.path.join(root, cfg.versionfile_source) - - # extract version from first of: _version.py, VCS command (e.g. 'git - # describe'), parentdir. This is meant to work for developers using a - # source checkout, for users of a tarball created by 'setup.py sdist', - # and for users of a tarball/zipball created by 'git archive' or github's - # download-from-tag feature or the equivalent in other VCSes. - - get_keywords_f = handlers.get("get_keywords") - from_keywords_f = handlers.get("keywords") - if get_keywords_f and from_keywords_f: - try: - keywords = get_keywords_f(versionfile_abs) - ver = from_keywords_f(keywords, cfg.tag_prefix, verbose) - if verbose: - print("got version from expanded keyword %s" % ver) - return ver - except NotThisMethod: - pass - - try: - ver = versions_from_file(versionfile_abs) - if verbose: - print("got version from file %s %s" % (versionfile_abs, ver)) - return ver - except NotThisMethod: - pass - - from_vcs_f = handlers.get("pieces_from_vcs") - if from_vcs_f: - try: - pieces = from_vcs_f(cfg.tag_prefix, root, verbose) - ver = render(pieces, cfg.style) - if verbose: - print("got version from VCS %s" % ver) - return ver - except NotThisMethod: - pass - - try: - if cfg.parentdir_prefix: - ver = versions_from_parentdir(cfg.parentdir_prefix, root, verbose) - if verbose: - print("got version from parentdir %s" % ver) - return ver - except NotThisMethod: - pass - - if verbose: - print("unable to compute version") - - return {"version": "0+unknown", "full-revisionid": None, - "dirty": None, "error": "unable to compute version"} - - -def get_version(): - """Get the short version string for this project.""" - return get_versions()["version"] - - -def get_cmdclass(): - """Get the custom setuptools/distutils subclasses used by Versioneer.""" - if "versioneer" in sys.modules: - del sys.modules["versioneer"] - # this fixes the "python setup.py develop" case (also 'install' and - # 'easy_install .'), in which subdependencies of the main project are - # built (using setup.py bdist_egg) in the same python process. Assume - # a main project A and a dependency B, which use different versions - # of Versioneer. A's setup.py imports A's Versioneer, leaving it in - # sys.modules by the time B's setup.py is executed, causing B to run - # with the wrong versioneer. Setuptools wraps the sub-dep builds in a - # sandbox that restores sys.modules to it's pre-build state, so the - # parent is protected against the child's "import versioneer". By - # removing ourselves from sys.modules here, before the child build - # happens, we protect the child from the parent's versioneer too. - # Also see https://github.com/warner/python-versioneer/issues/52 - - cmds = {} - - # we add "version" to both distutils and setuptools - from distutils.core import Command - - class cmd_version(Command): - description = "report generated version string" - user_options = [] - boolean_options = [] - - def initialize_options(self): - pass - - def finalize_options(self): - pass - - def run(self): - vers = get_versions(verbose=True) - print("Version: %s" % vers["version"]) - print(" full-revisionid: %s" % vers.get("full-revisionid")) - print(" dirty: %s" % vers.get("dirty")) - if vers["error"]: - print(" error: %s" % vers["error"]) - cmds["version"] = cmd_version - - # we override "build_py" in both distutils and setuptools - # - # most invocation pathways end up running build_py: - # distutils/build -> build_py - # distutils/install -> distutils/build ->.. - # setuptools/bdist_wheel -> distutils/install ->.. - # setuptools/bdist_egg -> distutils/install_lib -> build_py - # setuptools/install -> bdist_egg ->.. - # setuptools/develop -> ? - - # we override different "build_py" commands for both environments - if "setuptools" in sys.modules: - from setuptools.command.build_py import build_py as _build_py - else: - from distutils.command.build_py import build_py as _build_py - - class cmd_build_py(_build_py): - def run(self): - root = get_root() - cfg = get_config_from_root(root) - versions = get_versions() - _build_py.run(self) - # now locate _version.py in the new build/ directory and replace - # it with an updated value - if cfg.versionfile_build: - target_versionfile = os.path.join(self.build_lib, - cfg.versionfile_build) - print("UPDATING %s" % target_versionfile) - write_to_version_file(target_versionfile, versions) - cmds["build_py"] = cmd_build_py - - if "cx_Freeze" in sys.modules: # cx_freeze enabled? - from cx_Freeze.dist import build_exe as _build_exe - - class cmd_build_exe(_build_exe): - def run(self): - root = get_root() - cfg = get_config_from_root(root) - versions = get_versions() - target_versionfile = cfg.versionfile_source - print("UPDATING %s" % target_versionfile) - write_to_version_file(target_versionfile, versions) - - _build_exe.run(self) - os.unlink(target_versionfile) - with open(cfg.versionfile_source, "w") as f: - LONG = LONG_VERSION_PY[cfg.VCS] - f.write(LONG % - {"DOLLAR": "$", - "STYLE": cfg.style, - "TAG_PREFIX": cfg.tag_prefix, - "PARENTDIR_PREFIX": cfg.parentdir_prefix, - "VERSIONFILE_SOURCE": cfg.versionfile_source, - }) - cmds["build_exe"] = cmd_build_exe - del cmds["build_py"] - - # we override different "sdist" commands for both environments - if "setuptools" in sys.modules: - from setuptools.command.sdist import sdist as _sdist - else: - from distutils.command.sdist import sdist as _sdist - - class cmd_sdist(_sdist): - def run(self): - versions = get_versions() - self._versioneer_generated_versions = versions - # unless we update this, the command will keep using the old - # version - self.distribution.metadata.version = versions["version"] - return _sdist.run(self) - - def make_release_tree(self, base_dir, files): - root = get_root() - cfg = get_config_from_root(root) - _sdist.make_release_tree(self, base_dir, files) - # now locate _version.py in the new base_dir directory - # (remembering that it may be a hardlink) and replace it with an - # updated value - target_versionfile = os.path.join(base_dir, cfg.versionfile_source) - print("UPDATING %s" % target_versionfile) - write_to_version_file(target_versionfile, - self._versioneer_generated_versions) - cmds["sdist"] = cmd_sdist - - return cmds - - -CONFIG_ERROR = """ -setup.cfg is missing the necessary Versioneer configuration. You need -a section like: - - [versioneer] - VCS = git - style = pep440 - versionfile_source = src/myproject/_version.py - versionfile_build = myproject/_version.py - tag_prefix = - parentdir_prefix = myproject- - -You will also need to edit your setup.py to use the results: - - import versioneer - setup(version=versioneer.get_version(), - cmdclass=versioneer.get_cmdclass(), ...) - -Please read the docstring in ./versioneer.py for configuration instructions, -edit setup.cfg, and re-run the installer or 'python versioneer.py setup'. -""" - -SAMPLE_CONFIG = """ -# See the docstring in versioneer.py for instructions. Note that you must -# re-run 'versioneer.py setup' after changing this section, and commit the -# resulting files. - -[versioneer] -#VCS = git -#style = pep440 -#versionfile_source = -#versionfile_build = -#tag_prefix = -#parentdir_prefix = - -""" - -INIT_PY_SNIPPET = """ -from ._version import get_versions -__version__ = get_versions()['version'] -del get_versions -""" - - -def do_setup(): - """Main VCS-independent setup function for installing Versioneer.""" - root = get_root() - try: - cfg = get_config_from_root(root) - except (EnvironmentError, configparser.NoSectionError, - configparser.NoOptionError) as e: - if isinstance(e, (EnvironmentError, configparser.NoSectionError)): - print("Adding sample versioneer config to setup.cfg", - file=sys.stderr) - with open(os.path.join(root, "setup.cfg"), "a") as f: - f.write(SAMPLE_CONFIG) - print(CONFIG_ERROR, file=sys.stderr) - return 1 - - print(" creating %s" % cfg.versionfile_source) - with open(cfg.versionfile_source, "w") as f: - LONG = LONG_VERSION_PY[cfg.VCS] - f.write(LONG % {"DOLLAR": "$", - "STYLE": cfg.style, - "TAG_PREFIX": cfg.tag_prefix, - "PARENTDIR_PREFIX": cfg.parentdir_prefix, - "VERSIONFILE_SOURCE": cfg.versionfile_source, - }) - - ipy = os.path.join(os.path.dirname(cfg.versionfile_source), - "__init__.py") - if os.path.exists(ipy): - try: - with open(ipy, "r") as f: - old = f.read() - except EnvironmentError: - old = "" - if INIT_PY_SNIPPET not in old: - print(" appending to %s" % ipy) - with open(ipy, "a") as f: - f.write(INIT_PY_SNIPPET) - else: - print(" %s unmodified" % ipy) - else: - print(" %s doesn't exist, ok" % ipy) - ipy = None - - # Make sure both the top-level "versioneer.py" and versionfile_source - # (PKG/_version.py, used by runtime code) are in MANIFEST.in, so - # they'll be copied into source distributions. Pip won't be able to - # install the package without this. - manifest_in = os.path.join(root, "MANIFEST.in") - simple_includes = set() - try: - with open(manifest_in, "r") as f: - for line in f: - if line.startswith("include "): - for include in line.split()[1:]: - simple_includes.add(include) - except EnvironmentError: - pass - # That doesn't cover everything MANIFEST.in can do - # (http://docs.python.org/2/distutils/sourcedist.html#commands), so - # it might give some false negatives. Appending redundant 'include' - # lines is safe, though. - if "versioneer.py" not in simple_includes: - print(" appending 'versioneer.py' to MANIFEST.in") - with open(manifest_in, "a") as f: - f.write("include versioneer.py\n") - else: - print(" 'versioneer.py' already in MANIFEST.in") - if cfg.versionfile_source not in simple_includes: - print(" appending versionfile_source ('%s') to MANIFEST.in" % - cfg.versionfile_source) - with open(manifest_in, "a") as f: - f.write("include %s\n" % cfg.versionfile_source) - else: - print(" versionfile_source already in MANIFEST.in") - - # Make VCS-specific changes. For git, this means creating/changing - # .gitattributes to mark _version.py for export-time keyword - # substitution. - do_vcs_install(manifest_in, cfg.versionfile_source, ipy) - return 0 - - -def scan_setup_py(): - """Validate the contents of setup.py against Versioneer's expectations.""" - found = set() - setters = False - errors = 0 - with open("setup.py", "r") as f: - for line in f.readlines(): - if "import versioneer" in line: - found.add("import") - if "versioneer.get_cmdclass()" in line: - found.add("cmdclass") - if "versioneer.get_version()" in line: - found.add("get_version") - if "versioneer.VCS" in line: - setters = True - if "versioneer.versionfile_source" in line: - setters = True - if len(found) != 3: - print("") - print("Your setup.py appears to be missing some important items") - print("(but I might be wrong). Please make sure it has something") - print("roughly like the following:") - print("") - print(" import versioneer") - print(" setup( version=versioneer.get_version(),") - print(" cmdclass=versioneer.get_cmdclass(), ...)") - print("") - errors += 1 - if setters: - print("You should remove lines like 'versioneer.VCS = ' and") - print("'versioneer.versionfile_source = ' . This configuration") - print("now lives in setup.cfg, and should be removed from setup.py") - print("") - errors += 1 - return errors - -if __name__ == "__main__": - cmd = sys.argv[1] - if cmd == "setup": - errors = do_setup() - errors += scan_setup_py() - if errors: - sys.exit(1) diff --git a/src/leap/bitmask/mail/__init__.py b/src/leap/bitmask/mail/__init__.py new file mode 100644 index 0000000..4b25fe6 --- /dev/null +++ b/src/leap/bitmask/mail/__init__.py @@ -0,0 +1,26 @@ +# -*- coding: utf-8 -*- +# __init__.py +# Copyright (C) 2013 LEAP +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . + + +""" +Client mail bits. +""" + + +from ._version import get_versions +__version__ = get_versions()['version'] +del get_versions diff --git a/src/leap/bitmask/mail/_version.py b/src/leap/bitmask/mail/_version.py new file mode 100644 index 0000000..954f488 --- /dev/null +++ b/src/leap/bitmask/mail/_version.py @@ -0,0 +1,484 @@ + +# This file helps to compute a version number in source trees obtained from +# git-archive tarball (such as those provided by githubs download-from-tag +# feature). Distribution tarballs (built by setup.py sdist) and build +# directories (produced by setup.py build) will contain a much shorter file +# that just contains the computed version number. + +# This file is released into the public domain. Generated by +# versioneer-0.16 (https://github.com/warner/python-versioneer) + +"""Git implementation of _version.py.""" + +import errno +import os +import re +import subprocess +import sys + + +def get_keywords(): + """Get the keywords needed to look up the version information.""" + # these strings will be replaced by git during git-archive. + # setup.py/versioneer.py will grep for the variable names, so they must + # each be defined on a line of their own. _version.py will just call + # get_keywords(). + git_refnames = "$Format:%d$" + git_full = "$Format:%H$" + keywords = {"refnames": git_refnames, "full": git_full} + return keywords + + +class VersioneerConfig: + """Container for Versioneer configuration parameters.""" + + +def get_config(): + """Create, populate and return the VersioneerConfig() object.""" + # these strings are filled in when 'setup.py versioneer' creates + # _version.py + cfg = VersioneerConfig() + cfg.VCS = "git" + cfg.style = "pep440" + cfg.tag_prefix = "" + cfg.parentdir_prefix = "None" + cfg.versionfile_source = "src/leap/mail/_version.py" + cfg.verbose = False + return cfg + + +class NotThisMethod(Exception): + """Exception raised if a method is not valid for the current scenario.""" + + +LONG_VERSION_PY = {} +HANDLERS = {} + + +def register_vcs_handler(vcs, method): # decorator + """Decorator to mark a method as the handler for a particular VCS.""" + def decorate(f): + """Store f in HANDLERS[vcs][method].""" + if vcs not in HANDLERS: + HANDLERS[vcs] = {} + HANDLERS[vcs][method] = f + return f + return decorate + + +def run_command(commands, args, cwd=None, verbose=False, hide_stderr=False): + """Call the given command(s).""" + assert isinstance(commands, list) + p = None + for c in commands: + try: + dispcmd = str([c] + args) + # remember shell=False, so use git.cmd on windows, not just git + p = subprocess.Popen([c] + args, cwd=cwd, stdout=subprocess.PIPE, + stderr=(subprocess.PIPE if hide_stderr + else None)) + break + except EnvironmentError: + e = sys.exc_info()[1] + if e.errno == errno.ENOENT: + continue + if verbose: + print("unable to run %s" % dispcmd) + print(e) + return None + else: + if verbose: + print("unable to find command, tried %s" % (commands,)) + return None + stdout = p.communicate()[0].strip() + if sys.version_info[0] >= 3: + stdout = stdout.decode() + if p.returncode != 0: + if verbose: + print("unable to run %s (error)" % dispcmd) + return None + return stdout + + +def versions_from_parentdir(parentdir_prefix, root, verbose): + """Try to determine the version from the parent directory name. + + Source tarballs conventionally unpack into a directory that includes + both the project name and a version string. + """ + dirname = os.path.basename(root) + if not dirname.startswith(parentdir_prefix): + if verbose: + print("guessing rootdir is '%s', but '%s' doesn't start with " + "prefix '%s'" % (root, dirname, parentdir_prefix)) + raise NotThisMethod("rootdir doesn't start with parentdir_prefix") + return {"version": dirname[len(parentdir_prefix):], + "full-revisionid": None, + "dirty": False, "error": None} + + +@register_vcs_handler("git", "get_keywords") +def git_get_keywords(versionfile_abs): + """Extract version information from the given file.""" + # the code embedded in _version.py can just fetch the value of these + # keywords. When used from setup.py, we don't want to import _version.py, + # so we do it with a regexp instead. This function is not used from + # _version.py. + keywords = {} + try: + f = open(versionfile_abs, "r") + for line in f.readlines(): + if line.strip().startswith("git_refnames ="): + mo = re.search(r'=\s*"(.*)"', line) + if mo: + keywords["refnames"] = mo.group(1) + if line.strip().startswith("git_full ="): + mo = re.search(r'=\s*"(.*)"', line) + if mo: + keywords["full"] = mo.group(1) + f.close() + except EnvironmentError: + pass + return keywords + + +@register_vcs_handler("git", "keywords") +def git_versions_from_keywords(keywords, tag_prefix, verbose): + """Get version information from git keywords.""" + if not keywords: + raise NotThisMethod("no keywords at all, weird") + refnames = keywords["refnames"].strip() + if refnames.startswith("$Format"): + if verbose: + print("keywords are unexpanded, not using") + raise NotThisMethod("unexpanded keywords, not a git-archive tarball") + refs = set([r.strip() for r in refnames.strip("()").split(",")]) + # starting in git-1.8.3, tags are listed as "tag: foo-1.0" instead of + # just "foo-1.0". If we see a "tag: " prefix, prefer those. + TAG = "tag: " + tags = set([r[len(TAG):] for r in refs if r.startswith(TAG)]) + if not tags: + # Either we're using git < 1.8.3, or there really are no tags. We use + # a heuristic: assume all version tags have a digit. The old git %d + # expansion behaves like git log --decorate=short and strips out the + # refs/heads/ and refs/tags/ prefixes that would let us distinguish + # between branches and tags. By ignoring refnames without digits, we + # filter out many common branch names like "release" and + # "stabilization", as well as "HEAD" and "master". + tags = set([r for r in refs if re.search(r'\d', r)]) + if verbose: + print("discarding '%s', no digits" % ",".join(refs-tags)) + if verbose: + print("likely tags: %s" % ",".join(sorted(tags))) + for ref in sorted(tags): + # sorting will prefer e.g. "2.0" over "2.0rc1" + if ref.startswith(tag_prefix): + r = ref[len(tag_prefix):] + if verbose: + print("picking %s" % r) + return {"version": r, + "full-revisionid": keywords["full"].strip(), + "dirty": False, "error": None + } + # no suitable tags, so version is "0+unknown", but full hex is still there + if verbose: + print("no suitable tags, using unknown + full revision id") + return {"version": "0+unknown", + "full-revisionid": keywords["full"].strip(), + "dirty": False, "error": "no suitable tags"} + + +@register_vcs_handler("git", "pieces_from_vcs") +def git_pieces_from_vcs(tag_prefix, root, verbose, run_command=run_command): + """Get version from 'git describe' in the root of the source tree. + + This only gets called if the git-archive 'subst' keywords were *not* + expanded, and _version.py hasn't already been rewritten with a short + version string, meaning we're inside a checked out source tree. + """ + if not os.path.exists(os.path.join(root, ".git")): + if verbose: + print("no .git in %s" % root) + raise NotThisMethod("no .git directory") + + GITS = ["git"] + if sys.platform == "win32": + GITS = ["git.cmd", "git.exe"] + # if there is a tag matching tag_prefix, this yields TAG-NUM-gHEX[-dirty] + # if there isn't one, this yields HEX[-dirty] (no NUM) + describe_out = run_command(GITS, ["describe", "--tags", "--dirty", + "--always", "--long", + "--match", "%s*" % tag_prefix], + cwd=root) + # --long was added in git-1.5.5 + if describe_out is None: + raise NotThisMethod("'git describe' failed") + describe_out = describe_out.strip() + full_out = run_command(GITS, ["rev-parse", "HEAD"], cwd=root) + if full_out is None: + raise NotThisMethod("'git rev-parse' failed") + full_out = full_out.strip() + + pieces = {} + pieces["long"] = full_out + pieces["short"] = full_out[:7] # maybe improved later + pieces["error"] = None + + # parse describe_out. It will be like TAG-NUM-gHEX[-dirty] or HEX[-dirty] + # TAG might have hyphens. + git_describe = describe_out + + # look for -dirty suffix + dirty = git_describe.endswith("-dirty") + pieces["dirty"] = dirty + if dirty: + git_describe = git_describe[:git_describe.rindex("-dirty")] + + # now we have TAG-NUM-gHEX or HEX + + if "-" in git_describe: + # TAG-NUM-gHEX + mo = re.search(r'^(.+)-(\d+)-g([0-9a-f]+)$', git_describe) + if not mo: + # unparseable. Maybe git-describe is misbehaving? + pieces["error"] = ("unable to parse git-describe output: '%s'" + % describe_out) + return pieces + + # tag + full_tag = mo.group(1) + if not full_tag.startswith(tag_prefix): + if verbose: + fmt = "tag '%s' doesn't start with prefix '%s'" + print(fmt % (full_tag, tag_prefix)) + pieces["error"] = ("tag '%s' doesn't start with prefix '%s'" + % (full_tag, tag_prefix)) + return pieces + pieces["closest-tag"] = full_tag[len(tag_prefix):] + + # distance: number of commits since tag + pieces["distance"] = int(mo.group(2)) + + # commit: short hex revision ID + pieces["short"] = mo.group(3) + + else: + # HEX: no tags + pieces["closest-tag"] = None + count_out = run_command(GITS, ["rev-list", "HEAD", "--count"], + cwd=root) + pieces["distance"] = int(count_out) # total number of commits + + return pieces + + +def plus_or_dot(pieces): + """Return a + if we don't already have one, else return a .""" + if "+" in pieces.get("closest-tag", ""): + return "." + return "+" + + +def render_pep440(pieces): + """Build up version string, with post-release "local version identifier". + + Our goal: TAG[+DISTANCE.gHEX[.dirty]] . Note that if you + get a tagged build and then dirty it, you'll get TAG+0.gHEX.dirty + + Exceptions: + 1: no tags. git_describe was just HEX. 0+untagged.DISTANCE.gHEX[.dirty] + """ + if pieces["closest-tag"]: + rendered = pieces["closest-tag"] + if pieces["distance"] or pieces["dirty"]: + rendered += plus_or_dot(pieces) + rendered += "%d.g%s" % (pieces["distance"], pieces["short"]) + if pieces["dirty"]: + rendered += ".dirty" + else: + # exception #1 + rendered = "0+untagged.%d.g%s" % (pieces["distance"], + pieces["short"]) + if pieces["dirty"]: + rendered += ".dirty" + return rendered + + +def render_pep440_pre(pieces): + """TAG[.post.devDISTANCE] -- No -dirty. + + Exceptions: + 1: no tags. 0.post.devDISTANCE + """ + if pieces["closest-tag"]: + rendered = pieces["closest-tag"] + if pieces["distance"]: + rendered += ".post.dev%d" % pieces["distance"] + else: + # exception #1 + rendered = "0.post.dev%d" % pieces["distance"] + return rendered + + +def render_pep440_post(pieces): + """TAG[.postDISTANCE[.dev0]+gHEX] . + + The ".dev0" means dirty. Note that .dev0 sorts backwards + (a dirty tree will appear "older" than the corresponding clean one), + but you shouldn't be releasing software with -dirty anyways. + + Exceptions: + 1: no tags. 0.postDISTANCE[.dev0] + """ + if pieces["closest-tag"]: + rendered = pieces["closest-tag"] + if pieces["distance"] or pieces["dirty"]: + rendered += ".post%d" % pieces["distance"] + if pieces["dirty"]: + rendered += ".dev0" + rendered += plus_or_dot(pieces) + rendered += "g%s" % pieces["short"] + else: + # exception #1 + rendered = "0.post%d" % pieces["distance"] + if pieces["dirty"]: + rendered += ".dev0" + rendered += "+g%s" % pieces["short"] + return rendered + + +def render_pep440_old(pieces): + """TAG[.postDISTANCE[.dev0]] . + + The ".dev0" means dirty. + + Eexceptions: + 1: no tags. 0.postDISTANCE[.dev0] + """ + if pieces["closest-tag"]: + rendered = pieces["closest-tag"] + if pieces["distance"] or pieces["dirty"]: + rendered += ".post%d" % pieces["distance"] + if pieces["dirty"]: + rendered += ".dev0" + else: + # exception #1 + rendered = "0.post%d" % pieces["distance"] + if pieces["dirty"]: + rendered += ".dev0" + return rendered + + +def render_git_describe(pieces): + """TAG[-DISTANCE-gHEX][-dirty]. + + Like 'git describe --tags --dirty --always'. + + Exceptions: + 1: no tags. HEX[-dirty] (note: no 'g' prefix) + """ + if pieces["closest-tag"]: + rendered = pieces["closest-tag"] + if pieces["distance"]: + rendered += "-%d-g%s" % (pieces["distance"], pieces["short"]) + else: + # exception #1 + rendered = pieces["short"] + if pieces["dirty"]: + rendered += "-dirty" + return rendered + + +def render_git_describe_long(pieces): + """TAG-DISTANCE-gHEX[-dirty]. + + Like 'git describe --tags --dirty --always -long'. + The distance/hash is unconditional. + + Exceptions: + 1: no tags. HEX[-dirty] (note: no 'g' prefix) + """ + if pieces["closest-tag"]: + rendered = pieces["closest-tag"] + rendered += "-%d-g%s" % (pieces["distance"], pieces["short"]) + else: + # exception #1 + rendered = pieces["short"] + if pieces["dirty"]: + rendered += "-dirty" + return rendered + + +def render(pieces, style): + """Render the given version pieces into the requested style.""" + if pieces["error"]: + return {"version": "unknown", + "full-revisionid": pieces.get("long"), + "dirty": None, + "error": pieces["error"]} + + if not style or style == "default": + style = "pep440" # the default + + if style == "pep440": + rendered = render_pep440(pieces) + elif style == "pep440-pre": + rendered = render_pep440_pre(pieces) + elif style == "pep440-post": + rendered = render_pep440_post(pieces) + elif style == "pep440-old": + rendered = render_pep440_old(pieces) + elif style == "git-describe": + rendered = render_git_describe(pieces) + elif style == "git-describe-long": + rendered = render_git_describe_long(pieces) + else: + raise ValueError("unknown style '%s'" % style) + + return {"version": rendered, "full-revisionid": pieces["long"], + "dirty": pieces["dirty"], "error": None} + + +def get_versions(): + """Get version information or return default if unable to do so.""" + # I am in _version.py, which lives at ROOT/VERSIONFILE_SOURCE. If we have + # __file__, we can work backwards from there to the root. Some + # py2exe/bbfreeze/non-CPython implementations don't do __file__, in which + # case we can only use expanded keywords. + + cfg = get_config() + verbose = cfg.verbose + + try: + return git_versions_from_keywords(get_keywords(), cfg.tag_prefix, + verbose) + except NotThisMethod: + pass + + try: + root = os.path.realpath(__file__) + # versionfile_source is the relative path from the top of the source + # tree (where the .git directory might live) to this file. Invert + # this to find the root from __file__. + for i in cfg.versionfile_source.split('/'): + root = os.path.dirname(root) + except NameError: + return {"version": "0+unknown", "full-revisionid": None, + "dirty": None, + "error": "unable to find root of source tree"} + + try: + pieces = git_pieces_from_vcs(cfg.tag_prefix, root, verbose) + return render(pieces, cfg.style) + except NotThisMethod: + pass + + try: + if cfg.parentdir_prefix: + return versions_from_parentdir(cfg.parentdir_prefix, root, verbose) + except NotThisMethod: + pass + + return {"version": "0+unknown", "full-revisionid": None, + "dirty": None, + "error": "unable to compute version"} diff --git a/src/leap/bitmask/mail/adaptors/__init__.py b/src/leap/bitmask/mail/adaptors/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/src/leap/bitmask/mail/adaptors/models.py b/src/leap/bitmask/mail/adaptors/models.py new file mode 100644 index 0000000..49460f7 --- /dev/null +++ b/src/leap/bitmask/mail/adaptors/models.py @@ -0,0 +1,123 @@ +# -*- coding: utf-8 -*- +# models.py +# Copyright (C) 2014 LEAP +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . +""" +Generic Models to be used by the Document Adaptors. +""" +import copy + + +class SerializableModel(object): + """ + A Generic document model, that can be serialized into a dictionary. + + Subclasses of this `SerializableModel` are meant to be added as class + attributes of classes inheriting from DocumentWrapper. + + A subclass __meta__ of this SerializableModel might exist, and contain info + relative to particularities of this model. + + For instance, the use of `__meta__.index` marks the existence of a primary + index in the model, which will be used to do unique queries (in which case + all the other indexed fields in the underlying document will be filled with + the default info contained in the model definition). + """ + + @classmethod + def serialize(klass): + """ + Get a dictionary representation of the public attributes in the model + class. To avoid collisions with builtin functions, any occurrence of an + attribute ended in '_' (like 'type_') will be normalized by removing + the trailing underscore. + + This classmethod is used from within the serialized method of a + DocumentWrapper instance: it provides defaults for the + empty document. + """ + assert isinstance(klass, type) + return _normalize_dict(klass.__dict__) + + +class DocumentWrapper(object): + """ + A Wrapper object that can be manipulated, passed around, and serialized in + a format that the store understands. + It is related to a SerializableModel, which must be specified as the + ``model`` class attribute. The instance of this DocumentWrapper will not + allow any other *public* attributes than those defined in the corresponding + model. + """ + # TODO we could do some very basic type checking here + # TODO set a dirty flag (on __setattr__, whenever the value is != from + # before) + # TODO we could enforce the existence of a correct "model" attribute + # in some other way (other than in the initializer) + + def __init__(self, **kwargs): + if not getattr(self, 'model', None): + raise RuntimeError( + 'DocumentWrapper class needs a model attribute') + + defaults = self.model.serialize() + + if kwargs: + values = copy.deepcopy(defaults) + values.update(_normalize_dict(kwargs)) + else: + values = defaults + + for k, v in values.items(): + k = k.replace('-', '_') + setattr(self, k, v) + + def __setattr__(self, attr, value): + normalized = _normalize_dict(self.model.__dict__) + if not attr.startswith('_') and attr not in normalized: + raise RuntimeError( + "Cannot set attribute because it's not defined " + "in the model %s: %s" % (self.__class__, attr)) + object.__setattr__(self, attr, value) + + def serialize(self): + return _normalize_dict(self.__dict__) + + def create(self): + raise NotImplementedError() + + def update(self): + raise NotImplementedError() + + def delete(self): + raise NotImplementedError() + + @classmethod + def get_or_create(self): + raise NotImplementedError() + + @classmethod + def get_all(self): + raise NotImplementedError() + + +def _normalize_dict(_dict): + items = _dict.items() + items = filter(lambda (k, v): not callable(v), items) + items = filter(lambda (k, v): not k.startswith('_'), items) + items = [(k, v) if not k.endswith('_') else (k[:-1], v) + for (k, v) in items] + items = [(k.replace('-', '_'), v) for (k, v) in items] + return dict(items) diff --git a/src/leap/bitmask/mail/adaptors/soledad.py b/src/leap/bitmask/mail/adaptors/soledad.py new file mode 100644 index 0000000..ca8f741 --- /dev/null +++ b/src/leap/bitmask/mail/adaptors/soledad.py @@ -0,0 +1,1268 @@ +# soledad.py +# Copyright (C) 2014 LEAP +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . +""" +Soledadad MailAdaptor module. +""" +import logging +import re + +from collections import defaultdict +from email import message_from_string + +from twisted.internet import defer +from twisted.python import log +from zope.interface import implements +from leap.soledad.common import l2db + +from leap.common.check import leap_assert, leap_assert_type + +from leap.mail import constants +from leap.mail import walk +from leap.mail.adaptors import soledad_indexes as indexes +from leap.mail.constants import INBOX_NAME +from leap.mail.adaptors import models +from leap.mail.imap.mailbox import normalize_mailbox +from leap.mail.utils import lowerdict, first +from leap.mail.utils import stringify_parts_map +from leap.mail.interfaces import IMailAdaptor, IMessageWrapper + +from leap.soledad.common.document import SoledadDocument + + +logger = logging.getLogger(__name__) + +# TODO +# [ ] Convenience function to create mail specifying subject, date, etc? + + +_MSGID_PATTERN = r"""<([\w@.]+)>""" +_MSGID_RE = re.compile(_MSGID_PATTERN) + + +class DuplicatedDocumentError(Exception): + """ + Raised when a duplicated document is detected. + """ + pass + + +def cleanup_deferred_locks(): + """ + Need to use this from within trial to cleanup the reactor before + each run. + """ + SoledadDocumentWrapper._k_locks = defaultdict(defer.DeferredLock) + + +class SoledadDocumentWrapper(models.DocumentWrapper): + """ + A Wrapper object that can be manipulated, passed around, and serialized in + a format that the Soledad Store understands. + + It ensures atomicity of the document operations on creation, update and + deletion. + """ + # TODO we could also use a _dirty flag (in models) + # TODO add a get_count() method ??? -- that is extended over l2db. + + # We keep a dictionary with DeferredLocks, that will be + # unique to every subclass of SoledadDocumentWrapper. + _k_locks = defaultdict(defer.DeferredLock) + + @classmethod + def _get_klass_lock(cls): + """ + Get a DeferredLock that is unique for this subclass name. + Used to lock the access to indexes in the `get_or_create` call + for a particular DocumentWrapper. + """ + return cls._k_locks[cls.__name__] + + def __init__(self, doc_id=None, future_doc_id=None, **kwargs): + self._doc_id = doc_id + self._future_doc_id = future_doc_id + self._lock = defer.DeferredLock() + super(SoledadDocumentWrapper, self).__init__(**kwargs) + + @property + def doc_id(self): + return self._doc_id + + @property + def future_doc_id(self): + return self._future_doc_id + + def set_future_doc_id(self, doc_id): + self._future_doc_id = doc_id + + def create(self, store, is_copy=False): + """ + Create the documents for this wrapper. + Since this method will not check for duplication, the + responsibility of avoiding duplicates is left to the caller. + + You might be interested in using `get_or_create` classmethod + instead (that's the preferred way of creating documents from + the wrapper object). + + :return: a deferred that will fire when the underlying + Soledad document has been created. + :rtype: Deferred + """ + leap_assert(self._doc_id is None, + "This document already has a doc_id!") + + def update_doc_id(doc): + self._doc_id = doc.doc_id + self.set_future_doc_id(None) + return doc + + def update_wrapper(failure): + # In the case of some copies (for instance, from one folder to + # another and back to the original folder), the document that we + # want to insert already exists. In this case, putting it + # and overwriting the document with that doc_id is the right thing + # to do. + failure.trap(l2db.errors.RevisionConflict) + self._doc_id = self.future_doc_id + self._future_doc_id = None + return self.update(store) + + if self.future_doc_id is None: + d = store.create_doc(self.serialize()) + else: + d = store.create_doc(self.serialize(), + doc_id=self.future_doc_id) + d.addCallback(update_doc_id) + + if is_copy: + d.addErrback(update_wrapper) + else: + d.addErrback(self._catch_revision_conflict, self.future_doc_id) + return d + + def update(self, store): + """ + Update the documents for this wrapper. + + :return: a deferred that will fire when the underlying + Soledad document has been updated. + :rtype: Deferred + """ + # the deferred lock guards against revision conflicts + return self._lock.run(self._update, store) + + def _update(self, store): + leap_assert(self._doc_id is not None, + "Need to create doc before updating") + + def update_and_put_doc(doc): + doc.content.update(self.serialize()) + d = store.put_doc(doc) + d.addErrback(self._catch_revision_conflict, doc.doc_id) + return d + + d = store.get_doc(self._doc_id) + d.addCallback(update_and_put_doc) + return d + + def _catch_revision_conflict(self, failure, doc_id): + # XXX We can have some RevisionConflicts if we try + # to put the docs that are already there. + # This can happen right now when creating/saving the cdocs + # during a copy. Instead of catching and ignoring this + # error, we should mark them in the copy so there is no attempt to + # create/update them. + failure.trap(l2db.errors.RevisionConflict) + logger.debug("Got conflict while putting %s" % doc_id) + + def delete(self, store): + """ + Delete the documents for this wrapper. + + :return: a deferred that will fire when the underlying + Soledad document has been deleted. + :rtype: Deferred + """ + # the deferred lock guards against conflicts while updating + return self._lock.run(self._delete, store) + + def _delete(self, store): + leap_assert(self._doc_id is not None, + "Need to create doc before deleting") + # XXX might want to flag this DocumentWrapper to avoid + # updating it by mistake. This could go in models.DocumentWrapper + + def delete_doc(doc): + return store.delete_doc(doc) + + d = store.get_doc(self._doc_id) + d.addCallback(delete_doc) + return d + + @classmethod + def get_or_create(cls, store, index, value): + """ + Get a unique DocumentWrapper by index, or create a new one if the + matching query does not exist. + + :param index: the primary index for the model. + :type index: str + :param value: the value to query the primary index. + :type value: str + + :return: a deferred that will be fired with the SoledadDocumentWrapper + matching the index query, either existing or just created. + :rtype: Deferred + """ + return cls._get_klass_lock().run( + cls._get_or_create, store, index, value) + + @classmethod + def _get_or_create(cls, store, index, value): + # TODO shorten this method. + assert store is not None + assert index is not None + assert value is not None + + def get_main_index(): + try: + return cls.model.__meta__.index + except AttributeError: + raise RuntimeError("The model is badly defined") + + # TODO separate into another method? + def try_to_get_doc_from_index(indexes): + values = [] + idx_def = dict(indexes)[index] + if len(idx_def) == 1: + values = [value] + else: + main_index = get_main_index() + fields = cls.model.serialize() + for field in idx_def: + if field == main_index: + values.append(value) + else: + values.append(fields[field]) + d = store.get_from_index(index, *values) + return d + + def get_first_doc_if_any(docs): + if not docs: + return None + if len(docs) > 1: + raise DuplicatedDocumentError + return docs[0] + + def wrap_existing_or_create_new(doc): + if doc: + return cls(doc_id=doc.doc_id, **doc.content) + else: + return create_and_wrap_new_doc() + + def create_and_wrap_new_doc(): + # XXX use closure to store indexes instead of + # querying for them again. + d = store.list_indexes() + d.addCallback(get_wrapper_instance_from_index) + d.addCallback(return_wrapper_when_created) + return d + + def get_wrapper_instance_from_index(indexes): + init_values = {} + idx_def = dict(indexes)[index] + if len(idx_def) == 1: + init_value = {idx_def[0]: value} + return cls(**init_value) + main_index = get_main_index() + fields = cls.model.serialize() + for field in idx_def: + if field == main_index: + init_values[field] = value + else: + init_values[field] = fields[field] + return cls(**init_values) + + def return_wrapper_when_created(wrapper): + d = wrapper.create(store) + d.addCallback(lambda doc: wrapper) + return d + + d = store.list_indexes() + d.addCallback(try_to_get_doc_from_index) + d.addCallback(get_first_doc_if_any) + d.addCallback(wrap_existing_or_create_new) + return d + + @classmethod + def get_all(cls, store): + """ + Get a collection of wrappers around all the documents belonging + to this kind. + + For this to work, the model.__meta__ needs to include a tuple with + the index to be used for listing purposes, and which is the field to be + used to query the index. + + Note that this method only supports indexes of a single field at the + moment. It also might be too expensive to return all the documents + matching the query, so handle with care. + + class __meta__(object): + index = "name" + list_index = ("by-type", "type_") + + :return: a deferred that will be fired with an iterable containing + as many SoledadDocumentWrapper are matching the index defined + in the model as the `list_index`. + :rtype: Deferred + """ + # TODO LIST (get_all) + # [ ] extend support to indexes with n-ples + # [ ] benchmark the cost of querying and returning indexes in a big + # database. This might badly need pagination before being put to + # serious use. + return cls._get_klass_lock().run(cls._get_all, store) + + @classmethod + def _get_all(cls, store): + try: + list_index, list_attr = cls.model.__meta__.list_index + except AttributeError: + raise RuntimeError("The model is badly defined: no list_index") + try: + index_value = getattr(cls.model, list_attr) + except AttributeError: + raise RuntimeError("The model is badly defined: " + "no attribute matching list_index") + + def wrap_docs(docs): + return (cls(doc_id=doc.doc_id, **doc.content) for doc in docs) + + d = store.get_from_index(list_index, index_value) + d.addCallback(wrap_docs) + return d + + def __repr__(self): + try: + idx = getattr(self, self.model.__meta__.index) + except AttributeError: + idx = "" + return "<%s: %s (%s)>" % (self.__class__.__name__, + idx, self._doc_id) + + +# +# Message documents +# + +class FlagsDocWrapper(SoledadDocumentWrapper): + + class model(models.SerializableModel): + type_ = "flags" + chash = "" + + mbox_uuid = "" + seen = False + deleted = False + recent = False + flags = [] + tags = [] + size = 0 + multi = False + + class __meta__(object): + index = "mbox" + + def set_mbox_uuid(self, mbox_uuid): + # XXX raise error if already created, should use copy instead + mbox_uuid = mbox_uuid.replace('-', '_') + new_id = constants.FDOCID.format(mbox_uuid=mbox_uuid, chash=self.chash) + self._future_doc_id = new_id + self.mbox_uuid = mbox_uuid + + def get_flags(self): + """ + Get the flags for this message (as a tuple of strings, not unicode). + """ + return map(str, self.flags) + + +class HeaderDocWrapper(SoledadDocumentWrapper): + + class model(models.SerializableModel): + type_ = "head" + chash = "" + + date = "" + subject = "" + headers = {} + part_map = {} + body = "" # link to phash of body + msgid = "" + multi = False + + class __meta__(object): + index = "chash" + + +class ContentDocWrapper(SoledadDocumentWrapper): + + class model(models.SerializableModel): + type_ = "cnt" + phash = "" + + ctype = "" # XXX index by ctype too? + lkf = [] # XXX not implemented yet! + raw = "" + + content_disposition = "" + content_transfer_encoding = "" + content_type = "" + + class __meta__(object): + index = "phash" + + +class MetaMsgDocWrapper(SoledadDocumentWrapper): + + class model(models.SerializableModel): + type_ = "meta" + fdoc = "" + hdoc = "" + cdocs = [] + + def set_mbox_uuid(self, mbox_uuid): + # XXX raise error if already created, should use copy instead + mbox_uuid = mbox_uuid.replace('-', '_') + chash = re.findall(constants.FDOCID_CHASH_RE, self.fdoc)[0] + new_id = constants.METAMSGID.format(mbox_uuid=mbox_uuid, chash=chash) + new_fdoc_id = constants.FDOCID.format(mbox_uuid=mbox_uuid, chash=chash) + self._future_doc_id = new_id + self.fdoc = new_fdoc_id + + +class MessageWrapper(object): + + # This could benefit of a DeferredLock to create/update all the + # documents at the same time maybe, and defend against concurrent updates? + + implements(IMessageWrapper) + + def __init__(self, mdoc, fdoc, hdoc, cdocs=None, is_copy=False): + """ + Need at least a metamsg-document, a flag-document and a header-document + to instantiate a MessageWrapper. Content-documents can be retrieved + lazily. + + cdocs, if any, should be a dictionary in which the keys are ascending + integers, beginning at one, and the values are dictionaries with the + content of the content-docs. + + is_copy, if set to True, will only attempt to create mdoc and fdoc + (because hdoc and cdocs are supposed to exist already) + """ + self._is_copy = is_copy + + def get_doc_wrapper(doc, cls): + if isinstance(doc, SoledadDocument): + doc_id = doc.doc_id + doc = doc.content + else: + doc_id = None + if not doc: + doc = {} + return cls(doc_id=doc_id, **doc) + + self.mdoc = get_doc_wrapper(mdoc, MetaMsgDocWrapper) + + self.fdoc = get_doc_wrapper(fdoc, FlagsDocWrapper) + self.fdoc.set_future_doc_id(self.mdoc.fdoc) + + self.hdoc = get_doc_wrapper(hdoc, HeaderDocWrapper) + self.hdoc.set_future_doc_id(self.mdoc.hdoc) + + if cdocs is None: + cdocs = {} + cdocs_keys = cdocs.keys() + assert sorted(cdocs_keys) == range(1, len(cdocs_keys) + 1) + self.cdocs = dict([ + (key, get_doc_wrapper(doc, ContentDocWrapper)) + for (key, doc) in cdocs.items()]) + for doc_id, cdoc in zip(self.mdoc.cdocs, self.cdocs.values()): + if cdoc.raw == "": + log.msg("Empty raw field in cdoc %s" % doc_id) + cdoc.set_future_doc_id(doc_id) + + def create(self, store, notify_just_mdoc=False, pending_inserts_dict=None): + """ + Create all the parts for this message in the store. + + :param store: an instance of Soledad + + :param notify_just_mdoc: + if set to True, this method will return *only* the deferred + corresponding to the creation of the meta-message document. + Be warned that in that case there will be no record of failures + when creating the other part-documents. + + Otherwise, this method will return a deferred that will wait for + the creation of all the part documents. + + Setting this flag to True is mostly a convenient workaround for the + fact that massive serial appends will take too much time, and in + most of the cases the MUA will only switch to the mailbox where the + appends have happened after a certain time, which in most of the + times will be enough to have all the queued insert operations + finished. + :type notify_just_mdoc: bool + :param pending_inserts_dict: + a dictionary with the pending inserts ids. + :type pending_inserts_dict: dict + + :return: a deferred whose callback will be called when either all the + part documents have been written, or just the metamsg-doc, + depending on the value of the notify_just_mdoc flag + :rtype: defer.Deferred + """ + if pending_inserts_dict is None: + pending_inserts_dict = {} + + leap_assert(self.cdocs, + "Need non empty cdocs to create the " + "MessageWrapper documents") + leap_assert(self.mdoc.doc_id is None, + "Cannot create: mdoc has a doc_id") + leap_assert(self.fdoc.doc_id is None, + "Cannot create: fdoc has a doc_id") + + def unblock_pending_insert(result): + if pending_inserts_dict: + ci_headers = lowerdict(self.hdoc.headers) + msgid = ci_headers.get('message-id', None) + try: + d = pending_inserts_dict[msgid] + d.callback(msgid) + except KeyError: + pass + return result + + # TODO check that the doc_ids in the mdoc are coherent + self.d = [] + + mdoc_created = self.mdoc.create(store, is_copy=self._is_copy) + fdoc_created = self.fdoc.create(store, is_copy=self._is_copy) + + mdoc_created.addErrback(lambda f: log.err(f)) + fdoc_created.addErrback(lambda f: log.err(f)) + + self.d.append(mdoc_created) + self.d.append(fdoc_created) + + if not self._is_copy: + if self.hdoc.doc_id is None: + self.d.append(self.hdoc.create(store)) + for cdoc in self.cdocs.values(): + if cdoc.doc_id is not None: + # we could be just linking to an existing + # content-doc. + continue + self.d.append(cdoc.create(store)) + + def log_all_inserted(result): + log.msg("All parts inserted for msg!") + return result + + self.all_inserted_d = defer.gatherResults(self.d, consumeErrors=True) + self.all_inserted_d.addCallback(log_all_inserted) + self.all_inserted_d.addCallback(unblock_pending_insert) + self.all_inserted_d.addErrback(lambda failure: log.err(failure)) + + if notify_just_mdoc: + return mdoc_created + else: + return self.all_inserted_d + + def update(self, store): + """ + Update the only mutable parts, which are within the flags document. + """ + return self.fdoc.update(store) + + def delete(self, store): + # TODO + # Eventually this would have to do the duplicate search or send for the + # garbage collector. At least mdoc and t the mdoc and fdoc can be + # unlinked. + d = [] + if self.mdoc.doc_id: + d.append(self.mdoc.delete(store)) + d.append(self.fdoc.delete(store)) + return defer.gatherResults(d) + + def copy(self, store, new_mbox_uuid): + """ + Return a copy of this MessageWrapper in a new mailbox. + + :param store: an instance of Soledad, or anything that behaves alike. + :param new_mbox_uuid: the uuid of the mailbox where we are copying this + message to. + :type new_mbox_uuid: str + :rtype: MessageWrapper + """ + new_mdoc = self.mdoc.serialize() + new_fdoc = self.fdoc.serialize() + + # the future doc_ids is properly set because we modified + # the pointers in mdoc, which has precedence. + new_wrapper = MessageWrapper(new_mdoc, new_fdoc, None, None, + is_copy=True) + new_wrapper.hdoc = self.hdoc + new_wrapper.cdocs = self.cdocs + new_wrapper.set_mbox_uuid(new_mbox_uuid) + + # XXX could flag so that it only creates mdoc/fdoc... + + d = new_wrapper.create(store) + d.addCallback(lambda result: new_wrapper) + d.addErrback(lambda failure: log.err(failure)) + return d + + def set_mbox_uuid(self, mbox_uuid): + """ + Set the mailbox for this wrapper. + This method should only be used before the Documents for the + MessageWrapper have been created, will raise otherwise. + """ + mbox_uuid = mbox_uuid.replace('-', '_') + self.mdoc.set_mbox_uuid(mbox_uuid) + self.fdoc.set_mbox_uuid(mbox_uuid) + + def set_flags(self, flags): + # TODO serialize the get + update + if flags is None: + flags = tuple() + leap_assert_type(flags, tuple) + self.fdoc.flags = list(flags) + self.fdoc.deleted = "\\Deleted" in flags + self.fdoc.seen = "\\Seen" in flags + self.fdoc.recent = "\\Recent" in flags + + def set_tags(self, tags): + # TODO serialize the get + update + if tags is None: + tags = tuple() + leap_assert_type(tags, tuple) + self.fdoc.tags = list(tags) + + def set_date(self, date): + # XXX assert valid date format + self.hdoc.date = date + + def get_subpart_dict(self, index): + """ + :param index: the part to lookup, 1-indexed + :type index: int + :rtype: dict + """ + return self.hdoc.part_map[str(index)] + + def get_subpart_indexes(self): + return self.hdoc.part_map.keys() + + def get_body(self, store): + """ + :rtype: deferred + """ + body_phash = self.hdoc.body + if body_phash: + d = store.get_doc('C-' + body_phash) + d.addCallback(lambda doc: ContentDocWrapper(**doc.content)) + return d + elif self.cdocs: + return self.cdocs[1] + else: + return '' + +# +# Mailboxes +# + + +class MailboxWrapper(SoledadDocumentWrapper): + + class model(models.SerializableModel): + type_ = "mbox" + mbox = INBOX_NAME + uuid = None + flags = [] + recent = [] + created = 1 + closed = False + subscribed = False + + class __meta__(object): + index = "mbox" + list_index = (indexes.TYPE_IDX, 'type_') + + +# +# Soledad Adaptor +# + +class SoledadIndexMixin(object): + """ + This will need a class attribute `indexes`, that is a dictionary containing + the index definitions for the underlying l2db store underlying soledad. + + It needs to be in the following format: + {'index-name': ['field1', 'field2']} + + You can also add a class attribute `wait_for_indexes` to any class + inheriting from this Mixin, that should be a list of strings representing + the methods that need to wait until the indexes have been initialized + before being able to work properly. + """ + # TODO move this mixin to soledad itself + # so that each application can pass a set of indexes for their data model. + + # TODO could have a wrapper class for indexes, supporting introspection + # and __getattr__ + + # TODO make this an interface? + + indexes = {} + wait_for_indexes = [] + store_ready = False + + def initialize_store(self, store): + """ + Initialize the indexes in the database. + + :param store: store + :returns: a Deferred that will fire when the store is correctly + initialized. + :rtype: deferred + """ + # TODO I think we *should* get another deferredLock in here, but + # global to the soledad namespace, to protect from several points + # initializing soledad indexes at the same time. + self._wait_for_indexes() + + d = self._init_indexes(store) + d.addCallback(self._restore_waiting_methods) + return d + + def _init_indexes(self, store): + """ + Initialize the database indexes. + """ + leap_assert(store, "Cannot init indexes with null soledad") + leap_assert_type(self.indexes, dict) + + def _create_index(name, expression): + return store.create_index(name, *expression) + + def init_idexes(indexes): + deferreds = [] + db_indexes = dict(indexes) + # Loop through the indexes we expect to find. + for name, expression in self.indexes.items(): + if name not in db_indexes: + # The index does not yet exist. + d = _create_index(name, expression) + deferreds.append(d) + elif expression != db_indexes[name]: + # The index exists but the definition is not what expected, + # so we delete it and add the proper index expression. + d = store.delete_index(name) + d.addCallback( + lambda _: _create_index(name, *expression)) + deferreds.append(d) + return defer.gatherResults(deferreds, consumeErrors=True) + + def store_ready(whatever): + self.store_ready = True + return whatever + + self.deferred_indexes = store.list_indexes() + self.deferred_indexes.addCallback(init_idexes) + self.deferred_indexes.addCallback(store_ready) + return self.deferred_indexes + + def _wait_for_indexes(self): + """ + Make the marked methods to wait for the indexes to be ready. + Heavily based on + http://blogs.fluidinfo.com/terry/2009/05/11/a-mixin-class-allowing-python-__init__-methods-to-work-with-twisted-deferreds/ + + :param methods: methods that need to wait for the indexes to be ready + :type methods: tuple(str) + """ + leap_assert_type(self.wait_for_indexes, list) + methods = self.wait_for_indexes + + self.waiting = [] + self.stored = {} + + def makeWrapper(method): + def wrapper(*args, **kw): + d = defer.Deferred() + d.addCallback(lambda _: self.stored[method](*args, **kw)) + self.waiting.append(d) + return d + return wrapper + + for method in methods: + self.stored[method] = getattr(self, method) + setattr(self, method, makeWrapper(method)) + + def _restore_waiting_methods(self, _): + for method in self.stored: + setattr(self, method, self.stored[method]) + for d in self.waiting: + d.callback(None) + + +class SoledadMailAdaptor(SoledadIndexMixin): + + implements(IMailAdaptor) + store = None + + indexes = indexes.MAIL_INDEXES + wait_for_indexes = ['get_or_create_mbox', 'update_mbox', 'get_all_mboxes'] + + mboxwrapper_klass = MailboxWrapper + + def __init__(self): + SoledadIndexMixin.__init__(self) + + # Message handling + + def get_msg_from_string(self, MessageClass, raw_msg): + """ + Get an instance of a MessageClass initialized with a MessageWrapper + that contains all the parts obtained from parsing the raw string for + the message. + + :param MessageClass: any Message class that can be initialized passing + an instance of an IMessageWrapper implementor. + :type MessageClass: type + :param raw_msg: a string containing the raw email message. + :type raw_msg: str + :rtype: MessageClass instance. + """ + assert(MessageClass is not None) + mdoc, fdoc, hdoc, cdocs = _split_into_parts(raw_msg) + return self.get_msg_from_docs( + MessageClass, mdoc, fdoc, hdoc, cdocs) + + def get_msg_from_docs(self, MessageClass, mdoc, fdoc, hdoc, cdocs=None, + uid=None): + """ + Get an instance of a MessageClass initialized with a MessageWrapper + that contains the passed part documents. + + This is not the recommended way of obtaining a message, unless you know + how to take care of ensuring the internal consistency between the part + documents, or unless you are glueing together the part documents that + have been previously generated by `get_msg_from_string`. + + :param MessageClass: any Message class that can be initialized passing + an instance of an IMessageWrapper implementor. + :type MessageClass: type + :param fdoc: a dictionary containing values from which a + FlagsDocWrapper can be initialized + :type fdoc: dict + :param hdoc: a dictionary containing values from which a + HeaderDocWrapper can be initialized + :type hdoc: dict + :param cdocs: None, or a dictionary mapping integers (1-indexed) to + dicts from where a ContentDocWrapper can be initialized. + :type cdocs: dict, or None + + :rtype: MessageClass instance. + """ + assert(MessageClass is not None) + return MessageClass(MessageWrapper(mdoc, fdoc, hdoc, cdocs), uid=uid) + + def get_msg_from_mdoc_id(self, MessageClass, store, mdoc_id, + uid=None, get_cdocs=False): + + def wrap_meta_doc(doc): + cls = MetaMsgDocWrapper + return cls(doc_id=doc.doc_id, **doc.content) + + def get_part_docs_from_mdoc_wrapper(wrapper): + d_docs = [] + d_docs.append(store.get_doc(wrapper.fdoc)) + d_docs.append(store.get_doc(wrapper.hdoc)) + for cdoc in wrapper.cdocs: + d_docs.append(store.get_doc(cdoc)) + + def add_mdoc(doc_list): + return [wrapper.serialize()] + doc_list + + d = defer.gatherResults(d_docs) + d.addCallback(add_mdoc) + return d + + def get_parts_doc_from_mdoc_id(): + mbox = re.findall(constants.METAMSGID_MBOX_RE, mdoc_id)[0] + chash = re.findall(constants.METAMSGID_CHASH_RE, mdoc_id)[0] + + def _get_fdoc_id_from_mdoc_id(): + return constants.FDOCID.format(mbox_uuid=mbox, chash=chash) + + def _get_hdoc_id_from_mdoc_id(): + return constants.HDOCID.format(mbox_uuid=mbox, chash=chash) + + d_docs = [] + fdoc_id = _get_fdoc_id_from_mdoc_id() + hdoc_id = _get_hdoc_id_from_mdoc_id() + + d_docs.append(store.get_doc(mdoc_id)) + d_docs.append(store.get_doc(fdoc_id)) + d_docs.append(store.get_doc(hdoc_id)) + + d = defer.gatherResults(d_docs) + return d + + def _err_log_failure_part_docs(failure): + # See https://leap.se/code/issues/7495. + # This avoids blocks, but the real cause still needs to be + # isolated (0.9.0rc3) -- kali + log.msg("BUG ---------------------------------------------------") + log.msg("BUG: Error while retrieving part docs for mdoc id %s" % + mdoc_id) + log.err(failure) + log.msg("BUG (please report above info) ------------------------") + return [] + + def _err_log_cannot_find_msg(failure): + log.msg("BUG: Error while getting msg (uid=%s)" % uid) + return None + + if get_cdocs: + d = store.get_doc(mdoc_id) + d.addCallback(wrap_meta_doc) + d.addCallback(get_part_docs_from_mdoc_wrapper) + d.addErrback(_err_log_failure_part_docs) + + else: + d = get_parts_doc_from_mdoc_id() + + d.addCallback(self._get_msg_from_variable_doc_list, + msg_class=MessageClass, uid=uid) + d.addErrback(_err_log_cannot_find_msg) + return d + + def _get_msg_from_variable_doc_list(self, doc_list, msg_class, uid=None): + if len(doc_list) == 3: + mdoc, fdoc, hdoc = doc_list + cdocs = None + elif len(doc_list) > 3: + # XXX is this case used? + mdoc, fdoc, hdoc = doc_list[:3] + cdocs = dict(enumerate(doc_list[3:], 1)) + return self.get_msg_from_docs( + msg_class, mdoc, fdoc, hdoc, cdocs, uid=uid) + + def get_flags_from_mdoc_id(self, store, mdoc_id): + """ + # XXX stuff here... + """ + mbox = re.findall(constants.METAMSGID_MBOX_RE, mdoc_id)[0] + chash = re.findall(constants.METAMSGID_CHASH_RE, mdoc_id)[0] + + def _get_fdoc_id_from_mdoc_id(): + return constants.FDOCID.format(mbox_uuid=mbox, chash=chash) + + fdoc_id = _get_fdoc_id_from_mdoc_id() + + def wrap_fdoc(doc): + if not doc: + return + cls = FlagsDocWrapper + return cls(doc_id=doc.doc_id, **doc.content) + + def get_flags(fdoc_wrapper): + if not fdoc_wrapper: + return [] + return fdoc_wrapper.get_flags() + + d = store.get_doc(fdoc_id) + d.addCallback(wrap_fdoc) + d.addCallback(get_flags) + return d + + def create_msg(self, store, msg): + """ + :param store: an instance of soledad, or anything that behaves alike + :param msg: a Message object. + + :return: a Deferred that is fired when all the underlying documents + have been created. + :rtype: defer.Deferred + """ + wrapper = msg.get_wrapper() + return wrapper.create(store) + + def update_msg(self, store, msg): + """ + :param msg: a Message object. + :param store: an instance of soledad, or anything that behaves alike + :return: a Deferred that is fired when all the underlying documents + have been updated (actually, it's only the fdoc that's allowed + to update). + :rtype: defer.Deferred + """ + wrapper = msg.get_wrapper() + return wrapper.update(store) + + # batch deletion + + def del_all_flagged_messages(self, store, mbox_uuid): + """ + Delete all messages flagged as deleted. + """ + def err(failure): + log.err(failure) + + def delete_fdoc_and_mdoc_flagged(fdocs): + # low level here, not using the wrappers... + # get meta doc ids from the flag doc ids + fdoc_ids = [doc.doc_id for doc in fdocs] + mdoc_ids = map(lambda s: "M" + s[1:], fdoc_ids) + + def delete_all_docs(mdocs, fdocs): + mdocs = list(mdocs) + doc_ids = [m.doc_id for m in mdocs] + _d = [] + docs = mdocs + fdocs + for doc in docs: + _d.append(store.delete_doc(doc)) + d = defer.gatherResults(_d) + # return the mdocs ids only + d.addCallback(lambda _: doc_ids) + return d + + d = store.get_docs(mdoc_ids) + d.addCallback(delete_all_docs, fdocs) + d.addErrback(err) + return d + + type_ = FlagsDocWrapper.model.type_ + uuid = mbox_uuid.replace('-', '_') + deleted_index = indexes.TYPE_MBOX_DEL_IDX + + d = store.get_from_index(deleted_index, type_, uuid, "1") + d.addCallbacks(delete_fdoc_and_mdoc_flagged, err) + return d + + # count messages + + def get_count_unseen(self, store, mbox_uuid): + """ + Get the number of unseen messages for a given mailbox. + + :param store: instance of Soledad. + :param mbox_uuid: the uuid for this mailbox. + :rtype: int + """ + type_ = FlagsDocWrapper.model.type_ + uuid = mbox_uuid.replace('-', '_') + + unseen_index = indexes.TYPE_MBOX_SEEN_IDX + + d = store.get_count_from_index(unseen_index, type_, uuid, "0") + d.addErrback(self._errback) + return d + + def get_count_recent(self, store, mbox_uuid): + """ + Get the number of recent messages for a given mailbox. + + :param store: instance of Soledad. + :param mbox_uuid: the uuid for this mailbox. + :rtype: int + """ + type_ = FlagsDocWrapper.model.type_ + uuid = mbox_uuid.replace('-', '_') + + recent_index = indexes.TYPE_MBOX_RECENT_IDX + + d = store.get_count_from_index(recent_index, type_, uuid, "1") + d.addErrback(self._errback) + return d + + # search api + + def get_mdoc_id_from_msgid(self, store, mbox_uuid, msgid): + """ + Get the UID for a message with the passed msgid (the one in the headers + msg-id). + This is used by the MUA to retrieve the recently saved draft. + """ + type_ = HeaderDocWrapper.model.type_ + uuid = mbox_uuid.replace('-', '_') + + msgid_index = indexes.TYPE_MSGID_IDX + + def get_mdoc_id(hdoc): + if not hdoc: + log.msg("Could not find a HDOC with MSGID %s" % msgid) + return None + hdoc = hdoc[0] + mdoc_id = hdoc.doc_id.replace("H-", "M-%s-" % uuid) + return mdoc_id + + d = store.get_from_index(msgid_index, type_, msgid) + d.addCallback(get_mdoc_id) + return d + + # Mailbox handling + + def get_or_create_mbox(self, store, name): + """ + Get the mailbox with the given name, or create one if it does not + exist. + + :param store: instance of Soledad + :param name: the name of the mailbox + :type name: str + """ + index = indexes.TYPE_MBOX_IDX + mbox = normalize_mailbox(name) + return MailboxWrapper.get_or_create(store, index, mbox) + + def update_mbox(self, store, mbox_wrapper): + """ + Update the documents for a given mailbox. + :param mbox_wrapper: MailboxWrapper instance + :type mbox_wrapper: MailboxWrapper + :return: a Deferred that will be fired when the mailbox documents + have been updated. + :rtype: defer.Deferred + """ + leap_assert_type(mbox_wrapper, SoledadDocumentWrapper) + return mbox_wrapper.update(store) + + def delete_mbox(self, store, mbox_wrapper): + leap_assert_type(mbox_wrapper, SoledadDocumentWrapper) + return mbox_wrapper.delete(store) + + def get_all_mboxes(self, store): + """ + Retrieve a list with wrappers for all the mailboxes. + + :return: a deferred that will be fired with a list of all the + MailboxWrappers found. + :rtype: defer.Deferred + """ + return MailboxWrapper.get_all(store) + + def _errback(self, failure): + log.err(failure) + + +def _split_into_parts(raw): + # TODO signal that we can delete the original message!----- + # when all the processing is done. + # TODO add the linked-from info ! + # TODO add reference to the original message? + # TODO populate Default FLAGS/TAGS (unseen?) + # TODO seed propely the content_docs with defaults?? + + msg, chash, multi = _parse_msg(raw) + size = len(msg.as_string()) + + parts_map = walk.get_tree(msg) + cdocs_list = list(walk.get_raw_docs(msg)) + cdocs_phashes = [c['phash'] for c in cdocs_list] + body_phash = walk.get_body_phash(msg) + + mdoc = _build_meta_doc(chash, cdocs_phashes) + fdoc = _build_flags_doc(chash, size, multi) + hdoc = _build_headers_doc(msg, chash, body_phash, parts_map) + + # The MessageWrapper expects a dict, one-indexed + cdocs = dict(enumerate(cdocs_list, 1)) + + return mdoc, fdoc, hdoc, cdocs + + +def _parse_msg(raw): + msg = message_from_string(raw) + chash = walk.get_hash(raw) + multi = msg.is_multipart() + return msg, chash, multi + + +def _build_meta_doc(chash, cdocs_phashes): + _mdoc = MetaMsgDocWrapper() + # FIXME passing the inbox name because we don't have the uuid at this + # point. + + _mdoc.fdoc = constants.FDOCID.format(mbox_uuid=INBOX_NAME, chash=chash) + _mdoc.hdoc = constants.HDOCID.format(chash=chash) + _mdoc.cdocs = [constants.CDOCID.format(phash=p) for p in cdocs_phashes] + + return _mdoc.serialize() + + +def _build_flags_doc(chash, size, multi): + _fdoc = FlagsDocWrapper(chash=chash, size=size, multi=multi) + return _fdoc.serialize() + + +def _build_headers_doc(msg, chash, body_phash, parts_map): + """ + Assemble a headers document from the original parsed message, the + content-hash, and the parts map. + + It takes into account possibly repeated headers. + """ + headers = defaultdict(list) + for k, v in msg.items(): + headers[k].append(v) + # "fix" for repeated headers (as in "Received:" + for k, v in headers.items(): + newline = "\n%s: " % (k.lower(),) + headers[k] = newline.join(v) + + lower_headers = lowerdict(dict(headers)) + msgid = first(_MSGID_RE.findall( + lower_headers.get('message-id', ''))) + + _hdoc = HeaderDocWrapper( + chash=chash, headers=headers, body=body_phash, + msgid=msgid) + + def copy_attr(headers, key, doc): + if key in headers: + setattr(doc, key, headers[key]) + + copy_attr(lower_headers, "subject", _hdoc) + copy_attr(lower_headers, "date", _hdoc) + + hdoc = _hdoc.serialize() + # add some of the attr from the parts map to header doc + for key in parts_map: + if key in ('body', 'multi', 'part_map'): + hdoc[key] = parts_map[key] + return stringify_parts_map(hdoc) diff --git a/src/leap/bitmask/mail/adaptors/soledad_indexes.py b/src/leap/bitmask/mail/adaptors/soledad_indexes.py new file mode 100644 index 0000000..eec7d28 --- /dev/null +++ b/src/leap/bitmask/mail/adaptors/soledad_indexes.py @@ -0,0 +1,106 @@ +# -*- coding: utf-8 -*- +# soledad_indexes.py +# Copyright (C) 2013, 2014 LEAP +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . +""" +Soledad Indexes for Mail Documents. +""" + +# TODO +# [ ] hide most of the constants here + +# Document Type, for indexing + +TYPE = "type" +MBOX = "mbox" +MBOX_UUID = "mbox_uuid" +FLAGS = "flags" +HEADERS = "head" +CONTENT = "cnt" +RECENT = "rct" +HDOCS_SET = "hdocset" + +INCOMING_KEY = "incoming" +ERROR_DECRYPTING_KEY = "errdecr" + +# indexing keys +CONTENT_HASH = "chash" +PAYLOAD_HASH = "phash" +MSGID = "msgid" +UID = "uid" + + +# Index types +# -------------- + +TYPE_IDX = 'by-type' +TYPE_MBOX_IDX = 'by-type-and-mbox' +TYPE_MBOX_UUID_IDX = 'by-type-and-mbox-uuid' +TYPE_SUBS_IDX = 'by-type-and-subscribed' +TYPE_MSGID_IDX = 'by-type-and-message-id' +TYPE_MBOX_SEEN_IDX = 'by-type-and-mbox-and-seen' +TYPE_MBOX_RECENT_IDX = 'by-type-and-mbox-and-recent' +TYPE_MBOX_DEL_IDX = 'by-type-and-mbox-and-deleted' +TYPE_MBOX_C_HASH_IDX = 'by-type-and-mbox-and-contenthash' +TYPE_C_HASH_IDX = 'by-type-and-contenthash' +TYPE_C_HASH_PART_IDX = 'by-type-and-contenthash-and-partnumber' +TYPE_P_HASH_IDX = 'by-type-and-payloadhash' + +# Soledad index for incoming mail, without decrypting errors. +# and the backward-compatible index, will be deprecated at 0.7 +JUST_MAIL_IDX = "just-mail" +JUST_MAIL_COMPAT_IDX = "just-mail-compat" + + +# TODO +# it would be nice to measure the cost of indexing +# by many fields. + +# TODO +# make the indexes dict more readable! + +MAIL_INDEXES = { + # generic + TYPE_IDX: [TYPE], + TYPE_MBOX_IDX: [TYPE, MBOX], + TYPE_MBOX_UUID_IDX: [TYPE, MBOX_UUID], + + # XXX deprecate 0.4.0 + # TYPE_MBOX_UID_IDX: [TYPE, MBOX, UID], + + # mailboxes + TYPE_SUBS_IDX: [TYPE, 'bool(subscribed)'], + + # fdocs uniqueness + TYPE_MBOX_C_HASH_IDX: [TYPE, MBOX, CONTENT_HASH], + + # headers doc - search by msgid. + TYPE_MSGID_IDX: [TYPE, MSGID], + + # content, headers doc + TYPE_C_HASH_IDX: [TYPE, CONTENT_HASH], + + # attachment payload dedup + TYPE_P_HASH_IDX: [TYPE, PAYLOAD_HASH], + + # messages + TYPE_MBOX_SEEN_IDX: [TYPE, MBOX_UUID, 'bool(seen)'], + TYPE_MBOX_RECENT_IDX: [TYPE, MBOX_UUID, 'bool(recent)'], + TYPE_MBOX_DEL_IDX: [TYPE, MBOX_UUID, 'bool(deleted)'], + + # incoming queue + JUST_MAIL_IDX: ["bool(%s)" % (INCOMING_KEY,), + "bool(%s)" % (ERROR_DECRYPTING_KEY,)], +} diff --git a/src/leap/bitmask/mail/adaptors/tests/rfc822.message b/src/leap/bitmask/mail/adaptors/tests/rfc822.message new file mode 120000 index 0000000..b19cc28 --- /dev/null +++ b/src/leap/bitmask/mail/adaptors/tests/rfc822.message @@ -0,0 +1 @@ +../../tests/rfc822.message \ No newline at end of file diff --git a/src/leap/bitmask/mail/adaptors/tests/test_models.py b/src/leap/bitmask/mail/adaptors/tests/test_models.py new file mode 100644 index 0000000..b82cfad --- /dev/null +++ b/src/leap/bitmask/mail/adaptors/tests/test_models.py @@ -0,0 +1,106 @@ +# -*- coding: utf-8 -*- +# test_models.py +# Copyright (C) 2014 LEAP +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . +""" +Tests for the leap.mail.adaptors.models module. +""" +from twisted.trial import unittest + +from leap.mail.adaptors import models + + +class SerializableModelsTestCase(unittest.TestCase): + + def test_good_serialized_model(self): + + class M(models.SerializableModel): + foo = 42 + bar = 33 + baaz_ = None + _nope = 0 + __nope = 0 + + def not_today(self): + pass + + class IgnoreMe(object): + pass + + def killmeplease(x): + return x + + serialized = M.serialize() + expected = {'foo': 42, 'bar': 33, 'baaz': None} + self.assertEqual(serialized, expected) + + +class DocumentWrapperTestCase(unittest.TestCase): + + def test_wrapper_defaults(self): + + class Wrapper(models.DocumentWrapper): + class model(models.SerializableModel): + foo = 42 + bar = 11 + + wrapper = Wrapper() + wrapper._ignored = True + serialized = wrapper.serialize() + expected = {'foo': 42, 'bar': 11} + self.assertEqual(serialized, expected) + + def test_initialized_wrapper(self): + + class Wrapper(models.DocumentWrapper): + class model(models.SerializableModel): + foo = 42 + bar_ = 11 + + wrapper = Wrapper(foo=0, bar=-1) + serialized = wrapper.serialize() + expected = {'foo': 0, 'bar': -1} + self.assertEqual(serialized, expected) + + wrapper.foo = 23 + serialized = wrapper.serialize() + expected = {'foo': 23, 'bar': -1} + self.assertEqual(serialized, expected) + + wrapper = Wrapper(foo=0) + serialized = wrapper.serialize() + expected = {'foo': 0, 'bar': 11} + self.assertEqual(serialized, expected) + + def test_invalid_initialized_wrapper(self): + + class Wrapper(models.DocumentWrapper): + class model(models.SerializableModel): + foo = 42 + + def getwrapper(): + return Wrapper(bar=1) + self.assertRaises(RuntimeError, getwrapper) + + def test_no_model_wrapper(self): + + class Wrapper(models.DocumentWrapper): + pass + + def getwrapper(): + w = Wrapper() + w.foo = None + + self.assertRaises(RuntimeError, getwrapper) diff --git a/src/leap/bitmask/mail/adaptors/tests/test_soledad_adaptor.py b/src/leap/bitmask/mail/adaptors/tests/test_soledad_adaptor.py new file mode 100644 index 0000000..73eaf16 --- /dev/null +++ b/src/leap/bitmask/mail/adaptors/tests/test_soledad_adaptor.py @@ -0,0 +1,529 @@ +# -*- coding: utf-8 -*- +# test_soledad_adaptor.py +# Copyright (C) 2014 LEAP +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . +""" +Tests for the Soledad Adaptor module - leap.mail.adaptors.soledad +""" +import os +from functools import partial + +from twisted.internet import defer + +from leap.mail.adaptors import models +from leap.mail.adaptors.soledad import SoledadDocumentWrapper +from leap.mail.adaptors.soledad import SoledadIndexMixin +from leap.mail.adaptors.soledad import SoledadMailAdaptor +from leap.mail.testing.common import SoledadTestMixin + +from email.MIMEMultipart import MIMEMultipart +from email.mime.text import MIMEText + +# DEBUG +# import logging +# logging.basicConfig(level=logging.DEBUG) + + +class CounterWrapper(SoledadDocumentWrapper): + class model(models.SerializableModel): + counter = 0 + flag = None + + +class CharacterWrapper(SoledadDocumentWrapper): + class model(models.SerializableModel): + name = "" + age = 20 + + +class ActorWrapper(SoledadDocumentWrapper): + class model(models.SerializableModel): + type_ = "actor" + name = None + + class __meta__(object): + index = "name" + list_index = ("by-type", "type_") + + +class TestAdaptor(SoledadIndexMixin): + indexes = {'by-name': ['name'], + 'by-type-and-name': ['type', 'name'], + 'by-type': ['type']} + + +class SoledadDocWrapperTestCase(SoledadTestMixin): + """ + Tests for the SoledadDocumentWrapper. + """ + def assert_num_docs(self, num, docs): + self.assertEqual(len(docs[1]), num) + + def test_create_single(self): + + store = self._soledad + wrapper = CounterWrapper() + + def assert_one_doc(docs): + self.assertEqual(docs[0], 1) + + d = wrapper.create(store) + d.addCallback(lambda _: store.get_all_docs()) + d.addCallback(assert_one_doc) + return d + + def test_create_many(self): + + store = self._soledad + w1 = CounterWrapper() + w2 = CounterWrapper(counter=1) + w3 = CounterWrapper(counter=2) + w4 = CounterWrapper(counter=3) + w5 = CounterWrapper(counter=4) + + d1 = [w1.create(store), + w2.create(store), + w3.create(store), + w4.create(store), + w5.create(store)] + + d = defer.gatherResults(d1) + d.addCallback(lambda _: store.get_all_docs()) + d.addCallback(partial(self.assert_num_docs, 5)) + return d + + def test_multiple_updates(self): + + store = self._soledad + wrapper = CounterWrapper(counter=1) + MAX = 100 + + def assert_doc_id(doc): + self.assertTrue(wrapper._doc_id is not None) + return doc + + def assert_counter_initial_ok(doc): + self.assertEqual(wrapper.counter, 1) + + def increment_counter(ignored): + d1 = [] + + def record_revision(revision): + rev = int(revision.split(':')[1]) + self.results.append(rev) + + for i in list(range(MAX)): + wrapper.counter += 1 + wrapper.flag = i % 2 == 0 + d = wrapper.update(store) + d.addCallback(record_revision) + d1.append(d) + + return defer.gatherResults(d1) + + def assert_counter_final_ok(doc): + self.assertEqual(doc.content['counter'], MAX + 1) + self.assertEqual(doc.content['flag'], False) + + def assert_results_ordered_list(ignored): + self.assertEqual(self.results, sorted(range(2, MAX + 2))) + + d = wrapper.create(store) + d.addCallback(assert_doc_id) + d.addCallback(assert_counter_initial_ok) + d.addCallback(increment_counter) + d.addCallback(lambda _: store.get_doc(wrapper._doc_id)) + d.addCallback(assert_counter_final_ok) + d.addCallback(assert_results_ordered_list) + return d + + def test_delete(self): + adaptor = TestAdaptor() + store = self._soledad + + wrapper_list = [] + + def get_or_create_bob(ignored): + def add_to_list(wrapper): + wrapper_list.append(wrapper) + return wrapper + wrapper = CharacterWrapper.get_or_create( + store, 'by-name', 'bob') + wrapper.addCallback(add_to_list) + return wrapper + + def delete_bob(ignored): + wrapper = wrapper_list[0] + return wrapper.delete(store) + + d = adaptor.initialize_store(store) + d.addCallback(lambda _: store.get_all_docs()) + d.addCallback(partial(self.assert_num_docs, 0)) + + # this should create bob document + d.addCallback(get_or_create_bob) + d.addCallback(lambda _: store.get_all_docs()) + d.addCallback(partial(self.assert_num_docs, 1)) + + d.addCallback(delete_bob) + d.addCallback(lambda _: store.get_all_docs()) + d.addCallback(partial(self.assert_num_docs, 0)) + return d + + def test_get_or_create(self): + adaptor = TestAdaptor() + store = self._soledad + + def get_or_create_bob(ignored): + wrapper = CharacterWrapper.get_or_create( + store, 'by-name', 'bob') + return wrapper + + d = adaptor.initialize_store(store) + d.addCallback(lambda _: store.get_all_docs()) + d.addCallback(partial(self.assert_num_docs, 0)) + + # this should create bob document + d.addCallback(get_or_create_bob) + d.addCallback(lambda _: store.get_all_docs()) + d.addCallback(partial(self.assert_num_docs, 1)) + + # this should get us bob document + d.addCallback(get_or_create_bob) + d.addCallback(lambda _: store.get_all_docs()) + d.addCallback(partial(self.assert_num_docs, 1)) + return d + + def test_get_or_create_multi_index(self): + adaptor = TestAdaptor() + store = self._soledad + + def get_or_create_actor_harry(ignored): + wrapper = ActorWrapper.get_or_create( + store, 'by-type-and-name', 'harrison') + return wrapper + + def create_director_harry(ignored): + wrapper = ActorWrapper(name="harrison", type="director") + return wrapper.create(store) + + d = adaptor.initialize_store(store) + d.addCallback(lambda _: store.get_all_docs()) + d.addCallback(partial(self.assert_num_docs, 0)) + + # this should create harrison document + d.addCallback(get_or_create_actor_harry) + d.addCallback(lambda _: store.get_all_docs()) + d.addCallback(partial(self.assert_num_docs, 1)) + + # this should get us harrison document + d.addCallback(get_or_create_actor_harry) + d.addCallback(lambda _: store.get_all_docs()) + d.addCallback(partial(self.assert_num_docs, 1)) + + # create director harry, should create new doc + d.addCallback(create_director_harry) + d.addCallback(lambda _: store.get_all_docs()) + d.addCallback(partial(self.assert_num_docs, 2)) + + # this should get us harrison document, still 2 docs + d.addCallback(get_or_create_actor_harry) + d.addCallback(lambda _: store.get_all_docs()) + d.addCallback(partial(self.assert_num_docs, 2)) + return d + + def test_get_all(self): + adaptor = TestAdaptor() + store = self._soledad + actor_names = ["harry", "carrie", "mark", "david"] + + def create_some_actors(ignored): + deferreds = [] + for name in actor_names: + dw = ActorWrapper.get_or_create( + store, 'by-type-and-name', name) + deferreds.append(dw) + return defer.gatherResults(deferreds) + + d = adaptor.initialize_store(store) + d.addCallback(lambda _: store.get_all_docs()) + d.addCallback(partial(self.assert_num_docs, 0)) + + d.addCallback(create_some_actors) + + d.addCallback(lambda _: store.get_all_docs()) + d.addCallback(partial(self.assert_num_docs, 4)) + + def assert_actor_list_is_expected(res): + got = set([actor.name for actor in res]) + expected = set(actor_names) + self.assertEqual(got, expected) + + d.addCallback(lambda _: ActorWrapper.get_all(store)) + d.addCallback(assert_actor_list_is_expected) + return d + +HERE = os.path.split(os.path.abspath(__file__))[0] + + +class MessageClass(object): + def __init__(self, wrapper, uid): + self.wrapper = wrapper + self.uid = uid + + def get_wrapper(self): + return self.wrapper + + +class SoledadMailAdaptorTestCase(SoledadTestMixin): + """ + Tests for the SoledadMailAdaptor. + """ + + def get_adaptor(self): + adaptor = SoledadMailAdaptor() + adaptor.store = self._soledad + return adaptor + + def assert_num_docs(self, num, docs): + self.assertEqual(len(docs[1]), num) + + def test_mail_adaptor_init(self): + adaptor = self.get_adaptor() + self.assertTrue(isinstance(adaptor.indexes, dict)) + self.assertTrue(len(adaptor.indexes) != 0) + + # Messages + + def test_get_msg_from_string(self): + adaptor = self.get_adaptor() + + with open(os.path.join(HERE, "rfc822.message")) as f: + raw = f.read() + + msg = adaptor.get_msg_from_string(MessageClass, raw) + + chash = ("D27B2771C0DCCDCB468EE65A4540438" + "09DBD11588E87E951545BE0CBC321C308") + phash = ("64934534C1C80E0D4FA04BE1CCBA104" + "F07BCA5F469C86E2C0ABE1D41310B7299") + subject = ("[Twisted-commits] rebuild now works on " + "python versions from 2.2.0 and up.") + self.assertTrue(msg.wrapper.fdoc is not None) + self.assertTrue(msg.wrapper.hdoc is not None) + self.assertTrue(msg.wrapper.cdocs is not None) + self.assertEquals(len(msg.wrapper.cdocs), 1) + self.assertEquals(msg.wrapper.fdoc.chash, chash) + self.assertEquals(msg.wrapper.fdoc.size, 3837) + self.assertEquals(msg.wrapper.hdoc.chash, chash) + self.assertEqual(dict(msg.wrapper.hdoc.headers)['Subject'], + subject) + self.assertEqual(msg.wrapper.hdoc.subject, subject) + self.assertEqual(msg.wrapper.cdocs[1].phash, phash) + + def test_get_msg_from_string_multipart(self): + msg = MIMEMultipart() + msg['Subject'] = 'Test multipart mail' + msg.attach(MIMEText(u'a utf8 message', _charset='utf-8')) + adaptor = self.get_adaptor() + + msg = adaptor.get_msg_from_string(MessageClass, msg.as_string()) + + self.assertEqual( + 'base64', msg.wrapper.cdocs[1].content_transfer_encoding) + self.assertEqual( + 'text/plain', msg.wrapper.cdocs[1].content_type) + self.assertEqual( + 'YSB1dGY4IG1lc3NhZ2U=\n', msg.wrapper.cdocs[1].raw) + + def test_get_msg_from_docs(self): + adaptor = self.get_adaptor() + mdoc = dict( + fdoc="F-Foobox-deadbeef", + hdoc="H-deadbeef", + cdocs=["C-deadabad"]) + fdoc = dict( + mbox_uuid="Foobox", + flags=('\Seen', '\Nice'), + tags=('Personal', 'TODO'), + seen=False, deleted=False, + recent=False, multi=False) + hdoc = dict( + chash="deadbeef", + subject="Test Msg") + cdocs = { + 1: dict( + raw='This is a test message')} + + msg = adaptor.get_msg_from_docs( + MessageClass, mdoc, fdoc, hdoc, cdocs=cdocs) + self.assertEqual(msg.wrapper.fdoc.flags, + ('\Seen', '\Nice')) + self.assertEqual(msg.wrapper.fdoc.tags, + ('Personal', 'TODO')) + self.assertEqual(msg.wrapper.fdoc.mbox_uuid, "Foobox") + self.assertEqual(msg.wrapper.hdoc.multi, False) + self.assertEqual(msg.wrapper.hdoc.subject, + "Test Msg") + self.assertEqual(msg.wrapper.cdocs[1].raw, + "This is a test message") + + def test_get_msg_from_metamsg_doc_id(self): + # TODO complete-me! + pass + + test_get_msg_from_metamsg_doc_id.skip = "Not yet implemented" + + def test_create_msg(self): + adaptor = self.get_adaptor() + + with open(os.path.join(HERE, "rfc822.message")) as f: + raw = f.read() + msg = adaptor.get_msg_from_string(MessageClass, raw) + + def check_create_result(created): + # that's one mdoc, one hdoc, one fdoc, one cdoc + self.assertEqual(len(created), 4) + for doc in created: + self.assertTrue( + doc.__class__.__name__, + "SoledadDocument") + + d = adaptor.create_msg(adaptor.store, msg) + d.addCallback(check_create_result) + return d + + def test_update_msg(self): + adaptor = self.get_adaptor() + with open(os.path.join(HERE, "rfc822.message")) as f: + raw = f.read() + + def assert_msg_has_doc_id(ignored, msg): + wrapper = msg.get_wrapper() + self.assertTrue(wrapper.fdoc.doc_id is not None) + + def assert_msg_has_no_flags(ignored, msg): + wrapper = msg.get_wrapper() + self.assertEqual(wrapper.fdoc.flags, []) + + def update_msg_flags(ignored, msg): + wrapper = msg.get_wrapper() + wrapper.fdoc.flags = ["This", "That"] + return wrapper.update(adaptor.store) + + def assert_msg_has_flags(ignored, msg): + wrapper = msg.get_wrapper() + self.assertEqual(wrapper.fdoc.flags, ["This", "That"]) + + def get_fdoc_and_check_flags(ignored): + def assert_doc_has_flags(doc): + self.assertEqual(doc.content['flags'], + ['This', 'That']) + wrapper = msg.get_wrapper() + d = adaptor.store.get_doc(wrapper.fdoc.doc_id) + d.addCallback(assert_doc_has_flags) + return d + + msg = adaptor.get_msg_from_string(MessageClass, raw) + d = adaptor.create_msg(adaptor.store, msg) + d.addCallback(lambda _: adaptor.store.get_all_docs()) + d.addCallback(partial(self.assert_num_docs, 4)) + d.addCallback(assert_msg_has_doc_id, msg) + d.addCallback(assert_msg_has_no_flags, msg) + + # update it! + d.addCallback(update_msg_flags, msg) + d.addCallback(assert_msg_has_flags, msg) + d.addCallback(get_fdoc_and_check_flags) + return d + + # Mailboxes + + def test_get_or_create_mbox(self): + adaptor = self.get_adaptor() + + def get_or_create_mbox(ignored): + d = adaptor.get_or_create_mbox(adaptor.store, "Trash") + return d + + def assert_good_doc(mbox_wrapper): + self.assertTrue(mbox_wrapper.doc_id is not None) + self.assertEqual(mbox_wrapper.mbox, "Trash") + self.assertEqual(mbox_wrapper.type, "mbox") + self.assertEqual(mbox_wrapper.closed, False) + self.assertEqual(mbox_wrapper.subscribed, False) + + d = adaptor.initialize_store(adaptor.store) + d.addCallback(get_or_create_mbox) + d.addCallback(assert_good_doc) + d.addCallback(lambda _: adaptor.store.get_all_docs()) + d.addCallback(partial(self.assert_num_docs, 1)) + return d + + def test_update_mbox(self): + adaptor = self.get_adaptor() + + wrapper_ref = [] + + def get_or_create_mbox(ignored): + d = adaptor.get_or_create_mbox(adaptor.store, "Trash") + return d + + def update_wrapper(wrapper, wrapper_ref): + wrapper_ref.append(wrapper) + wrapper.subscribed = True + wrapper.closed = True + d = adaptor.update_mbox(adaptor.store, wrapper) + return d + + def get_mbox_doc_and_check_flags(res, wrapper_ref): + wrapper = wrapper_ref[0] + + def assert_doc_has_flags(doc): + self.assertEqual(doc.content['subscribed'], True) + self.assertEqual(doc.content['closed'], True) + d = adaptor.store.get_doc(wrapper.doc_id) + d.addCallback(assert_doc_has_flags) + return d + + d = adaptor.initialize_store(adaptor.store) + d.addCallback(get_or_create_mbox) + d.addCallback(update_wrapper, wrapper_ref) + d.addCallback(get_mbox_doc_and_check_flags, wrapper_ref) + return d + + def test_get_all_mboxes(self): + adaptor = self.get_adaptor() + mboxes = ("Sent", "Trash", "Personal", "ListFoo") + + def get_or_create_mboxes(ignored): + d = [] + for mbox in mboxes: + d.append(adaptor.get_or_create_mbox( + adaptor.store, mbox)) + return defer.gatherResults(d) + + def get_all_mboxes(ignored): + return adaptor.get_all_mboxes(adaptor.store) + + def assert_mboxes_match_expected(wrappers): + names = [m.mbox for m in wrappers] + self.assertEqual(set(names), set(mboxes)) + + d = adaptor.initialize_store(adaptor.store) + d.addCallback(get_or_create_mboxes) + d.addCallback(get_all_mboxes) + d.addCallback(assert_mboxes_match_expected) + return d diff --git a/src/leap/bitmask/mail/constants.py b/src/leap/bitmask/mail/constants.py new file mode 100644 index 0000000..4ef42cb --- /dev/null +++ b/src/leap/bitmask/mail/constants.py @@ -0,0 +1,52 @@ +# *- coding: utf-8 -*- +# constants.py +# Copyright (C) 2014 LEAP +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . +""" +Constants for leap.mail. +""" + +INBOX_NAME = "INBOX" + +# Regular expressions for the identifiers to be used in the Message Data Layer. + +METAMSGID = "M-{mbox_uuid}-{chash}" +METAMSGID_RE = "M\-{mbox_uuid}\-[0-9a-fA-F]+" +METAMSGID_CHASH_RE = "M\-\w+\-([0-9a-fA-F]+)" +METAMSGID_MBOX_RE = "M\-(\w+)\-[0-9a-fA-F]+" + +FDOCID = "F-{mbox_uuid}-{chash}" +FDOCID_RE = "F\-{mbox_uuid}\-[0-9a-fA-F]+" +FDOCID_CHASH_RE = "F\-\w+\-([0-9a-fA-F]+)" + +HDOCID = "H-{chash}" +HDOCID_RE = "H\-[0-9a-fA-F]+" + +CDOCID = "C-{phash}" +CDOCID_RE = "C\-[0-9a-fA-F]+" + + +class MessageFlags(object): + """ + Flags used in Message and Mailbox. + """ + SEEN_FLAG = "\\Seen" + RECENT_FLAG = "\\Recent" + ANSWERED_FLAG = "\\Answered" + FLAGGED_FLAG = "\\Flagged" # yo dawg + DELETED_FLAG = "\\Deleted" + DRAFT_FLAG = "\\Draft" + NOSELECT_FLAG = "\\Noselect" + LIST_FLAG = "List" # is this OK? (no \. ie, no system flag) diff --git a/src/leap/bitmask/mail/cred.py b/src/leap/bitmask/mail/cred.py new file mode 100644 index 0000000..7eab1f0 --- /dev/null +++ b/src/leap/bitmask/mail/cred.py @@ -0,0 +1,80 @@ +# -*- coding: utf-8 -*- +# cred.py +# Copyright (C) 2015 LEAP +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . +""" +Credentials handling. +""" + +from zope.interface import implementer +from twisted.cred.checkers import ICredentialsChecker +from twisted.cred.credentials import IUsernamePassword +from twisted.cred.error import UnauthorizedLogin +from twisted.internet import defer + + +@implementer(ICredentialsChecker) +class LocalSoledadTokenChecker(object): + + """ + A Credentials Checker for a LocalSoledad store. + + It checks that: + + 1) The Local SoledadStorage has been correctly unlocked for the given + user. This currently means that the right passphrase has been passed + to the Local SoledadStorage. + + 2) The password passed in the credentials matches whatever token has + been stored in the local encrypted SoledadStorage, associated to the + Protocol that is requesting the authentication. + """ + + credentialInterfaces = (IUsernamePassword,) + service = None + + def __init__(self, soledad_sessions): + """ + :param soledad_sessions: a dict-like object, containing instances + of a Store (soledad instances), indexed by + userid. + """ + self._soledad_sessions = soledad_sessions + + def requestAvatarId(self, credentials): + if self.service is None: + raise NotImplementedError( + "this checker has not defined its service name") + username, password = credentials.username, credentials.password + d = self.checkSoledadToken(username, password, self.service) + d.addErrback(lambda f: defer.fail(UnauthorizedLogin())) + return d + + def checkSoledadToken(self, username, password, service): + soledad = self._soledad_sessions.get(username) + if not soledad: + return defer.fail(Exception("No soledad")) + + def match_token(token): + if token is None: + raise RuntimeError('no token') + if token == password: + return username + else: + raise RuntimeError('bad token') + + d = soledad.get_or_create_service_token(service) + d.addCallback(match_token) + return d diff --git a/src/leap/bitmask/mail/decorators.py b/src/leap/bitmask/mail/decorators.py new file mode 100644 index 0000000..5105de9 --- /dev/null +++ b/src/leap/bitmask/mail/decorators.py @@ -0,0 +1,149 @@ +# -*- coding: utf-8 -*- +# decorators.py +# Copyright (C) 2013 LEAP +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . +""" +Useful decorators for mail package. +""" +import logging +import os + +from functools import wraps + +from twisted.internet.threads import deferToThread + + +logger = logging.getLogger(__name__) + + +# TODO +# Should write a helper to be able to pass a timeout argument. +# See this answer: http://stackoverflow.com/a/19019648/1157664 +# And the notes by glyph and jpcalderone + +def deferred_to_thread(f): + """ + Decorator, for deferring methods to Threads. + + It will do a deferToThread of the decorated method + unless the environment variable LEAPMAIL_DEBUG is set. + + It uses a descriptor to delay the definition of the + method wrapper. + """ + class descript(object): + """ + The class to be used as decorator. + + It takes any method as the passed object. + """ + + def __init__(self, f): + """ + Initializes the decorator object. + + :param f: the decorated function + :type f: callable + """ + self.f = f + + def __get__(self, instance, klass): + """ + Descriptor implementation. + + At creation time, the decorated `method` is unbound. + + It will dispatch the make_unbound method if we still do not + have an instance available, and the make_bound method when the + method has already been bound to the instance. + + :param instance: the instance of the class, or None if not exist. + :type instance: instantiated class or None. + """ + if instance is None: + # Class method was requested + return self.make_unbound(klass) + return self.make_bound(instance) + + def _errback(self, failure): + """ + Errorback that logs the exception catched. + + :param failure: a twisted failure + :type failure: Failure + """ + logger.warning('Error in method: %s' % (self.f.__name__)) + logger.exception(failure.getTraceback()) + + def make_unbound(self, klass): + """ + Return a wrapped function with the unbound call, during the + early access to the decortad method. This gets passed + only the class (not the instance since it does not yet exist). + + :param klass: the class to which the still unbound method belongs + :type klass: type + """ + + @wraps(self.f) + def wrapper(*args, **kwargs): + """ + We're temporarily wrapping the decorated method, but this + should not be called, since our application should use + the bound-wrapped method after this decorator class has been + used. + + This documentation will vanish at runtime. + """ + raise TypeError( + 'unbound method {}() must be called with {} instance ' + 'as first argument (got nothing instead)'.format( + self.f.__name__, + klass.__name__) + ) + return wrapper + + def make_bound(self, instance): + """ + Return a function that wraps the bound method call, + after we are able to access the instance object. + + :param instance: an instance of the class the decorated method, + now bound, belongs to. + :type instance: object + """ + + @wraps(self.f) + def wrapper(*args, **kwargs): + """ + Do a proper function wrapper that defers the decorated method + call to a separated thread if the LEAPMAIL_DEBUG + environment variable is set. + + This documentation will vanish at runtime. + """ + if not os.environ.get('LEAPMAIL_DEBUG'): + d = deferToThread(self.f, instance, *args, **kwargs) + d.addErrback(self._errback) + return d + else: + return self.f(instance, *args, **kwargs) + + # This instance does not need the descriptor anymore, + # let it find the wrapper directly next time: + setattr(instance, self.f.__name__, wrapper) + return wrapper + + return descript(f) diff --git a/src/leap/bitmask/mail/errors.py b/src/leap/bitmask/mail/errors.py new file mode 100644 index 0000000..2f18e87 --- /dev/null +++ b/src/leap/bitmask/mail/errors.py @@ -0,0 +1,27 @@ +# -*- coding: utf-8 -*- +# errors.py +# Copyright (C) 2015 LEAP +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . +""" +Exceptions for leap.mail +""" + + +class AuthenticationError(Exception): + pass + + +class ConfigurationError(Exception): + pass diff --git a/src/leap/bitmask/mail/generator.py b/src/leap/bitmask/mail/generator.py new file mode 100644 index 0000000..bb3f26e --- /dev/null +++ b/src/leap/bitmask/mail/generator.py @@ -0,0 +1,23 @@ +from email.generator import Generator as EmailGenerator + + +class Generator(EmailGenerator): + """ + Generates output from a Message object tree, keeping signatures. + + This code was extracted from Mailman.Generator.Generator, version 2.1.4: + + Most other Generator will be created not setting the foldheader flag, + as we do not overwrite clone(). The original clone() does not + set foldheaders. + + So you need to set foldheaders if you want the toplevel to fold headers + + TODO: Python 3.3 is patched against this problems. See issue 1590744 on + python bug tracker. + """ + def _write_headers(self, msg): + for h, v in msg.items(): + print >> self._fp, '%s:' % h, + print >> self._fp, v + print >> self._fp diff --git a/src/leap/bitmask/mail/imap/__init__.py b/src/leap/bitmask/mail/imap/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/src/leap/bitmask/mail/imap/account.py b/src/leap/bitmask/mail/imap/account.py new file mode 100644 index 0000000..e795c1b --- /dev/null +++ b/src/leap/bitmask/mail/imap/account.py @@ -0,0 +1,498 @@ +# -*- coding: utf-8 -*- +# account.py +# Copyright (C) 2013-2015 LEAP +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . +""" +Soledad Backed IMAP Account. +""" +import logging +import os +import time +from functools import partial + +from twisted.internet import defer +from twisted.mail import imap4 +from twisted.python import log +from zope.interface import implements + +from leap.common.check import leap_assert, leap_assert_type + +from leap.mail.constants import MessageFlags +from leap.mail.mail import Account +from leap.mail.imap.mailbox import IMAPMailbox, normalize_mailbox +from leap.soledad.client import Soledad + +logger = logging.getLogger(__name__) + +PROFILE_CMD = os.environ.get('LEAP_PROFILE_IMAPCMD', False) + +if PROFILE_CMD: + def _debugProfiling(result, cmdname, start): + took = (time.time() - start) * 1000 + log.msg("CMD " + cmdname + " TOOK: " + str(took) + " msec") + return result + + +####################################### +# Soledad IMAP Account +####################################### + + +class IMAPAccount(object): + """ + An implementation of an imap4 Account + that is backed by Soledad Encrypted Documents. + """ + + implements(imap4.IAccount, imap4.INamespacePresenter) + + selected = None + + def __init__(self, store, user_id, d=defer.Deferred()): + """ + Keeps track of the mailboxes and subscriptions handled by this account. + + The account is not ready to be used, since the store needs to be + initialized and we also need to do some initialization routines. + You can either pass a deferred to this constructor, or use + `callWhenReady` method. + + :param store: a Soledad instance. + :type store: Soledad + + :param user_id: The identifier of the user this account belongs to + (user id, in the form user@provider). + :type user_id: str + + + :param d: a deferred that will be fired with this IMAPAccount instance + when the account is ready to be used. + :type d: defer.Deferred + """ + leap_assert(store, "Need a store instance to initialize") + leap_assert_type(store, Soledad) + + # TODO assert too that the name matches the user/uuid with which + # soledad has been initialized. Although afaik soledad doesn't know + # about user_id, only the client backend. + + self.user_id = user_id + self.account = Account( + store, user_id, ready_cb=lambda: d.callback(self)) + + def end_session(self): + """ + Used to mark when the session has closed, and we should not allow any + more commands from the client. + + Right now it's called from the client backend. + """ + # TODO move its use to the service shutdown in leap.mail + self.account.end_session() + + @property + def session_ended(self): + return self.account.session_ended + + def callWhenReady(self, cb, *args, **kw): + """ + Execute callback when the account is ready to be used. + XXX note that this callback will be called with a first ignored + parameter. + """ + # TODO ignore the first parameter and change tests accordingly. + d = self.account.callWhenReady(cb, *args, **kw) + return d + + def getMailbox(self, name): + """ + Return a Mailbox with that name, without selecting it. + + :param name: name of the mailbox + :type name: str + + :returns: an IMAPMailbox instance + :rtype: IMAPMailbox + """ + name = normalize_mailbox(name) + + def check_it_exists(mailboxes): + if name not in mailboxes: + raise imap4.MailboxException("No such mailbox: %r" % name) + return True + + d = self.account.list_all_mailbox_names() + d.addCallback(check_it_exists) + d.addCallback(lambda _: self.account.get_collection_by_mailbox(name)) + d.addCallback(self._return_mailbox_from_collection) + return d + + def _return_mailbox_from_collection(self, collection, readwrite=1): + if collection is None: + return None + mbox = IMAPMailbox(collection, rw=readwrite) + return mbox + + # + # IAccount + # + + def addMailbox(self, name, creation_ts=None): + """ + Add a mailbox to the account. + + :param name: the name of the mailbox + :type name: str + + :param creation_ts: an optional creation timestamp to be used as + mailbox id. A timestamp will be used if no + one is provided. + :type creation_ts: int + + :returns: a Deferred that will contain the document if successful. + :rtype: defer.Deferred + """ + name = normalize_mailbox(name) + + # FIXME --- return failure instead of AssertionError + # See AccountTestCase... + leap_assert(name, "Need a mailbox name to create a mailbox") + + def check_it_does_not_exist(mailboxes): + if name in mailboxes: + raise imap4.MailboxCollision, repr(name) + return mailboxes + + d = self.account.list_all_mailbox_names() + d.addCallback(check_it_does_not_exist) + d.addCallback(lambda _: self.account.add_mailbox( + name, creation_ts=creation_ts)) + d.addCallback(lambda _: self.account.get_collection_by_mailbox(name)) + d.addCallback(self._return_mailbox_from_collection) + return d + + def create(self, pathspec): + """ + Create a new mailbox from the given hierarchical name. + + :param pathspec: + The full hierarchical name of a new mailbox to create. + If any of the inferior hierarchical names to this one + do not exist, they are created as well. + :type pathspec: str + + :return: + A deferred that will fire with a true value if the creation + succeeds. The deferred might fail with a MailboxException + if the mailbox cannot be added. + :rtype: Deferred + + """ + def pass_on_collision(failure): + failure.trap(imap4.MailboxCollision) + return True + + def handle_collision(failure): + failure.trap(imap4.MailboxCollision) + if not pathspec.endswith('/'): + return defer.succeed(False) + else: + return defer.succeed(True) + + def all_good(result): + return all(result) + + paths = filter(None, normalize_mailbox(pathspec).split('/')) + subs = [] + sep = '/' + + for accum in range(1, len(paths)): + partial_path = sep.join(paths[:accum]) + d = self.addMailbox(partial_path) + d.addErrback(pass_on_collision) + subs.append(d) + + df = self.addMailbox(sep.join(paths)) + df.addErrback(handle_collision) + subs.append(df) + + d1 = defer.gatherResults(subs) + d1.addCallback(all_good) + return d1 + + def select(self, name, readwrite=1): + """ + Selects a mailbox. + + :param name: the mailbox to select + :type name: str + + :param readwrite: 1 for readwrite permissions. + :type readwrite: int + + :rtype: IMAPMailbox + """ + name = normalize_mailbox(name) + + def check_it_exists(mailboxes): + if name not in mailboxes: + logger.warning("SELECT: No such mailbox!") + return None + return name + + def set_selected(_): + self.selected = name + + def get_collection(name): + if name is None: + return None + return self.account.get_collection_by_mailbox(name) + + d = self.account.list_all_mailbox_names() + d.addCallback(check_it_exists) + d.addCallback(get_collection) + d.addCallback(partial( + self._return_mailbox_from_collection, readwrite=readwrite)) + return d + + def delete(self, name, force=False): + """ + Deletes a mailbox. + + :param name: the mailbox to be deleted + :type name: str + + :param force: + if True, it will not check for noselect flag or inferior + names. use with care. + :type force: bool + :rtype: Deferred + """ + name = normalize_mailbox(name) + _mboxes = None + + def check_it_exists(mailboxes): + global _mboxes + _mboxes = mailboxes + if name not in mailboxes: + raise imap4.MailboxException("No such mailbox: %r" % name) + + def get_mailbox(_): + return self.getMailbox(name) + + def destroy_mailbox(mbox): + return mbox.destroy() + + def check_can_be_deleted(mbox): + global _mboxes + # See if this box is flagged \Noselect + mbox_flags = mbox.getFlags() + if MessageFlags.NOSELECT_FLAG in mbox_flags: + # Check for hierarchically inferior mailboxes with this one + # as part of their root. + for others in _mboxes: + if others != name and others.startswith(name): + raise imap4.MailboxException( + "Hierarchically inferior mailboxes " + "exist and \\Noselect is set") + return mbox + + d = self.account.list_all_mailbox_names() + d.addCallback(check_it_exists) + d.addCallback(get_mailbox) + if not force: + d.addCallback(check_can_be_deleted) + d.addCallback(destroy_mailbox) + return d + + # FIXME --- not honoring the inferior names... + # if there are no hierarchically inferior names, we will + # delete it from our ken. + # XXX is this right? + # if self._inferiorNames(name) > 1: + # self._index.removeMailbox(name) + + def rename(self, oldname, newname): + """ + Renames a mailbox. + + :param oldname: old name of the mailbox + :type oldname: str + + :param newname: new name of the mailbox + :type newname: str + """ + oldname = normalize_mailbox(oldname) + newname = normalize_mailbox(newname) + + def rename_inferiors((inferiors, mailboxes)): + rename_deferreds = [] + inferiors = [ + (o, o.replace(oldname, newname, 1)) for o in inferiors] + + for (old, new) in inferiors: + if new in mailboxes: + raise imap4.MailboxCollision(repr(new)) + + for (old, new) in inferiors: + d = self.account.rename_mailbox(old, new) + rename_deferreds.append(d) + + d1 = defer.gatherResults(rename_deferreds, consumeErrors=True) + return d1 + + d1 = self._inferiorNames(oldname) + d2 = self.account.list_all_mailbox_names() + + d = defer.gatherResults([d1, d2]) + d.addCallback(rename_inferiors) + return d + + def _inferiorNames(self, name): + """ + Return hierarchically inferior mailboxes. + + :param name: name of the mailbox + :rtype: list + """ + # XXX use wildcard query instead + def filter_inferiors(mailboxes): + inferiors = [] + for infname in mailboxes: + if infname.startswith(name): + inferiors.append(infname) + return inferiors + + d = self.account.list_all_mailbox_names() + d.addCallback(filter_inferiors) + return d + + def listMailboxes(self, ref, wildcard): + """ + List the mailboxes. + + from rfc 3501: + returns a subset of names from the complete set + of all names available to the client. Zero or more untagged LIST + replies are returned, containing the name attributes, hierarchy + delimiter, and name. + + :param ref: reference name + :type ref: str + + :param wildcard: mailbox name with possible wildcards + :type wildcard: str + """ + wildcard = imap4.wildcardToRegexp(wildcard, '/') + + def get_list(mboxes, mboxes_names): + return zip(mboxes_names, mboxes) + + def filter_inferiors(ref): + mboxes = [mbox for mbox in ref if wildcard.match(mbox)] + mbox_d = defer.gatherResults([self.getMailbox(m) for m in mboxes]) + + mbox_d.addCallback(get_list, mboxes) + return mbox_d + + d = self._inferiorNames(normalize_mailbox(ref)) + d.addCallback(filter_inferiors) + return d + + # + # The rest of the methods are specific for leap.mail.imap.account.Account + # + + def isSubscribed(self, name): + """ + Returns True if user is subscribed to this mailbox. + + :param name: the mailbox to be checked. + :type name: str + + :rtype: Deferred (will fire with bool) + """ + name = normalize_mailbox(name) + + def get_subscribed(mbox): + return mbox.collection.get_mbox_attr("subscribed") + + d = self.getMailbox(name) + d.addCallback(get_subscribed) + return d + + def subscribe(self, name): + """ + Subscribe to this mailbox if not already subscribed. + + :param name: name of the mailbox + :type name: str + :rtype: Deferred + """ + name = normalize_mailbox(name) + + def set_subscribed(mbox): + return mbox.collection.set_mbox_attr("subscribed", True) + + d = self.getMailbox(name) + d.addCallback(set_subscribed) + return d + + def unsubscribe(self, name): + """ + Unsubscribe from this mailbox + + :param name: name of the mailbox + :type name: str + :rtype: Deferred + """ + # TODO should raise MailboxException if attempted to unsubscribe + # from a mailbox that is not currently subscribed. + # TODO factor out with subscribe method. + name = normalize_mailbox(name) + + def set_unsubscribed(mbox): + return mbox.collection.set_mbox_attr("subscribed", False) + + d = self.getMailbox(name) + d.addCallback(set_unsubscribed) + return d + + def getSubscriptions(self): + def get_subscribed(mailboxes): + return [x.mbox for x in mailboxes if x.subscribed] + + d = self.account.get_all_mailboxes() + d.addCallback(get_subscribed) + return d + + # + # INamespacePresenter + # + + def getPersonalNamespaces(self): + return [["", "/"]] + + def getSharedNamespaces(self): + return None + + def getOtherNamespaces(self): + return None + + def __repr__(self): + """ + Representation string for this object. + """ + return "" % self.user_id diff --git a/src/leap/bitmask/mail/imap/mailbox.py b/src/leap/bitmask/mail/imap/mailbox.py new file mode 100644 index 0000000..e70a1d8 --- /dev/null +++ b/src/leap/bitmask/mail/imap/mailbox.py @@ -0,0 +1,970 @@ +# *- coding: utf-8 -*- +# mailbox.py +# Copyright (C) 2013-2015 LEAP +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . +""" +IMAP Mailbox. +""" +import re +import logging +import os +import cStringIO +import StringIO +import time + +from collections import defaultdict +from email.utils import formatdate + +from twisted.internet import defer +from twisted.internet import reactor +from twisted.python import log + +from twisted.mail import imap4 +from zope.interface import implements + +from leap.common.check import leap_assert +from leap.common.check import leap_assert_type +from leap.mail.constants import INBOX_NAME, MessageFlags +from leap.mail.imap.messages import IMAPMessage + +logger = logging.getLogger(__name__) + +# TODO LIST +# [ ] Restore profile_cmd instrumentation +# [ ] finish the implementation of IMailboxListener +# [ ] implement the rest of ISearchableMailbox + + +""" +If the environment variable `LEAP_SKIPNOTIFY` is set, we avoid +notifying clients of new messages. Use during stress tests. +""" +NOTIFY_NEW = not os.environ.get('LEAP_SKIPNOTIFY', False) +PROFILE_CMD = os.environ.get('LEAP_PROFILE_IMAPCMD', False) + +if PROFILE_CMD: + + def _debugProfiling(result, cmdname, start): + took = (time.time() - start) * 1000 + log.msg("CMD " + cmdname + " TOOK: " + str(took) + " msec") + return result + + def do_profile_cmd(d, name): + """ + Add the profiling debug to the passed callback. + :param d: deferred + :param name: name of the command + :type name: str + """ + d.addCallback(_debugProfiling, name, time.time()) + d.addErrback(lambda f: log.msg(f.getTraceback())) + +INIT_FLAGS = (MessageFlags.SEEN_FLAG, MessageFlags.ANSWERED_FLAG, + MessageFlags.FLAGGED_FLAG, MessageFlags.DELETED_FLAG, + MessageFlags.DRAFT_FLAG, MessageFlags.RECENT_FLAG, + MessageFlags.LIST_FLAG) + + +def make_collection_listener(mailbox): + """ + Wrap a mailbox in a class that can be hashed according to the mailbox name. + + This means that dicts or sets will use this new equality rule, so we won't + collect multiple instances of the same mailbox in collections like the + MessageCollection set where we keep track of listeners. + """ + + class HashableMailbox(object): + + def __init__(self, mbox): + self.mbox = mbox + + # See #8083, pixelated adaptor seems to be misusing this class. + self.mailbox_name = self.mbox.mbox_name + + def __hash__(self): + return hash(self.mbox.mbox_name) + + def __eq__(self, other): + return self.mbox.mbox_name == other.mbox.mbox_name + + def notify_new(self): + self.mbox.notify_new() + + return HashableMailbox(mailbox) + + +class IMAPMailbox(object): + """ + A Soledad-backed IMAP mailbox. + + Implements the high-level method needed for the Mailbox interfaces. + The low-level database methods are contained in the generic + MessageCollection class. We receive an instance of it and it is made + accessible in the `collection` attribute. + """ + implements( + imap4.IMailbox, + imap4.IMailboxInfo, + imap4.ISearchableMailbox, + # XXX I think we do not need to implement CloseableMailbox, do we? + # We could remove ourselves from the collectionListener, although I + # think it simply will be garbage collected. + # imap4.ICloseableMailbox + imap4.IMessageCopier) + + init_flags = INIT_FLAGS + + CMD_MSG = "MESSAGES" + CMD_RECENT = "RECENT" + CMD_UIDNEXT = "UIDNEXT" + CMD_UIDVALIDITY = "UIDVALIDITY" + CMD_UNSEEN = "UNSEEN" + + # TODO we should turn this into a datastructure with limited capacity + _listeners = defaultdict(set) + + def __init__(self, collection, rw=1): + """ + :param collection: instance of MessageCollection + :type collection: MessageCollection + + :param rw: read-and-write flag for this mailbox + :type rw: int + """ + self.rw = rw + self._uidvalidity = None + self.collection = collection + self.collection.addListener(make_collection_listener(self)) + + @property + def mbox_name(self): + return self.collection.mbox_name + + @property + def listeners(self): + """ + Returns listeners for this mbox. + + The server itself is a listener to the mailbox. + so we can notify it (and should!) after changes in flags + and number of messages. + + :rtype: set + """ + return self._listeners[self.mbox_name] + + def get_imap_message(self, message): + d = defer.Deferred() + IMAPMessage(message, store=self.collection.store, d=d) + return d + + # FIXME this grows too crazily when many instances are fired, like + # during imaptest stress testing. Should have a queue of limited size + # instead. + + def addListener(self, listener): + """ + Add a listener to the listeners queue. + The server adds itself as a listener when there is a SELECT, + so it can send EXIST commands. + + :param listener: listener to add + :type listener: an object that implements IMailboxListener + """ + if not NOTIFY_NEW: + return + + listeners = self.listeners + logger.debug('adding mailbox listener: %s. Total: %s' % ( + listener, len(listeners))) + listeners.add(listener) + + def removeListener(self, listener): + """ + Remove a listener from the listeners queue. + + :param listener: listener to remove + :type listener: an object that implements IMailboxListener + """ + self.listeners.remove(listener) + + def getFlags(self): + """ + Returns the flags defined for this mailbox. + + :returns: tuple of flags for this mailbox + :rtype: tuple of str + """ + flags = self.collection.mbox_wrapper.flags + if not flags: + flags = self.init_flags + flags_str = map(str, flags) + return flags_str + + def setFlags(self, flags): + """ + Sets flags for this mailbox. + + :param flags: a tuple with the flags + :type flags: tuple of str + """ + # XXX this is setting (overriding) old flags. + # Better pass a mode flag + leap_assert(isinstance(flags, tuple), + "flags expected to be a tuple") + return self.collection.set_mbox_attr("flags", flags) + + def getUIDValidity(self): + """ + Return the unique validity identifier for this mailbox. + + :return: unique validity identifier + :rtype: int + """ + return self.collection.get_mbox_attr("created") + + def getUID(self, message_number): + """ + Return the UID of a message in the mailbox + + .. note:: this implementation does not make much sense RIGHT NOW, + but in the future will be useful to get absolute UIDs from + message sequence numbers. + + + :param message: the message sequence number. + :type message: int + + :rtype: int + :return: the UID of the message. + + """ + # TODO support relative sequences. The (imap) message should + # receive a sequence number attribute: a deferred is not expected + return message_number + + def getUIDNext(self): + """ + Return the likely UID for the next message added to this + mailbox. Currently it returns the higher UID incremented by + one. + + :return: deferred with int + :rtype: Deferred + """ + d = self.collection.get_uid_next() + return d + + def getMessageCount(self): + """ + Returns the total count of messages in this mailbox. + + :return: deferred with int + :rtype: Deferred + """ + return self.collection.count() + + def getUnseenCount(self): + """ + Returns the number of messages with the 'Unseen' flag. + + :return: count of messages flagged `unseen` + :rtype: int + """ + return self.collection.count_unseen() + + def getRecentCount(self): + """ + Returns the number of messages with the 'Recent' flag. + + :return: count of messages flagged `recent` + :rtype: int + """ + return self.collection.count_recent() + + def isWriteable(self): + """ + Get the read/write status of the mailbox. + + :return: 1 if mailbox is read-writeable, 0 otherwise. + :rtype: int + """ + # XXX We don't need to store it in the mbox doc, do we? + # return int(self.collection.get_mbox_attr('rw')) + return self.rw + + def getHierarchicalDelimiter(self): + """ + Returns the character used to delimite hierarchies in mailboxes. + + :rtype: str + """ + return '/' + + def requestStatus(self, names): + """ + Handles a status request by gathering the output of the different + status commands. + + :param names: a list of strings containing the status commands + :type names: iter + """ + r = {} + maybe = defer.maybeDeferred + if self.CMD_MSG in names: + r[self.CMD_MSG] = maybe(self.getMessageCount) + if self.CMD_RECENT in names: + r[self.CMD_RECENT] = maybe(self.getRecentCount) + if self.CMD_UIDNEXT in names: + r[self.CMD_UIDNEXT] = maybe(self.getUIDNext) + if self.CMD_UIDVALIDITY in names: + r[self.CMD_UIDVALIDITY] = maybe(self.getUIDValidity) + if self.CMD_UNSEEN in names: + r[self.CMD_UNSEEN] = maybe(self.getUnseenCount) + + def as_a_dict(values): + return dict(zip(r.keys(), values)) + + d = defer.gatherResults(r.values()) + d.addCallback(as_a_dict) + return d + + def addMessage(self, message, flags, date=None, notify_just_mdoc=True): + """ + Adds a message to this mailbox. + + :param message: the raw message + :type message: str + + :param flags: flag list + :type flags: list of str + + :param date: timestamp + :type date: str, or None + + :param notify_just_mdoc: + boolean passed to the wrapper.create method, to indicate whether + we're insterested in being notified right after the mdoc has been + written (as it's the first doc to be written, and quite small, this + is faster, though potentially unsafe). + Setting it to True improves a *lot* the responsiveness of the + APPENDS: we just need to be notified when the mdoc is saved, and + let's just expect that the other parts are doing just fine. This + will not catch any errors when the inserts of the other parts + fail, but on the other hand allows us to return very quickly, + which seems a good compromise given that we have to serialize the + appends. + However, some operations like the saving of drafts need to wait for + all the parts to be saved, so if some heuristics are met down in + the call chain a Draft message will unconditionally set this flag + to False, and therefore ignoring the setting of this flag here. + :type notify_just_mdoc: bool + + :return: a deferred that will be triggered with the UID of the added + message. + """ + # TODO should raise ReadOnlyMailbox if not rw. + # TODO have a look at the cases for internal date in the rfc + # XXX we could treat the message as an IMessage from here + + # TODO change notify_just_mdoc to something more meaningful, like + # fast_insert_notify? + + # TODO notify_just_mdoc *sometimes* make the append tests fail. + # have to find a better solution for this. A workaround could probably + # be to have a list of the ongoing deferreds related to append, so that + # we queue for later all the requests having to do with these. + + # A better solution will probably involve implementing MULTIAPPEND + # extension or patching imap server to support pipelining. + + if isinstance(message, (cStringIO.OutputType, StringIO.StringIO)): + message = message.getvalue() + + leap_assert_type(message, basestring) + + if flags is None: + flags = tuple() + else: + flags = tuple(str(flag) for flag in flags) + + if date is None: + date = formatdate(time.time()) + + d = self.collection.add_msg(message, flags, date=date, + notify_just_mdoc=notify_just_mdoc) + d.addErrback(lambda failure: log.err(failure)) + return d + + def notify_new(self, *args): + """ + Notify of new messages to all the listeners. + + This will be called indirectly by the underlying collection, that will + notify this IMAPMailbox whenever there are changes in the number of + messages in the collection, since we have added ourselves to the + collection listeners. + + :param args: ignored. + """ + if not NOTIFY_NEW: + return + + def cbNotifyNew(result): + exists, recent = result + for listener in self.listeners: + listener.newMessages(exists, recent) + + d = self._get_notify_count() + d.addCallback(cbNotifyNew) + d.addCallback(self.collection.cb_signal_unread_to_ui) + d.addErrback(lambda failure: log.err(failure)) + + def _get_notify_count(self): + """ + Get message count and recent count for this mailbox. + + :return: a deferred that will fire with a tuple, with number of + messages and number of recent messages. + :rtype: Deferred + """ + # XXX this is way too expensive in cases like multiple APPENDS. + # We should have a way of keep a cache or do a self-increment for that + # kind of calls. + d_exists = defer.maybeDeferred(self.getMessageCount) + d_recent = defer.maybeDeferred(self.getRecentCount) + d_list = [d_exists, d_recent] + + def log_num_msg(result): + exists, recent = tuple(result) + logger.debug("NOTIFY (%r): there are %s messages, %s recent" % ( + self.mbox_name, exists, recent)) + return result + + d = defer.gatherResults(d_list) + d.addCallback(log_num_msg) + return d + + # commands, do not rename methods + + def destroy(self): + """ + Called before this mailbox is permanently deleted. + + Should cleanup resources, and set the \\Noselect flag + on the mailbox. + + """ + # XXX this will overwrite all the existing flags + # should better simply addFlag + self.setFlags((MessageFlags.NOSELECT_FLAG,)) + + def remove_mbox(_): + uuid = self.collection.mbox_uuid + d = self.collection.mbox_wrapper.delete(self.collection.store) + d.addCallback( + lambda _: self.collection.mbox_indexer.delete_table(uuid)) + return d + + d = self.deleteAllDocs() + d.addCallback(remove_mbox) + return d + + def expunge(self): + """ + Remove all messages flagged \\Deleted + """ + if not self.isWriteable(): + raise imap4.ReadOnlyMailbox + return self.collection.delete_all_flagged() + + def _get_message_fun(self, uid): + """ + Return the proper method to get a message for this mailbox, depending + on the passed uid flag. + + :param uid: If true, the IDs specified in the query are UIDs; + otherwise they are message sequence IDs. + :type uid: bool + :rtype: callable + """ + get_message_fun = [ + self.collection.get_message_by_sequence_number, + self.collection.get_message_by_uid][uid] + return get_message_fun + + def _get_messages_range(self, messages_asked, uid=True): + + def get_range(messages_asked): + return self._filter_msg_seq(messages_asked) + + d = self._bound_seq(messages_asked, uid) + if uid: + d.addCallback(get_range) + d.addErrback(lambda f: log.err(f)) + return d + + def _bound_seq(self, messages_asked, uid): + """ + Put an upper bound to a messages sequence if this is open. + + :param messages_asked: IDs of the messages. + :type messages_asked: MessageSet + :return: a Deferred that will fire with a MessageSet + """ + + def set_last_uid(last_uid): + messages_asked.last = last_uid + return messages_asked + + def set_last_seq(all_uid): + messages_asked.last = len(all_uid) + return messages_asked + + if not messages_asked.last: + try: + iter(messages_asked) + except TypeError: + # looks like we cannot iterate + if uid: + d = self.collection.get_last_uid() + d.addCallback(set_last_uid) + else: + d = self.collection.all_uid_iter() + d.addCallback(set_last_seq) + return d + return defer.succeed(messages_asked) + + def _filter_msg_seq(self, messages_asked): + """ + Filter a message sequence returning only the ones that do exist in the + collection. + + :param messages_asked: IDs of the messages. + :type messages_asked: MessageSet + :rtype: set + """ + # TODO we could pass the asked sequence to the indexer + # all_uid_iter, and bound the sql query instead. + def filter_by_asked(all_msg_uid): + set_asked = set(messages_asked) + set_exist = set(all_msg_uid) + return set_asked.intersection(set_exist) + + d = self.collection.all_uid_iter() + d.addCallback(filter_by_asked) + return d + + def fetch(self, messages_asked, uid): + """ + Retrieve one or more messages in this mailbox. + + from rfc 3501: The data items to be fetched can be either a single atom + or a parenthesized list. + + :param messages_asked: IDs of the messages to retrieve information + about + :type messages_asked: MessageSet + + :param uid: If true, the IDs are UIDs. They are message sequence IDs + otherwise. + :type uid: bool + + :rtype: deferred with a generator that yields... + """ + get_msg_fun = self._get_message_fun(uid) + getimapmsg = self.get_imap_message + + def get_imap_messages_for_range(msg_range): + + def _get_imap_msg(messages): + d_imapmsg = [] + # just in case we got bad data in here + for msg in filter(None, messages): + d_imapmsg.append(getimapmsg(msg)) + return defer.gatherResults(d_imapmsg, consumeErrors=True) + + def _zip_msgid(imap_messages): + zipped = zip( + list(msg_range), imap_messages) + return (item for item in zipped) + + # XXX not called?? + def _unset_recent(sequence): + reactor.callLater(0, self.unset_recent_flags, sequence) + return sequence + + d_msg = [] + for msgid in msg_range: + # XXX We want cdocs because we "probably" are asked for the + # body. We should be smarter at do_FETCH and pass a parameter + # to this method in order not to prefetch cdocs if they're not + # going to be used. + d_msg.append(get_msg_fun(msgid, get_cdocs=True)) + + d = defer.gatherResults(d_msg, consumeErrors=True) + d.addCallback(_get_imap_msg) + d.addCallback(_zip_msgid) + d.addErrback(lambda failure: log.err(failure)) + return d + + d = self._get_messages_range(messages_asked, uid) + d.addCallback(get_imap_messages_for_range) + d.addErrback(lambda failure: log.err(failure)) + return d + + def fetch_flags(self, messages_asked, uid): + """ + A fast method to fetch all flags, tricking just the + needed subset of the MIME interface that's needed to satisfy + a generic FLAGS query. + + Given how LEAP Mail is supposed to work without local cache, + this query is going to be quite common, and also we expect + it to be in the form 1:* at the beginning of a session, so + it's not bad to fetch all the FLAGS docs at once. + + :param messages_asked: IDs of the messages to retrieve information + about + :type messages_asked: MessageSet + + :param uid: If 1, the IDs are UIDs. They are message sequence IDs + otherwise. + :type uid: int + + :return: A tuple of two-tuples of message sequence numbers and + flagsPart, which is a only a partial implementation of + MessagePart. + :rtype: tuple + """ + # is_sequence = True if uid == 0 else False + # XXX FIXME ----------------------------------------------------- + # imap/tests, or muas like mutt, it will choke until we implement + # sequence numbers. This is an easy hack meanwhile. + is_sequence = False + # --------------------------------------------------------------- + + if is_sequence: + raise NotImplementedError( + "FETCH FLAGS NOT IMPLEMENTED FOR MESSAGE SEQUENCE NUMBERS YET") + + d = defer.Deferred() + reactor.callLater(0, self._do_fetch_flags, messages_asked, uid, d) + if PROFILE_CMD: + do_profile_cmd(d, "FETCH-ALL-FLAGS") + return d + + def _do_fetch_flags(self, messages_asked, uid, d): + """ + :param messages_asked: IDs of the messages to retrieve information + about + :type messages_asked: MessageSet + + :param uid: If 1, the IDs are UIDs. They are message sequence IDs + otherwise. + :type uid: int + :param d: deferred whose callback will be called with result. + :type d: Deferred + + :rtype: A generator that yields two-tuples of message sequence numbers + and flagsPart + """ + class flagsPart(object): + def __init__(self, uid, flags): + self.uid = uid + self.flags = flags + + def getUID(self): + return self.uid + + def getFlags(self): + return map(str, self.flags) + + def pack_flags(result): + _uid, _flags = result + return _uid, flagsPart(_uid, _flags) + + def get_flags_for_seq(sequence): + d_all_flags = [] + for msgid in sequence: + # TODO implement sequence numbers here too + d_flags_per_uid = self.collection.get_flags_by_uid(msgid) + d_flags_per_uid.addCallback(pack_flags) + d_all_flags.append(d_flags_per_uid) + gotflags = defer.gatherResults(d_all_flags) + gotflags.addCallback(get_uid_flag_generator) + return gotflags + + def get_uid_flag_generator(result): + generator = (item for item in result) + d.callback(generator) + + d_seq = self._get_messages_range(messages_asked, uid) + d_seq.addCallback(get_flags_for_seq) + return d_seq + + @defer.inlineCallbacks + def fetch_headers(self, messages_asked, uid): + """ + A fast method to fetch all headers, tricking just the + needed subset of the MIME interface that's needed to satisfy + a generic HEADERS query. + + Given how LEAP Mail is supposed to work without local cache, + this query is going to be quite common, and also we expect + it to be in the form 1:* at the beginning of a session, so + **MAYBE** it's not too bad to fetch all the HEADERS docs at once. + + :param messages_asked: IDs of the messages to retrieve information + about + :type messages_asked: MessageSet + + :param uid: If true, the IDs are UIDs. They are message sequence IDs + otherwise. + :type uid: bool + + :return: A tuple of two-tuples of message sequence numbers and + headersPart, which is a only a partial implementation of + MessagePart. + :rtype: tuple + """ + # TODO implement sequences + is_sequence = True if uid == 0 else False + if is_sequence: + raise NotImplementedError( + "FETCH HEADERS NOT IMPLEMENTED FOR SEQUENCE NUMBER YET") + + class headersPart(object): + def __init__(self, uid, headers): + self.uid = uid + self.headers = headers + + def getUID(self): + return self.uid + + def getHeaders(self, _): + return dict( + (str(key), str(value)) + for key, value in + self.headers.items()) + + messages_asked = yield self._bound_seq(messages_asked, uid) + seq_messg = yield self._filter_msg_seq(messages_asked) + + result = [] + for msgid in seq_messg: + msg = yield self.collection.get_message_by_uid(msgid) + headers = headersPart(msgid, msg.get_headers()) + result.append((msgid, headers)) + defer.returnValue(iter(result)) + + def store(self, messages_asked, flags, mode, uid): + """ + Sets the flags of one or more messages. + + :param messages: The identifiers of the messages to set the flags + :type messages: A MessageSet object with the list of messages requested + + :param flags: The flags to set, unset, or add. + :type flags: sequence of str + + :param mode: If mode is -1, these flags should be removed from the + specified messages. If mode is 1, these flags should be + added to the specified messages. If mode is 0, all + existing flags should be cleared and these flags should be + added. + :type mode: -1, 0, or 1 + + :param uid: If true, the IDs specified in the query are UIDs; + otherwise they are message sequence IDs. + :type uid: bool + + :return: A deferred, that will be called with a dict mapping message + sequence numbers to sequences of str representing the flags + set on the message after this operation has been performed. + :rtype: deferred + + :raise ReadOnlyMailbox: Raised if this mailbox is not open for + read-write. + """ + if not self.isWriteable(): + log.msg('read only mailbox!') + raise imap4.ReadOnlyMailbox + + d = defer.Deferred() + reactor.callLater(0, self._do_store, messages_asked, flags, + mode, uid, d) + if PROFILE_CMD: + do_profile_cmd(d, "STORE") + + d.addCallback(self.collection.cb_signal_unread_to_ui) + d.addErrback(lambda f: log.err(f)) + return d + + def _do_store(self, messages_asked, flags, mode, uid, observer): + """ + Helper method, invoke set_flags method in the IMAPMessageCollection. + + See the documentation for the `store` method for the parameters. + + :param observer: a deferred that will be called with the dictionary + mapping UIDs to flags after the operation has been + done. + :type observer: deferred + """ + # TODO we should prevent client from setting Recent flag + get_msg_fun = self._get_message_fun(uid) + leap_assert(not isinstance(flags, basestring), + "flags cannot be a string") + flags = tuple(flags) + + def set_flags_for_seq(sequence): + def return_result_dict(list_of_flags): + result = dict(zip(list(sequence), list_of_flags)) + observer.callback(result) + return result + + d_all_set = [] + for msgid in sequence: + d = get_msg_fun(msgid) + d.addCallback(lambda msg: self.collection.update_flags( + msg, flags, mode)) + d_all_set.append(d) + got_flags_setted = defer.gatherResults(d_all_set) + got_flags_setted.addCallback(return_result_dict) + return got_flags_setted + + d_seq = self._get_messages_range(messages_asked, uid) + d_seq.addCallback(set_flags_for_seq) + return d_seq + + # ISearchableMailbox + + def search(self, query, uid): + """ + Search for messages that meet the given query criteria. + + Warning: this is half-baked, and it might give problems since + it offers the SearchableInterface. + We'll be implementing it asap. + + :param query: The search criteria + :type query: list + + :param uid: If true, the IDs specified in the query are UIDs; + otherwise they are message sequence IDs. + :type uid: bool + + :return: A list of message sequence numbers or message UIDs which + match the search criteria or a C{Deferred} whose callback + will be invoked with such a list. + :rtype: C{list} or C{Deferred} + """ + # TODO see if we can raise w/o interrupting flow + # :raise IllegalQueryError: Raised when query is not valid. + # example query: + # ['UNDELETED', 'HEADER', 'Message-ID', + # XXX fixme, does not exist + # '52D44F11.9060107@dev.bitmask.net'] + + # TODO hardcoding for now! -- we'll support generic queries later on + # but doing a quickfix for avoiding duplicate saves in the draft + # folder. # See issue #4209 + + if len(query) > 2: + if query[1] == 'HEADER' and query[2].lower() == "message-id": + msgid = str(query[3]).strip() + logger.debug("Searching for %s" % (msgid,)) + + d = self.collection.get_uid_from_msgid(str(msgid)) + d.addCallback(lambda result: [result]) + return d + + # nothing implemented for any other query + logger.warning("Cannot process query: %s" % (query,)) + return [] + + # IMessageCopier + + def copy(self, message): + """ + Copy the given message object into this mailbox. + + :param message: an IMessage implementor + :type message: LeapMessage + :return: a deferred that will be fired with the message + uid when the copy succeed. + :rtype: Deferred + """ + # if PROFILE_CMD: + # do_profile_cmd(d, "COPY") + + # A better place for this would be the COPY/APPEND dispatcher + # in server.py, but qtreactor hangs when I do that, so this seems + # to work fine for now. + # d.addCallback(lambda r: self.reactor.callLater(0, self.notify_new)) + # deferLater(self.reactor, 0, self._do_copy, message, d) + # return d + + d = self.collection.copy_msg(message.message, + self.collection.mbox_uuid) + return d + + # convenience fun + + def deleteAllDocs(self): + """ + Delete all docs in this mailbox + """ + # FIXME not implemented + return self.collection.delete_all_docs() + + def unset_recent_flags(self, uid_seq): + """ + Unset Recent flag for a sequence of UIDs. + """ + # FIXME not implemented + return self.collection.unset_recent_flags(uid_seq) + + def __repr__(self): + """ + Representation string for this mailbox. + """ + return u"" % ( + self.mbox_name, self.collection.count()) + + +_INBOX_RE = re.compile(INBOX_NAME, re.IGNORECASE) + + +def normalize_mailbox(name): + """ + Return a normalized representation of the mailbox ``name``. + + This method ensures that an eventual initial 'inbox' part of a + mailbox name is made uppercase. + + :param name: the name of the mailbox + :type name: unicode + + :rtype: unicode + """ + # XXX maybe it would make sense to normalize common folders too: + # trash, sent, drafts, etc... + if _INBOX_RE.match(name): + # ensure inital INBOX is uppercase + return INBOX_NAME + name[len(INBOX_NAME):] + return name diff --git a/src/leap/bitmask/mail/imap/messages.py b/src/leap/bitmask/mail/imap/messages.py new file mode 100644 index 0000000..d1c7b93 --- /dev/null +++ b/src/leap/bitmask/mail/imap/messages.py @@ -0,0 +1,254 @@ +# -*- coding: utf-8 -*- +# imap/messages.py +# Copyright (C) 2013-2015 LEAP +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . +""" +IMAPMessage implementation. +""" +import logging +from twisted.mail import imap4 +from twisted.internet import defer +from zope.interface import implements + +from leap.mail.utils import find_charset, CaseInsensitiveDict + + +logger = logging.getLogger(__name__) + +# TODO +# [ ] Add ref to incoming message during add_msg. + + +class IMAPMessage(object): + """ + The main representation of a message as seen by the IMAP Server. + This class implements the semantics specific to IMAP specification. + """ + implements(imap4.IMessage) + + def __init__(self, message, prefetch_body=True, + store=None, d=defer.Deferred()): + """ + Get an IMAPMessage. A mail.Message is needed, since many of the methods + are proxied to that object. + + + If you do not need to prefetch the body of the message, you can set + `prefetch_body` to False, but the current imap server implementation + expect the getBodyFile method to return inmediately. + + When the prefetch_body option is used, a deferred is also expected as a + parameter, and this will fire when the deferred initialization has + taken place, with this instance of IMAPMessage as a parameter. + + :param message: the abstract message + :type message: mail.Message + :param prefetch_body: Whether to prefetch the content doc for the body. + :type prefetch_body: bool + :param store: an instance of soledad, or anything that behaves like it. + :param d: an optional deferred, that will be fired with the instance of + the IMAPMessage being initialized + :type d: defer.Deferred + """ + # TODO substitute the use of the deferred initialization by a factory + # function, maybe. + + self.message = message + self.__body_fd = None + self.store = store + if prefetch_body: + gotbody = self.__prefetch_body_file() + gotbody.addCallback(lambda _: d.callback(self)) + + # IMessage implementation + + def getUID(self): + """ + Retrieve the unique identifier associated with this Message. + + :return: uid for this message + :rtype: int + """ + return self.message.get_uid() + + def getFlags(self): + """ + Retrieve the flags associated with this Message. + + :return: The flags, represented as strings + :rtype: tuple + """ + return self.message.get_flags() + + def getInternalDate(self): + """ + Retrieve the date internally associated with this message + + According to the spec, this is NOT the date and time in the + RFC-822 header, but rather a date and time that reflects when the + message was received. + + * In SMTP, date and time of final delivery. + * In COPY, internal date/time of the source message. + * In APPEND, date/time specified. + + :return: An RFC822-formatted date string. + :rtype: str + """ + return self.message.get_internal_date() + + # + # IMessagePart + # + + def getBodyFile(self, store=None): + """ + Retrieve a file object containing only the body of this message. + + :return: file-like object opened for reading + :rtype: a deferred that will fire with a StringIO object. + """ + if self.__body_fd is not None: + fd = self.__body_fd + fd.seek(0) + return fd + + if store is None: + store = self.store + return self.message.get_body_file(store) + + def getSize(self): + """ + Return the total size, in octets, of this message. + + :return: size of the message, in octets + :rtype: int + """ + return self.message.get_size() + + def getHeaders(self, negate, *names): + """ + Retrieve a group of message headers. + + :param names: The names of the headers to retrieve or omit. + :type names: tuple of str + + :param negate: If True, indicates that the headers listed in names + should be omitted from the return value, rather + than included. + :type negate: bool + + :return: A mapping of header field names to header field values + :rtype: dict + """ + headers = self.message.get_headers() + return _format_headers(headers, negate, *names) + + def isMultipart(self): + """ + Return True if this message is multipart. + """ + return self.message.is_multipart() + + def getSubPart(self, part): + """ + Retrieve a MIME submessage + + :type part: C{int} + :param part: The number of the part to retrieve, indexed from 0. + :raise IndexError: Raised if the specified part does not exist. + :raise TypeError: Raised if this message is not multipart. + :rtype: Any object implementing C{IMessagePart}. + :return: The specified sub-part. + """ + subpart = self.message.get_subpart(part + 1) + return IMAPMessagePart(subpart) + + def __prefetch_body_file(self): + def assign_body_fd(fd): + self.__body_fd = fd + return fd + d = self.getBodyFile() + d.addCallback(assign_body_fd) + return d + + +class IMAPMessagePart(object): + + def __init__(self, message_part): + self.message_part = message_part + + def getBodyFile(self, store=None): + return self.message_part.get_body_file() + + def getSize(self): + return self.message_part.get_size() + + def getHeaders(self, negate, *names): + headers = self.message_part.get_headers() + return _format_headers(headers, negate, *names) + + def isMultipart(self): + return self.message_part.is_multipart() + + def getSubPart(self, part): + subpart = self.message_part.get_subpart(part + 1) + return IMAPMessagePart(subpart) + + +def _format_headers(headers, negate, *names): + # current server impl. expects content-type to be present, so if for + # some reason we do not have headers, we have to return at least that + # one + if not headers: + logger.warning("No headers found") + return {str('content-type'): str('')} + + names = map(lambda s: s.upper(), names) + + if negate: + def cond(key): + return key.upper() not in names + else: + def cond(key): + return key.upper() in names + + if isinstance(headers, list): + headers = dict(headers) + + # default to most likely standard + charset = find_charset(headers, "utf-8") + + # We will return a copy of the headers dictionary that + # will allow case-insensitive lookups. In some parts of the twisted imap + # server code the keys are expected to be in lower case, and in this way + # we avoid having to convert them. + + _headers = CaseInsensitiveDict() + for key, value in headers.items(): + if not isinstance(key, str): + key = key.encode(charset, 'replace') + if not isinstance(value, str): + value = value.encode(charset, 'replace') + + if value.endswith(";"): + # bastards + value = value[:-1] + + # filter original dict by negate-condition + if cond(key): + _headers[key] = value + + return _headers diff --git a/src/leap/bitmask/mail/imap/server.py b/src/leap/bitmask/mail/imap/server.py new file mode 100644 index 0000000..5a63af0 --- /dev/null +++ b/src/leap/bitmask/mail/imap/server.py @@ -0,0 +1,693 @@ +# -*- coding: utf-8 -*- +# server.py +# Copyright (C) 2014 LEAP +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . +""" +LEAP IMAP4 Server Implementation. +""" +import StringIO +from copy import copy + +from twisted.internet.defer import maybeDeferred +from twisted.mail import imap4 +from twisted.python import log + +# imports for LITERAL+ patch +from twisted.internet import defer, interfaces +from twisted.mail.imap4 import IllegalClientResponse +from twisted.mail.imap4 import LiteralString, LiteralFile + +from leap.common.events import emit_async, catalog + + +def _getContentType(msg): + """ + Return a two-tuple of the main and subtype of the given message. + """ + attrs = None + mm = msg.getHeaders(False, 'content-type').get('content-type', None) + if mm: + mm = ''.join(mm.splitlines()) + mimetype = mm.split(';') + if mimetype: + type = mimetype[0].split('/', 1) + if len(type) == 1: + major = type[0] + minor = None + elif len(type) == 2: + major, minor = type + else: + major = minor = None + # XXX patched --------------------------------------------- + attrs = dict(x.strip().split('=', 1) for x in mimetype[1:]) + # XXX patched --------------------------------------------- + else: + major = minor = None + else: + major = minor = None + return major, minor, attrs + +# Monkey-patch _getContentType to avoid bug that passes lower-case boundary in +# BODYSTRUCTURE response. +imap4._getContentType = _getContentType + + +class LEAPIMAPServer(imap4.IMAP4Server): + """ + An IMAP4 Server with a LEAP Storage Backend. + """ + + ############################################################# + # + # Twisted imap4 patch to workaround bad mime rendering in TB. + # See https://leap.se/code/issues/6773 + # and https://bugzilla.mozilla.org/show_bug.cgi?id=149771 + # Still unclear if this is a thunderbird bug. + # TODO send this patch upstream + # + ############################################################# + + def spew_body(self, part, id, msg, _w=None, _f=None): + if _w is None: + _w = self.transport.write + for p in part.part: + if msg.isMultipart(): + msg = msg.getSubPart(p) + elif p > 0: + # Non-multipart messages have an implicit first part but no + # other parts - reject any request for any other part. + raise TypeError("Requested subpart of non-multipart message") + + if part.header: + hdrs = msg.getHeaders(part.header.negate, *part.header.fields) + hdrs = imap4._formatHeaders(hdrs) + # PATCHED ########################################## + _w(str(part) + ' ' + imap4._literal(hdrs + "\r\n")) + # PATCHED ########################################## + elif part.text: + _w(str(part) + ' ') + _f() + return imap4.FileProducer( + msg.getBodyFile() + ).beginProducing(self.transport) + elif part.mime: + hdrs = imap4._formatHeaders(msg.getHeaders(True)) + + # PATCHED ########################################## + _w(str(part) + ' ' + imap4._literal(hdrs + "\r\n")) + # END PATCHED ###################################### + + elif part.empty: + _w(str(part) + ' ') + _f() + if part.part: + # PATCHED ############################################# + # implement partial FETCH + # TODO implement boundary checks + # TODO see if there's a more efficient way, without + # copying the original content into a new buffer. + fd = msg.getBodyFile() + begin = getattr(part, "partialBegin", None) + _len = getattr(part, "partialLength", None) + if begin is not None and _len is not None: + _fd = StringIO.StringIO() + fd.seek(part.partialBegin) + _fd.write(fd.read(part.partialLength)) + _fd.seek(0) + else: + _fd = fd + return imap4.FileProducer( + _fd + # END PATCHED #########################3 + ).beginProducing(self.transport) + else: + mf = imap4.IMessageFile(msg, None) + if mf is not None: + return imap4.FileProducer( + mf.open()).beginProducing(self.transport) + return imap4.MessageProducer( + msg, None, self._scheduler).beginProducing(self.transport) + + else: + _w('BODY ' + + imap4.collapseNestedLists([imap4.getBodyStructure(msg)])) + + ################################################################## + # + # END Twisted imap4 patch to workaround bad mime rendering in TB. + # #6773 + # + ################################################################## + + def lineReceived(self, line): + """ + Attempt to parse a single line from the server. + + :param line: the line from the server, without the line delimiter. + :type line: str + """ + if "login" in line.lower(): + # avoid to log the pass, even though we are using a dummy auth + # by now. + msg = line[:7] + " [...]" + else: + msg = copy(line) + log.msg('rcv (%s): %s' % (self.state, msg)) + imap4.IMAP4Server.lineReceived(self, line) + + def close_server_connection(self): + """ + Send a BYE command so that the MUA at least knows that we're closing + the connection. + """ + self.sendLine( + '* BYE LEAP IMAP Proxy is shutting down; ' + 'so long and thanks for all the fish') + self.transport.loseConnection() + if self.mbox: + self.mbox.removeListener(self) + self.mbox = None + self.state = 'unauth' + + def do_FETCH(self, tag, messages, query, uid=0): + """ + Overwritten fetch dispatcher to use the fast fetch_flags + method + """ + if not query: + self.sendPositiveResponse(tag, 'FETCH complete') + return + + cbFetch = self._IMAP4Server__cbFetch + ebFetch = self._IMAP4Server__ebFetch + + if len(query) == 1 and str(query[0]) == "flags": + self._oldTimeout = self.setTimeout(None) + # no need to call iter, we get a generator + maybeDeferred( + self.mbox.fetch_flags, messages, uid=uid + ).addCallback( + cbFetch, tag, query, uid + ).addErrback(ebFetch, tag) + + elif len(query) == 1 and str(query[0]) == "rfc822.header": + self._oldTimeout = self.setTimeout(None) + # no need to call iter, we get a generator + maybeDeferred( + self.mbox.fetch_headers, messages, uid=uid + ).addCallback( + cbFetch, tag, query, uid + ).addErrback(ebFetch, tag) + else: + self._oldTimeout = self.setTimeout(None) + # no need to call iter, we get a generator + maybeDeferred( + self.mbox.fetch, messages, uid=uid + ).addCallback( + cbFetch, tag, query, uid + ).addErrback( + ebFetch, tag) + + select_FETCH = (do_FETCH, imap4.IMAP4Server.arg_seqset, + imap4.IMAP4Server.arg_fetchatt) + + def _cbSelectWork(self, mbox, cmdName, tag): + """ + Callback for selectWork + + * patched to avoid conformance errors due to incomplete UIDVALIDITY + line. + * patched to accept deferreds for messagecount and recent count + """ + if mbox is None: + self.sendNegativeResponse(tag, 'No such mailbox') + return + if '\\noselect' in [s.lower() for s in mbox.getFlags()]: + self.sendNegativeResponse(tag, 'Mailbox cannot be selected') + return + + d1 = defer.maybeDeferred(mbox.getMessageCount) + d2 = defer.maybeDeferred(mbox.getRecentCount) + return defer.gatherResults([d1, d2]).addCallback( + self.__cbSelectWork, mbox, cmdName, tag) + + def __cbSelectWork(self, ((msg_count, recent_count)), mbox, cmdName, tag): + flags = mbox.getFlags() + self.sendUntaggedResponse('FLAGS (%s)' % ' '.join(flags)) + + # Patched ------------------------------------------------------- + # accept deferreds for the count + self.sendUntaggedResponse(str(msg_count) + ' EXISTS') + self.sendUntaggedResponse(str(recent_count) + ' RECENT') + # ---------------------------------------------------------------- + + # Patched ------------------------------------------------------- + # imaptest was complaining about the incomplete line, we're adding + # "UIDs valid" here. + self.sendPositiveResponse( + None, '[UIDVALIDITY %d] UIDs valid' % mbox.getUIDValidity()) + # ---------------------------------------------------------------- + + s = mbox.isWriteable() and 'READ-WRITE' or 'READ-ONLY' + mbox.addListener(self) + self.sendPositiveResponse(tag, '[%s] %s successful' % (s, cmdName)) + self.state = 'select' + self.mbox = mbox + + def checkpoint(self): + """ + Called when the client issues a CHECK command. + + This should perform any checkpoint operations required by the server. + It may be a long running operation, but may not block. If it returns + a deferred, the client will only be informed of success (or failure) + when the deferred's callback (or errback) is invoked. + """ + # TODO implement a collection of ongoing deferreds? + return None + + ############################################################# + # + # Twisted imap4 patch to support LITERAL+ extension + # TODO send this patch upstream asap! + # + ############################################################# + + def capabilities(self): + cap = {'AUTH': self.challengers.keys()} + if self.ctx and self.canStartTLS: + t = self.transport + ti = interfaces.ISSLTransport + if not self.startedTLS and ti(t, None) is None: + cap['LOGINDISABLED'] = None + cap['STARTTLS'] = None + cap['NAMESPACE'] = None + cap['IDLE'] = None + # patched ############ + cap['LITERAL+'] = None + ###################### + return cap + + def _stringLiteral(self, size, literal_plus=False): + if size > self._literalStringLimit: + raise IllegalClientResponse( + "Literal too long! I accept at most %d octets" % + (self._literalStringLimit,)) + d = defer.Deferred() + self.parseState = 'pending' + self._pendingLiteral = LiteralString(size, d) + # Patched ########################################################### + if not literal_plus: + self.sendContinuationRequest('Ready for %d octets of text' % size) + ##################################################################### + self.setRawMode() + return d + + def _fileLiteral(self, size, literal_plus=False): + d = defer.Deferred() + self.parseState = 'pending' + self._pendingLiteral = LiteralFile(size, d) + if not literal_plus: + self.sendContinuationRequest('Ready for %d octets of data' % size) + self.setRawMode() + return d + + def arg_astring(self, line): + """ + Parse an astring from the line, return (arg, rest), possibly + via a deferred (to handle literals) + """ + line = line.strip() + if not line: + raise IllegalClientResponse("Missing argument") + d = None + arg, rest = None, None + if line[0] == '"': + try: + spam, arg, rest = line.split('"', 2) + rest = rest[1:] # Strip space + except ValueError: + raise IllegalClientResponse("Unmatched quotes") + elif line[0] == '{': + # literal + if line[-1] != '}': + raise IllegalClientResponse("Malformed literal") + + # Patched ################ + if line[-2] == "+": + literalPlus = True + size_end = -2 + else: + literalPlus = False + size_end = -1 + + try: + size = int(line[1:size_end]) + except ValueError: + raise IllegalClientResponse( + "Bad literal size: " + line[1:size_end]) + d = self._stringLiteral(size, literalPlus) + ########################## + else: + arg = line.split(' ', 1) + if len(arg) == 1: + arg.append('') + arg, rest = arg + return d or (arg, rest) + + def arg_literal(self, line): + """ + Parse a literal from the line + """ + if not line: + raise IllegalClientResponse("Missing argument") + + if line[0] != '{': + raise IllegalClientResponse("Missing literal") + + if line[-1] != '}': + raise IllegalClientResponse("Malformed literal") + + # Patched ################## + if line[-2] == "+": + literalPlus = True + size_end = -2 + else: + literalPlus = False + size_end = -1 + + try: + size = int(line[1:size_end]) + except ValueError: + raise IllegalClientResponse( + "Bad literal size: " + line[1:size_end]) + + return self._fileLiteral(size, literalPlus) + ############################# + + # --------------------------------- isSubscribed patch + # TODO -- send patch upstream. + # There is a bug in twisted implementation: + # in cbListWork, it's assumed that account.isSubscribed IS a callable, + # although in the interface documentation it's stated that it can be + # a deferred. + + def _listWork(self, tag, ref, mbox, sub, cmdName): + mbox = self._parseMbox(mbox) + mailboxes = maybeDeferred(self.account.listMailboxes, ref, mbox) + mailboxes.addCallback(self._cbSubscribed) + mailboxes.addCallback( + self._cbListWork, tag, sub, cmdName, + ).addErrback(self._ebListWork, tag) + + def _cbSubscribed(self, mailboxes): + subscribed = [ + maybeDeferred(self.account.isSubscribed, name) + for (name, box) in mailboxes] + + def get_mailboxes_and_subs(result): + subscribed = [i[0] for i, yes in zip(mailboxes, result) if yes] + return mailboxes, subscribed + + d = defer.gatherResults(subscribed) + d.addCallback(get_mailboxes_and_subs) + return d + + def _cbListWork(self, mailboxes_subscribed, tag, sub, cmdName): + mailboxes, subscribed = mailboxes_subscribed + + for (name, box) in mailboxes: + if not sub or name in subscribed: + flags = box.getFlags() + delim = box.getHierarchicalDelimiter() + resp = (imap4.DontQuoteMe(cmdName), + map(imap4.DontQuoteMe, flags), + delim, name.encode('imap4-utf-7')) + self.sendUntaggedResponse( + imap4.collapseNestedLists(resp)) + self.sendPositiveResponse(tag, '%s completed' % (cmdName,)) + # -------------------- end isSubscribed patch ----------- + + # TODO subscribe method had also to be changed to accomodate deferred + def do_SUBSCRIBE(self, tag, name): + name = self._parseMbox(name) + + def _subscribeCb(_): + self.sendPositiveResponse(tag, 'Subscribed') + + def _subscribeEb(failure): + m = failure.value + log.err() + if failure.check(imap4.MailboxException): + self.sendNegativeResponse(tag, str(m)) + else: + self.sendBadResponse( + tag, + "Server error encountered while subscribing to mailbox") + + d = self.account.subscribe(name) + d.addCallbacks(_subscribeCb, _subscribeEb) + return d + + auth_SUBSCRIBE = (do_SUBSCRIBE, arg_astring) + select_SUBSCRIBE = auth_SUBSCRIBE + + def do_UNSUBSCRIBE(self, tag, name): + # unsubscribe method had also to be changed to accomodate + # deferred + name = self._parseMbox(name) + + def _unsubscribeCb(_): + self.sendPositiveResponse(tag, 'Unsubscribed') + + def _unsubscribeEb(failure): + m = failure.value + log.err() + if failure.check(imap4.MailboxException): + self.sendNegativeResponse(tag, str(m)) + else: + self.sendBadResponse( + tag, + "Server error encountered while unsubscribing " + "from mailbox") + + d = self.account.unsubscribe(name) + d.addCallbacks(_unsubscribeCb, _unsubscribeEb) + return d + + auth_UNSUBSCRIBE = (do_UNSUBSCRIBE, arg_astring) + select_UNSUBSCRIBE = auth_UNSUBSCRIBE + + def do_RENAME(self, tag, oldname, newname): + oldname, newname = [self._parseMbox(n) for n in oldname, newname] + if oldname.lower() == 'inbox' or newname.lower() == 'inbox': + self.sendNegativeResponse( + tag, + 'You cannot rename the inbox, or ' + 'rename another mailbox to inbox.') + return + + def _renameCb(_): + self.sendPositiveResponse(tag, 'Mailbox renamed') + + def _renameEb(failure): + m = failure.value + if failure.check(TypeError): + self.sendBadResponse(tag, 'Invalid command syntax') + elif failure.check(imap4.MailboxException): + self.sendNegativeResponse(tag, str(m)) + else: + log.err() + self.sendBadResponse( + tag, + "Server error encountered while " + "renaming mailbox") + + d = self.account.rename(oldname, newname) + d.addCallbacks(_renameCb, _renameEb) + return d + + auth_RENAME = (do_RENAME, arg_astring, arg_astring) + select_RENAME = auth_RENAME + + def do_CREATE(self, tag, name): + name = self._parseMbox(name) + + def _createCb(result): + if result: + self.sendPositiveResponse(tag, 'Mailbox created') + else: + self.sendNegativeResponse(tag, 'Mailbox not created') + + def _createEb(failure): + c = failure.value + if failure.check(imap4.MailboxException): + self.sendNegativeResponse(tag, str(c)) + else: + log.err() + self.sendBadResponse( + tag, "Server error encountered while creating mailbox") + + d = self.account.create(name) + d.addCallbacks(_createCb, _createEb) + return d + + auth_CREATE = (do_CREATE, arg_astring) + select_CREATE = auth_CREATE + + def do_DELETE(self, tag, name): + name = self._parseMbox(name) + if name.lower() == 'inbox': + self.sendNegativeResponse(tag, 'You cannot delete the inbox') + return + + def _deleteCb(result): + self.sendPositiveResponse(tag, 'Mailbox deleted') + + def _deleteEb(failure): + m = failure.value + if failure.check(imap4.MailboxException): + self.sendNegativeResponse(tag, str(m)) + else: + print "SERVER: other error" + log.err() + self.sendBadResponse( + tag, + "Server error encountered while deleting mailbox") + + d = self.account.delete(name) + d.addCallbacks(_deleteCb, _deleteEb) + return d + + auth_DELETE = (do_DELETE, arg_astring) + select_DELETE = auth_DELETE + + # ----------------------------------------------------------------------- + # Patched just to allow __cbAppend to receive a deferred from messageCount + # TODO format and send upstream. + def do_APPEND(self, tag, mailbox, flags, date, message): + mailbox = self._parseMbox(mailbox) + maybeDeferred(self.account.select, mailbox).addCallback( + self._cbAppendGotMailbox, tag, flags, date, message).addErrback( + self._ebAppendGotMailbox, tag) + + def __ebAppend(self, failure, tag): + self.sendBadResponse(tag, 'APPEND failed: ' + str(failure.value)) + + def _cbAppendGotMailbox(self, mbox, tag, flags, date, message): + if not mbox: + self.sendNegativeResponse(tag, '[TRYCREATE] No such mailbox') + return + + d = mbox.addMessage(message, flags, date) + d.addCallback(self.__cbAppend, tag, mbox) + d.addErrback(self.__ebAppend, tag) + + def _ebAppendGotMailbox(self, failure, tag): + self.sendBadResponse( + tag, "Server error encountered while opening mailbox.") + log.err(failure) + + def __cbAppend(self, result, tag, mbox): + + # XXX patched --------------------------------- + def send_response(count): + self.sendUntaggedResponse('%d EXISTS' % count) + self.sendPositiveResponse(tag, 'APPEND complete') + + d = mbox.getMessageCount() + d.addCallback(send_response) + return d + # XXX patched --------------------------------- + # ----------------------------------------------------------------------- + + auth_APPEND = (do_APPEND, arg_astring, imap4.IMAP4Server.opt_plist, + imap4.IMAP4Server.opt_datetime, arg_literal) + select_APPEND = auth_APPEND + + # Need to override the command table after patching + # arg_astring and arg_literal, except on the methods that we are already + # overriding. + + # TODO -------------------------------------------- + # Check if we really need to override these + # methods, or we can monkeypatch. + # do_DELETE = imap4.IMAP4Server.do_DELETE + # do_CREATE = imap4.IMAP4Server.do_CREATE + # do_RENAME = imap4.IMAP4Server.do_RENAME + # do_SUBSCRIBE = imap4.IMAP4Server.do_SUBSCRIBE + # do_UNSUBSCRIBE = imap4.IMAP4Server.do_UNSUBSCRIBE + # do_APPEND = imap4.IMAP4Server.do_APPEND + # ------------------------------------------------- + do_LOGIN = imap4.IMAP4Server.do_LOGIN + do_STATUS = imap4.IMAP4Server.do_STATUS + do_COPY = imap4.IMAP4Server.do_COPY + + _selectWork = imap4.IMAP4Server._selectWork + + arg_plist = imap4.IMAP4Server.arg_plist + arg_seqset = imap4.IMAP4Server.arg_seqset + opt_plist = imap4.IMAP4Server.opt_plist + opt_datetime = imap4.IMAP4Server.opt_datetime + + unauth_LOGIN = (do_LOGIN, arg_astring, arg_astring) + + auth_SELECT = (_selectWork, arg_astring, 1, 'SELECT') + select_SELECT = auth_SELECT + + auth_CREATE = (do_CREATE, arg_astring) + select_CREATE = auth_CREATE + + auth_EXAMINE = (_selectWork, arg_astring, 0, 'EXAMINE') + select_EXAMINE = auth_EXAMINE + + # TODO ----------------------------------------------- + # re-add if we stop overriding DELETE + # auth_DELETE = (do_DELETE, arg_astring) + # select_DELETE = auth_DELETE + # auth_APPEND = (do_APPEND, arg_astring, opt_plist, opt_datetime, + # arg_literal) + # select_APPEND = auth_APPEND + + # ---------------------------------------------------- + + auth_RENAME = (do_RENAME, arg_astring, arg_astring) + select_RENAME = auth_RENAME + + auth_SUBSCRIBE = (do_SUBSCRIBE, arg_astring) + select_SUBSCRIBE = auth_SUBSCRIBE + + auth_UNSUBSCRIBE = (do_UNSUBSCRIBE, arg_astring) + select_UNSUBSCRIBE = auth_UNSUBSCRIBE + + auth_LIST = (_listWork, arg_astring, arg_astring, 0, 'LIST') + select_LIST = auth_LIST + + auth_LSUB = (_listWork, arg_astring, arg_astring, 1, 'LSUB') + select_LSUB = auth_LSUB + + auth_STATUS = (do_STATUS, arg_astring, arg_plist) + select_STATUS = auth_STATUS + + select_COPY = (do_COPY, arg_seqset, arg_astring) + + ############################################################# + # END of Twisted imap4 patch to support LITERAL+ extension + ############################################################# + + def authenticateLogin(self, user, passwd): + result = imap4.IMAP4Server.authenticateLogin(self, user, passwd) + emit_async(catalog.IMAP_CLIENT_LOGIN, str(user)) + return result diff --git a/src/leap/bitmask/mail/imap/service/README.rst b/src/leap/bitmask/mail/imap/service/README.rst new file mode 100644 index 0000000..2cca9b3 --- /dev/null +++ b/src/leap/bitmask/mail/imap/service/README.rst @@ -0,0 +1,39 @@ +testing the service +=================== + +Run the twisted service:: + + twistd -n -y imap-server.tac + +And use offlineimap for tests:: + + offlineimap -c LEAPofflineimapRC-tests + +minimal offlineimap configuration +--------------------------------- + +[general] +accounts = leap-local + +[Account leap-local] +localrepository = LocalLeap +remoterepository = RemoteLeap + +[Repository LocalLeap] +type = Maildir +localfolders = ~/LEAPMail/Mail + +[Repository RemoteLeap] +type = IMAP +ssl = no +remotehost = localhost +remoteport = 9930 +remoteuser = user +remotepass = pass + +debugging +--------- + +Use ngrep to obtain logs of the sequences:: + + sudo ngrep -d lo -W byline port 9930 diff --git a/src/leap/bitmask/mail/imap/service/__init__.py b/src/leap/bitmask/mail/imap/service/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/src/leap/bitmask/mail/imap/service/imap-server.tac b/src/leap/bitmask/mail/imap/service/imap-server.tac new file mode 100644 index 0000000..c4d602d --- /dev/null +++ b/src/leap/bitmask/mail/imap/service/imap-server.tac @@ -0,0 +1,145 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- +# imap-server.tac +# Copyright (C) 2013,2014 LEAP +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . +""" +TAC file for initialization of the imap service using twistd. + +Use this for debugging and testing the imap server using a native reactor. + +For now, and for debugging/testing purposes, you need +to pass a config file with the following structure: + +[leap_mail] +userid = 'user@provider' +uuid = 'deadbeefdeadabad' +passwd = 'supersecret' # optional, will get prompted if not found. +""" + +# TODO -- this .tac file should be deprecated in favor of bitmask.core.bitmaskd + +import ConfigParser +import getpass +import os +import sys + +from leap.keymanager import KeyManager +from leap.mail.imap.service import imap +from leap.soledad.client import Soledad + +from twisted.application import service, internet + + +# TODO should get this initializers from some authoritative mocked source +# We might want to put them the soledad itself. + +def initialize_soledad(uuid, email, passwd, + secrets, localdb, + gnupg_home, tempdir): + """ + Initializes soledad by hand + + :param email: ID for the user + :param gnupg_home: path to home used by gnupg + :param tempdir: path to temporal dir + :rtype: Soledad instance + """ + server_url = "http://provider" + cert_file = "" + + soledad = Soledad( + uuid, + passwd, + secrets, + localdb, + server_url, + cert_file, + syncable=False) + + return soledad + +###################################################################### +# Remember to set your config files, see module documentation above! +###################################################################### + +print "[+] Running LEAP IMAP Service" + + +bmconf = os.environ.get("LEAP_MAIL_CONFIG", "") +if not bmconf: + print ("[-] Please set LEAP_MAIL_CONFIG environment variable " + "pointing to your config.") + sys.exit(1) + +SECTION = "leap_mail" +cp = ConfigParser.ConfigParser() +cp.read(bmconf) + +userid = cp.get(SECTION, "userid") +uuid = cp.get(SECTION, "uuid") +passwd = unicode(cp.get(SECTION, "passwd")) + +# XXX get this right from the environment variable !!! +port = 1984 + +if not userid or not uuid: + print "[-] Config file missing userid or uuid field" + sys.exit(1) + +if not passwd: + passwd = unicode(getpass.getpass("Soledad passphrase: ")) + + +secrets = os.path.expanduser("~/.config/leap/soledad/%s.secret" % (uuid,)) +localdb = os.path.expanduser("~/.config/leap/soledad/%s.db" % (uuid,)) + +# XXX Is this really used? Should point it to user var dirs defined in xdg? +gnupg_home = "/tmp/" +tempdir = "/tmp/" + +################################################### + +# Ad-hoc soledad/keymanager initialization. + +print "[~] user:", userid +soledad = initialize_soledad(uuid, userid, passwd, secrets, + localdb, gnupg_home, tempdir, userid=userid) +km_args = (userid, "https://localhost", soledad) +km_kwargs = { + "token": "", + "ca_cert_path": "", + "api_uri": "", + "api_version": "", + "uid": uuid, + "gpgbinary": "/usr/bin/gpg" +} +keymanager = KeyManager(*km_args, **km_kwargs) + +################################################## + +# Ok, let's expose the application object for the twistd application +# framework to pick up from here... + + +def getIMAPService(): + soledad_sessions = {userid: soledad} + factory = imap.LeapIMAPFactory(soledad_sessions) + return internet.TCPServer(port, factory, interface="localhost") + + +application = service.Application("LEAP IMAP Application") +service = getIMAPService() +service.setServiceParent(application) diff --git a/src/leap/bitmask/mail/imap/service/imap.py b/src/leap/bitmask/mail/imap/service/imap.py new file mode 100644 index 0000000..4663854 --- /dev/null +++ b/src/leap/bitmask/mail/imap/service/imap.py @@ -0,0 +1,208 @@ +# -*- coding: utf-8 -*- +# imap.py +# Copyright (C) 2013-2015 LEAP +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . +""" +IMAP Service Initialization. +""" +import logging +import os + +from collections import defaultdict + +from twisted.cred.portal import Portal, IRealm +from twisted.mail.imap4 import IAccount +from twisted.internet import defer +from twisted.internet import reactor +from twisted.internet.error import CannotListenError +from twisted.internet.protocol import ServerFactory +from twisted.python import log +from zope.interface import implementer + +from leap.common.events import emit_async, catalog +from leap.mail.cred import LocalSoledadTokenChecker +from leap.mail.imap.account import IMAPAccount +from leap.mail.imap.server import LEAPIMAPServer + +# TODO: leave only an implementor of IService in here + +logger = logging.getLogger(__name__) + +DO_MANHOLE = os.environ.get("LEAP_MAIL_MANHOLE", None) +if DO_MANHOLE: + from leap.mail.imap.service import manhole + +# The default port in which imap service will run + +IMAP_PORT = 1984 + +# +# Credentials Handling +# + + +@implementer(IRealm) +class LocalSoledadIMAPRealm(object): + + _encoding = 'utf-8' + + def __init__(self, soledad_sessions): + """ + :param soledad_sessions: a dict-like object, containing instances + of a Store (soledad instances), indexed by + userid. + """ + self._soledad_sessions = soledad_sessions + + def requestAvatar(self, avatarId, mind, *interfaces): + if isinstance(avatarId, str): + avatarId = avatarId.decode(self._encoding) + + def gotSoledad(soledad): + for iface in interfaces: + if iface is IAccount: + avatar = IMAPAccount(soledad, avatarId) + return (IAccount, avatar, + getattr(avatar, 'logout', lambda: None)) + raise NotImplementedError(self, interfaces) + + return self.lookupSoledadInstance(avatarId).addCallback(gotSoledad) + + def lookupSoledadInstance(self, userid): + soledad = self._soledad_sessions[userid] + # XXX this should return the instance after whenReady callback + return defer.succeed(soledad) + + +class IMAPTokenChecker(LocalSoledadTokenChecker): + """A credentials checker that will lookup a token for the IMAP service. + For now it will be using the same identifier than SMTPTokenChecker""" + + service = 'mail_auth' + + +class LocalSoledadIMAPServer(LEAPIMAPServer): + + """ + An IMAP Server that authenticates against a LocalSoledad store. + """ + + def __init__(self, soledad_sessions, *args, **kw): + + LEAPIMAPServer.__init__(self, *args, **kw) + + realm = LocalSoledadIMAPRealm(soledad_sessions) + portal = Portal(realm) + checker = IMAPTokenChecker(soledad_sessions) + self.checker = checker + self.portal = portal + portal.registerChecker(checker) + + +class LeapIMAPFactory(ServerFactory): + + """ + Factory for a IMAP4 server with soledad remote sync and gpg-decryption + capabilities. + """ + + protocol = LocalSoledadIMAPServer + + def __init__(self, soledad_sessions): + """ + Initializes the server factory. + + :param soledad_sessions: a dict-like object, containing instances + of a Store (soledad instances), indexed by + userid. + """ + self._soledad_sessions = soledad_sessions + self._connections = defaultdict() + + def buildProtocol(self, addr): + """ + Return a protocol suitable for the job. + + :param addr: remote ip address + :type addr: str + """ + # TODO should reject anything from addr != localhost, + # just in case. + log.msg("Building protocol for connection %s" % addr) + imapProtocol = self.protocol(self._soledad_sessions) + self._connections[addr] = imapProtocol + return imapProtocol + + def stopFactory(self): + # say bye! + for conn, proto in self._connections.items(): + log.msg("Closing connections for %s" % conn) + proto.close_server_connection() + + def doStop(self): + """ + Stops imap service (fetcher, factory and port). + """ + return ServerFactory.doStop(self) + + +def run_service(soledad_sessions, port=IMAP_PORT): + """ + Main entry point to run the service from the client. + + :param soledad_sessions: a dict-like object, containing instances + of a Store (soledad instances), indexed by userid. + + :returns: the port as returned by the reactor when starts listening, and + the factory for the protocol. + :rtype: tuple + """ + factory = LeapIMAPFactory(soledad_sessions) + + try: + interface = "localhost" + # don't bind just to localhost if we are running on docker since we + # won't be able to access imap from the host + if os.environ.get("LEAP_DOCKERIZED"): + interface = '' + + # TODO use Endpoints !!! + tport = reactor.listenTCP(port, factory, + interface=interface) + except CannotListenError: + logger.error("IMAP Service failed to start: " + "cannot listen in port %s" % (port,)) + except Exception as exc: + logger.error("Error launching IMAP service: %r" % (exc,)) + else: + # all good. + + if DO_MANHOLE: + # TODO get pass from env var.too. + manhole_factory = manhole.getManholeFactory( + {'f': factory, + 'gm': factory.theAccount.getMailbox}, + "boss", "leap") + # TODO use Endpoints !!! + reactor.listenTCP(manhole.MANHOLE_PORT, manhole_factory, + interface="127.0.0.1") + logger.debug("IMAP4 Server is RUNNING in port %s" % (port,)) + emit_async(catalog.IMAP_SERVICE_STARTED, str(port)) + + # FIXME -- change service signature + return tport, factory + + # not ok, signal error. + emit_async(catalog.IMAP_SERVICE_FAILED_TO_START, str(port)) diff --git a/src/leap/bitmask/mail/imap/service/manhole.py b/src/leap/bitmask/mail/imap/service/manhole.py new file mode 100644 index 0000000..c83ae89 --- /dev/null +++ b/src/leap/bitmask/mail/imap/service/manhole.py @@ -0,0 +1,130 @@ +# -*- coding: utf-8 -*- +# manhole.py +# Copyright (C) 2014 LEAP +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . +""" +Utilities for enabling the manhole administrative interface into the +LEAP Mail application. +""" +MANHOLE_PORT = 2222 + + +def getManholeFactory(namespace, user, secret): + """ + Get an administrative manhole into the application. + + :param namespace: the namespace to show in the manhole + :type namespace: dict + :param user: the user to authenticate into the administrative shell. + :type user: str + :param secret: pass for this manhole + :type secret: str + """ + import string + + from twisted.cred.portal import Portal + from twisted.conch import manhole, manhole_ssh + from twisted.conch.insults import insults + from twisted.cred.checkers import ( + InMemoryUsernamePasswordDatabaseDontUse as MemoryDB) + + from rlcompleter import Completer + + class EnhancedColoredManhole(manhole.ColoredManhole): + """ + A Manhole with some primitive autocomplete support. + """ + # TODO use introspection to make life easier + + def find_common(self, l): + """ + find common parts in thelist items + ex: 'ab' for ['abcd','abce','abf'] + requires an ordered list + """ + if len(l) == 1: + return l[0] + + init = l[0] + for item in l[1:]: + for i, (x, y) in enumerate(zip(init, item)): + if x != y: + init = "".join(init[:i]) + break + + if not init: + return None + return init + + def handle_TAB(self): + """ + Trap the TAB keystroke. + """ + necessarypart = "".join(self.lineBuffer).split(' ')[-1] + completer = Completer(globals()) + if completer.complete(necessarypart, 0): + matches = list(set(completer.matches)) # has multiples + + if len(matches) == 1: + length = len(necessarypart) + self.lineBuffer = self.lineBuffer[:-length] + self.lineBuffer.extend(matches[0]) + self.lineBufferIndex = len(self.lineBuffer) + else: + matches.sort() + commons = self.find_common(matches) + if commons: + length = len(necessarypart) + self.lineBuffer = self.lineBuffer[:-length] + self.lineBuffer.extend(commons) + self.lineBufferIndex = len(self.lineBuffer) + + self.terminal.nextLine() + while matches: + matches, part = matches[4:], matches[:4] + for item in part: + self.terminal.write('%s' % item.ljust(30)) + self.terminal.write('\n') + self.terminal.nextLine() + + self.terminal.eraseLine() + self.terminal.cursorBackward(self.lineBufferIndex + 5) + self.terminal.write("%s %s" % ( + self.ps[self.pn], "".join(self.lineBuffer))) + + def keystrokeReceived(self, keyID, modifier): + """ + Act upon any keystroke received. + """ + self.keyHandlers.update({'\b': self.handle_BACKSPACE}) + m = self.keyHandlers.get(keyID) + if m is not None: + m() + elif keyID in string.printable: + self.characterReceived(keyID, False) + + sshRealm = manhole_ssh.TerminalRealm() + + def chainedProtocolFactory(): + return insults.ServerProtocol(EnhancedColoredManhole, namespace) + + sshRealm = manhole_ssh.TerminalRealm() + sshRealm.chainedProtocolFactory = chainedProtocolFactory + + portal = Portal( + sshRealm, [MemoryDB(**{user: secret})]) + + f = manhole_ssh.ConchFactory(portal) + return f diff --git a/src/leap/bitmask/mail/imap/service/notes.txt b/src/leap/bitmask/mail/imap/service/notes.txt new file mode 100644 index 0000000..623e122 --- /dev/null +++ b/src/leap/bitmask/mail/imap/service/notes.txt @@ -0,0 +1,81 @@ +T 127.0.0.1:9930 -> 127.0.0.1:42866 [AP] +* OK [CAPABILITY IMAP4rev1 IDLE NAMESPACE] Twisted IMAP4rev1 Ready. + +## +T 127.0.0.1:42866 -> 127.0.0.1:9930 [AP] +NCLJ1 CAPABILITY. + +## +T 127.0.0.1:9930 -> 127.0.0.1:42866 [AP] +* CAPABILITY IMAP4rev1 IDLE NAMESPACE. +NCLJ1 OK CAPABILITY completed. + +## +T 127.0.0.1:42866 -> 127.0.0.1:9930 [AP] +NCLJ2 LOGIN user "pass". + +# +T 127.0.0.1:9930 -> 127.0.0.1:42866 [AP] +NCLJ2 OK LOGIN succeeded. + +## +T 127.0.0.1:42866 -> 127.0.0.1:9930 [AP] +NCLJ3 CAPABILITY. + +# +T 127.0.0.1:9930 -> 127.0.0.1:42866 [AP] +* CAPABILITY IMAP4rev1 IDLE NAMESPACE. +NCLJ3 OK CAPABILITY completed. + +# +T 127.0.0.1:42866 -> 127.0.0.1:9930 [AP] +NCLJ4 LIST "" "". + +## +T 127.0.0.1:9930 -> 127.0.0.1:42866 [AP] +* LIST (\Seen \Answered \Flagged \Deleted \Draft \Recent List) "/" "INBOX". +NCLJ4 OK LIST completed. + +# +T 127.0.0.1:42866 -> 127.0.0.1:9930 [AP] +NCLJ5 LIST "" "*". + +## +T 127.0.0.1:9930 -> 127.0.0.1:42866 [AP] +* LIST (\Seen \Answered \Flagged \Deleted \Draft \Recent List) "/" "INBOX". +NCLJ5 OK LIST completed. + +# +T 127.0.0.1:42866 -> 127.0.0.1:9930 [AP] +NCLJ6 SELECT INBOX. + +# +T 127.0.0.1:9930 -> 127.0.0.1:42866 [AP] +* 0 EXISTS. +* 3 RECENT. +* FLAGS (\Seen \Answered \Flagged \Deleted \Draft \Recent List). +* OK [UIDVALIDITY 42]. +NCLJ6 OK [READ-WRITE] SELECT successful. + +# +T 127.0.0.1:42866 -> 127.0.0.1:9930 [AP] +NCLJ7 EXAMINE INBOX. + +## +T 127.0.0.1:9930 -> 127.0.0.1:42866 [AP] +* 0 EXISTS. +* 3 RECENT. +* FLAGS (\Seen \Answered \Flagged \Deleted \Draft \Recent List). +* OK [UIDVALIDITY 42]. +NCLJ7 OK [READ-ONLY] EXAMINE successful. + +# +T 127.0.0.1:42866 -> 127.0.0.1:9930 [AP] +NCLJ8 LOGOUT. + +## +T 127.0.0.1:9930 -> 127.0.0.1:42866 [AP] +* BYE Nice talking to you. +NCLJ8 OK LOGOUT successful. + + diff --git a/src/leap/bitmask/mail/imap/service/rfc822.message b/src/leap/bitmask/mail/imap/service/rfc822.message new file mode 100644 index 0000000..ee97ab9 --- /dev/null +++ b/src/leap/bitmask/mail/imap/service/rfc822.message @@ -0,0 +1,86 @@ +Return-Path: +Delivered-To: exarkun@meson.dyndns.org +Received: from localhost [127.0.0.1] + by localhost with POP3 (fetchmail-6.2.1) + for exarkun@localhost (single-drop); Thu, 20 Mar 2003 14:50:20 -0500 (EST) +Received: from pyramid.twistedmatrix.com (adsl-64-123-27-105.dsl.austtx.swbell.net [64.123.27.105]) + by intarweb.us (Postfix) with ESMTP id 4A4A513EA4 + for ; Thu, 20 Mar 2003 14:49:27 -0500 (EST) +Received: from localhost ([127.0.0.1] helo=pyramid.twistedmatrix.com) + by pyramid.twistedmatrix.com with esmtp (Exim 3.35 #1 (Debian)) + id 18w648-0007Vl-00; Thu, 20 Mar 2003 13:51:04 -0600 +Received: from acapnotic by pyramid.twistedmatrix.com with local (Exim 3.35 #1 (Debian)) + id 18w63j-0007VK-00 + for ; Thu, 20 Mar 2003 13:50:39 -0600 +To: twisted-commits@twistedmatrix.com +From: etrepum CVS +Reply-To: twisted-python@twistedmatrix.com +X-Mailer: CVSToys +Message-Id: +Subject: [Twisted-commits] rebuild now works on python versions from 2.2.0 and up. +Sender: twisted-commits-admin@twistedmatrix.com +Errors-To: twisted-commits-admin@twistedmatrix.com +X-BeenThere: twisted-commits@twistedmatrix.com +X-Mailman-Version: 2.0.11 +Precedence: bulk +List-Help: +List-Post: +List-Subscribe: , + +List-Id: +List-Unsubscribe: , + +List-Archive: +Date: Thu, 20 Mar 2003 13:50:39 -0600 + +Modified files: +Twisted/twisted/python/rebuild.py 1.19 1.20 + +Log message: +rebuild now works on python versions from 2.2.0 and up. + + +ViewCVS links: +http://twistedmatrix.com/users/jh.twistd/viewcvs/cgi/viewcvs.cgi/twisted/python/rebuild.py.diff?r1=text&tr1=1.19&r2=text&tr2=1.20&cvsroot=Twisted + +Index: Twisted/twisted/python/rebuild.py +diff -u Twisted/twisted/python/rebuild.py:1.19 Twisted/twisted/python/rebuild.py:1.20 +--- Twisted/twisted/python/rebuild.py:1.19 Fri Jan 17 13:50:49 2003 ++++ Twisted/twisted/python/rebuild.py Thu Mar 20 11:50:08 2003 +@@ -206,15 +206,27 @@ + clazz.__dict__.clear() + clazz.__getattr__ = __getattr__ + clazz.__module__ = module.__name__ ++ if newclasses: ++ import gc ++ if (2, 2, 0) <= sys.version_info[:3] < (2, 2, 2): ++ hasBrokenRebuild = 1 ++ gc_objects = gc.get_objects() ++ else: ++ hasBrokenRebuild = 0 + for nclass in newclasses: + ga = getattr(module, nclass.__name__) + if ga is nclass: + log.msg("WARNING: new-class %s not replaced by reload!" % reflect.qual(nclass)) + else: +- import gc +- for r in gc.get_referrers(nclass): +- if isinstance(r, nclass): ++ if hasBrokenRebuild: ++ for r in gc_objects: ++ if not getattr(r, '__class__', None) is nclass: ++ continue + r.__class__ = ga ++ else: ++ for r in gc.get_referrers(nclass): ++ if getattr(r, '__class__', None) is nclass: ++ r.__class__ = ga + if doLog: + log.msg('') + log.msg(' (fixing %s): ' % str(module.__name__)) + + +_______________________________________________ +Twisted-commits mailing list +Twisted-commits@twistedmatrix.com +http://twistedmatrix.com/cgi-bin/mailman/listinfo/twisted-commits diff --git a/src/leap/bitmask/mail/imap/tests/.gitignore b/src/leap/bitmask/mail/imap/tests/.gitignore new file mode 100644 index 0000000..60baa9c --- /dev/null +++ b/src/leap/bitmask/mail/imap/tests/.gitignore @@ -0,0 +1 @@ +data/* diff --git a/src/leap/bitmask/mail/imap/tests/getmail b/src/leap/bitmask/mail/imap/tests/getmail new file mode 100755 index 0000000..dd3fa0b --- /dev/null +++ b/src/leap/bitmask/mail/imap/tests/getmail @@ -0,0 +1,344 @@ +#!/usr/bin/env python + +# Copyright (c) Twisted Matrix Laboratories. +# See LICENSE in twisted for details. + +# Modifications by LEAP Developers 2014 to fit +# Bitmask configuration settings. +""" +Simple IMAP4 client which displays the subjects of all messages in a +particular mailbox. +""" + +import os +import sys + +from twisted.internet import protocol +from twisted.internet import ssl +from twisted.internet import defer +from twisted.internet import stdio +from twisted.mail import imap4 +from twisted.protocols import basic +from twisted.python import log + +# Global options stored here from main +_opts = {} + + +class TrivialPrompter(basic.LineReceiver): + from os import linesep as delimiter + + promptDeferred = None + + def prompt(self, msg): + assert self.promptDeferred is None + self.display(msg) + self.promptDeferred = defer.Deferred() + return self.promptDeferred + + def display(self, msg): + self.transport.write(msg) + + def lineReceived(self, line): + if self.promptDeferred is None: + return + d, self.promptDeferred = self.promptDeferred, None + d.callback(line) + + +class SimpleIMAP4Client(imap4.IMAP4Client): + """ + A client with callbacks for greeting messages from an IMAP server. + """ + greetDeferred = None + + def serverGreeting(self, caps): + self.serverCapabilities = caps + if self.greetDeferred is not None: + d, self.greetDeferred = self.greetDeferred, None + d.callback(self) + + +class SimpleIMAP4ClientFactory(protocol.ClientFactory): + usedUp = False + + protocol = SimpleIMAP4Client + + def __init__(self, username, onConn): + self.ctx = ssl.ClientContextFactory() + + self.username = username + self.onConn = onConn + + def buildProtocol(self, addr): + """ + Initiate the protocol instance. Since we are building a simple IMAP + client, we don't bother checking what capabilities the server has. We + just add all the authenticators twisted.mail has. + """ + assert not self.usedUp + self.usedUp = True + + p = self.protocol(self.ctx) + p.factory = self + p.greetDeferred = self.onConn + + p.registerAuthenticator(imap4.PLAINAuthenticator(self.username)) + p.registerAuthenticator(imap4.LOGINAuthenticator(self.username)) + p.registerAuthenticator( + imap4.CramMD5ClientAuthenticator(self.username)) + + return p + + def clientConnectionFailed(self, connector, reason): + d, self.onConn = self.onConn, None + d.errback(reason) + + +def cbServerGreeting(proto, username, password): + """ + Initial callback - invoked after the server sends us its greet message. + """ + # Hook up stdio + tp = TrivialPrompter() + stdio.StandardIO(tp) + + # And make it easily accessible + proto.prompt = tp.prompt + proto.display = tp.display + + # Try to authenticate securely + return proto.authenticate( + password).addCallback( + cbAuthentication, + proto).addErrback( + ebAuthentication, proto, username, password + ) + + +def ebConnection(reason): + """ + Fallback error-handler. If anything goes wrong, log it and quit. + """ + log.startLogging(sys.stdout) + log.err(reason) + return reason + + +def cbAuthentication(result, proto): + """ + Callback after authentication has succeeded. + + Lists a bunch of mailboxes. + """ + return proto.list("", "*" + ).addCallback(cbMailboxList, proto + ) + + +def ebAuthentication(failure, proto, username, password): + """ + Errback invoked when authentication fails. + + If it failed because no SASL mechanisms match, offer the user the choice + of logging in insecurely. + + If you are trying to connect to your Gmail account, you will be here! + """ + failure.trap(imap4.NoSupportedAuthentication) + return InsecureLogin(proto, username, password) + + +def InsecureLogin(proto, username, password): + """ + insecure-login. + """ + return proto.login(username, password + ).addCallback(cbAuthentication, proto + ) + + +def cbMailboxList(result, proto): + """ + Callback invoked when a list of mailboxes has been retrieved. + If we have a selected mailbox in the global options, we directly pick it. + Otherwise, we offer a prompt to let user choose one. + """ + all_mbox_list = [e[2] for e in result] + s = '\n'.join(['%d. %s' % (n + 1, m) for (n, m) in zip(range(len(all_mbox_list)), all_mbox_list)]) + if not s: + return defer.fail(Exception("No mailboxes exist on server!")) + + selected_mailbox = _opts.get('mailbox') + + if not selected_mailbox: + return proto.prompt(s + "\nWhich mailbox? [1] " + ).addCallback(cbPickMailbox, proto, all_mbox_list + ) + else: + mboxes_lower = map(lambda s: s.lower(), all_mbox_list) + index = mboxes_lower.index(selected_mailbox.lower()) + 1 + return cbPickMailbox(index, proto, all_mbox_list) + + +def cbPickMailbox(result, proto, mboxes): + """ + When the user selects a mailbox, "examine" it. + """ + mbox = mboxes[int(result or '1') - 1] + return proto.examine(mbox + ).addCallback(cbExamineMbox, proto + ) + + +def cbExamineMbox(result, proto): + """ + Callback invoked when examine command completes. + + Retrieve the subject header of every message in the mailbox. + """ + return proto.fetchSpecific('1:*', + headerType='HEADER.FIELDS', + headerArgs=['SUBJECT'], + ).addCallback(cbFetch, proto, + ) + + +def cbFetch(result, proto): + """ + Display a listing of the messages in the mailbox, based on the collected + headers. + """ + selected_subject = _opts.get('subject', None) + index = None + + if result: + keys = result.keys() + keys.sort() + + if selected_subject: + for k in keys: + # remove 'Subject: ' preffix plus eol + subject = result[k][0][2][9:].rstrip('\r\n') + if subject.lower() == selected_subject.lower(): + index = k + break + else: + for k in keys: + proto.display('%s %s' % (k, result[k][0][2])) + else: + print "Hey, an empty mailbox!" + + if not index: + return proto.prompt("\nWhich message? [1] (Q quits) " + ).addCallback(cbPickMessage, proto) + else: + return cbPickMessage(index, proto) + + +def cbPickMessage(result, proto): + """ + Pick a message. + """ + if result == "Q": + print "Bye!" + return proto.logout() + + return proto.fetchSpecific( + '%s' % result, + headerType='', + headerArgs=['BODY.PEEK[]'], + ).addCallback(cbShowmessage, proto) + + +def cbShowmessage(result, proto): + """ + Display message. + """ + if result: + keys = result.keys() + keys.sort() + for k in keys: + proto.display('%s %s' % (k, result[k][0][2])) + else: + print "Hey, an empty message!" + + return proto.logout() + + +def cbClose(result): + """ + Close the connection when we finish everything. + """ + from twisted.internet import reactor + reactor.stop() + + +def main(): + import argparse + import ConfigParser + import sys + from twisted.internet import reactor + + description = ( + 'Get messages from a LEAP IMAP Proxy.\nThis is a ' + 'debugging tool, do not use this to retrieve any sensitive ' + 'information, or we will send ninjas to your house!') + epilog = ( + 'In case you want to automate the usage of this utility ' + 'you can place your credentials in a file pointed by ' + 'BITMASK_CREDENTIALS. You need to have a [Credentials] ' + 'section, with username= and password fields') + + parser = argparse.ArgumentParser(description=description, epilog=epilog) + credentials = os.environ.get('BITMASK_CREDENTIALS') + + if credentials: + try: + config = ConfigParser.ConfigParser() + config.read(credentials) + username = config.get('Credentials', 'username') + password = config.get('Credentials', 'password') + except Exception, e: + print "Error reading credentials file: {0}".format(e) + sys.exit() + else: + parser.add_argument('username', type=str) + parser.add_argument('password', type=str) + + parser.add_argument('--mailbox', dest='mailbox', default=None, + help='Which mailbox to retrieve. Empty for interactive prompt.') + parser.add_argument('--subject', dest='subject', default=None, + help='A subject for retrieve a mail that matches. Empty for interactive prompt.') + + ns = parser.parse_args() + + if not credentials: + username = ns.username + password = ns.password + + _opts['mailbox'] = ns.mailbox + _opts['subject'] = ns.subject + + hostname = "localhost" + port = "1984" + + onConn = defer.Deferred( + ).addCallback(cbServerGreeting, username, password + ).addErrback(ebConnection + ).addBoth(cbClose) + + factory = SimpleIMAP4ClientFactory(username, onConn) + + if port == '993': + reactor.connectSSL( + hostname, int(port), factory, ssl.ClientContextFactory()) + else: + if not port: + port = 143 + reactor.connectTCP(hostname, int(port), factory) + reactor.run() + + +if __name__ == '__main__': + main() diff --git a/src/leap/bitmask/mail/imap/tests/imapclient.py b/src/leap/bitmask/mail/imap/tests/imapclient.py new file mode 100755 index 0000000..c353cee --- /dev/null +++ b/src/leap/bitmask/mail/imap/tests/imapclient.py @@ -0,0 +1,207 @@ +#!/usr/bin/env python + +# Copyright (c) Twisted Matrix Laboratories. +# See LICENSE for details. + + +""" +Simple IMAP4 client which connects to our custome +IMAP4 server: imapserver.py. +""" + +import sys + +from twisted.internet import protocol +from twisted.internet import defer +from twisted.internet import stdio +from twisted.mail import imap4 +from twisted.protocols import basic +from twisted.python import util +from twisted.python import log + + +class TrivialPrompter(basic.LineReceiver): + # from os import linesep as delimiter + + promptDeferred = None + + def prompt(self, msg): + assert self.promptDeferred is None + self.display(msg) + self.promptDeferred = defer.Deferred() + return self.promptDeferred + + def display(self, msg): + self.transport.write(msg) + + def lineReceived(self, line): + if self.promptDeferred is None: + return + d, self.promptDeferred = self.promptDeferred, None + d.callback(line) + + +class SimpleIMAP4Client(imap4.IMAP4Client): + + """ + Add callbacks when the client receives greeting messages from + an IMAP server. + """ + greetDeferred = None + + def serverGreeting(self, caps): + self.serverCapabilities = caps + if self.greetDeferred is not None: + d, self.greetDeferred = self.greetDeferred, None + d.callback(self) + + +class SimpleIMAP4ClientFactory(protocol.ClientFactory): + usedUp = False + protocol = SimpleIMAP4Client + + def __init__(self, username, onConn): + self.username = username + self.onConn = onConn + + def buildProtocol(self, addr): + assert not self.usedUp + self.usedUp = True + + p = self.protocol() + p.factory = self + p.greetDeferred = self.onConn + + p.registerAuthenticator(imap4.PLAINAuthenticator(self.username)) + p.registerAuthenticator(imap4.LOGINAuthenticator(self.username)) + p.registerAuthenticator( + imap4.CramMD5ClientAuthenticator(self.username)) + + return p + + def clientConnectionFailed(self, connector, reason): + d, self.onConn = self.onConn, None + d.errback(reason) + + +def cbServerGreeting(proto, username, password): + """ + Initial callback - invoked after the server sends us its greet message. + """ + # Hook up stdio + tp = TrivialPrompter() + stdio.StandardIO(tp) + + # And make it easily accessible + proto.prompt = tp.prompt + proto.display = tp.display + + # Try to authenticate securely + return proto.authenticate( + password).addCallback( + cbAuthentication, proto).addErrback( + ebAuthentication, proto, username, password) + + +def ebConnection(reason): + """ + Fallback error-handler. If anything goes wrong, log it and quit. + """ + log.startLogging(sys.stdout) + log.err(reason) + return reason + + +def cbAuthentication(result, proto): + """ + Callback after authentication has succeeded. + List a bunch of mailboxes. + """ + return proto.list("", "*" + ).addCallback(cbMailboxList, proto + ) + + +def ebAuthentication(failure, proto, username, password): + """ + Errback invoked when authentication fails. + If it failed because no SASL mechanisms match, offer the user the choice + of logging in insecurely. + If you are trying to connect to your Gmail account, you will be here! + """ + failure.trap(imap4.NoSupportedAuthentication) + return proto.prompt( + "No secure authentication available. Login insecurely? (y/N) " + ).addCallback(cbInsecureLogin, proto, username, password + ) + + +def cbInsecureLogin(result, proto, username, password): + """ + Callback for "insecure-login" prompt. + """ + if result.lower() == "y": + # If they said yes, do it. + return proto.login(username, password + ).addCallback(cbAuthentication, proto + ) + return defer.fail(Exception("Login failed for security reasons.")) + + +def cbMailboxList(result, proto): + """ + Callback invoked when a list of mailboxes has been retrieved. + """ + result = [e[2] for e in result] + s = '\n'.join( + ['%d. %s' % (n + 1, m) for (n, m) in zip(range(len(result)), result)]) + if not s: + return defer.fail(Exception("No mailboxes exist on server!")) + return proto.prompt(s + "\nWhich mailbox? [1] " + ).addCallback(cbPickMailbox, proto, result + ) + + +def cbPickMailbox(result, proto, mboxes): + """ + When the user selects a mailbox, "examine" it. + """ + mbox = mboxes[int(result or '1') - 1] + return proto.status(mbox, 'MESSAGES', 'UNSEEN' + ).addCallback(cbMboxStatus, proto) + + +def cbMboxStatus(result, proto): + print "You have %s messages (%s unseen)!" % ( + result['MESSAGES'], result['UNSEEN']) + return proto.logout() + + +def cbClose(result): + """ + Close the connection when we finish everything. + """ + from twisted.internet import reactor + reactor.stop() + + +def main(): + hostname = raw_input('IMAP4 Server Hostname: ') + port = raw_input('IMAP4 Server Port (the default is 143): ') + username = raw_input('IMAP4 Username: ') + password = util.getPassword('IMAP4 Password: ') + + onConn = defer.Deferred( + ).addCallback(cbServerGreeting, username, password + ).addErrback(ebConnection + ).addBoth(cbClose) + + factory = SimpleIMAP4ClientFactory(username, onConn) + + from twisted.internet import reactor + conn = reactor.connectTCP(hostname, int(port), factory) + reactor.run() + + +if __name__ == '__main__': + main() diff --git a/src/leap/bitmask/mail/imap/tests/regressions_mime_struct b/src/leap/bitmask/mail/imap/tests/regressions_mime_struct new file mode 100755 index 0000000..0332664 --- /dev/null +++ b/src/leap/bitmask/mail/imap/tests/regressions_mime_struct @@ -0,0 +1,461 @@ +#!/usr/bin/env python + +# -*- coding: utf-8 -*- +# regression_mime_struct +# Copyright (C) 2014 LEAP +# Copyright (c) Twisted Matrix Laboratories. +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . +""" +Simple Regression Tests for checking MIME struct handling using IMAP4 client. + +Iterates trough all mails under a given folder and tries to APPEND them to +the server being tested. After FETCHING the pushed message, it compares +the received version with the one that was saved, and exits with an error +code if they do not match. +""" +import os +import StringIO +import sys + +from email.parser import Parser + +from twisted.internet import protocol +from twisted.internet import ssl +from twisted.internet import defer +from twisted.internet import stdio +from twisted.mail import imap4 +from twisted.protocols import basic +from twisted.python import log + + +REGRESSIONS_FOLDER = os.environ.get( + "REGRESSIONS_FOLDER", "regressions_test") +print "[+] Using regressions folder:", REGRESSIONS_FOLDER + +parser = Parser() + + +def get_msg_parts(raw): + """ + Return a representation of the parts of a message suitable for + comparison. + + :param raw: string for the message + :type raw: str + """ + m = parser.parsestr(raw) + return [dict(part.items()) + if part.is_multipart() + else part.get_payload() + for part in m.walk()] + + +def compare_msg_parts(a, b): + """ + Compare two sequences of parts of messages. + + :param a: part sequence for message a + :param b: part sequence for message b + + :return: True if both message sequences are equivalent. + :rtype: bool + """ + # XXX This could be smarter and show the differences in the + # different parts when/where they differ. + #import pprint; pprint.pprint(a[0]) + #import pprint; pprint.pprint(b[0]) + + def lowerkey(d): + return dict((k.lower(), v.replace('\r', '')) + for k, v in d.iteritems()) + + def eq(x, y): + # For dicts, we compare a variation with their keys + # in lowercase, and \r removed from their values + if all(map(lambda i: isinstance(i, dict), (x, y))): + x, y = map(lowerkey, (x, y)) + return x == y + + compare_vector = map(lambda tup: eq(tup[0], tup[1]), zip(a, b)) + all_match = all(compare_vector) + + if not all_match: + print "PARTS MISMATCH!" + print "vector: ", compare_vector + index = compare_vector.index(False) + from pprint import pprint + print "Expected:" + pprint(a[index]) + print ("***") + print "Found:" + pprint(b[index]) + print + + return all_match + + +def get_fd(string): + """ + Return a file descriptor with the passed string + as content. + """ + fd = StringIO.StringIO() + fd.write(string) + fd.seek(0) + return fd + + +class TrivialPrompter(basic.LineReceiver): + promptDeferred = None + + def prompt(self, msg): + assert self.promptDeferred is None + self.display(msg) + self.promptDeferred = defer.Deferred() + return self.promptDeferred + + def display(self, msg): + self.transport.write(msg) + + def lineReceived(self, line): + if self.promptDeferred is None: + return + d, self.promptDeferred = self.promptDeferred, None + d.callback(line) + + +class SimpleIMAP4Client(imap4.IMAP4Client): + """ + A client with callbacks for greeting messages from an IMAP server. + """ + greetDeferred = None + + def serverGreeting(self, caps): + self.serverCapabilities = caps + if self.greetDeferred is not None: + d, self.greetDeferred = self.greetDeferred, None + d.callback(self) + + +class SimpleIMAP4ClientFactory(protocol.ClientFactory): + usedUp = False + protocol = SimpleIMAP4Client + + def __init__(self, username, onConn): + self.ctx = ssl.ClientContextFactory() + + self.username = username + self.onConn = onConn + + def buildProtocol(self, addr): + """ + Initiate the protocol instance. Since we are building a simple IMAP + client, we don't bother checking what capabilities the server has. We + just add all the authenticators twisted.mail has. Note: Gmail no + longer uses any of the methods below, it's been using XOAUTH since + 2010. + """ + assert not self.usedUp + self.usedUp = True + + p = self.protocol(self.ctx) + p.factory = self + p.greetDeferred = self.onConn + + p.registerAuthenticator(imap4.PLAINAuthenticator(self.username)) + p.registerAuthenticator(imap4.LOGINAuthenticator(self.username)) + p.registerAuthenticator( + imap4.CramMD5ClientAuthenticator(self.username)) + + return p + + def clientConnectionFailed(self, connector, reason): + d, self.onConn = self.onConn, None + d.errback(reason) + + +def cbServerGreeting(proto, username, password): + """ + Initial callback - invoked after the server sends us its greet message. + """ + # Hook up stdio + tp = TrivialPrompter() + stdio.StandardIO(tp) + + # And make it easily accessible + proto.prompt = tp.prompt + proto.display = tp.display + + # Try to authenticate securely + return proto.authenticate( + password).addCallback( + cbAuthentication, + proto).addErrback( + ebAuthentication, proto, username, password + ) + + +def ebConnection(reason): + """ + Fallback error-handler. If anything goes wrong, log it and quit. + """ + log.startLogging(sys.stdout) + log.err(reason) + return reason + + +def cbAuthentication(result, proto): + """ + Callback after authentication has succeeded. + + Lists a bunch of mailboxes. + """ + return proto.select( + REGRESSIONS_FOLDER + ).addCallback( + cbSelectMbox, proto + ).addErrback( + ebSelectMbox, proto, REGRESSIONS_FOLDER) + + +def ebAuthentication(failure, proto, username, password): + """ + Errback invoked when authentication fails. + + If it failed because no SASL mechanisms match, offer the user the choice + of logging in insecurely. + + If you are trying to connect to your Gmail account, you will be here! + """ + failure.trap(imap4.NoSupportedAuthentication) + return InsecureLogin(proto, username, password) + + +def InsecureLogin(proto, username, password): + """ + Raise insecure-login error. + """ + return proto.login( + username, password + ).addCallback( + cbAuthentication, proto) + + +def cbSelectMbox(result, proto): + """ + Callback invoked when select command finishes successfully. + + If any message is in the test folder, it will flag them as deleted and + expunge. + If no messages found, it will start with the APPEND tests. + """ + print "SELECT: %s EXISTS " % result.get("EXISTS", "??") + + if result["EXISTS"] != 0: + # Flag as deleted, expunge, and do an examine again. + print "There is mail here, will delete..." + return cbDeleteAndExpungeTestFolder(proto) + + else: + return cbAppendNextMessage(proto) + + +def ebSelectMbox(failure, proto, folder): + """ + Errback invoked when the examine command fails. + + Creates the folder. + """ + log.err(failure) + log.msg("Folder %r does not exist. Creating..." % (folder,)) + return proto.create(folder).addCallback(cbAuthentication, proto) + + +def ebExpunge(failure): + log.err(failure) + + +def cbDeleteAndExpungeTestFolder(proto): + """ + Callback invoked fom cbExamineMbox when the number of messages in the + mailbox is not zero. It flags all messages as deleted and expunge the + mailbox. + """ + return proto.setFlags( + "1:*", ("\\Deleted",) + ).addCallback( + lambda r: proto.expunge() + ).addCallback( + cbExpunge, proto + ).addErrback( + ebExpunge) + + +def cbExpunge(result, proto): + return proto.select( + REGRESSIONS_FOLDER + ).addCallback( + cbSelectMbox, proto + ).addErrback(ebSettingDeleted, proto) + + +def ebSettingDeleted(failure, proto): + """ + Report errors during deletion of messages in the mailbox. + """ + print failure.getTraceback() + + +def cbAppendNextMessage(proto): + """ + Appends the next message in the global queue to the test folder. + """ + # 1. Get the next test message from global tuple. + try: + next_sample = SAMPLES.pop() + except IndexError: + # we're done! + return proto.logout() + + print "\nAPPEND %s" % (next_sample,) + raw = open(next_sample).read() + msg = get_fd(raw) + return proto.append( + REGRESSIONS_FOLDER, msg + ).addCallback( + lambda r: proto.select(REGRESSIONS_FOLDER) + ).addCallback( + cbAppend, proto, raw + ).addErrback( + ebAppend, proto, raw) + + +def cbAppend(result, proto, orig_msg): + """ + Fetches the message right after an append. + """ + # XXX keep account of highest UID + uid = "1:*" + + return proto.fetchSpecific( + '%s' % uid, + headerType='', + headerArgs=['BODY.PEEK[]'], + ).addCallback( + cbCompareMessage, proto, orig_msg + ).addErrback(ebAppend, proto, orig_msg) + + +def ebAppend(failure, proto, raw): + """ + Errorback for the append operation + """ + print "ERROR WHILE APPENDING!" + print failure.getTraceback() + + +def cbPickMessage(result, proto): + """ + Pick a message. + """ + return proto.fetchSpecific( + '%s' % result, + headerType='', + headerArgs=['BODY.PEEK[]'], + ).addCallback(cbCompareMessage, proto) + + +def cbCompareMessage(result, proto, raw): + """ + Display message and compare it with the original one. + """ + parts_orig = get_msg_parts(raw) + + if result: + keys = result.keys() + keys.sort() + else: + print "[-] GOT NO RESULT" + return proto.logout() + + latest = max(keys) + + fetched_msg = result[latest][0][2] + parts_fetched = get_msg_parts(fetched_msg) + + equal = compare_msg_parts( + parts_orig, + parts_fetched) + + if equal: + print "[+] MESSAGES MATCH" + return cbAppendNextMessage(proto) + else: + print "[-] ERROR: MESSAGES DO NOT MATCH !!!" + print " ABORTING COMPARISON..." + # FIXME logout and print the subject ... + return proto.logout() + + +def cbClose(result): + """ + Close the connection when we finish everything. + """ + from twisted.internet import reactor + reactor.stop() + + +def main(): + import glob + import sys + + if len(sys.argv) != 4: + print "Usage: regressions " + sys.exit() + + hostname = "localhost" + port = "1984" + username = sys.argv[1] + password = sys.argv[2] + + samplesdir = sys.argv[3] + + if not os.path.isdir(samplesdir): + print ("Could not find samples folder! " + "Make sure of copying mail_breaker contents there.") + sys.exit() + + samples = glob.glob(samplesdir + '/*') + + global SAMPLES + SAMPLES = [] + SAMPLES += samples + + onConn = defer.Deferred( + ).addCallback( + cbServerGreeting, username, password + ).addErrback( + ebConnection + ).addBoth(cbClose) + + factory = SimpleIMAP4ClientFactory(username, onConn) + + from twisted.internet import reactor + reactor.connectTCP(hostname, int(port), factory) + reactor.run() + + +if __name__ == '__main__': + main() diff --git a/src/leap/bitmask/mail/imap/tests/rfc822.message b/src/leap/bitmask/mail/imap/tests/rfc822.message new file mode 120000 index 0000000..b19cc28 --- /dev/null +++ b/src/leap/bitmask/mail/imap/tests/rfc822.message @@ -0,0 +1 @@ +../../tests/rfc822.message \ No newline at end of file diff --git a/src/leap/bitmask/mail/imap/tests/rfc822.multi-minimal.message b/src/leap/bitmask/mail/imap/tests/rfc822.multi-minimal.message new file mode 120000 index 0000000..e0aa678 --- /dev/null +++ b/src/leap/bitmask/mail/imap/tests/rfc822.multi-minimal.message @@ -0,0 +1 @@ +../../tests/rfc822.multi-minimal.message \ No newline at end of file diff --git a/src/leap/bitmask/mail/imap/tests/rfc822.multi-nested.message b/src/leap/bitmask/mail/imap/tests/rfc822.multi-nested.message new file mode 120000 index 0000000..306d0de --- /dev/null +++ b/src/leap/bitmask/mail/imap/tests/rfc822.multi-nested.message @@ -0,0 +1 @@ +../../tests/rfc822.multi-nested.message \ No newline at end of file diff --git a/src/leap/bitmask/mail/imap/tests/rfc822.multi-signed.message b/src/leap/bitmask/mail/imap/tests/rfc822.multi-signed.message new file mode 120000 index 0000000..4172244 --- /dev/null +++ b/src/leap/bitmask/mail/imap/tests/rfc822.multi-signed.message @@ -0,0 +1 @@ +../../tests/rfc822.multi-signed.message \ No newline at end of file diff --git a/src/leap/bitmask/mail/imap/tests/rfc822.multi.message b/src/leap/bitmask/mail/imap/tests/rfc822.multi.message new file mode 120000 index 0000000..62057d2 --- /dev/null +++ b/src/leap/bitmask/mail/imap/tests/rfc822.multi.message @@ -0,0 +1 @@ +../../tests/rfc822.multi.message \ No newline at end of file diff --git a/src/leap/bitmask/mail/imap/tests/rfc822.plain.message b/src/leap/bitmask/mail/imap/tests/rfc822.plain.message new file mode 120000 index 0000000..5bab0e8 --- /dev/null +++ b/src/leap/bitmask/mail/imap/tests/rfc822.plain.message @@ -0,0 +1 @@ +../../tests/rfc822.plain.message \ No newline at end of file diff --git a/src/leap/bitmask/mail/imap/tests/stress_tests_imap.zsh b/src/leap/bitmask/mail/imap/tests/stress_tests_imap.zsh new file mode 100755 index 0000000..544faca --- /dev/null +++ b/src/leap/bitmask/mail/imap/tests/stress_tests_imap.zsh @@ -0,0 +1,178 @@ +#!/bin/zsh +# BATCH STRESS TEST FOR IMAP ---------------------- +# http://imgs.xkcd.com/comics/science.jpg +# +# Run imaptest against a LEAP IMAP server +# for a fixed period of time, and collect output. +# +# Author: Kali Kaneko +# Date: 2014 01 26 +# +# To run, you need to have `imaptest` in your path. +# See: +# http://www.imapwiki.org/ImapTest/Installation +# +# For the tests, I'm using a 10MB file sample that +# can be downloaded from: +# http://www.dovecot.org/tmp/dovecot-crlf +# +# Want to contribute to benchmarking? +# +# 1. Create a pristine account in a bitmask provider. +# +# 2. Launch your bitmask client, with different flags +# if you desire. +# +# For example to try the nosync flag in sqlite: +# +# LEAP_SQLITE_NOSYNC=1 bitmask --debug -N --offline -l /tmp/leap.log +# +# 3. Run at several points in time (ie: just after +# launching the bitmask client. one minute after, +# ten minutes after) +# +# mkdir data +# cd data +# ../leap_tests_imap.zsh | tee sqlite_nosync_run2.log +# +# 4. Submit your results to: kali at leap dot se +# together with the logs of the bitmask run. +# +# Please provide also details about your system, and +# the type of hard disk setup you are running against. +# + +# ------------------------------------------------ +# Edit these variables if you are too lazy to pass +# the user and mbox as parameters. Like me. + +USER="test_f14@dev.bitmask.net" +MBOX="~/leap/imaptest/data/dovecot-crlf" + +HOST="localhost" +PORT="1984" + +# in case you have it aliased +GREP="/bin/grep" +IMAPTEST="imaptest" + +# ----------------------------------------------- +# +# These should be kept constant across benchmarking +# runs across different machines, for comparability. + +DURATION=200 +NUM_MSG=200 + + +# TODO add another function, and a cli flag, to be able +# to take several aggretates spaced in time, along a period +# of several minutes. + +imaptest_cmd() { + stdbuf -o0 ${IMAPTEST} user=${USER} pass=1234 host=${HOST} \ + port=${PORT} mbox=${MBOX} clients=1 msgs=${NUM_MSG} \ + no_pipelining 2>/dev/null +} + +stress_imap() { + mkfifo imap_pipe + cat imap_pipe | tee output & + imaptest_cmd >> imap_pipe +} + +wait_and_kill() { + while : + do + sleep $DURATION + pkill -2 imaptest + rm imap_pipe + break + done +} + +print_results() { + sleep 1 + echo + echo + echo "AGGREGATED RESULTS" + echo "----------------------" + echo "\tavg\tstdev" + $GREP "avg" ./output | sed -e 's/^ *//g' -e 's/ *$//g' | \ + gawk ' +function avg(data, count) { + sum=0; + for( x=0; x <= count-1; x++) { + sum += data[x]; + } + return sum/count; +} +function std_dev(data, count) { + sum=0; + for( x=0; x <= count-1; x++) { + sum += data[x]; + } + average = sum/count; + + sumsq=0; + for( x=0; x <= count-1; x++) { + sumsq += (data[x] - average)^2; + } + return sqrt(sumsq/count); +} +BEGIN { + cnt = 0 +} END { + +printf("LOGI:\t%04.2lf\t%04.2f\n", avg(array[1], NR), std_dev(array[1], NR)); +printf("LIST:\t%04.2lf\t%04.2f\n", avg(array[2], NR), std_dev(array[2], NR)); +printf("STAT:\t%04.2lf\t%04.2f\n", avg(array[3], NR), std_dev(array[3], NR)); +printf("SELE:\t%04.2lf\t%04.2f\n", avg(array[4], NR), std_dev(array[4], NR)); +printf("FETC:\t%04.2lf\t%04.2f\n", avg(array[5], NR), std_dev(array[5], NR)); +printf("FET2:\t%04.2lf\t%04.2f\n", avg(array[6], NR), std_dev(array[6], NR)); +printf("STOR:\t%04.2lf\t%04.2f\n", avg(array[7], NR), std_dev(array[7], NR)); +printf("DELE:\t%04.2lf\t%04.2f\n", avg(array[8], NR), std_dev(array[8], NR)); +printf("EXPU:\t%04.2lf\t%04.2f\n", avg(array[9], NR), std_dev(array[9], NR)); +printf("APPE:\t%04.2lf\t%04.2f\n", avg(array[10], NR), std_dev(array[10], NR)); +printf("LOGO:\t%04.2lf\t%04.2f\n", avg(array[11], NR), std_dev(array[11], NR)); + +print "" +print "TOT samples", NR; +} +{ + it = cnt++; + array[1][it] = $1; + array[2][it] = $2; + array[3][it] = $3; + array[4][it] = $4; + array[5][it] = $5; + array[6][it] = $6; + array[7][it] = $7; + array[8][it] = $8; + array[9][it] = $9; + array[10][it] = $10; + array[11][it] = $11; +}' +} + + +{ test $1 = "--help" } && { + echo "Usage: $0 [user@provider] [/path/to/sample.mbox]" + exit 0 +} + +# If the first parameter is passed, take it as the user +{ test $1 } && { + USER=$1 +} + +# If the second parameter is passed, take it as the mbox +{ test $2 } && { + MBOX=$2 +} + +echo "[+] LEAP IMAP TESTS" +echo "[+] Running imaptest for $DURATION seconds with $NUM_MSG messages" +wait_and_kill & +stress_imap +print_results diff --git a/src/leap/bitmask/mail/imap/tests/test_imap.py b/src/leap/bitmask/mail/imap/tests/test_imap.py new file mode 100644 index 0000000..9cca17f --- /dev/null +++ b/src/leap/bitmask/mail/imap/tests/test_imap.py @@ -0,0 +1,1060 @@ +# -*- coding: utf-8 -*- +# test_imap.py +# Copyright (C) 2013 LEAP +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . +""" +Test case for leap.email.imap.server +TestCases taken from twisted tests and modified to make them work +against our implementation of the IMAPAccount. + +@authors: Kali Kaneko, +XXX add authors from the original twisted tests. + +@license: GPLv3, see included LICENSE file +""" +# XXX review license of the original tests!!! +import os +import string +import types + + +from twisted.mail import imap4 +from twisted.internet import defer +from twisted.python import util +from twisted.python import failure + +from twisted import cred + +from leap.mail.imap.mailbox import IMAPMailbox +from leap.mail.imap.messages import CaseInsensitiveDict +from leap.mail.testing.imap import IMAP4HelperMixin + + +TEST_USER = "testuser@leap.se" +TEST_PASSWD = "1234" + + +def strip(f): + return lambda result, f=f: f() + + +def sortNest(l): + l = l[:] + l.sort() + for i in range(len(l)): + if isinstance(l[i], types.ListType): + l[i] = sortNest(l[i]) + elif isinstance(l[i], types.TupleType): + l[i] = tuple(sortNest(list(l[i]))) + return l + + +class TestRealm: + """ + A minimal auth realm for testing purposes only + """ + theAccount = None + + def requestAvatar(self, avatarId, mind, *interfaces): + return imap4.IAccount, self.theAccount, lambda: None + +# +# TestCases +# + +# DEBUG --- +# from twisted.internet.base import DelayedCall +# DelayedCall.debug = True + + +class LEAPIMAP4ServerTestCase(IMAP4HelperMixin): + + """ + Tests for the generic behavior of the LEAPIMAP4Server + which, right now, it's just implemented in this test file as + LEAPIMAPServer. We will move the implementation, together with + authentication bits, to leap.mail.imap.server so it can be instantiated + from the tac file. + + Right now this TestCase tries to mimmick as close as possible the + organization from the twisted.mail.imap tests so we can achieve + a complete implementation. The order in which they appear reflect + the intended order of implementation. + """ + + # + # mailboxes operations + # + + def testCreate(self): + """ + Test whether we can create mailboxes + """ + succeed = ('testbox', 'test/box', 'test/', 'test/box/box', 'foobox') + fail = ('testbox', 'test/box') + acc = self.server.theAccount + + def cb(): + self.result.append(1) + + def eb(failure): + self.result.append(0) + + def login(): + return self.client.login(TEST_USER, TEST_PASSWD) + + def create(): + create_deferreds = [] + for name in succeed + fail: + d = self.client.create(name) + d.addCallback(strip(cb)).addErrback(eb) + create_deferreds.append(d) + dd = defer.gatherResults(create_deferreds) + dd.addCallbacks(self._cbStopClient, self._ebGeneral) + return dd + + self.result = [] + d1 = self.connected.addCallback(strip(login)) + d1.addCallback(strip(create)) + d2 = self.loopback() + d = defer.gatherResults([d1, d2], consumeErrors=True) + d.addCallback(lambda _: acc.account.list_all_mailbox_names()) + return d.addCallback(self._cbTestCreate, succeed, fail) + + def _cbTestCreate(self, mailboxes, succeed, fail): + self.assertEqual(self.result, [1] * len(succeed) + [0] * len(fail)) + + answers = ([u'INBOX', u'testbox', u'test/box', u'test', + u'test/box/box', 'foobox']) + self.assertEqual(sorted(mailboxes), sorted([a for a in answers])) + + def testDelete(self): + """ + Test whether we can delete mailboxes + """ + def add_mailbox(): + return self.server.theAccount.addMailbox('test-delete/me') + + def login(): + return self.client.login(TEST_USER, TEST_PASSWD) + + def delete(): + return self.client.delete('test-delete/me') + + acc = self.server.theAccount.account + + d1 = self.connected.addCallback(add_mailbox) + d1.addCallback(strip(login)) + d1.addCallbacks(strip(delete), self._ebGeneral) + d1.addCallbacks(self._cbStopClient, self._ebGeneral) + d2 = self.loopback() + d = defer.gatherResults([d1, d2]) + d.addCallback(lambda _: acc.list_all_mailbox_names()) + d.addCallback(lambda mboxes: self.assertEqual( + mboxes, ['INBOX'])) + return d + + def testIllegalInboxDelete(self): + """ + Test what happens if we try to delete the user Inbox. + We expect that operation to fail. + """ + self.stashed = None + + def login(): + return self.client.login(TEST_USER, TEST_PASSWD) + + def delete(): + return self.client.delete('inbox') + + def stash(result): + self.stashed = result + + d1 = self.connected.addCallback(strip(login)) + d1.addCallbacks(strip(delete), self._ebGeneral) + d1.addBoth(stash) + d1.addCallbacks(self._cbStopClient, self._ebGeneral) + d2 = self.loopback() + d = defer.gatherResults([d1, d2]) + d.addCallback(lambda _: self.failUnless(isinstance(self.stashed, + failure.Failure))) + return d + + def testNonExistentDelete(self): + """ + Test what happens if we try to delete a non-existent mailbox. + We expect an error raised stating 'No such mailbox' + """ + def login(): + return self.client.login(TEST_USER, TEST_PASSWD) + + def delete(): + return self.client.delete('delete/me') + self.failure = failure + + def deleteFailed(failure): + self.failure = failure + + self.failure = None + d1 = self.connected.addCallback(strip(login)) + d1.addCallback(strip(delete)).addErrback(deleteFailed) + d1.addCallbacks(self._cbStopClient, self._ebGeneral) + d2 = self.loopback() + d = defer.gatherResults([d1, d2]) + d.addCallback(lambda _: self.assertTrue( + str(self.failure.value).startswith('No such mailbox'))) + return d + + def testIllegalDelete(self): + """ + Try deleting a mailbox with sub-folders, and \NoSelect flag set. + An exception is expected. + """ + acc = self.server.theAccount + + def login(): + return self.client.login(TEST_USER, TEST_PASSWD) + + def create_mailboxes(): + d1 = acc.addMailbox('delete') + d2 = acc.addMailbox('delete/me') + d = defer.gatherResults([d1, d2]) + return d + + def get_noselect_mailbox(mboxes): + mbox = mboxes[0] + return mbox.setFlags((r'\Noselect',)) + + def delete_mbox(ignored): + return self.client.delete('delete') + + def deleteFailed(failure): + self.failure = failure + + self.failure = None + + d1 = self.connected.addCallback(strip(login)) + d1.addCallback(strip(create_mailboxes)) + d1.addCallback(get_noselect_mailbox) + + d1.addCallback(delete_mbox).addErrback(deleteFailed) + d1.addCallbacks(self._cbStopClient, self._ebGeneral) + d2 = self.loopback() + d = defer.gatherResults([d1, d2]) + expected = ("Hierarchically inferior mailboxes exist " + "and \\Noselect is set") + d.addCallback(lambda _: + self.assertTrue(self.failure is not None)) + d.addCallback(lambda _: + self.assertEqual(str(self.failure.value), expected)) + return d + + # FIXME --- this test sometimes FAILS (timing issue). + # Some of the deferreds used in the rename op is not waiting for the + # operations properly + def testRename(self): + """ + Test whether we can rename a mailbox + """ + def create_mbox(): + return self.server.theAccount.addMailbox('oldmbox') + + def login(): + return self.client.login(TEST_USER, TEST_PASSWD) + + def rename(): + return self.client.rename('oldmbox', 'newname') + + d1 = self.connected.addCallback(strip(create_mbox)) + d1.addCallback(strip(login)) + d1.addCallbacks(strip(rename), self._ebGeneral) + d1.addCallbacks(self._cbStopClient, self._ebGeneral) + d2 = self.loopback() + d = defer.gatherResults([d1, d2]) + d.addCallback(lambda _: + self.server.theAccount.account.list_all_mailbox_names()) + d.addCallback(lambda mboxes: + self.assertItemsEqual(mboxes, ['INBOX', 'newname'])) + return d + + def testIllegalInboxRename(self): + """ + Try to rename inbox. We expect it to fail. Then it would be not + an inbox anymore, would it? + """ + self.stashed = None + + def login(): + return self.client.login(TEST_USER, TEST_PASSWD) + + def rename(): + return self.client.rename('inbox', 'frotz') + + def stash(stuff): + self.stashed = stuff + + d1 = self.connected.addCallback(strip(login)) + d1.addCallbacks(strip(rename), self._ebGeneral) + d1.addBoth(stash) + d1.addCallbacks(self._cbStopClient, self._ebGeneral) + d2 = self.loopback() + d = defer.gatherResults([d1, d2]) + d.addCallback(lambda _: + self.failUnless(isinstance( + self.stashed, failure.Failure))) + return d + + def testHierarchicalRename(self): + """ + Try to rename hierarchical mailboxes + """ + acc = self.server.theAccount + + def add_mailboxes(): + return defer.gatherResults([ + acc.addMailbox('oldmbox/m1'), + acc.addMailbox('oldmbox/m2')]) + + def login(): + return self.client.login(TEST_USER, TEST_PASSWD) + + def rename(): + return self.client.rename('oldmbox', 'newname') + + d1 = self.connected.addCallback(strip(add_mailboxes)) + d1.addCallback(strip(login)) + d1.addCallbacks(strip(rename), self._ebGeneral) + d1.addCallbacks(self._cbStopClient, self._ebGeneral) + d2 = self.loopback() + d = defer.gatherResults([d1, d2]) + d.addCallback(lambda _: acc.account.list_all_mailbox_names()) + return d.addCallback(self._cbTestHierarchicalRename) + + def _cbTestHierarchicalRename(self, mailboxes): + expected = ['INBOX', 'newname/m1', 'newname/m2'] + self.assertEqual(sorted(mailboxes), sorted([s for s in expected])) + + def testSubscribe(self): + """ + Test whether we can mark a mailbox as subscribed to + """ + acc = self.server.theAccount + + def add_mailbox(): + return acc.addMailbox('this/mbox') + + def login(): + return self.client.login(TEST_USER, TEST_PASSWD) + + def subscribe(): + return self.client.subscribe('this/mbox') + + def get_subscriptions(ignored): + return self.server.theAccount.getSubscriptions() + + d1 = self.connected.addCallback(strip(add_mailbox)) + d1.addCallback(strip(login)) + d1.addCallbacks(strip(subscribe), self._ebGeneral) + d1.addCallbacks(self._cbStopClient, self._ebGeneral) + d2 = self.loopback() + d = defer.gatherResults([d1, d2]) + d.addCallback(get_subscriptions) + d.addCallback(lambda subscriptions: + self.assertEqual(subscriptions, + ['this/mbox'])) + return d + + def testUnsubscribe(self): + """ + Test whether we can unsubscribe from a set of mailboxes + """ + acc = self.server.theAccount + + def add_mailboxes(): + return defer.gatherResults([ + acc.addMailbox('this/mbox'), + acc.addMailbox('that/mbox')]) + + def dc1(): + return acc.subscribe('this/mbox') + + def dc2(): + return acc.subscribe('that/mbox') + + def login(): + return self.client.login(TEST_USER, TEST_PASSWD) + + def unsubscribe(): + return self.client.unsubscribe('this/mbox') + + def get_subscriptions(ignored): + return acc.getSubscriptions() + + d1 = self.connected.addCallback(strip(add_mailboxes)) + d1.addCallback(strip(login)) + d1.addCallback(strip(dc1)) + d1.addCallback(strip(dc2)) + d1.addCallbacks(strip(unsubscribe), self._ebGeneral) + d1.addCallbacks(self._cbStopClient, self._ebGeneral) + d2 = self.loopback() + d = defer.gatherResults([d1, d2]) + d.addCallback(get_subscriptions) + d.addCallback(lambda subscriptions: + self.assertEqual(subscriptions, + ['that/mbox'])) + return d + + def testSelect(self): + """ + Try to select a mailbox + """ + mbox_name = "TESTMAILBOXSELECT" + self.selectedArgs = None + + acc = self.server.theAccount + + def add_mailbox(): + return acc.addMailbox(mbox_name, creation_ts=42) + + def login(): + return self.client.login(TEST_USER, TEST_PASSWD) + + def select(): + def selected(args): + self.selectedArgs = args + self._cbStopClient(None) + d = self.client.select(mbox_name) + d.addCallback(selected) + return d + + d1 = self.connected.addCallback(strip(add_mailbox)) + d1.addCallback(strip(login)) + d1.addCallback(strip(select)) + # d1.addErrback(self._ebGeneral) + + d2 = self.loopback() + + d = defer.gatherResults([d1, d2]) + d.addCallback(self._cbTestSelect) + return d + + def _cbTestSelect(self, ignored): + self.assertTrue(self.selectedArgs is not None) + + self.assertEqual(self.selectedArgs, { + 'EXISTS': 0, 'RECENT': 0, 'UIDVALIDITY': 42, + 'FLAGS': ('\\Seen', '\\Answered', '\\Flagged', + '\\Deleted', '\\Draft', '\\Recent', 'List'), + 'READ-WRITE': True + }) + + # + # capabilities + # + + def testCapability(self): + caps = {} + + def getCaps(): + def gotCaps(c): + caps.update(c) + self.server.transport.loseConnection() + return self.client.getCapabilities().addCallback(gotCaps) + + d1 = self.connected + d1.addCallback( + strip(getCaps)).addErrback(self._ebGeneral) + + d = defer.gatherResults([self.loopback(), d1]) + expected = {'IMAP4rev1': None, 'NAMESPACE': None, 'LITERAL+': None, + 'IDLE': None} + d.addCallback(lambda _: self.assertEqual(expected, caps)) + return d + + def testCapabilityWithAuth(self): + caps = {} + self.server.challengers[ + 'CRAM-MD5'] = cred.credentials.CramMD5Credentials + + def getCaps(): + def gotCaps(c): + caps.update(c) + self.server.transport.loseConnection() + return self.client.getCapabilities().addCallback(gotCaps) + d1 = self.connected.addCallback( + strip(getCaps)).addErrback(self._ebGeneral) + + d = defer.gatherResults([self.loopback(), d1]) + + expCap = {'IMAP4rev1': None, 'NAMESPACE': None, + 'IDLE': None, 'LITERAL+': None, + 'AUTH': ['CRAM-MD5']} + + d.addCallback(lambda _: self.assertEqual(expCap, caps)) + return d + + # + # authentication + # + + def testLogout(self): + """ + Test log out + """ + self.loggedOut = 0 + + def logout(): + def setLoggedOut(): + self.loggedOut = 1 + self.client.logout().addCallback(strip(setLoggedOut)) + self.connected.addCallback(strip(logout)).addErrback(self._ebGeneral) + d = self.loopback() + return d.addCallback(lambda _: self.assertEqual(self.loggedOut, 1)) + + def testNoop(self): + """ + Test noop command + """ + self.responses = None + + def noop(): + def setResponses(responses): + self.responses = responses + self.server.transport.loseConnection() + self.client.noop().addCallback(setResponses) + self.connected.addCallback(strip(noop)).addErrback(self._ebGeneral) + d = self.loopback() + return d.addCallback(lambda _: self.assertEqual(self.responses, [])) + + def testLogin(self): + """ + Test login + """ + def login(): + d = self.client.login(TEST_USER, TEST_PASSWD) + d.addCallback(self._cbStopClient) + d1 = self.connected.addCallback( + strip(login)).addErrback(self._ebGeneral) + d = defer.gatherResults([d1, self.loopback()]) + return d.addCallback(self._cbTestLogin) + + def _cbTestLogin(self, ignored): + self.assertEqual(self.server.state, 'auth') + + def testFailedLogin(self): + """ + Test bad login + """ + def login(): + d = self.client.login("bad_user@leap.se", TEST_PASSWD) + d.addBoth(self._cbStopClient) + + d1 = self.connected.addCallback( + strip(login)).addErrback(self._ebGeneral) + d2 = self.loopback() + d = defer.gatherResults([d1, d2]) + return d.addCallback(self._cbTestFailedLogin) + + def _cbTestFailedLogin(self, ignored): + self.assertEqual(self.server.state, 'unauth') + self.assertEqual(self.server.account, None) + + def testLoginRequiringQuoting(self): + """ + Test login requiring quoting + """ + self.server.checker.userid = '{test}user@leap.se' + self.server.checker.password = '{test}password' + + def login(): + d = self.client.login('{test}user@leap.se', '{test}password') + d.addBoth(self._cbStopClient) + + d1 = self.connected.addCallback( + strip(login)).addErrback(self._ebGeneral) + d = defer.gatherResults([self.loopback(), d1]) + return d.addCallback(self._cbTestLoginRequiringQuoting) + + def _cbTestLoginRequiringQuoting(self, ignored): + self.assertEqual(self.server.state, 'auth') + + # + # Inspection + # + + def testNamespace(self): + """ + Test retrieving namespace + """ + self.namespaceArgs = None + + def login(): + return self.client.login(TEST_USER, TEST_PASSWD) + + def namespace(): + def gotNamespace(args): + self.namespaceArgs = args + self._cbStopClient(None) + return self.client.namespace().addCallback(gotNamespace) + + d1 = self.connected.addCallback(strip(login)) + d1.addCallback(strip(namespace)) + d1.addErrback(self._ebGeneral) + d2 = self.loopback() + d = defer.gatherResults([d1, d2]) + d.addCallback(lambda _: self.assertEqual(self.namespaceArgs, + [[['', '/']], [], []])) + return d + + def testExamine(self): + """ + L{IMAP4Client.examine} issues an I{EXAMINE} command to the server and + returns a L{Deferred} which fires with a C{dict} with as many of the + following keys as the server includes in its response: C{'FLAGS'}, + C{'EXISTS'}, C{'RECENT'}, C{'UNSEEN'}, C{'READ-WRITE'}, C{'READ-ONLY'}, + C{'UIDVALIDITY'}, and C{'PERMANENTFLAGS'}. + + Unfortunately the server doesn't generate all of these so it's hard to + test the client's handling of them here. See + L{IMAP4ClientExamineTests} below. + + See U{RFC 3501}, section 6.3.2, + for details. + """ + # TODO implement the IMAP4ClientExamineTests testcase. + mbox_name = "test_mailbox_e" + acc = self.server.theAccount + self.examinedArgs = None + + def add_mailbox(): + return acc.addMailbox(mbox_name, creation_ts=42) + + def login(): + return self.client.login(TEST_USER, TEST_PASSWD) + + def examine(): + def examined(args): + self.examinedArgs = args + self._cbStopClient(None) + d = self.client.examine(mbox_name) + d.addCallback(examined) + return d + + d1 = self.connected.addCallback(strip(add_mailbox)) + d1.addCallback(strip(login)) + d1.addCallback(strip(examine)) + d1.addErrback(self._ebGeneral) + d2 = self.loopback() + d = defer.gatherResults([d1, d2]) + return d.addCallback(self._cbTestExamine) + + def _cbTestExamine(self, ignored): + self.assertEqual(self.examinedArgs, { + 'EXISTS': 0, 'RECENT': 0, 'UIDVALIDITY': 42, + 'FLAGS': ('\\Seen', '\\Answered', '\\Flagged', + '\\Deleted', '\\Draft', '\\Recent', 'List'), + 'READ-WRITE': False}) + + def _listSetup(self, f, f2=None): + + acc = self.server.theAccount + + def dc1(): + return acc.addMailbox('root_subthing', creation_ts=42) + + def dc2(): + return acc.addMailbox('root_another_thing', creation_ts=42) + + def dc3(): + return acc.addMailbox('non_root_subthing', creation_ts=42) + + def login(): + return self.client.login(TEST_USER, TEST_PASSWD) + + def listed(answers): + self.listed = answers + + self.listed = None + d1 = self.connected.addCallback(strip(login)) + d1.addCallback(strip(dc1)) + d1.addCallback(strip(dc2)) + d1.addCallback(strip(dc3)) + + if f2 is not None: + d1.addCallback(f2) + + d1.addCallbacks(strip(f), self._ebGeneral) + d1.addCallbacks(listed, self._ebGeneral) + d1.addCallbacks(self._cbStopClient, self._ebGeneral) + d2 = self.loopback() + return defer.gatherResults([d1, d2]).addCallback(lambda _: self.listed) + + def testList(self): + """ + Test List command + """ + def list(): + return self.client.list('root', '%') + + d = self._listSetup(list) + d.addCallback(lambda listed: self.assertEqual( + sortNest(listed), + sortNest([ + (IMAPMailbox.init_flags, "/", "root_subthing"), + (IMAPMailbox.init_flags, "/", "root_another_thing") + ]) + )) + return d + + def testLSub(self): + """ + Test LSub command + """ + acc = self.server.theAccount + + def subs_mailbox(): + # why not client.subscribe instead? + return acc.subscribe('root_subthing') + + def lsub(): + return self.client.lsub('root', '%') + + d = self._listSetup(lsub, strip(subs_mailbox)) + d.addCallback(self.assertEqual, + [(IMAPMailbox.init_flags, "/", "root_subthing")]) + return d + + def testStatus(self): + """ + Test Status command + """ + acc = self.server.theAccount + + def add_mailbox(): + return acc.addMailbox('root_subthings') + + # XXX FIXME ---- should populate this a little bit, + # with unseen etc... + + def login(): + return self.client.login(TEST_USER, TEST_PASSWD) + + def status(): + return self.client.status( + 'root_subthings', 'MESSAGES', 'UIDNEXT', 'UNSEEN') + + def statused(result): + self.statused = result + + self.statused = None + + d1 = self.connected.addCallback(strip(add_mailbox)) + d1.addCallback(strip(login)) + d1.addCallbacks(strip(status), self._ebGeneral) + d1.addCallbacks(statused, self._ebGeneral) + d1.addCallbacks(self._cbStopClient, self._ebGeneral) + d2 = self.loopback() + d = defer.gatherResults([d1, d2]) + d.addCallback(lambda _: self.assertEqual( + self.statused, + {'MESSAGES': 0, 'UIDNEXT': '1', 'UNSEEN': 0} + )) + return d + + def testFailedStatus(self): + """ + Test failed status command with a non-existent mailbox + """ + def login(): + return self.client.login(TEST_USER, TEST_PASSWD) + + def status(): + return self.client.status( + 'root/nonexistent', 'MESSAGES', 'UIDNEXT', 'UNSEEN') + + def statused(result): + self.statused = result + + def failed(failure): + self.failure = failure + + self.statused = self.failure = None + d1 = self.connected.addCallback(strip(login)) + d1.addCallbacks(strip(status), self._ebGeneral) + d1.addCallbacks(statused, failed) + d1.addCallbacks(self._cbStopClient, self._ebGeneral) + d2 = self.loopback() + return defer.gatherResults([d1, d2]).addCallback( + self._cbTestFailedStatus) + + def _cbTestFailedStatus(self, ignored): + self.assertEqual( + self.statused, None + ) + self.assertEqual( + self.failure.value.args, + ('Could not open mailbox',) + ) + + # + # messages + # + + def testFullAppend(self): + """ + Test appending a full message to the mailbox + """ + infile = util.sibpath(__file__, 'rfc822.message') + message = open(infile) + acc = self.server.theAccount + mailbox_name = "appendmbox/subthing" + + def add_mailbox(): + return acc.addMailbox(mailbox_name) + + def login(): + return self.client.login(TEST_USER, TEST_PASSWD) + + def append(): + return self.client.append( + mailbox_name, message, + ('\\SEEN', '\\DELETED'), + 'Tue, 17 Jun 2003 11:22:16 -0600 (MDT)', + ) + + d1 = self.connected.addCallback(strip(add_mailbox)) + d1.addCallback(strip(login)) + d1.addCallbacks(strip(append), self._ebGeneral) + d1.addCallbacks(self._cbStopClient, self._ebGeneral) + d2 = self.loopback() + d = defer.gatherResults([d1, d2]) + + d.addCallback(lambda _: acc.getMailbox(mailbox_name)) + d.addCallback(lambda mb: mb.fetch(imap4.MessageSet(start=1), True)) + return d.addCallback(self._cbTestFullAppend, infile) + + def _cbTestFullAppend(self, fetched, infile): + fetched = list(fetched) + self.assertTrue(len(fetched) == 1) + self.assertTrue(len(fetched[0]) == 2) + uid, msg = fetched[0] + parsed = self.parser.parse(open(infile)) + expected_body = parsed.get_payload() + expected_headers = CaseInsensitiveDict(parsed.items()) + + def assert_flags(flags): + self.assertEqual( + set(('\\SEEN', '\\DELETED')), + set(flags)) + + def assert_date(date): + self.assertEqual( + 'Tue, 17 Jun 2003 11:22:16 -0600 (MDT)', + date) + + def assert_body(body): + gotbody = body.read() + self.assertEqual(expected_body, gotbody) + + def assert_headers(headers): + self.assertItemsEqual(map(string.lower, expected_headers), headers) + + d = defer.maybeDeferred(msg.getFlags) + d.addCallback(assert_flags) + + d.addCallback(lambda _: defer.maybeDeferred(msg.getInternalDate)) + d.addCallback(assert_date) + + d.addCallback( + lambda _: defer.maybeDeferred( + msg.getBodyFile, self._soledad)) + d.addCallback(assert_body) + + d.addCallback(lambda _: defer.maybeDeferred(msg.getHeaders, True)) + d.addCallback(assert_headers) + + return d + + def testPartialAppend(self): + """ + Test partially appending a message to the mailbox + """ + # TODO this test sometimes will fail because of the notify_just_mdoc + infile = util.sibpath(__file__, 'rfc822.message') + + acc = self.server.theAccount + + def add_mailbox(): + return acc.addMailbox('PARTIAL/SUBTHING') + + def login(): + return self.client.login(TEST_USER, TEST_PASSWD) + + def append(): + message = file(infile) + return self.client.sendCommand( + imap4.Command( + 'APPEND', + 'PARTIAL/SUBTHING (\\SEEN) "Right now" ' + '{%d}' % os.path.getsize(infile), + (), self.client._IMAP4Client__cbContinueAppend, message + ) + ) + d1 = self.connected.addCallback(strip(add_mailbox)) + d1.addCallback(strip(login)) + d1.addCallbacks(strip(append), self._ebGeneral) + d1.addCallbacks(self._cbStopClient, self._ebGeneral) + d2 = self.loopback() + d = defer.gatherResults([d1, d2]) + + d.addCallback(lambda _: acc.getMailbox("PARTIAL/SUBTHING")) + d.addCallback(lambda mb: mb.fetch(imap4.MessageSet(start=1), True)) + return d.addCallback( + self._cbTestPartialAppend, infile) + + def _cbTestPartialAppend(self, fetched, infile): + fetched = list(fetched) + self.assertTrue(len(fetched) == 1) + self.assertTrue(len(fetched[0]) == 2) + uid, msg = fetched[0] + parsed = self.parser.parse(open(infile)) + expected_body = parsed.get_payload() + + def assert_flags(flags): + self.assertEqual( + set((['\\SEEN'])), set(flags)) + + def assert_body(body): + gotbody = body.read() + self.assertEqual(expected_body, gotbody) + + d = defer.maybeDeferred(msg.getFlags) + d.addCallback(assert_flags) + + d.addCallback(lambda _: defer.maybeDeferred(msg.getBodyFile)) + d.addCallback(assert_body) + return d + + def testCheck(self): + """ + Test check command + """ + def add_mailbox(): + return self.server.theAccount.addMailbox('root/subthing') + + def login(): + return self.client.login(TEST_USER, TEST_PASSWD) + + def select(): + return self.client.select('root/subthing') + + def check(): + return self.client.check() + + d = self.connected.addCallbacks( + strip(add_mailbox), self._ebGeneral) + d.addCallbacks(lambda _: login(), self._ebGeneral) + d.addCallbacks(strip(select), self._ebGeneral) + d.addCallbacks(strip(check), self._ebGeneral) + d.addCallbacks(self._cbStopClient, self._ebGeneral) + d2 = self.loopback() + return defer.gatherResults([d, d2]) + + # Okay, that was much fun indeed + + def testExpunge(self): + """ + Test expunge command + """ + acc = self.server.theAccount + mailbox_name = 'mailboxexpunge' + + def add_mailbox(): + return acc.addMailbox(mailbox_name) + + def login(): + return self.client.login(TEST_USER, TEST_PASSWD) + + def select(): + return self.client.select(mailbox_name) + + def save_mailbox(mailbox): + self.mailbox = mailbox + + def get_mailbox(): + d = acc.getMailbox(mailbox_name) + d.addCallback(save_mailbox) + return d + + def add_messages(): + d = self.mailbox.addMessage( + 'test 1', flags=('\\Deleted', 'AnotherFlag'), + notify_just_mdoc=False) + d.addCallback(lambda _: self.mailbox.addMessage( + 'test 2', flags=('AnotherFlag',), + notify_just_mdoc=False)) + d.addCallback(lambda _: self.mailbox.addMessage( + 'test 3', flags=('\\Deleted',), + notify_just_mdoc=False)) + return d + + def expunge(): + return self.client.expunge() + + def expunged(results): + self.failIf(self.server.mbox is None) + self.results = results + + self.results = None + d1 = self.connected.addCallback(strip(add_mailbox)) + d1.addCallback(strip(login)) + d1.addCallback(strip(get_mailbox)) + d1.addCallbacks(strip(add_messages), self._ebGeneral) + d1.addCallbacks(strip(select), self._ebGeneral) + d1.addCallbacks(strip(expunge), self._ebGeneral) + d1.addCallbacks(expunged, self._ebGeneral) + d1.addCallbacks(self._cbStopClient, self._ebGeneral) + d2 = self.loopback() + d = defer.gatherResults([d1, d2]) + d.addCallback(lambda _: self.mailbox.getMessageCount()) + return d.addCallback(self._cbTestExpunge) + + def _cbTestExpunge(self, count): + # we only left 1 mssage with no deleted flag + self.assertEqual(count, 1) + # the uids of the deleted messages + self.assertItemsEqual(self.results, [1, 3]) + + +class AccountTestCase(IMAP4HelperMixin): + """ + Test the Account. + """ + def _create_empty_mailbox(self): + return self.server.theAccount.addMailbox('') + + def _create_one_mailbox(self): + return self.server.theAccount.addMailbox('one') + + def test_illegalMailboxCreate(self): + self.assertRaises(AssertionError, self._create_empty_mailbox) + + +class IMAP4ServerSearchTestCase(IMAP4HelperMixin): + """ + Tests for the behavior of the search_* functions in L{imap5.IMAP4Server}. + """ + # XXX coming soon to your screens! + pass diff --git a/src/leap/bitmask/mail/imap/tests/walktree.py b/src/leap/bitmask/mail/imap/tests/walktree.py new file mode 100644 index 0000000..f259a55 --- /dev/null +++ b/src/leap/bitmask/mail/imap/tests/walktree.py @@ -0,0 +1,127 @@ +# -*- coding: utf-8 -*- +# walktree.py +# Copyright (C) 2013 LEAP +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . +""" +Tests for the walktree module. +""" +import os +import sys +import pprint +from email import parser + +from leap.mail import walk as W + +DEBUG = os.environ.get("BITMASK_MAIL_DEBUG") + + +p = parser.Parser() + +# TODO pass an argument of the type of message + +################################################## +# Input from hell + +if len(sys.argv) > 1: + FILENAME = sys.argv[1] +else: + FILENAME = "rfc822.multi-signed.message" + +""" +FILENAME = "rfc822.plain.message" +FILENAME = "rfc822.multi-minimal.message" +""" + +msg = p.parse(open(FILENAME)) +DO_CHECK = False +################################################# + +parts = W.get_parts(msg) + +if DEBUG: + def trim(item): + item = item[:10] + [trim(part["phash"]) for part in parts if part.get('phash', None)] + +raw_docs = list(W.get_raw_docs(msg, parts)) + +body_phash_fun = [W.get_body_phash_simple, + W.get_body_phash_multi][int(msg.is_multipart())] +body_phash = body_phash_fun(W.get_payloads(msg)) +parts_map = W.walk_msg_tree(parts, body_phash=body_phash) + + +# TODO add missing headers! +expected = { + 'body': '1ddfa80485', + 'multi': True, + 'part_map': { + 1: { + 'headers': {'Content-Disposition': 'inline', + 'Content-Type': 'multipart/mixed; ' + 'boundary="z0eOaCaDLjvTGF2l"'}, + 'multi': True, + 'part_map': {1: {'ctype': 'text/plain', + 'headers': [ + ('Content-Type', + 'text/plain; charset=utf-8'), + ('Content-Disposition', + 'inline'), + ('Content-Transfer-Encoding', + 'quoted-printable')], + 'multi': False, + 'parts': 1, + 'phash': '1ddfa80485', + 'size': 206}, + 2: {'ctype': 'text/plain', + 'headers': [('Content-Type', + 'text/plain; charset=us-ascii'), + ('Content-Disposition', + 'attachment; ' + 'filename="attach.txt"')], + 'multi': False, + 'parts': 1, + 'phash': '7a94e4d769', + 'size': 133}, + 3: {'ctype': 'application/octet-stream', + 'headers': [('Content-Type', + 'application/octet-stream'), + ('Content-Disposition', + 'attachment; filename="hack.ico"'), + ('Content-Transfer-Encoding', + 'base64')], + 'multi': False, + 'parts': 1, + 'phash': 'c42cccebbd', + 'size': 12736}}}, + 2: {'ctype': 'application/pgp-signature', + 'headers': [('Content-Type', 'application/pgp-signature')], + 'multi': False, + 'parts': 1, + 'phash': '8f49fbf749', + 'size': 877}}} + +if DEBUG and DO_CHECK: + # TODO turn this into a proper unittest + assert(parts_map == expected) + print "Structure: OK" + + +print +print "RAW DOCS" +pprint.pprint(raw_docs) +print +print "PARTS MAP" +pprint.pprint(parts_map) diff --git a/src/leap/bitmask/mail/incoming/__init__.py b/src/leap/bitmask/mail/incoming/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/src/leap/bitmask/mail/incoming/service.py b/src/leap/bitmask/mail/incoming/service.py new file mode 100644 index 0000000..1e20862 --- /dev/null +++ b/src/leap/bitmask/mail/incoming/service.py @@ -0,0 +1,844 @@ +# -*- coding: utf-8 -*- +# service.py +# Copyright (C) 2015 LEAP +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . +""" +Incoming mail fetcher. +""" +import copy +import logging +import shlex +import time +import warnings + +from email.parser import Parser +from email.utils import parseaddr +from email.utils import formatdate +from StringIO import StringIO +from urlparse import urlparse + +from twisted.application.service import Service +from twisted.python import log +from twisted.python.failure import Failure +from twisted.internet import defer, reactor +from twisted.internet.task import LoopingCall +from twisted.internet.task import deferLater + +from leap.common.events import emit_async, catalog +from leap.common.check import leap_assert, leap_assert_type +from leap.common.mail import get_email_charset +from leap.keymanager import errors as keymanager_errors +from leap.mail.adaptors import soledad_indexes as fields +from leap.mail.generator import Generator +from leap.mail.utils import json_loads, empty +from leap.soledad.client import Soledad +from leap.soledad.common.crypto import ENC_SCHEME_KEY, ENC_JSON_KEY +from leap.soledad.common.errors import InvalidAuthTokenError + + +logger = logging.getLogger(__name__) + +MULTIPART_ENCRYPTED = "multipart/encrypted" +MULTIPART_SIGNED = "multipart/signed" +PGP_BEGIN = "-----BEGIN PGP MESSAGE-----" +PGP_END = "-----END PGP MESSAGE-----" + +# The period between succesive checks of the incoming mail +# queue (in seconds) +INCOMING_CHECK_PERIOD = 60 + + +class MalformedMessage(Exception): + """ + Raised when a given message is not well formed. + """ + pass + + +class IncomingMail(Service): + """ + Fetches and process mail from the incoming pool. + + This object implements IService interface, has public methods + startService and stopService that will actually initiate a + LoopingCall with check_period recurrency. + The LoopingCall itself will invoke the fetch method each time + that the check_period expires. + + This loop will sync the soledad db with the remote server and + process all the documents found tagged as incoming mail. + """ + # TODO implements IService? + + name = "IncomingMail" + + RECENT_FLAG = "\\Recent" + CONTENT_KEY = "content" + + LEAP_SIGNATURE_HEADER = 'X-Leap-Signature' + LEAP_ENCRYPTION_HEADER = 'X-Leap-Encryption' + """ + Header added to messages when they are decrypted by the fetcher, + which states the validity of an eventual signature that might be included + in the encrypted blob. + """ + LEAP_SIGNATURE_VALID = 'valid' + LEAP_SIGNATURE_INVALID = 'invalid' + LEAP_SIGNATURE_COULD_NOT_VERIFY = 'could not verify' + + LEAP_ENCRYPTION_DECRYPTED = 'decrypted' + + def __init__(self, keymanager, soledad, inbox, userid, + check_period=INCOMING_CHECK_PERIOD): + + """ + Initialize IncomingMail.. + + :param keymanager: a keymanager instance + :type keymanager: keymanager.KeyManager + + :param soledad: a soledad instance + :type soledad: Soledad + + :param inbox: the collection for the inbox where the new emails will be + stored + :type inbox: MessageCollection + + :param check_period: the period to fetch new mail, in seconds. + :type check_period: int + """ + leap_assert(keymanager, "need a keymanager to initialize") + leap_assert_type(soledad, Soledad) + leap_assert(check_period, "need a period to check incoming mail") + leap_assert_type(check_period, int) + leap_assert(userid, "need a userid to initialize") + + self._keymanager = keymanager + self._soledad = soledad + self._inbox_collection = inbox + self._userid = userid + + self._listeners = [] + self._loop = None + self._check_period = check_period + + # initialize a mail parser only once + self._parser = Parser() + + def add_listener(self, listener): + """ + Add a listener to inbox insertions. + + This listener function will be called for each message added to the + inbox with its uid as parameter. This function should not be blocking + or it will block the incoming queue. + + :param listener: the listener function + :type listener: callable + """ + self._listeners.append(listener) + + # + # Public API: fetch, start_loop, stop. + # + + def fetch(self): + """ + Fetch incoming mail, to be called periodically. + + Calls a deferred that will execute the fetch callback. + """ + def _sync_errback(failure): + log.err(failure) + + def syncSoledadCallback(_): + # XXX this should be moved to adaptors + d = self._soledad.get_from_index( + fields.JUST_MAIL_IDX, "1", "0") + d.addCallback(self._process_doclist) + d.addErrback(_sync_errback) + return d + + logger.debug("fetching mail for: %s %s" % ( + self._soledad.uuid, self._userid)) + d = self._sync_soledad() + d.addCallbacks(syncSoledadCallback, self._errback) + d.addCallbacks(self._signal_fetch_to_ui, self._errback) + return d + + def startService(self): + """ + Starts a loop to fetch mail. + + :returns: A Deferred whose callback will be invoked with + the LoopingCall instance when loop.stop is called, or + whose errback will be invoked when the function raises an + exception or returned a deferred that has its errback + invoked. + """ + Service.startService(self) + if self._loop is None: + self._loop = LoopingCall(self.fetch) + stop_deferred = self._loop.start(self._check_period) + return stop_deferred + else: + logger.warning("Tried to start an already running fetching loop.") + return defer.fail(Failure('Already running loop.')) + + def stopService(self): + """ + Stops the loop that fetches mail. + """ + if self._loop and self._loop.running is True: + self._loop.stop() + self._loop = None + Service.stopService(self) + + # + # Private methods. + # + + # synchronize incoming mail + + def _errback(self, failure): + log.err(failure) + + def _sync_soledad(self): + """ + Synchronize with remote soledad. + + :returns: a list of LeapDocuments, or None. + :rtype: iterable or None + """ + def _log_synced(result): + log.msg('FETCH soledad SYNCED.') + return result + + def _signal_invalid_auth(failure): + failure.trap(InvalidAuthTokenError) + # if the token is invalid, send an event so the GUI can + # disable mail and show an error message. + emit_async(catalog.SOLEDAD_INVALID_AUTH_TOKEN, self._userid) + + log.msg('FETCH: syncing soledad...') + d = self._soledad.sync() + d.addCallbacks(_log_synced, _signal_invalid_auth) + return d + + def _signal_fetch_to_ui(self, doclist): + """ + Send leap events to ui. + + :param doclist: iterable with msg documents. + :type doclist: iterable. + :returns: doclist + :rtype: iterable + """ + if doclist: + fetched_ts = time.mktime(time.gmtime()) + num_mails = len(doclist) if doclist is not None else 0 + if num_mails != 0: + log.msg("there are %s mails" % (num_mails,)) + emit_async(catalog.MAIL_FETCHED_INCOMING, self._userid, + str(num_mails), str(fetched_ts)) + return doclist + + def _signal_unread_to_ui(self, *args): + """ + Sends unread event to ui. + """ + emit_async(catalog.MAIL_UNREAD_MESSAGES, self._userid, + str(self._inbox_collection.count_unseen())) + + # process incoming mail. + + def _process_doclist(self, doclist): + """ + Iterates through the doclist, checks if each doc + looks like a message, and yields a deferred that will decrypt and + process the message. + + :param doclist: iterable with msg documents. + :type doclist: iterable. + :returns: a list of deferreds for individual messages. + """ + log.msg('processing doclist') + if not doclist: + logger.debug("no docs found") + return + num_mails = len(doclist) + + deferreds = [] + for index, doc in enumerate(doclist): + logger.debug("processing doc %d of %d" % (index + 1, num_mails)) + emit_async(catalog.MAIL_MSG_PROCESSING, self._userid, + str(index), str(num_mails)) + + keys = doc.content.keys() + + # TODO Compatibility check with the index in pre-0.6 mx + # that does not write the ERROR_DECRYPTING_KEY + # This should be removed in 0.7 + + has_errors = doc.content.get(fields.ERROR_DECRYPTING_KEY, None) + + if has_errors is None: + warnings.warn("JUST_MAIL_COMPAT_IDX will be deprecated!", + DeprecationWarning) + + if has_errors: + logger.debug("skipping msg with decrypting errors...") + elif self._is_msg(keys): + # TODO this pipeline is a bit obscure! + d = self._decrypt_doc(doc) + d.addCallback(self._maybe_extract_keys) + d.addCallbacks(self._add_message_locally, self._errback) + deferreds.append(d) + + d = defer.gatherResults(deferreds, consumeErrors=True) + d.addCallback(lambda _: doclist) + return d + + # + # operations on individual messages + # + + def _decrypt_doc(self, doc): + """ + Decrypt the contents of a document. + + :param doc: A document containing an encrypted message. + :type doc: SoledadDocument + + :return: A Deferred that will be fired with the document and the + decrypted message. + :rtype: SoledadDocument, str + """ + log.msg('decrypting msg') + + def process_decrypted(res): + if isinstance(res, tuple): + decrdata, _ = res + success = True + else: + decrdata = "" + success = False + + emit_async(catalog.MAIL_MSG_DECRYPTED, self._userid, + "1" if success else "0") + return self._process_decrypted_doc(doc, decrdata) + + d = self._keymanager.decrypt(doc.content[ENC_JSON_KEY], self._userid) + d.addErrback(self._errback) + d.addCallback(process_decrypted) + d.addCallback(lambda data: (doc, data)) + return d + + def _process_decrypted_doc(self, doc, data): + """ + Process a document containing a succesfully decrypted message. + + :param doc: the incoming message + :type doc: SoledadDocument + :param data: the json-encoded, decrypted content of the incoming + message + :type data: str + + :return: a Deferred that will be fired with an str of the proccessed + data. + :rtype: Deferred + """ + log.msg('processing decrypted doc') + + # XXX turn this into an errBack for each one of + # the deferreds that would process an individual document + try: + msg = json_loads(data) + except UnicodeError as exc: + logger.error("Error while decrypting %s" % (doc.doc_id,)) + logger.exception(exc) + + # we flag the message as "with decrypting errors", + # to avoid further decryption attempts during sync + # cycles until we're prepared to deal with that. + # What is the same, when Ivan deals with it... + # A new decrypting attempt event could be triggered by a + # future a library upgrade, or a cli flag to the client, + # we just `defer` that for now... :) + doc.content[fields.ERROR_DECRYPTING_KEY] = True + deferLater(reactor, 0, self._update_incoming_message, doc) + + # FIXME this is just a dirty hack to delay the proper + # deferred organization here... + # and remember, boys, do not do this at home. + return [] + + if not isinstance(msg, dict): + defer.returnValue(False) + if not msg.get(fields.INCOMING_KEY, False): + defer.returnValue(False) + + # ok, this is an incoming message + rawmsg = msg.get(self.CONTENT_KEY, None) + if rawmsg is None: + return "" + return self._maybe_decrypt_msg(rawmsg) + + def _update_incoming_message(self, doc): + """ + Do a put for a soledad document. This probably has been called only + in the case that we've needed to update the ERROR_DECRYPTING_KEY + flag in an incoming message, to get it out of the decrypting queue. + + :param doc: the SoledadDocument to update + :type doc: SoledadDocument + """ + log.msg("Updating Incoming MSG: SoledadDoc %s" % (doc.doc_id)) + return self._soledad.put_doc(doc) + + def _delete_incoming_message(self, doc): + """ + Delete document. + + :param doc: the SoledadDocument to delete + :type doc: SoledadDocument + """ + log.msg("Deleting Incoming message: %s" % (doc.doc_id,)) + return self._soledad.delete_doc(doc) + + def _maybe_decrypt_msg(self, data): + """ + Tries to decrypt a gpg message if data looks like one. + + :param data: the text to be decrypted. + :type data: str + + :return: a Deferred that will be fired with an str of data, possibly + decrypted. + :rtype: Deferred + """ + leap_assert_type(data, str) + log.msg('maybe decrypting doc') + + # parse the original message + encoding = get_email_charset(data) + msg = self._parser.parsestr(data) + + fromHeader = msg.get('from', None) + senderAddress = None + + if (fromHeader is not None and + (msg.get_content_type() == MULTIPART_ENCRYPTED or + msg.get_content_type() == MULTIPART_SIGNED)): + senderAddress = parseaddr(fromHeader)[1] + + def add_leap_header(ret): + decrmsg, signkey = ret + if (senderAddress is None or signkey is None or + isinstance(signkey, keymanager_errors.KeyNotFound)): + decrmsg.add_header( + self.LEAP_SIGNATURE_HEADER, + self.LEAP_SIGNATURE_COULD_NOT_VERIFY) + elif isinstance(signkey, keymanager_errors.InvalidSignature): + decrmsg.add_header( + self.LEAP_SIGNATURE_HEADER, + self.LEAP_SIGNATURE_INVALID) + else: + self._add_verified_signature_header(decrmsg, + signkey.fingerprint) + return decrmsg.as_string() + + if msg.get_content_type() == MULTIPART_ENCRYPTED: + d = self._decrypt_multipart_encrypted_msg( + msg, encoding, senderAddress) + elif msg.get_content_type() == MULTIPART_SIGNED: + d = self._verify_signature_not_encrypted_msg(msg, senderAddress) + else: + d = self._maybe_decrypt_inline_encrypted_msg( + msg, encoding, senderAddress) + d.addCallback(add_leap_header) + return d + + def _add_verified_signature_header(self, decrmsg, fingerprint): + decrmsg.add_header( + self.LEAP_SIGNATURE_HEADER, + self.LEAP_SIGNATURE_VALID, + pubkey=fingerprint) + + def _add_decrypted_header(self, msg): + msg.add_header(self.LEAP_ENCRYPTION_HEADER, + self.LEAP_ENCRYPTION_DECRYPTED) + + def _decrypt_multipart_encrypted_msg(self, msg, encoding, senderAddress): + """ + Decrypt a message with content-type 'multipart/encrypted'. + + :param msg: The original encrypted message. + :type msg: Message + :param encoding: The encoding of the email message. + :type encoding: str + :param senderAddress: The email address of the sender of the message. + :type senderAddress: str + + :return: A Deferred that will be fired with a tuple containing a + decrypted Message and the signing OpenPGPKey if the signature + is valid or InvalidSignature or KeyNotFound. + :rtype: Deferred + """ + log.msg('decrypting multipart encrypted msg') + msg = copy.deepcopy(msg) + self._msg_multipart_sanity_check(msg) + + # parse message and get encrypted content + pgpencmsg = msg.get_payload()[1] + encdata = pgpencmsg.get_payload() + + # decrypt or fail gracefully + def build_msg(res): + decrdata, signkey = res + + decrmsg = self._parser.parsestr(decrdata) + # remove original message's multipart/encrypted content-type + del(msg['content-type']) + + # replace headers back in original message + for hkey, hval in decrmsg.items(): + try: + # this will raise KeyError if header is not present + msg.replace_header(hkey, hval) + except KeyError: + msg[hkey] = hval + + # all ok, replace payload by unencrypted payload + msg.set_payload(decrmsg.get_payload()) + self._add_decrypted_header(msg) + return (msg, signkey) + + def verify_signature_after_decrypt_an_email(res): + decrdata, signkey = res + if decrdata.get_content_type() == MULTIPART_SIGNED: + res = self._verify_signature_not_encrypted_msg(decrdata, + senderAddress) + return res + + d = self._keymanager.decrypt( + encdata, self._userid, verify=senderAddress) + d.addCallbacks(build_msg, self._decryption_error, errbackArgs=(msg,)) + d.addCallbacks(verify_signature_after_decrypt_an_email) + return d + + def _maybe_decrypt_inline_encrypted_msg(self, origmsg, encoding, + senderAddress): + """ + Possibly decrypt an inline OpenPGP encrypted message. + + :param origmsg: The original, possibly encrypted message. + :type origmsg: Message + :param encoding: The encoding of the email message. + :type encoding: str + :param senderAddress: The email address of the sender of the message. + :type senderAddress: str + + :return: A Deferred that will be fired with a tuple containing a + decrypted Message and the signing OpenPGPKey if the signature + is valid or InvalidSignature or KeyNotFound. + :rtype: Deferred + """ + log.msg('maybe decrypting inline encrypted msg') + + data = self._serialize_msg(origmsg) + + def decrypted_data(res): + decrdata, signkey = res + replaced_data = data.replace(pgp_message, decrdata) + self._add_decrypted_header(origmsg) + return replaced_data, signkey + + def encode_and_return(res): + data, signkey = res + if isinstance(data, unicode): + data = data.encode(encoding, 'replace') + return (self._parser.parsestr(data), signkey) + + # handle exactly one inline PGP message + if PGP_BEGIN in data: + begin = data.find(PGP_BEGIN) + end = data.find(PGP_END) + pgp_message = data[begin:end + len(PGP_END)] + d = self._keymanager.decrypt( + pgp_message, self._userid, verify=senderAddress) + d.addCallbacks(decrypted_data, self._decryption_error, + errbackArgs=(data,)) + else: + d = defer.succeed((data, None)) + d.addCallback(encode_and_return) + return d + + def _verify_signature_not_encrypted_msg(self, origmsg, sender_address): + """ + Possibly decrypt an inline OpenPGP encrypted message. + + :param origmsg: The original, possibly encrypted message. + :type origmsg: Message + :param sender_address: The email address of the sender of the message. + :type sender_address: str + + :return: A Deferred that will be fired with a tuple containing a + signed Message and the signing OpenPGPKey if the signature + is valid or InvalidSignature. + :rtype: Deferred + """ + msg = copy.deepcopy(origmsg) + data = self._serialize_msg(msg.get_payload(0)) + detached_sig = self._extract_signature(msg) + d = self._keymanager.verify(data, sender_address, detached_sig) + + d.addCallback(lambda sign_key: (msg, sign_key)) + d.addErrback(lambda _: (msg, keymanager_errors.InvalidSignature())) + return d + + def _serialize_msg(self, origmsg): + buf = StringIO() + g = Generator(buf) + g.flatten(origmsg) + return buf.getvalue() + + def _extract_signature(self, msg): + body = msg.get_payload(0).get_payload() + + if isinstance(body, str): + body = msg.get_payload(0) + + detached_sig = msg.get_payload(1).get_payload() + msg.set_payload(body) + return detached_sig + + def _decryption_error(self, failure, msg): + """ + Check for known decryption errors + """ + if failure.check(keymanager_errors.DecryptError): + logger.warning('Failed to decrypt encrypted message (%s). ' + 'Storing message without modifications.' + % str(failure.value)) + return (msg, None) + elif failure.check(keymanager_errors.KeyNotFound): + logger.error('Failed to find private key for decryption (%s). ' + 'Storing message without modifications.' + % str(failure.value)) + return (msg, None) + else: + return failure + + @defer.inlineCallbacks + def _maybe_extract_keys(self, msgtuple): + """ + Retrieve attached keys to the mesage and parse message headers for an + *OpenPGP* header as described on the `IETF draft + ` + only urls with https and the same hostname than the email are supported + for security reasons. + + :param msgtuple: a tuple consisting of a SoledadDocument + instance containing the incoming message + and data, the json-encoded, decrypted content of the + incoming message + :type msgtuple: (SoledadDocument, str) + + :return: A Deferred that will be fired with msgtuple when key + extraction finishes + :rtype: Deferred + """ + OpenPGP_HEADER = 'OpenPGP' + doc, data = msgtuple + + # XXX the parsing of the message is done in mailbox.addMessage, maybe + # we should do it in this module so we don't need to parse it again + # here + msg = self._parser.parsestr(data) + _, fromAddress = parseaddr(msg['from']) + + valid_attachment = False + if msg.is_multipart(): + valid_attachment = yield self._maybe_extract_attached_key( + msg.get_payload(), fromAddress) + + if not valid_attachment: + header = msg.get(OpenPGP_HEADER, None) + if header is not None: + yield self._maybe_extract_openpgp_header(header, fromAddress) + + defer.returnValue(msgtuple) + + def _maybe_extract_openpgp_header(self, header, address): + """ + Import keys from the OpenPGP header + + :param header: OpenPGP header string + :type header: str + :param address: email address in the from header + :type address: str + + :return: A Deferred that will be fired when header extraction is done + :rtype: Deferred + """ + d = defer.succeed(None) + fields = dict([f.strip(' ').split('=') for f in header.split(';')]) + if 'url' in fields: + url = shlex.split(fields['url'])[0] # remove quotations + urlparts = urlparse(url) + addressHostname = address.split('@')[1] + if ( + urlparts.scheme == 'https' and + urlparts.hostname == addressHostname + ): + def fetch_error(failure): + if failure.check(keymanager_errors.KeyNotFound): + logger.warning("Url from OpenPGP header %s failed" + % (url,)) + elif failure.check(keymanager_errors.KeyAttributesDiffer): + logger.warning("Key from OpenPGP header url %s didn't " + "match the from address %s" + % (url, address)) + else: + return failure + + d = self._keymanager.fetch_key(address, url) + d.addCallback( + lambda _: + logger.info("Imported key from header %s" % (url,))) + d.addErrback(fetch_error) + else: + logger.debug("No valid url on OpenPGP header %s" % (url,)) + else: + logger.debug("There is no url on the OpenPGP header: %s" + % (header,)) + return d + + def _maybe_extract_attached_key(self, attachments, address): + """ + Import keys from the attachments + + :param attachments: email attachment list + :type attachments: list(email.Message) + :param address: email address in the from header + :type address: str + + :return: A Deferred that will be fired when all the keys are stored + with a boolean: True if there was a valid key attached, or + False otherwise. + :rtype: Deferred + """ + MIME_KEY = "application/pgp-keys" + + def log_key_added(ignored): + logger.debug('Added key found in attachment for %s' % address) + return True + + def failed_put_key(failure): + logger.info("An error has ocurred adding attached key for %s: %s" + % (address, failure.getErrorMessage())) + return False + + deferreds = [] + for attachment in attachments: + if MIME_KEY == attachment.get_content_type(): + d = self._keymanager.put_raw_key( + attachment.get_payload(decode=True), address=address) + d.addCallbacks(log_key_added, failed_put_key) + deferreds.append(d) + d = defer.gatherResults(deferreds) + d.addCallback(lambda result: any(result)) + return d + + def _add_message_locally(self, msgtuple): + """ + Adds a message to local inbox and delete it from the incoming db + in soledad. + + :param msgtuple: a tuple consisting of a SoledadDocument + instance containing the incoming message + and data, the json-encoded, decrypted content of the + incoming message + :type msgtuple: (SoledadDocument, str) + + :return: A Deferred that will be fired when the messages is stored + :rtype: Defferred + """ + doc, raw_data = msgtuple + insertion_date = formatdate(time.time()) + log.msg('adding message %s to local db' % (doc.doc_id,)) + + def msgSavedCallback(result): + if empty(result): + return + + for listener in self._listeners: + listener(result) + + def signal_deleted(doc_id): + emit_async(catalog.MAIL_MSG_DELETED_INCOMING, + self._userid) + return doc_id + + emit_async(catalog.MAIL_MSG_SAVED_LOCALLY, self._userid) + d = self._delete_incoming_message(doc) + d.addCallback(signal_deleted) + return d + + d = self._inbox_collection.add_msg( + raw_data, (self.RECENT_FLAG,), date=insertion_date, + notify_just_mdoc=True) + d.addCallbacks(msgSavedCallback, self._errback) + return d + + # + # helpers + # + + def _msg_multipart_sanity_check(self, msg): + """ + Performs a sanity check against a multipart encrypted msg + + :param msg: The original encrypted message. + :type msg: Message + """ + # sanity check + payload = msg.get_payload() + if len(payload) != 2: + raise MalformedMessage( + 'Multipart/encrypted messages should have exactly 2 body ' + 'parts (instead of %d).' % len(payload)) + if payload[0].get_content_type() != 'application/pgp-encrypted': + raise MalformedMessage( + "Multipart/encrypted messages' first body part should " + "have content type equal to 'application/pgp-encrypted' " + "(instead of %s)." % payload[0].get_content_type()) + if payload[1].get_content_type() != 'application/octet-stream': + raise MalformedMessage( + "Multipart/encrypted messages' second body part should " + "have content type equal to 'octet-stream' (instead of " + "%s)." % payload[1].get_content_type()) + + def _is_msg(self, keys): + """ + Checks if the keys of a dictionary match the signature + of the document type we use for messages. + + :param keys: iterable containing the strings to match. + :type keys: iterable of strings. + :rtype: bool + """ + return ENC_SCHEME_KEY in keys and ENC_JSON_KEY in keys diff --git a/src/leap/bitmask/mail/incoming/tests/rfc822.multi-encrypt-signed.message b/src/leap/bitmask/mail/incoming/tests/rfc822.multi-encrypt-signed.message new file mode 100644 index 0000000..98304f2 --- /dev/null +++ b/src/leap/bitmask/mail/incoming/tests/rfc822.multi-encrypt-signed.message @@ -0,0 +1,61 @@ +Content-Type: multipart/encrypted; + boundary="Apple-Mail=_C01A1464-6C43-43BF-8F62-157335B7E25B"; + protocol="application/pgp-encrypted"; +Subject: Enc signed +Mime-Version: 1.0 (Mac OS X Mail 9.3 \(3124\)) +From: Leap Test Key +Date: Tue, 24 May 2016 11:47:24 -0300 +Content-Description: OpenPGP encrypted message +To: leap@leap.se + +This is an OpenPGP/MIME encrypted message (RFC 2440 and 3156) +--Apple-Mail=_C01A1464-6C43-43BF-8F62-157335B7E25B +Content-Type: application/pgp-encrypted +Content-Description: PGP/MIME Versions Identification + +--Apple-Mail=_C01A1464-6C43-43BF-8F62-157335B7E25B +Content-Disposition: inline; + filename=encrypted.asc +Content-Type: application/octet-stream; + name=encrypted.asc +Content-Description: OpenPGP encrypted message + +-----BEGIN PGP MESSAGE----- +Version: GnuPG v2 + +hQIMAyj9aG/xtZOwAQ/9Gft0KmOpgzL6z4wmVlLm2aeAvHolXmxWb7N/ByL/dZ4n +YZd/GPRj42X3BwUrDEL5aO3Mcp+rqq8ACh9hsZXiau0Q9cs1K7Gr55Y06qLrIjom +2fLqwLFBxCL2sAX1dvClgStyfsRFk9Y/+5tX+IjWaD8dAoRdxCO8IbUDuYGnaKld +bB9h0NMfKVddCAvuQvX1Zc1Nx0Yb3Hd+ocDD7i9BVgX1BBiGu4/ElS3d32TAVCFs +Na3tjitWB2G472CYu1O6exY7h1F5V4FHfXH6iMRJSYnvV2Jr+oPZENzNdEEA5H/H +fUbpWrpKzPafjho9S5rJBBM/tqtmBQFBIdgFVcBVb+bXO6DJ8SMTLiiGcVUvvm1b +9N2VQIhsxtZ8DpcHHSqFVgT2Gt4UkSrEleSoReg36TzS1s8Uw0oU068PwTe3K0Gx +2pLMdT9NA6X/t7movpXP6tih1l6P5z62dxFl6W12J9OcegISCt0Q7gex1gk/a8zM +rzBJC3mVxRiFlvHPBgD6oUKarnTJPQx5f5dFXg8DXBWR1Eh/aFjPQIzhZBYpmOi8 +HqgjcAA+WhMQ7v5c0enJoJJS+8Xfai/MK2vTUGsfAT6HqHLw1HSIn6XQGEf4sQ/U +NfLeFHHbe9rTk8QhyjrSl2vvek2H4EBQVLF08/FUrAfPELUttOFtysQfC3+M0+PS +6QGyeIlUjKpBJG7HBd4ibuKMQ5vnA+ACsg/TySYeCO6P85xsN+Lmqlr8cAICn/hR +ezFSzlibaIelRgfDEDJdjVyCsa7qBMjhRCvGYBdkyTzIRq53qwD9pkhrQ6nwWQrv +bBzyLrl+NVR8CTEOwbeFLI6qf68kblojk3lwo3Qi3psmeMJdiaV9uevsHrgmEFTH +lZ3rFECPWzmrkMSfVjWu5d8jJqMcqa4lnGzFQKaB76I8BzGhCWrnuvHPB9c9SVhI +AnAwNw3gY5xgsbXMxZhnPgYeBSViPkQkgRCWl8Jz41eiAJ3Gtj8QSSFWGHpX+MgP +ohBaPHz6Fnkhz7Lok97e2AcuRZrDVKV6i28r8mizI3B2Mah6ZV0Yuv0EYNtzBv/v +yV3nu4DWuOOU0301CXBayxJGX0h07z1Ycv7jWD6LNiBXa1vahtbU4WSYNkF0OJaz +nf8O3CZy5twMq5kQYoPacdNNLregAmWquvE1nxqWbtHFMjtXitP7czxzUTU/DE+C +jr+irDoYEregEKg9xov91UCRPZgxL+TML71+tSYOMO3JG6lbGw77PQ8s2So7xore +8+FeDFPaaJqh6uhF5LETRSx8x/haZiXLd+WtO7wF8S3+Vz7AJIFIe8MUadZrYwnH +wfMAktQKbep3iHCeZ5jHYA461AOhnCca2y+GoyHZUDDFwS1pC1RN4lMkafSE1AgH +cmEcjLYsw1gqT0+DfqrvjbXmMjGgkgnkMybJH7df5TKu36Q0Nqvcbc2XLFkalr5V +Vk0SScqKYnKL+cJjabqA8rKkeAh22E2FBCpKPqxSS3te2bRb3XBX26bP0LshkJuy +GPu6LKvwmUn0obPKCnLJvb9ImIGZToXu6Fb/Cd2c3DG1IK5PptQz4f7ZRW98huPO +2w59Bswwt5q4lQqsMEzVRnIDH45MmnhEUeS4NaxqLTO7eJpMpb4VxT2u/Ac3XWKp +o2RE6CbqTyJ+n8tY9OwBRMKzdVd9RFAMqMHTzWTAuU4BgW2vT2sHYZdAsX8sktBr +5mo9P3MqvgdPNpg8+AOB03JlIv0dzrAFWCZxxLLGIIIz0eXsjghHzQ9QjGfr0xFH +Z79AKDjsoRisWyWCnadS2oM9fdAg4T/h1STnfxc44o7N1+ym7u58ODICFi+Kg8IR +JBHIp3CK02JLTLd/WFhUVyWgc6l8gn+oBK+r7Dw+FTWhqX2/ZHCO8qKK1ZK3NIMn +MBcSVvHSnTPtppb+oND5nk38xazVVHnwxNHaIh7g3NxDB4hl5rBhrWsgTNuqDDRU +w7ufvMYr1AOV+8e92cHCEKPM19nFKEgaBFECEptEObesGI3QZPAESlojzQ3cDeBa +=tEyc +-----END PGP MESSAGE----- + +--Apple-Mail=_C01A1464-6C43-43BF-8F62-157335B7E25B-- \ No newline at end of file diff --git a/src/leap/bitmask/mail/incoming/tests/test_incoming_mail.py b/src/leap/bitmask/mail/incoming/tests/test_incoming_mail.py new file mode 100644 index 0000000..29422ec --- /dev/null +++ b/src/leap/bitmask/mail/incoming/tests/test_incoming_mail.py @@ -0,0 +1,391 @@ +# -*- coding: utf-8 -*- +# test_incoming_mail.py +# Copyright (C) 2015 LEAP +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . +""" +Test case for leap.mail.incoming.service + +@authors: Ruben Pollan, + +@license: GPLv3, see included LICENSE file +""" + +import json +import os +import tempfile +import uuid + +from email.mime.application import MIMEApplication +from email.mime.multipart import MIMEMultipart +from email.parser import Parser +from mock import Mock + +from twisted.internet import defer +from twisted.python import log + +from leap.keymanager.errors import KeyAddressMismatch +from leap.mail.adaptors import soledad_indexes as fields +from leap.mail.adaptors.soledad import cleanup_deferred_locks +from leap.mail.adaptors.soledad import SoledadMailAdaptor +from leap.mail.mail import MessageCollection +from leap.mail.mailbox_indexer import MailboxIndexer + +from leap.mail.incoming.service import IncomingMail +from leap.mail.rfc3156 import MultipartEncrypted, PGPEncrypted +from leap.mail.testing import KeyManagerWithSoledadTestCase +from leap.mail.testing import ADDRESS, ADDRESS_2 +from leap.soledad.common.document import SoledadDocument +from leap.soledad.common.crypto import ( + EncryptionSchemes, + ENC_JSON_KEY, + ENC_SCHEME_KEY, +) + +HERE = os.path.split(os.path.abspath(__file__))[0] + +# TODO: add some tests for encrypted, unencrypted, signed and unsgined messages + + +class IncomingMailTestCase(KeyManagerWithSoledadTestCase): + """ + Tests for the incoming mail parser + """ + NICKSERVER = "http://domain" + BODY = """ +Governments of the Industrial World, you weary giants of flesh and steel, I +come from Cyberspace, the new home of Mind. On behalf of the future, I ask +you of the past to leave us alone. You are not welcome among us. You have +no sovereignty where we gather. + """ + EMAIL = """from: Test from SomeDomain <%(from)s> +to: %(to)s +subject: independence of cyberspace + +%(body)s + """ % { + "from": ADDRESS_2, + "to": ADDRESS, + "body": BODY + } + + def setUp(self): + cleanup_deferred_locks() + try: + del self._soledad + del self.km + except AttributeError: + pass + + # pytest handles correctly the setupEnv for the class, + # but trial ignores it. + if not getattr(self, 'tempdir', None): + self.tempdir = tempfile.mkdtemp() + + def getCollection(_): + adaptor = SoledadMailAdaptor() + store = self._soledad + adaptor.store = store + mbox_indexer = MailboxIndexer(store) + mbox_name = "INBOX" + mbox_uuid = str(uuid.uuid4()) + + def get_collection_from_mbox_wrapper(wrapper): + wrapper.uuid = mbox_uuid + return MessageCollection( + adaptor, store, + mbox_indexer=mbox_indexer, mbox_wrapper=wrapper) + + d = adaptor.initialize_store(store) + d.addCallback(lambda _: mbox_indexer.create_table(mbox_uuid)) + d.addCallback( + lambda _: adaptor.get_or_create_mbox(store, mbox_name)) + d.addCallback(get_collection_from_mbox_wrapper) + return d + + def setUpFetcher(inbox_collection): + self.fetcher = IncomingMail( + self.km, + self._soledad, + inbox_collection, + ADDRESS) + + # The messages don't exist on soledad will fail on deletion + self.fetcher._delete_incoming_message = Mock( + return_value=defer.succeed(None)) + + d = KeyManagerWithSoledadTestCase.setUp(self) + d.addCallback(getCollection) + d.addCallback(setUpFetcher) + d.addErrback(log.err) + return d + + def tearDown(self): + d = KeyManagerWithSoledadTestCase.tearDown(self) + return d + + def testExtractOpenPGPHeader(self): + """ + Test the OpenPGP header key extraction + """ + KEYURL = "https://leap.se/key.txt" + OpenPGP = "id=12345678; url=\"%s\"; preference=signencrypt" % (KEYURL,) + + message = Parser().parsestr(self.EMAIL) + message.add_header("OpenPGP", OpenPGP) + self.fetcher._keymanager.fetch_key = Mock( + return_value=defer.succeed(None)) + + def fetch_key_called(ret): + self.fetcher._keymanager.fetch_key.assert_called_once_with( + ADDRESS_2, KEYURL) + + d = self._create_incoming_email(message.as_string()) + d.addCallback( + lambda email: + self._mock_soledad_get_from_index(fields.JUST_MAIL_IDX, [email])) + d.addCallback(lambda _: self.fetcher.fetch()) + d.addCallback(fetch_key_called) + return d + + def testExtractOpenPGPHeaderInvalidUrl(self): + """ + Test the OpenPGP header key extraction + """ + KEYURL = "https://someotherdomain.com/key.txt" + OpenPGP = "id=12345678; url=\"%s\"; preference=signencrypt" % (KEYURL,) + + message = Parser().parsestr(self.EMAIL) + message.add_header("OpenPGP", OpenPGP) + self.fetcher._keymanager.fetch_key = Mock() + + def fetch_key_called(ret): + self.assertFalse(self.fetcher._keymanager.fetch_key.called) + + d = self._create_incoming_email(message.as_string()) + d.addCallback( + lambda email: + self._mock_soledad_get_from_index(fields.JUST_MAIL_IDX, [email])) + d.addCallback(lambda _: self.fetcher.fetch()) + d.addCallback(fetch_key_called) + return d + + def testExtractAttachedKey(self): + KEY = "-----BEGIN PGP PUBLIC KEY BLOCK-----\n..." + + message = MIMEMultipart() + message.add_header("from", ADDRESS_2) + key = MIMEApplication("", "pgp-keys") + key.set_payload(KEY) + message.attach(key) + self.fetcher._keymanager.put_raw_key = Mock( + return_value=defer.succeed(None)) + + def put_raw_key_called(_): + self.fetcher._keymanager.put_raw_key.assert_called_once_with( + KEY, address=ADDRESS_2) + + d = self._do_fetch(message.as_string()) + d.addCallback(put_raw_key_called) + return d + + def testExtractInvalidAttachedKey(self): + KEY = "-----BEGIN PGP PUBLIC KEY BLOCK-----\n..." + + message = MIMEMultipart() + message.add_header("from", ADDRESS_2) + key = MIMEApplication("", "pgp-keys") + key.set_payload(KEY) + message.attach(key) + self.fetcher._keymanager.put_raw_key = Mock( + return_value=defer.fail(KeyAddressMismatch())) + + def put_raw_key_called(_): + self.fetcher._keymanager.put_raw_key.assert_called_once_with( + KEY, address=ADDRESS_2) + + d = self._do_fetch(message.as_string()) + d.addCallback(put_raw_key_called) + d.addErrback(log.err) + return d + + def testExtractAttachedKeyAndNotOpenPGPHeader(self): + KEY = "-----BEGIN PGP PUBLIC KEY BLOCK-----\n..." + KEYURL = "https://leap.se/key.txt" + OpenPGP = "id=12345678; url=\"%s\"; preference=signencrypt" % (KEYURL,) + + message = MIMEMultipart() + message.add_header("from", ADDRESS_2) + message.add_header("OpenPGP", OpenPGP) + key = MIMEApplication("", "pgp-keys") + key.set_payload(KEY) + message.attach(key) + + self.fetcher._keymanager.put_raw_key = Mock( + return_value=defer.succeed(None)) + self.fetcher._keymanager.fetch_key = Mock() + + def put_raw_key_called(_): + self.fetcher._keymanager.put_raw_key.assert_called_once_with( + KEY, address=ADDRESS_2) + self.assertFalse(self.fetcher._keymanager.fetch_key.called) + + d = self._do_fetch(message.as_string()) + d.addCallback(put_raw_key_called) + return d + + def testExtractOpenPGPHeaderIfInvalidAttachedKey(self): + KEY = "-----BEGIN PGP PUBLIC KEY BLOCK-----\n..." + KEYURL = "https://leap.se/key.txt" + OpenPGP = "id=12345678; url=\"%s\"; preference=signencrypt" % (KEYURL,) + + message = MIMEMultipart() + message.add_header("from", ADDRESS_2) + message.add_header("OpenPGP", OpenPGP) + key = MIMEApplication("", "pgp-keys") + key.set_payload(KEY) + message.attach(key) + + self.fetcher._keymanager.put_raw_key = Mock( + return_value=defer.fail(KeyAddressMismatch())) + self.fetcher._keymanager.fetch_key = Mock() + + def put_raw_key_called(_): + self.fetcher._keymanager.put_raw_key.assert_called_once_with( + KEY, address=ADDRESS_2) + self.fetcher._keymanager.fetch_key.assert_called_once_with( + ADDRESS_2, KEYURL) + + d = self._do_fetch(message.as_string()) + d.addCallback(put_raw_key_called) + return d + + def testAddDecryptedHeader(self): + class DummyMsg(): + + def __init__(self): + self.headers = {} + + def add_header(self, k, v): + self.headers[k] = v + + msg = DummyMsg() + self.fetcher._add_decrypted_header(msg) + + self.assertEquals(msg.headers['X-Leap-Encryption'], 'decrypted') + + def testDecryptEmail(self): + + self.fetcher._decryption_error = Mock() + self.fetcher._add_decrypted_header = Mock() + + def create_encrypted_message(encstr): + message = Parser().parsestr(self.EMAIL) + newmsg = MultipartEncrypted('application/pgp-encrypted') + for hkey, hval in message.items(): + newmsg.add_header(hkey, hval) + + encmsg = MIMEApplication( + encstr, _subtype='octet-stream', _encoder=lambda x: x) + encmsg.add_header('content-disposition', 'attachment', + filename='msg.asc') + # create meta message + metamsg = PGPEncrypted() + metamsg.add_header('Content-Disposition', 'attachment') + # attach pgp message parts to new message + newmsg.attach(metamsg) + newmsg.attach(encmsg) + return newmsg + + def decryption_error_not_called(_): + self.assertFalse(self.fetcher._decryption_error.called, + "There was some errors with decryption") + + def add_decrypted_header_called(_): + self.assertTrue(self.fetcher._add_decrypted_header.called, + "There was some errors with decryption") + + d = self.km.encrypt(self.EMAIL, ADDRESS, sign=ADDRESS_2) + d.addCallback(create_encrypted_message) + d.addCallback( + lambda message: + self._do_fetch(message.as_string())) + d.addCallback(decryption_error_not_called) + d.addCallback(add_decrypted_header_called) + return d + + def testValidateSignatureFromEncryptedEmailFromAppleMail(self): + enc_signed_file = os.path.join( + HERE, 'rfc822.multi-encrypt-signed.message') + self.fetcher._add_verified_signature_header = Mock() + + def add_verified_signature_header_called(_): + self.assertTrue(self.fetcher._add_verified_signature_header.called, + "There was some errors verifying signature") + + with open(enc_signed_file) as f: + enc_signed_raw = f.read() + + d = self._do_fetch(enc_signed_raw) + d.addCallback(add_verified_signature_header_called) + return d + + def testListener(self): + self.called = False + + def listener(uid): + self.called = True + + def listener_called(_): + self.assertTrue(self.called) + + self.fetcher.add_listener(listener) + d = self._do_fetch(self.EMAIL) + d.addCallback(listener_called) + return d + + def _do_fetch(self, message): + d = self._create_incoming_email(message) + d.addCallback( + lambda email: + self._mock_soledad_get_from_index(fields.JUST_MAIL_IDX, [email])) + d.addCallback(lambda _: self.fetcher.fetch()) + return d + + def _create_incoming_email(self, email_str): + email = SoledadDocument() + data = json.dumps( + {"incoming": True, "content": email_str}, + ensure_ascii=False) + + def set_email_content(encr_data): + email.content = { + fields.INCOMING_KEY: True, + fields.ERROR_DECRYPTING_KEY: False, + ENC_SCHEME_KEY: EncryptionSchemes.PUBKEY, + ENC_JSON_KEY: encr_data + } + return email + d = self.km.encrypt(data, ADDRESS, fetch_remote=False) + d.addCallback(set_email_content) + return d + + def _mock_soledad_get_from_index(self, index_name, value): + get_from_index = self._soledad.get_from_index + + def soledad_mock(idx_name, *key_values): + if index_name == idx_name: + return defer.succeed(value) + return get_from_index(idx_name, *key_values) + self.fetcher._soledad.get_from_index = Mock(side_effect=soledad_mock) diff --git a/src/leap/bitmask/mail/interfaces.py b/src/leap/bitmask/mail/interfaces.py new file mode 100644 index 0000000..10f5123 --- /dev/null +++ b/src/leap/bitmask/mail/interfaces.py @@ -0,0 +1,215 @@ +# -*- coding: utf-8 -*- +# interfaces.py +# Copyright (C) 2014,2015 LEAP +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . +""" +Interfaces for the leap.mail module. +""" +from zope.interface import Interface, Attribute + + +class IMessageWrapper(Interface): + """ + I know how to access the different parts into which a given message is + splitted into. + + :ivar fdoc: dict with flag document. + :ivar hdoc: dict with flag document. + :ivar cdocs: dict with content-documents, one-indexed. + """ + + fdoc = Attribute('A dictionaly-like containing the flags document ' + '(mutable)') + hdoc = Attribute('A dictionary-like containing the headers document ' + '(immutable)') + cdocs = Attribute('A dictionary with the content-docs, one-indexed') + + def create(self, store, notify_just_mdoc=False, pending_inserts_dict={}): + """ + Create the underlying wrapper. + """ + + def update(self, store): + """ + Update the only mutable parts, which are within the flags document. + """ + + def delete(self, store): + """ + Delete the parts for this wrapper that are not referenced from anywhere + else. + """ + + def copy(self, store, new_mbox_uuid): + """ + Return a copy of this IMessageWrapper in a new mailbox. + """ + + def set_mbox_uuid(self, mbox_uuid): + """ + Set the mailbox for this wrapper. + """ + + def set_flags(self, flags): + """ + """ + + def set_tags(self, tags): + """ + """ + + def set_date(self, date): + """ + """ + + def get_subpart_dict(self, index): + """ + :param index: the part to lookup, 1-indexed + """ + + def get_subpart_indexes(self): + """ + """ + + def get_body(self, store): + """ + """ + + +# TODO -- split into smaller interfaces? separate mailbox interface at least? + +class IMailAdaptor(Interface): + """ + I know how to store the standard representation for messages and mailboxes, + and how to update the relevant mutable parts when needed. + """ + + def initialize_store(self, store): + """ + Performs whatever initialization is needed before the store can be + used (creating indexes, sanity checks, etc). + + :param store: store + :returns: a Deferred that will fire when the store is correctly + initialized. + :rtype: deferred + """ + + def get_msg_from_string(self, MessageClass, raw_msg): + """ + Get an instance of a MessageClass initialized with a MessageWrapper + that contains all the parts obtained from parsing the raw string for + the message. + + :param MessageClass: an implementor of IMessage + :type raw_msg: str + :rtype: implementor of leap.mail.IMessage + """ + + def get_msg_from_docs(self, MessageClass, mdoc, fdoc, hdoc, cdocs=None, + uid=None): + """ + Get an instance of a MessageClass initialized with a MessageWrapper + that contains the passed part documents. + + This is not the recommended way of obtaining a message, unless you know + how to take care of ensuring the internal consistency between the part + documents, or unless you are glueing together the part documents that + have been previously generated by `get_msg_from_string`. + """ + + def get_flags_from_mdoc_id(self, store, mdoc_id): + """ + """ + + def create_msg(self, store, msg): + """ + :param store: an instance of soledad, or anything that behaves alike + :param msg: a Message object. + + :return: a Deferred that is fired when all the underlying documents + have been created. + :rtype: defer.Deferred + """ + + def update_msg(self, store, msg): + """ + :param msg: a Message object. + :param store: an instance of soledad, or anything that behaves alike + :return: a Deferred that is fired when all the underlying documents + have been updated (actually, it's only the fdoc that's allowed + to update). + :rtype: defer.Deferred + """ + + def get_count_unseen(self, store, mbox_uuid): + """ + Get the number of unseen messages for a given mailbox. + + :param store: instance of Soledad. + :param mbox_uuid: the uuid for this mailbox. + :rtype: int + """ + + def get_count_recent(self, store, mbox_uuid): + """ + Get the number of recent messages for a given mailbox. + + :param store: instance of Soledad. + :param mbox_uuid: the uuid for this mailbox. + :rtype: int + """ + + def get_mdoc_id_from_msgid(self, store, mbox_uuid, msgid): + """ + Get the UID for a message with the passed msgid (the one in the headers + msg-id). + This is used by the MUA to retrieve the recently saved draft. + """ + + # mbox handling + + def get_or_create_mbox(self, store, name): + """ + Get the mailbox with the given name, or create one if it does not + exist. + + :param store: instance of Soledad + :param name: the name of the mailbox + :type name: str + """ + + def update_mbox(self, store, mbox_wrapper): + """ + Update the documents for a given mailbox. + :param mbox_wrapper: MailboxWrapper instance + :type mbox_wrapper: MailboxWrapper + :return: a Deferred that will be fired when the mailbox documents + have been updated. + :rtype: defer.Deferred + """ + + def delete_mbox(self, store, mbox_wrapper): + """ + """ + + def get_all_mboxes(self, store): + """ + Retrieve a list with wrappers for all the mailboxes. + + :return: a deferred that will be fired with a list of all the + MailboxWrappers found. + :rtype: defer.Deferred + """ diff --git a/src/leap/bitmask/mail/load_tests.py b/src/leap/bitmask/mail/load_tests.py new file mode 100644 index 0000000..be65b8d --- /dev/null +++ b/src/leap/bitmask/mail/load_tests.py @@ -0,0 +1,29 @@ +# -*- coding: utf-8 -*- +# tests.py +# Copyright (C) 2013 LEAP +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . +""" +Provide a function for loading tests. +""" +import unittest + + +def load_tests(): + suite = unittest.TestSuite() + for test in unittest.defaultTestLoader.discover( + './src/leap/mail/', + top_level_dir='./src/'): + suite.addTest(test) + return suite diff --git a/src/leap/bitmask/mail/mail.py b/src/leap/bitmask/mail/mail.py new file mode 100644 index 0000000..2fde3a1 --- /dev/null +++ b/src/leap/bitmask/mail/mail.py @@ -0,0 +1,1070 @@ +# -*- coding: utf-8 -*- +# mail.py +# Copyright (C) 2014,2015 LEAP +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . +""" +Generic Access to Mail objects. + +This module holds the public LEAP Mail API, which should be viewed as the main +entry point for message and account manipulation, in a protocol-agnostic way. + +In the future, pluggable transports will expose this generic API. +""" +import itertools +import uuid +import logging +import StringIO +import time +import weakref + +from collections import defaultdict + +from twisted.internet import defer +from twisted.python import log + +from leap.common.check import leap_assert_type +from leap.common.events import emit_async, catalog + +from leap.mail.adaptors.soledad import SoledadMailAdaptor +from leap.mail.constants import INBOX_NAME +from leap.mail.constants import MessageFlags +from leap.mail.mailbox_indexer import MailboxIndexer +from leap.mail.plugins import soledad_sync_hooks +from leap.mail.utils import find_charset, CaseInsensitiveDict +from leap.mail.utils import lowerdict + +logger = logging.getLogger(name=__name__) + + +# TODO LIST +# [ ] Probably change the name of this module to "api" or "account", mail is +# too generic (there's also IncomingMail, and OutgoingMail +# [ ] Profile add_msg. + +def _get_mdoc_id(mbox, chash): + """ + Get the doc_id for the metamsg document. + """ + return "M+{mbox}+{chash}".format(mbox=mbox, chash=chash) + + +def _write_and_rewind(payload): + fd = StringIO.StringIO() + fd.write(payload) + fd.seek(0) + return fd + + +def _encode_payload(payload, ctype=""): + """ + Properly encode an unicode payload (which can be string or unicode) as a + string. + + :param payload: the payload to encode. currently soledad returns unicode + strings. + :type payload: basestring + :param ctype: optional, the content of the content-type header for this + payload. + :type ctype: str + :rtype: str + """ + # TODO Related, it's proposed that we're able to pass + # the encoding to the soledad documents. Better to store the charset there? + # FIXME ----------------------------------------------- + # this need a dedicated test-suite + charset = find_charset(ctype) + + # XXX get from mail headers if not multipart! + # Beware also that we should pass the proper encoding to + # soledad when it's creating the documents. + # if not charset: + # charset = get_email_charset(payload) + # ----------------------------------------------------- + + if not charset: + charset = "utf-8" + + try: + if isinstance(payload, unicode): + payload = payload.encode(charset) + except UnicodeError as exc: + logger.error( + "Unicode error, using 'replace'. {0!r}".format(exc)) + payload = payload.encode(charset, 'replace') + return payload + + +def _unpack_headers(headers_dict): + """ + Take a "packed" dict containing headers (with repeated keys represented as + line breaks inside each value, preceded by the header key) and return a + list of tuples in which each repeated key has a different tuple. + """ + headers_l = headers_dict.items() + for i, (k, v) in enumerate(headers_l): + splitted = v.split(k.lower() + ": ") + if len(splitted) != 1: + inner = zip( + itertools.cycle([k]), + map(lambda l: l.rstrip('\n'), splitted)) + headers_l = headers_l[:i] + inner + headers_l[i + 1:] + return headers_l + + +class MessagePart(object): + # TODO This class should be better abstracted from the data model. + # TODO support arbitrarily nested multiparts (right now we only support + # the trivial case) + """ + Represents a part of a multipart MIME Message. + """ + + def __init__(self, part_map, cdocs=None, nested=False): + """ + :param part_map: a dictionary mapping the subparts for + this MessagePart (1-indexed). + :type part_map: dict + + The format for the part_map is as follows: + + {u'ctype': u'text/plain', + u'headers': [[u'Content-Type', u'text/plain; charset="utf-8"'], + [u'Content-Transfer-Encoding', u'8bit']], + u'multi': False, + u'parts': 1, + u'phash': u'02D82B29F6BB0C8612D1C', + u'size': 132} + + :param cdocs: optional, a reference to the top-level dict of wrappers + for content-docs (1-indexed). + """ + if cdocs is None: + cdocs = {} + self._pmap = part_map + self._cdocs = cdocs + self._nested = nested + + def get_size(self): + """ + Size of the body, in octets. + """ + total = self._pmap['size'] + _h = self.get_headers() + headers = len( + '\n'.join(["%s: %s" % (k, v) for k, v in dict(_h).items()])) + # have to subtract 2 blank lines + return total - headers - 2 + + def get_body_file(self): + payload = "" + pmap = self._pmap + + multi = pmap.get('multi') + if not multi: + payload = self._get_payload(pmap.get('phash')) + if payload: + payload = _encode_payload(payload) + + return _write_and_rewind(payload) + + def get_headers(self): + return CaseInsensitiveDict(self._pmap.get("headers", [])) + + def is_multipart(self): + return self._pmap.get("multi", False) + + def get_subpart(self, part): + if not self.is_multipart(): + raise TypeError + sub_pmap = self._pmap.get("part_map", {}) + + try: + part_map = sub_pmap[str(part)] + except KeyError: + log.msg("getSubpart for %s: KeyError" % (part,)) + raise IndexError + return MessagePart(part_map, cdocs=self._cdocs, nested=True) + + def _get_payload(self, phash): + for cdocw in self._cdocs.values(): + if cdocw.phash == phash: + return cdocw.raw + return "" + + +class Message(object): + """ + Represents a single message, and gives access to all its attributes. + """ + + def __init__(self, wrapper, uid=None): + """ + :param wrapper: an instance of an implementor of IMessageWrapper + :param uid: + :type uid: int + """ + self._wrapper = wrapper + self._uid = uid + + def get_wrapper(self): + """ + Get the wrapper for this message. + """ + return self._wrapper + + def get_uid(self): + """ + Get the (optional) UID. + """ + return self._uid + + # imap.IMessage methods + + def get_flags(self): + """ + Get flags for this message. + :rtype: tuple + """ + return self._wrapper.fdoc.get_flags() + + def get_internal_date(self): + """ + Retrieve the date internally associated with this message + + According to the spec, this is NOT the date and time in the + RFC-822 header, but rather a date and time that reflects when the + message was received. + + * In SMTP, date and time of final delivery. + * In COPY, internal date/time of the source message. + * In APPEND, date/time specified. + + :return: An RFC822-formatted date string. + :rtype: str + """ + return self._wrapper.hdoc.date + + # imap.IMessageParts + + def get_headers(self): + """ + Get the raw headers document. + """ + return CaseInsensitiveDict(self._wrapper.hdoc.headers) + + def get_body_file(self, store): + """ + Get a file descriptor with the body content. + """ + def write_and_rewind_if_found(cdoc): + payload = cdoc.raw if cdoc else "" + # XXX pass ctype from headers if not multipart? + if payload: + payload = _encode_payload(payload, ctype=cdoc.content_type) + return _write_and_rewind(payload) + + d = defer.maybeDeferred(self._wrapper.get_body, store) + d.addCallback(write_and_rewind_if_found) + return d + + def get_size(self): + """ + Size of the whole message, in octets (including headers). + """ + total = self._wrapper.fdoc.size + return total + + def is_multipart(self): + """ + Return True if this message is multipart. + """ + return self._wrapper.fdoc.multi + + def get_subpart(self, part): + """ + :param part: The number of the part to retrieve, indexed from 1. + :type part: int + :rtype: MessagePart + """ + if not self.is_multipart(): + raise TypeError + try: + subpart_dict = self._wrapper.get_subpart_dict(part) + except KeyError: + raise IndexError + + return MessagePart( + subpart_dict, cdocs=self._wrapper.cdocs) + + # Custom methods. + + def get_tags(self): + """ + Get the tags for this message. + """ + return tuple(self._wrapper.fdoc.tags) + + +class Flagsmode(object): + """ + Modes for setting the flags/tags. + """ + APPEND = 1 + REMOVE = -1 + SET = 0 + + +class MessageCollection(object): + """ + A generic collection of messages. It can be messages sharing the same + mailbox, tag, the result of a given query, or just a bunch of ids for + master documents. + + Since LEAP Mail is primarily oriented to store mail in Soledad, the default + (and, so far, only) implementation of the store is contained in the + Soledad Mail Adaptor, which is passed to every collection on creation by + the root Account object. If you need to use a different adaptor, change the + adaptor class attribute in your Account object. + + Store is a reference to a particular instance of the message store (soledad + instance or proxy, for instance). + """ + + # TODO LIST + # [ ] look at IMessageSet methods + # [ ] make constructor with a per-instance deferredLock to use on + # creation/deletion? + # [ ] instead of a mailbox, we could pass an arbitrary container with + # pointers to different doc_ids (type: foo) + # [ ] To guarantee synchronicity of the documents sent together during a + # sync, we could get hold of a deferredLock that inhibits + # synchronization while we are updating (think more about this!) + # [ ] review the serveral count_ methods. I think it's better to patch + # server to accept deferreds. + # [ ] Use inheritance for the mailbox-collection instead of handling the + # special cases everywhere? + # [ ] or maybe a mailbox_only decorator... + + # Account should provide an adaptor instance when creating this collection. + adaptor = None + store = None + messageklass = Message + + _pending_inserts = dict() + + def __init__(self, adaptor, store, mbox_indexer=None, mbox_wrapper=None): + """ + Constructor for a MessageCollection. + """ + self.adaptor = adaptor + self.store = store + + # XXX think about what to do when there is no mbox passed to + # the initialization. We could still get the MetaMsg by index, instead + # of by doc_id. See get_message_by_content_hash + self.mbox_indexer = mbox_indexer + self.mbox_wrapper = mbox_wrapper + self._listeners = set([]) + + def is_mailbox_collection(self): + """ + Return True if this collection represents a Mailbox. + :rtype: bool + """ + return bool(self.mbox_wrapper) + + @property + def mbox_name(self): + # TODO raise instead? + if self.mbox_wrapper is None: + return None + return self.mbox_wrapper.mbox + + @property + def mbox_uuid(self): + # TODO raise instead? + if self.mbox_wrapper is None: + return None + return self.mbox_wrapper.uuid + + def get_mbox_attr(self, attr): + if self.mbox_wrapper is None: + raise RuntimeError("This is not a mailbox collection") + return getattr(self.mbox_wrapper, attr) + + def set_mbox_attr(self, attr, value): + if self.mbox_wrapper is None: + raise RuntimeError("This is not a mailbox collection") + setattr(self.mbox_wrapper, attr, value) + return self.mbox_wrapper.update(self.store) + + # Get messages + + def get_message_by_content_hash(self, chash, get_cdocs=False): + """ + Retrieve a message by its content hash. + :rtype: Deferred + """ + if not self.is_mailbox_collection(): + # TODO instead of getting the metamsg by chash, in this case we + # should query by (meta) index or use the internal collection of + # pointers-to-docs. + raise NotImplementedError() + + metamsg_id = _get_mdoc_id(self.mbox_name, chash) + + return self.adaptor.get_msg_from_mdoc_id( + self.messageklass, self.store, + metamsg_id, get_cdocs=get_cdocs) + + def get_message_by_sequence_number(self, msn, get_cdocs=False): + """ + Retrieve a message by its Message Sequence Number. + :rtype: Deferred + """ + def get_uid_for_msn(all_uid): + return all_uid[msn - 1] + d = self.all_uid_iter() + d.addCallback(get_uid_for_msn) + d.addCallback( + lambda uid: self.get_message_by_uid( + uid, get_cdocs=get_cdocs)) + d.addErrback(lambda f: log.err(f)) + return d + + def get_message_by_uid(self, uid, absolute=True, get_cdocs=False): + """ + Retrieve a message by its Unique Identifier. + + If this is a Mailbox collection, that is the message UID, unique for a + given mailbox, or a relative sequence number depending on the absolute + flag. For now, only absolute identifiers are supported. + :rtype: Deferred + """ + # TODO deprecate absolute flag, it doesn't make sense UID and + # !absolute. use _by_sequence_number instead. + if not absolute: + raise NotImplementedError("Does not support relative ids yet") + + get_doc_fun = self.mbox_indexer.get_doc_id_from_uid + + def get_msg_from_mdoc_id(doc_id): + if doc_id is None: + return None + return self.adaptor.get_msg_from_mdoc_id( + self.messageklass, self.store, + doc_id, uid=uid, get_cdocs=get_cdocs) + + def cleanup_and_get_doc_after_pending_insert(result): + for key in result: + self._pending_inserts.pop(key, None) + return get_doc_fun(self.mbox_uuid, uid) + + if not self._pending_inserts: + d = get_doc_fun(self.mbox_uuid, uid) + else: + d = defer.gatherResults(self._pending_inserts.values()) + d.addCallback(cleanup_and_get_doc_after_pending_insert) + d.addCallback(get_msg_from_mdoc_id) + return d + + def get_flags_by_uid(self, uid, absolute=True): + # TODO use sequence numbers + if not absolute: + raise NotImplementedError("Does not support relative ids yet") + + def get_flags_from_mdoc_id(doc_id): + if doc_id is None: # XXX needed? or bug? + return None + return self.adaptor.get_flags_from_mdoc_id( + self.store, doc_id) + + def wrap_in_tuple(flags): + return (uid, flags) + + d = self.mbox_indexer.get_doc_id_from_uid(self.mbox_uuid, uid) + d.addCallback(get_flags_from_mdoc_id) + d.addCallback(wrap_in_tuple) + return d + + def count(self): + """ + Count the messages in this collection. + :return: a Deferred that will fire with the integer for the count. + :rtype: Deferred + """ + if not self.is_mailbox_collection(): + raise NotImplementedError() + + d = self.mbox_indexer.count(self.mbox_uuid) + return d + + def count_recent(self): + """ + Count the recent messages in this collection. + :return: a Deferred that will fire with the integer for the count. + :rtype: Deferred + """ + if not self.is_mailbox_collection(): + raise NotImplementedError() + return self.adaptor.get_count_recent(self.store, self.mbox_uuid) + + def count_unseen(self): + """ + Count the unseen messages in this collection. + :return: a Deferred that will fire with the integer for the count. + :rtype: Deferred + """ + if not self.is_mailbox_collection(): + raise NotImplementedError() + return self.adaptor.get_count_unseen(self.store, self.mbox_uuid) + + def get_uid_next(self): + """ + Get the next integer beyond the highest UID count for this mailbox. + + :return: a Deferred that will fire with the integer for the next uid. + :rtype: Deferred + """ + return self.mbox_indexer.get_next_uid(self.mbox_uuid) + + def get_last_uid(self): + """ + Get the last UID for this mailbox. + """ + return self.mbox_indexer.get_last_uid(self.mbox_uuid) + + def all_uid_iter(self): + """ + Iterator through all the uids for this collection. + """ + return self.mbox_indexer.all_uid_iter(self.mbox_uuid) + + def get_uid_from_msgid(self, msgid): + """ + Return the UID(s) of the matching msg-ids for this mailbox collection. + """ + if not self.is_mailbox_collection(): + raise NotImplementedError() + + def get_uid(mdoc_id): + if not mdoc_id: + return None + d = self.mbox_indexer.get_uid_from_doc_id( + self.mbox_uuid, mdoc_id) + return d + + d = self.adaptor.get_mdoc_id_from_msgid( + self.store, self.mbox_uuid, msgid) + d.addCallback(get_uid) + return d + + # Manipulate messages + + def add_msg(self, raw_msg, flags=tuple(), tags=tuple(), date="", + notify_just_mdoc=False): + """ + Add a message to this collection. + + :param raw_msg: the raw message + :param flags: tuple of flags for this message + :param tags: tuple of tags for this message + :param date: + formatted date, it will be used to retrieve the internal + date for this message. According to the spec, this is NOT the date + and time in the RFC-822 header, but rather a date and time that + reflects when the message was received. + :type date: str + :param notify_just_mdoc: + boolean passed to the wrapper.create method, to indicate whether + we're insterested in being notified right after the mdoc has been + written (as it's the first doc to be written, and quite small, this + is faster, though potentially unsafe), or on the contrary we want + to wait untill all the parts have been written. + Used by the imap mailbox implementation to get faster responses. + This will be ignored (and set to False) if a heuristic for a Draft + message is met, which currently is a specific mozilla header. + :type notify_just_mdoc: bool + + :returns: a deferred that will fire with the UID of the inserted + message. + :rtype: deferred + """ + # TODO watch out if the use of this method in IMAP COPY/APPEND is + # passing the right date. + # XXX mdoc ref is a leaky abstraction here. generalize. + leap_assert_type(flags, tuple) + leap_assert_type(date, str) + + msg = self.adaptor.get_msg_from_string(Message, raw_msg) + wrapper = msg.get_wrapper() + + headers = lowerdict(msg.get_headers()) + moz_draft_hdr = "X-Mozilla-Draft-Info" + if moz_draft_hdr.lower() in headers: + log.msg("Setting fast notify to False, Draft detected") + notify_just_mdoc = False + + if notify_just_mdoc: + msgid = headers.get('message-id') + if msgid: + self._pending_inserts[msgid] = defer.Deferred() + + if not self.is_mailbox_collection(): + raise NotImplementedError() + + else: + mbox_id = self.mbox_uuid + wrapper.set_mbox_uuid(mbox_id) + wrapper.set_flags(flags) + wrapper.set_tags(tags) + wrapper.set_date(date) + + def insert_mdoc_id(_, wrapper): + doc_id = wrapper.mdoc.doc_id + if not doc_id: + # --- BUG ----------------------------------------- + # XXX watch out, sometimes mdoc doesn't have doc_id + # but it has future_id. Should be solved already. + logger.error("BUG: (please report) Null doc_id for " + "document %s" % + (wrapper.mdoc.serialize(),)) + return defer.succeed("mdoc_id not inserted") + # XXX BUG ----------------------------------------- + + # XXX BUG sometimes the table is not yet created, + # so workaround is to make sure we always check for it before + # inserting the doc. I should debug into the real cause. + d = self.mbox_indexer.create_table(self.mbox_uuid) + d.addBoth(lambda _: self.mbox_indexer.insert_doc( + self.mbox_uuid, doc_id)) + return d + + d = wrapper.create( + self.store, + notify_just_mdoc=notify_just_mdoc, + pending_inserts_dict=self._pending_inserts) + d.addCallback(insert_mdoc_id, wrapper) + d.addCallback(self.cb_signal_unread_to_ui) + d.addCallback(self.notify_new_to_listeners) + d.addErrback(lambda failure: log.err(failure)) + + return d + + # Listeners + + def addListener(self, listener): + self._listeners.add(listener) + + def removeListener(self, listener): + self._listeners.remove(listener) + + def notify_new_to_listeners(self, result): + for listener in self._listeners: + listener.notify_new() + return result + + def cb_signal_unread_to_ui(self, result): + """ + Sends an unread event to ui, passing *only* the number of unread + messages if *this* is the inbox. This event is catched, for instance, + in the Bitmask client that displays a message with the number of unread + mails in the INBOX. + + Used as a callback in several commands. + + :param result: ignored + """ + # TODO it might make sense to modify the event so that + # it receives both the mailbox name AND the number of unread messages. + if self.mbox_name.lower() == "inbox": + d = defer.maybeDeferred(self.count_unseen) + d.addCallback(self.__cb_signal_unread_to_ui) + return result + + def __cb_signal_unread_to_ui(self, unseen): + """ + Send the unread signal to UI. + :param unseen: number of unseen messages. + :type unseen: int + """ + emit_async(catalog.MAIL_UNREAD_MESSAGES, self.store.uuid, str(unseen)) + + def copy_msg(self, msg, new_mbox_uuid): + """ + Copy the message to another collection. (it only makes sense for + mailbox collections) + """ + # TODO should CHECK first if the mdoc is present in the mailbox + # WITH a Deleted flag... and just simply remove the flag... + # Another option is to delete the previous mdoc if it already exists + # (so we get a new UID) + + if not self.is_mailbox_collection(): + raise NotImplementedError() + + def delete_mdoc_entry_and_insert(failure, mbox_uuid, doc_id): + d = self.mbox_indexer.delete_doc_by_hash(mbox_uuid, doc_id) + d.addCallback(lambda _: self.mbox_indexer.insert_doc( + new_mbox_uuid, doc_id)) + return d + + def insert_copied_mdoc_id(wrapper_new_msg): + # XXX FIXME -- since this is already saved, the future_doc_id + # should be already copied into the doc_id! + # Investigate why we are not receiving the already saved doc_id + doc_id = wrapper_new_msg.mdoc.doc_id + if not doc_id: + doc_id = wrapper_new_msg.mdoc._future_doc_id + + def insert_conditionally(uid, mbox_uuid, doc_id): + indexer = self.mbox_indexer + if uid: + d = indexer.delete_doc_by_hash(mbox_uuid, doc_id) + d.addCallback(lambda _: indexer.insert_doc( + new_mbox_uuid, doc_id)) + return d + else: + d = indexer.insert_doc(mbox_uuid, doc_id) + return d + + def log_result(result): + return result + + def insert_doc(_, mbox_uuid, doc_id): + d = self.mbox_indexer.get_uid_from_doc_id(mbox_uuid, doc_id) + d.addCallback(insert_conditionally, mbox_uuid, doc_id) + d.addErrback(lambda err: log.failure(err)) + d.addCallback(log_result) + return d + + d = self.mbox_indexer.create_table(new_mbox_uuid) + d.addBoth(insert_doc, new_mbox_uuid, doc_id) + return d + + wrapper = msg.get_wrapper() + + d = wrapper.copy(self.store, new_mbox_uuid) + d.addCallback(insert_copied_mdoc_id) + d.addCallback(self.notify_new_to_listeners) + return d + + def delete_msg(self, msg): + """ + Delete this message. + """ + wrapper = msg.get_wrapper() + + def delete_mdoc_id(_, wrapper): + doc_id = wrapper.mdoc.doc_id + return self.mbox_indexer.delete_doc_by_hash( + self.mbox_uuid, doc_id) + d = wrapper.delete(self.store) + d.addCallback(delete_mdoc_id, wrapper) + return d + + def delete_all_flagged(self): + """ + Delete all messages flagged as \\Deleted. + Used from IMAPMailbox.expunge() + """ + def get_uid_list(hashes): + d = [] + for h in hashes: + d.append(self.mbox_indexer.get_uid_from_doc_id( + self.mbox_uuid, h)) + return defer.gatherResults(d), hashes + + def delete_uid_entries((uids, hashes)): + d = [] + for h in hashes: + d.append(self.mbox_indexer.delete_doc_by_hash( + self.mbox_uuid, h)) + + def return_uids_when_deleted(ignored): + return uids + + all_deleted = defer.gatherResults(d).addCallback( + return_uids_when_deleted) + return all_deleted + + mdocs_deleted = self.adaptor.del_all_flagged_messages( + self.store, self.mbox_uuid) + mdocs_deleted.addCallback(get_uid_list) + mdocs_deleted.addCallback(delete_uid_entries) + mdocs_deleted.addErrback(lambda f: log.err(f)) + return mdocs_deleted + + # TODO should add a delete-by-uid to collection? + + def delete_all_docs(self): + def del_all_uid(uid_list): + deferreds = [] + for uid in uid_list: + d = self.get_message_by_uid(uid) + d.addCallback(lambda msg: msg.delete()) + deferreds.append(d) + return defer.gatherResults(deferreds) + + d = self.all_uid_iter() + d.addCallback(del_all_uid) + return d + + def update_flags(self, msg, flags, mode): + """ + Update flags for a given message. + """ + wrapper = msg.get_wrapper() + current = wrapper.fdoc.flags + newflags = map(str, self._update_flags_or_tags(current, flags, mode)) + wrapper.fdoc.flags = newflags + + wrapper.fdoc.seen = MessageFlags.SEEN_FLAG in newflags + wrapper.fdoc.deleted = MessageFlags.DELETED_FLAG in newflags + + d = self.adaptor.update_msg(self.store, msg) + d.addCallback(lambda _: newflags) + return d + + def update_tags(self, msg, tags, mode): + """ + Update tags for a given message. + """ + wrapper = msg.get_wrapper() + current = wrapper.fdoc.tags + newtags = self._update_flags_or_tags(current, tags, mode) + + wrapper.fdoc.tags = newtags + d = self.adaptor.update_msg(self.store, msg) + d.addCallback(newtags) + return d + + def _update_flags_or_tags(self, old, new, mode): + if mode == Flagsmode.APPEND: + final = list((set(tuple(old) + new))) + elif mode == Flagsmode.REMOVE: + final = list(set(old).difference(set(new))) + elif mode == Flagsmode.SET: + final = new + return final + + +class Account(object): + """ + Account is the top level abstraction to access collections of messages + associated with a LEAP Mail Account. + + It primarily handles creation and access of Mailboxes, which will be the + basic collection handled by traditional MUAs, but it can also handle other + types of Collections (tag based, for instance). + + leap.mail.imap.IMAPAccount partially proxies methods in this + class. + """ + + # Adaptor is passed to the returned MessageCollections, so if you want to + # use a different adaptor this is the place to change it, by subclassing + # the Account class. + + adaptor_class = SoledadMailAdaptor + + # this is a defaultdict, indexed by userid, that returns a + # WeakValueDictionary mapping to collection instances so that we always + # return a reference to them instead of creating new ones. however, + # being a dictionary of weakrefs values, they automagically vanish + # from the dict when no hard refs is left to them (so they can be + # garbage collected) this is important because the different wrappers + # rely on several kinds of deferredlocks that are kept as class or + # instance variables. + + # We need it to be a class property because we create more than one Account + # object in the current usage pattern (ie, one in the mail service, and + # another one in the IncomingMailService). When we move to a proper service + # tree we can let it be an instance attribute. + _collection_mapping = defaultdict(weakref.WeakValueDictionary) + + def __init__(self, store, user_id, ready_cb=None): + self.store = store + self.user_id = user_id + self.adaptor = self.adaptor_class() + + self.mbox_indexer = MailboxIndexer(self.store) + + # This flag is only used from the imap service for the moment. + # In the future, we should prevent any public method to continue if + # this is set to True. Also, it would be good to plug to the + # authentication layer. + self.session_ended = False + + self.deferred_initialization = defer.Deferred() + self._ready_cb = ready_cb + + self._init_d = self._initialize_storage() + self._initialize_sync_hooks() + + def _initialize_storage(self): + + def add_mailbox_if_none(mboxes): + if not mboxes: + return self.add_mailbox(INBOX_NAME) + + def finish_initialization(result): + self.deferred_initialization.callback(None) + if self._ready_cb is not None: + self._ready_cb() + + d = self.adaptor.initialize_store(self.store) + d.addCallback(lambda _: self.list_all_mailbox_names()) + d.addCallback(add_mailbox_if_none) + d.addCallback(finish_initialization) + return d + + def callWhenReady(self, cb, *args, **kw): + """ + Execute the callback when the initialization of the Account is ready. + Note that the callback will receive a first meaningless parameter. + """ + # TODO this should ignore the first parameter explicitely + # lambda _: cb(*args, **kw) + self.deferred_initialization.addCallback(cb, *args, **kw) + return self.deferred_initialization + + # Sync hooks + + def _initialize_sync_hooks(self): + soledad_sync_hooks.post_sync_uid_reindexer.set_account(self) + + def _teardown_sync_hooks(self): + soledad_sync_hooks.post_sync_uid_reindexer.set_account(None) + + # + # Public API Starts + # + + def list_all_mailbox_names(self): + + def filter_names(mboxes): + return [m.mbox for m in mboxes] + + d = self.get_all_mailboxes() + d.addCallback(filter_names) + return d + + def get_all_mailboxes(self): + d = self.adaptor.get_all_mboxes(self.store) + return d + + def add_mailbox(self, name, creation_ts=None): + + if creation_ts is None: + # by default, we pass an int value + # taken from the current time + # we make sure to take enough decimals to get a unique + # mailbox-uidvalidity. + creation_ts = int(time.time() * 10E2) + + def set_creation_ts(wrapper): + wrapper.created = creation_ts + d = wrapper.update(self.store) + d.addCallback(lambda _: wrapper) + return d + + def create_uuid(wrapper): + if not wrapper.uuid: + wrapper.uuid = str(uuid.uuid4()) + d = wrapper.update(self.store) + d.addCallback(lambda _: wrapper) + return d + return wrapper + + def create_uid_table_cb(wrapper): + d = self.mbox_indexer.create_table(wrapper.uuid) + d.addCallback(lambda _: wrapper) + return d + + d = self.adaptor.get_or_create_mbox(self.store, name) + d.addCallback(set_creation_ts) + d.addCallback(create_uuid) + d.addCallback(create_uid_table_cb) + return d + + def delete_mailbox(self, name): + + def delete_uid_table_cb(wrapper): + d = self.mbox_indexer.delete_table(wrapper.uuid) + d.addCallback(lambda _: wrapper) + return d + + d = self.adaptor.get_or_create_mbox(self.store, name) + d.addCallback(delete_uid_table_cb) + d.addCallback( + lambda wrapper: self.adaptor.delete_mbox(self.store, wrapper)) + return d + + def rename_mailbox(self, oldname, newname): + + def _rename_mbox(wrapper): + wrapper.mbox = newname + d = wrapper.update(self.store) + d.addCallback(lambda result: wrapper) + return d + + d = self.adaptor.get_or_create_mbox(self.store, oldname) + d.addCallback(_rename_mbox) + return d + + # Get Collections + + def get_collection_by_mailbox(self, name): + """ + :rtype: deferred + :return: a deferred that will fire with a MessageCollection + """ + collection = self._collection_mapping[self.user_id].get( + name, None) + if collection: + return defer.succeed(collection) + + # imap select will use this, passing the collection to SoledadMailbox + def get_collection_for_mailbox(mbox_wrapper): + collection = MessageCollection( + self.adaptor, self.store, self.mbox_indexer, mbox_wrapper) + self._collection_mapping[self.user_id][name] = collection + return collection + + d = self.adaptor.get_or_create_mbox(self.store, name) + d.addCallback(get_collection_for_mailbox) + return d + + def get_collection_by_docs(self, docs): + """ + :rtype: MessageCollection + """ + # get a collection of docs by a list of doc_id + # get.docs(...) --> it should be a generator. does it behave in the + # threadpool? + raise NotImplementedError() + + def get_collection_by_tag(self, tag): + """ + :rtype: MessageCollection + """ + raise NotImplementedError() + + # Session handling + + def end_session(self): + self._teardown_sync_hooks() + self.session_ended = True diff --git a/src/leap/bitmask/mail/mailbox_indexer.py b/src/leap/bitmask/mail/mailbox_indexer.py new file mode 100644 index 0000000..c49f808 --- /dev/null +++ b/src/leap/bitmask/mail/mailbox_indexer.py @@ -0,0 +1,327 @@ +# -*- coding: utf-8 -*- +# mailbox_indexer.py +# Copyright (C) 2014 LEAP +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . +""" +.. :py:module::mailbox_indexer + +Local tables to store the message Unique Identifiers for a given mailbox. +""" +import re +import uuid + +from leap.mail.constants import METAMSGID_RE + + +def _maybe_first_query_item(thing): + """ + Return the first item the returned query result, or None + if empty. + """ + try: + return thing[0][0] + except (TypeError, IndexError): + return None + + +class WrongMetaDocIDError(Exception): + pass + + +def sanitize(mailbox_uuid): + return mailbox_uuid.replace("-", "_") + + +def check_good_uuid(mailbox_uuid): + """ + Check that the passed mailbox identifier is a valid UUID. + :param mailbox_uuid: the uuid to check + :type mailbox_uuid: str + :return: None + :raises: AssertionError if a wrong uuid was passed. + """ + try: + uuid.UUID(str(mailbox_uuid)) + except (AttributeError, ValueError): + raise AssertionError( + "the mbox_id is not a valid uuid: %s" % mailbox_uuid) + + +class MailboxIndexer(object): + """ + This class contains the commands needed to create, modify and alter the + local-only UID tables for a given mailbox. + + Its purpouse is to keep a local-only index with the messages in each + mailbox, mainly to satisfy the demands of the IMAP specification, but + useful too for any effective listing of the messages in a mailbox. + + Since the incoming mail can be processed at any time in any replica, it's + preferred not to attempt to maintain a global chronological global index. + + These indexes are Message Attributes needed for the IMAP specification (rfc + 3501), although they can be useful for other non-imap store + implementations. + + """ + # The uids are expected to be 32-bits values, but the ROWIDs in sqlite + # are 64-bit values. I *don't* think it really matters for any + # practical use, but it's good to remember we've got that difference going + # on. + + store = None + table_preffix = "leapmail_uid_" + + def __init__(self, store): + self.store = store + + def _query(self, *args, **kw): + assert self.store is not None + return self.store.raw_sqlcipher_query(*args, **kw) + + def _operation(self, *args, **kw): + assert self.store is not None + return self.store.raw_sqlcipher_operation(*args, **kw) + + def create_table(self, mailbox_uuid): + """ + Create the UID table for a given mailbox. + :param mailbox: the mailbox identifier. + :type mailbox: str + :rtype: Deferred + """ + check_good_uuid(mailbox_uuid) + sql = ("CREATE TABLE if not exists {preffix}{name}( " + "uid INTEGER PRIMARY KEY AUTOINCREMENT, " + "hash TEXT UNIQUE NOT NULL)".format( + preffix=self.table_preffix, name=sanitize(mailbox_uuid))) + return self._operation(sql) + + def delete_table(self, mailbox_uuid): + """ + Delete the UID table for a given mailbox. + :param mailbox: the mailbox name + :type mailbox: str + :rtype: Deferred + """ + check_good_uuid(mailbox_uuid) + sql = ("DROP TABLE if exists {preffix}{name}".format( + preffix=self.table_preffix, name=sanitize(mailbox_uuid))) + return self._operation(sql) + + def insert_doc(self, mailbox_uuid, doc_id): + """ + Insert the doc_id for a MetaMsg in the UID table for a given mailbox. + + The doc_id must be in the format: + + M-- + + :param mailbox: the mailbox name + :type mailbox: str + :param doc_id: the doc_id for the MetaMsg + :type doc_id: str + :return: a deferred that will fire with the uid of the newly inserted + document. + :rtype: Deferred + """ + check_good_uuid(mailbox_uuid) + assert doc_id + mailbox_uuid = mailbox_uuid.replace('-', '_') + + if not re.findall(METAMSGID_RE.format(mbox_uuid=mailbox_uuid), doc_id): + raise WrongMetaDocIDError("Wrong format for the MetaMsg doc_id") + + def get_rowid(result): + return _maybe_first_query_item(result) + + sql = ("INSERT INTO {preffix}{name} VALUES (" + "NULL, ?)".format( + preffix=self.table_preffix, name=sanitize(mailbox_uuid))) + values = (doc_id,) + + sql_last = ("SELECT MAX(rowid) FROM {preffix}{name} " + "LIMIT 1;").format( + preffix=self.table_preffix, name=sanitize(mailbox_uuid)) + + d = self._operation(sql, values) + d.addCallback(lambda _: self._query(sql_last)) + d.addCallback(get_rowid) + d.addErrback(lambda f: f.printTraceback()) + return d + + def delete_doc_by_uid(self, mailbox_uuid, uid): + """ + Delete the entry for a MetaMsg in the UID table for a given mailbox. + + :param mailbox_uuid: the mailbox uuid + :type mailbox: str + :param uid: the UID of the message. + :type uid: int + :rtype: Deferred + """ + check_good_uuid(mailbox_uuid) + assert uid + sql = ("DELETE FROM {preffix}{name} " + "WHERE uid=?".format( + preffix=self.table_preffix, name=sanitize(mailbox_uuid))) + values = (uid,) + return self._query(sql, values) + + def delete_doc_by_hash(self, mailbox_uuid, doc_id): + """ + Delete the entry for a MetaMsg in the UID table for a given mailbox. + + The doc_id must be in the format: + + M-- + + :param mailbox_uuid: the mailbox uuid + :type mailbox: str + :param doc_id: the doc_id for the MetaMsg + :type doc_id: str + :return: a deferred that will fire when the deletion has succed. + :rtype: Deferred + """ + check_good_uuid(mailbox_uuid) + assert doc_id + sql = ("DELETE FROM {preffix}{name} " + "WHERE hash=?".format( + preffix=self.table_preffix, name=sanitize(mailbox_uuid))) + values = (doc_id,) + return self._query(sql, values) + + def get_doc_id_from_uid(self, mailbox_uuid, uid): + """ + Get the doc_id for a MetaMsg in the UID table for a given mailbox. + + :param mailbox_uuid: the mailbox uuid + :type mailbox: str + :param uid: the uid for the MetaMsg for this mailbox + :type uid: int + :rtype: Deferred + """ + check_good_uuid(mailbox_uuid) + mailbox_uuid = mailbox_uuid.replace('-', '_') + + def get_hash(result): + return _maybe_first_query_item(result) + + sql = ("SELECT hash from {preffix}{name} " + "WHERE uid=?".format( + preffix=self.table_preffix, name=sanitize(mailbox_uuid))) + values = (uid,) + d = self._query(sql, values) + d.addCallback(get_hash) + return d + + def get_uid_from_doc_id(self, mailbox_uuid, doc_id): + check_good_uuid(mailbox_uuid) + mailbox_uuid = mailbox_uuid.replace('-', '_') + + def get_uid(result): + return _maybe_first_query_item(result) + + sql = ("SELECT uid from {preffix}{name} " + "WHERE hash=?".format( + preffix=self.table_preffix, name=sanitize(mailbox_uuid))) + values = (doc_id,) + d = self._query(sql, values) + d.addCallback(get_uid) + return d + + def get_doc_ids_from_uids(self, mailbox_uuid, uids): + # For IMAP relative numbering /sequences. + # XXX dereference the range (n,*) + raise NotImplementedError() + + def count(self, mailbox_uuid): + """ + Get the number of entries in the UID table for a given mailbox. + + :param mailbox_uuid: the mailbox uuid + :type mailbox_uuid: str + :return: a deferred that will fire with an integer returning the count. + :rtype: Deferred + """ + check_good_uuid(mailbox_uuid) + + def get_count(result): + return _maybe_first_query_item(result) + + sql = ("SELECT Count(*) FROM {preffix}{name};".format( + preffix=self.table_preffix, name=sanitize(mailbox_uuid))) + d = self._query(sql) + d.addCallback(get_count) + d.addErrback(lambda _: 0) + return d + + def get_next_uid(self, mailbox_uuid): + """ + Get the next integer beyond the highest UID count for a given mailbox. + + This is expected by the IMAP implementation. There are no guarantees + that a document to be inserted in the future gets the returned UID: the + only thing that can be assured is that it will be equal or greater than + the value returned. + + :param mailbox_uuid: the mailbox uuid + :type mailbox: str + :return: a deferred that will fire with an integer returning the next + uid. + :rtype: Deferred + """ + check_good_uuid(mailbox_uuid) + d = self.get_last_uid(mailbox_uuid) + d.addCallback(lambda uid: uid + 1) + return d + + def get_last_uid(self, mailbox_uuid): + """ + Get the highest UID for a given mailbox. + """ + check_good_uuid(mailbox_uuid) + sql = ("SELECT MAX(rowid) FROM {preffix}{name} " + "LIMIT 1;").format( + preffix=self.table_preffix, name=sanitize(mailbox_uuid)) + + def getit(result): + rowid = _maybe_first_query_item(result) + if not rowid: + rowid = 0 + return rowid + + d = self._query(sql) + d.addCallback(getit) + return d + + def all_uid_iter(self, mailbox_uuid): + """ + Get a sequence of all the uids in this mailbox. + + :param mailbox_uuid: the mailbox uuid + :type mailbox_uuid: str + """ + check_good_uuid(mailbox_uuid) + + sql = ("SELECT uid from {preffix}{name} ").format( + preffix=self.table_preffix, name=sanitize(mailbox_uuid)) + + def get_results(result): + return [x[0] for x in result] + + d = self._query(sql) + d.addCallback(get_results) + return d diff --git a/src/leap/bitmask/mail/outgoing/__init__.py b/src/leap/bitmask/mail/outgoing/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/src/leap/bitmask/mail/outgoing/service.py b/src/leap/bitmask/mail/outgoing/service.py new file mode 100644 index 0000000..8b02f2e --- /dev/null +++ b/src/leap/bitmask/mail/outgoing/service.py @@ -0,0 +1,518 @@ +# -*- coding: utf-8 -*- +# outgoing/service.py +# Copyright (C) 2013-2015 LEAP +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . + +""" +OutgoingMail module. + +The OutgoingMail class allows to send mail, and encrypts/signs it if needed. +""" + +import os.path +import re +from StringIO import StringIO +from copy import deepcopy +from email.parser import Parser +from email.mime.application import MIMEApplication +from email.mime.multipart import MIMEMultipart +from email.mime.text import MIMEText + +from OpenSSL import SSL + +from twisted.mail import smtp +from twisted.internet import reactor +from twisted.internet import defer +from twisted.protocols.amp import ssl +from twisted.python import log + +from leap.common.check import leap_assert_type, leap_assert +from leap.common.events import emit_async, catalog +from leap.keymanager.errors import KeyNotFound, KeyAddressMismatch +from leap.mail import __version__ +from leap.mail import errors +from leap.mail.utils import validate_address +from leap.mail.rfc3156 import MultipartEncrypted +from leap.mail.rfc3156 import MultipartSigned +from leap.mail.rfc3156 import encode_base64_rec +from leap.mail.rfc3156 import RFC3156CompliantGenerator +from leap.mail.rfc3156 import PGPSignature +from leap.mail.rfc3156 import PGPEncrypted + +# TODO +# [ ] rename this module to something else, service should be the implementor +# of IService + + +class SSLContextFactory(ssl.ClientContextFactory): + def __init__(self, cert, key): + self.cert = cert + self.key = key + + def getContext(self): + # FIXME -- we should use sslv23 to allow for tlsv1.2 + # and, if possible, explicitely disable sslv3 clientside. + # Servers should avoid sslv3 + self.method = SSL.TLSv1_METHOD # SSLv23_METHOD + ctx = ssl.ClientContextFactory.getContext(self) + ctx.use_certificate_file(self.cert) + ctx.use_privatekey_file(self.key) + return ctx + + +def outgoingFactory(userid, keymanager, opts, check_cert=True, bouncer=None): + + cert = unicode(opts.cert) + key = unicode(opts.key) + hostname = str(opts.hostname) + port = opts.port + + if check_cert: + if not os.path.isfile(cert): + raise errors.ConfigurationError( + 'No valid SMTP certificate could be found for %s!' % userid) + + return OutgoingMail( + str(userid), keymanager, cert, key, hostname, port, + bouncer) + + +class OutgoingMail(object): + """ + Sends Outgoing Mail, encrypting and signing if needed. + """ + + def __init__(self, from_address, keymanager, cert, key, host, port, + bouncer=None): + """ + Initialize the outgoing mail service. + + :param from_address: The sender address. + :type from_address: str + :param keymanager: A KeyManager for retrieving recipient's keys. + :type keymanager: leap.common.keymanager.KeyManager + :param cert: The client certificate for SSL authentication. + :type cert: str + :param key: The client private key for SSL authentication. + :type key: str + :param host: The hostname of the remote SMTP server. + :type host: str + :param port: The port of the remote SMTP server. + :type port: int + """ + + # assert params + leap_assert_type(from_address, str) + leap_assert('@' in from_address) + + # XXX it can be a zope.proxy too + # leap_assert_type(keymanager, KeyManager) + + leap_assert_type(host, str) + leap_assert(host != '') + leap_assert_type(port, int) + leap_assert(port is not 0) + leap_assert_type(cert, unicode) + leap_assert(cert != '') + leap_assert_type(key, unicode) + leap_assert(key != '') + + self._port = port + self._host = host + self._key = key + self._cert = cert + self._from_address = from_address + self._keymanager = keymanager + self._bouncer = bouncer + + def send_message(self, raw, recipient): + """ + Sends a message to a recipient. Maybe encrypts and signs. + + :param raw: The raw message + :type raw: str + :param recipient: The recipient for the message + :type recipient: smtp.User + :return: a deferred which delivers the message when fired + """ + d = self._maybe_encrypt_and_sign(raw, recipient) + d.addCallback(self._route_msg, raw) + d.addErrback(self.sendError, raw) + return d + + def sendSuccess(self, smtp_sender_result): + """ + Callback for a successful send. + + :param smtp_sender_result: The result from the ESMTPSender from + _route_msg + :type smtp_sender_result: tuple(int, list(tuple)) + """ + dest_addrstr = smtp_sender_result[1][0][0] + fromaddr = self._from_address + log.msg('Message sent from %s to %s' % (fromaddr, dest_addrstr)) + emit_async(catalog.SMTP_SEND_MESSAGE_SUCCESS, + fromaddr, dest_addrstr) + + def sendError(self, failure, origmsg): + """ + Callback for an unsuccessfull send. + + :param failure: The result from the last errback. + :type failure: anything + :param origmsg: the original, unencrypted, raw message, to be passed to + the bouncer. + :type origmsg: str + """ + # XXX: need to get the address from the original message to send signal + # emit_async(catalog.SMTP_SEND_MESSAGE_ERROR, self._from_address, + # self._user.dest.addrstr) + + # TODO when we implement outgoing queues/long-term-retries, we could + # examine the error *here* and delay the notification if it's just a + # temporal error. We might want to notify the permanent errors + # differently. + + err = failure.value + log.err(err) + + if self._bouncer: + self._bouncer.bounce_message( + err.message, to=self._from_address, + orig=origmsg) + else: + raise err + + def _route_msg(self, encrypt_and_sign_result, raw): + """ + Sends the msg using the ESMTPSenderFactory. + + :param encrypt_and_sign_result: A tuple containing the 'maybe' + encrypted message and the recipient + :type encrypt_and_sign_result: tuple + """ + message, recipient = encrypt_and_sign_result + log.msg("Connecting to SMTP server %s:%s" % (self._host, self._port)) + msg = message.as_string(False) + + # we construct a defer to pass to the ESMTPSenderFactory + d = defer.Deferred() + d.addCallback(self.sendSuccess) + d.addErrback(self.sendError, raw) + # we don't pass an ssl context factory to the ESMTPSenderFactory + # because ssl will be handled by reactor.connectSSL() below. + factory = smtp.ESMTPSenderFactory( + "", # username is blank, no client auth here + "", # password is blank, no client auth here + self._from_address, + recipient.dest.addrstr, + StringIO(msg), + d, + heloFallback=True, + requireAuthentication=False, + requireTransportSecurity=True) + factory.domain = bytes('leap.mail-' + __version__) + emit_async(catalog.SMTP_SEND_MESSAGE_START, + self._from_address, recipient.dest.addrstr) + reactor.connectSSL( + self._host, self._port, factory, + contextFactory=SSLContextFactory(self._cert, self._key)) + + def _maybe_encrypt_and_sign(self, raw, recipient, fetch_remote=True): + """ + Attempt to encrypt and sign the outgoing message. + + The behaviour of this method depends on: + + 1. the original message's content-type, and + 2. the availability of the recipient's public key. + + If the original message's content-type is "multipart/encrypted", then + the original message is not altered. For any other content-type, the + method attempts to fetch the recipient's public key. If the + recipient's public key is available, the message is encrypted and + signed; otherwise it is only signed. + + Note that, if the C{encrypted_only} configuration is set to True and + the recipient's public key is not available, then the recipient + address would have been rejected in SMTPDelivery.validateTo(). + + The following table summarizes the overall behaviour of the gateway: + + +---------------------------------------------------+----------------+ + | content-type | rcpt pubkey | enforce encr. | action | + +---------------------+-------------+---------------+----------------+ + | multipart/encrypted | any | any | pass | + | other | available | any | encrypt + sign | + | other | unavailable | yes | reject | + | other | unavailable | no | sign | + +---------------------+-------------+---------------+----------------+ + + :param raw: The raw message + :type raw: str + :param recipient: The recipient for the message + :type: recipient: smtp.User + + :return: A Deferred that will be fired with a MIMEMultipart message + and the original recipient Message + :rtype: Deferred + """ + # pass if the original message's content-type is "multipart/encrypted" + origmsg = Parser().parsestr(raw) + + if origmsg.get_content_type() == 'multipart/encrypted': + return defer.succeed((origmsg, recipient)) + + from_address = validate_address(self._from_address) + username, domain = from_address.split('@') + to_address = validate_address(recipient.dest.addrstr) + + def maybe_encrypt_and_sign(message): + d = self._encrypt_and_sign( + message, to_address, from_address, + fetch_remote=fetch_remote) + d.addCallbacks(signal_encrypt_sign, + if_key_not_found_send_unencrypted, + errbackArgs=(message,)) + return d + + def signal_encrypt_sign(newmsg): + emit_async(catalog.SMTP_END_ENCRYPT_AND_SIGN, + self._from_address, + "%s,%s" % (self._from_address, to_address)) + return newmsg, recipient + + def if_key_not_found_send_unencrypted(failure, message): + failure.trap(KeyNotFound, KeyAddressMismatch) + + log.msg('Will send unencrypted message to %s.' % to_address) + emit_async(catalog.SMTP_START_SIGN, self._from_address, to_address) + d = self._sign(message, from_address) + d.addCallback(signal_sign) + return d + + def signal_sign(newmsg): + emit_async(catalog.SMTP_END_SIGN, self._from_address) + return newmsg, recipient + + log.msg("Will encrypt the message with %s and sign with %s." + % (to_address, from_address)) + emit_async(catalog.SMTP_START_ENCRYPT_AND_SIGN, + self._from_address, + "%s,%s" % (self._from_address, to_address)) + d = self._maybe_attach_key(origmsg, from_address, to_address) + d.addCallback(maybe_encrypt_and_sign) + return d + + def _maybe_attach_key(self, origmsg, from_address, to_address): + filename = "%s-email-key.asc" % (from_address,) + + def attach_if_address_hasnt_encrypted(to_key): + # if the sign_used flag is true that means that we got an encrypted + # email from this address, because we conly check signatures on + # encrypted emails. In this case we don't attach. + # XXX: this might not be true some time in the future + if to_key.sign_used: + return origmsg + return get_key_and_attach(None) + + def get_key_and_attach(_): + d = self._keymanager.get_key(from_address, fetch_remote=False) + d.addCallback(attach_key) + return d + + def attach_key(from_key): + msg = origmsg + if not origmsg.is_multipart(): + msg = MIMEMultipart() + for h, v in origmsg.items(): + msg.add_header(h, v) + msg.attach(MIMEText(origmsg.get_payload())) + + keymsg = MIMEApplication(from_key.key_data, _subtype='pgp-keys', + _encoder=lambda x: x) + keymsg.add_header('content-disposition', 'attachment', + filename=filename) + msg.attach(keymsg) + return msg + + d = self._keymanager.get_key(to_address, fetch_remote=False) + d.addCallbacks(attach_if_address_hasnt_encrypted, get_key_and_attach) + d.addErrback(lambda _: origmsg) + return d + + def _encrypt_and_sign(self, origmsg, encrypt_address, sign_address, + fetch_remote=True): + """ + Create an RFC 3156 compliang PGP encrypted and signed message using + C{encrypt_address} to encrypt and C{sign_address} to sign. + + :param origmsg: The original message + :type origmsg: email.message.Message + :param encrypt_address: The address used to encrypt the message. + :type encrypt_address: str + :param sign_address: The address used to sign the message. + :type sign_address: str + + :return: A Deferred with the MultipartEncrypted message + :rtype: Deferred + """ + # create new multipart/encrypted message with 'pgp-encrypted' protocol + + def encrypt(res): + newmsg, origmsg = res + d = self._keymanager.encrypt( + origmsg.as_string(unixfrom=False), + encrypt_address, sign=sign_address, + fetch_remote=fetch_remote) + d.addCallback(lambda encstr: (newmsg, encstr)) + return d + + def create_encrypted_message(res): + newmsg, encstr = res + encmsg = MIMEApplication( + encstr, _subtype='octet-stream', _encoder=lambda x: x) + encmsg.add_header('content-disposition', 'attachment', + filename='msg.asc') + # create meta message + metamsg = PGPEncrypted() + metamsg.add_header('Content-Disposition', 'attachment') + # attach pgp message parts to new message + newmsg.attach(metamsg) + newmsg.attach(encmsg) + return newmsg + + d = self._fix_headers( + origmsg, + MultipartEncrypted('application/pgp-encrypted'), + sign_address) + d.addCallback(encrypt) + d.addCallback(create_encrypted_message) + return d + + def _sign(self, origmsg, sign_address): + """ + Create an RFC 3156 compliant PGP signed MIME message using + C{sign_address}. + + :param origmsg: The original message + :type origmsg: email.message.Message + :param sign_address: The address used to sign the message. + :type sign_address: str + + :return: A Deferred with the MultipartSigned message. + :rtype: Deferred + """ + # apply base64 content-transfer-encoding + encode_base64_rec(origmsg) + # get message text with headers and replace \n for \r\n + fp = StringIO() + g = RFC3156CompliantGenerator( + fp, mangle_from_=False, maxheaderlen=76) + g.flatten(origmsg) + msgtext = re.sub('\r?\n', '\r\n', fp.getvalue()) + # make sure signed message ends with \r\n as per OpenPGP stantard. + if origmsg.is_multipart(): + if not msgtext.endswith("\r\n"): + msgtext += "\r\n" + + def create_signed_message(res): + (msg, _), signature = res + sigmsg = PGPSignature(signature) + # attach original message and signature to new message + msg.attach(origmsg) + msg.attach(sigmsg) + return msg + + dh = self._fix_headers( + origmsg, + MultipartSigned('application/pgp-signature', 'pgp-sha512'), + sign_address) + ds = self._keymanager.sign( + msgtext, sign_address, digest_algo='SHA512', + clearsign=False, detach=True, binary=False) + d = defer.gatherResults([dh, ds]) + d.addCallback(create_signed_message) + return d + + def _fix_headers(self, msg, newmsg, sign_address): + """ + Move some headers from C{origmsg} to C{newmsg}, delete unwanted + headers from C{origmsg} and add new headers to C{newms}. + + Outgoing messages are either encrypted and signed or just signed + before being sent. Because of that, they are packed inside new + messages and some manipulation has to be made on their headers. + + Allowed headers for passing through: + + - From + - Date + - To + - Subject + - Reply-To + - References + - In-Reply-To + - Cc + + Headers to be added: + + - Message-ID (i.e. should not use origmsg's Message-Id) + - Received (this is added automatically by twisted smtp API) + - OpenPGP (see #4447) + + Headers to be deleted: + + - User-Agent + + :param msg: The original message. + :type msg: email.message.Message + :param newmsg: The new message being created. + :type newmsg: email.message.Message + :param sign_address: The address used to sign C{newmsg} + :type sign_address: str + + :return: A Deferred with a touple: + (new Message with the unencrypted headers, + original Message with headers removed) + :rtype: Deferred + """ + origmsg = deepcopy(msg) + # move headers from origmsg to newmsg + headers = origmsg.items() + passthrough = [ + 'from', 'date', 'to', 'subject', 'reply-to', 'references', + 'in-reply-to', 'cc' + ] + headers = filter(lambda x: x[0].lower() in passthrough, headers) + for hkey, hval in headers: + newmsg.add_header(hkey, hval) + del (origmsg[hkey]) + # add a new message-id to newmsg + newmsg.add_header('Message-Id', smtp.messageid()) + # delete user-agent from origmsg + del (origmsg['user-agent']) + + def add_openpgp_header(signkey): + username, domain = sign_address.split('@') + newmsg.add_header( + 'OpenPGP', 'id=%s' % signkey.fingerprint, + url='https://%s/key/%s' % (domain, username), + preference='signencrypt') + return newmsg, origmsg + + d = self._keymanager.get_key(sign_address, private=True) + d.addCallback(add_openpgp_header) + return d diff --git a/src/leap/bitmask/mail/outgoing/tests/test_outgoing.py b/src/leap/bitmask/mail/outgoing/tests/test_outgoing.py new file mode 100644 index 0000000..dd053c1 --- /dev/null +++ b/src/leap/bitmask/mail/outgoing/tests/test_outgoing.py @@ -0,0 +1,263 @@ +# -*- coding: utf-8 -*- +# test_gateway.py +# Copyright (C) 2013 LEAP +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . + + +""" +SMTP gateway tests. +""" +import re +from copy import deepcopy +from StringIO import StringIO +from email.parser import Parser +from datetime import datetime +from twisted.internet.defer import fail +from twisted.mail.smtp import User +from twisted.python import log + +from mock import Mock + +from leap.mail.rfc3156 import RFC3156CompliantGenerator +from leap.mail.outgoing.service import OutgoingMail +from leap.mail.testing import ADDRESS, ADDRESS_2, PUBLIC_KEY_2 +from leap.mail.testing import KeyManagerWithSoledadTestCase +from leap.mail.testing.smtp import getSMTPFactory +from leap.keymanager import errors + + +BEGIN_PUBLIC_KEY = "-----BEGIN PGP PUBLIC KEY BLOCK-----" + +TEST_USER = u'anotheruser@leap.se' + + +class TestOutgoingMail(KeyManagerWithSoledadTestCase): + EMAIL_DATA = ['HELO gateway.leap.se', + 'MAIL FROM: <%s>' % ADDRESS_2, + 'RCPT TO: <%s>' % ADDRESS, + 'DATA', + 'From: User <%s>' % ADDRESS_2, + 'To: Leap <%s>' % ADDRESS, + 'Date: ' + datetime.now().strftime('%c'), + 'Subject: test message', + '', + 'This is a secret message.', + 'Yours,', + 'A.', + '', + '.', + 'QUIT'] + + def setUp(self): + self.lines = [line for line in self.EMAIL_DATA[4:12]] + self.lines.append('') # add a trailing newline + self.raw = '\r\n'.join(self.lines) + self.expected_body = '\r\n'.join(self.EMAIL_DATA[9:12]) + "\r\n" + self.fromAddr = ADDRESS_2 + + class opts: + cert = u'/tmp/cert' + key = u'/tmp/cert' + hostname = 'remote' + port = 666 + self.opts = opts + + def init_outgoing_and_proto(_): + self.outgoing_mail = OutgoingMail( + self.fromAddr, self.km, opts.cert, + opts.key, opts.hostname, opts.port) + + user = TEST_USER + + # TODO -- this shouldn't need SMTP to be tested!? or does it? + self.proto = getSMTPFactory( + {user: None}, {user: self.km}, {user: None}) + self.dest = User(ADDRESS, 'gateway.leap.se', self.proto, ADDRESS_2) + + d = KeyManagerWithSoledadTestCase.setUp(self) + d.addCallback(init_outgoing_and_proto) + return d + + def test_message_encrypt(self): + """ + Test if message gets encrypted to destination email. + """ + def check_decryption(res): + decrypted, _ = res + self.assertEqual( + '\n' + self.expected_body, + decrypted, + 'Decrypted text differs from plaintext.') + + d = self._set_sign_used(ADDRESS) + d.addCallback( + lambda _: + self.outgoing_mail._maybe_encrypt_and_sign(self.raw, self.dest)) + d.addCallback(self._assert_encrypted) + d.addCallback(lambda message: self.km.decrypt( + message.get_payload(1).get_payload(), ADDRESS)) + d.addCallback(check_decryption) + return d + + def test_message_encrypt_sign(self): + """ + Test if message gets encrypted to destination email and signed with + sender key. + '""" + def check_decryption_and_verify(res): + decrypted, signkey = res + self.assertEqual( + '\n' + self.expected_body, + decrypted, + 'Decrypted text differs from plaintext.') + self.assertTrue(ADDRESS_2 in signkey.address, + "Verification failed") + + d = self._set_sign_used(ADDRESS) + d.addCallback( + lambda _: + self.outgoing_mail._maybe_encrypt_and_sign(self.raw, self.dest)) + d.addCallback(self._assert_encrypted) + d.addCallback(lambda message: self.km.decrypt( + message.get_payload(1).get_payload(), ADDRESS, verify=ADDRESS_2)) + d.addCallback(check_decryption_and_verify) + return d + + def test_message_sign(self): + """ + Test if message is signed with sender key. + """ + # mock the key fetching + self.km._fetch_keys_from_server = Mock( + return_value=fail(errors.KeyNotFound())) + recipient = User('ihavenopubkey@nonleap.se', + 'gateway.leap.se', self.proto, ADDRESS) + self.outgoing_mail = OutgoingMail( + self.fromAddr, self.km, self.opts.cert, self.opts.key, + self.opts.hostname, self.opts.port) + + def check_signed(res): + message, _ = res + self.assertTrue('Content-Type' in message) + self.assertEqual('multipart/signed', message.get_content_type()) + self.assertEqual('application/pgp-signature', + message.get_param('protocol')) + self.assertEqual('pgp-sha512', message.get_param('micalg')) + # assert content of message + body = (message.get_payload(0) + .get_payload(0) + .get_payload(decode=True)) + self.assertEqual(self.expected_body, + body) + # assert content of signature + self.assertTrue( + message.get_payload(1).get_payload().startswith( + '-----BEGIN PGP SIGNATURE-----\n'), + 'Message does not start with signature header.') + self.assertTrue( + message.get_payload(1).get_payload().endswith( + '-----END PGP SIGNATURE-----\n'), + 'Message does not end with signature footer.') + return message + + def verify(message): + # replace EOL before verifying (according to rfc3156) + fp = StringIO() + g = RFC3156CompliantGenerator( + fp, mangle_from_=False, maxheaderlen=76) + g.flatten(message.get_payload(0)) + signed_text = re.sub('\r?\n', '\r\n', + fp.getvalue()) + + def assert_verify(key): + self.assertTrue(ADDRESS_2 in key.address, + 'Signature could not be verified.') + + d = self.km.verify( + signed_text, ADDRESS_2, + detached_sig=message.get_payload(1).get_payload()) + d.addCallback(assert_verify) + return d + + d = self.outgoing_mail._maybe_encrypt_and_sign(self.raw, recipient) + d.addCallback(check_signed) + d.addCallback(verify) + return d + + def test_attach_key(self): + d = self.outgoing_mail._maybe_encrypt_and_sign(self.raw, self.dest) + d.addCallback(self._assert_encrypted) + d.addCallback(self._check_headers, self.lines[:4]) + d.addCallback(lambda message: self.km.decrypt( + message.get_payload(1).get_payload(), ADDRESS)) + d.addCallback(lambda (decrypted, _): + self._check_key_attachment(Parser().parsestr(decrypted))) + return d + + def test_attach_key_not_known(self): + unknown_address = "someunknownaddress@somewhere.com" + lines = deepcopy(self.lines) + lines[1] = "To: <%s>" % (unknown_address,) + raw = '\r\n'.join(lines) + dest = User(unknown_address, 'gateway.leap.se', self.proto, ADDRESS_2) + + d = self.outgoing_mail._maybe_encrypt_and_sign( + raw, dest, fetch_remote=False) + d.addCallback(lambda (message, _): + self._check_headers(message, lines[:4])) + d.addCallback(self._check_key_attachment) + d.addErrback(log.err) + return d + + def _check_headers(self, message, headers): + msgstr = message.as_string(unixfrom=False) + for header in headers: + self.assertTrue(header in msgstr, + "Missing header: %s" % (header,)) + return message + + def _check_key_attachment(self, message): + for payload in message.get_payload(): + if payload.is_multipart(): + return self._check_key_attachment(payload) + if 'application/pgp-keys' == payload.get_content_type(): + keylines = PUBLIC_KEY_2.split('\n') + key = BEGIN_PUBLIC_KEY + '\n\n' + '\n'.join(keylines[4:-1]) + self.assertTrue(key in payload.get_payload(decode=True), + "Key attachment don't match") + return + self.fail("No public key attachment found") + + def _set_sign_used(self, address): + def set_sign(key): + key.sign_used = True + return self.km.put_key(key) + + d = self.km.get_key(address, fetch_remote=False) + d.addCallback(set_sign) + return d + + def _assert_encrypted(self, res): + message, _ = res + self.assertTrue('Content-Type' in message) + self.assertEqual('multipart/encrypted', message.get_content_type()) + self.assertEqual('application/pgp-encrypted', + message.get_param('protocol')) + self.assertEqual(2, len(message.get_payload())) + self.assertEqual('application/pgp-encrypted', + message.get_payload(0).get_content_type()) + self.assertEqual('application/octet-stream', + message.get_payload(1).get_content_type()) + return message diff --git a/src/leap/bitmask/mail/plugins/__init__.py b/src/leap/bitmask/mail/plugins/__init__.py new file mode 100644 index 0000000..ddb8691 --- /dev/null +++ b/src/leap/bitmask/mail/plugins/__init__.py @@ -0,0 +1,3 @@ +from twisted.plugin import pluginPackagePaths +__path__.extend(pluginPackagePaths(__name__)) +__all__ = [] diff --git a/src/leap/bitmask/mail/plugins/soledad_sync_hooks.py b/src/leap/bitmask/mail/plugins/soledad_sync_hooks.py new file mode 100644 index 0000000..9d48126 --- /dev/null +++ b/src/leap/bitmask/mail/plugins/soledad_sync_hooks.py @@ -0,0 +1,19 @@ +# -*- coding: utf-8 -*- +# soledad_sync_hooks.py +# Copyright (C) 2015 LEAP +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . + +from leap.mail.sync_hooks import MailProcessingPostSyncHook +post_sync_uid_reindexer = MailProcessingPostSyncHook() diff --git a/src/leap/bitmask/mail/rfc3156.py b/src/leap/bitmask/mail/rfc3156.py new file mode 100644 index 0000000..7d7bc0f --- /dev/null +++ b/src/leap/bitmask/mail/rfc3156.py @@ -0,0 +1,390 @@ +# -*- coding: utf-8 -*- +# rfc3156.py +# Copyright (C) 2013 LEAP +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . + +""" +Implements RFC 3156: MIME Security with OpenPGP. +""" + +import base64 +from StringIO import StringIO + +from twisted.python import log +from email.mime.application import MIMEApplication +from email.mime.multipart import MIMEMultipart +from email import errors +from email.generator import ( + Generator, + fcre, + NL, + _make_boundary, +) + + +# +# A generator that solves http://bugs.python.org/issue14983 +# + +class RFC3156CompliantGenerator(Generator): + """ + An email generator that addresses Python's issue #14983 for multipart + messages. + + This is just a copy of email.generator.Generator which fixes the following + bug: http://bugs.python.org/issue14983 + """ + + def _handle_multipart(self, msg): + """ + A multipart handling implementation that addresses issue #14983. + + This is just a copy of the parent's method which fixes the following + bug: http://bugs.python.org/issue14983 (see the line marked with + "(***)"). + + :param msg: The multipart message to be handled. + :type msg: email.message.Message + """ + # The trick here is to write out each part separately, merge them all + # together, and then make sure that the boundary we've chosen isn't + # present in the payload. + msgtexts = [] + subparts = msg.get_payload() + if subparts is None: + subparts = [] + elif isinstance(subparts, basestring): + # e.g. a non-strict parse of a message with no starting boundary. + self._fp.write(subparts) + return + elif not isinstance(subparts, list): + # Scalar payload + subparts = [subparts] + for part in subparts: + s = StringIO() + g = self.clone(s) + g.flatten(part, unixfrom=False) + msgtexts.append(s.getvalue()) + # BAW: What about boundaries that are wrapped in double-quotes? + boundary = msg.get_boundary() + if not boundary: + # Create a boundary that doesn't appear in any of the + # message texts. + alltext = NL.join(msgtexts) + boundary = _make_boundary(alltext) + msg.set_boundary(boundary) + # If there's a preamble, write it out, with a trailing CRLF + if msg.preamble is not None: + preamble = msg.preamble + if self._mangle_from_: + preamble = fcre.sub('>From ', msg.preamble) + self._fp.write(preamble + '\n') + # dash-boundary transport-padding CRLF + self._fp.write('--' + boundary + '\n') + # body-part + if msgtexts: + self._fp.write(msgtexts.pop(0)) + # *encapsulation + # --> delimiter transport-padding + # --> CRLF body-part + for body_part in msgtexts: + # delimiter transport-padding CRLF + self._fp.write('\n--' + boundary + '\n') + # body-part + self._fp.write(body_part) + # close-delimiter transport-padding + self._fp.write('\n--' + boundary + '--' + '\n') # (***) Solve #14983 + if msg.epilogue is not None: + self._fp.write('\n') + epilogue = msg.epilogue + if self._mangle_from_: + epilogue = fcre.sub('>From ', msg.epilogue) + self._fp.write(epilogue) + + +# +# Base64 encoding: these are almost the same as python's email.encoder +# solution, but a bit modified. +# + +def _bencode(s): + """ + Encode C{s} in base64. + + :param s: The string to be encoded. + :type s: str + """ + # We can't quite use base64.encodestring() since it tacks on a "courtesy + # newline". Blech! + if not s: + return s + value = base64.encodestring(s) + return value[:-1] + + +def encode_base64(msg): + """ + Encode a non-multipart message's payload in Base64 (in place). + + This method modifies the message contents in place and adds or replaces an + appropriate Content-Transfer-Encoding header. + + :param msg: The non-multipart message to be encoded. + :type msg: email.message.Message + """ + encoding = msg.get('Content-Transfer-Encoding', None) + if encoding is not None: + encoding = encoding.lower() + # XXX Python's email module can only decode quoted-printable, base64 and + # uuencoded data, so we might have to implement other decoding schemes in + # order to support RFC 3156 properly and correctly calculate signatures + # for multipart attachments (eg. 7bit or 8bit encoded attachments). For + # now, if content is already encoded as base64 or if it is encoded with + # some unknown encoding, we just pass. + if encoding in [None, 'quoted-printable', 'x-uuencode', 'uue', 'x-uue']: + orig = msg.get_payload(decode=True) + encdata = _bencode(orig) + msg.set_payload(encdata) + # replace or set the Content-Transfer-Encoding header. + try: + msg.replace_header('Content-Transfer-Encoding', 'base64') + except KeyError: + msg['Content-Transfer-Encoding'] = 'base64' + elif encoding is not 'base64': + log.err('Unknown content-transfer-encoding: %s' % encoding) + + +def encode_base64_rec(msg): + """ + Encode (possibly multipart) messages in base64 (in place). + + This method modifies the message contents in place. + + :param msg: The non-multipart message to be encoded. + :type msg: email.message.Message + """ + if not msg.is_multipart(): + encode_base64(msg) + else: + for sub in msg.get_payload(): + encode_base64_rec(sub) + + +# +# RFC 1847: multipart/signed and multipart/encrypted +# + +class MultipartSigned(MIMEMultipart): + """ + Multipart/Signed MIME message according to RFC 1847. + + 2.1. Definition of Multipart/Signed + + (1) MIME type name: multipart + (2) MIME subtype name: signed + (3) Required parameters: boundary, protocol, and micalg + (4) Optional parameters: none + (5) Security considerations: Must be treated as opaque while in + transit + + The multipart/signed content type contains exactly two body parts. + The first body part is the body part over which the digital signature + was created, including its MIME headers. The second body part + contains the control information necessary to verify the digital + signature. The first body part may contain any valid MIME content + type, labeled accordingly. The second body part is labeled according + to the value of the protocol parameter. + + When the OpenPGP digital signature is generated: + + (1) The data to be signed MUST first be converted to its content- + type specific canonical form. For text/plain, this means + conversion to an appropriate character set and conversion of + line endings to the canonical sequence. + + (2) An appropriate Content-Transfer-Encoding is then applied; see + section 3. In particular, line endings in the encoded data + MUST use the canonical sequence where appropriate + (note that the canonical line ending may or may not be present + on the last line of encoded data and MUST NOT be included in + the signature if absent). + + (3) MIME content headers are then added to the body, each ending + with the canonical sequence. + + (4) As described in section 3 of this document, any trailing + whitespace MUST then be removed from the signed material. + + (5) As described in [2], the digital signature MUST be calculated + over both the data to be signed and its set of content headers. + + (6) The signature MUST be generated detached from the signed data + so that the process does not alter the signed data in any way. + """ + + def __init__(self, protocol, micalg, boundary=None, _subparts=None): + """ + Initialize the multipart/signed message. + + :param boundary: the multipart boundary string. By default it is + calculated as needed. + :type boundary: str + :param _subparts: a sequence of initial subparts for the payload. It + must be an iterable object, such as a list. You can always + attach new subparts to the message by using the attach() method. + :type _subparts: iterable + """ + MIMEMultipart.__init__( + self, _subtype='signed', boundary=boundary, + _subparts=_subparts) + self.set_param('protocol', protocol) + self.set_param('micalg', micalg) + + def attach(self, payload): + """ + Add the C{payload} to the current payload list. + + Also prevent from adding payloads with wrong Content-Type and from + exceeding a maximum of 2 payloads. + + :param payload: The payload to be attached. + :type payload: email.message.Message + """ + # second payload's content type must be equal to the protocol + # parameter given on object creation + if len(self.get_payload()) == 1: + if payload.get_content_type() != self.get_param('protocol'): + raise errors.MultipartConversionError( + 'Wrong content type %s.' % payload.get_content_type) + # prevent from adding more payloads + if len(self._payload) == 2: + raise errors.MultipartConversionError( + 'Cannot have more than two subparts.') + MIMEMultipart.attach(self, payload) + + +class MultipartEncrypted(MIMEMultipart): + """ + Multipart/encrypted MIME message according to RFC 1847. + + 2.2. Definition of Multipart/Encrypted + + (1) MIME type name: multipart + (2) MIME subtype name: encrypted + (3) Required parameters: boundary, protocol + (4) Optional parameters: none + (5) Security considerations: none + + The multipart/encrypted content type contains exactly two body parts. + The first body part contains the control information necessary to + decrypt the data in the second body part and is labeled according to + the value of the protocol parameter. The second body part contains + the data which was encrypted and is always labeled + application/octet-stream. + """ + + def __init__(self, protocol, boundary=None, _subparts=None): + """ + :param protocol: The encryption protocol to be added as a parameter to + the Content-Type header. + :type protocol: str + :param boundary: the multipart boundary string. By default it is + calculated as needed. + :type boundary: str + :param _subparts: a sequence of initial subparts for the payload. It + must be an iterable object, such as a list. You can always + attach new subparts to the message by using the attach() method. + :type _subparts: iterable + """ + MIMEMultipart.__init__( + self, _subtype='encrypted', boundary=boundary, + _subparts=_subparts) + self.set_param('protocol', protocol) + + def attach(self, payload): + """ + Add the C{payload} to the current payload list. + + Also prevent from adding payloads with wrong Content-Type and from + exceeding a maximum of 2 payloads. + + :param payload: The payload to be attached. + :type payload: email.message.Message + """ + # first payload's content type must be equal to the protocol parameter + # given on object creation + if len(self._payload) == 0: + if payload.get_content_type() != self.get_param('protocol'): + raise errors.MultipartConversionError( + 'Wrong content type.') + # second payload is always application/octet-stream + if len(self._payload) == 1: + if payload.get_content_type() != 'application/octet-stream': + raise errors.MultipartConversionError( + 'Wrong content type %s.' % payload.get_content_type) + # prevent from adding more payloads + if len(self._payload) == 2: + raise errors.MultipartConversionError( + 'Cannot have more than two subparts.') + MIMEMultipart.attach(self, payload) + + +# +# RFC 3156: application/pgp-encrypted, application/pgp-signed and +# application-pgp-signature. +# + +class PGPEncrypted(MIMEApplication): + """ + Application/pgp-encrypted MIME media type according to RFC 3156. + + * MIME media type name: application + * MIME subtype name: pgp-encrypted + * Required parameters: none + * Optional parameters: none + """ + + def __init__(self, version=1): + data = "Version: %d" % version + MIMEApplication.__init__(self, data, 'pgp-encrypted') + + +class PGPSignature(MIMEApplication): + """ + Application/pgp-signature MIME media type according to RFC 3156. + + * MIME media type name: application + * MIME subtype name: pgp-signature + * Required parameters: none + * Optional parameters: none + """ + def __init__(self, _data, name='signature.asc'): + MIMEApplication.__init__(self, _data, 'pgp-signature', + _encoder=lambda x: x, name=name) + self.add_header('Content-Description', 'OpenPGP Digital Signature') + + +class PGPKeys(MIMEApplication): + """ + Application/pgp-keys MIME media type according to RFC 3156. + + * MIME media type name: application + * MIME subtype name: pgp-keys + * Required parameters: none + * Optional parameters: none + """ + + def __init__(self, _data): + MIMEApplication.__init__(self, _data, 'pgp-keys') diff --git a/src/leap/bitmask/mail/size.py b/src/leap/bitmask/mail/size.py new file mode 100644 index 0000000..c9eaabd --- /dev/null +++ b/src/leap/bitmask/mail/size.py @@ -0,0 +1,57 @@ +# -*- coding: utf-8 -*- +# size.py +# Copyright (C) 2014 LEAP +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . +""" +Recursively get size of objects. +""" +from gc import collect +from itertools import chain +from sys import getsizeof + + +def _get_size(item, seen): + known_types = {dict: lambda d: chain.from_iterable(d.items())} + default_size = getsizeof(0) + + def size_walk(item): + if id(item) in seen: + return 0 + seen.add(id(item)) + s = getsizeof(item, default_size) + for _type, fun in known_types.iteritems(): + if isinstance(item, _type): + s += sum(map(size_walk, fun(item))) + break + return s + + return size_walk(item) + + +def get_size(item): + """ + Return the cumulative size of a given object. + + Currently it supports only dictionaries, and seemingly leaks + some memory, so use with care. + + :param item: the item which size wants to be computed + :rtype: int + """ + seen = set() + size = _get_size(item, seen) + del seen + collect() + return size diff --git a/src/leap/bitmask/mail/smtp/README.rst b/src/leap/bitmask/mail/smtp/README.rst new file mode 100644 index 0000000..1d3a903 --- /dev/null +++ b/src/leap/bitmask/mail/smtp/README.rst @@ -0,0 +1,44 @@ +Leap SMTP Gateway +================= + +The Bitmask Client runs a thin SMTP gateway on the user's device, which +intends to encrypt and sign outgoing messages to achieve point to point +encryption. + +The gateway is bound to localhost and the user's MUA should be configured to +send messages to it. After doing its thing, the gateway will relay the +messages to the remote SMTP server. + +Outgoing mail workflow: + + * SMTP gateway receives a message from the MUA. + + * SMTP gateway queries Key Manager for the user's private key. + + * For each recipient (including addresses in "To", "Cc" anc "Bcc" fields), + the following happens: + + - The recipient's address is validated against RFC2822. + + - An attempt is made to fetch the recipient's public PGP key. + + - If key is not found: + + - If the gateway is configured to only send encrypted messages the + recipient is rejected. + + - Otherwise, the message is signed and sent as plain text. + + - If the key is found, the message is encrypted to the recipient and + signed with the sender's private PGP key. + + * Finally, one message for each recipient is gatewayed to provider's SMTP + server. + + +Running tests +------------- + +Tests are run using Twisted's Trial API, like this:: + + python setup.py test -s leap.mail.gateway.tests diff --git a/src/leap/bitmask/mail/smtp/__init__.py b/src/leap/bitmask/mail/smtp/__init__.py new file mode 100644 index 0000000..9fab70a --- /dev/null +++ b/src/leap/bitmask/mail/smtp/__init__.py @@ -0,0 +1,73 @@ +# -*- coding: utf-8 -*- +# __init__.py +# Copyright (C) 2013 LEAP +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . + +""" +SMTP gateway helper function. +""" +import logging +import os + +from twisted.internet import reactor +from twisted.internet.error import CannotListenError + +from leap.common.events import emit_async, catalog + +from leap.mail.smtp.gateway import SMTPFactory + +logger = logging.getLogger(__name__) + + +SMTP_PORT = 2013 + + +def run_service(soledad_sessions, keymanager_sessions, sendmail_opts, + port=SMTP_PORT): + """ + Main entry point to run the service from the client. + + :param soledad_sessions: a dict-like object, containing instances + of a Store (soledad instances), indexed by userid. + :param keymanager_sessions: a dict-like object, containing instances + of Keymanager, indexed by userid. + :param sendmail_opts: a dict-like object of sendmailOptions. + + :returns: the port as returned by the reactor when starts listening, and + the factory for the protocol. + :rtype: tuple + """ + factory = SMTPFactory(soledad_sessions, keymanager_sessions, + sendmail_opts) + + try: + interface = "localhost" + # don't bind just to localhost if we are running on docker since we + # won't be able to access smtp from the host + if os.environ.get("LEAP_DOCKERIZED"): + interface = '' + + # TODO Use Endpoints instead -------------------------------- + tport = reactor.listenTCP(port, factory, interface=interface) + emit_async(catalog.SMTP_SERVICE_STARTED, str(port)) + + return factory, tport + except CannotListenError: + logger.error("STMP Service failed to start: " + "cannot listen in port %s" % port) + emit_async(catalog.SMTP_SERVICE_FAILED_TO_START, str(port)) + except Exception as exc: + logger.error("Unhandled error while launching smtp gateway service") + logger.exception(exc) diff --git a/src/leap/bitmask/mail/smtp/bounces.py b/src/leap/bitmask/mail/smtp/bounces.py new file mode 100644 index 0000000..7a4674b --- /dev/null +++ b/src/leap/bitmask/mail/smtp/bounces.py @@ -0,0 +1,90 @@ +# -*- coding: utf-8 -*- +# bounces.py +# Copyright (C) 2016 LEAP +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . +""" +Deliver bounces to the user Inbox. +""" +import time +from email.message import Message +from email.utils import formatdate + +from leap.mail.constants import INBOX_NAME +from leap.mail.mail import Account + + +# TODO implement localization for this template. + +BOUNCE_TEMPLATE = """This is your local Bitmask Mail Agent running at localhost. + +I'm sorry to have to inform you that your message could not be delivered to one +or more recipients. + +The reasons I got for the error are: + +{raw_error} + +If the problem persists and it's not a network connectivity issue, you might +want to contact your provider ({provider}) with this information (remove any +sensitive data before). + +--- Original message (*before* it was encrypted by bitmask) below ----: + +{orig}""" + + +class Bouncer(object): + """ + Implements a mechanism to deliver bounces to user inbox. + """ + # TODO this should follow RFC 6522, and compose a correct multipart + # attaching the report and the original message. Leaving this for a future + # iteration. + + def __init__(self, inbox_collection): + self._inbox_collection = inbox_collection + + def bounce_message(self, error_data, to, date=None, orig=''): + if not date: + date = formatdate(time.time()) + + raw_data = self._format_msg(error_data, to, date, orig) + d = self._inbox_collection.add_msg( + raw_data, ('\\Recent',), date=date) + return d + + def _format_msg(self, error_data, to, date, orig): + provider = to.split('@')[1] + + msg = Message() + msg.add_header( + 'From', 'bitmask-bouncer@localhost (Bitmask Local Agent)') + msg.add_header('To', to) + msg.add_header('Subject', 'Undelivered Message') + msg.add_header('Date', date) + msg.set_payload(BOUNCE_TEMPLATE.format( + raw_error=error_data, + provider=provider, + orig=orig)) + + return msg.as_string() + + +def bouncerFactory(soledad): + user_id = soledad.uuid + acc = Account(soledad, user_id) + d = acc.callWhenReady(lambda _: acc.get_collection_by_mailbox(INBOX_NAME)) + d.addCallback(lambda inbox: Bouncer(inbox)) + return d diff --git a/src/leap/bitmask/mail/smtp/gateway.py b/src/leap/bitmask/mail/smtp/gateway.py new file mode 100644 index 0000000..e49bbe8 --- /dev/null +++ b/src/leap/bitmask/mail/smtp/gateway.py @@ -0,0 +1,413 @@ +# -*- coding: utf-8 -*- +# gateway.py +# Copyright (C) 2013 LEAP +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . +""" +LEAP SMTP encrypted gateway. + +The following classes comprise the SMTP gateway service: + + * SMTPFactory - A twisted.internet.protocol.ServerFactory that provides + the SMTPDelivery protocol. + + * SMTPDelivery - A twisted.mail.smtp.IMessageDelivery implementation. It + knows how to validate sender and receiver of messages and it generates + an EncryptedMessage for each recipient. + + * EncryptedMessage - An implementation of twisted.mail.smtp.IMessage that + knows how to encrypt/sign itself before sending. +""" +from email.Header import Header + +from zope.interface import implements +from zope.interface import implementer + +from twisted.cred.portal import Portal, IRealm +from twisted.mail import smtp +from twisted.mail.imap4 import LOGINCredentials, PLAINCredentials +from twisted.internet import defer, protocol +from twisted.python import log + +from leap.common.check import leap_assert_type +from leap.common.events import emit_async, catalog +from leap.mail import errors +from leap.mail.cred import LocalSoledadTokenChecker +from leap.mail.utils import validate_address +from leap.mail.rfc3156 import RFC3156CompliantGenerator +from leap.mail.outgoing.service import outgoingFactory +from leap.mail.smtp.bounces import bouncerFactory +from leap.keymanager.errors import KeyNotFound + +# replace email generator with a RFC 3156 compliant one. +from email import generator + +generator.Generator = RFC3156CompliantGenerator + + +LOCAL_FQDN = "bitmask.local" + + +@implementer(IRealm) +class LocalSMTPRealm(object): + + _encoding = 'utf-8' + + def __init__(self, keymanager_sessions, soledad_sessions, sendmail_opts, + encrypted_only=False): + """ + :param keymanager_sessions: a dict-like object, containing instances + of a Keymanager objects, indexed by + userid. + """ + self._keymanager_sessions = keymanager_sessions + self._soledad_sessions = soledad_sessions + self._sendmail_opts = sendmail_opts + self.encrypted_only = encrypted_only + + def requestAvatar(self, avatarId, mind, *interfaces): + + if isinstance(avatarId, str): + avatarId = avatarId.decode(self._encoding) + + def gotKeymanagerAndSoledad(result): + keymanager, soledad = result + d = bouncerFactory(soledad) + d.addCallback(lambda bouncer: (keymanager, soledad, bouncer)) + return d + + def getMessageDelivery(result): + keymanager, soledad, bouncer = result + # TODO use IMessageDeliveryFactory instead ? + # it could reuse the connections. + if smtp.IMessageDelivery in interfaces: + userid = avatarId + opts = self.getSendingOpts(userid) + + outgoing = outgoingFactory( + userid, keymanager, opts, bouncer=bouncer) + avatar = SMTPDelivery(userid, keymanager, self.encrypted_only, + outgoing) + + return (smtp.IMessageDelivery, avatar, + getattr(avatar, 'logout', lambda: None)) + + raise NotImplementedError(self, interfaces) + + d1 = self.lookupKeymanagerInstance(avatarId) + d2 = self.lookupSoledadInstance(avatarId) + d = defer.gatherResults([d1, d2]) + d.addCallback(gotKeymanagerAndSoledad) + d.addCallback(getMessageDelivery) + return d + + def lookupKeymanagerInstance(self, userid): + try: + keymanager = self._keymanager_sessions[userid] + except: + raise errors.AuthenticationError( + 'No keymanager session found for user %s. Is it authenticated?' + % userid) + # XXX this should return the instance after whenReady callback + return defer.succeed(keymanager) + + def lookupSoledadInstance(self, userid): + try: + soledad = self._soledad_sessions[userid] + except: + raise errors.AuthenticationError( + 'No soledad session found for user %s. Is it authenticated?' + % userid) + # XXX this should return the instance after whenReady callback + return defer.succeed(soledad) + + def getSendingOpts(self, userid): + try: + opts = self._sendmail_opts[userid] + except KeyError: + raise errors.ConfigurationError( + 'No sendingMail options found for user %s' % userid) + return opts + + +class SMTPTokenChecker(LocalSoledadTokenChecker): + """A credentials checker that will lookup a token for the SMTP service. + For now it will be using the same identifier than IMAPTokenChecker""" + + service = 'mail_auth' + + # TODO besides checking for token credential, + # we could also verify the certificate here. + + +class LEAPInitMixin(object): + + """ + A Mixin that takes care of initialization of all the data needed to access + LEAP sessions. + """ + def __init__(self, soledad_sessions, keymanager_sessions, sendmail_opts, + encrypted_only=False): + realm = LocalSMTPRealm( + keymanager_sessions, soledad_sessions, sendmail_opts, + encrypted_only) + portal = Portal(realm) + + checker = SMTPTokenChecker(soledad_sessions) + self.checker = checker + self.portal = portal + portal.registerChecker(checker) + + +class LocalSMTPServer(smtp.ESMTP, LEAPInitMixin): + """ + The Production ESMTP Server: Authentication Needed. + Authenticates against SMTP Token stored in Local Soledad instance. + The Realm will produce a Delivery Object that handles encryption/signing. + """ + + # TODO: implement Queue using twisted.mail.mail.MailService + + def __init__(self, soledads, keyms, sendmailopts, *args, **kw): + encrypted_only = kw.pop('encrypted_only', False) + + LEAPInitMixin.__init__(self, soledads, keyms, sendmailopts, + encrypted_only) + smtp.ESMTP.__init__(self, *args, **kw) + + +# TODO implement retries -- see smtp.SenderMixin +class SMTPFactory(protocol.ServerFactory): + """ + Factory for an SMTP server with encrypted gatewaying capabilities. + """ + + protocol = LocalSMTPServer + domain = LOCAL_FQDN + timeout = 600 + encrypted_only = False + + def __init__(self, soledad_sessions, keymanager_sessions, sendmail_opts, + deferred=None, retries=3): + + self._soledad_sessions = soledad_sessions + self._keymanager_sessions = keymanager_sessions + self._sendmail_opts = sendmail_opts + + def buildProtocol(self, addr): + p = self.protocol( + self._soledad_sessions, self._keymanager_sessions, + self._sendmail_opts, encrypted_only=self.encrypted_only) + p.factory = self + p.host = LOCAL_FQDN + p.challengers = {"LOGIN": LOGINCredentials, "PLAIN": PLAINCredentials} + return p + + +# +# SMTPDelivery +# + +@implementer(smtp.IMessageDelivery) +class SMTPDelivery(object): + """ + Validate email addresses and handle message delivery. + """ + + def __init__(self, userid, keymanager, encrypted_only, outgoing_mail): + """ + Initialize the SMTP delivery object. + + :param userid: The user currently logged in + :type userid: unicode + :param keymanager: A Key Manager from where to get recipients' public + keys. + :param encrypted_only: Whether the SMTP gateway should send unencrypted + mail or not. + :type encrypted_only: bool + :param outgoing_mail: The outgoing mail to send the message + :type outgoing_mail: leap.mail.outgoing.service.OutgoingMail + """ + self._userid = userid + self._outgoing_mail = outgoing_mail + self._km = keymanager + self._encrypted_only = encrypted_only + self._origin = None + + def receivedHeader(self, helo, origin, recipients): + """ + Generate the 'Received:' header for a message. + + :param helo: The argument to the HELO command and the client's IP + address. + :type helo: (str, str) + :param origin: The address the message is from. + :type origin: twisted.mail.smtp.Address + :param recipients: A list of the addresses for which this message is + bound. + :type: list of twisted.mail.smtp.User + + @return: The full "Received" header string. + :type: str + """ + myHostname, clientIP = helo + headerValue = "by bitmask.local from %s with ESMTP ; %s" % ( + clientIP, smtp.rfc822date()) + # email.Header.Header used for automatic wrapping of long lines + return "Received: %s" % Header(s=headerValue, header_name='Received') + + def validateTo(self, user): + """ + Validate the address of a recipient of the message, possibly + rejecting it if the recipient key is not available. + + This method is called once for each recipient, i.e. for each SMTP + protocol line beginning with "RCPT TO:", which includes all addresses + in "To", "Cc" and "Bcc" MUA fields. + + The recipient's address is validated against the RFC 2822 definition. + If self._encrypted_only is True and no key is found for a recipient, + then that recipient is rejected. + + The method returns an encrypted message object that is able to send + itself to the user's address. + + :param user: The user whose address we wish to validate. + :type: twisted.mail.smtp.User + + @return: A callable which takes no arguments and returns an + encryptedMessage. + @rtype: no-argument callable + + @raise SMTPBadRcpt: Raised if messages to the address are not to be + accepted. + """ + # try to find recipient's public key + address = validate_address(user.dest.addrstr) + + # verify if recipient key is available in keyring + def found(_): + log.msg("Accepting mail for %s..." % user.dest.addrstr) + emit_async(catalog.SMTP_RECIPIENT_ACCEPTED_ENCRYPTED, + self._userid, user.dest.addrstr) + + def not_found(failure): + failure.trap(KeyNotFound) + + # if key was not found, check config to see if will send anyway + if self._encrypted_only: + emit_async(catalog.SMTP_RECIPIENT_REJECTED, self._userid, + user.dest.addrstr) + raise smtp.SMTPBadRcpt(user.dest.addrstr) + log.msg("Warning: will send an unencrypted message (because " + "encrypted_only' is set to False).") + emit_async( + catalog.SMTP_RECIPIENT_ACCEPTED_UNENCRYPTED, + self._userid, user.dest.addrstr) + + def encrypt_func(_): + return lambda: EncryptedMessage(user, self._outgoing_mail) + + d = self._km.get_key(address) + d.addCallbacks(found, not_found) + d.addCallback(encrypt_func) + return d + + def validateFrom(self, helo, origin): + """ + Validate the address from which the message originates. + + :param helo: The argument to the HELO command and the client's IP + address. + :type: (str, str) + :param origin: The address the message is from. + :type origin: twisted.mail.smtp.Address + + @return: origin or a Deferred whose callback will be passed origin. + @rtype: Deferred or Address + + @raise twisted.mail.smtp.SMTPBadSender: Raised if messages from this + address are not to be accepted. + """ + # accept mail from anywhere. To reject an address, raise + # smtp.SMTPBadSender here. + if str(origin) != str(self._userid): + log.msg("Rejecting sender {0}, expected {1}".format(origin, + self._userid)) + raise smtp.SMTPBadSender(origin) + self._origin = origin + return origin + + +# +# EncryptedMessage +# + +class EncryptedMessage(object): + """ + Receive plaintext from client, encrypt it and send message to a + recipient. + """ + implements(smtp.IMessage) + + def __init__(self, user, outgoing_mail): + """ + Initialize the encrypted message. + + :param user: The recipient of this message. + :type user: twisted.mail.smtp.User + :param outgoing_mail: The outgoing mail to send the message + :type outgoing_mail: leap.mail.outgoing.service.OutgoingMail + """ + # assert params + leap_assert_type(user, smtp.User) + + self._user = user + self._lines = [] + self._outgoing_mail = outgoing_mail + + def lineReceived(self, line): + """ + Handle another line. + + :param line: The received line. + :type line: str + """ + self._lines.append(line) + + def eomReceived(self): + """ + Handle end of message. + + This method will encrypt and send the message. + + :returns: a deferred + """ + log.msg("Message data complete.") + self._lines.append('') # add a trailing newline + raw_mail = '\r\n'.join(self._lines) + + return self._outgoing_mail.send_message(raw_mail, self._user) + + def connectionLost(self): + """ + Log an error when the connection is lost. + """ + log.msg("Connection lost unexpectedly!") + log.err() + emit_async(catalog.SMTP_CONNECTION_LOST, self._userid, + self._user.dest.addrstr) + # unexpected loss of connection; don't save + + self._lines = [] diff --git a/src/leap/bitmask/mail/smtp/tests/185CA770.key b/src/leap/bitmask/mail/smtp/tests/185CA770.key new file mode 100644 index 0000000..587b416 --- /dev/null +++ b/src/leap/bitmask/mail/smtp/tests/185CA770.key @@ -0,0 +1,79 @@ +-----BEGIN PGP PRIVATE KEY BLOCK----- +Version: GnuPG v1.4.10 (GNU/Linux) + +lQIVBFCJNL4BEADFsI1TCD4yq7ZqL7VhdVviTuX6JUps8/mVEhRVOZhojLcTYaqQ +gs6T6WabRxcK7ymOnf4K8NhYdz6HFoJN46BT87etokx7J/Sl2OhpiqBQEY+jW8Rp ++3MSGrGmvFw0s1lGrz/cXzM7UNgWSTOnYZ5nJS1veMhy0jseZOUK7ekp2oEDjGZh +pzgd3zICCR2SvlpLIXB2Nr/CUcuRWTcc5LlKmbjMybu0E/uuY14st3JL+7qI6QX0 +atFm0VhFVpagOl0vWKxakUx4hC7j1wH2ADlCvSZPG0StSLUyHkJx3UPsmYxOZFao +ATED3Okjwga6E7PJEbzyqAkvzw/M973kaZCUSH75ZV0cQnpdgXV3DK1gSa3d3gug +W1lE0V7pwnN2NTOYfBMi+WloCs/bp4iZSr4QP1duZ3IqKraeBDCk7MoFo4A9Wk07 +kvqPwF9IBgatu62WVEZIzwyViN+asFUGfgp+8D7gtnlWAw0V6y/lSTzyl+dnLP98 +Hfr2eLBylFs+Kl3Pivpg2uHw09LLCrjeLEN3dj9SfBbA9jDIo9Zhs1voiIK/7Shx +E0BRJaBgG3C4QaytYEu7RFFOKuvBai9w2Y5OfsKFo8rA7v4dxFFDvzKGujCtNnwf +oyaGlZmMBU5MUmHUNiG8ON21COZBtK5oMScuY1VC9CQonj3OClg3IbU9SQARAQAB +/gNlAkdOVQG0JGRyZWJzIChncGcgdGVzdCBrZXkpIDxkcmVic0BsZWFwLnNlPokC +OAQTAQIAIgUCUIk0vgIbAwYLCQgHAwIGFQgCCQoLBBYCAwECHgECF4AACgkQty9e +xhhcp3Bdhw//bdPUNbp6rgIjRRuwYvGJ6IuiFuFWJQ0m3iAuuAoZo5GHAPqZAuGk +dMVYu0dtCtZ68MJ/QpjBCT9RRL+mgIgfLfUSj2ZknP4nb6baiG5u28l0KId/e5IC +iQKBnIsjxKxhLBVHSzRaS1P+vZeF2C2R9XyNy0eCnAwyCMcD0R8TVROGQ7i4ZQsM +bMj1LPpOwhV/EGp23nD+upWOVbn/wQHOYV2kMiA/8fizmWRIWsV4/68uMA+WDP4L +40AnJ0fcs04f9deM9P6pjlm00VD7qklYEGw6Mpr2g/M73kGh1nlAv+ImQBGlLMle +RXyzHY3WAhzmRKWO4koFuKeR9Q0EMzk2R4/kuagdWEpM+bhwE4xPV1tPZhn9qFTz +pQD4p/VT4qNQKOD0+aTFWre65Rt2cFFMLI7UmEHNLi0NB9JCIAi4+l+b9WQNlmaO +C8EhOGwRzmehUyHmXM3BNW28MnyKFJ7bBFMd7uJz+vAPOrr6OzuNvVCv2I2ICkTs +ihIj/zw5GXxkPO7YbMu9rKG0nKF1N3JB1gUJ78DHmhbjeaGSvHw85sPD0/1dPZK4 +8Gig8i62aCxf8OlJPlt8ZhBBolzs6ITUNa75Rw9fJsj3UWuv2VFaIuR57bFWmY3s +A9KPgdf7jVQlAZKlVyli7IkyaZmxDZNFQoTdIC9uo0aggIDP8zKv0n2dBz4EUIk0 +vgEQAOO8BAR7sBdqj2RRMRNeWSA4S9GuHfV3YQARnqYsbITs1jRgAo7jx9Z5C80c +ZOxOUVK7CJjtTqU0JB9QP/zwV9hk5i6y6aQTysclQyTNN10aXu/3zJla5Duhz+Cs ++5UcVAmNJX9FgTMVvhKDEIY/LNmb9MoBLMut1CkDx+WPCV45WOIBCDdj2HpIjie4 +phs0/65SWjPiVg3WsFZljVxpJCGXP48Eet2bf8afYH1lx3sQMcNbyJACIPtz+YKz +c7jIKwKSWzg1VyYikbk9eWCxcz6VKNJKi94YH9c7U8X3TdZ8G0kGYUldjYDvesyl +nuQlcGCtSGKOAhrN/Bu2R0gpFgYl247u79CmjotefMdv8BGUDW6u9/Sep9xN3dW8 +S87h6M/tvs0ChlkDDpJedzCd7ThdikGvFRJfW/8sT/+qoTKskySQaDIeNJnxZuyK +wELLMBvCZGpamwmnkEGhvuZWq0h/DwyTs4QAE8OVHXJSM3UN7hM4lJIUh+sRKJ1F +AXXTdSY4cUNaS+OKtj2LJ85zFqhfAZ4pFwLCgYbJtU5hej2LnMJNbYcSkjxbk+c5 +IjkoZRF+ExjZlc0VLYNT57ZriwZ/pX42ofjOyMR/dkHQuFik/4K7v1ZemfaTdm07 +SEMBknR6OZsy/5+viEtXiih3ptTMaT9row+g+cFoxdXkisKvABEBAAH+AwMCIlVK +Xs3x0Slgwx03cTNIoWXmishkPCJlEEdcjldz2VyQF9hjdp1VIe+npI26chKwCZqm +U8yYbJh4UBrugUUzKKd4EfnmKfu+/BsJciFRVKwBtiolIiUImzcHPWktYLwo9yzX +W42teShXXVgWmsJN1/6FqJdsLg8dxWesXMKoaNF4n1P7zx6vKBmDHTRz7PToaI/d +5/nKrjED7ZT1h+qR5i9UUgbvF0ySp8mlqk/KNqHUSLDB9kf/JDg4XVtPHGGd9Ik/ +60UJ7aDfohi4Z0VgwWmfLBwcQ3It+ENtnPFufH3WHW8c1UA4wVku9tOTqyrRG6tP +TZGiRfuwsv7Hq3pWT6rntbDkTiVgESM4C1fiZblc98iWUKGXSHqm+te1TwXOUCci +J/gryXcjQFM8A0rwA/m+EvsoWuzoqIl3x++p3/3/mGux6UD4O7OhJNRVRz+8Mhq1 +ksrR9XkQzpq3Yv3ulTHz7l+WCRRXxw5+XWAkRHHF47Vf/na38NJQHcsCBbRIuLYR +wBzS48cYzYkF6VejKThdQmdYJ0/fUrlUBCAJWgrfqCihFLDa1s4jJ16/fqi8a97Y +4raVy2hrF2vFc/wet13hsaddVn4rPRAMDEGdgEmJX7MmU1emT/yaIG9lvjMpI2c5 +ADXGF2yYYa7H8zPIFyHU1RSavlT0S/K9yzIZvv+jA5KbNeGp+WWFT8MLZs0IhoCZ +d1EgLUYAt7LPUSm2lBy1w/IL+VtYuyn/UVFo2xWiHd1ABiNWl1ji3X9Ki5613QqH +bvn4z46voCzdZ02rYkAwrdqDr92fiBR8ctwA0AudaG6nf2ztmFKtM3E/RPMkPgKF +8NHYc7QxS2jruJxXBtjRBMtoIaZ0+AXUO6WuEJrDLDHWaM08WKByQMm808xNCbRr +CpiK8qyR3SwkfaOMCp22mqViirQ2KfuVvBpBT2pBYlgDKs50nE+stDjUMv+FDKAo +5NtiyPfNtaBOYnXAEQb/hjjW5bKq7JxHSxIWAYKbNKIWgftJ3ACZAsBMHfaOCFNH ++XLojAoxOI+0zbN6FtjN+YMU1XrLd6K49v7GEiJQZVQSfLCecVDhDU9paNROA/Xq +/3nDCTKhd3stTPnc8ymLAwhTP0bSoFh/KtU96D9ZMC2cu9XZ+UcSQYES/ncZWcLw +wTKrt+VwBG1z3DbV2O0ruUiXTLcZMsrwbUSDx1RVhmKZ0i42AttMdauFQ9JaX2CS +2ddqFBS1b4X6+VCy44KkpdXsmp0NWMgm/PM3PTisCxrha7bI5/LqfXG0b+GuIFb4 +h/lEA0Ae0gMgkzm3ePAPPVlRj7kFl5Osjxm3YVRW23WWGDRF5ywIROlBjbdozA0a +MyMgXlG9hhJseIpFveoiwqenNE5Wxg0yQbnhMUTKeCQ0xskG82P+c9bvDsevAQUR +uv1JAGGxDd1/4nk0M5m9/Gf4Bn0uLAz29LdMg0FFUvAm2ol3U3uChm7OISU8dqFy +JdCFACKBMzAREiXfgH2TrTxAhpy5uVcUSQV8x5J8qJ/mUoTF1WE3meXEm9CIvIAF +Mz49KKebLS3zGFixMcKLAOKA+s/tUWO7ZZoJyQjvQVerLyDo6UixVb11LQUJQOXb +ZIuSKV7deCgBDQ26C42SpF3rHfEQa7XH7j7tl1IIW/9DfYJYVQHaz1NTq6zcjWS2 +e+cUexBPhxbadGn0zelXr6DLJqQT7kaVeYOHlkYUHkZXdHE4CWoHqOboeB02uM/A +e7nge1rDi57ySrsF4AVl59QJYBPR43AOVbCJAh8EGAECAAkFAlCJNL4CGwwACgkQ +ty9exhhcp3DetA/8D/IscSBlWY3TjCD2P7t3+X34USK8EFD3QJse9dnCWOLcskFQ +IoIfhRM752evFu2W9owEvxSQdG+otQAOqL72k1EH2g7LsADuV8I4LOYOnLyeIE9I +b+CFPBkmzTEzrdYp6ITUU7qqgkhcgnltKGHoektIjxE8gtxCKEdyxkzazum6nCQQ +kSBZOXVU3ezm+A2QHHP6XT1GEbdKbJ0tIuJR8ADu08pBx2c/LDBBreVStrrt1Dbz +uR+U8MJsfLVcYX/Rw3V+KA24oLRzg91y3cfi3sNU/kmd5Cw42Tj00B+FXQny51Mq +s4KyqHobj62II68eL5HRB2pcGsoaedQyxu2cYSeVyarBOiUPNYkoGDJoKdDyZRIB +NNK0W+ASTf0zeHhrY/okt1ybTVtvbt6wkTEbKVePUaYmNmhre1cAj4uNwFzYjkzJ +cm+8XWftD+TV8cE5DyVdnF00SPDuPzodRAPXaGpQUMLkE4RPr1TAwcuoPH9aFHZ/ +se6rw6TQHLd0vMk0U/DocikXpSJ1N6caE3lRwI/+nGfXNiCr8MIdofgkBeO86+G7 +k0UXS4v5FKk1nwTyt4PkFJDvAJX6rZPxIZ9NmtA5ao5vyu1DT5IhoXgDzwurAe8+ +R+y6gtA324hXIweFNt7SzYPfI4SAjunlmm8PIBf3owBrk3j+w6EQoaCreK4= +=6HcJ +-----END PGP PRIVATE KEY BLOCK----- diff --git a/src/leap/bitmask/mail/smtp/tests/185CA770.pub b/src/leap/bitmask/mail/smtp/tests/185CA770.pub new file mode 100644 index 0000000..38af19f --- /dev/null +++ b/src/leap/bitmask/mail/smtp/tests/185CA770.pub @@ -0,0 +1,52 @@ +-----BEGIN PGP PUBLIC KEY BLOCK----- +Version: GnuPG v1.4.10 (GNU/Linux) + +mQINBFCJNL4BEADFsI1TCD4yq7ZqL7VhdVviTuX6JUps8/mVEhRVOZhojLcTYaqQ +gs6T6WabRxcK7ymOnf4K8NhYdz6HFoJN46BT87etokx7J/Sl2OhpiqBQEY+jW8Rp ++3MSGrGmvFw0s1lGrz/cXzM7UNgWSTOnYZ5nJS1veMhy0jseZOUK7ekp2oEDjGZh +pzgd3zICCR2SvlpLIXB2Nr/CUcuRWTcc5LlKmbjMybu0E/uuY14st3JL+7qI6QX0 +atFm0VhFVpagOl0vWKxakUx4hC7j1wH2ADlCvSZPG0StSLUyHkJx3UPsmYxOZFao +ATED3Okjwga6E7PJEbzyqAkvzw/M973kaZCUSH75ZV0cQnpdgXV3DK1gSa3d3gug +W1lE0V7pwnN2NTOYfBMi+WloCs/bp4iZSr4QP1duZ3IqKraeBDCk7MoFo4A9Wk07 +kvqPwF9IBgatu62WVEZIzwyViN+asFUGfgp+8D7gtnlWAw0V6y/lSTzyl+dnLP98 +Hfr2eLBylFs+Kl3Pivpg2uHw09LLCrjeLEN3dj9SfBbA9jDIo9Zhs1voiIK/7Shx +E0BRJaBgG3C4QaytYEu7RFFOKuvBai9w2Y5OfsKFo8rA7v4dxFFDvzKGujCtNnwf +oyaGlZmMBU5MUmHUNiG8ON21COZBtK5oMScuY1VC9CQonj3OClg3IbU9SQARAQAB +tCRkcmVicyAoZ3BnIHRlc3Qga2V5KSA8ZHJlYnNAbGVhcC5zZT6JAjgEEwECACIF +AlCJNL4CGwMGCwkIBwMCBhUIAgkKCwQWAgMBAh4BAheAAAoJELcvXsYYXKdwXYcP +/23T1DW6eq4CI0UbsGLxieiLohbhViUNJt4gLrgKGaORhwD6mQLhpHTFWLtHbQrW +evDCf0KYwQk/UUS/poCIHy31Eo9mZJz+J2+m2ohubtvJdCiHf3uSAokCgZyLI8Ss +YSwVR0s0WktT/r2XhdgtkfV8jctHgpwMMgjHA9EfE1UThkO4uGULDGzI9Sz6TsIV +fxBqdt5w/rqVjlW5/8EBzmFdpDIgP/H4s5lkSFrFeP+vLjAPlgz+C+NAJydH3LNO +H/XXjPT+qY5ZtNFQ+6pJWBBsOjKa9oPzO95BodZ5QL/iJkARpSzJXkV8sx2N1gIc +5kSljuJKBbinkfUNBDM5NkeP5LmoHVhKTPm4cBOMT1dbT2YZ/ahU86UA+Kf1U+Kj +UCjg9PmkxVq3uuUbdnBRTCyO1JhBzS4tDQfSQiAIuPpfm/VkDZZmjgvBIThsEc5n +oVMh5lzNwTVtvDJ8ihSe2wRTHe7ic/rwDzq6+js7jb1Qr9iNiApE7IoSI/88ORl8 +ZDzu2GzLvayhtJyhdTdyQdYFCe/Ax5oW43mhkrx8PObDw9P9XT2SuPBooPIutmgs +X/DpST5bfGYQQaJc7OiE1DWu+UcPXybI91Frr9lRWiLkee2xVpmN7APSj4HX+41U +JQGSpVcpYuyJMmmZsQ2TRUKE3SAvbqNGoICAz/Myr9J9uQINBFCJNL4BEADjvAQE +e7AXao9kUTETXlkgOEvRrh31d2EAEZ6mLGyE7NY0YAKO48fWeQvNHGTsTlFSuwiY +7U6lNCQfUD/88FfYZOYusumkE8rHJUMkzTddGl7v98yZWuQ7oc/grPuVHFQJjSV/ +RYEzFb4SgxCGPyzZm/TKASzLrdQpA8fljwleOVjiAQg3Y9h6SI4nuKYbNP+uUloz +4lYN1rBWZY1caSQhlz+PBHrdm3/Gn2B9Zcd7EDHDW8iQAiD7c/mCs3O4yCsCkls4 +NVcmIpG5PXlgsXM+lSjSSoveGB/XO1PF903WfBtJBmFJXY2A73rMpZ7kJXBgrUhi +jgIazfwbtkdIKRYGJduO7u/Qpo6LXnzHb/ARlA1urvf0nqfcTd3VvEvO4ejP7b7N +AoZZAw6SXncwne04XYpBrxUSX1v/LE//qqEyrJMkkGgyHjSZ8WbsisBCyzAbwmRq +WpsJp5BBob7mVqtIfw8Mk7OEABPDlR1yUjN1De4TOJSSFIfrESidRQF103UmOHFD +WkvjirY9iyfOcxaoXwGeKRcCwoGGybVOYXo9i5zCTW2HEpI8W5PnOSI5KGURfhMY +2ZXNFS2DU+e2a4sGf6V+NqH4zsjEf3ZB0LhYpP+Cu79WXpn2k3ZtO0hDAZJ0ejmb +Mv+fr4hLV4ood6bUzGk/a6MPoPnBaMXV5IrCrwARAQABiQIfBBgBAgAJBQJQiTS+ +AhsMAAoJELcvXsYYXKdw3rQP/A/yLHEgZVmN04wg9j+7d/l9+FEivBBQ90CbHvXZ +wlji3LJBUCKCH4UTO+dnrxbtlvaMBL8UkHRvqLUADqi+9pNRB9oOy7AA7lfCOCzm +Dpy8niBPSG/ghTwZJs0xM63WKeiE1FO6qoJIXIJ5bShh6HpLSI8RPILcQihHcsZM +2s7pupwkEJEgWTl1VN3s5vgNkBxz+l09RhG3SmydLSLiUfAA7tPKQcdnPywwQa3l +Ura67dQ287kflPDCbHy1XGF/0cN1figNuKC0c4Pdct3H4t7DVP5JneQsONk49NAf +hV0J8udTKrOCsqh6G4+tiCOvHi+R0QdqXBrKGnnUMsbtnGEnlcmqwTolDzWJKBgy +aCnQ8mUSATTStFvgEk39M3h4a2P6JLdcm01bb27esJExGylXj1GmJjZoa3tXAI+L +jcBc2I5MyXJvvF1n7Q/k1fHBOQ8lXZxdNEjw7j86HUQD12hqUFDC5BOET69UwMHL +qDx/WhR2f7Huq8Ok0By3dLzJNFPw6HIpF6UidTenGhN5UcCP/pxn1zYgq/DCHaH4 +JAXjvOvhu5NFF0uL+RSpNZ8E8reD5BSQ7wCV+q2T8SGfTZrQOWqOb8rtQ0+SIaF4 +A88LqwHvPkfsuoLQN9uIVyMHhTbe0s2D3yOEgI7p5ZpvDyAX96MAa5N4/sOhEKGg +q3iu +=RChS +-----END PGP PUBLIC KEY BLOCK----- diff --git a/src/leap/bitmask/mail/smtp/tests/cert/server.crt b/src/leap/bitmask/mail/smtp/tests/cert/server.crt new file mode 100644 index 0000000..a27391c --- /dev/null +++ b/src/leap/bitmask/mail/smtp/tests/cert/server.crt @@ -0,0 +1,29 @@ +-----BEGIN CERTIFICATE----- +MIIFBjCCAu4CCQCWn3oMoQrDJTANBgkqhkiG9w0BAQUFADBFMQswCQYDVQQGEwJV +UzETMBEGA1UECAwKU29tZS1TdGF0ZTEhMB8GA1UECgwYSW50ZXJuZXQgV2lkZ2l0 +cyBQdHkgTHRkMB4XDTEzMTAyMzE0NDUwNFoXDTE2MDcxOTE0NDUwNFowRTELMAkG +A1UEBhMCVVMxEzARBgNVBAgMClNvbWUtU3RhdGUxITAfBgNVBAoMGEludGVybmV0 +IFdpZGdpdHMgUHR5IEx0ZDCCAiIwDQYJKoZIhvcNAQEBBQADggIPADCCAgoCggIB +APexTvEvG7cSmZdAERHt9TB11cSor54Y/F7NmYMdSOJNi4Y0kwkSslpdfipi+mt/ +NFg/uGKi1mcgvuXdVbVPZ9rCgVpIzMncO8RAP7a5+I2zKUzqMCCbLH16sYpo/rDk +VQ5V15TwLsTzOFGG8Cgp68TR8zHuZ4Edf2zMGC1IaiJ6W38LTnJgsowYOCFDAF3z +L36kxMO5gNGEUYV6tjltx+rAcXka3po+xiAgvW6q65UUgDHcIdEGG2dc9bkxxPl7 +RkprF2RwwADNzYS7Tn+Hpmjy06pfYZHNME+Iw515bCRF3GQFUU4BpGnY7EO+h4P9 +Kb1h948gUT9/oswXG+q2Kwk8AoggMJkUOWDFiCa5UjW1GBoxxb7VtZ+QTJXxlFWc +M2VzT7M/HX+P4b05vY4MXJjxPAFKrAGS7J8DKW8WJNUnXa9XSDBHg5qijDzZ/zGm +HTdG6iADnJLmOHBQgFQ12a/n9mYV2GPVC6FlgDzG9f0/SUPBUCafyWYz1LwKY4VM +2NLx/iwYMQsNIMSZQfNmufNDBr70+BShe3ZpbmKB/J33d87AuJd2HjnsThTEAAr+ +6CejyYmwFutoDUCF8IaKGJEp7OGP2//ub4nt5WwW8DYLRi8EqtzEnxPo5ZiayHMY +GHR1jpX1O5JVJFUE79bZCFFHKmtJc4kVZS4m4rTLsk83AgMBAAEwDQYJKoZIhvcN +AQEFBQADggIBAEt4PIRqVuALQSdgZ+GiZYuvEVjxoDVtMSc/ym93Gi8R7DDivFH9 +4suQc5QUiuEF8lpEtkmh+PZ+oFdQkjhBH80h7p4BUSyBy5Yi6dy7ATTlBAqwzCYZ +4wzHeJzu1SI6FinZLksoULbcw04n410aGHkLa6I9O3vCC4kXSnBlwU1sUsJphxM2 +3pkHBpvv79XYf5kFqZPzF16aO7rxFuVvqgXLyzwuyP9kH5zMA21Kioxs/pNyg1lm +5h0VinpHLPse+4tYih1L1WLMpEZiSwZgFhoRtlcdIVXokZPaX4G2EkdrMmSQruWg +Uz8Av6LEYHmRfbYwYM2kEX/+AF8thpTQDbvxjqYk5oyGX4wpKGpih1ac/jYu3O8B +VLhbxZlBYcLxCqqNsGJrWaiHj2Jf4GhUB0O9hXfaZDMqEGXT9GzOz0yF6b3pDQVy +H0lKIBb+kQzB/jhZKu4vrTAowXtt/av5d7D+rpAU1SxfUhBOPNSRoJUI5NSBbokp +a7u4azdB2IQETX3d2rhDk09EbG1XmMi5Vg1oa8nxfMOWXZnDMusJoZClKjrthmwd +rtR5et44XYhX6p217RBkYMDOVFT7aZpu4SaFeqZIuarVYodSmgXToOFXPsrLppRQ +adOT0FpU64RPNrQz5NF1bSIjqrHSaRVacf8yr7qqxNnpMsrtkDJzsMBz +-----END CERTIFICATE----- diff --git a/src/leap/bitmask/mail/smtp/tests/cert/server.key b/src/leap/bitmask/mail/smtp/tests/cert/server.key new file mode 100644 index 0000000..197a449 --- /dev/null +++ b/src/leap/bitmask/mail/smtp/tests/cert/server.key @@ -0,0 +1,51 @@ +-----BEGIN RSA PRIVATE KEY----- +MIIJKQIBAAKCAgEA97FO8S8btxKZl0AREe31MHXVxKivnhj8Xs2Zgx1I4k2LhjST +CRKyWl1+KmL6a380WD+4YqLWZyC+5d1VtU9n2sKBWkjMydw7xEA/trn4jbMpTOow +IJssfXqximj+sORVDlXXlPAuxPM4UYbwKCnrxNHzMe5ngR1/bMwYLUhqInpbfwtO +cmCyjBg4IUMAXfMvfqTEw7mA0YRRhXq2OW3H6sBxeRremj7GICC9bqrrlRSAMdwh +0QYbZ1z1uTHE+XtGSmsXZHDAAM3NhLtOf4emaPLTql9hkc0wT4jDnXlsJEXcZAVR +TgGkadjsQ76Hg/0pvWH3jyBRP3+izBcb6rYrCTwCiCAwmRQ5YMWIJrlSNbUYGjHF +vtW1n5BMlfGUVZwzZXNPsz8df4/hvTm9jgxcmPE8AUqsAZLsnwMpbxYk1Sddr1dI +MEeDmqKMPNn/MaYdN0bqIAOckuY4cFCAVDXZr+f2ZhXYY9ULoWWAPMb1/T9JQ8FQ +Jp/JZjPUvApjhUzY0vH+LBgxCw0gxJlB82a580MGvvT4FKF7dmluYoH8nfd3zsC4 +l3YeOexOFMQACv7oJ6PJibAW62gNQIXwhooYkSns4Y/b/+5vie3lbBbwNgtGLwSq +3MSfE+jlmJrIcxgYdHWOlfU7klUkVQTv1tkIUUcqa0lziRVlLibitMuyTzcCAwEA +AQKCAgAFQdcqGVTeQt/NrQdvuPw+RhH+dZIcqe0ZWgXLGaEFZJ30gEMqqyHr9xYJ +ckZcZ7vFr7yLI2enkrNaj6MVERVkOEKkluz5z9nY5YA0safL4iSbRFE3L/P2ydbg +2C+ns4D2p+3GdH6ZoYvtdw6723/skoQ16Bh8ThL5TS+qLmJKTwyIGsZUeSbxAEaY +tiJY3COC7Z5bhSFt0QAl9B/QAjt/CQyfhGl7Hp/36Jn8slYDuQariD+TfyyvufJh +NuQ2Y15vj+xULmx01+lnys30uP1YNuc1M4cPoCpJVd7JBd28u1rdKJu8Kx7BPGBv +Y6jerU3ofh7SA96VmXDsIgVuquUo51Oklspe6a9VaDmzLvjYqJsBKQ7BH3J2f07x +NiOob56CGXykX51Ig3WBK1wKn+pA69FL62DbkEa6SykGCqdZPdgBF/kiMc0TESsl +867Em63Yx/2hq+mG3Dknnq8jWXf+Es/zZSSak6N4154IxPOD3m1hzuUq73PP7Ptt +KFe6NfU0DmAuTJL3FqNli8F8lFfvJfuwmW2qk5iTMfwPxybSd8FPbGxi7aRgoZdh +7fIbTFJ0X2f83/SO+9rCzV+B091+d7TM8AaOJ4dEoS74rlRZg53EgmAU0phVnE+l +taMNKGHy2kpJrv9IHX3w5Gm6CjNJj5t4ccS0J18NFFJ+j077eQKCAQEA/RJNRUBS +mI5l0eirl78Q9uDPh1usChZpQiLsvscIJITWQ1vtXSRCvP0hVQRRv8+4CtrZr2rX +v0afkzg/3HNFaNsjYT6aHjgnombFqfpyS/NZN/p3gOzi2h+1Sujzz5fBUGhNLVgZ +F2GLnJbiIHnM1BmKA6597pHpXcRMh1E3DSjDMQAEEsBgF6MyS+MT9WfNwHvJukii +k028tNzR4wRq3Xo3WTfvXZRjbX54Ew9Zy3+TFiu19j2FmuOoqyj+ZvMic4EYmTaY +BWm7viDff4dW34dR9sYCuTWWehLtMJGroA38e7lTLfNOHNDGaUZWkfxs4uJCsxvP +0fPp3xlbU3NUGwKCAQEA+o8SeHwEN+VN2dZvC3wFvbnRvWLc1aLnNcndRE9QLVcC +B4LMRuQMpxaNYRiSQPppoPTNq6zWbo6FEjUO5Md7R8I8dbg1vHo4PzuHOu2wXNcm +DEicocCpSKShSS27NCK6uoSsTqTIlG4u+1x9/R2gJEjlTqjeIkOQkPv7PbWhrUyt +XqvzPy4bewOz9Brmd6ryi8ZLtNbUSNwMyd64s9b1V4A6JRlYZrMDOQ6kXEZo+mbL +ynet0vuj7lYxsAZvxoPIq+Gi5i0CrDYtze6JCg+kGahjMX0zXRjXrYh/YID8NWYT +0GXr2+a0V5pXg86YCDp/jpr3lq75HJJ+vIvm2VHLFQKCAQATEm0GWgmfe6PKxPkh +j4GsyVZ6gfseK4A1PsKOwhsn/WbUXrotuczZx03axV+P0AyzrLiZErk9rgnao3OU +no9Njq5E5t3ghyTdhVdCLyCr/qPrpxGYgsG55IfaJGIzc+FauPGQCEKj03MdEvXp +sqQwG9id3GmbMB3hNij6TbGTaU4EhFbKPvs+7Mqek3dumCsWZX3Xbx/pcANXsgiT +TkLrfAltzNxaNhOkLdLIxPBkeLHSCutEqnBGMwAEHivGAG7JO6Jp8YZVahl/A6U0 +TDPM1rrjmRqdcJ9thb2gWmoPvt4XSOku3lY1r7o0NtvRVq+yDZEvRFpOHU6zxIpw +aJGfAoIBAQDiTvvF62379pc8nJwr6VdeKEozHuqL49mmEbBTFLg8W4wvsIpFtZFg +EdSc0I65NfTWNobV+wSrUvsKmPXc2fiVtfDZ+wo+NL49Ds10Al/7WzC4g5VF3DiK +rngnGrEtw/iYo2Dmn5uzxVmWG9KIHowYeeb0Bz6sAA7BhXdGI5nmZ41oJzNL659S +muOdJfboO3Vbnj2fFzMio+7BHvQBK7Tp1Z2vCJd6G1Jb5Me7uLT1BognVbWhDTzh +9uRmM0oeKcXEycZS1HDHjyAMEtmgRsRXkGoXtxf/jIKx8MnsJlSm/o4C+yvvsQ9O +2M8W9DEJrZys93eNmHjUv9TNBCf8Pg6JAoIBAQDDItnQPLntCUgd7dy0dDjQYBGN +4wVRJNINpgjqwJj0hVjB/dmvrcxkXcOG4VAH+iNH8A25qLU+RTDcNipuL3uEFKbF +O4DSjFih3qL1Y8otTXSrPeqZOMvYpY8dXS5uyI7DSWQQZyZ9bMpeWbxgx4LHqPPH +rdcVJy9Egw1ZIOA7JBFM02uGn9TVwFzNUJk0G/3xwVHzDxYNbJ98vDfflc2vD4CH +OAN6un0pOuol2h200F6zFgc5mbETWHCPIom+ZMXIX3bq7g341c/cgqIELPTk8DLS +s+AgrZ4qYmskrFaD0PHakWsQNHGC8yOh80lgE3Gl4nxSGAvkcR7dkSmsIQFL +-----END RSA PRIVATE KEY----- diff --git a/src/leap/bitmask/mail/smtp/tests/mail.txt b/src/leap/bitmask/mail/smtp/tests/mail.txt new file mode 100644 index 0000000..9542047 --- /dev/null +++ b/src/leap/bitmask/mail/smtp/tests/mail.txt @@ -0,0 +1,10 @@ +HELO drebs@riseup.net +MAIL FROM: drebs@riseup.net +RCPT TO: drebs@riseup.net +RCPT TO: drebs@leap.se +DATA +Subject: leap test + +Hello world! +. +QUIT diff --git a/src/leap/bitmask/mail/smtp/tests/test_gateway.py b/src/leap/bitmask/mail/smtp/tests/test_gateway.py new file mode 100644 index 0000000..9d88afb --- /dev/null +++ b/src/leap/bitmask/mail/smtp/tests/test_gateway.py @@ -0,0 +1,181 @@ +# -*- coding: utf-8 -*- +# test_gateway.py +# Copyright (C) 2013 LEAP +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . + +""" +SMTP gateway tests. +""" +import re +import tempfile +from datetime import datetime + +from twisted.internet import reactor +from twisted.internet.defer import inlineCallbacks, fail, succeed, Deferred +from twisted.test import proto_helpers + +from mock import Mock + +from leap.keymanager import openpgp, errors +from leap.mail.testing import KeyManagerWithSoledadTestCase +from leap.mail.testing import ADDRESS, ADDRESS_2 +from leap.mail.testing.smtp import getSMTPFactory, TEST_USER + + +# some regexps +IP_REGEX = "(([0-9]|[1-9][0-9]|1[0-9]{2}|2[0-4][0-9]|25[0-5])\.){3}" + \ + "([0-9]|[1-9][0-9]|1[0-9]{2}|2[0-4][0-9]|25[0-5])" +HOSTNAME_REGEX = "(([a-zA-Z0-9]|[a-zA-Z0-9][a-zA-Z0-9\-]*[a-zA-Z0-9])\.)*" + \ + "([A-Za-z0-9]|[A-Za-z0-9][A-Za-z0-9\-]*[A-Za-z0-9])" +IP_OR_HOST_REGEX = '(' + IP_REGEX + '|' + HOSTNAME_REGEX + ')' + + +class TestSmtpGateway(KeyManagerWithSoledadTestCase): + + EMAIL_DATA = ['HELO gateway.leap.se', + 'MAIL FROM: <%s>' % ADDRESS_2, + 'RCPT TO: <%s>' % ADDRESS, + 'DATA', + 'From: User <%s>' % ADDRESS_2, + 'To: Leap <%s>' % ADDRESS, + 'Date: ' + datetime.now().strftime('%c'), + 'Subject: test message', + '', + 'This is a secret message.', + 'Yours,', + 'A.', + '', + '.', + 'QUIT'] + + def setUp(self): + # pytest handles correctly the setupEnv for the class, + # but trial ignores it. + if not getattr(self, 'tempdir', None): + self.tempdir = tempfile.mkdtemp() + return KeyManagerWithSoledadTestCase.setUp(self) + + def tearDown(self): + return KeyManagerWithSoledadTestCase.tearDown(self) + + def assertMatch(self, string, pattern, msg=None): + if not re.match(pattern, string): + msg = self._formatMessage(msg, '"%s" does not match pattern "%s".' + % (string, pattern)) + raise self.failureException(msg) + + @inlineCallbacks + def test_gateway_accepts_valid_email(self): + """ + Test if SMTP server responds correctly for valid interaction. + """ + + SMTP_ANSWERS = ['220 ' + IP_OR_HOST_REGEX + + ' NO UCE NO UBE NO RELAY PROBES', + '250 ' + IP_OR_HOST_REGEX + ' Hello ' + + IP_OR_HOST_REGEX + ', nice to meet you', + '250 Sender address accepted', + '250 Recipient address accepted', + '354 Continue'] + + user = TEST_USER + proto = getSMTPFactory({user: None}, {user: self.km}, {user: None}) + transport = proto_helpers.StringTransport() + proto.makeConnection(transport) + reply = "" + for i, line in enumerate(self.EMAIL_DATA): + reply += yield self.getReply(line + '\r\n', proto, transport) + self.assertMatch(reply, '\r\n'.join(SMTP_ANSWERS), + 'Did not get expected answer from gateway.') + proto.setTimeout(None) + + @inlineCallbacks + def test_missing_key_rejects_address(self): + """ + Test if server rejects to send unencrypted when 'encrypted_only' is + True. + """ + # remove key from key manager + pubkey = yield self.km.get_key(ADDRESS) + pgp = openpgp.OpenPGPScheme( + self._soledad, gpgbinary=self.gpg_binary_path) + yield pgp.delete_key(pubkey) + # mock the key fetching + self.km._fetch_keys_from_server = Mock( + return_value=fail(errors.KeyNotFound())) + user = TEST_USER + proto = getSMTPFactory( + {user: None}, {user: self.km}, {user: None}, + encrypted_only=True) + transport = proto_helpers.StringTransport() + proto.makeConnection(transport) + yield self.getReply(self.EMAIL_DATA[0] + '\r\n', proto, transport) + yield self.getReply(self.EMAIL_DATA[1] + '\r\n', proto, transport) + reply = yield self.getReply(self.EMAIL_DATA[2] + '\r\n', + proto, transport) + # ensure the address was rejected + self.assertEqual( + '550 Cannot receive for specified address\r\n', + reply, + 'Address should have been rejected with appropriate message.') + proto.setTimeout(None) + + @inlineCallbacks + def test_missing_key_accepts_address(self): + """ + Test if server accepts to send unencrypted when 'encrypted_only' is + False. + """ + # remove key from key manager + pubkey = yield self.km.get_key(ADDRESS) + pgp = openpgp.OpenPGPScheme( + self._soledad, gpgbinary=self.gpg_binary_path) + yield pgp.delete_key(pubkey) + # mock the key fetching + self.km._fetch_keys_from_server = Mock( + return_value=fail(errors.KeyNotFound())) + user = TEST_USER + proto = getSMTPFactory({user: None}, {user: self.km}, {user: None}) + transport = proto_helpers.StringTransport() + proto.makeConnection(transport) + yield self.getReply(self.EMAIL_DATA[0] + '\r\n', proto, transport) + yield self.getReply(self.EMAIL_DATA[1] + '\r\n', proto, transport) + reply = yield self.getReply(self.EMAIL_DATA[2] + '\r\n', + proto, transport) + # ensure the address was accepted + self.assertEqual( + '250 Recipient address accepted\r\n', + reply, + 'Address should have been accepted with appropriate message.') + proto.setTimeout(None) + + def getReply(self, line, proto, transport): + proto.lineReceived(line) + + if line[:4] not in ['HELO', 'MAIL', 'RCPT', 'DATA']: + return succeed("") + + def check_transport(_): + reply = transport.value() + if reply: + transport.clear() + return succeed(reply) + + d = Deferred() + d.addCallback(check_transport) + reactor.callLater(0, lambda: d.callback(None)) + return d + + return check_transport(None) diff --git a/src/leap/bitmask/mail/sync_hooks.py b/src/leap/bitmask/mail/sync_hooks.py new file mode 100644 index 0000000..8efbb7c --- /dev/null +++ b/src/leap/bitmask/mail/sync_hooks.py @@ -0,0 +1,120 @@ +# -*- coding: utf-8 -*- +# sync_hooks.py +# Copyright (C) 2015 LEAP +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . +""" +Soledad PostSync Hooks. + +Process every new document of interest after every soledad synchronization, +using the hooks that soledad exposes via plugins. +""" +import logging + +from re import compile as regex_compile + +from zope.interface import implements +from twisted.internet import defer +from twisted.plugin import IPlugin +from twisted.python import log + +from leap.soledad.client.interfaces import ISoledadPostSyncPlugin +from leap.mail import constants + +logger = logging.getLogger(__name__) + + +def _get_doc_type_preffix(s): + return s[:2] + + +class MailProcessingPostSyncHook(object): + implements(IPlugin, ISoledadPostSyncPlugin) + + META_DOC_PREFFIX = _get_doc_type_preffix(constants.METAMSGID) + watched_doc_types = (META_DOC_PREFFIX, ) + + _account = None + _pending_docs = [] + _processing_deferreds = [] + + def process_received_docs(self, doc_id_list): + if self._has_configured_account(): + process_fun = self._make_uid_index + else: + self._processing_deferreds = [] + process_fun = self._queue_doc_id + + for doc_id in doc_id_list: + if _get_doc_type_preffix(doc_id) in self.watched_doc_types: + log.msg("Mail post-sync hook: processing %s" % doc_id) + process_fun(doc_id) + + return defer.gatherResults(self._processing_deferreds) + + def set_account(self, account): + self._account = account + if account: + self._process_queued_docs() + + def _has_configured_account(self): + return self._account is not None + + def _queue_doc_id(self, doc_id): + self._pending_docs.append(doc_id) + + def _make_uid_index(self, mdoc_id): + indexer = self._account.mbox_indexer + mbox_uuid = _get_mbox_uuid(mdoc_id) + if mbox_uuid: + chash = _get_chash_from_mdoc(mdoc_id) + logger.debug("Making index table for %s:%s" % (mbox_uuid, chash)) + index_docid = constants.METAMSGID.format( + mbox_uuid=mbox_uuid.replace('-', '_'), + chash=chash) + # XXX could avoid creating table if I track which ones I already + # have seen -- but make sure *it's already created* before + # inserting the index entry!. + d = indexer.create_table(mbox_uuid) + d.addBoth(lambda _: indexer.insert_doc(mbox_uuid, index_docid)) + self._processing_deferreds.append(d) + + def _process_queued_docs(self): + assert(self._has_configured_account()) + pending = self._pending_docs + log.msg("Mail post-sync hook: processing queued docs") + + def remove_pending_docs(res): + self._pending_docs = [] + return res + + d = self.process_received_docs(pending) + d.addCallback(remove_pending_docs) + return d + + +_mbox_uuid_regex = regex_compile(constants.METAMSGID_MBOX_RE) +_mdoc_chash_regex = regex_compile(constants.METAMSGID_CHASH_RE) + + +def _get_mbox_uuid(doc_id): + matches = _mbox_uuid_regex.findall(doc_id) + if matches: + return matches[0].replace('_', '-') + + +def _get_chash_from_mdoc(doc_id): + matches = _mdoc_chash_regex.findall(doc_id) + if matches: + return matches[0] diff --git a/src/leap/bitmask/mail/testing/__init__.py b/src/leap/bitmask/mail/testing/__init__.py new file mode 100644 index 0000000..8822a5c --- /dev/null +++ b/src/leap/bitmask/mail/testing/__init__.py @@ -0,0 +1,353 @@ +# -*- coding: utf-8 -*- +# __init__.py +# Copyright (C) 2013 LEAP +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . +""" +Base classes and keys for leap.mail tests. +""" +import os +import distutils.spawn +from mock import Mock +from twisted.internet.defer import gatherResults +from twisted.trial import unittest +from twisted.internet import defer + + +from leap.soledad.client import Soledad +from leap.keymanager import KeyManager + + +from leap.common.testing.basetest import BaseLeapTest + +ADDRESS = 'leap@leap.se' +ADDRESS_2 = 'anotheruser@leap.se' + + +class defaultMockSharedDB(object): + get_doc = Mock(return_value=None) + put_doc = Mock(side_effect=None) + open = Mock(return_value=None) + close = Mock(return_value=None) + syncable = True + + def __call__(self): + return self + + +class KeyManagerWithSoledadTestCase(unittest.TestCase, BaseLeapTest): + + def setUp(self): + self.gpg_binary_path = self._find_gpg() + + self._soledad = Soledad( + u"leap@leap.se", + u"123456", + secrets_path=self.tempdir + "/secret.gpg", + local_db_path=self.tempdir + "/soledad.u1db", + server_url='', + cert_file=None, + auth_token=None, + shared_db=defaultMockSharedDB(), + syncable=False) + + self.km = self._key_manager() + + class Response(object): + code = 200 + phrase = '' + + def deliverBody(self, x): + return '' + + self.km._async_client_pinned.request = Mock( + return_value=defer.succeed(Response())) + + d1 = self.km.put_raw_key(PRIVATE_KEY, ADDRESS) + d2 = self.km.put_raw_key(PRIVATE_KEY_2, ADDRESS_2) + return gatherResults([d1, d2]) + + def tearDown(self): + km = self._key_manager() + # wait for the indexes to be ready for the tear down + d = km._openpgp.deferred_init + d.addCallback(lambda _: self.delete_all_keys(km)) + d.addCallback(lambda _: self._soledad.close()) + return d + + def delete_all_keys(self, km): + def delete_keys(keys): + deferreds = [] + for key in keys: + d = km._openpgp.delete_key(key) + deferreds.append(d) + return gatherResults(deferreds) + + def check_deleted(_, private): + d = km.get_all_keys(private=private) + d.addCallback(lambda keys: self.assertEqual(keys, [])) + return d + + deferreds = [] + for private in [True, False]: + d = km.get_all_keys(private=private) + d.addCallback(delete_keys) + d.addCallback(check_deleted, private) + deferreds.append(d) + return gatherResults(deferreds) + + def _key_manager(self, user=ADDRESS, url='', token=None, + ca_cert_path=None): + return KeyManager(user, url, self._soledad, token=token, + gpgbinary=self.gpg_binary_path, + ca_cert_path=ca_cert_path) + + def _find_gpg(self): + gpg_path = distutils.spawn.find_executable('gpg') + if gpg_path is not None: + return os.path.realpath(gpg_path) + else: + return "/usr/bin/gpg" + + def get_public_binary_key(self): + with open(PATH + '/fixtures/public_key.bin', 'r') as binary_public_key: + return binary_public_key.read() + + def get_private_binary_key(self): + with open( + PATH + '/fixtures/private_key.bin', 'r') as binary_private_key: + return binary_private_key.read() + + +# key 24D18DDF: public key "Leap Test Key " +KEY_FINGERPRINT = "E36E738D69173C13D709E44F2F455E2824D18DDF" +PUBLIC_KEY = """ +-----BEGIN PGP PUBLIC KEY BLOCK----- +Version: GnuPG v1.4.10 (GNU/Linux) + +mQINBFC9+dkBEADNRfwV23TWEoGc/x0wWH1P7PlXt8MnC2Z1kKaKKmfnglVrpOiz +iLWoiU58sfZ0L5vHkzXHXCBf6Eiy/EtUIvdiWAn+yASJ1mk5jZTBKO/WMAHD8wTO +zpMsFmWyg3xc4DkmFa9KQ5EVU0o/nqPeyQxNMQN7px5pPwrJtJFmPxnxm+aDkPYx +irDmz/4DeDNqXliazGJKw7efqBdlwTHkl9Akw2gwy178pmsKwHHEMOBOFFvX61AT +huKqHYmlCGSliwbrJppTG7jc1/ls3itrK+CWTg4txREkSpEVmfcASvw/ZqLbjgfs +d/INMwXnR9U81O8+7LT6yw/ca4ppcFoJD7/XJbkRiML6+bJ4Dakiy6i727BzV17g +wI1zqNvm5rAhtALKfACha6YO43aJzairO4II1wxVHvRDHZn2IuKDDephQ3Ii7/vb +hUOf6XCSmchkAcpKXUOvbxm1yfB1LRa64mMc2RcZxf4mW7KQkulBsdV5QG2276lv +U2UUy2IutXcGP5nXC+f6sJJGJeEToKJ57yiO/VWJFjKN8SvP+7AYsQSqINUuEf6H +T5gCPCraGMkTUTPXrREvu7NOohU78q6zZNaL3GW8ai7eSeANSuQ8Vzffx7Wd8Y7i +Pw9sYj0SMFs1UgjbuL6pO5ueHh+qyumbtAq2K0Bci0kqOcU4E9fNtdiovQARAQAB +tBxMZWFwIFRlc3QgS2V5IDxsZWFwQGxlYXAuc2U+iQI3BBMBCAAhBQJQvfnZAhsD +BQsJCAcDBRUKCQgLBRYCAwEAAh4BAheAAAoJEC9FXigk0Y3fT7EQAKH3IuRniOpb +T/DDIgwwjz3oxB/W0DDMyPXowlhSOuM0rgGfntBpBb3boezEXwL86NPQxNGGruF5 +hkmecSiuPSvOmQlqlS95NGQp6hNG0YaKColh+Q5NTspFXCAkFch9oqUje0LdxfSP +QfV9UpeEvGyPmk1I9EJV/YDmZ4+Djge1d7qhVZInz4Rx1NrSyF/Tc2EC0VpjQFsU +Y9Kb2YBBR7ivG6DBc8ty0jJXi7B4WjkFcUEJviQpMF2dCLdonCehYs1PqsN1N7j+ +eFjQd+hqVMJgYuSGKjvuAEfClM6MQw7+FmFwMyLgK/Ew/DttHEDCri77SPSkOGSI +txCzhTg6798f6mJr7WcXmHX1w1Vcib5FfZ8vTDFVhz/XgAgArdhPo9V6/1dgSSiB +KPQ/spsco6u5imdOhckERE0lnAYvVT6KE81TKuhF/b23u7x+Wdew6kK0EQhYA7wy +7LmlaNXc7rMBQJ9Z60CJ4JDtatBWZ0kNrt2VfdDHVdqBTOpl0CraNUjWE5YMDasr +K2dF5IX8D3uuYtpZnxqg0KzyLg0tzL0tvOL1C2iudgZUISZNPKbS0z0v+afuAAnx +2pTC3uezbh2Jt8SWTLhll4i0P4Ps5kZ6HQUO56O+/Z1cWovX+mQekYFmERySDR9n +3k1uAwLilJmRmepGmvYbB8HloV8HqwgguQINBFC9+dkBEAC0I/xn1uborMgDvBtf +H0sEhwnXBC849/32zic6udB6/3Efk9nzbSpL3FSOuXITZsZgCHPkKarnoQ2ztMcS +sh1ke1C5gQGms75UVmM/nS+2YI4vY8OX/GC/on2vUyncqdH+bR6xH5hx4NbWpfTs +iQHmz5C6zzS/kuabGdZyKRaZHt23WQ7JX/4zpjqbC99DjHcP9BSk7tJ8wI4bkMYD +uFVQdT9O6HwyKGYwUU4sAQRAj7XCTGvVbT0dpgJwH4RmrEtJoHAx4Whg8mJ710E0 +GCmzf2jqkNuOw76ivgk27Kge+Hw00jmJjQhHY0yVbiaoJwcRrPKzaSjEVNgrpgP3 +lXPRGQArgESsIOTeVVHQ8fhK2YtTeCY9rIiO+L0OX2xo9HK7hfHZZWL6rqymXdyS +fhzh/f6IPyHFWnvj7Brl7DR8heMikygcJqv+ed2yx7iLyCUJ10g12I48+aEj1aLe +dP7lna32iY8/Z0SHQLNH6PXO9SlPcq2aFUgKqE75A/0FMk7CunzU1OWr2ZtTLNO1 +WT/13LfOhhuEq9jTyTosn0WxBjJKq18lnhzCXlaw6EAtbA7CUwsD3CTPR56aAXFK +3I7KXOVAqggrvMe5Tpdg5drfYpI8hZovL5aAgb+7Y5ta10TcJdUhS5K3kFAWe/td +U0cmWUMDP1UMSQ5Jg6JIQVWhSwARAQABiQIfBBgBCAAJBQJQvfnZAhsMAAoJEC9F +Xigk0Y3fRwsP/i0ElYCyxeLpWJTwo1iCLkMKz2yX1lFVa9nT1BVTPOQwr/IAc5OX +NdtbJ14fUsKL5pWgW8OmrXtwZm1y4euI1RPWWubG01ouzwnGzv26UcuHeqC5orZj +cOnKtL40y8VGMm8LoicVkRJH8blPORCnaLjdOtmA3rx/v2EXrJpSa3AhOy0ZSRXk +ZSrK68AVNwamHRoBSYyo0AtaXnkPX4+tmO8X8BPfj125IljubvwZPIW9VWR9UqCE +VPfDR1XKegVb6VStIywF7kmrknM1C5qUY28rdZYWgKorw01hBGV4jTW0cqde3N51 +XT1jnIAa+NoXUM9uQoGYMiwrL7vNsLlyyiW5ayDyV92H/rIuiqhFgbJsHTlsm7I8 +oGheR784BagAA1NIKD1qEO9T6Kz9lzlDaeWS5AUKeXrb7ZJLI1TTCIZx5/DxjLqM +Tt/RFBpVo9geZQrvLUqLAMwdaUvDXC2c6DaCPXTh65oCZj/hqzlJHH+RoTWWzKI+ +BjXxgUWF9EmZUBrg68DSmI+9wuDFsjZ51BcqvJwxyfxtTaWhdoYqH/UQS+D1FP3/ +diZHHlzwVwPICzM9ooNTgbrcDzyxRkIVqsVwBq7EtzcvgYUyX53yG25Giy6YQaQ2 +ZtQ/VymwFL3XdUWV6B/hU4PVAFvO3qlOtdJ6TpE+nEWgcWjCv5g7RjXX +=MuOY +-----END PGP PUBLIC KEY BLOCK----- +""" +PRIVATE_KEY = """ +-----BEGIN PGP PRIVATE KEY BLOCK----- +Version: GnuPG v1.4.10 (GNU/Linux) + +lQcYBFC9+dkBEADNRfwV23TWEoGc/x0wWH1P7PlXt8MnC2Z1kKaKKmfnglVrpOiz +iLWoiU58sfZ0L5vHkzXHXCBf6Eiy/EtUIvdiWAn+yASJ1mk5jZTBKO/WMAHD8wTO +zpMsFmWyg3xc4DkmFa9KQ5EVU0o/nqPeyQxNMQN7px5pPwrJtJFmPxnxm+aDkPYx +irDmz/4DeDNqXliazGJKw7efqBdlwTHkl9Akw2gwy178pmsKwHHEMOBOFFvX61AT +huKqHYmlCGSliwbrJppTG7jc1/ls3itrK+CWTg4txREkSpEVmfcASvw/ZqLbjgfs +d/INMwXnR9U81O8+7LT6yw/ca4ppcFoJD7/XJbkRiML6+bJ4Dakiy6i727BzV17g +wI1zqNvm5rAhtALKfACha6YO43aJzairO4II1wxVHvRDHZn2IuKDDephQ3Ii7/vb +hUOf6XCSmchkAcpKXUOvbxm1yfB1LRa64mMc2RcZxf4mW7KQkulBsdV5QG2276lv +U2UUy2IutXcGP5nXC+f6sJJGJeEToKJ57yiO/VWJFjKN8SvP+7AYsQSqINUuEf6H +T5gCPCraGMkTUTPXrREvu7NOohU78q6zZNaL3GW8ai7eSeANSuQ8Vzffx7Wd8Y7i +Pw9sYj0SMFs1UgjbuL6pO5ueHh+qyumbtAq2K0Bci0kqOcU4E9fNtdiovQARAQAB +AA/+JHtlL39G1wsH9R6UEfUQJGXR9MiIiwZoKcnRB2o8+DS+OLjg0JOh8XehtuCs +E/8oGQKtQqa5bEIstX7IZoYmYFiUQi9LOzIblmp2vxOm+HKkxa4JszWci2/ZmC3t +KtaA4adl9XVnshoQ7pijuCMUKB3naBEOAxd8s9d/JeReGIYkJErdrnVfNk5N71Ds +FmH5Ll3XtEDvgBUQP3nkA6QFjpsaB94FHjL3gDwum/cxzj6pCglcvHOzEhfY0Ddb +J967FozQTaf2JW3O+w3LOqtcKWpq87B7+O61tVidQPSSuzPjCtFF0D2LC9R/Hpky +KTMQ6CaKja4MPhjwywd4QPcHGYSqjMpflvJqi+kYIt8psUK/YswWjnr3r4fbuqVY +VhtiHvnBHQjz135lUqWvEz4hM3Xpnxydx7aRlv5NlevK8+YIO5oFbWbGNTWsPZI5 +jpoFBpSsnR1Q5tnvtNHauvoWV+XN2qAOBTG+/nEbDYH6Ak3aaE9jrpTdYh0CotYF +q7csANsDy3JvkAzeU6WnYpsHHaAjqOGyiZGsLej1UcXPFMosE/aUo4WQhiS8Zx2c +zOVKOi/X5vQ2GdNT9Qolz8AriwzsvFR+bxPzyd8V6ALwDsoXvwEYinYBKK8j0OPv +OOihSR6HVsuP9NUZNU9ewiGzte/+/r6pNXHvR7wTQ8EWLcEIAN6Zyrb0bHZTIlxt +VWur/Ht2mIZrBaO50qmM5RD3T5oXzWXi/pjLrIpBMfeZR9DWfwQwjYzwqi7pxtYx +nJvbMuY505rfnMoYxb4J+cpRXV8MS7Dr1vjjLVUC9KiwSbM3gg6emfd2yuA93ihv +Pe3mffzLIiQa4mRE3wtGcioC43nWuV2K2e1KjxeFg07JhrezA/1Cak505ab/tmvP +4YmjR5c44+yL/YcQ3HdFgs4mV+nVbptRXvRcPpolJsgxPccGNdvHhsoR4gwXMS3F +RRPD2z6x8xeN73Q4KH3bm01swQdwFBZbWVfmUGLxvN7leCdfs9+iFJyqHiCIB6Iv +mQfp8F0IAOwSo8JhWN+V1dwML4EkIrM8wUb4yecNLkyR6TpPH/qXx4PxVMC+vy6x +sCtjeHIwKE+9vqnlhd5zOYh7qYXEJtYwdeDDmDbL8oks1LFfd+FyAuZXY33DLwn0 +cRYsr2OEZmaajqUB3NVmj3H4uJBN9+paFHyFSXrH68K1Fk2o3n+RSf2EiX+eICwI +L6rqoF5sSVUghBWdNegV7qfy4anwTQwrIMGjgU5S6PKW0Dr/3iO5z3qQpGPAj5OW +ATqPWkDICLbObPxD5cJlyyNE2wCA9VVc6/1d6w4EVwSq9h3/WTpATEreXXxTGptd +LNiTA1nmakBYNO2Iyo3djhaqBdWjk+EIAKtVEnJH9FAVwWOvaj1RoZMA5DnDMo7e +SnhrCXl8AL7Z1WInEaybasTJXn1uQ8xY52Ua4b8cbuEKRKzw/70NesFRoMLYoHTO +dyeszvhoDHberpGRTciVmpMu7Hyi33rM31K9epA4ib6QbbCHnxkWOZB+Bhgj1hJ8 +xb4RBYWiWpAYcg0+DAC3w9gfxQhtUlZPIbmbrBmrVkO2GVGUj8kH6k4UV6kUHEGY +HQWQR0HcbKcXW81ZXCCD0l7ROuEWQtTe5Jw7dJ4/QFuqZnPutXVRNOZqpl6eRShw +7X2/a29VXBpmHA95a88rSQsL+qm7Fb3prqRmuMCtrUZgFz7HLSTuUMR867QcTGVh +cCBUZXN0IEtleSA8bGVhcEBsZWFwLnNlPokCNwQTAQgAIQUCUL352QIbAwULCQgH +AwUVCgkICwUWAgMBAAIeAQIXgAAKCRAvRV4oJNGN30+xEACh9yLkZ4jqW0/wwyIM +MI896MQf1tAwzMj16MJYUjrjNK4Bn57QaQW926HsxF8C/OjT0MTRhq7heYZJnnEo +rj0rzpkJapUveTRkKeoTRtGGigqJYfkOTU7KRVwgJBXIfaKlI3tC3cX0j0H1fVKX +hLxsj5pNSPRCVf2A5mePg44HtXe6oVWSJ8+EcdTa0shf03NhAtFaY0BbFGPSm9mA +QUe4rxugwXPLctIyV4uweFo5BXFBCb4kKTBdnQi3aJwnoWLNT6rDdTe4/nhY0Hfo +alTCYGLkhio77gBHwpTOjEMO/hZhcDMi4CvxMPw7bRxAwq4u+0j0pDhkiLcQs4U4 +Ou/fH+pia+1nF5h19cNVXIm+RX2fL0wxVYc/14AIAK3YT6PVev9XYEkogSj0P7Kb +HKOruYpnToXJBERNJZwGL1U+ihPNUyroRf29t7u8flnXsOpCtBEIWAO8Muy5pWjV +3O6zAUCfWetAieCQ7WrQVmdJDa7dlX3Qx1XagUzqZdAq2jVI1hOWDA2rKytnReSF +/A97rmLaWZ8aoNCs8i4NLcy9Lbzi9QtornYGVCEmTTym0tM9L/mn7gAJ8dqUwt7n +s24dibfElky4ZZeItD+D7OZGeh0FDuejvv2dXFqL1/pkHpGBZhEckg0fZ95NbgMC +4pSZkZnqRpr2GwfB5aFfB6sIIJ0HGARQvfnZARAAtCP8Z9bm6KzIA7wbXx9LBIcJ +1wQvOPf99s4nOrnQev9xH5PZ820qS9xUjrlyE2bGYAhz5Cmq56ENs7THErIdZHtQ +uYEBprO+VFZjP50vtmCOL2PDl/xgv6J9r1Mp3KnR/m0esR+YceDW1qX07IkB5s+Q +us80v5LmmxnWcikWmR7dt1kOyV/+M6Y6mwvfQ4x3D/QUpO7SfMCOG5DGA7hVUHU/ +Tuh8MihmMFFOLAEEQI+1wkxr1W09HaYCcB+EZqxLSaBwMeFoYPJie9dBNBgps39o +6pDbjsO+or4JNuyoHvh8NNI5iY0IR2NMlW4mqCcHEazys2koxFTYK6YD95Vz0RkA +K4BErCDk3lVR0PH4StmLU3gmPayIjvi9Dl9saPRyu4Xx2WVi+q6spl3ckn4c4f3+ +iD8hxVp74+wa5ew0fIXjIpMoHCar/nndsse4i8glCddINdiOPPmhI9Wi3nT+5Z2t +9omPP2dEh0CzR+j1zvUpT3KtmhVICqhO+QP9BTJOwrp81NTlq9mbUyzTtVk/9dy3 +zoYbhKvY08k6LJ9FsQYySqtfJZ4cwl5WsOhALWwOwlMLA9wkz0eemgFxStyOylzl +QKoIK7zHuU6XYOXa32KSPIWaLy+WgIG/u2ObWtdE3CXVIUuSt5BQFnv7XVNHJllD +Az9VDEkOSYOiSEFVoUsAEQEAAQAP/1AagnZQZyzHDEgw4QELAspYHCWLXE5aZInX +wTUJhK31IgIXNn9bJ0hFiSpQR2xeMs9oYtRuPOu0P8oOFMn4/z374fkjZy8QVY3e +PlL+3EUeqYtkMwlGNmVw5a/NbNuNfm5Darb7pEfbYd1gPcni4MAYw7R2SG/57GbC +9gucvspHIfOSfBNLBthDzmK8xEKe1yD2eimfc2T7IRYb6hmkYfeds5GsqvGI6mwI +85h4uUHWRc5JOlhVM6yX8hSWx0L60Z3DZLChmc8maWnFXd7C8eQ6P1azJJbW71Ih +7CoK0XW4LE82vlQurSRFgTwfl7wFYszW2bOzCuhHDDtYnwH86Nsu0DC78ZVRnvxn +E8Ke/AJgrdhIOo4UAyR+aZD2+2mKd7/waOUTUrUtTzc7i8N3YXGi/EIaNReBXaq+ +ZNOp24BlFzRp+FCF/pptDW9HjPdiV09x0DgICmeZS4Gq/4vFFIahWctg52NGebT0 +Idxngjj+xDtLaZlLQoOz0n5ByjO/Wi0ANmMv1sMKCHhGvdaSws2/PbMR2r4caj8m +KXpIgdinM/wUzHJ5pZyF2U/qejsRj8Kw8KH/tfX4JCLhiaP/mgeTuWGDHeZQERAT +xPmRFHaLP9/ZhvGNh6okIYtrKjWTLGoXvKLHcrKNisBLSq+P2WeFrlme1vjvJMo/ +jPwLT5o9CADQmcbKZ+QQ1ZM9v99iDZol7SAMZX43JC019sx6GK0u6xouJBcLfeB4 +OXacTgmSYdTa9RM9fbfVpti01tJ84LV2SyL/VJq/enJF4XQPSynT/tFTn1PAor6o +tEAAd8fjKdJ6LnD5wb92SPHfQfXqI84rFEO8rUNIE/1ErT6DYifDzVCbfD2KZdoF +cOSp7TpD77sY1bs74ocBX5ejKtd+aH99D78bJSMM4pSDZsIEwnomkBHTziubPwJb +OwnATy0LmSMAWOw5rKbsh5nfwCiUTM20xp0t5JeXd+wPVWbpWqI2EnkCEN+RJr9i +7dp/ymDQ+Yt5wrsN3NwoyiexPOG91WQVCADdErHsnglVZZq9Z8Wx7KwecGCUurJ2 +H6lKudv5YOxPnAzqZS5HbpZd/nRTMZh2rdXCr5m2YOuewyYjvM757AkmUpM09zJX +MQ1S67/UX2y8/74TcRF97Ncx9HeELs92innBRXoFitnNguvcO6Esx4BTe1OdU6qR +ER3zAmVf22Le9ciXbu24DN4mleOH+OmBx7X2PqJSYW9GAMTsRB081R6EWKH7romQ +waxFrZ4DJzZ9ltyosEJn5F32StyLrFxpcrdLUoEaclZCv2qka7sZvi0EvovDVEBU +e10jOx9AOwf8Gj2ufhquQ6qgVYCzbP+YrodtkFrXRS3IsljIchj1M2ffB/0bfoUs +rtER9pLvYzCjBPg8IfGLw0o754Qbhh/ReplCRTusP/fQMybvCvfxreS3oyEriu/G +GufRomjewZ8EMHDIgUsLcYo2UHZsfF7tcazgxMGmMvazp4r8vpgrvW/8fIN/6Adu +tF+WjWDTvJLFJCe6O+BFJOWrssNrrra1zGtLC1s8s+Wfpe+bGPL5zpHeebGTwH1U +22eqgJArlEKxrfarz7W5+uHZJHSjF/K9ZvunLGD0n9GOPMpji3UO3zeM8IYoWn7E +/EWK1XbjnssNemeeTZ+sDh+qrD7BOi+vCX1IyBxbfqnQfJZvmcPWpruy1UsO+aIC +0GY8Jr3OL69dDQ21jueJAh8EGAEIAAkFAlC9+dkCGwwACgkQL0VeKCTRjd9HCw/+ +LQSVgLLF4ulYlPCjWIIuQwrPbJfWUVVr2dPUFVM85DCv8gBzk5c121snXh9Swovm +laBbw6ate3BmbXLh64jVE9Za5sbTWi7PCcbO/bpRy4d6oLmitmNw6cq0vjTLxUYy +bwuiJxWREkfxuU85EKdouN062YDevH+/YResmlJrcCE7LRlJFeRlKsrrwBU3BqYd +GgFJjKjQC1peeQ9fj62Y7xfwE9+PXbkiWO5u/Bk8hb1VZH1SoIRU98NHVcp6BVvp +VK0jLAXuSauSczULmpRjbyt1lhaAqivDTWEEZXiNNbRyp17c3nVdPWOcgBr42hdQ +z25CgZgyLCsvu82wuXLKJblrIPJX3Yf+si6KqEWBsmwdOWybsjygaF5HvzgFqAAD +U0goPWoQ71PorP2XOUNp5ZLkBQp5etvtkksjVNMIhnHn8PGMuoxO39EUGlWj2B5l +Cu8tSosAzB1pS8NcLZzoNoI9dOHrmgJmP+GrOUkcf5GhNZbMoj4GNfGBRYX0SZlQ +GuDrwNKYj73C4MWyNnnUFyq8nDHJ/G1NpaF2hiof9RBL4PUU/f92JkceXPBXA8gL +Mz2ig1OButwPPLFGQhWqxXAGrsS3Ny+BhTJfnfIbbkaLLphBpDZm1D9XKbAUvdd1 +RZXoH+FTg9UAW87eqU610npOkT6cRaBxaMK/mDtGNdc= +=JTFu +-----END PGP PRIVATE KEY BLOCK----- +""" + +# key 7FEE575A: public key "anotheruser " +PUBLIC_KEY_2 = """ +-----BEGIN PGP PUBLIC KEY BLOCK----- +Version: GnuPG v1.4.10 (GNU/Linux) + +mI0EUYwJXgEEAMbTKHuPJ5/Gk34l9Z06f+0WCXTDXdte1UBoDtZ1erAbudgC4MOR +gquKqoj3Hhw0/ILqJ88GcOJmKK/bEoIAuKaqlzDF7UAYpOsPZZYmtRfPC2pTCnXq +Z1vdeqLwTbUspqXflkCkFtfhGKMq5rH8GV5a3tXZkRWZhdNwhVXZagC3ABEBAAG0 +IWFub3RoZXJ1c2VyIDxhbm90aGVydXNlckBsZWFwLnNlPoi4BBMBAgAiBQJRjAle +AhsDBgsJCAcDAgYVCAIJCgsEFgIDAQIeAQIXgAAKCRB/nfpof+5XWotuA/4tLN4E +gUr7IfLy2HkHAxzw7A4rqfMN92DIM9mZrDGaWRrOn3aVF7VU1UG7MDkHfPvp/cFw +ezoCw4s4IoHVc/pVlOkcHSyt4/Rfh248tYEJmFCJXGHpkK83VIKYJAithNccJ6Q4 +JE/o06Mtf4uh/cA1HUL4a4ceqUhtpLJULLeKo7iNBFGMCV4BBADsyQI7GR0wSAxz +VayLjuPzgT+bjbFeymIhjuxKIEwnIKwYkovztW+4bbOcQs785k3Lp6RzvigTpQQt +Z/hwcLOqZbZw8t/24+D+Pq9mMP2uUvCFFqLlVvA6D3vKSQ/XNN+YB919WQ04jh63 +yuRe94WenT1RJd6xU1aaUff4rKizuQARAQABiJ8EGAECAAkFAlGMCV4CGwwACgkQ +f536aH/uV1rPZQQAqCzRysOlu8ez7PuiBD4SebgRqWlxa1TF1ujzfLmuPivROZ2X +Kw5aQstxgGSjoB7tac49s0huh4X8XK+BtJBfU84JS8Jc2satlfwoyZ35LH6sDZck +I+RS/3we6zpMfHs3vvp9xgca6ZupQxivGtxlJs294TpJorx+mFFqbV17AzQ= +=Thdu +-----END PGP PUBLIC KEY BLOCK----- +""" + +PRIVATE_KEY_2 = """ +-----BEGIN PGP PRIVATE KEY BLOCK----- +Version: GnuPG v1.4.10 (GNU/Linux) + +lQHYBFGMCV4BBADG0yh7jyefxpN+JfWdOn/tFgl0w13bXtVAaA7WdXqwG7nYAuDD +kYKriqqI9x4cNPyC6ifPBnDiZiiv2xKCALimqpcwxe1AGKTrD2WWJrUXzwtqUwp1 +6mdb3Xqi8E21LKal35ZApBbX4RijKuax/BleWt7V2ZEVmYXTcIVV2WoAtwARAQAB +AAP7BLuSAx7tOohnimEs74ks8l/L6dOcsFQZj2bqs4AoY3jFe7bV0tHr4llypb/8 +H3/DYvpf6DWnCjyUS1tTnXSW8JXtx01BUKaAufSmMNg9blKV6GGHlT/Whe9uVyks +7XHk/+9mebVMNJ/kNlqq2k+uWqJohzC8WWLRK+d1tBeqDsECANZmzltPaqUsGV5X +C3zszE3tUBgptV/mKnBtopKi+VH+t7K6fudGcG+bAcZDUoH/QVde52mIIjjIdLje +uajJuHUCAO1mqh+vPoGv4eBLV7iBo3XrunyGXiys4a39eomhxTy3YktQanjjx+ty +GltAGCs5PbWGO6/IRjjvd46wh53kzvsCAO0J97gsWhzLuFnkxFAJSPk7RRlyl7lI +1XS/x0Og6j9XHCyY1OYkfBm0to3UlCfkgirzCYlTYObCofzdKFIPDmSqHbQhYW5v +dGhlcnVzZXIgPGFub3RoZXJ1c2VyQGxlYXAuc2U+iLgEEwECACIFAlGMCV4CGwMG +CwkIBwMCBhUIAgkKCwQWAgMBAh4BAheAAAoJEH+d+mh/7ldai24D/i0s3gSBSvsh +8vLYeQcDHPDsDiup8w33YMgz2ZmsMZpZGs6fdpUXtVTVQbswOQd8++n9wXB7OgLD +izgigdVz+lWU6RwdLK3j9F+Hbjy1gQmYUIlcYemQrzdUgpgkCK2E1xwnpDgkT+jT +oy1/i6H9wDUdQvhrhx6pSG2kslQst4qjnQHYBFGMCV4BBADsyQI7GR0wSAxzVayL +juPzgT+bjbFeymIhjuxKIEwnIKwYkovztW+4bbOcQs785k3Lp6RzvigTpQQtZ/hw +cLOqZbZw8t/24+D+Pq9mMP2uUvCFFqLlVvA6D3vKSQ/XNN+YB919WQ04jh63yuRe +94WenT1RJd6xU1aaUff4rKizuQARAQABAAP9EyElqJ3dq3EErXwwT4mMnbd1SrVC +rUJrNWQZL59mm5oigS00uIyR0SvusOr+UzTtd8ysRuwHy5d/LAZsbjQStaOMBILx +77TJveOel0a1QK0YSMF2ywZMCKvquvjli4hAtWYz/EwfuzQN3t23jc5ny+GqmqD2 +3FUxLJosFUfLNmECAO9KhVmJi+L9dswIs+2Dkjd1eiRQzNOEVffvYkGYZyKxNiXF +UA5kvyZcB4iAN9sWCybE4WHZ9jd4myGB0MPDGxkCAP1RsXJbbuD6zS7BXe5gwunO +2q4q7ptdSl/sJYQuTe1KNP5d/uGsvlcFfsYjpsopasPjFBIncc/2QThMKlhoEaEB +/0mVAxpT6SrEvUbJ18z7kna24SgMPr3OnPMxPGfvNLJY/Xv/A17YfoqjmByCvsKE +JCDjopXtmbcrZyoEZbEht9mko4ifBBgBAgAJBQJRjAleAhsMAAoJEH+d+mh/7lda +z2UEAKgs0crDpbvHs+z7ogQ+Enm4EalpcWtUxdbo83y5rj4r0TmdlysOWkLLcYBk +o6Ae7WnOPbNIboeF/FyvgbSQX1POCUvCXNrGrZX8KMmd+Sx+rA2XJCPkUv98Hus6 +THx7N776fcYHGumbqUMYrxrcZSbNveE6SaK8fphRam1dewM0 +=a5gs +-----END PGP PRIVATE KEY BLOCK----- +""" diff --git a/src/leap/bitmask/mail/testing/__init__.py~ b/src/leap/bitmask/mail/testing/__init__.py~ new file mode 100644 index 0000000..ffaadd8 --- /dev/null +++ b/src/leap/bitmask/mail/testing/__init__.py~ @@ -0,0 +1,358 @@ +# -*- coding: utf-8 -*- +# __init__.py +# Copyright (C) 2013 LEAP +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . +""" +Base classes and keys for leap.mail tests. +""" +import os +import distutils.spawn +from mock import Mock +from twisted.internet.defer import gatherResults, succeed +from twisted.trial import unittest +from twisted.web.client import Response +from twisted.internet import defer +from twisted.python import log + + +from leap.soledad.client import Soledad +from leap.keymanager import KeyManager + + +from leap.common.testing.basetest import BaseLeapTest + +ADDRESS = 'leap@leap.se' +ADDRESS_2 = 'anotheruser@leap.se' + +class defaultMockSharedDB(object): + get_doc = Mock(return_value=None) + put_doc = Mock(side_effect=None) + open = Mock(return_value=None) + close = Mock(return_value=None) + syncable = True + + def __call__(self): + return self + + +class KeyManagerWithSoledadTestCase(unittest.TestCase, BaseLeapTest): + + def setUp(self): + self.gpg_binary_path = self._find_gpg() + + self._soledad = Soledad( + u"leap@leap.se", + u"123456", + secrets_path=self.tempdir + "/secret.gpg", + local_db_path=self.tempdir + "/soledad.u1db", + server_url='', + cert_file=None, + auth_token=None, + shared_db=defaultMockSharedDB(), + syncable=False) + + self.km = self._key_manager() + + class Response(object): + code = 200 + phrase = '' + def deliverBody(self, x): + return + + # XXX why the fuck is this needed? ------------------------ + self.km._async_client_pinned.request = Mock( + return_value=defer.succeed(Response())) + #self.km._async_client.request = Mock(return_value='') + #self.km._async_client_pinned.request = Mock( + #return_value='') + # ------------------------------------------------------- + + d1 = self.km.put_raw_key(PRIVATE_KEY, ADDRESS) + d2 = self.km.put_raw_key(PRIVATE_KEY_2, ADDRESS_2) + return gatherResults([d1, d2]) + + def tearDown(self): + km = self._key_manager() + # wait for the indexes to be ready for the tear down + d = km._openpgp.deferred_init + d.addCallback(lambda _: self.delete_all_keys(km)) + d.addCallback(lambda _: self._soledad.close()) + return d + + def delete_all_keys(self, km): + def delete_keys(keys): + deferreds = [] + for key in keys: + d = km._openpgp.delete_key(key) + deferreds.append(d) + return gatherResults(deferreds) + + def check_deleted(_, private): + d = km.get_all_keys(private=private) + d.addCallback(lambda keys: self.assertEqual(keys, [])) + return d + + deferreds = [] + for private in [True, False]: + d = km.get_all_keys(private=private) + d.addCallback(delete_keys) + d.addCallback(check_deleted, private) + deferreds.append(d) + return gatherResults(deferreds) + + def _key_manager(self, user=ADDRESS, url='', token=None, + ca_cert_path=None): + return KeyManager(user, url, self._soledad, token=token, + gpgbinary=self.gpg_binary_path, + ca_cert_path=ca_cert_path) + + def _find_gpg(self): + gpg_path = distutils.spawn.find_executable('gpg') + if gpg_path is not None: + return os.path.realpath(gpg_path) + else: + return "/usr/bin/gpg" + + def get_public_binary_key(self): + with open(PATH + '/fixtures/public_key.bin', 'r') as binary_public_key: + return binary_public_key.read() + + def get_private_binary_key(self): + with open( + PATH + '/fixtures/private_key.bin', 'r') as binary_private_key: + return binary_private_key.read() + + +# key 24D18DDF: public key "Leap Test Key " +KEY_FINGERPRINT = "E36E738D69173C13D709E44F2F455E2824D18DDF" +PUBLIC_KEY = """ +-----BEGIN PGP PUBLIC KEY BLOCK----- +Version: GnuPG v1.4.10 (GNU/Linux) + +mQINBFC9+dkBEADNRfwV23TWEoGc/x0wWH1P7PlXt8MnC2Z1kKaKKmfnglVrpOiz +iLWoiU58sfZ0L5vHkzXHXCBf6Eiy/EtUIvdiWAn+yASJ1mk5jZTBKO/WMAHD8wTO +zpMsFmWyg3xc4DkmFa9KQ5EVU0o/nqPeyQxNMQN7px5pPwrJtJFmPxnxm+aDkPYx +irDmz/4DeDNqXliazGJKw7efqBdlwTHkl9Akw2gwy178pmsKwHHEMOBOFFvX61AT +huKqHYmlCGSliwbrJppTG7jc1/ls3itrK+CWTg4txREkSpEVmfcASvw/ZqLbjgfs +d/INMwXnR9U81O8+7LT6yw/ca4ppcFoJD7/XJbkRiML6+bJ4Dakiy6i727BzV17g +wI1zqNvm5rAhtALKfACha6YO43aJzairO4II1wxVHvRDHZn2IuKDDephQ3Ii7/vb +hUOf6XCSmchkAcpKXUOvbxm1yfB1LRa64mMc2RcZxf4mW7KQkulBsdV5QG2276lv +U2UUy2IutXcGP5nXC+f6sJJGJeEToKJ57yiO/VWJFjKN8SvP+7AYsQSqINUuEf6H +T5gCPCraGMkTUTPXrREvu7NOohU78q6zZNaL3GW8ai7eSeANSuQ8Vzffx7Wd8Y7i +Pw9sYj0SMFs1UgjbuL6pO5ueHh+qyumbtAq2K0Bci0kqOcU4E9fNtdiovQARAQAB +tBxMZWFwIFRlc3QgS2V5IDxsZWFwQGxlYXAuc2U+iQI3BBMBCAAhBQJQvfnZAhsD +BQsJCAcDBRUKCQgLBRYCAwEAAh4BAheAAAoJEC9FXigk0Y3fT7EQAKH3IuRniOpb +T/DDIgwwjz3oxB/W0DDMyPXowlhSOuM0rgGfntBpBb3boezEXwL86NPQxNGGruF5 +hkmecSiuPSvOmQlqlS95NGQp6hNG0YaKColh+Q5NTspFXCAkFch9oqUje0LdxfSP +QfV9UpeEvGyPmk1I9EJV/YDmZ4+Djge1d7qhVZInz4Rx1NrSyF/Tc2EC0VpjQFsU +Y9Kb2YBBR7ivG6DBc8ty0jJXi7B4WjkFcUEJviQpMF2dCLdonCehYs1PqsN1N7j+ +eFjQd+hqVMJgYuSGKjvuAEfClM6MQw7+FmFwMyLgK/Ew/DttHEDCri77SPSkOGSI +txCzhTg6798f6mJr7WcXmHX1w1Vcib5FfZ8vTDFVhz/XgAgArdhPo9V6/1dgSSiB +KPQ/spsco6u5imdOhckERE0lnAYvVT6KE81TKuhF/b23u7x+Wdew6kK0EQhYA7wy +7LmlaNXc7rMBQJ9Z60CJ4JDtatBWZ0kNrt2VfdDHVdqBTOpl0CraNUjWE5YMDasr +K2dF5IX8D3uuYtpZnxqg0KzyLg0tzL0tvOL1C2iudgZUISZNPKbS0z0v+afuAAnx +2pTC3uezbh2Jt8SWTLhll4i0P4Ps5kZ6HQUO56O+/Z1cWovX+mQekYFmERySDR9n +3k1uAwLilJmRmepGmvYbB8HloV8HqwgguQINBFC9+dkBEAC0I/xn1uborMgDvBtf +H0sEhwnXBC849/32zic6udB6/3Efk9nzbSpL3FSOuXITZsZgCHPkKarnoQ2ztMcS +sh1ke1C5gQGms75UVmM/nS+2YI4vY8OX/GC/on2vUyncqdH+bR6xH5hx4NbWpfTs +iQHmz5C6zzS/kuabGdZyKRaZHt23WQ7JX/4zpjqbC99DjHcP9BSk7tJ8wI4bkMYD +uFVQdT9O6HwyKGYwUU4sAQRAj7XCTGvVbT0dpgJwH4RmrEtJoHAx4Whg8mJ710E0 +GCmzf2jqkNuOw76ivgk27Kge+Hw00jmJjQhHY0yVbiaoJwcRrPKzaSjEVNgrpgP3 +lXPRGQArgESsIOTeVVHQ8fhK2YtTeCY9rIiO+L0OX2xo9HK7hfHZZWL6rqymXdyS +fhzh/f6IPyHFWnvj7Brl7DR8heMikygcJqv+ed2yx7iLyCUJ10g12I48+aEj1aLe +dP7lna32iY8/Z0SHQLNH6PXO9SlPcq2aFUgKqE75A/0FMk7CunzU1OWr2ZtTLNO1 +WT/13LfOhhuEq9jTyTosn0WxBjJKq18lnhzCXlaw6EAtbA7CUwsD3CTPR56aAXFK +3I7KXOVAqggrvMe5Tpdg5drfYpI8hZovL5aAgb+7Y5ta10TcJdUhS5K3kFAWe/td +U0cmWUMDP1UMSQ5Jg6JIQVWhSwARAQABiQIfBBgBCAAJBQJQvfnZAhsMAAoJEC9F +Xigk0Y3fRwsP/i0ElYCyxeLpWJTwo1iCLkMKz2yX1lFVa9nT1BVTPOQwr/IAc5OX +NdtbJ14fUsKL5pWgW8OmrXtwZm1y4euI1RPWWubG01ouzwnGzv26UcuHeqC5orZj +cOnKtL40y8VGMm8LoicVkRJH8blPORCnaLjdOtmA3rx/v2EXrJpSa3AhOy0ZSRXk +ZSrK68AVNwamHRoBSYyo0AtaXnkPX4+tmO8X8BPfj125IljubvwZPIW9VWR9UqCE +VPfDR1XKegVb6VStIywF7kmrknM1C5qUY28rdZYWgKorw01hBGV4jTW0cqde3N51 +XT1jnIAa+NoXUM9uQoGYMiwrL7vNsLlyyiW5ayDyV92H/rIuiqhFgbJsHTlsm7I8 +oGheR784BagAA1NIKD1qEO9T6Kz9lzlDaeWS5AUKeXrb7ZJLI1TTCIZx5/DxjLqM +Tt/RFBpVo9geZQrvLUqLAMwdaUvDXC2c6DaCPXTh65oCZj/hqzlJHH+RoTWWzKI+ +BjXxgUWF9EmZUBrg68DSmI+9wuDFsjZ51BcqvJwxyfxtTaWhdoYqH/UQS+D1FP3/ +diZHHlzwVwPICzM9ooNTgbrcDzyxRkIVqsVwBq7EtzcvgYUyX53yG25Giy6YQaQ2 +ZtQ/VymwFL3XdUWV6B/hU4PVAFvO3qlOtdJ6TpE+nEWgcWjCv5g7RjXX +=MuOY +-----END PGP PUBLIC KEY BLOCK----- +""" +PRIVATE_KEY = """ +-----BEGIN PGP PRIVATE KEY BLOCK----- +Version: GnuPG v1.4.10 (GNU/Linux) + +lQcYBFC9+dkBEADNRfwV23TWEoGc/x0wWH1P7PlXt8MnC2Z1kKaKKmfnglVrpOiz +iLWoiU58sfZ0L5vHkzXHXCBf6Eiy/EtUIvdiWAn+yASJ1mk5jZTBKO/WMAHD8wTO +zpMsFmWyg3xc4DkmFa9KQ5EVU0o/nqPeyQxNMQN7px5pPwrJtJFmPxnxm+aDkPYx +irDmz/4DeDNqXliazGJKw7efqBdlwTHkl9Akw2gwy178pmsKwHHEMOBOFFvX61AT +huKqHYmlCGSliwbrJppTG7jc1/ls3itrK+CWTg4txREkSpEVmfcASvw/ZqLbjgfs +d/INMwXnR9U81O8+7LT6yw/ca4ppcFoJD7/XJbkRiML6+bJ4Dakiy6i727BzV17g +wI1zqNvm5rAhtALKfACha6YO43aJzairO4II1wxVHvRDHZn2IuKDDephQ3Ii7/vb +hUOf6XCSmchkAcpKXUOvbxm1yfB1LRa64mMc2RcZxf4mW7KQkulBsdV5QG2276lv +U2UUy2IutXcGP5nXC+f6sJJGJeEToKJ57yiO/VWJFjKN8SvP+7AYsQSqINUuEf6H +T5gCPCraGMkTUTPXrREvu7NOohU78q6zZNaL3GW8ai7eSeANSuQ8Vzffx7Wd8Y7i +Pw9sYj0SMFs1UgjbuL6pO5ueHh+qyumbtAq2K0Bci0kqOcU4E9fNtdiovQARAQAB +AA/+JHtlL39G1wsH9R6UEfUQJGXR9MiIiwZoKcnRB2o8+DS+OLjg0JOh8XehtuCs +E/8oGQKtQqa5bEIstX7IZoYmYFiUQi9LOzIblmp2vxOm+HKkxa4JszWci2/ZmC3t +KtaA4adl9XVnshoQ7pijuCMUKB3naBEOAxd8s9d/JeReGIYkJErdrnVfNk5N71Ds +FmH5Ll3XtEDvgBUQP3nkA6QFjpsaB94FHjL3gDwum/cxzj6pCglcvHOzEhfY0Ddb +J967FozQTaf2JW3O+w3LOqtcKWpq87B7+O61tVidQPSSuzPjCtFF0D2LC9R/Hpky +KTMQ6CaKja4MPhjwywd4QPcHGYSqjMpflvJqi+kYIt8psUK/YswWjnr3r4fbuqVY +VhtiHvnBHQjz135lUqWvEz4hM3Xpnxydx7aRlv5NlevK8+YIO5oFbWbGNTWsPZI5 +jpoFBpSsnR1Q5tnvtNHauvoWV+XN2qAOBTG+/nEbDYH6Ak3aaE9jrpTdYh0CotYF +q7csANsDy3JvkAzeU6WnYpsHHaAjqOGyiZGsLej1UcXPFMosE/aUo4WQhiS8Zx2c +zOVKOi/X5vQ2GdNT9Qolz8AriwzsvFR+bxPzyd8V6ALwDsoXvwEYinYBKK8j0OPv +OOihSR6HVsuP9NUZNU9ewiGzte/+/r6pNXHvR7wTQ8EWLcEIAN6Zyrb0bHZTIlxt +VWur/Ht2mIZrBaO50qmM5RD3T5oXzWXi/pjLrIpBMfeZR9DWfwQwjYzwqi7pxtYx +nJvbMuY505rfnMoYxb4J+cpRXV8MS7Dr1vjjLVUC9KiwSbM3gg6emfd2yuA93ihv +Pe3mffzLIiQa4mRE3wtGcioC43nWuV2K2e1KjxeFg07JhrezA/1Cak505ab/tmvP +4YmjR5c44+yL/YcQ3HdFgs4mV+nVbptRXvRcPpolJsgxPccGNdvHhsoR4gwXMS3F +RRPD2z6x8xeN73Q4KH3bm01swQdwFBZbWVfmUGLxvN7leCdfs9+iFJyqHiCIB6Iv +mQfp8F0IAOwSo8JhWN+V1dwML4EkIrM8wUb4yecNLkyR6TpPH/qXx4PxVMC+vy6x +sCtjeHIwKE+9vqnlhd5zOYh7qYXEJtYwdeDDmDbL8oks1LFfd+FyAuZXY33DLwn0 +cRYsr2OEZmaajqUB3NVmj3H4uJBN9+paFHyFSXrH68K1Fk2o3n+RSf2EiX+eICwI +L6rqoF5sSVUghBWdNegV7qfy4anwTQwrIMGjgU5S6PKW0Dr/3iO5z3qQpGPAj5OW +ATqPWkDICLbObPxD5cJlyyNE2wCA9VVc6/1d6w4EVwSq9h3/WTpATEreXXxTGptd +LNiTA1nmakBYNO2Iyo3djhaqBdWjk+EIAKtVEnJH9FAVwWOvaj1RoZMA5DnDMo7e +SnhrCXl8AL7Z1WInEaybasTJXn1uQ8xY52Ua4b8cbuEKRKzw/70NesFRoMLYoHTO +dyeszvhoDHberpGRTciVmpMu7Hyi33rM31K9epA4ib6QbbCHnxkWOZB+Bhgj1hJ8 +xb4RBYWiWpAYcg0+DAC3w9gfxQhtUlZPIbmbrBmrVkO2GVGUj8kH6k4UV6kUHEGY +HQWQR0HcbKcXW81ZXCCD0l7ROuEWQtTe5Jw7dJ4/QFuqZnPutXVRNOZqpl6eRShw +7X2/a29VXBpmHA95a88rSQsL+qm7Fb3prqRmuMCtrUZgFz7HLSTuUMR867QcTGVh +cCBUZXN0IEtleSA8bGVhcEBsZWFwLnNlPokCNwQTAQgAIQUCUL352QIbAwULCQgH +AwUVCgkICwUWAgMBAAIeAQIXgAAKCRAvRV4oJNGN30+xEACh9yLkZ4jqW0/wwyIM +MI896MQf1tAwzMj16MJYUjrjNK4Bn57QaQW926HsxF8C/OjT0MTRhq7heYZJnnEo +rj0rzpkJapUveTRkKeoTRtGGigqJYfkOTU7KRVwgJBXIfaKlI3tC3cX0j0H1fVKX +hLxsj5pNSPRCVf2A5mePg44HtXe6oVWSJ8+EcdTa0shf03NhAtFaY0BbFGPSm9mA +QUe4rxugwXPLctIyV4uweFo5BXFBCb4kKTBdnQi3aJwnoWLNT6rDdTe4/nhY0Hfo +alTCYGLkhio77gBHwpTOjEMO/hZhcDMi4CvxMPw7bRxAwq4u+0j0pDhkiLcQs4U4 +Ou/fH+pia+1nF5h19cNVXIm+RX2fL0wxVYc/14AIAK3YT6PVev9XYEkogSj0P7Kb +HKOruYpnToXJBERNJZwGL1U+ihPNUyroRf29t7u8flnXsOpCtBEIWAO8Muy5pWjV +3O6zAUCfWetAieCQ7WrQVmdJDa7dlX3Qx1XagUzqZdAq2jVI1hOWDA2rKytnReSF +/A97rmLaWZ8aoNCs8i4NLcy9Lbzi9QtornYGVCEmTTym0tM9L/mn7gAJ8dqUwt7n +s24dibfElky4ZZeItD+D7OZGeh0FDuejvv2dXFqL1/pkHpGBZhEckg0fZ95NbgMC +4pSZkZnqRpr2GwfB5aFfB6sIIJ0HGARQvfnZARAAtCP8Z9bm6KzIA7wbXx9LBIcJ +1wQvOPf99s4nOrnQev9xH5PZ820qS9xUjrlyE2bGYAhz5Cmq56ENs7THErIdZHtQ +uYEBprO+VFZjP50vtmCOL2PDl/xgv6J9r1Mp3KnR/m0esR+YceDW1qX07IkB5s+Q +us80v5LmmxnWcikWmR7dt1kOyV/+M6Y6mwvfQ4x3D/QUpO7SfMCOG5DGA7hVUHU/ +Tuh8MihmMFFOLAEEQI+1wkxr1W09HaYCcB+EZqxLSaBwMeFoYPJie9dBNBgps39o +6pDbjsO+or4JNuyoHvh8NNI5iY0IR2NMlW4mqCcHEazys2koxFTYK6YD95Vz0RkA +K4BErCDk3lVR0PH4StmLU3gmPayIjvi9Dl9saPRyu4Xx2WVi+q6spl3ckn4c4f3+ +iD8hxVp74+wa5ew0fIXjIpMoHCar/nndsse4i8glCddINdiOPPmhI9Wi3nT+5Z2t +9omPP2dEh0CzR+j1zvUpT3KtmhVICqhO+QP9BTJOwrp81NTlq9mbUyzTtVk/9dy3 +zoYbhKvY08k6LJ9FsQYySqtfJZ4cwl5WsOhALWwOwlMLA9wkz0eemgFxStyOylzl +QKoIK7zHuU6XYOXa32KSPIWaLy+WgIG/u2ObWtdE3CXVIUuSt5BQFnv7XVNHJllD +Az9VDEkOSYOiSEFVoUsAEQEAAQAP/1AagnZQZyzHDEgw4QELAspYHCWLXE5aZInX +wTUJhK31IgIXNn9bJ0hFiSpQR2xeMs9oYtRuPOu0P8oOFMn4/z374fkjZy8QVY3e +PlL+3EUeqYtkMwlGNmVw5a/NbNuNfm5Darb7pEfbYd1gPcni4MAYw7R2SG/57GbC +9gucvspHIfOSfBNLBthDzmK8xEKe1yD2eimfc2T7IRYb6hmkYfeds5GsqvGI6mwI +85h4uUHWRc5JOlhVM6yX8hSWx0L60Z3DZLChmc8maWnFXd7C8eQ6P1azJJbW71Ih +7CoK0XW4LE82vlQurSRFgTwfl7wFYszW2bOzCuhHDDtYnwH86Nsu0DC78ZVRnvxn +E8Ke/AJgrdhIOo4UAyR+aZD2+2mKd7/waOUTUrUtTzc7i8N3YXGi/EIaNReBXaq+ +ZNOp24BlFzRp+FCF/pptDW9HjPdiV09x0DgICmeZS4Gq/4vFFIahWctg52NGebT0 +Idxngjj+xDtLaZlLQoOz0n5ByjO/Wi0ANmMv1sMKCHhGvdaSws2/PbMR2r4caj8m +KXpIgdinM/wUzHJ5pZyF2U/qejsRj8Kw8KH/tfX4JCLhiaP/mgeTuWGDHeZQERAT +xPmRFHaLP9/ZhvGNh6okIYtrKjWTLGoXvKLHcrKNisBLSq+P2WeFrlme1vjvJMo/ +jPwLT5o9CADQmcbKZ+QQ1ZM9v99iDZol7SAMZX43JC019sx6GK0u6xouJBcLfeB4 +OXacTgmSYdTa9RM9fbfVpti01tJ84LV2SyL/VJq/enJF4XQPSynT/tFTn1PAor6o +tEAAd8fjKdJ6LnD5wb92SPHfQfXqI84rFEO8rUNIE/1ErT6DYifDzVCbfD2KZdoF +cOSp7TpD77sY1bs74ocBX5ejKtd+aH99D78bJSMM4pSDZsIEwnomkBHTziubPwJb +OwnATy0LmSMAWOw5rKbsh5nfwCiUTM20xp0t5JeXd+wPVWbpWqI2EnkCEN+RJr9i +7dp/ymDQ+Yt5wrsN3NwoyiexPOG91WQVCADdErHsnglVZZq9Z8Wx7KwecGCUurJ2 +H6lKudv5YOxPnAzqZS5HbpZd/nRTMZh2rdXCr5m2YOuewyYjvM757AkmUpM09zJX +MQ1S67/UX2y8/74TcRF97Ncx9HeELs92innBRXoFitnNguvcO6Esx4BTe1OdU6qR +ER3zAmVf22Le9ciXbu24DN4mleOH+OmBx7X2PqJSYW9GAMTsRB081R6EWKH7romQ +waxFrZ4DJzZ9ltyosEJn5F32StyLrFxpcrdLUoEaclZCv2qka7sZvi0EvovDVEBU +e10jOx9AOwf8Gj2ufhquQ6qgVYCzbP+YrodtkFrXRS3IsljIchj1M2ffB/0bfoUs +rtER9pLvYzCjBPg8IfGLw0o754Qbhh/ReplCRTusP/fQMybvCvfxreS3oyEriu/G +GufRomjewZ8EMHDIgUsLcYo2UHZsfF7tcazgxMGmMvazp4r8vpgrvW/8fIN/6Adu +tF+WjWDTvJLFJCe6O+BFJOWrssNrrra1zGtLC1s8s+Wfpe+bGPL5zpHeebGTwH1U +22eqgJArlEKxrfarz7W5+uHZJHSjF/K9ZvunLGD0n9GOPMpji3UO3zeM8IYoWn7E +/EWK1XbjnssNemeeTZ+sDh+qrD7BOi+vCX1IyBxbfqnQfJZvmcPWpruy1UsO+aIC +0GY8Jr3OL69dDQ21jueJAh8EGAEIAAkFAlC9+dkCGwwACgkQL0VeKCTRjd9HCw/+ +LQSVgLLF4ulYlPCjWIIuQwrPbJfWUVVr2dPUFVM85DCv8gBzk5c121snXh9Swovm +laBbw6ate3BmbXLh64jVE9Za5sbTWi7PCcbO/bpRy4d6oLmitmNw6cq0vjTLxUYy +bwuiJxWREkfxuU85EKdouN062YDevH+/YResmlJrcCE7LRlJFeRlKsrrwBU3BqYd +GgFJjKjQC1peeQ9fj62Y7xfwE9+PXbkiWO5u/Bk8hb1VZH1SoIRU98NHVcp6BVvp +VK0jLAXuSauSczULmpRjbyt1lhaAqivDTWEEZXiNNbRyp17c3nVdPWOcgBr42hdQ +z25CgZgyLCsvu82wuXLKJblrIPJX3Yf+si6KqEWBsmwdOWybsjygaF5HvzgFqAAD +U0goPWoQ71PorP2XOUNp5ZLkBQp5etvtkksjVNMIhnHn8PGMuoxO39EUGlWj2B5l +Cu8tSosAzB1pS8NcLZzoNoI9dOHrmgJmP+GrOUkcf5GhNZbMoj4GNfGBRYX0SZlQ +GuDrwNKYj73C4MWyNnnUFyq8nDHJ/G1NpaF2hiof9RBL4PUU/f92JkceXPBXA8gL +Mz2ig1OButwPPLFGQhWqxXAGrsS3Ny+BhTJfnfIbbkaLLphBpDZm1D9XKbAUvdd1 +RZXoH+FTg9UAW87eqU610npOkT6cRaBxaMK/mDtGNdc= +=JTFu +-----END PGP PRIVATE KEY BLOCK----- +""" + +# key 7FEE575A: public key "anotheruser " +PUBLIC_KEY_2 = """ +-----BEGIN PGP PUBLIC KEY BLOCK----- +Version: GnuPG v1.4.10 (GNU/Linux) + +mI0EUYwJXgEEAMbTKHuPJ5/Gk34l9Z06f+0WCXTDXdte1UBoDtZ1erAbudgC4MOR +gquKqoj3Hhw0/ILqJ88GcOJmKK/bEoIAuKaqlzDF7UAYpOsPZZYmtRfPC2pTCnXq +Z1vdeqLwTbUspqXflkCkFtfhGKMq5rH8GV5a3tXZkRWZhdNwhVXZagC3ABEBAAG0 +IWFub3RoZXJ1c2VyIDxhbm90aGVydXNlckBsZWFwLnNlPoi4BBMBAgAiBQJRjAle +AhsDBgsJCAcDAgYVCAIJCgsEFgIDAQIeAQIXgAAKCRB/nfpof+5XWotuA/4tLN4E +gUr7IfLy2HkHAxzw7A4rqfMN92DIM9mZrDGaWRrOn3aVF7VU1UG7MDkHfPvp/cFw +ezoCw4s4IoHVc/pVlOkcHSyt4/Rfh248tYEJmFCJXGHpkK83VIKYJAithNccJ6Q4 +JE/o06Mtf4uh/cA1HUL4a4ceqUhtpLJULLeKo7iNBFGMCV4BBADsyQI7GR0wSAxz +VayLjuPzgT+bjbFeymIhjuxKIEwnIKwYkovztW+4bbOcQs785k3Lp6RzvigTpQQt +Z/hwcLOqZbZw8t/24+D+Pq9mMP2uUvCFFqLlVvA6D3vKSQ/XNN+YB919WQ04jh63 +yuRe94WenT1RJd6xU1aaUff4rKizuQARAQABiJ8EGAECAAkFAlGMCV4CGwwACgkQ +f536aH/uV1rPZQQAqCzRysOlu8ez7PuiBD4SebgRqWlxa1TF1ujzfLmuPivROZ2X +Kw5aQstxgGSjoB7tac49s0huh4X8XK+BtJBfU84JS8Jc2satlfwoyZ35LH6sDZck +I+RS/3we6zpMfHs3vvp9xgca6ZupQxivGtxlJs294TpJorx+mFFqbV17AzQ= +=Thdu +-----END PGP PUBLIC KEY BLOCK----- +""" + +PRIVATE_KEY_2 = """ +-----BEGIN PGP PRIVATE KEY BLOCK----- +Version: GnuPG v1.4.10 (GNU/Linux) + +lQHYBFGMCV4BBADG0yh7jyefxpN+JfWdOn/tFgl0w13bXtVAaA7WdXqwG7nYAuDD +kYKriqqI9x4cNPyC6ifPBnDiZiiv2xKCALimqpcwxe1AGKTrD2WWJrUXzwtqUwp1 +6mdb3Xqi8E21LKal35ZApBbX4RijKuax/BleWt7V2ZEVmYXTcIVV2WoAtwARAQAB +AAP7BLuSAx7tOohnimEs74ks8l/L6dOcsFQZj2bqs4AoY3jFe7bV0tHr4llypb/8 +H3/DYvpf6DWnCjyUS1tTnXSW8JXtx01BUKaAufSmMNg9blKV6GGHlT/Whe9uVyks +7XHk/+9mebVMNJ/kNlqq2k+uWqJohzC8WWLRK+d1tBeqDsECANZmzltPaqUsGV5X +C3zszE3tUBgptV/mKnBtopKi+VH+t7K6fudGcG+bAcZDUoH/QVde52mIIjjIdLje +uajJuHUCAO1mqh+vPoGv4eBLV7iBo3XrunyGXiys4a39eomhxTy3YktQanjjx+ty +GltAGCs5PbWGO6/IRjjvd46wh53kzvsCAO0J97gsWhzLuFnkxFAJSPk7RRlyl7lI +1XS/x0Og6j9XHCyY1OYkfBm0to3UlCfkgirzCYlTYObCofzdKFIPDmSqHbQhYW5v +dGhlcnVzZXIgPGFub3RoZXJ1c2VyQGxlYXAuc2U+iLgEEwECACIFAlGMCV4CGwMG +CwkIBwMCBhUIAgkKCwQWAgMBAh4BAheAAAoJEH+d+mh/7ldai24D/i0s3gSBSvsh +8vLYeQcDHPDsDiup8w33YMgz2ZmsMZpZGs6fdpUXtVTVQbswOQd8++n9wXB7OgLD +izgigdVz+lWU6RwdLK3j9F+Hbjy1gQmYUIlcYemQrzdUgpgkCK2E1xwnpDgkT+jT +oy1/i6H9wDUdQvhrhx6pSG2kslQst4qjnQHYBFGMCV4BBADsyQI7GR0wSAxzVayL +juPzgT+bjbFeymIhjuxKIEwnIKwYkovztW+4bbOcQs785k3Lp6RzvigTpQQtZ/hw +cLOqZbZw8t/24+D+Pq9mMP2uUvCFFqLlVvA6D3vKSQ/XNN+YB919WQ04jh63yuRe +94WenT1RJd6xU1aaUff4rKizuQARAQABAAP9EyElqJ3dq3EErXwwT4mMnbd1SrVC +rUJrNWQZL59mm5oigS00uIyR0SvusOr+UzTtd8ysRuwHy5d/LAZsbjQStaOMBILx +77TJveOel0a1QK0YSMF2ywZMCKvquvjli4hAtWYz/EwfuzQN3t23jc5ny+GqmqD2 +3FUxLJosFUfLNmECAO9KhVmJi+L9dswIs+2Dkjd1eiRQzNOEVffvYkGYZyKxNiXF +UA5kvyZcB4iAN9sWCybE4WHZ9jd4myGB0MPDGxkCAP1RsXJbbuD6zS7BXe5gwunO +2q4q7ptdSl/sJYQuTe1KNP5d/uGsvlcFfsYjpsopasPjFBIncc/2QThMKlhoEaEB +/0mVAxpT6SrEvUbJ18z7kna24SgMPr3OnPMxPGfvNLJY/Xv/A17YfoqjmByCvsKE +JCDjopXtmbcrZyoEZbEht9mko4ifBBgBAgAJBQJRjAleAhsMAAoJEH+d+mh/7lda +z2UEAKgs0crDpbvHs+z7ogQ+Enm4EalpcWtUxdbo83y5rj4r0TmdlysOWkLLcYBk +o6Ae7WnOPbNIboeF/FyvgbSQX1POCUvCXNrGrZX8KMmd+Sx+rA2XJCPkUv98Hus6 +THx7N776fcYHGumbqUMYrxrcZSbNveE6SaK8fphRam1dewM0 +=a5gs +-----END PGP PRIVATE KEY BLOCK----- +""" diff --git a/src/leap/bitmask/mail/testing/common.py b/src/leap/bitmask/mail/testing/common.py new file mode 100644 index 0000000..1bf1de2 --- /dev/null +++ b/src/leap/bitmask/mail/testing/common.py @@ -0,0 +1,109 @@ +# -*- coding: utf-8 -*- +# common.py +# Copyright (C) 2014 LEAP +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . +""" +Common utilities for testing Soledad. +""" +import os +import tempfile + +from twisted.internet import defer +from twisted.trial import unittest + +from leap.common.testing.basetest import BaseLeapTest +from leap.soledad.client import Soledad + +# TODO move to common module, or Soledad itself +# XXX remove duplication + +TEST_USER = "testuser@leap.se" +TEST_PASSWD = "1234" + + +def _initialize_soledad(email, gnupg_home, tempdir): + """ + Initializes soledad by hand + + :param email: ID for the user + :param gnupg_home: path to home used by gnupg + :param tempdir: path to temporal dir + :rtype: Soledad instance + """ + + uuid = "foobar-uuid" + passphrase = u"verysecretpassphrase" + secret_path = os.path.join(tempdir, "secret.gpg") + local_db_path = os.path.join(tempdir, "soledad.u1db") + server_url = "https://provider" + cert_file = "" + + soledad = Soledad( + uuid, + passphrase, + secret_path, + local_db_path, + server_url, + cert_file, + syncable=False) + + return soledad + + +class SoledadTestMixin(unittest.TestCase, BaseLeapTest): + """ + It is **VERY** important that this base is added *AFTER* unittest.TestCase + """ + + def setUp(self): + self.results = [] + self.setUpEnv() + + # pytest handles correctly the setupEnv for the class, + # but trial ignores it. + if not getattr(self, 'tempdir', None): + self.tempdir = tempfile.gettempdir() + + # Soledad: config info + self.gnupg_home = "%s/gnupg" % self.tempdir + self.email = 'leap@leap.se' + + # initialize soledad by hand so we can control keys + self._soledad = _initialize_soledad( + self.email, + self.gnupg_home, + self.tempdir) + + return defer.succeed(True) + + def tearDown(self): + """ + tearDown method called after each test. + """ + self.results = [] + try: + self._soledad.close() + except Exception: + print "ERROR WHILE CLOSING SOLEDAD" + # logging.exception(exc) + self.tearDownEnv() + + @classmethod + def setUpClass(self): + pass + + @classmethod + def tearDownClass(self): + pass diff --git a/src/leap/bitmask/mail/testing/imap.py b/src/leap/bitmask/mail/testing/imap.py new file mode 100644 index 0000000..72acbf2 --- /dev/null +++ b/src/leap/bitmask/mail/testing/imap.py @@ -0,0 +1,186 @@ +# -*- coding: utf-8 -*- +# utils.py +# Copyright (C) 2014, 2015 LEAP +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . +""" +Common utilities for testing Soledad IMAP Server. +""" +from email import parser + +from mock import Mock +from twisted.cred.checkers import ICredentialsChecker +from twisted.cred.credentials import IUsernamePassword +from twisted.cred.error import UnauthorizedLogin +from twisted.cred.portal import Portal, IRealm +from twisted.mail import imap4 +from twisted.internet import defer +from twisted.protocols import loopback +from twisted.python import log +from zope.interface import implementer + +from leap.mail.adaptors import soledad as soledad_adaptor +from leap.mail.imap.account import IMAPAccount +from leap.mail.imap.server import LEAPIMAPServer +from leap.mail.testing.common import SoledadTestMixin + +TEST_USER = "testuser@leap.se" +TEST_PASSWD = "1234" + + +# +# Simple IMAP4 Client for testing +# + +class SimpleClient(imap4.IMAP4Client): + """ + A Simple IMAP4 Client to test our + Soledad-LEAPServer + """ + + def __init__(self, deferred, contextFactory=None): + imap4.IMAP4Client.__init__(self, contextFactory) + self.deferred = deferred + self.events = [] + + def serverGreeting(self, caps): + self.deferred.callback(None) + + def modeChanged(self, writeable): + self.events.append(['modeChanged', writeable]) + self.transport.loseConnection() + + def flagsChanged(self, newFlags): + self.events.append(['flagsChanged', newFlags]) + self.transport.loseConnection() + + def newMessages(self, exists, recent): + self.events.append(['newMessages', exists, recent]) + self.transport.loseConnection() + +# +# Dummy credentials for tests +# + + +@implementer(IRealm) +class TestRealm(object): + + def __init__(self, account): + self._account = account + + def requestAvatar(self, avatarId, mind, *interfaces): + avatar = self._account + return (imap4.IAccount, avatar, + getattr(avatar, 'logout', lambda: None)) + + +@implementer(ICredentialsChecker) +class TestCredentialsChecker(object): + + credentialInterfaces = (IUsernamePassword,) + + userid = TEST_USER + password = TEST_PASSWD + + def requestAvatarId(self, credentials): + username, password = credentials.username, credentials.password + d = self.checkTestCredentials(username, password) + d.addErrback(lambda f: defer.fail(UnauthorizedLogin())) + return d + + def checkTestCredentials(self, username, password): + if username == self.userid and password == self.password: + return defer.succeed(username) + else: + return defer.fail(Exception("Wrong credentials")) + + +class TestSoledadIMAPServer(LEAPIMAPServer): + + def __init__(self, account, *args, **kw): + + LEAPIMAPServer.__init__(self, *args, **kw) + + realm = TestRealm(account) + portal = Portal(realm) + checker = TestCredentialsChecker() + self.checker = checker + self.portal = portal + portal.registerChecker(checker) + + +class IMAP4HelperMixin(SoledadTestMixin): + """ + MixIn containing several utilities to be shared across + different TestCases + """ + serverCTX = None + clientCTX = None + + def setUp(self): + + soledad_adaptor.cleanup_deferred_locks() + + USERID = TEST_USER + + def setup_server(account): + self.server = TestSoledadIMAPServer( + account=account, + contextFactory=self.serverCTX) + self.server.theAccount = account + + d_server_ready = defer.Deferred() + self.client = SimpleClient( + d_server_ready, contextFactory=self.clientCTX) + self.connected = d_server_ready + + def setup_account(_): + self.parser = parser.Parser() + + # XXX this should be fixed in soledad. + # Soledad sync makes trial block forever. The sync it's mocked to + # fix this problem. _mock_soledad_get_from_index can be used from + # the tests to provide documents. + # TODO see here, possibly related? + # -- http://www.pythoneye.com/83_20424875/ + self._soledad.sync = Mock() + + d = defer.Deferred() + self.acc = IMAPAccount(self._soledad, USERID, d=d) + return d + + d = super(IMAP4HelperMixin, self).setUp() + d.addCallback(setup_account) + d.addCallback(setup_server) + return d + + def tearDown(self): + SoledadTestMixin.tearDown(self) + del self._soledad + del self.client + del self.server + del self.connected + + def _cbStopClient(self, ignore): + self.client.transport.loseConnection() + + def _ebGeneral(self, failure): + self.client.transport.loseConnection() + self.server.transport.loseConnection() + if hasattr(self, 'function'): + log.err(failure, "Problem with %r" % (self.function,)) + + def loopback(self): + return loopback.loopbackAsync(self.server, self.client) diff --git a/src/leap/bitmask/mail/testing/smtp.py b/src/leap/bitmask/mail/testing/smtp.py new file mode 100644 index 0000000..d8690f1 --- /dev/null +++ b/src/leap/bitmask/mail/testing/smtp.py @@ -0,0 +1,51 @@ +from twisted.mail import smtp + +from leap.mail.smtp.gateway import SMTPFactory, LOCAL_FQDN +from leap.mail.smtp.gateway import SMTPDelivery +from leap.mail.outgoing.service import outgoingFactory + +TEST_USER = u'anotheruser@leap.se' + + +class UnauthenticatedSMTPServer(smtp.SMTP): + + encrypted_only = False + + def __init__(self, soledads, keyms, opts, encrypted_only=False): + smtp.SMTP.__init__(self) + + userid = TEST_USER + keym = keyms[userid] + + class Opts: + cert = '/tmp/cert' + key = '/tmp/cert' + hostname = 'remote' + port = 666 + + outgoing = outgoingFactory( + userid, keym, Opts, check_cert=False) + avatar = SMTPDelivery(userid, keym, encrypted_only, outgoing) + self.delivery = avatar + + def validateFrom(self, helo, origin): + return origin + + +class UnauthenticatedSMTPFactory(SMTPFactory): + """ + A Factory that produces a SMTP server that does not authenticate user. + Only for tests! + """ + protocol = UnauthenticatedSMTPServer + domain = LOCAL_FQDN + encrypted_only = False + + +def getSMTPFactory(soledad_s, keymanager_s, sendmail_opts, + encrypted_only=False): + factory = UnauthenticatedSMTPFactory + factory.encrypted_only = encrypted_only + proto = factory( + soledad_s, keymanager_s, sendmail_opts).buildProtocol(('127.0.0.1', 0)) + return proto diff --git a/src/leap/bitmask/mail/tests/rfc822.bounce.message b/src/leap/bitmask/mail/tests/rfc822.bounce.message new file mode 100644 index 0000000..7a51ac0 --- /dev/null +++ b/src/leap/bitmask/mail/tests/rfc822.bounce.message @@ -0,0 +1,152 @@ +Return-Path: <> +X-Original-To: yoyo@dev.pixelated-project.org +Delivered-To: a6973ec1af0a6d1e2a1e4db4ff85f6c2@deliver.local +Received: by dev1.dev.pixelated-project.org (Postfix) + id 92CEA83164; Thu, 16 Jun 2016 14:53:34 +0200 (CEST) +Date: Thu, 16 Jun 2016 14:53:34 +0200 (CEST) +From: MAILER-DAEMON@dev1.dev.pixelated-project.org (Mail Delivery System) +Subject: Undelivered Mail Returned to Sender +To: yoyo@dev.pixelated-project.org +Auto-Submitted: auto-replied +MIME-Version: 1.0 +Content-Type: multipart/report; report-type=delivery-status; + boundary="8F60183010.1466081614/dev1.dev.pixelated-project.org" +Message-Id: <20160616125334.92CEA83164@dev1.dev.pixelated-project.org> + +This is a MIME-encapsulated message. + +--8F60183010.1466081614/dev1.dev.pixelated-project.org +Content-Description: Notification +Content-Type: text/plain; charset=us-ascii + +This is the mail system at host dev1.dev.pixelated-project.org. + +I'm sorry to have to inform you that your message could not +be delivered to one or more recipients. It's attached below. + +For further assistance, please send mail to postmaster. + +If you do so, please include this problem report. You can +delete your own text from the attached returned message. + + The mail system + +: host caribou.leap.se[176.53.69.122] said: 550 5.1.1 + : Recipient address rejected: User unknown in virtual alias + table (in reply to RCPT TO command) + +--8F60183010.1466081614/dev1.dev.pixelated-project.org +Content-Description: Delivery report +Content-Type: message/delivery-status + +Reporting-MTA: dns; dev1.dev.pixelated-project.org +X-Postfix-Queue-ID: 8F60183010 +X-Postfix-Sender: rfc822; yoyo@dev.pixelated-project.org +Arrival-Date: Thu, 16 Jun 2016 14:53:33 +0200 (CEST) + +Final-Recipient: rfc822; nobody@leap.se +Original-Recipient: rfc822;nobody@leap.se +Action: failed +Status: 5.1.1 +Remote-MTA: dns; caribou.leap.se +Diagnostic-Code: smtp; 550 5.1.1 : Recipient address rejected: + User unknown in virtual alias table + +--8F60183010.1466081614/dev1.dev.pixelated-project.org +Content-Description: Undelivered Message +Content-Type: message/rfc822 + +Return-Path: +Received: from leap.mail-0.4.0rc1+111.g736ea86 (localhost [127.0.0.1]) + (using TLSv1 with cipher ECDHE-RSA-AES128-SHA (128/128 bits)) + (Client CN "yoyo@dev.pixelated-project.org", Issuer "Pixelated Project Root CA (client certificates only!)" (verified OK)) + by dev1.dev.pixelated-project.org (Postfix) with ESMTPS id 8F60183010 + for ; Thu, 16 Jun 2016 14:53:33 +0200 (CEST) +MIME-Version: 1.0 +Content-Type: multipart/signed; protocol="application/pgp-signature"; + micalg="pgp-sha512"; boundary="===============7598747164910592838==" +To: nobody@leap.se +Subject: vrgg +From: yoyo@dev.pixelated-project.org +Date: Thu, 16 Jun 2016 13:53:32 -0000 +Message-Id: <20160616125332.16961.677041909.5@dev1.dev.pixelated-project.org> +OpenPGP: id=CB546109E857BC34DFF2BCB3288870B39C400C24; + url="https://dev.pixelated-project.org/key/yoyo"; preference="signencrypt" + +--===============7598747164910592838== +Content-Type: multipart/mixed; boundary="===============3737055506052708210==" +MIME-Version: 1.0 +To: nobody@leap.se +Subject: vrgg +From: yoyo@dev.pixelated-project.org +Date: Thu, 16 Jun 2016 13:53:32 -0000 + +--===============3737055506052708210== +Content-Type: text/plain; charset="utf-8" +MIME-Version: 1.0 +Content-Transfer-Encoding: base64 + + +--===============3737055506052708210== +Content-Type: application/pgp-keys +MIME-Version: 1.0 +content-disposition: attachment; filename="yoyo@dev.pixelated-project.org-email-key.asc" +Content-Transfer-Encoding: base64 + +LS0tLS1CRUdJTiBQR1AgUFVCTElDIEtFWSBCTE9DSy0tLS0tCgptUUlOQkZkZ01BZ0JFQURIWWpU +T20wcTdOT0lYVUpoTmlHVXg2S05OZ1M0Q0I2VlMvbGtab2UvYjZuRjdCSENmCkFnRVkxeFlxMkIv +MzA3YzBtNTZWMEZvOWt2ZmZCUWhQckU5WG9rckI5blRlN1RsSDZUNTdiV09LSWMyMHhNSy8KSlVU +djZ3UEpybjdLN0VyNEdxbzdrUmpWcFVBcWlBbGFxMkhVYllGd2NEMnBIb0VENmU2L01CZDBVUTFX +b2s4QQpPNURDc2ZmeWhBZ0NFU1poK2w2VHlsVEJXYTJDTmJvUTl0SWtPZ0ZWTk9kTW9uWkxoTk1N +Y0tIeU54dmF5bUdCCjhjQlRISVE2UWhGRThvR2JDRTdvczdZWWhyTmNmcUsyMzJJQllzTHNXN3Vk +QmdwRTA0YkpwQWlvbW1zTHBCYmwKV0pCSjdqeEhwWmhJR3JGL1ltejNsSXpkbm9Mb3BSSWJyS0pC +MmxaVDhIUHBlTVVJdVE2eHErd3RhQXFJVzlPTgo5U29uZWYyVU5BL3VseW1LeDRkOFhxbEwxY3hE +aDFQU1E5YVlPcVg0RDlrMklmOXZmR2hET0xVMzR2Y2VFOC8vCnM1WGdTY2ZFbHg2SWlEVWZHdGx2 +aE5zQUM4TmhhUU1sOHJjUXVoRDA2RFdvSUowMVhkeFJVM2JSVVZkc0I1NWMKcXRWSHJMbVBVb256 +NU13MGFURzlTZzZudUlQcU1QOVNKRlBzbVpzR3ZYVnZWbCtSNzl1SFBlc25yWkoyTjZqOQpNaUth +S045NFBhL1dJUnRoYWdzVnpHeHNtd2orTVZCRkZKRmh0TUtnNlFzYUsvbzRLNGJFR1ZLdWNXQk1i +MnNxCldmd0o0SndTcHcrOHgyS3p6aXhWTllTZXhRdm9oMkc3RDRmRXdISDJzazNST3k3dTlldjhs +bEVqUFFBUkFRQUIKdEQ5NWIzbHZRR1JsZGk1d2FYaGxiR0YwWldRdGNISnZhbVZqZEM1dmNtY2dQ +SGx2ZVc5QVpHVjJMbkJwZUdWcwpZWFJsWkMxd2NtOXFaV04wTG05eVp6NkpBajRFRXdFQ0FDZ0ZB +bGRnTUFnQ0d5OEZDUUhnTUZnR0N3a0lCd01DCkJoVUlBZ2tLQ3dRV0FnTUJBaDRCQWhlQUFBb0pF +Q2lJY0xPY1FBd2s4djBQL2o2MmNyNjRUMlZPMVNKdHp1RlEKWjVpeVJsVFVHSGN2NW5hQjlUSDdI +VVB3cTVwekZiTkg5SnhNRjVFRWtvZjdvV0hWeldWVTFBM1NDdzVNZ2FFbwppWTk5ZFBGNzdHazJ4 +ZEczNXZlWmIwWkg2WkVLdks1S042VXBucG5IeStxaVZVc1FLcE9DdUZKNkF0UlVEOTRJClJ2YnUv +S1hsMHdORDlzVXFlYkJZN1BBSlRNY1RjLzVEdWpIT1Erd3VlSkFtaFZZbEozVnpZK1lBS2t5U05B +QVoKZ3VVenNyUm5xQWU5SmU5TGgrcERpcVpHT2tEK1Z3b2kvRlVPQXJwbWFnNzZONTVjR3hiK2VG +QUlzRHYrM1NNOQpjUDFyQkFON2lEaGgvdkdJeHgzMFlrYUlpMmpmcXg3VXUydnNwSXh6K0NsWWdi +dm1wZm1CWmFqVzYzR0FsK3YvCngrby92eFZmVTMraTZ3alFjRS8vRTBTR2pvY3lQdUw0ZTZLNERy +S3k2SHQycjBQckdHVFZ0dUZPaWU2dnVzbVcKL09sdVB1dGszU3o1S1BmRDFpRXBobmpPQ0pNRkZx +Z2xRM1pPa3MweG00WGdwWW1ycnpQcXc1WWlzK1NEVjhobwp6anlrSzRWUlcrcC9IcUVzU29GQm5a +MG5XSmg2Q1pZOExIeVNiMVJwaFlMRFpWd21JRXd1OW12Vm1ISVIyWUZVCllNZEx4UExiOFZNei9t +QWpMb2Q0OGNSSzdSTzBSZ1RoMTUyK0VieXRGR3k5Y2tiS3VzRmJzVTFCQjN2MFJyUlUKenozTTcx +T3hjcFhVQ0tpWlI0MEVYZnErSnVtZVFudm1wSWdZdUNaQkh5MzJwQUJuOHNDdUlrMStyQnp4bXdt +bgp0WGh0K0RvNlExYXYyVjZYR00xV2xoKzEKPU8zaHEKLS0tLS1FTkQgUEdQIFBVQkxJQyBLRVkg +QkxPQ0stLS0tLQo= +--===============3737055506052708210==-- + +--===============7598747164910592838== +Content-Type: application/pgp-signature; name="signature.asc" +MIME-Version: 1.0 +Content-Description: OpenPGP Digital Signature + +-----BEGIN PGP SIGNATURE----- + +iQIcBAABCgAGBQJXYqFNAAoJECiIcLOcQAwkDEIQAL67/XJXDv+lusoy18jr7Ony +WQEP0pIRLp4GywGpH3dAITFAkuamO4VX3QEdVGjOHNoaT8VkSVWf9mnsYLl+Mh2v +1OIwMv0u8WyVtrcxyXijIznnJv8X1RgyCzpUJcmOh04VZcDyxKbnFHWSDMfJ4Jtq +qnXDONcfEeT8pwrGjP5qzTgcF/irG3w5svyQjEtj6kycddYtqUc9Hx3cMaRIzsHg +kuUzznSzU/6P0Z345q/kXyYvU9rlcsP9vogrsqL2ueLwYSipxUJQUrRWG82FYoCo +PAKNdGIt0xl2gEW+xWZkJqFarPiUFCx//+bVBelKrqj6rjwbj+E7mHJW318JYVHQ +en3Smv7pEWlT4hZHXnoe8ng6TAvKzQjf7/bUxq2JpKSycp2hDO3Qz3Tv+kc+jC/r +5UDWe/flR+syq8lAQTRSn6057g3BgDG2RtAwsjedg1aTFSrljSxbKlK4vsj5Muek +Olq9+MUdMFSE3Jj/JC2COcS3rlt/Qt+JLDYXKahU3CodaSgF2dobikDe1bW0/QNS +7O4Ng2PK0pA416RCFRUgPXerUnMGiWAiq7BoRHeym9y7fkHYhIYGpPVKXJ6t67y5 +JjvuzwfwG8SZTp4Wy2pg1Mr6znm6uVBxUDxTHyP3BjciI1zpEigOIg9UwJ9nCDxL +uUGz4VqipNKbkpRkjLLW +=3IaF +-----END PGP SIGNATURE----- + +--===============7598747164910592838==-- + +--8F60183010.1466081614/dev1.dev.pixelated-project.org-- diff --git a/src/leap/bitmask/mail/tests/rfc822.message b/src/leap/bitmask/mail/tests/rfc822.message new file mode 100644 index 0000000..ee97ab9 --- /dev/null +++ b/src/leap/bitmask/mail/tests/rfc822.message @@ -0,0 +1,86 @@ +Return-Path: +Delivered-To: exarkun@meson.dyndns.org +Received: from localhost [127.0.0.1] + by localhost with POP3 (fetchmail-6.2.1) + for exarkun@localhost (single-drop); Thu, 20 Mar 2003 14:50:20 -0500 (EST) +Received: from pyramid.twistedmatrix.com (adsl-64-123-27-105.dsl.austtx.swbell.net [64.123.27.105]) + by intarweb.us (Postfix) with ESMTP id 4A4A513EA4 + for ; Thu, 20 Mar 2003 14:49:27 -0500 (EST) +Received: from localhost ([127.0.0.1] helo=pyramid.twistedmatrix.com) + by pyramid.twistedmatrix.com with esmtp (Exim 3.35 #1 (Debian)) + id 18w648-0007Vl-00; Thu, 20 Mar 2003 13:51:04 -0600 +Received: from acapnotic by pyramid.twistedmatrix.com with local (Exim 3.35 #1 (Debian)) + id 18w63j-0007VK-00 + for ; Thu, 20 Mar 2003 13:50:39 -0600 +To: twisted-commits@twistedmatrix.com +From: etrepum CVS +Reply-To: twisted-python@twistedmatrix.com +X-Mailer: CVSToys +Message-Id: +Subject: [Twisted-commits] rebuild now works on python versions from 2.2.0 and up. +Sender: twisted-commits-admin@twistedmatrix.com +Errors-To: twisted-commits-admin@twistedmatrix.com +X-BeenThere: twisted-commits@twistedmatrix.com +X-Mailman-Version: 2.0.11 +Precedence: bulk +List-Help: +List-Post: +List-Subscribe: , + +List-Id: +List-Unsubscribe: , + +List-Archive: +Date: Thu, 20 Mar 2003 13:50:39 -0600 + +Modified files: +Twisted/twisted/python/rebuild.py 1.19 1.20 + +Log message: +rebuild now works on python versions from 2.2.0 and up. + + +ViewCVS links: +http://twistedmatrix.com/users/jh.twistd/viewcvs/cgi/viewcvs.cgi/twisted/python/rebuild.py.diff?r1=text&tr1=1.19&r2=text&tr2=1.20&cvsroot=Twisted + +Index: Twisted/twisted/python/rebuild.py +diff -u Twisted/twisted/python/rebuild.py:1.19 Twisted/twisted/python/rebuild.py:1.20 +--- Twisted/twisted/python/rebuild.py:1.19 Fri Jan 17 13:50:49 2003 ++++ Twisted/twisted/python/rebuild.py Thu Mar 20 11:50:08 2003 +@@ -206,15 +206,27 @@ + clazz.__dict__.clear() + clazz.__getattr__ = __getattr__ + clazz.__module__ = module.__name__ ++ if newclasses: ++ import gc ++ if (2, 2, 0) <= sys.version_info[:3] < (2, 2, 2): ++ hasBrokenRebuild = 1 ++ gc_objects = gc.get_objects() ++ else: ++ hasBrokenRebuild = 0 + for nclass in newclasses: + ga = getattr(module, nclass.__name__) + if ga is nclass: + log.msg("WARNING: new-class %s not replaced by reload!" % reflect.qual(nclass)) + else: +- import gc +- for r in gc.get_referrers(nclass): +- if isinstance(r, nclass): ++ if hasBrokenRebuild: ++ for r in gc_objects: ++ if not getattr(r, '__class__', None) is nclass: ++ continue + r.__class__ = ga ++ else: ++ for r in gc.get_referrers(nclass): ++ if getattr(r, '__class__', None) is nclass: ++ r.__class__ = ga + if doLog: + log.msg('') + log.msg(' (fixing %s): ' % str(module.__name__)) + + +_______________________________________________ +Twisted-commits mailing list +Twisted-commits@twistedmatrix.com +http://twistedmatrix.com/cgi-bin/mailman/listinfo/twisted-commits diff --git a/src/leap/bitmask/mail/tests/rfc822.multi-minimal.message b/src/leap/bitmask/mail/tests/rfc822.multi-minimal.message new file mode 100644 index 0000000..582297c --- /dev/null +++ b/src/leap/bitmask/mail/tests/rfc822.multi-minimal.message @@ -0,0 +1,16 @@ +Content-Type: multipart/mixed; boundary="===============6203542367371144092==" +MIME-Version: 1.0 +Subject: [TEST] 010 - Inceptos cum lorem risus congue +From: testmailbitmaskspam@gmail.com +To: test_c5@dev.bitmask.net + +--===============6203542367371144092== +Content-Type: text/plain; charset="us-ascii" +MIME-Version: 1.0 +Content-Transfer-Encoding: 7bit + +Howdy from python! +The subject: [TEST] 010 - Inceptos cum lorem risus congue +Current date & time: Wed Jan 8 16:36:21 2014 +Trying to attach: [] +--===============6203542367371144092==-- diff --git a/src/leap/bitmask/mail/tests/rfc822.multi-nested.message b/src/leap/bitmask/mail/tests/rfc822.multi-nested.message new file mode 100644 index 0000000..694bef5 --- /dev/null +++ b/src/leap/bitmask/mail/tests/rfc822.multi-nested.message @@ -0,0 +1,619 @@ +From: TEST +Content-Type: multipart/alternative; + boundary="Apple-Mail=_F4EF9C8E-2E66-4FC6-8840-F435ADBED5C8" +X-Smtp-Server: smtp.example.com:test.bitmask +Subject: test simple attachment +X-Universally-Unique-Identifier: 0ea1b4b2-cdb8-43c3-b54c-dc88a19c6e0a +Date: Wed, 8 Jul 2015 04:25:56 +0900 +Message-Id: <47278179-628A-43F5-95C9-BC7E1753C521@example.com> +To: test_alpha14_001@dev.bitmask.net +Mime-Version: 1.0 (Apple Message framework v1251.1) + + +--Apple-Mail=_F4EF9C8E-2E66-4FC6-8840-F435ADBED5C8 +Content-Transfer-Encoding: 7bit +Content-Type: text/plain; + charset=us-ascii + +this is a simple attachment +--Apple-Mail=_F4EF9C8E-2E66-4FC6-8840-F435ADBED5C8 +Content-Type: multipart/related; + type="text/html"; + boundary="Apple-Mail=_C7D5288F-B043-4A7F-AF3F-1EDF1A78438B" + + +--Apple-Mail=_C7D5288F-B043-4A7F-AF3F-1EDF1A78438B +Content-Transfer-Encoding: 7bit +Content-Type: text/html; + charset=us-ascii + +this is a simple attachment +--Apple-Mail=_C7D5288F-B043-4A7F-AF3F-1EDF1A78438B +Content-Transfer-Encoding: base64 +Content-Disposition: inline; + filename="saing_ergol.jpg" +Content-Type: image/jpg; + x-mac-hide-extension=yes; + x-unix-mode=0600; + name="saint_ergol.jpg" +Content-Id: <163B7957-4342-485F-8FD6-D46A4A53A2C1> + +/9j/4AAQSkZJRgABAQEAYABgAAD/4QCURXhpZgAASUkqAAgAAAACADEBAgALAAAAJgAAAGmHBAAB +AAAAMgAAAAAAAABQaWNhc2EgMy4wAAAEAAKgBAABAAAALAEAAAOgBAABAAAAHgEAAACQBwAEAAAA +MDIxMAWgBAABAAAAaAAAAAAAAAACAAEAAgAFAAAAhgAAAAIABwAEAAAAMDEwMAAAAAAgICAgAAD/ +7QAcUGhvdG9zaG9wIDMuMAA4QklNBAQAAAAAAAD//gAmRmlsZSB3cml0dGVuIGJ5IEFkb2JlIFBo +b3Rvc2hvcKggNS4y/9sAQwAFAwQEBAMFBAQEBQUFBgcMCAcHBwcPCwsJDBEPEhIRDxERExYcFxMU +GhURERghGBodHR8fHxMXIiQiHiQcHh8e/9sAQwEFBQUHBgcOCAgOHhQRFB4eHh4eHh4eHh4eHh4e +Hh4eHh4eHh4eHh4eHh4eHh4eHh4eHh4eHh4eHh4eHh4eHh4e/8AAEQgBHgEsAwEiAAIRAQMRAf/E +AB0AAAEFAQEBAQAAAAAAAAAAAAUCAwQGBwgAAQn/xABQEAABAwIEAwUFBQUGBAQFAgcCAwQFARIA +BhETFCEiBzEyQVEVI0JSYSQzYnGBCBZDcqElNIKRkqJTscHRRGOy0jVUc4PhF2SkwsPT4vDy/8QA +FAEBAAAAAAAAAAAAAAAAAAAAAP/EABQRAQAAAAAAAAAAAAAAAAAAAAD/2gAMAwEAAhEDEQA/ANW2 +3RMhJzJOiUK4up+4Lv8A4JUt/rgjBZfmN1JRBsaDchEklXfSaQ15GkdfFX6YteR4tNtDiuuCC7gh +6iHpt89O+vLFg+ARv8WAQ0bizSSaD1CkNo4fuL58J2/BdhXSJ+PAex64sL+PCDTEjK47vw4D1qnz +49b1+Pw4bMrf+IOFWj/NgPdV5Wn/AIceSUuMus/5sKMRw2Y+Lo6i8OAV4kiETtL5sMGpaACofUXh +/F+WPN1vvei0R/1YV7tRLpC4fxYBvcT2hJMztIenxXD/AP76YVvCICQnd+G/COpMNwvu/iMvh/Om +Bkgs63QXYtk+IIv4h9FuAJm4+0XCdpdN4+Lp+v8A3xGBZYXBJ3mVt1nzW/T1w6qmperb0koNoeG8 +f+44Emo6QBJN9YoSnVv+FJUvr5p4CVxBKXEmsdxdNvwCPpT5S+mIvGSBN1d4zbOkytuLqEgH1p/1 +phDtZwg1VIt+0LbwEKXjr6j/ABKfWmGGjhQmSvE/CVqVp3Xaf8Mq+Gv0rgLEyUW4JJVT4vx3D+mv +w4dN04IVyRv6Oq6y4fr3YQ0IXMeLkQMRtutIOr/LAqY4VNJqMgYJo2/iG3/TgGJDMyyCS5JuQEh6 +UhctlQH9a24EuJbjnFzGShyISuV3HiqRidPpUfBhSpEoBptjfKIp+7AUXgFdp8WhjiM3FYd1y5fr +kj/+7igI0h/FZXATmSz7dIkW0omSfUYIyQGCo18x6sNHMSjFVVZ28mEESK1tcwFURL5a2692BgLM +RBJMloRe4iIbQWQJL8Q99448kzjyVVJNtFqXCJf/ABJUAX+nOnTgGpiWeO0l0F5K5NTqVFRFZsqg +VO46Wj4cRop8pxC7FN4+EhSEr05gDScj+C8qFStPyxMWEuldBaUUWTIhDZeJKkkVP4daENLh9MMG +Kj5IWOy6ZFaSi7ZaNAyIfmoQlTAS28lME0/vkpvUuFuuTluYpDX/AIlL+/DDSQmkFVV1JibJqmRE +ZKMwIkD866hWuqeGHCjdNp7SF4xJO0UUC9lGIL+VqlKH5YdSKNjY8XJLNdkfERM1ht15W1pd4cA0 +bjMSkO1TXkl7R98u54ZUCS6unwjzGuGnr6U2lVBklEBUEb7XLojSLyU0t8OBTh8oSSCaaLVNqur0 +ELlZK3q0tKlRLpri6/um4FoRIGgIiQ2JEe6QpU/g+XTrgKe79rPrU1JWR4xMbiJs8dW20LkuOo86 +euLXkyPnEGntKSknwrOfAgm5JUEh7q8lNO/lXBiJg4tsDYkGa+8gO4Jl/wCmnV3YK+8USVLZ6vh6 +PX9cA3/aCaVvtJdRa3wiil1f7sKbuH3FluP1yH+QLS8vmx401ib7lnw22eHCVk1t0RFY+rqO0BL/ +AK+eAc4p0nu778y+XoDp/rhYLOCMS4ldNNPqUuttLUfzxBj94jESbGSY9VpIj1f7sPpKOBcEPiR8 +I22l3eeAfCSJNv8AfXeK0yWG78PnhJksropxT1O6mtorDp/zwhuKyiRD4VBL8FuHiJ0VdRqadPl6 +cBToqUKPcIFwx2q9JG5tAiL5SKnQWL5GPEXzIXYgpaXzYzA0U2J7DZYLrSIjK9iRad415bZV/wAW +Lbld4oLtBMUdtuokV1yJCV/0INRL/PAWsxHCTG48KuIvD1Y8ZF04BIF1+D/Vh3wn4MNH81/+rHkl +FLyuDAKtuAui7DG5aYiQdP8A6cP9VmEqjcA3dJCXTbgHbv8Adho07rh/04i++Td9R/Z/iH4h/LEq +4f5R8WAauIlbbOr5sNJIrClcXVb8v/TEhZMd3cI/F0/LiPcKYffGPwl8PV8v0r9cB5VToLr/AAl/ +3w2bckw93Z7zxB4bvy+WuEGPujITu6rb7P8AbXDpimolaQWl8v4cBFSWTLpUW3xTK0yILSSLy19M +J4cve9ZkXSN/iK38VPOmJjtPwiSxipb0mPUQ/wDfDTdmSYbdgW9V3i/1U/PADDTWEFSQC7cK4Osv +Ll7s/g/LFbnSeEqKaaNyYn9qVIPCXounT/1ji7pR6nUQncRdRJWdKunnWnrhjg7URVH3dtxXl4kv +yKvfT6VwDGWnlzUWy5nb4UNzquGnfYfxYmSo7bQiT8SfUIpmIjZX+bWlcDJXejW5Wsw2SIulO4ki +9Knp1BX60w5BOJJTpK9dmRWmC3jQ/wDudyg4AYlH+9XJyz30yt6lGHr8VydKcsNfYU2/EpmDZQSF +Mi49VLp9DpUsWl6mok4FZAA92Nto3D0V/Lz9MQ9kSbiShuhu6SEljLqr9K0wFb2RJXiWjl7t/Akm +5SPq86p1KlS/TDSqJCkkx3nztFUSWICYIkJD8vIeRUxMerJppCg5O1MS6QWATEvxDU9LcDHCm84J +sotHEiRD0u0UhEiqPK6oFUqVwC3DFumqlILhHDtESaS7mNJIh8tK1DTDjRSPdpW3tUFhuG4X5pEk +XzUpUudK4S9jfY2X9xcDQapl9pVbPFRJIfmr088KbpuvZiqyDx0u3QtUQJN4KpD+KnPUh591cAwH +Ctm5Nr0C3SuMBmCucl8wal34YVki4dqgUk6+0+AikgK4e+xTX4sSjGYUZIERrkiJEoR3pWkdS5VD +UuRU+XEVVGSkHpFs76atyZbkakQql5jqBVtL64B9wimQCuKxjtD1A5lfuh8k66d419cXBJwptIKc +GaC20IkIgJj4eY618WBOX4V42PcI1+DBINseGAC8VemulcWAy6/dhaJXdI2Db/XAI6VQLYDbLwkJ +AIkWvnzwwkTwXApkshaVvQNtwjTyxISIdolFPDbdaVn/AErjLc5zXs3tryVEoOTLjUFxVSTBK0Q6 +a3F1YDVne8QCQrAh8XgHqH64jKl96oRgIj1FcAlb+euHHAjYkmR3DddulZ4qfrjAP2jc/SBOEsoZ +QACcO1RRVGwS4nXltAdK8vxYAjCdpEl2g9qbWCyS5XbQMWRLSL4Ww3LlTltalTQRrz6sbU3RtutC +34hGwBIv9uM77Gsh/uLk9nHqAHHEJOHRpoo+Ov47uqlMXreWvJRS8fitLa9/08q63eeAmfcJWqBc +RfDYHh/yw8KgFTVR0FS8/By/pga7HfAVCcroDd0j7rw/L4sRyXSGtqbghEeVKJrJW/8AqwAqHb9D +wRBBNFUusGjwXIJLXaXCmoOtLq9/PEHedRbhrw14iToU3IIgQKiXz7ZXUrSv81MTNtuILryiICok +vtkT1htEvr8eoUpX/Fgrl8dj3fGLrt0uoy4wHNw1/PUsBZEpBO9dN2sCbhLqMRO60fLyxNPwfNis +N3DUnBKNt9QkyK4rFi6q/StcSmT5RNJK5suSZdV4omQ3fStfLAGrbvEGEqjaZW9JW3Ya4gbBu3BI +urqAsKbum6hlaYEVvhE7iup5aUwDqRDtdXi+LCjttx4BEgust+a3Dd2302GWAcMRsu+L/wBOGNxM +XBCXT8mJXwXYQqnuWlZ1D4S+IcBD3hTPpDcu6enqtH6YdNNEjJT4bbbvFcP5YYVuG5AQuIuoPhu9 +dPr9MeScEKQWmChEPSRBbd6/kXrgPJKfZ/uTEfiEerp/7Y8BCQDaBl8tvi/w4TasQEW9bcQiN3hH +/tXD7RNTiOo9serpEOkvzwDrdEhD357iheIvw4UZWmPXaNuFH8OB0hxCjhL7xAfDfvANuAlGp8Im +Al8JYGPVhbRi4qLIcQp8qxCJfrpX/liY7uF2P2k0xttHrH/rgE4UFRJUk3LohTIRsTchd/lgCPHN +xSVInIbJJCVhH0W07x8OJzQkRbjsBanaJdJ9PPu0xXjUUct0k0wXXJUesSRAwVG3zqI9OFRjxxGp +L3AuLceoS4AhtKnLnpTSuAKO+OT30yNclFLRSuciJF+tB6cQnpPBb2+/Hbt97xI3fkXThYSm4xIn +exxCdqgincHTXnd1/wBcQwkmq6tzaYBe64TMTStH/wAutNP64CO4cOC3RHilRTESEBcpF0111tuH +nXEF66TFUSdn94NoGswEwIbddCqBU0KmIOY2/wDYSqbYFyRQIVC4ZEF10Du8VtB1KnPvwwkxWsJy +ViZKiKaqAmq2FyFP4yfVpgCDRwjcP2xqoip4hK9Akh/lOh0KnpiU4FYj8Cgo23CAtgXD86VAg/yx +GbxrjhLWz90797uAuRi5EUqFrs1Guv6Urh9uiixVEk2xoN1BJR4AomgSRU7ipTy78AObrM7+GUbM +dsi3PdrEBEVC5EmBjpQ6fniL7NeNJAlF5VdMiPbQ3rhSXHv+8Ctaa0/lwb2SFwkmJm7akO8oCljw +efcVKVpUqaYRw6KIDcFpeIQbLKobpV53CmdbaF9MBKhJoU24i5WXXT+Ix97bp6+Gv+3BoH0W52iY +uQUEhu6ekuX0rirTCKZHuF7sRG5VdRtZ/wDxCendiGqo3kFUl00eGFoQ3P8AeErht15Kj02187sB +fEU00EiTU6iH8HixgvbLJIse1iFmbwIotdoN42ltCtuUK6nTrroPnjaoeWTctBtPc6RHqWSEhL8v ++WOY/wBo1qnJZgmpCPA3b5SRZxjVIQEw3QuKpnpy+LAbJ2wdozHKGWnjRMFymFUi4YSRG0ip6dVe +eMr/AGX8llO5gddpE62kRTTVL2ULvrAhLxndypr+mM37SE3Ehn1DK+aJv3iSqZTrps2E0m2gD9zQ +afJprSvcXPHQX/6pdnOR4GHyvl8150U0uGQZsAWclcI8x8VeeA15X3jgSTR27vFuIjaWn+L+mBzg +nC73qZqKIh4fcjbb6eLA7JMo+norjpbKr2H1+6auESuKlfi/pixNExELtlBsiXhSURtL/wDGAU4J +QgEU23u+nw22jiNSig1KgpbdNfD0f+3D52iGwIAJfONvhr9MOJN0doOIVQUU05lshz/pgKvk8lFz +VUFmCagpEo2BQ1hEdf4ZXjTSmJwC+GPXJjv9VvUpduj1cxpy50wJdSDx63SU3uJJUhECUDqJGnMh +U0KminLBSPTeNGQoILe5V6QcE2L3WpcwL3uAfVFwrH+Bds8EiEDELhtr8NfUa4fjGJF9rXRQFS0h +JIWxDcXxFrTCQZkTRW6xsI3dSiJ9I0/I+6uByKg2L75tbum0RWVSIS+bnrgH3AppgK6CKaZJ+7Ax +WVQO30PpxKSdPOLSUQDdTtLdEdpX8+dCuL/LEe5w2cWpuXyjhRK4myblJcSD56ajTnhi4SbjuGgT +dQvdEo2JIhKnKo3AXTWn9cBa4yQRcpGRGA+ERHqD+h0piZ8H8uK9GLcc3VUFYE9vwCR7/VTz0rQS +wdaERNxu6VLbiEfDgFbnXb4RHqLouwoyHpK/xYYcbZJXXn09XThO4NgkXhU8VuAfcJpqBtl1CXVg +c4TtcESnURl/qKnd+VfriVdtq2j4bekvl/8A8cIaKIuTVtv3E+kx/wCX+H64BIbJN+pbcISJMiIL +bvw1/wC+GnBOlHbXYM00RLqIfEJW/HTzH64mXeL8PUY/DhaRJkrb0XD8vit9MA4Y9YXdXy/zeuIb +jb4hLrtHq6Cs/wAu/E8y6Pw4iqkmVwqI7lxeGzAQTFwKvUZkn/wyAOn/AHYrr1PfkEF1D21Ej3Ac +popWkXop1f0xaXu8JpJoLbZXeKwekfm78AnCaKDhXrAhckN4rGAi5DuqQ/iwEG14mqSbYNxQUh4p +mTYRBXXvIK0r54UDdmpHtXKCIL7dyYlsmNpV+A7aV7sLcTBNIx8pe64dLpEhRG9IaFpb4ueKVlLO +0LmLMBRbaSapzyAkso2EDteI15Xjofl50wF1cOkWjRIWPFfMYC5VAhGneVNR8OuIAOnTlVBBiG4S +hWqr3pGC4V7+8qdWBj3bF17QjX/92LZJVS4dg689sqVu7/LD/tR0xkHTYTNNuyS60hWASI+/3fuu +dNO/ALVUYk0JymsCaKDrhh3GYXCXmFbCr04cBTgXDoUwPbQEVthNYwND0LQhp04Y4ckHaBDYo82F +FhDZExV10r16EPOmEx7MY+PSUXcroKLqk4VeEBCSQV52c7u/y9MAo3SjaQFpH3uZJ21EnI+6MHI9 +1fi13KUx5u+dNgERcgTe4UwdKXttoqeJI7ht/rgK92ZSP33Ky/Bqq7zoLBI0BEukkzGwvFjzdq+9 +sJJrtgFq594qwJyQk5DyWoJUPAGHr4V27qZURNOPRIiEyRFU0iEtPEFa1tuxFip5u+BUXZuouQbf +AS3uhKo6X0opbdTzxYjhWq6RIIHc12OHFIvAqN1dddK06sAnDUk5Uk2KwXDcmVywmIhQvuq0IfP+ +bATm7N8pGKrw7BBR4ncLpSOW2CVV7+jToLX88VtkNsgKd5xcgr4mrkCZkRVHUxu+7V/K7ErMUoSG +TYpPLrnhm6zpcXBX7H2ig60Cteq3XyxnPY/2mDmjKUxF9qMkbtvHKkK/EIjutgqdaCpRYdNa0r0+ +H4cBfXs5H5Fy4q9/u0ekVtt5ILqq17kaWUIVDrXlTSuMjnc0PMkuFc3yDMHeYph4LlJqp9+x1HSq +NCDWu5pt8jtwf7WG62TnCGYM2vAkW7RcigOH6mz4qhXkuPz2/wAQe6uAfYpl192n57LtEzeCCka0 +K1mwcrEHWOltdaD1269/ngCGTOynOHaQD7NGf5J1Cx8ouKhxgomKq4jyEj0Hl3Y3jKnZ/k3LcU1a +QmXmrZG7cMyRI1SK3kWteeDEYs3UcERGC/iESTvLp9MSDWETSQFsdviG5Eur6eLAIdtWqloi29yn +aXUF135YlJC3La2D6bbQ9zb03eHDUg3UXO0um23q2S6S/wBWFJDa48B3CN1+zaN3n8WAYVaqL+Hq +IS8JMwL/AA8ywpOrlMKDukNfOgtktNf9WPo7yO+oXSSnh9z1D/uwtFOhJ0qoNCL1FHl/zwFO9k7j +Rdjw32pdUiExRRISGnKleQ+PTExu1TbGQuUTURFIRO6NMBL/AEfH+WJziFRcu1UyYIdXUJcMBjdT +vLmV2EybNmRigIG2IhG4RBUbtC+YRrTAQ0nEgs0SYNHMcSbsis23JpKiPkHVXvpg+CaijcUxcuvl +L3wL2/QvXXENgK3tAVOMkRuK1UU3IKj/AJV6ta1w6qXRwyjZdQSVtAl2Y2l5W9+v64CL7LT6uJZg +O3aKSpMLbTu8N4UpqGPJM2qavwJ7he9Fs5VSC6heLSpYdVcC2aJJp8K2RHpvT3Q7tenSg1078Smj +wrBUI/fWimKAvBu8XfoemAkt2ai0vxithDZaIltKXfioVPDgoBEoFyVlvV1YQyIiAeu64viMSIfp +ywsFveql4huEREeq71wDlvw9H83iwwbVQTAkztG7q/CWF2rCJEO2RfD0fDhpJwN6qafwlcfuS/7Y +BXD7Ye7C3DAD767wlaQl8wj5a/hxOBQSDx4S4RIvuz27vDb8vy4BhJQiNXcC23wl4ht+anqP0wwk +jaZEhfb/AD+H8q+mJwC3TtH/AIf4+kdcJuTEBEQtTLp6Qu/5YBzq2hEvF8VuEqkXUJeH/EX/ACwO +OQtVFAjQtUO26+20qfniUagqAuh19Nt1wWjbXz1wA6QTH3q5AHu/CWzcVv5V78cO/tF5mkn3aLIx +t5sGrQxRsTAgAituvT07u/yx3ce2SVrYNzb8I+L+uMy7YOzGBzxDnxKJoPkklCQFFYrhOvxDbSv6 +0rgOaA7ZMwNsqOstyyISSyrNNuMin0iQcrhP1L8eAHYllOSzb2kRLaJM40RLiFV0zMSQAS1v1pXz +7sCc8ZHlss5lGJcrA9HwpKiZCJaDqVNC0rTBPsizxIZCzGM3HxpySl3CGkodo7NedRGlO8qaYDtF +2PAzo8MiabpISUVQUNUUnw28iHnbWtMQZNRvMmkPEgW6uK1yh2mx0LXqofw64g5S7RIHOzRV9ls9 +xqyt3+g+JYlUdbtKU8OJztZRo9Vkhc2oqJCNxbtr4bfS3kWAYVUTkGk0SgbbpRcW6pJgkREAc6KJ +20uw5Jk4kIol48zUtS2WwiCw8SlQeevPSheuILJaQTj1Xz4EE1I+5RmgIB039wqXFStcDknDpCPQ +UFtt2tVFJHoIeDMir1UqF3Td3YCHNvHSdqjFYF3X90YCpYXeXUioFerlXBaCJYdpdofDJqK7Krlb +dI0CHxDSuultcQ527LcU1cvgaqEggTl0k5edSqp9G8G5b82AEnIZils1rwmVI105FNJNpvpgCSQg +Y31PfCtev1wF3kM8ZbYvUkJZ5HNm5CooIEsiJidDLQuXPAKMzZE5tMUI+YQHiVxbGxTvG0R67yvr +pSug4qmcOyOaQdrzcyftJwo1TRSjE1ituoWlDVLTrGmObZhHMmX5NWQ3jjnC6pJhwhiJX09RGtef +0wHZMK+Yy3tDJahoC+kEifxxJgFhKp1tr4KaXW4wrKLHJ8tHv3c+/Xi3kI+X4tspq3SeJEqVqJqj +omZUPXvxXYftokGhtXCDBBo+ZvE3aZprHtlaNqgCnSmlx0xdf2pURm8nwXaDle9zluYK1dIQtJo4 +Itba0p6nry+bAEOzDN2X88LSPZZma9eHWIih3ghyaHXwhUx8yH0xdVe0px2ayC+Uhio4W8Paja/m +EkDVRt8YBWt35YwXKSOXYl6xEWexMKrp2MFrVV17kipRbcqVBS6vEHiHG3ZwJrm3ITybUh8uzWds +otxbvOJuVbJJFzopS2lbyHTADFf2hnhNFZlTIy4sbiEFxniSBUaFp0jUuf1wYaftCErk8syM8mOj +jWx7bhdaVtJL1rTq1PT6Y5omHHD5cXbOcvQ7aWc+8J4udV3TwVCvpVNGn3I2lrz8sPqsYtOPZsYJ +zl17JCQtjBPdJVyZBWhFQ1BEQGn/AKsB1UH7RWU49ukhmCEn4VZVIVrVmxq9Fe4/5cSm/b92XruN +v95F7i6R/s1a0tO7TXGB5oymcW+y3HlA5eczCqVq8EymCNddGo8ycOK0tAR05DrimRXGRsxPxooy +jBNcdkmcY5ScpEN1dBWXu0Cn1wHbDTPGVZtuxdxs3HKIyglwt3SatvfpSv8AXBU3xt6AndbS2laU +2fLHBEfGtyBiIg1zHFtECWkQjjJsbYSLXQlDtuL8sEY6NynJJG6mO0CQg16qVoDAk1XBIp/CNVB5 +FXTzwHfLgkWjvcXNBMi94S5ImIEXy60LH33K6pKILXCPVamsQld81taYHAoiVqHGAmoQ7hEm8MRI +bvQywg953aKDyR2WxkIriiC5CVPhry10rrgJircrxUUsLxdZIiYjr9QqOE3Ji3LrNdr0ioCZkYjp +3cq1oVP88BxcOFAJBNsxSTH3hoCCzUyG7xJlXQcE2ixEsQrhvolbwypbS+6NvrTrp+WAD5gtTNJB +8bHqVJS29Ud0KDyGvf7z0w/Dvickk5E91MhEWwkYGqOg8xr7umLAYpkySacS6QvK3pMw+vnzw+yj +yJ6q7XM/utkTvuEgu+SvxU+bADuIkuNSQbAZIiO4vaA+6O3y00xMZJuCtc8MumpaX3iJDb/u88Fg +FuhbaiFw/wCrCjcCNokZ/wCgsBW5XjEw3E0UFOq0bgMSEfTShYU3U2zFSwLbeohckI8h76UrSuLE +bpMQIusbvwFiCq8TUZELkwHc8XwiOndzr64Ac3dPHIAKJmV5WiqVq4/49KDg+BEKQ7h+8+IhC0eQ +/rgY3dMyblw2wmsXjJMwEh/PElJ8iqAknYSalw2isFpFTv0/LAJSj1hcKqqLApu23iSJF/W7DUgJ +IBxIo9Q9NxLEI/0w+ahDcuJmQlb4QEsOt3gk4Ftfa4EbiEukrfXTABVpJqmaSDkHS6aolaQgBj/W +lK64aSISMUxNBQh6htvSIR/FrUqYLTEaMk32uhD8RIiZf1pgA7FRNVJB3Zw4+7ESvQ3fw6F0lXAH +G6xEyEhO0RPrAguL+bkXdj0rJM4mHXdvjBNMEiUIhC278sVlw8GEaOncksxaIppXWFYNofmGla4w +ftI7YGso1eSEei6QZsEPdEsaoiurUtEqbZV08XxYDMO0gX072tlGtHi7twuuLQFyMeo1CtU8uXfi +29tfYf8Auy4/eDI58SzSFNNdgiZqqpHQaX6d3u6+eGP2QstupntT/eBf7SnG3EuQheIrGPLn3c+/ +HXOaBTFwTlN+u2WttIk3Ipd/cVR88Bz7+y/2fuo/L6+ZnYBFupJ1cglsqkIpCX3ZUup01r640aYb +uIsCtA1FFnQqGxTckIthp3Kp1IS11xfmjxFCPtF4gXT1qk8G0jp5VpSvLFWNw8KbfO1DBRNNJNMQ +RkrDH8SZGXd64BjbUdtHRXoLqSFok5JyBJKhTnbzGnOnngBl9jISUfLLsWdzVR4mmBuQNIeFEaCp +Sp39Y8vTAztleKZd7MppQTTXFJmVwEaSlxqFyup81PWmMo7D83ZwhJOAybNrIO4GSSEUHKhkQpDX +4aV88A7+0ApJZdmIwpts1dkquo4SapuSVQ2e8RKtaa/pjqPscnonNeQY6Zi0UGyKo2mgj0ikdOVR +rypzw1nXIcPmnK/AkCBN10veKiFxEGt1LP8AFTXAWHZw/Zrld409sAgN3ELmodypDXl+dacsAW7T +WakzCSMM2kgbFaKjxfwkklXlqNaefpjjjtDheEy5ttg2BTeJqM2ajkbyCpW7ltRvIirXW6qn6Yum +YO1bicvyKDGedEirIkntCwVAHI1PSiZrH4Nfp4cV6dzQ6YuDXm8vQ79FdcU3C8iZqAkr82zSvvCp +T1wGPzYqcaF3uyEtnYTRt2redvfX19caP2F9o0XFoyOQ88Gf7nTvTVVPxR61e5UK110/6Yoz2aUX +lXT5tY2RUXEngCijYJ9VKbIUHSg24BSCwru1VEkQTTU8IfKNOXOn1wGvZ17OEsmZ9QUmlgex7tcS +iiYPNpWSRrTQVRUMSpuXVG/F3/Z4lEYvtVQj1DfPWsw1KHkzWAOHFal9QBNQdL6UpXTw4pnY92kw +q8Gl2adosX7VywfUwclaS8atQdakNa/DjXshZByu2ze17SkM+Qj/ACwxVTcoCmAJEJAJDQbB00Pn +gMwzaz9jTs1lJMPZKjZ0omIRkUTlyulQtRGq6p17vSnw4qTduK/Zk8gpCbmxeRZE4OKbQgECVvxE +4pW6nV3/AFxrL2Jku0bND6ZUWQfw71cmDAE5Uw4NZQ613KgBeQ1540fIWT4WJhyaZLeLuU5Z+LAT +dnckg1bnQlST/CVuAweT7Jc9LysO0fxuVoseA41BVBZUeJCtLy3CqV1a8vw+LFn7MuyEnmeGK+b4 +eOUhZ2HN+q2ZOTQQZhTW3purUtdO+tca6xcOs2vppKQZoMlJmTGFaq9RAqxQG6ulfmrrin5tfOGs +rcxm2rkZaWHLisWwAiXYx6HKtEbeq+lOoq4AZlfsty64exzR8xdTjqSQcroA593wzIPBubdt1alb +1Y+9k+R8szGQo54tm4YdX3iZs0yREUqioVNNFBIu6lO+uCrJ44TcTXsebeu2+ZHiUPliTICEWyQc +1EK+VtLfF8WM17Ycms8ydoEh7IynGsvZdsc7obsU91wnSlTUpQOWlb6c68+WA6bRfOmwqsf7bQIx +EjQcMBck0qXylSvUOHYq7qTj0Ytyool71dRFVrv6fCVLeWKeqs6THbTMF1F3hNxXcrOGptv/ACi6 +i6dfPGgx4vBjxXXNdk4JAhVaovANJUacrqEY19cApwTxNuPArGKdvUkm8EiSKny6+VcCnEgiLjfc +tnW2V3VwAqiqXdcFUyraWBLuQlhbuicg+bR6CRJ8U/ZgqIl+aZDy0x5kUW5cPHaxxaCLZAViFNZZ +C3T4/FUcBeoov7PSTEwIfhtMgMf8J0pzwRVU21RJQzES8IiZdReflilQT58o3auRcunYqD0E2eIr +3fWuoULzwa450SpIyRmKIW2kszIBE/xFQ68/0wDUxnyFi8wDCSRmm66Ss2VS3Rr4Oqg201+tcFHc +oJASiBh0/ESN4/y10xmmeGL6NnWeeMttuPUYCXtEOPvBVGg89AIfHTvxYsnrZfncuK5kykYOWrtX +cVSUM7R08dnV3YC0gXFt99dHY6rvd33Fp5jW3COOInFwrIWq9NxLGQ+mmhDTA5u6auWQin/cfF1X +gaQ/NTqrityEw6d2pwjlBR9cRAYvBFJUKFzHqTr7yuAtjhZZFVVS8BTSStEiMCLn5U58+7EdJ069 +n7gguTciFQSEEiG36UvxBh3Cz4NwQfcOkXUKjMCJA6fDrrzHnhTh43TVdXbnvEulsTAhFUafxB0L +ASqqLKEIkTJdYri3SbKpDbQta30pSumPJSTxA9tA7vmBFyKto9/TQ7f6YAKuBT4VpxgCsqraDlNY +wt0/grUK7nicZEo3ErHTZNoqQ3+6MxV/QaahgLTCSRPmiBR5oLpkBEBiBiH+LpwnMbgULXK6xijt +FYArWkRelOWmAkYTxNwKhM0EFPESXUhvj849RDg/GCpIe8Jb7OmIiKBAJdX1rp5YAAlDpzKTOSkr +yRH7pJRG01R8r/yxzN+0b2d5mj5hVzEw67mJekSxigAiKStPSl2vdjtAExEOoLrR6R8Q/pgBnMm4 +tEt899S0iSQ2b7it9KVp/wA8ByX2BZ4cdnMezYzsI6Qg5R4XErleKu7UtKFSlR6gpT0x1bMPk0AF +2ma5AraILtgAejyqQHWmuOUs6tXGf+0OMy/lmKek6YPEyVMXO77q64zKlelIraaUpTHU1roWl3vy +RQHbHbZ9XL1oZVwCm6inDoJke24ckRWkYbRDTnzqNa86jirTcojFgRPeOTZgqo7XSJZI7RpzpVJS +71+HFkBR8lH742JkRdR8MApfnXn008sZF+0BnDKsW7jsuyjldRRJVNZVsmAEKQ99C6fFTz0wGV5t +lJbtYzB/Z8lKxuX36pCqCzYEiV+QaUu67dCrX0xecqdk8S0ewqEXNryjGJSInK5GRJb1eeqdKd/P +/LFGzX+7ck9SnSmDUg2F1qVhWrq9OvMCGzv/AFxfMpdsWUVAVhFwNo3SakRk2RtSSSoOpDTn0cu/ +xYC7uMxPISPKSQzICEKhamSFhHb1adWlOWvlijdrbeazXkyRmcszzGShU0hLYK1QV7br+fwlTXGI +NO1DM0FNSycO5aqRMo6JTg3CO5tbnOhCNa89MDgzpPSkYWX2zlrGxqxC2EUQ4YCGpaqFWvO3u5+m +Ahw6cauaRKImg3JISSIXIiVwDfUqHWvRWpVwSz5KrLxJi9Bom6cEmk5QTtqu5NOnPW2taAkJfXqr +gN9jbOjbQz81yXXFDg+GEwVSE61oW4X/AD0prgY72U4z3bb7Urt2rl1EI0HU7K00+L1wEN6s4ILn +bnfWuuG4yIh+tNKaYYVFGwRTO4h8Zjdb+mEGI2EQmA+HovuLnh1u3cOd3hkbhRSJQ7flp8WAmZcj +XktNNYuLZqPZBc+hvYJbpU56aVrTWnLHRvYR2Px9PZ55ohJh0+cy224aqGKTBAgG60wqVx9/y4uP +7PuScsw+So6WGNXQlm1sobpw2LfVK21MNdem6teQ4uj2cLLpyDZ2ALy0XGKSjm463KOnJWpiH5UH +AO5XynFxMPc0ZsY1x7RkH5N0UbRIRIgDWvcGlBxFhk28fmjJrRg/BdbL0Au5etRutVSLpvp611rg +J+/yJRAkig6e5fb5d2J5cQ0SbOz16OfVUr/Fz5YrQR/7vvc0sZDM5/vZGwiDbL6qP3pN1g5IV+E9 +DtHXAWPJiR25IqJIL5eeuXz1q5QO3gVfx6+nriN2eos5jMTB9Hnt5uj2cgslxN1r65wqNSv0tPp8 +6d2Bzh1IQTKamYTLfARLCJbR00xerW2rLFosSI87CprStfmw48FpFu5WJeLOXsfHtm0Pld41C0UH +CydK1Cp0+MrvFgG27zJ8S4YxCiK7iHlIBzahfVUo18Neszr/AA9a9xYsuRs8ZZylkuHh84ZffNpm +jWii+raq9VKVKtp1UGlaFWo0pz1wEcNXjYcxSy8UyZOsrw4wr6OI7ReAuH3255/TpxSs3yfbBD0h +o9nmJoTdOKR2qptrumtSrSla611rSmlPLu7sBvbt8+cuECEHRFdtoLtJVJUFQt8KiZlUe/Dhk8JI +k3ca+TbkVrls5ihMufxBtjz7sDAUWlgJyTYyUEi2gUYAok5C3ysOtL8SmTpSNbk2QBD3hEiYuWyo +GzuHWzpErqcu/AOgUeobVsoiDS4x2leDcJA5G3S2tKcrsDpNNiWTJaPmcyNYlEhuamKytyHVqGoK +V5h64hw75mu7dMXbbhk0FxUPZfqggudvIk7g78DO2BqtLZXVzAxlTFZpc4bKjJFYI0GtaCsnb318 +sBRcuZuyeLtqnm0GsTNOUuIbzsUAqtXmhW9QV1prXTGzwTN49iUH0bmaONS4C3VDMe8uXh8q4xMH +2RZmCVhs2g+y88FJO102MTYuSMdR7vXSvfpg/lLsxybGuI6Wgu1ddMVCTWXbk5EgeCP3gn1eHywG +pTWR1loCWDMkws9QfhYItEbditfQ/FX9cRf2Ymfs/Ji7NS/cbPlUwuAQ6KaacqcrsaQ9WJONFZsC +bkSEbB+EhqXi/LzxWcnt1ISdXaKAvsu7VErrdsjqVddPxYAFNsXmWc1ul2xmqzfiSiW4ssZpKkVd +wk9K+HTyxBBFqMZ7PXW3ESElHKQuQsXGni27+odMXDPEgm5BWJFyug4X+ztl0ekxPzK7T4fPHKXb +xmDPmUJ5rCN8yLv492gL1AXaIkZABeKulfiwHQ0Y6byTIXaDMxjR6W7rgw9+VOVito8tMOpN9twL +tzY2WuEgbE5cIKpH3ahQi58vLHJeR+2iciZVV7NxsdmGHdl9sakiQFb8JUrTkOnljoLJXadl/MWw +5aPzTdbu2k248hNmVvLVMw5j+VeWA0GScPlNpBj9pkF+o9t+BJLjTxaUUr4qUx6VYltJLoLL7KA2 +++jRMf5S0p1054CK5wYtJAWhTDFzIF7x0kmsiaRacq2VMxtKnfpgjl99EzqROWhoOxUIrz4MhK6n +rQa1pWuAlMkxFLcFHY2BJQGye8kQ+pJ0+WvlgrCPm8fxUlIPNtqQ9a6jkiAdPz+L1xmHaB2jR8I0 +XGC2J2SESFBg2MhVZlXoutrS4hrWnOmKKGX8zdoOWkpbP8qom1IRFqzvIWbnq5ipUepuf5jgNjnu +37s1i+JTGVXeqIEQ/ZESMVSpbyvp0/FjIe2Dt2azbjhMshIpkfu26qjbbAXHwa106ufr04Mfui3h +FUigoEPbEb769yF75AKjz5U1Byly9dcYZnXPk1JNF3cgzjmUW9ebiDFozAAfLInaRLalcn3c+WA2 +f9kKHY/uo6za+eIO5JdUkxBNYjVEaFpW6lOd1fTF67UO17KOR/7NlpV0TxVDcBq2bGSpfz7nhpXH +Ks3mqcyFmh0hlSSdRryQa/2ogQBtidedBCytR6acqYpntK5w6cyRunssSorA/JzcSRfNpXkWA1yT +z52idqvthpDOYHLkftCmux4xJInNLuVOqvj/AJeWMsk4V4KpLiEiusIiiLn70COnKo0UT5VpTFp2 +ZCZOJzFLZk9pLO91RfeZkqDFEen3tQ5ipWlOXLBaCy/ltSVdS2zl1eLuFFmKjxaMNIg5VqI2V6/M +q1wGWu3jxoyViRld9mv7xVIQtuOnffSvPliG0UIkiTIAVRFJQhSUOwRGo+Kn5Y3HOGR4UocY8Tn4 +VwLVNZq1ethkGdlSOu6LhMq1oPfrdTFNmMhtUuMXbSUc7GPFInIRRmuNhANd7W3/AP5wFbh414Nz +xSNB2KAisYCAmJXDoI1rT8+dPLD6pNU4f7YCiaaaBWJJ3g2XVqXhPX4x89Pw4GMotQnrxEXOwLIr +lTLUkr/DQLw1pTXXTDTtq3QBVcVkLSG5AbyIyKvLly0rpgFhMELjdFEy6RtVvK4Sp3fQvpriZmCU +ZKSRC2hWsOqKRJqVTvExr53p16afp04rpiJK2+IvDh9JqSgJbBm5WXK1JBMCI1dO/XAJSTUduEEU +kffEIpjthbd389Mat2FZRy7mTtaistzbAHLUUFBXJot0kdOoiMq+n4MGOx/9n/MmavZUtOouo6De +qkiJDaS6Zc9CINaVGmOlezzJbHsnyk6ThIcHLrj9gVbxVUckQ0ChVrXTb+o4CzTcKtINI6ICSuj3 +MiLlxsrEJi2R60xCvfpcI0xW+0KPWfNYxy9Uatvt6s0u4W16EkBtBHT0rUsW2RB86kpMmtlVl004 +9sSev2bUdVO/TFGmG6a6XCRLw5ZrJFwBLisQBHx6ZarK93jvwAyCasZDK8i3m2a6DMUlJybjhvHd +MyIkqDr/AA7benFYzXKFsZNicxN4tpLOf7TQnWhiRJNkAqYifxeVnpgwjF5oLMDObjX+1JPZEhbe ++/v0S15UQ/mK27FSexKcsD7NCiKjaJzvIkJ8Mjc5jGrfrM9NenWzq0wBNlNZmkHpOXbM085ZgXT9 +pxdg8OvFJalRdPX0pT/dh/8AeaDcxr6LYsJVCBzfOpezJMj2towEAU7uoLSHzxGSdSGa3sYxkIqR +TzJmMTRh5RsskkaESjpStdLuV/xD54hqrM4admFN6HdumSow8dBJgXDJAQCBuhGlLjL5vTAOx7VH +MGeJoUZVf2lJZiQQYKuLwGTatOag1qPir04NS6mR835kmXzp08iF2j0mSjZJZK0STEedPpzwOzA3 +9l5ShYqAePZBjDvCSy/NsrRVKQMap0Rt+USLmVfhw8lm7KGXboOeyvVWdZ125VROy1RzpSplzrr5 +0wF93G+0kmobFoR3CbMkVkFROvKpp2lbSuFw6nCSqQsTi3LghJvxSMqqBl3aCompfzxYYyBFKMSE +pV8wFAyWbf2qKto+mt3ViUrEpqPVXKDlBRwukIqmQNy3dPOtMBXQUWdyYx6iMoTdNcidCLlurtad +xDTb18WDBIqOyXeoIqJoubhtIEVUFdS+MaDStP8APHm+XxXMehiKyZWmqLNvfb3+VMTEoFu0t3Gy +GyXSYjGjYJU8+kfPAYfmPs/GNz7wmXXMim4ciT1WMJmBtbrdOYkXxelMBIpx2hRMgxhk8kwD9R28 +EbhZ2mkHmRj8AjjXs4ZNg52P2FzQTkBVuYP2TZUV0NC8NCGndTASVynnZKPXaKdp0qmiqI/afZR8 +SOheEdBwFszB2qZTyo9Z5Zm8yRzKSFr78bCsEre7kX+WKjm3t4yfHxSrmLmAeqJEO+QtiP6Uspd3 +fXBPLnZTktSMtfM46aklPeOZNbeJUir8+vg/lxXnH7PvZm5kN9BHhritJDiXCSRF52lWuAfjO1SN +mYwVCB1tkKYkkQCJFryBKnmF5eIsYZ+0tLN5bPqUPlt+g76dl4qif8YyoO3rXwpjrpTSuBna7leW +7O84Euxm+LR3SJqSLm5XaEdLSrrqWg8sVbs5j3WYsykXsEJFmQkifRtpCZgVA+l2tfPAezQKKb2O +TEweikgmjJs4zwCYEVKdfdzuwhxl2ej99ZaBXbLXEInxNpoBQdyvPypVMhxenr5EjSj3MlHQTNyS +aJtmACRiksJDXSvdVShtx10+fEyQRZzcqzhG0VIuyJBJy+VcrEkIk36V6VofxVSEBwGTTcMoi0Xe +iKI7a4p1ETIlSM6X86/Fbpprg12aZyzNBSDptDTxsE3KRe9WvMUir00KmhUsrz01xq+aHzFskWX3 +LZi9JOHGJJVkYkZGoO4ianyaENA5/NjHcrxu7mhncBshUMbEBDirQp86ddTNO5Pn6YDcOzRNjDO3 +xJooKOhJJRVq/WudLkI0uVZOaeIty/UK40Z7m4W0Yk+bLG/uVJuq+FsIgqVDsseofBS6um5TGTSq +y0M9YxLZm1i/aCqb8mbtYBSfXnrvtFf/AA31HpxufZ+xh42Q9koLA/lhako8NZEQkVUT53KDXk4p ++LqwFbeymfvaboU8sAnHtvdpNifikbYrdaG1cW9YFr4SxzvnXL+ZI126zM59lcUkqIvEiMLxSUHT +3iPOlda95jXG+dsuSZ7NbJ0ojmpjCw6g/wDzJhHLkPcCievu1PzxzhP5fy+0Vaw0PmF1IzYobNjo +w4MdeeqKlOXKvngKjsuLCT3gUtEVCK/15aa151LBXK+XXkzvu7DQh2FvtN+IXi2AuVK1p54smX+y +uWdsvaksDpNmLVJ+SrSxURaVKmpfnTGjZHy3FyjKMF2TLbRVOOkWTJYox+Il1N11R6dz9cBJ7H8k +wPBCmq2ay3GkqwIhWVbLktdUm6hBdb1jUNNaYub/ACyyWn2LsmLpszIVHLKPk2YEgnICFRVQUUrT +uUpdgnGR/wDZls3x0pIKEhGOR6UHzZUC0buK1pp8NupYNSTpxKNVY1u8jicSlrd7Ey5+8QkAG4Kh +9K24DJ89rReRTdT2yvBOJJLiIl02WNdiuPKizJVK7QfpjOZicarsmMfHsGMOiW6/XVYLbAuWVRvJ +Dc5jUrqlTmOLX2kSxLzSUAhJOsqx7ISJzHSLO9ik+AeoAOtNOuleWMHcCK6S8goANk1CUJqkmBEF +1S1qI1+HT64B+VdKIGTRia7SPJIREbLCco18BKUp01LT6YgmoS6pKEdpfD8o6fTyx5K5D3lgESvu +QBQLum3S4a+tMWfsvyS+z1mVrl2JWQTeK9RG4uEREe8dfWtMBEyFk+Yz1m1jleEBEnTu4h3j0AQp +3kX5Y7M7GuyNnkuRy8+JmnGzqrddJ6isjRcOXdtH8PPqw32UdmMHkn2dmRFgg7e0kybqul7gNsld +bSgU/mxrPGf3xBis6TUiVd9QVEamSiNR1tCnf+WAQxW/shjKe4lnTZUhVVZdA866EWnxF4cNyDNw +LSTJP3CzR1xIiytM1xt8VRr8WI6ScW9YrwSPu0ZZInbBJFEkLRoVKkN3z3Yr2Zc3IoO2eZLHQoxZ +cPOt2R3Ls9O5Q6U5kl64AxlREVz2FJUHMogahDcZCSorc7lA8unAfPDyJj5VjDMXhsnUoJQ7VBEA +2hAupVxX8tNNcVHtrzY6guz95M5dkoQnQvCYEq5RsV2lPB1U7ra89ccoXN1pOVLMWZJFzKMmYptV +WixOTcuCLUhEi15UH5eWA6NzpmrIeW8iTSbSadFwy4xOX0kD3HKAB94oPnpU7tcNuM4QMzl11nRs +8h2QzTwYdVqssQWsg++UEKF0GQ0xzetGx7kJCbgeEg04ZqgtsOn9CdOVa0pzRqNddfit+HDrLJst +JBKyTZG5jCtUnr5eVAhItwh6qJ+Ihrry+mA6BdosSyFO5oGeapov0k2WXWvEikUQ1qVaVWurS+6u +3gV2VZ6iZSS9sNm7V3N5SjnKEYigjti+bWaGssZa2mXixgjd4pPCvJLM3eYZZBdN65cXltC2DkQE +Ff8ABg2bNnKRM7MNHy45sdr7iELCI3ICy8St9Q5VGg4DpKEgY8v3Zy+pKoKwbRgvmtd8hcKSCxDX +poQlTpuK+mtPhxKyoz7O3eWmEp2hMWimY5BMnLpRzXQ1KVMqAX5VARxyzJZmzMmq8KWmHTIphJsg +5jhbVTJWP0upbpTQE6DTw0w52lTMzmzNKkzlaKzA3hTRSSZhUSPoAaDrr+dK4D9ApCWJiyLaYOnq +hdSQJswC4fUNS0rpiuhmBRR0cWg/fETAvtS/sfrQ6dbS1rS6ldfhwFcIpuZB45TCHdsUepVqUqtc +JULkaVdeQ/TFdZTDOJZcSQLisuvw7V40myJQdda2qApStMBdWWbCd/aX1jISVJFsKkOZJPPTQh1I +f8sSm8wzT+1oItd63qFQFmyo6d9eQldSlcU521kBSHL/ALKn2zh2qLm1OVAkHOo8ttTTo/TuwWXW +nRcKtiRznwrUwHifcmq0ER500s1MK4C6qyDdy0QIuolREhtcpEJF5UG4hLX9MPgoo7aAgu2Xeoiu +PSVwqj9dNNP64zSkhHrv15Jw+uV2hG+RhCSQIOeg1qNelX8sfcvyDcW68g2coKOLiEUE3izZygNC +58lCOh0p/wAsBa8xqFDuFU09gXRKiQkW7db+KohWluH1U1CbquU1uGLxEk2kiEVfOpDcHf8ATDTR +wT5uKC7ldRZyIqCScqBDbT01D/OmDG2m7DYvejcXSSeyuIlTurXppgMK/a6h2st2coTblF9vMVd5 +i6TciokQmPvEz7rK4yPs3i4Fj2eJTLk5h64epL77NtcIbyJXgOunSW2JV1xrn7UBFIJZdyTIOV4l +u5kUydOiAUklUSOy7l08tNe7GV51ecXxOWxeTD9wgWyDZojsAubfpqXLuubbutcBSMvqAUw2YkjH +QqTtW0V1A3XHWV4FStNddCT0/wAWLRHqSzTtIdZiXigcyyAlIuuPeCkKoh/eAER18VbqaYp+Wk48 +p15KO3nCNU1U0RQ+/dJDXWoKBbpQbTAf9WFuH3t2VkXyhxyDwRUckq5c3kSwD7wenT7wufy4A/Hv +Nr2jHtph1JRMgJNBZsg9+KtfeJjQq/CKop0rprgJ2SIqFnJdBjxzZ8kNzVdMBJdJahiFp6102yuO +hYH5aWJpxX9pOkEbVEySaBcrbQrxKlfl3KBrUbcK7N3gx+ZUnLnikyXVUTMiuFBfUhrVI606uf5+ +K3AbBmBYWPaEu8QRdRqkA63HTUmwqoRBVAQuOl1dxuoVPLw4tv7OrhTM2aprMzaN2ExXJRy2HxNt +OSazMu+o28iHFEzFmx9GlxUWunuHus2b9yFoKpU6jYOhrrU1OfKuNA/Z49m5bhH0zvPmEeO0JKrg +QlFLEOpAQ+betcBG/aVzYo2cEhGrINBXSJQ3hdTCVAdKVTrSn8al+KB2aN27Ro6Fszjm0kyQF7w0 +rdYuSB1WBIFCGmlwVHp88Fs+PNJ/jXzYHca7XEn8ZZ0lvXgLhkPwhWtBxZM0Q7okozJqEkg2cEIk +8GcRvJJ23SHh6XjbbQxoA4CoquHDRw1YyEIu2br7sm6dZfWuAopyNtR56fdEY91MXfs4bt5dornS +YRa5pRjx9mLk3bbTwUaF9ncc/FpjM4SQKC35RMJSHlFXigsSaBxjYmv/AIlqVK8h07+WNpaM3CLS +MaOVvaaKTUl20xCmIE8ji+83EqfEkWAkPXQiAuZtZeWRXL2TLNlLUpBsdVSq2W1u59NRxTMwZsWY +wjopQ/a0optxzmOeo8LIC48Ld6Ned5YBSeYo+Seus3yxxeZYWHEYU3LTdSeKjX+7uqhd7whrXTux +nmaFn0obGSlJv94czSCXALtnNwPmKqJe75fXAO9pco6bQ6GW1ZibXkl7VJ2MmkRJVs6HWgkB+hCX +PFFNwomqJCz2G4qipwal2wqQ9J6curu54Ovvbk3uqLzwSUo7AnEik7MRXSNDptur9K8vXEFJvJKR +6DGQW2BTtKOZufEqKneSf4fPvwEGQInyqTZMzQUVX22bMjHYSAy+f0u79cdr9kmXW/Zdk9dRys64 +qNeJrvBRWFcVxMRpdTlcKfP/AG45O7KomPlM4MfaUachDpltyZKARglr4CrQK07yx2Tm6Sh04JnP +Ciu7y3LsxinhRAEa/LUU+XoPPAEc2yEXCCUI9mzGPn19yMdJ9RtnNSvp/h53UxVp7tBl2GRoLNAt +XUhLtnCLBddvaSbsqlQVRVTpXWnTqeAvahFtXfZ6/aSje5jlkmhQuYm95GkFo6b4UrQqVDzLAU8v +i+zxNRb7NQNnDtdKYi5ODAuFuT6bFh50Cta1wF0zlmRGLOai5J++lYtyz41g4ikfexi1fhM6V6aa +088AJWJmJdWMc5sc8W4ncuiizmIY6bguEhqr9oSpXQtRIe6pYlzEg0lX2Xczt1nuUnk4t7HeOxQB +WOkkqULmr8I6lTliM4y2xyFlxBo5eKQE5HyhLDNtDM2LYFz01KhVrZ0aUtwEbMCeYMwZMSzt/Zyj +p/lhRo8S4Yb1VrqAIop61tP1+XGJw8C+Ty+8mYsGLB5lUuLfSah7rpUlx29nQ7R93p/vxsTLNGYB +ZDFwkOcopkiT2ym2RgQKokXLdCtPFXvrjPc1SUPlXtNGJ7Tma7tEnjmYdNmH3BGuAcONRpzOmoc+ +eAo80WRYn2EtEwq889YoEU05WWIW6rg/uu6lcMrs4mUCFfPsyS87NS6SqktHNek0ACmoCVSrSnLT +E6cdZjdrA9zB7Ljsv5hoMsuzRAU/ctjqFltKUK/o8OEPVE5ZaThcrZGWi/az8ZFs+vO9CPoNaV59 +9vO6tcBEn2843gIzMDSUQbp5qQUYKRsZpv7KZU0Egp6+eD2SW2didx9MloxeXRzKkWXyoJ9R7I+9 +VurTp566liI3DKGVo6dpAS9ZPNNJFJpCl4bUbfen+HW7TAIo9mWT3j9bMjptKRcmLCLiU1qmaomV +yxCdNPMsBbcrzUSxkJ+GY5Y9pSU1Gex2JktxXDOKdBERlbSgkPUOnhwf7LU89SOTmwU7Vo/L6DEi +Zt2pElStQTrpd1VpWutalT9MZ+oxdJuJvNGR33sGHi3gNEkH61AWBRYbS6dK92terHyQkMt5Wl30 +MyZ5WnWyS1yb18moZqaiNS0rSult12mA7DNN5HtEsuj7bXTQuWXFaHFddtqWoEnURqJjSvpgPmBm +3lJVBd2jCKN0iTUGRe5eVDfGnKqahCGlOde/BBW5o99qWIOUzuJs5aT1y7ET9aENOnCTmJCLV4Rd +nMP5J2kVzZvMImk5C74btOfPAAHZQqcg8kpuEgItq2XHhWqm8kSpAFK7iJ/T5aYkwjNP2Oq9YzDX +7SqNkxHTxJLoalcKawKFTXniwcGsukqJHnMmLa29JZZFVdidPi7/AE7sfFW71eTWkicSiZCIi0X9 +ipCguJDySOgnW7TzrgIgN5InfsYQmBcOUBXcoNJ5IgXG7QySKpa3euJKQvnzhBym5zEMfH7ggus2 +SeGloOm2ppqZDy8WAcemom3XjVNgnjkRJditCGkqzCpFqaZUKtCHl3a4NOPZ6IthjVoQniZdbwQc +IJK6d4LDbWndgHeMkkDVc8HuLLq9LUoEiSc6fxk6iOtC/PFhb8PxYkojHDtDcuqpFKpXF/NSnI8V +ZwnEuXvu20cm1SH+6+0lkhJXvGqNahp+uuJDJ0QsiUiXi/FOfdgqUr1JFT/jCYUr+umAxDtozEjJ +dpEnIJv0EGrRrwy4E2NURSIdFNsa06NB6q/NjK2TrMk+9VHem3LwldlquTmwCWt5VrrW2nuNylLc +aD2sSikhnV9BSG+yJVUbVeJCwnVtEwUqpTuDp0trTnjNcryTFtmiRTlDAWLv7ETp2ZEbYiL77p77 +aYATGL1bTSbdoxRXBYT3UB0I7CLQx3S7q0s8WJSojHZ0S99FIikKayRJt6rp6UGhpjSg0rW6vdcO +JKrVEnHtBi/aqMRdWr3e6bX07+impbZ0AefrgZm4kyV4hoZizVMVG6QtiSFIi6zS1r5BrywHm6hI +OF10DkXqKqRCBCj0Kj3q9/OlKV54GR6zy+1s5feLcEEzLxVIaUrWtPX1xYnyLcR4lRm63thJFwo5 +eD0u1OtRYaU52EFD8sV+0mL0hQcmg4Iegkwt7++mtfywGsnAySm++Js6fouUFU3gSqxIARgiPQI1 +57g/Cf642Psqdey4p+io8azDN2km2YvnVojIDZpwLsK/dq0+b4sc3qziLmPXkBB9LS3UJryPS3sF +uFKGNNdKKhp3fFTGrZKzV+7qpMc5Ngko2WYIO3XAWkC6VdKcYGuhbwVr5DgLRIMY9lmhq8LLAPYt +sRsF2r1baeR5kI6CN3eI1rqGBWcPbhOUsly0q19sTqqbB+3mg90SVOTVwmqPiIvPFuSnC9poCJx2 +bCbMFVHL7wjJxVSGlNSry309e/GaZ9nEcr8Y2Xee2m67UWiDCaO41WKhVNFVJYdbKpVPTATGUG6T +zQ1IY11khNgqJbROQOOKVQHw6a9AKjQufnhrMEx7GhUF20bw00/JWRy2/wAvgRilcVjhsaXy+fpi +hOMzoRPDskMxyUS8oahzKVnFIKukepuv1acjrQaVwGLNUtLO5HNrlm+bzSio+y3EZ0oIOP4nR+Mf +TAJza42XCEWo8ayLWLX3OPadLwt3romXrtlX9K4qzh06dqqvnyxvXSokQqEt1pFTuxMbuikm6SCs +Ugu8VdKkR+A1yW5ac+XIu6uGFkXDSNFpwaI75itvuQtVuDpMB58g19cARkBg+I/styuqzMUCQ40N +pVU6iV4606dKV88QpBRQjHccm5ERtDr3REaF4Rr8uLY3kvZKAIRbUBZlHJXk6bA5tCp13FKlSvRT +ny88V0eFbTcj7Jcg/R3VU25KI2gqlXuKnp9MBpX7N6hITGZE05hdg4SYC9bG2WEhVNEqHQSGvjHl +jeLJRGCmBWkv3cTlzTmI542MQbLq23KN6VpzDWzz/FjmHsceN4ftCh5CQeMWylyZMzco7qCtxUEk +ldK9PKv+rG/59mE4KTfISBuk0Y+fQdoZbegRpPEVAqFKNFadPVUi6a4A1O5kWzQ4gnbZFTKo5mYq +tnKoo8S1eOiDRMDMdR0qXLXxYEQgyWX5PIexsZRlnoqxM1J7KRtnJgNSpSpV+OpiPixAcC4HImZk +xZh+7sPmJJwOXn5iLpijS2qg6ULw693PDqrhQssZ1yc0bIv2MS+SlIyJkUSSdEjUqKKbR879K8qY +CBmuPfQ3ZnP5eeNnXtKDmE13oogarGZGp3cw/g9+Ck6pmb94HMIKzXKMHmyEJZIXv25qurbzAj51 +R0HFhZTyjTtA/d/Ksr7DFeCJ4ULNh7pVxXTp3C5Dp81LsDoThUHGSJIkTym8FVdF+T1EVY5yag21 +06rdCLuPXAAIx06j8ykxUikMoxecWqTYXzY0nMe5cIh7ylacxompSleunVgZnvslmJvMTV3lvY3H +48RIvCMV0A2PuxbnX1592Dcgj+6mZZFynMBDx6UiJTCorA8jBEuVABCuhBXnz0xVu2ftOfZOWLL+ +T34Nm7Yk1EnEc/FdmSRcxGg94FzU19MBTc0Ry6E4zRztFryjuj1N4s4aHurizMddqiQdA3V/33YI +TD6ed5Ris7DmAIKJeoFltmyaGRuSYpczI/P4fLniY3F1m+elY3LL+Oyr7QgN6VQTWE2zkadd4HWu +t3Pn8pYocetAxuWpbjo2RZZiUJsUOQnupII/xSqfdrXAWTPcejKGxzBknJgQuW5BgMcz9yO+5JMr +lnFNPBWlKeIsR0pDJLGZnmuQ8vTWYTfQ4NIp4sFSUFyQ6qr2fMOvl54EKzTeSaRkNJP5TM4x5KIx +0S0MgSED51PcpS4i9enGqZSedpTRxlYWmQzjW8ag5btn1gG5JqQ8ysu8XPxYCi5Fjch/vBIscySr +rMyyrBMWrBgiQC5kFBtoHT4qhr48VrMyT5F2gxcFGRq7JCjdVAUTcHQhIvvCpStKHpWlK08tMaOB +M8s/u9JZL9lu57LMiqnJuXrbbN46cFYnTbrWlS8XfTw4nNx7JYWUmG3bQo6d5xVkFF3SzQ67ZCYj +UdLa6cudP0wGuKqChNryintHeepE3bLrZeAmznpuHWzTQtKaYVKy0a0AicsIddRTbI2ZRSqTlnr0 +kQaF5fTEFWPdOXqUbBAaiLYSKTim2aritoNdCT1PlWpYfaFMCDWQcts4D0kjFPhnkVdoq/w1dD0L +n5lgGnsxEoSrWEYv4FZqVpFKbzpK4qcqAp161/zxKCeatlXke0Wjk49ICJdqnNmW6NedNute7XHy +MTzZDuV2BLZi42UAVXDNyi3XQVqPItsufV8VuPswM9Ht9tocw/jUVRJUnOXgI2fT4k+nw09MBLSz +ADY2uYCRevUySFNur7bAjZiWtLVKWeHWmFqyE4g7SiSZzacw7SJwaCazdVBcPirrZyLTuwFcOhey +ass59iEi2ajtG9y2qCDnUtLS6PFr6YFm4btAdSC+W8tMnDkdvg1GbgVxO7TcR1HX/LAXBxCyD7g4 +ti2zEUS2IiXFQG97Y6lr0UIK6j64nyrgSMpbhnwuCLgmrrg0bS/CtSo8qcsZzIJsWyQxZOYTe3QT +VmE+ItEhLWgLcvp59+CMgMWM3aoGT2BIKpkK5IuCZvLh8VSoNl1f88BVO2XsXzBMoFmiFZqLTF32 +xi/NIAIBLWlUraUvprz54wfMEwjJH7Pkoq2aL7O5VTtARVDpT0HT079fFjrmKeRqkZIoKMMrSQq3 +DwDdsqqrf57dbeY6YfZQMfKSaCacCxU4JL3qrSBEDEq+FMtwacsBwsKaiZrjvdKfjHwj4vT9MWmM +i56ZS4ZSBkZLc8KiIGZIFUtaFprbppjrbL/ZjklebSevssQ6klvq8UKiwqhcXKmqSetpaeXdjbMu +Q8XCMhj4Zgg0appDdw1oiRUHTupgPzwkMn5mUh0F2mT3TZ0kuqJqkiRXCXLnUq1w7lTshzxmRVqK +cDIpt1PdkuQFakXrrXH6ImLMVeqwiIbSut8PrzwFdvBbKkV/DJ/GIgj1D5FgPz87QMt5iydmBLLu +YpUEiFcXAfZvddIUDd5/y24C5XfQLFwTl88kRkGzW6OVQ6gQdAXKlde9M8dS/tYKZTd5KJOWcoFM +L/3EhBIlyOncPRztr544/epuG32ZREB96XWPVu6eh/FpgLFIZ0mJaH9kqbDZvvquQBqsSQiR2XjX +8FbfDiLOzDPi0PZvFLs21nDIPwuFKlQpen81vlTnit/BcOJyRLPnou11kFFE+ohWP4RHkPVy+mmA +eScLGyXUbImInci/dKdfiOlac64fVlH14rtlgYNytUBJsZWoaDZXQfIipiKrIJ7Qi0RNp0+8AViI +FSEumtRr6YigoRKkSl5EoJdXxXYAjKyTqQcM0yWNZuxSFBsSiIgQpUK6mttKeuHQFaQcWoXuXjlV +NMWwgV6pl3cyrXTA5kLh2rsJ76rhQSTSAQvIit8OLXlqHFSMeO5JHaTZLtnLqTJYd1BIh+6Ee+4v +pgBSRNWaRfaXrZbhyFdLZ92q4A6e50+WnmXdh+6UzA3eSSbACFV0ncCIW9Zcuke7EVuIyQLlGooE +4SarkrvWiWzQx0PWvTfzwYbtYeNy+845hNoShPGiiSpBYLZvaNaqV0+f4cAFJ83hHsY+i0VxfNhL +juJREhJW7mOleVeWOqnqjfMTeTbNv7UazWVknqCEi5sEVUb/AO6lXuqNfixyS4TubpOxeAv94RId +RbAVLTXn82N67GnEpmjskKClIr2s3glSdteJbFdwVeSopqVppWlO/TAFuzpixksuvGwt32cBzJl0 +Xj5LwPkHCI2VoB/xPB4a9+CeXJaNzJnDJ6c68DNyhZbUbjwVyD9odCGhjUKF1HSnljzSNFePkSyL +MLuYPLLpB7HKkCoyKDUwAlE076aKDXXSuuD8bkWPXjJqZerLryEbKJSscbREGb4gr4xPXTTv5lTA +AuzprOSErCy04f72Novi4VzDPURSfM0SrcJqXa1ryHywWzbnDsvjcqHE+23T+BQfisvlh3aJilQu +Yp6jdaJfDriozXadNPms6xhMsLy0jv8AEnJulhB4KNO/bpTn+WmMwtZ5kz3HfY805jTlECHbfvwS +dEdvIQOlfDb83iwE/tDksvvpqRyyhnBCRy6RcawkWsbU1xVLvTUoNaXUEddSrgBk94WW2SuYIRtx +Lxirw66i8UK7HaLXqLXwF3Yiw7GacxMq+bMFxdQlopq8eKSrEKd47d1x0t86YPum76GyvGJMniDR +bMKW88eJzBKtnwU1rtLBr7uuAhw8HMSVkWTORaSUy+J22YiwERVRMfeKJqU500+X0xbofsb7VM1p +IZfs4CFZJFsG/tAUgvpqNdOZFrjeuwfIMDl/KrHOM62XXkE0AdM1yuV2hMeQI0r+HlSmLa4zMzUA +UYmEdPZK3ebwSYWigZeFRwfhDXvrrgAHZ12R9n/ZvDoZqYm6UkBSFMnjm0jVKvIhANOm4saek3ap +gMo5A2zzhR+MbmKVvhDlypjOWWallXq6Aj+9WZWyopvHg+7i4oq940KvLpp3+uCj3MzqfdIM8uma +yKCoDIyyaIiTmtC+5R9da95YCH2gdmMHnhJdyoi1bSnGEXEpo3CIiWtDPTQqVqGONu1ns+Xgc8vm +yKEo9ZrVos0Xq1JSppV5Urrr60L9NMfoI3XEW679K9Nsklc+e7Ni65o/h+LliUlFNn1KvFxRMV9F +ERJtzBOtOQ9351/XAc4KvGKEUggo/hFuLfbiqS0Csg6Yifg8OvL6YNRieX3b5JoLbJNW6DYhVETc +bTvSvK2tR92VOeuJTdOYFIZltmSY2XLoRjnnEt3JFdztPVPX9a4RBNZxoy9ksUZsX3FEtJoOY1uY +9WupI9HdXANmOWXbRV65hMrOWaH3V0qsKrYu4j8OttPpgg3j30Tlwtixyo7K0XyeYbiEO8KKVqGv +08OJUhHvHMm2T490KLZIrn37tpCCRDzqKnT50xOVUkrFXYo2qCltpAnAgKDz0KuvxeWArIfvI0ky +hkwP2SkhvOmaOYQVVV1+JO4KD/XA5xIE+SdTMstnMYOPIeFdFJN+JbFQdK0qFNbqa4LN2cgLImy4 +bjp+JEbN3l4RXQC6nhIK0rbTHnELJC4axK4MXe0qKhOW0CYqCdC1DcGp6FSlMBHis1QLFw+kpLMM +3x0jtJtSIESQc/L0UrprX64ek3m1lr2WutmVRR2vuJMHINSP/wC2d+mnpiTKotUHCssu24Zumqm2 +SQTYNwbKn4daiYkQ1wYgOzdZ26Xkm0ko33xK1s7Z3igRd9lfl/LAAIwVE3DVsL/MUkiySIiSUNFq +5E7uQ7lCrfph2PGWeZt4ko18om/Q2Wy8m8V3UtPhrYnbbjSIrJsShFey5BsDsU7bjURtK6n4/FSn +pixshUTZC3HpTQLpAjIitwETLUe4Yx+w5Bim6ErjVZdN3prrTXBVuoV/u/eD8REf/wCMeNNbiPdg +FvxdHw4C59zdA5Jy/wC1Jl+1ZCRbaQqHbcfpywBN3am3VUc2J9f8Y7REfm/7Ywft77dMt5UinkFl +8wfzSokmBtDEhQP8Rf8AbGNdrfbpmbPCLlWCWOJjQZi2ftVFh96VVR0IK1pz1+mMphGbds3XknzA +1xds1eBBPq97Qg6i86efVgDUw3fOeGls1LBLPljbEKDsyBVdIiurRLXw059RYgoFFCs8vP2SzVB8 +idzbfa601NNFLnrdrTv9cWY0XE2ynXb6eZSyiUSzJAVkS3VxHuRR8wGnd+LA9pl+YUzR7ETZrxb5 +R4V4uzE2LNJYCpXWtaeLngIGc4FjHhwblgEXLHwzlO1YlEEm5BXWqqlKeKpUp5YoppkPUQeLqEv8 +WmNVPJok0SUdrOmzM4xTbXJapjLum6vhEfIba4gTuVXQtBdiwXF4kgD8WyZiSAtFOZWU8Vw93fgK +GbNZOMGQJseyuqSaCvTaWneNfQqYYBMt3bs6h+H5S+uDRpyGW3aqCgfZVCFbach0rh5KDTEOQ4N2 +quszPgk9q4m6nivqXhCtO/ASsqPvZc2lLKIggmJKJgQhdadvlz5fzYdm5BZ8bVGbBASFq2TA23SC +QXU6lPmU078QFfajaQJfxOEEvEICQiNAp5d3Lzw1vfZ9hC9sQkmRJEF9x0Hxa+VPpgLVGuIeHNmU +lCBmGPTSckyTTebZKGVvvlNBrWg0qPIcCfbkwpHrrqTy6lzxJbaU6iVOgjShV5fBgOCniU+7U6iE +hDqIq/DyxaOyrIcpnrMCUXG/ZkSuFd4QXiloPdpy54B3svyXIZ6zGQqAunHkW4+eJtrtrq1rp9cd +rNI2DhMvw4sWz1BNgQpgkiz2isqNtSPnXWldcTsg5Dj8r5cSho+NBNNMbVSEFR3T8y8Xng29TTE/ +cHcNvSF5kQjXl1dXdgMj7bZLL+VJOMkk4tkLiLFRsmXH7S6iRDckfSNaaUKuMbb5+zJn2Pmoly2a +yxKx1zM1HgoOUjEqU8ddKH/LjSO3rs0eyT2MkYiHZKVaIW+8BUkypdeSapVPwVuxhKrMouP2pRaO +FGJV4lJiszK1cy5ElRelaVLTXlgI2SVGse6Z5km4prMRLBXhnzEnmw6VMrhpTlXqHXDsJDqzaM62 +iYqIYrR6nugUf7D5OlSqVoqV5KaeHy6cSwJvCNEkF4Fk/LNDW09yNVA4y4unZO7qLl54Hyb7jp2H +uABbwtqLp8jFWroJUOoe+GuoqcvpgLHOw8lG5fytmhRbKzlvbsk+bPLzVEipW50jTqP5K4v/AGP9 +n/HTbXPso5jkItgqSjX2UwMhkyPlUNs/CI6emKbkLIcfnTOEwSh7EHHvE1imGzYWYiZF/wAM6F5d +w46Imm5Rskll85J6yjbLQbNgEpGXoI/wRGlNoK69ReeAUWYM0T+bXgxsWCbOFMUWJJrDwLEqhS41 +y7jUp5CPSPrhcwxfPY8WSaMiMKo6tLhFhSfTzmveVS+BCmGcsCDU0GkkzNZ6venTLrJaxlGpVKtb +3KnmVvfdj7K5iTkJ1WSQcGhGtEuCKWEOi6v/AIdgn/Er6n5YATIZZnHMOllltEx1rZUVCjG7kgjG +ZXV63a3iWOnyYvWS4OLy62XzNMZgB8T8dk3BaoJICHKiLVH6lit8Uqmm1QUy86KTL3kTlNNb3hF5 +Onp/154cjmLpzmBjISrxjPvodUl5KRdnawjTr/CQTpoJGNPPAXqBRmpucOdlHBobqRDEw5XACaWv +NZTlzMv6YsbOOQvcGUu/MjWqRUFwIiFdKUtpT05YqXtQZlkC3tVeJy+ouHDPCP7U+K7XopXwp17u +7FoZLSJUV32LFjopWgJ1UEiqOlNCKvrXAY81hyeSrqUdsItYRLbBq4gSFquN3QVwjh2dFq0ZE5Jm +xJ4oW2ZDGuAXZiXnrQLraaU0xbmmT5BtFJM1odld1EXCPzQAi8iIKiWnP64aOBzQ2VSubSKjdsO5 +sJzAkV1O4R1T6+/zwFKjLVEmKe9DruFCIVVUXjhAlypypuh82nzYOyDxi2aIRbT92kEdpNZRs5kj +FIhoVKdOnT3+uDUfH5gUk0lXLbMSYgO8ZXo+KvwkPnh1JrKIKkmUVNrtV7uhZsir3/W6lmArLJrF +rSZPiZwim1btF7buXEq/8Ot3hwVBu3eq8W7YRyjpPcTQBY1lXP16qUrdyxKDLc4+ZcIoEi0R/FtJ +GI/Qqa4Iw+VyX2l5s3xOGCv2beciqRB566UHvwEDK+V49y3SUUbAmIqkQkLDaMht0tKhUofd54vk +ZFsY1kLZszaotx8ACFo3fNhYEJOBuMC2/hs8P9cSXCyKDcl3KwIopjcRKdIiP64BsGqI22+7EfDb +04CZwzhlfKrdJXMkq1ZCors3KdQ3/Wvl+uMP7bf2gCi3CENkB+1XcLlauvZeaWpWXJ07j7vXHOeb +fbE7Ie0p2YMnkhxKxGpcW6QiVBCiX8MqeHAbr2n/ALSBOUfZ+Q2N9FW5bij3pMeu2m2P8TX6YwqT +zVOPpvjp+SueJCuoJubrBKgWW0T8QFyx5WLko8yTFyDSUTdNNpmXU8G0CrQkuWnnzw1KyCbF2l7Q +bNXMhIEqLxUjuVLUq6mY91K4CrsmrVeSSduXMdw5Ok0yS6hEurxfgH1wfgWbVs8QbvtlNFzxjUhY +LbpuR6SEbKfw/rg7GZJzZmtJ5+6jNr7PX94e5YBkKfeI869NK4fylBvhmGcpEgBKKLk0YbgAIKnQ +feCfV0/T5sAdaTUe5jF49oaDuWd5fSZXqAIixMfgS+ZSvzYGLTzt8/22MPwUcLhBw3YqBfVy+Qro +e8VfWl3PBplkFu2cbjQAbJyjwmvtXZtOKc+SNKXcta4s2X+zNGQSJzKNuGar7kU6atnJBwL2pa0X +rqOtplTAQ4x9JL5rGZTZxyEwqkT+FZtFr2ot6jY5Tsryv88QTZtWjRr7lrJCwLi4VsSI3yqJnWpp +qUDyArun0xO/cPgpj7cC6cpvpsnD5Y7eDdpiRUUpp8JjbTXEbKgxvtNrLQz/AG5Rd4rtIEHTGSNC +qNe+v3a1KcqeHAVHtgyaOXYJi7UeLu30gkLtjt3ihwneTbq77O//AA4yllBvnaW8j8TVR2l0F1BQ +qUqNMbj2zuCmYVrleJWUXYOTN2wF17so94A+/QEvTv6cUd7CxKcYl7LeLprPYziYzbPoScCWi6P+ +L0wFGjykHYKimG41ZDvKpdNgjbpdZXl5YgKrEoBXABLKdRFf4vp6UwYZOIlSFdIqImMom6FZILOg +kf4iZUwHIRILrwtUIrh+L9aeWA1LsJ7I1s9SyTmQ308vp/emmsKSpfldWmOx4Ls/g8utEm0TDhti +kmIri2b320HlWpeeAH7N7MlOw+AklLxJVAi6Tuu0LTutxp+yRJCShmoXhH4em3lgBMeooQKioYEp +4blASErfXDCTVuglamG2REVpfZ+r/diUyRZiqruHt7YkJeL/ANuFAioNxJublPh6y8/8GAGO2qKa +SqCh9KgkJCoCJCQenf3YBPct5ZdtxbO42LdtytIgUbN7breVdK4siorJ3Fx7obRtG4yL/wDp4fbv +FkwFteBCI9RqGV3h/wDp4DDO1XsxlJSHVGGzCC8agO41jHLkEmzMR8k6J186+WMuyZ2eyUpMR0yM +r7YRkFfZ0mLB+YvGPwkoY16yHHWM7KLR7QXMe2XfqFamkzZHcapV/nClMZTKs5RDMq8s7W/dxw7X +QEkhfibldKpaWLogGlOX4sBcm7f3oxOXZVDjGyWzNZtdoiRFp3UD4VFfrTw4nTEe3jWDh+pmF1E+ +0rWwSNm/KPq/8NOlfAPpbgTCzAk+blMIoZjzImkS0XARCJA3Zh3UJQq0pS6vzFiaxlHrycSJq2az +mbDoXGPCPWOgk6d9KFWmhFT6YAItDrNpZNqUPxpF/cctIreI7uTqQV7v01w7QZAsxKsvaUc9lIdC +59IpgKUZlwa+KidK+NbT/wDOLTl5ipJUk0YSXdA1clbJz3/iXytOVjf0GndriBG5Ti0zGCXhduBa +WrNYfeI3L5xXvXdf/nAU9FwLZBzKISL5CHfq7DP45bMqtfkrXrTRxNeJOXqjPKqjBjIzTfbXVjiO +nszLqPfQ3H/FV+nfi0N2b5jJlEi/QTneFNxKTBJ0MIxvTkCSNO4a6d2IMSxGG9mNIlguUK5dXJC4 +6nU47IruIWLv2x1uwEl0+i4J6rmTMDx09sVSQZyaiP36tf8Aw7VtT17rtMfMy9oDJtKmMtnqDyg5 +KlD9mORE1gGvcSnylXzp5aYmzb5rmJ4q1i6pruI8th5NlaLWP1p7wUa15bn1piBl/KkO+jRdQOSc +vSTAyLbfTRVq5d1pXSqta176VrrpX6YDXwEbLS+XxEeEq2l1D1KeEcO3dFw//wA2Gj3PFYHi/FgE +AJCqQkd13gIrbhH/ACwkFFB3RK8tzq8Y4dVJbduTC78N/ThwfD1B4huL+bANgO4A3I2/zW4UaY+G +z+Yit/7Y9cVnTjztwLRkq5UPpSSJQrvoOAizchGwUUvLyCiCCKAXEfTdbT4afnjj/tY7UMydoO+c +a/OHy+oquixFYOle0R1oWnxVu8OC3aln5x2oybZoPHQ+X2zxsnvj1GkqZa0qQ08X0xT27OUQyuxi +WiKAtXa7695s3hICJB0080y5d+AqPCx8M9SaLsF01kFWZElfc6Z6JAdTTpT4da64am5xRtGcCpwI +iuKqgPy6l1xu1tU+QvTBLMqkXDZoEXMa+iU1CbOQVW63iWgD90pX4fTngc4h2Mw9JVOeais53CJy +7uA0FS50FXTzL8WAcAph89Zk0Dgk3rpJxGLufeurwHnRNWml3f4cWmPa5dTzw6cyFijctwvaKkad +yUhXuQWCtbaUr+WJmR41wnl9BWSilFY1svsNUhf+8inVPCuA1r93XXvxdY/Lco7jJYXz90TpchRz +OzIAO5Kpe6dpV/64Co5HiWbmMllF544laQdJovB2SD2eY/dq8u5Mq4ubjLcWo0XdqMGrIk9uOlTT +/wDDOqc0nQ0p30LzLFyy4izjY9djIPPanBCKD5WwbnzQ/u1/xKD54aOLdNJsmy7YHLdJAWD4iMbX +LI+aS9KfgrgIcOzbpwjxSZRNgL0uAndsyIUFqfcvE9ddNaW1rXFgaLSTmPMpY2q6g2x06SICJX0/ +u7sdfTENoxR4dWPmQXUKNH2TLGJ3AqxMdUXP6a484fC0VSaO1gtu9kzu4HXtGOjdz+etvPAQ8xrM +ZBwqpOgumJF7JlgLq2lbfszmmnrz54ocnDzCfHbiLGJREW0C/LhupdWhe6c0L4aW1HqwYm5R02zG ++bSxtSWSIYN+YnYJBUa1QdemtO7ATOE5INGW5KOd/wCx+xZYSO4L6fcrjX0+uAxjPDhOUzGSD43X +tbqTctm11qr6hWgVfS7D+bszNV4J+LbLy8W1XVTUY/CTOQSGgrf6vp+uEvk55zJ8FLP2UWm9dcI6 +kekPfD1pl6iNbcCc4IqJxQu5A13pOVRJR4oY3cSF1DEtPmGod+ATldrLIO2slHooPbhVeiFlxKkA ++8E9fyuwCklouxsMWC+4Ilvmp/F1LlXT1xZmmXXybRBzEzCDYlIdV6QLdKgiJFRZL866d2KbduJJ +IJtrrbi6QuIvzwHf37NXDl2KZbubOl0xQIkzTAwHxV/F/wAsaM4TFc/44ppj4CA7i05992M1/ZtE +WnYrllMg3CTQ8Sm0J21IuWla3UxfnCJO3Y7DY7R6bx2THn69WAUrxgpWkBlb4rWxl8XL4v8APEF2 +T5NkXCNjXIS6REFfF9OrE5JqimoTUQREfAoRbOhV8vPHm7cW1qCbbcT8IkIIj1V9OrAAOOmuHLiU +V7i6rU0Vv/fgnxT7h0hFgvcVolcBj3fWpYHvWMb1DwwJl1e9JFv0+XLUsUrtdz84yhkfcI0BcKCK +LYljbkQiRcyENdTpQeelMAMzXOReZMxlH/urMOU0FdkZVOSVj0t2pa7Y9XUVbcXlkms5fICKOWoA +nKRNyal9seKlbpT3tPTGLdh80nmIlynQi5SS4pQhKefkG6rSz3gt+9Pp+mN/yoo1afZoRyyeorqq +LPHTQBJBmVB16Cry78AJyu1a5SW9jqNuARcpWqmosSr+QK3S0R8VBp64hybOSXSVjVofeRXG5llh +qtRsCAUr986Wpzur8uBaVUWs8+kMpyXFunStkjmd/XiFbA6qos0ud3d4R6cSHD54WWnSctFO2jV8 +vti2Jzuy0qVfD3fdAX+2mAJtX0+vGkjl32VIyLRUkwep+4jIilRoJCH/ABa0wHZS0fCZYeKx8w+Q +Ypq2yeaX4Fvyat2m22pXx615Ut6Rw5KQryPjmLaWigfrIICMblSIMkmaF3eTlSug10/F+mBqKcxO +5iZu2z+HIou4eJRbVKOgQoNfubh0WXr9e7ASlt5y4atHeXl4mDVtcDE8SRScu6/h73O4E6aa1uw5 +MOHjl5JySz+2XaIbLldH+4wLb+IIF3EqQ4XBQ8e2buZBT2w5GSVJFWWX6X0qsXcknp90l04OSbXL +cW1jovMjsItND7WhCMlqlugFNffW+Pn54CgS66KjWIqpFGqyUSJPLGUbDSVXr/8AOOufSHxVu8se +aZeknqW+5j5rOrita0VkWL7hWol5ooAI6bYd1K+ddcWdim4nMyqi5jVGj2SSFebfkdvBtO5JkHnQ +i9MWFyhKt1OFr2iMcqgjTbTjWzZJWiYU8NSIud9ad+A0S7o6f92PGJCHT4vhwsOrqLHumy7ANXKX +9QBh0LvF1/y4909JeL8OIsrJRsM04uUfoNG9wp7qx2jfXuHAPqrN2jddd2tsopBcqqXTaP1xyd28 +dpUhmuaKPgni/sVk6UT2mi1irnQBPeEudCHn4cQO3DtWzFnGUk8ops30XBpAuK7ZoFzxWqdn31PJ +Hq15YpeT/wB3Y2PZsXKzV6m0dXLoIn7q1RELTFbv5VwEuM9sJpKyCCyHGNl4/hpEfumwW8hcBX4K +V7ywluxnotk6Xj3JucxNl11pMr72pN1NOpGmJ7eDnJ1psSDZAnTlqm2iXLBawSNDuSXoWtD105XY +tsJAymWYLLPDHvsydKEPEgIL8R8bRTX+GXlgK2yiZJ20jkHkU6kpJsKai43iYLtO8FU/xjixR+X1 +pR2qWyu/WU95MJEzFMXKNC+9T/8ANDzpg7lSBFtIIPmKK8a1dvCWauiMrox3TxM1PLbKvKmLarvL +yCD4nPsclXRJpJEdvBvh7hrp/CUwAVWDZsTXknwb7xghbZ8ErH106qUp8YYkg6WJug7iVkFFmCW4 +1VLxScZX7xOvzEHP8sRjavH1qaiJoOifEo1uMvsb2njQ5fw1R7sCXrV05kEJCEZrobW64Ztr+lm4 +Aq8S109DpgDsS4KNkEnLFmo7YtPtbE7x+0x6w+8RL+StcO5iUaxMggu7M3Ps9Ir/AISXjl//AO2W +AWYpD2EkzGL3yaikMtGAPgVRPku3H8NNcGDnE/Zi6ikbxIxoj0F1krHr6a86/KWAlQ7cRaf2g5BR +u0ujpW7+OxU5orV/LXA7OdwskmLta0Ykhjp0B61V2Rfdq0L6d+HTT9lhdIOTKNQSJg8EQuEminNB +YvPp1wEe5mFNk1kE4037xJJSDnUr7Ssr0pql6/8AtwFIzatGzOVxfKIrk+hyKOmBUMhNVqoXuF66 +U6qU08WKlmJ019scDNyQRaIoJxMml4iVtG5F1X0pzxa3ZFDO9vMD9BAd32LLeEt1oQ3pkNKd9Brj +KMyyCJSAtBhEHakfuMF3ThYrXIGdaIKV9NBt0wDEISMlNi0bLHLOH6oo77joFBa+m2pTv1qWJXaL +Le1M0JFGw4JyAkIrtkw6Rdj0qDSndXURphrL7N5JSr4ZQ+CZiPBKumwWg2dAOqJV09a0w12XrOCz +AkxXRBckiVepHeQqbolS8qV118NC/PAEcwQaL4J9SEeAm3YIJvT3ukiKoU32wV9Br5UxRW7wmwKq +JommRNdki3v0qXd3YueblmMpDz60XvoRaboHMWneJFtLlot+XPXFAMiFIrunpK7owH6F9ggvG3Y/ +lklzBRRRn4yW+G6v4MXlkRNktxy52/is3iMbv9FMVbs0ZjE9lWW2xAuntsExIRMrefPnp+eLYkQp +pCK5r2l1ERXXf564BSq3UShIoEPwnvEJFr/gww7cLJnsCFpEXwrFcI/6MRXr5HdVJB509IhcZdJe +fnhbclF26qm8Cl3hK9W39a0LuwENInBXFfcJDcFrkrrfyqljC/2k5hqpJsYld41TcJtTUFV6iSrY +RqFeWlBoVFKeWN7abiCVxH74riG01vDXy8WOeP2hXzdt2ipLprXOkWe4OyiRkzGvKp1E61osPPQw +8sBmGTM0I5fVffukaDZm5QEXLmTbCb4deSotT1tP6a2+LG65dTcZhyoxi4394l0XbUU2sY90bhaJ +dSq6g08Gn+rHO7Ru1ThxdxcJxqLR0oL6Vd//AA/r+Krf+F5Y6c7AlE22V44bZV28XIRbILPN3fSp +zKz0RGv9MBdDkIlhJpREAim9ogCiiT0QAWcMNB0rqp5lX6YGKz2V4dAW8XIKXL+/eTJBU1V/LRHl +1FX6eHE+byOnKNI6LFxttWC6qx8MG0xsv02lBHxFQcGsj5TiY9v7SFb2kS9ynHOQEdoe6gop06QH +8sAHcJvJ8UOJbLtotIRIIdM/tj7XkNXBfAniZDwck9Fq+mWbVJREi4CHa/3ZidvKq1afeH+mLO4T +Yxrfi0TQZboihxxdRkVS0Eda864oWcpKU4h4CbNeFy+SpdTc7X004qPgQD4KfiwBGWnFI9RVPLyL +aazOqkImO9QWbOo69VS0rQKU154pq3s32euUlnY2CJXDIyzQPtMg5IetJvqNS2h+lMQMvwLpiqMM +5hDkZd2qLn9245ztM4xLnYTtenUodfOhV6sX7LUGaL5WZfBHyuYU1RaJJNAHh4YflH09a4DNQWYx +7eJkI2NmxFddRbL8E5O59Juy8Ltf0SHy1xbzn5ZgXCQ/Zw6zTt/3ySMtdxzXmoNK6c6DWumv0xKk +6lAZxUXYM/3gnHSFs2/UPrQC3QG6FPgIq/DTy6sD3uXidEkpMzWZYtxt0oLGDoVW7UNa1onWuvUf +PWtfrTAbKAiPhDHjtEPBh0cJ54BCRDZ8v+3HIn7X+eJJ9m9vHwj/AG42NQVRXVbnRcCcW3+AdbCH +TlWuOoe0CY/d/IsxLCdqiDVTbH5jqPTSlPPH55cGIq8c7cuiuJDiVWx2GruJFeFKV1offpdgFhNL +RJiS5rpPFCUHaRW+03LAHXv/ABjy501xfsiZX4FoSko23EyY2yO2Hul0ajaRp/8AmI6eKmKj2Pps +SzR7ScogvHtEhTcoKASpIJEZUorr5EJbeNGOQcZZnXjknJi4UdEmLYgtSSdW6VoeuugLBbX/ABYC +5wWWYdQxfSD/AH3iAppya49IkNf7q/T07q911aYuQRayce+kJ1mu9UVIUZZIfB/5b5Gnd8uumK63 +mm8f7H9nw7XhVWpC1QUP79uX94Yl+MPEP8mLFFPkRkGrSPeLuSZNVHUYfVZIR9fvGta1+NOtMAAz +Q4UbOHSEzKmmmQi2mCT6Rsr/AHd8nr6fHiBmCQUIFXMle9UbCnHTZtguEh/gPaUph3tAkEyaFLII +oO24sxJIVg/vkeZe9RrT50ueA/taNy+qzj2J7jNNgLeTVL7pePXKgp6V56qBd+mAsTTNBNGSskoz +MXSiqbKRJQ7iSMeaLmn83dXFPk86EvmNedbb7Rwp0qpdIglIJ+L/AAqDhjNCgwj1JjJb66jYSaPC +E/vWp9SC3Lx26YxrNeYiJukuu/4lwJbLwU/wF7lan5jb1f0wGmZ1zNIcQKmX2G3FtBJ+ld1ESS3Q +sKf0pf8Ay4Hg6zBlJwqNhvWLBnauPzNHPhLX4rS8sCXc08Uyo6fQiJiMIqRBudRKslx0P/TWuGof +MWZlAdLyFijVkw9mOTI7t1FYtUlO74dPF/TAbPl8XBZMZtJkAXcdMLLFvWmLVQbm6nPnpS7vxVJV +ZZSPVUlpLbRSX9iyzMeki2x90ty/FQerzxKyEsop7MY5k4UhmGqkK+citdaYf3danp5YVmt0oSTU +iZsWjp+gpCzB32kg6H7tauvzad+AyTNuaFk5OMF2w2pJkko0kFVgtuKv3alafNQfXFRVcM5J2zUk +FnTl5ukm5JG7qSHkJU/Fj02soU2q5mX/ALSkt9Ru5SHp8OggV2PQTV8nxiHBgSyoi0uWO0kDIuRU +wBuCJOLyEqTZzxac6KjV01K33TgCuSP+mFdnMaQpSc2Jmo6gkk3CDZQ7CVC73g8++nP8sO9ocgmh +FRgkHDSRCSb8bB6XCHSBUp5aj/niz5S3I3syatpBFihIEqMixfEdxLslCqCwV9a0rpgKVnNq1aC6 +9pGbSUcr8RtJ2kgk1UAVUxoNO4+vFbj2sgvbGsWy6ijshEEk/ER+WlMWHtAGFjZD932hnJCwFQUn +wncTkSG5On+DXEFks8dyApxZoNk+P4hql1AQqiOtREtPDgP0PyOm+QyVGIPjuecGnviRq/CNKacx +xOcKLIbRCB/CJW7vn+mIMOs+KHY76IIqEgmoqJIjddZTWv3mH5AR4clLLiHwkmHi/wB+A89WWFUh +URX2x+IuI8X6Dhpo6dIOCHZ8NtvvlhHn60IaYSyTRdgP2YBISttUAh/xfe1xM4URMkyR2PiExDw/ +n11wDr1ZYUkiEF1BIrSFO8/r6Yxz9p3LLybyuxzEga6AxKvEFts1SXFKheVbf89cbKkis23U11kC +HpISJEh/y6sCcxwrObhHkavYui5SK3oIhIajz/iUwHJLtwhOspydJJbMq9jZyMwij9lZgNw2vGvn +Wmvi0xrv7IScehDzDSNmGr94Dol1XDICJszRr1WDfSnfjnzMrdbLshLQSDORJG3h0AFyKBiND199 +WmoKUGvdSunjxrH7Ms9MFnV004N6uioSRSL5EAbEkNB5I1R+Lq+KneOA6rboorAguPS13dxqCNwd +487vXHgWcJ1QTd7ZSRCptCjdtW0+avliSipaqQie6oRCQoeHYCuAc81nFEiicuufZokW44k1g3bQ +r3inTl1f8sBHzFmJvDAlx6KktMKpgSUSyDdLdpzoX4efxFgFlaJzNNSCWZMyOWSEskqSfAo+8CMb +lSvRSvmtXzLFnh4FrHtVUYu8U10ivkSW+0qHd3c6d2IuYJRuhHruXL4IWJtEvaN9qqytK06aBWnn +TAMQuVYeGZPo6CNRkzXIl3stxNy+7QqV0I689MB5iaWcpKqZbftcvQZFacmLa9d8t3e5D4/5sU9X +NTrPk8xi2gcBCgqSoQSlyDiQAf4y5/w0fw18WA/aHna12upk1YJaatJgxVbo3JCr/wAFmn4en41M +A7mDNEbEGuXttCAcJiQpM1LyXvU6auXNaU5KWVKwPXuwmAkO015FIuGGf4TIUeVPscZIMAJ0ol5L +q386Edda6emmEdmXZqUb/a2dFo6Smo10Lt+g7P3DMqjeSypV++UpTu+EcFs3Z8yCi8bKTEPGzyq7 +aiyT1+6FJVRKpFQenypyrp9MBvvx+PCvFj4Ajf3Y+15a6eXdgOef21c1DE5XjMvqMzXbyhksRitt +EFnPpr82OSMzPh3UOEeLuUWQ8MAEY2pWleOnzjpTHQ37Y7mQLPgtmpN1ExiVK2OQuFIfnT5clPrj +niPTkpmBbMGz6gIryQAILDr11Gtda179K178BrHZkKcRldisq5ZOU3KSj10aNu4qxUKgLJ10+JMq +AXLCpVmom7k4l3MOnLe1NGVc2CuRMq9TR0H8o1pTEuDi3ySqpRps29VGwv0E6JWp0EfdronSneJ0 +pStC76YrmYiFmo4BZRfaibQUogdRq6jVOfDnX5g1roXPXzwB/J8g8sdRcoZthTVFTitkruKoXuXQ +6+GivhP+fFuj5SWKCeC2cgyJdUlIwyusZvh+8b//AE1B54xxw+UXhXHBSkkiq1dIxyaihUKtY5au +qadfUxrSnfyp5YvPZ1OSUoJxrrbMFqVaODLqqLlKlyTkOXKunIqf1wBSVzA8kG8SMbGmLfj+JEiO +4Gbug/aGqn/ln8OKHnucYje0y2AezySUWSbEdxIJGNaKtS1+IddaY88zlKZlaydFUko9OQUFNxwZ +VCou0g04gPSpUpTWnLXz1xWmsWwWjk0UKrIqOo83Jr/GDgB5lSmulRLzp/TAWOQJ1Fx4zruVCWeQ +opCaSJiO+xUGtKcq/LjPOIasZtBRyw9z7wjKwT3UTLpLTw8sG7G45faS3CpaRZJgonSmm+gdNSCt +fLn5c6Yqjt0g+lG7ZFExZm6Mm4qncVAIuQl66YDR27daJj3S7ZY3rFgW2qF5CRMl/ARUr36VrTAV +V1IIN3jZewk0Ek2D74QVSMtUVOXkOuLhInXJCLriCJ8ghRSMdDTkSqRjWo6etaVpTWtef1xnjxN8 +3kF2ajlMkkwSarjZdQklK6hSmvnSvr+mAs2Qmb5tJycfKSX2hRUY4x6fcEI02luXw/XF57VWbeWj +Gs2+bLuXDlLgpNsn4kpABtTUqP1xnmX8ssOMRZLLOarlMnErGBaCoppqB189KV8+/F6gZprIQ1ZJ +03VApAVI2QBMqVEl0qag4CleV2vnXngMMlUVGxl9j4Ikvsy4F4t2nOpYNQ8SUkySlnkkajcnQsnl +txEgdvuir6jgVmhJYaJyLxyaq7xRSqtfFrUK6U7/AKYumSE2yOUgUqlU208ktHvEa+RhzSVGvrSv +f3fngKzm6SWlpMnLttwyxCKLnc6iVWEbalWlPDrX1xa80Ok2zTK0XHtvbUD1P2LYfvRG2lFkSLxV +0t10xRYtwqEok4TbtlXJGK4KLUuqKqZUMq93dXu07vWmLmbqSWziM3Bk3bK0aDKKJLhqFCK4STHS +ngrTy0wAKbavF5D3fAsmuwLlnVQxDfRqdSTEa+RU8PLvxFhE5Sbm2DHe6XbwVitttEqnShF0+HDe +amjhGdfbrmqttU6lry0pULqCP0p5YtHYvEm/zxEhHPFUttRBZxRbnQq7g00HT6euA7n2xQh2aZLI +ESaSfSRpCPIdPMce4xum3SQJsxIVCuG1yl/1HywaogKBkSKCQKiVhaEVtPoP0xJSZvCVvMkan8VL +yt/TlgBDckb7d5raPyrI+L5uQ+eHTdEmqNphvEVxANl1tPXpxPJo/TQNusadEKd20udC8X5YivnH +AK1SJV2e5TSlarkVtP1wCjeEuZKEBqD8IkiJdXp3YfBMRuTFHqHwe5G23/viIaTUgEbVKlTuqVcR +hdtA02xWpeNbeVOnXv8APAczftC5VTY9oTqQbNkH60ta7bRg3CkSoDWil4dxdNdenFB7NZ5qnmZJ +9LM15tZBAukZIkgbABVITTMa0qoIj01Avlx1J2y5ZaZ5yO9hUXKzV8nQKtHJDTRM7uWunPT1xyqG +XvbMs0bOHCQzbt3wIUogPB1KgiJGY9+te+6lNcB3BkWW9t5fYzKjYGzqQapuJECuE0BqHSP+WCqo +qKKhILouk02R2t0kTuFca6UuKmOUexfPS2Ss8o5RclJvSqqSEuuo8qsDkxPQSAD0oNKUx0a+zgqW +X46Rat9l5JOeDaUrXVNGl2lxU8+XlgDrgniDhmgKPFut9Qb07tpsFfn/AE7sZznXNULxaSe9FT2Y +I5JTiXqxiTOPIfGVAryJT8NOeLJPrPqSq8G2XpHgtRKi7lr0rKLHTS7XlbSlPTHOud8tqJyuVhlp +g0Ip46dxyKUYzTTWbGJXblD5VIiKlK1KvPAXTPce1aZCnXzGSN63XFstNkK21KvBIuYqKV+5S08v +lxj7+YmMoTb7NeX5iIQ9hPmzZiggtvpNmiojoIadFtteo9NSrimzWa4R07NzKUnnaziKqksqbqh7 +6gqHQLwrXSyg0py9cQ6ZkZSUll1lBZajIWWZtiaruRHdTdFdX3pDWnIvrzr9cAfzrmyYlM2lDz2f ++JbkZIqP2QUJrw6/OtKhpcVvpX8sQYma7I6sE0815azFNSiWqRvkX1oLCNdBIRrzGmmnLywh5x/a +T2oUbxMbE5bfE1rRbhiOiKhohdVTSlNbq9/59+KPmSUSeTTlZywboL3UFQWw2p1KlKUrWlPLXTX8 +64D/2Q== + +--Apple-Mail=_C7D5288F-B043-4A7F-AF3F-1EDF1A78438B-- + +--Apple-Mail=_F4EF9C8E-2E66-4FC6-8840-F435ADBED5C8-- diff --git a/src/leap/bitmask/mail/tests/rfc822.multi-signed.message b/src/leap/bitmask/mail/tests/rfc822.multi-signed.message new file mode 100644 index 0000000..9907c2d --- /dev/null +++ b/src/leap/bitmask/mail/tests/rfc822.multi-signed.message @@ -0,0 +1,238 @@ +Date: Mon, 6 Jan 2014 04:40:47 -0400 +From: Kali Kaneko +To: penguin@example.com +Subject: signed message +Message-ID: <20140106084047.GA21317@samsara.lan> +MIME-Version: 1.0 +Content-Type: multipart/signed; micalg=pgp-sha1; + protocol="application/pgp-signature"; boundary="z9ECzHErBrwFF8sy" +Content-Disposition: inline +User-Agent: Mutt/1.5.21 (2012-12-30) + + +--z9ECzHErBrwFF8sy +Content-Type: multipart/mixed; boundary="z0eOaCaDLjvTGF2l" +Content-Disposition: inline + + +--z0eOaCaDLjvTGF2l +Content-Type: text/plain; charset=utf-8 +Content-Disposition: inline +Content-Transfer-Encoding: quoted-printable + +This is an example of a signed message, +with attachments. + + +--=20 +Nihil sine chao! =E2=88=B4 + +--z0eOaCaDLjvTGF2l +Content-Type: text/plain; charset=us-ascii +Content-Disposition: attachment; filename="attach.txt" + +this is attachment in plain text. + +--z0eOaCaDLjvTGF2l +Content-Type: application/octet-stream +Content-Disposition: attachment; filename="hack.ico" +Content-Transfer-Encoding: base64 + +AAABAAMAEBAAAAAAAABoBQAANgAAACAgAAAAAAAAqAgAAJ4FAABAQAAAAAAAACgWAABGDgAA +KAAAABAAAAAgAAAAAQAIAAAAAABAAQAAAAAAAAAAAAAAAAAAAAAAAAAAAAD///8Ai4uLAEZG +RgDDw8MAJCQkAGVlZQDh4eEApqamADQ0NADw8PAADw8PAFVVVQDT09MAtLS0AJmZmQAaGhoA +PT09AMvLywAsLCwA+Pj4AAgICADp6ekA2traALy8vABeXl4An5+fAJOTkwAfHx8A9PT0AOXl +5QA4ODgAuLi4ALCwsACPj48ABQUFAPv7+wDt7e0AJycnADExMQDe3t4A0NDQAL+/vwCcnJwA +/f39ACkpKQDy8vIA6+vrADY2NgDn5+cAOjo6AOPj4wDc3NwASEhIANjY2ADV1dUAU1NTAMnJ +yQC6uroApKSkAAEBAQAGBgYAICAgAP7+/gD6+voA+fn5AC0tLQD19fUA8/PzAPHx8QDv7+8A +Pj4+AO7u7gDs7OwA6urqAOjo6ADk5OQAVFRUAODg4ADf398A3d3dANvb2wBfX18A2dnZAMrK +ygDCwsIAu7u7ALm5uQC3t7cAs7OzAKWlpQCdnZ0AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA +AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA +AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA +AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA +AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA +AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA +AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA +AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA +AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA +AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA +AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA +AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA +AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAABKRC5ESDRELi4uNEUhIhcK +LgEBAUEeAQEBAUYCAAATNC4BPwEUMwE/PwFOQgAAACsuAQEBQUwBAQEBSk0AABVWSCwBP0RP +QEFBFDNTUkdbLk4eOg0xEh5MTEw5RlEqLgdKTQAcGEYBAQEBJQ4QPBklWwAAAANKAT8/AUwy +AAAAOxoAAAA1LwE/PwEeEQAAAFpJGT0mVUgBAQE/SVYFFQZIKEtVNjFUJR4eSTlIKARET0gs +AT8dS1kJH1dINzgnGy5EAQEBASk+AAAtUAwAACNYLgE/AQEYFQAAC1UwAAAAW0QBAQEkMRkA +AAZDGwAAME8WRC5EJU4lOwhIT0UgD08KAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA +AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAACgAAAAgAAAAQAAAAAEACAAAAAAA +gAQAAAAAAAAAAAAAAAAAAAAAAAAAAAAA////AH9/fwC/v78APz8/AN/f3wBfX18An5+fAB0d +HQAuLi4A7+/vAM/PzwCvr68Ab29vAE5OTgAPDw8AkZGRAPf39wDn5+cAJiYmANfX1wA3NzcA +x8fHAFdXVwC3t7cAh4eHAAcHBwAWFhYAaGhoAEhISAClpaUAmZmZAHl5eQCMjIwAdHR0APv7 ++wALCwsA8/PzAOvr6wDj4+MAKioqANvb2wDT09MAy8vLAMPDwwBTU1MAu7u7AFtbWwBjY2MA +AwMDABkZGQAjIyMANDQ0ADw8PABCQkIAtLS0AEtLSwCioqIAnJycAGxsbAD9/f0ABQUFAPn5 ++QAJCQkA9fX1AA0NDQDx8fEAERERAO3t7QDp6ekA5eXlAOHh4QAsLCwA3d3dADAwMADZ2dkA +OTk5ANHR0QDNzc0AycnJAMXFxQDBwcEAUVFRAL29vQBZWVkAXV1dALKysgBycnIAk5OTAIqK +igABAQEABgYGAAwMDAD+/v4A/Pz8APr6+gAXFxcA+Pj4APb29gD09PQA8vLyACQkJADw8PAA +JycnAOzs7AApKSkA6urqAOjo6AAvLy8A5ubmAOTk5ADi4uIAODg4AODg4ADe3t4A3NzcANra +2gDY2NgA1tbWANTU1ABNTU0A0tLSANDQ0ABUVFQAzs7OAMzMzABYWFgAysrKAMjIyABcXFwA +xsbGAF5eXgDExMQAYGBgAMDAwABkZGQAuLi4AG1tbQC2trYAtbW1ALCwsACurq4Aenp6AKOj +owChoaEAoKCgAJ6engCdnZ0AmpqaAI2NjQCSkpIAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA +AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA +AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA +AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA +AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA +AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA +AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA +AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA +AAAAAAAAAAAAAAAAAAAAAAAAFHFvR3Fvb0dHJ1F0R0dHR29HR0YLf28nJkVraGtHBXMnAQEB +AQEBAQEBCxEBAQEBAQEBASdzASOMHHsZSQEBcnEBAV1dXV1dXQFOJQEBXV1dXV0BR0kBOwAA +AAAIUAFyJwFdXV1dXV1dAU4lAV1dXV1dXQFHbVgAAAAAAAAoaG5xAV1dXV1dXV0BfSUBXV1d +XV1dASd2HQAAAAAAAFoMEkcBXV1dXV1dXQFOZAEBXV1dXV0BbU8TAAAAAAAAAFkmcQFdXV1d +XV1dAU4lAV1dXV1dXQEnSzgAAAAAAABaN2tHAV1dXV1dXV0BTiUBXV1dXV1dAUdtHwAAAAAA +AEpEJycBXV1dXV1dAQFOJQFdAV1dAV0BRykBIgAAAABlfAFzJwEBAQEBAQEBAQtAAQEBAQEB +AQFuSQE8iFeBEG8BXUeGTn0LdnR3fH0LOYR8Tk5OTnxOeouNTQspJ0YFd30rgCljIwpTlCxm +X2KERWMlJSUlJSURFE1hPEYMBysRYSV0RwF3NT0AGjYpAQtjAQEBAQEBAQFvKQGKMzEAP4dC +AXESEmcAAAAAAEpEKiUBXV1dXV1dAUduLEEAAAAAAIFdcUSWAAAAAAAAADp1ZAFdXV1dXV0B +bwVVAAAAAAAAW4Jta34AAAAAAAAAhRQlAV1dXV1dAQFtK0gAAAAAAAAAEGtFhwAAAAAAAACJ +S2QBXV1dXV1dAW5NFQAAAAAAAACTa2geAAAAAAAAAAx0ZAFdXV1dXV0BR0YNAAAAAAAADxRu +J14tAAAAAAAvXQslAV1dXV1dXQFHcW4JAAAAAAAhAXFuAWMgbBsJAhEBTWIBAQEBAQEBAW5y +AW+DZWBwkQEBcQtHbWh2hnZEbm6LFG9HR21uR3FGgFFGa2oqFgVob3FNf0t0dAUncnR0SY1N +KW5xK01ucUlRLklyRksqR250S3pGAQEBAQEBAQEBeWIBUFRINA1uAUYFAQqOTGlSiAEBb0cB +XV1dAQFdAQF9I4pcAAAAABNHEnIKBAAAAAA9kAFJJwFdXV1dXV1dAXptZwAAAAAAAAZqbY4A +AAAAAAAbcm5HAV1dXV1dXV0BFFZbAAAAAAAAZ3pLNQAAAAAAAACPa0cBXV1dXV1dXQEpkgAA +AAAAAAAygHppAAAAAAAAAJVrcQFdXV1dXV1dAXl9QwAAAAAAADZxcRcAAAAAAAA9UW1vAV1d +XV1dXV0BC2EwAAAAAAAAkmhGGD0AAAAAAHg+cW8BAV1dAV1dAQFOESWBAAAAJJUBJykBkEMA +AAAOJgFzRwE8AV1dXV1dAX0lAV8WEDp1AQFxSwEBBTkhAxEBPHJzSXEFcnJJcnFyFnRycRJr +RW5ycXl8cXJuRSYScQVJcQAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA +AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA +AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAKAAAAEAAAACAAAAAAQAIAAAA +AAAAEgAAAAAAAAAAAAAAAAAAAAAAAAAAAAD///8Af39/AL+/vwA/Pz8A39/fAF9fXwCfn58A +Hx8fAO/v7wAvLy8Ab29vAI+PjwAPDw8A0NDQALCwsABQUFAA9/f3ABcXFwDn5+cAJycnAMjI +yABHR0cAqKioAGdnZwCXl5cAd3d3AIeHhwAHBwcA2NjYALi4uABXV1cANTU1ADo6OgD7+/sA +CwsLAPPz8wATExMA6+vrABsbGwDj4+MAIyMjANTU1AArKysAzMzMAMTExABLS0sAtLS0AKys +rABbW1sApKSkAGNjYwCbm5sAa2trAJOTkwBzc3MAi4uLAHt7ewCDg4MAAwMDANzc3AAyMjIA +vLy8AFNTUwD9/f0ABQUFAPn5+QAJCQkADQ0NAPHx8QDt7e0AFRUVAOnp6QAZGRkA5eXlAB0d +HQDh4eEAISEhACUlJQDa2toAKSkpANbW1gDS0tIAysrKADw8PADGxsYAwsLCAEVFRQBJSUkA +urq6ALa2tgCysrIArq6uAFlZWQCqqqoAXV1dAKampgBlZWUAoqKiAJ2dnQBtbW0AmZmZAHFx +cQCVlZUAeXl5AH19fQCJiYkAhYWFAAEBAQACAgIABAQEAP7+/gAGBgYA/Pz8AAgICAD6+voA +CgoKAPj4+AAMDAwA9vb2APT09AASEhIA8vLyABQUFADu7u4AFhYWAOzs7AAYGBgA6urqAOjo +6AAeHh4AICAgAOTk5AAiIiIA4uLiACQkJADg4OAAJiYmAN7e3gDd3d0AKCgoANvb2wAqKioA +2dnZACwsLADX19cALi4uANXV1QAxMTEA09PTADMzMwDR0dEANDQ0AM3NzQA5OTkAy8vLADs7 +OwDJyckAPT09AMfHxwBAQEAAxcXFAMPDwwDBwcEAwMDAAL6+vgBKSkoAvb29ALu7uwC5ubkA +UVFRALe3twBSUlIAtbW1AFRUVACzs7MAVlZWAFhYWABaWloAra2tAFxcXACrq6sAXl5eAKmp +qQCnp6cAZGRkAKOjowChoaEAaGhoAKCgoACenp4AnJycAG5ubgCampoAcHBwAJiYmABycnIA +lpaWAJSUlAB2dnYAkpKSAHh4eACQkJAAenp6AI6OjgB8fHwAjIyMAIiIiACCgoIAhISEAAAA +AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA +AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA +AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA +AAAAAAAAAAAAAAAAAAAAAAAAAAC1WlpaWlpaWlpaWlpaWlpaWlpaHjAHr6taWlpaWlpaWlpa +WlpaWlpaq68HMB5aWlpap6KlWzBaA6KoWlpaWlq1WgEBAQEBAQEBAQEBAQEBAQEBAQGXNkUB +AQEBAQEBAQEBAQEBAQEBAQFFNpcBAQEBASg4EI6HPa5lfgEBAQEBWloBAQEBAQEBAQEBAQEB +AQEBAQEBlzZFAQEBAQEBAQEBAQEBAQEBAQEBRTaXAQEBETpEAAAAAAAAAH/FbwEBAVpaAQEB +AQEBAQEBAQEBAQEBAQEBAZc2RQEBAQEBAQEBAQEBAQEBAQEBAUU2lwEBhFQAAAAAAAAAAAAA +ALJCAQFaWgEBAQEBAQEBAQEBAQEBAQEBAQGXNkUBAQEBAQEBAQEBAQEBAQEBAQFFNpcBeJoA +AAAAAAAAAAAAAAAAMQEBWloBAQEBAQEBAQEBAQEBAQEBAQEBlzZFAQEBAQEBAQEBAQEBAQEB +AQEBRTZSATUAAAAAAAAAAAAAAAAAAABnAVpaAQEBAQEBAQEBAQEBAQEBAQEBAZc2RQEBAQEB +AQEBAQEBAQEBAQEBAUU2Tx1wAAAAAAAAAAAAAAAAAAAAgkaoWgEBAQEBAQEBAQEBAQEBAQEB +AQGXNkUBAQEBAQEBAQEBAQEBAQEBAQFFNgVrAAAAAAAAAAAAAAAAAAAAAABioloBAQEBAQEB +AQEBAQEBAQEBAQEBlzZFAQEBAQEBAQEBAQEBAQEBAQEBRWcqngAAAAAAAAAAAAAAAAAAAAAA +tANaAQEBAQEBAQEBAQEBAQEBAQEBAZc2RQEBAQEBAQEBAQEBAQEBAQEBAUXDpIcAAAAAAAAA +AAAAAAAAAAAAAJRaWgEBAQEBAQEBAQEBAQEBAQEBAQGXNkUBAQEBAQEBAQEBAQEBAQEBAQFF +wa9HAAAAAAAAAAAAAAAAAAAAAABOMFoBAQEBAQEBAQEBAQEBAQEBAQEBlzZFAQEBAQEBAQEB +AQEBAQEBAQEBRWVZggAAAAAAAAAAAAAAAAAAAAAAjltaAQEBAQEBAQEBAQEBAQEBAQEBAZc2 +RQEBAQEBAQEBAQEBAQEBAQEBAUXFmZYAAAAAAAAAAAAAAAAAAAAAAKqlWgEBAQEBAQEBAQEB +AQEBAQEBAQGXNkUBAQEBAQEBAQEBAQEBAQEBAQFFNorHAAAAAAAAAAAAAAAAAAAAAABloloB +AQEBAQEBAQEBAQEBAQEBAQEBlzZFAQEBAQEBAQEBAQEBAQEBAQEBRTY8UwAAAAAAAAAAAAAA +AAAAAAASEz5aAQEBAQEBAQEBAQEBAQEBAQEBAZc2RQEBAQEBAQEBAQEBAQEBAQEBAUU2lQFd +AAAAAAAAAAAAAAAAAAAA0AFaWgEBAQEBAQEBAQEBAQEBAQEBAQGXNkUBAQEBAQEBAQEBAQEB +AQEBAQFFNpcBhoUAAAAAAAAAAAAAAAAAVxEBWloBAQEBAQEBAQEBAQEBAQEBAQEBlzZFAQEB +AQEBAQEBAQEBAQEBAQEBRTaXAQGXTQAAAAAAAAAAAAAAnCgBAVpaAQEBAQEBAQEBAQEBAQEB +AQEBAZc2RQEBAQEBAQEBAQEBAQEBAQEBAUU2lwEBASiwAAAAAAAAAAAcwncBAQFaWgEBAQEB +AQEBAQEBAQEBAQEBAQGXNkUBAQEBAQEBAQEBAQEBAQEBAQFFNpcBAQEBASy8khINgiFojQEB +AQEBWjCVl5eXl5eXl5dSUpeXl5eXl5eTHsWdlZeXl5eXl5eXl5eXl5eXl5eVncUek5eXl1I8 +ipsvs6iVBU9Sl5eXlTAHNjY2NjY2Zb1ivbtiY2c2NjY2NsVlxjY2NjY2NjY2NjY2NjY2NjY2 +NsZlxTY2NjY2xr8yFxcXusHGNjY2NjYHW3hFRUURAY8HC7Jh0ahFb3pFRRGdxkp4RUVFRUVF +RUVFRUVFRUVFRXhKxp0RRUVFIkKhDLkxwMiXInNFRUV4W1oBAQEBCcclAAAAAAAAnK0BAQEB +lzZFAQEBAQEBAQEBAQEBAQEBAQEBRTaXAQEBAQ4ucAAAAAAAdAaNAQEBAVpaAQEBpYMAAAAA +AAAAAAAAGHUBAZc2RQEBAQEBAQEBAQEBAQEBAQEBAUU2lwEBAWtwAAAAAAAAAAAADboBAQFa +WgEBHnIAAAAAAAAAAAAAAACxcwGXNkUBAQEBAQEBAQEBAQEBAQEBAQFFNpcBAcQAAAAAAAAA +AAAAAABtwQEBWloBiCcAAAAAAAAAAAAAAAAAAM0BUjZFAQEBAQEBAQEBAQEBAQEBAQEBRTaX +AbsAAAAAAAAAAAAAAAAAAHCiAVpaAQYAAAAAAAAAAAAAAAAAAAAck082RQEBAQEBAQEBAQEB +AQEBAQEBAUU2UUVLAAAAAAAAAAAAAAAAAAAAIQEePkoNAAAAAAAAAAAAAAAAAAAAAMCLxkUB +AQEBAQEBAQEBAQEBAQEBAQFFNgViAAAAAAAAAAAAAAAAAAAAAACppKK9AAAAAAAAAAAAAAAA +AAAAAACQnxlFAQEBAQEBAQEBAQEBAQEBAQEBRcZPrAAAAAAAAAAAAAAAAAAAAAAAZqOjCwAA +AAAAAAAAAAAAAAAAAAAAQ7i/RQEBAQEBAQEBAQEBAQEBAQEBAUUZVSsAAAAAAAAAAAAAAAAA +AAAAAFRZpT8AAAAAAAAAAAAAAAAAAAAAAADKvkUBAQEBAQEBAQEBAQEBAQEBAQFFZVpJAAAA +AAAAAAAAAAAAAAAAAAAUXKU/AAAAAAAAAAAAAAAAAAAAAAAAyr5FAQEBAQEBAQEBAQEBAQEB +AQEBRWVaSQAAAAAAAAAAAAAAAAAAAAAAFFyjCwAAAAAAAAAAAAAAAAAAAAAAdl40RQEBAQEB +AQEBAQEBAQEBAQEBAUUZVSsAAAAAAAAAAAAAAAAAAAAAAKCoVrcAAAAAAAAAAAAAAAAAAAAA +ACCZxUUBAQEBAQEBAQEBAQEBAQEBAQFFxo1fAAAAAAAAAAAAAAAAAAAAAABpVqh+fQAAAAAA +AAAAAAAAAAAAAADRijZFAQEBAQEBAQEBAQEBAQEBAQEBRTaKXAAAAAAAAAAAAAAAAAAAAAA7 +LANaAWgAAAAAAAAAAAAAAAAAAABJSJE2RQEBAQEBAQEBAQEBAQEBAQEBAUU2KgEKAAAAAAAA +AAAAAAAAAAAAHwGrWgF8kAAAAAAAAAAAAAAAAAAAZQGXNkUBAQEBAQEBAQEBAQEBAQEBAQFF +NpcBHm0AAAAAAAAAAAAAAAAAEk8BWloBAZVLAAAAAAAAAAAAAAAANwEBlzZFAQEBAQEBAQEB +AQEBAQEBAQEBRTaXAQHFAAAAAAAAAAAAAAAAQx4BAVpaAQEBj1QAAAAAAAAAAAByGQEBAZc2 +RQEBAQEBAQEBAQEBAQEBAQEBAUU2lwEBARcSAAAAAAAAAAAAjJkBAQFaWgEBAQFxuphuAAAA +ABK8jwEBAQGXNkUBAQEBAQEBAQEBAQEBAQEBAQFFNpcBAQEBSMlLAAAAAG0rDEUBAQEBWlt4 +RUVFeAFFLWU6DC8FcXNFRUURncZKeEVFRUVFRUVFRUVFRUVFRUV4SsadEUVFRXUBhC8MOmWi +JgF3RUVFeFsHNjY2NjY2Z7+9Yru+wzY2NjY2NsVlxjY2NjY2NsU0vr6/wzY2NjY2NsZlxTY2 +NjY2NmUytbO3Yhk2NjY2NjYHMJWXl5eXl5eXl5eXl5eXl5eXl5MexZ2Vl5eXHQWdXgwMYKKK +T5eXl5WdxR6Tl5eXKgWVrWfOvquPipWXl5eVMFoBAQEBAQEBAQEBAQEBAQEBAQEBlzZFAQEB +AYE5kHYAAEMpvJEBAQEBRTaXAQEBAXFiBEcAAG4Spi8BAQEBAVpaAQEBAQEBAQEBAQEBAQEB +AQEBAZc2RQEBAcF7AAAAAAAAAABBaUIBAUU2lwEBAZsgAAAAAAAAAAAAFooBAQFaWgEBAQEB +AQEBAQEBAQEBAQEBAQGXNkUBAQsAAAAAAAAAAAAAAACxcwFFNpcBAQ92AAAAAAAAAAAAAABN +UQEBWloBAQEBAQEBAQEBAQEBAQEBAQEBlzZFAcwAAAAAAAAAAAAAAAAAABgBejaXAZd5AAAA +AAAAAAAAAAAAAImAAVpaAQEBAQEBAQEBAQEBAQEBAQEBAZc2c1JDAAAAAAAAAAAAAAAAAAAA +W3E2KgGeAAAAAAAAAAAAAAAAAAAAMwGrWgEBAQEBAQEBAQEBAQEBAQEBAQGXNm9kAAAAAAAA +AAAAAAAAAAAAAAQJZ4ukAAAAAAAAAAAAAAAAAAAAAHKVpVoBAQEBAQEBAQEBAQEBAQEBAQEB +l8OGKQAAAAAAAAAAAAAAAAAAAAAcor+LNQAAAAAAAAAAAAAAAAAAAAAAaqJaAQEBAQEBAQEB +AQEBAQEBAQEBAZdjHmwAAAAAAAAAAAAAAAAAAAAAAM8ymT0AAAAAAAAAAAAAAAAAAAAAAFg+ +WgEBAQEBAQEBAQEBAQEBAQEBAQGXvWUAAAAAAAAAAAAAAAAAAAAAAABhuFmCAAAAAAAAAAAA +AAAAAAAAAACOW1oBAQEBAQEBAQEBAQEBAQEBAQEBl7vOAAAAAAAAAAAAAAAAAAAAAAAAtGCv +RwAAAAAAAAAAAAAAAAAAAAAATjBaAQEBAQEBAQEBAQEBAQEBAQEBAZcHYgAAAAAAAAAAAAAA +AAAAAAAAAAu4pIcAAAAAAAAAAAAAAAAAAAAAAD1aWgEBAQEBAQEBAQEBAQEBAQEBAQGXNBUj +AAAAAAAAAAAAAAAAAAAAAAAyvSpXAAAAAAAAAAAAAAAAAAAAAAAYpFoBAQEBAQEBAQEBAQEB +AQEBAQEBl2ckVAAAAAAAAAAAAAAAAAAAAACDiMMFzAAAAAAAAAAAAAAAAAAAAAAAr6NaAQEB +AQEBAQEBAQEBAQEBAQEBAZc2b7sAAAAAAAAAAAAAAAAAAAAAaW82HRMlAAAAAAAAAAAAAAAA +AAAAlECpWgEBAQEBAQEBAQEBAQEBAQEBAQGXNngBBAAAAAAAAAAAAAAAAAAAKUZ3NpcBzwAA +AAAAAAAAAAAAAAAAAA8BWloBAQEBAQEBAQEBAQEBAQEBAQEBlzZFAZGCAAAAAAAAAAAAAAAA +dC0BRTaXAXGwAAAAAAAAAAAAAAAAAAIBAVpaAQEBAQEBAQEBAQEBAQEBAQEBAZc2RQEBlY4A +AAAAAAAAAAAACD4BAUU2lwEBd7YAAAAAAAAAAAAAbmtvAQFaWgEBAQEBAQEBAQEBAQEBAQEB +AQGXNkUBAQEJyw0AAAAAAAB0M0wBAQFFNpcBAQEBF1AAAAAAAAAAVD4BAQEBWloBAQEBAQEB +AQEBAQEBAQEBAQEBlzZFAQEBAQETB7ymprxliwEBAQEBRTaXAQEBAQF1qxqsV7QbVXEBAQEB +AVq1WlpaWlpaWlpaWlpaWlpaWlpaHjAHr6taWlpaPqKkPj6kLadaWlpaq68HMB5aWlpaqaNW +pz4DLaQeWlpaWlq1AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA +AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA +AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA +AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA +AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA +AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA +AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA +AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA +AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA +AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA= + +--z0eOaCaDLjvTGF2l-- + +--z9ECzHErBrwFF8sy +Content-Type: application/pgp-signature + +-----BEGIN PGP SIGNATURE----- +Version: GnuPG v1.4.15 (GNU/Linux) + +iQIcBAEBAgAGBQJSymwPAAoJECNji/csWTvBhtcP/2AKF0uk6ljrfMWhNBSFwDqv +kYng3slREnF/pxnIGOpR2GAxPBPjRipZOuUU8QL+pXBwk5kWzb9RYpr26xMYWRtl +vXdVbob5NolNEYrqTkkQ1kejERQGFyescsUJDcEDXJl024czKWbxHTYYN4vlYJMK +PZ5mPSdADFn970PnVXfNix3Rjvv7SFQGammDBGjQzyROkoiDKPZcomp6dzm6zEXC +w8i42WfHU8GkyVVNvXZI52Xw3LUXiXsJ58B1V1O5U42facepG6S+S0DC/PWptqPw +sAM9/YGkvBNWrsJA/BavXPRLE1gVpu+hZZEsOqRvs244k7JTrVo54xDbdeOT2nTr +BDk4e88vmCVKGgE9MZjDbjgOHDZhmsxNQm4DBGRH2huF0noUc/8Sm4KhSO49S2mN +QjIT5QrPerQNiP5QtShHZRJX7ElXYZWX1SG/c9jQjfd0W1XK/cGtwClICe+lpprt +mLC2607yalbRhCxV9bQlVUnd2tY3NY4UgIKgCEiEwb1hf/k9jQDvpk16VuNWSZQJ +jFeg9F2WdNjQMp79cyvnayyhjS9o/K2LbSIgJi7KdlQcVZ/2DQfbMjCwByR7P9g8 +gcAKh8V7E6IpAu1mnvs4FDagipppK6hOTRj2s/I3xZzneprSK1WaVro/8LAWZe9X +sSdfcAhT7Tno7PB/Acoh +=+okv +-----END PGP SIGNATURE----- + +--z9ECzHErBrwFF8sy-- diff --git a/src/leap/bitmask/mail/tests/rfc822.multi.message b/src/leap/bitmask/mail/tests/rfc822.multi.message new file mode 100644 index 0000000..30f74e5 --- /dev/null +++ b/src/leap/bitmask/mail/tests/rfc822.multi.message @@ -0,0 +1,96 @@ +Date: Fri, 19 May 2000 09:55:48 -0400 (EDT) +From: Doug Sauder +To: Joe Blow +Subject: Test message from PINE +Message-ID: +MIME-Version: 1.0 +Content-Type: MULTIPART/MIXED; BOUNDARY="-1463757054-952513540-958744548=:8452" + + This message is in MIME format. The first part should be readable text, + while the remaining parts are likely unreadable without MIME-aware tools. + Send mail to mime@docserver.cac.washington.edu for more info. + +---1463757054-952513540-958744548=:8452 +Content-Type: TEXT/PLAIN; charset=US-ASCII + +This is a test message from PINE MUA. + + +---1463757054-952513540-958744548=:8452 +Content-Type: APPLICATION/octet-stream; name="redball.png" +Content-Transfer-Encoding: BASE64 +Content-ID: +Content-Description: A PNG graphic file +Content-Disposition: attachment; filename="redball.png" + +iVBORw0KGgoAAAANSUhEUgAAABsAAAAbCAMAAAC6CgRnAAADAFBMVEX///8A +AAABAAALAAAVAAAaAAAXAAARAAAKAAADAAAcAAAyAABEAABNAABIAAA9AAAj +AAAWAAAmAABhAAB7AACGAACHAAB9AAB0AABgAAA5AAAUAAAGAAAnAABLAABv +AACQAAClAAC7AAC/AACrAAChAACMAABzAABbAAAuAAAIAABMAAB3AACZAAC0 +GRnKODjVPT3bKSndBQW4AACoAAB5AAAxAAAYAAAEAABFAACaAAC7JCTRYWHf +hITmf3/mVlbqHx/SAAC5AACjAABdAABCAAAoAAAJAABnAAC6Dw/QVFTek5Pl +rKzpmZntZWXvJSXXAADBAACxAACcAABtAABTAAA2AAAbAAAFAABKAACBAADL +ICDdZ2fonJzrpqbtiorvUVHvFBTRAADDAAC2AAB4AABeAABAAAAiAABXAACS +AADCAADaGxvoVVXseHjveHjvV1fvJibhAADOAAC3AACnAACVAABHAAArAAAP +AACdAADFAADhBQXrKCjvPDzvNTXvGxvjAADQAADJAAC1AACXAACEAABsAABP +AAASAAACAABiAADpAADvAgLnAADYAADLAAC6AACwAABwAAATAAAkAABYAADI +AADTAADNAACzAACDAABuAAAeAAB+AADAAACkAACNAAB/AABpAABQAAAwAACR +AACpAAC8AACqAACbAABlAABJAAAqAAAOAAA0AACsAACvAACtAACmAACJAAB6 +AABrAABaAAA+AAApAABqAACCAACfAACeAACWAACPAAB8AAAZAAAHAABVAACO +AACKAAA4AAAQAAA/AAByAACAAABcAAA3AAAsAABmAABDAABWAAAgAAAzAAA8 +AAA6AAAfAAAMAAAdAAANAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA +AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA +AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAD8 +LtlFAAAAAXRSTlMAQObYZgAAABZ0RVh0U29mdHdhcmUAZ2lmMnBuZyAyLjAu +MT1evmgAAAIISURBVHicY2CAg/8QwIABmJhZWFnZ2Dk4MaU5uLh5eHn5+LkF +BDlQJf8zC/EIi4iKiUtI8koJScsgyf5nlpWTV1BUUlZRVVPX4NFk1UJIyghp +6+jq6RsYGhmbKJgK85mZW8Dk/rNaSlhZ29ja2Ts4Ojkr6Li4urFDNf53N/Ow +8vTy9vH18w8IDAoWDQkNC4+ASP5ni4wKio6JjYtPSExKTnFWSE1LF4A69n9G +ZlZ2Tm5efkFhUXFySWlZlEd5RSVY7j+TkGRVdU1tXX1DY1Ozcktpa1t7h2Yn +OAj+d7l1tyo79vT29SdNSJ44SbFVdHIo9xSIHNPUaWqTpifNSJrZnK00S0U1 +a/acUG5piNz/uXLzVJ2qm6dXz584S2WB1cJFi5cshZr539xVftnyFKUVTi2T +VjqvyhJLXb1m7TqoHPt6F/HW0g0bN63crGqVtWXrtu07BJihcsw71+zanRW8 +Z89eq337RQ/Ip60xO3gIElX/LbikDm8T36KwbNmRo7O3zpHkPSZwHBqL//8f +lz1x2OOkyKJTi7aqbzutfUZI2gIuF8F2lr/D5dw2+fZdwpl8YVOlI+CJ4/9/ +joOyYed5QzMvhGqnm2V0WiClm///D0lfXHtJ6vLlK9w7rx7vQk5SQJbFtSms +1y9evXid7QZacgOxmSxktNzdtSwwU+J/VICaCPFIYU3XAJhIOtjf5sfyAAAA +JXRFWHRDb21tZW50AGNsaXAyZ2lmIHYuMC42IGJ5IFl2ZXMgUGlndWV0NnM7 +vAAAAABJRU5ErkJggg== +---1463757054-952513540-958744548=:8452 +Content-Type: APPLICATION/octet-stream; name="blueball.png" +Content-Transfer-Encoding: BASE64 +Content-ID: +Content-Description: A PNG graphic file +Content-Disposition: attachment; filename="blueball.png" + +iVBORw0KGgoAAAANSUhEUgAAABsAAAAbCAMAAAC6CgRnAAADAFBMVEX///8A +AAgAABAAABgAAAAACCkAEEIAEEoACDEAEFIIIXMIKXsIKYQIIWsAGFoACDkI +IWMQOZwYQqUYQq0YQrUQOaUQMZQAGFIQMYwpUrU5Y8Y5Y84pWs4YSs4YQs4Y +Qr1Ca8Z7nNacvd6Mtd5jlOcxa94hUt4YStYYQsYQMaUAACHO5+/n7++cxu9S +hO8pWucQOa1Ke86tzt6lzu9ajO8QMZxahNat1ufO7++Mve9Ke+8YOaUYSsaM +vee15++Uve8AAClajOdzpe9rnO8IKYwxY+8pWu8IIXsAAAAAAAAAAAAAAAAA +AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA +AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA +AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA +AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA +AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA +AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA +AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA +AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA +AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA +AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA +AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA +AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAADB +Mg1VAAAAAXRSTlMAQObYZgAAABZ0RVh0U29mdHdhcmUAZ2lmMnBuZyAyLjAu +MT1evmgAAAGISURBVHicddJtV5swGAbgEk6AJhBSk4bMCUynBSLaqovbrG/b +fPn/vyh70lbsscebL5xznTsh5BmNhgQoRChwo50EOIohUYLDj4zHhKYQkrEo +Qdvock4ne0IKMVUpKZLQDeqSTIsv+18PyqqWUw2IBsRM7307PPp+fDJrWtnp +LDJvewYxnewfnvanZ+fzpmwXijC8KbqEa3Fx2ff91Y95U9XCUpaDeQwiMpHX +P/v+1++bWVPWQoGFawtjury9vru/f/C1Vi7ezT0WWpQHf/7+u/G71aLThK/M +jRxmT6KdzZ9fGk9yatMsTgZLl3XVgFRAC6spj/13enssqJVtWVa3NdBSacL8 ++VZmYqKmdd1CSYoOiMOSGwtzlqqlFFIuOqv0a1ZEZrUkWICLLFW266y1KvWE +1zV/iDAH1EopnVLCiygZCIomH3NCKX0lnI+B1iuuzCGTxwXjnDO4d7NpbX42 +YJJHkBwmAm2TxwAZg40J3+Xtbv1rgOAZwG0NxW62p+lT+Yi747sD/wEUVMzY +mWkOvwAAACV0RVh0Q29tbWVudABjbGlwMmdpZiB2LjAuNiBieSBZdmVzIFBp +Z3VldDZzO7wAAAAASUVORK5CYII= +---1463757054-952513540-958744548=:8452-- diff --git a/src/leap/bitmask/mail/tests/rfc822.plain.message b/src/leap/bitmask/mail/tests/rfc822.plain.message new file mode 100644 index 0000000..fc627c3 --- /dev/null +++ b/src/leap/bitmask/mail/tests/rfc822.plain.message @@ -0,0 +1,66 @@ +From pyar-bounces@python.org.ar Wed Jan 8 14:46:02 2014 +Return-Path: +X-Spam-Checker-Version: SpamAssassin 3.3.2 (2011-06-06) on spamd2.riseup.net +X-Spam-Level: ** +X-Spam-Pyzor: Reported 0 times. +X-Spam-Status: No, score=2.1 required=8.0 tests=AM_TRUNCATED,CK_419SIZE, + CK_NAIVER_NO_DNS,CK_NAIVE_NO_DNS,ENV_FROM_DIFF0,HAS_REPLY_TO,LINK_NR_TOP, + NO_REAL_NAME,RDNS_NONE,RISEUP_SPEAR_C shortcircuit=no autolearn=disabled + version=3.3.2 +Delivered-To: kali@leap.se +Received: from mx1.riseup.net (mx1-pn.riseup.net [10.0.1.33]) + (using TLSv1 with cipher DHE-RSA-AES256-SHA (256/256 bits)) + (Client CN "*.riseup.net", Issuer "Gandi Standard SSL CA" (not verified)) + by vireo.riseup.net (Postfix) with ESMTPS id 6C39A8F + for ; Wed, 8 Jan 2014 18:46:02 +0000 (UTC) +Received: from pyar.usla.org.ar (unknown [190.228.30.157]) + by mx1.riseup.net (Postfix) with ESMTP id F244C533F4 + for ; Wed, 8 Jan 2014 10:46:01 -0800 (PST) +Received: from [127.0.0.1] (localhost [127.0.0.1]) + by pyar.usla.org.ar (Postfix) with ESMTP id CC51D26A4F + for ; Wed, 8 Jan 2014 15:46:00 -0300 (ART) +MIME-Version: 1.0 +Content-Type: text/plain; charset="iso-8859-1" +Content-Transfer-Encoding: quoted-printable +From: pyar-request@python.org.ar +To: kali@leap.se +Subject: confirm 0e47e4342e4d42508e8c283175b05b3377148ac2 +Reply-To: pyar-request@python.org.ar +Auto-Submitted: auto-replied +Message-ID: +Date: Wed, 08 Jan 2014 15:45:59 -0300 +Precedence: bulk +X-BeenThere: pyar@python.org.ar +X-Mailman-Version: 2.1.15 +List-Id: Python Argentina +X-List-Administrivia: yes +Errors-To: pyar-bounces@python.org.ar +Sender: "pyar" +X-Virus-Scanned: clamav-milter 0.97.8 at mx1 +X-Virus-Status: Clean + +Mailing list subscription confirmation notice for mailing list pyar + +We have received a request de kaliyuga@riseup.net for subscription of +your email address, "kaliyuga@riseup.net", to the pyar@python.org.ar +mailing list. To confirm that you want to be added to this mailing +list, simply reply to this message, keeping the Subject: header +intact. Or visit this web page: + + http://listas.python.org.ar/confirm/pyar/0e47e4342e4d42508e8c283175b05b= +3377148ac2 + + +Or include the following line -- and only the following line -- in a +message to pyar-request@python.org.ar: + + confirm 0e47e4342e4d42508e8c283175b05b3377148ac2 + +Note that simply sending a `reply' to this message should work from +most mail readers, since that usually leaves the Subject: line in the +right form (additional "Re:" text in the Subject: is okay). + +If you do not wish to be subscribed to this list, please simply +disregard this message. If you think you are being maliciously +subscribed to the list, or have any other questions, send them to +pyar-owner@python.org.ar. diff --git a/src/leap/bitmask/mail/tests/test_mail.py b/src/leap/bitmask/mail/tests/test_mail.py new file mode 100644 index 0000000..f9cded2 --- /dev/null +++ b/src/leap/bitmask/mail/tests/test_mail.py @@ -0,0 +1,399 @@ +# -*- coding: utf-8 -*- +# test_mail.py +# Copyright (C) 2014 LEAP +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . +""" +Tests for the mail module. +""" +import os +import time +import uuid + +from functools import partial +from email.parser import Parser +from email.Utils import formatdate + +from leap.mail.adaptors.soledad import SoledadMailAdaptor +from leap.mail.mail import MessageCollection, Account, _unpack_headers +from leap.mail.mailbox_indexer import MailboxIndexer +from leap.mail.testing.common import SoledadTestMixin + +HERE = os.path.split(os.path.abspath(__file__))[0] + + +def _get_raw_msg(multi=False): + if multi: + sample = "rfc822.multi.message" + else: + sample = "rfc822.message" + with open(os.path.join(HERE, sample)) as f: + raw = f.read() + return raw + + +def _get_parsed_msg(multi=False): + mail_parser = Parser() + raw = _get_raw_msg(multi=multi) + return mail_parser.parsestr(raw) + + +def _get_msg_time(): + timestamp = time.mktime((2010, 12, 12, 1, 1, 1, 1, 1, 1)) + return formatdate(timestamp) + + +class CollectionMixin(object): + + def get_collection(self, mbox_collection=True, mbox_name=None, + mbox_uuid=None): + """ + Get a collection for tests. + """ + adaptor = SoledadMailAdaptor() + store = self._soledad + adaptor.store = store + + if mbox_collection: + mbox_indexer = MailboxIndexer(store) + mbox_name = mbox_name or "TestMbox" + mbox_uuid = mbox_uuid or str(uuid.uuid4()) + else: + mbox_indexer = mbox_name = None + + def get_collection_from_mbox_wrapper(wrapper): + wrapper.uuid = mbox_uuid + return MessageCollection( + adaptor, store, + mbox_indexer=mbox_indexer, mbox_wrapper=wrapper) + + d = adaptor.initialize_store(store) + if mbox_collection: + d.addCallback(lambda _: mbox_indexer.create_table(mbox_uuid)) + d.addCallback(lambda _: adaptor.get_or_create_mbox(store, mbox_name)) + d.addCallback(get_collection_from_mbox_wrapper) + return d + + +# TODO profile add_msg. Why are these tests so SLOW??! +class MessageTestCase(SoledadTestMixin, CollectionMixin): + """ + Tests for the Message class. + """ + msg_flags = ('\Recent', '\Unseen', '\TestFlag') + msg_tags = ('important', 'todo', 'wonderful') + internal_date = "19-Mar-2015 19:22:21 -0500" + + maxDiff = None + + def _do_insert_msg(self, multi=False): + """ + Inserts and return a regular message, for tests. + """ + def insert_message(collection): + self._mbox_uuid = collection.mbox_uuid + return collection.add_msg( + raw, flags=self.msg_flags, tags=self.msg_tags, + date=self.internal_date) + + raw = _get_raw_msg(multi=multi) + + d = self.get_collection() + d.addCallback(insert_message) + return d + + def get_inserted_msg(self, multi=False): + d = self._do_insert_msg(multi=multi) + d.addCallback(lambda _: self.get_collection(mbox_uuid=self._mbox_uuid)) + d.addCallback(lambda col: col.get_message_by_uid(1)) + return d + + def test_get_flags(self): + d = self.get_inserted_msg() + d.addCallback(self._test_get_flags_cb) + return d + + def _test_get_flags_cb(self, msg): + self.assertTrue(msg is not None) + self.assertEquals(tuple(msg.get_flags()), self.msg_flags) + + def test_get_internal_date(self): + d = self.get_inserted_msg() + d.addCallback(self._test_get_internal_date_cb) + + def _test_get_internal_date_cb(self, msg): + self.assertTrue(msg is not None) + self.assertDictEqual(msg.get_internal_date(), + self.internal_date) + + def test_get_headers(self): + d = self.get_inserted_msg() + d.addCallback(self._test_get_headers_cb) + return d + + def _test_get_headers_cb(self, msg): + self.assertTrue(msg is not None) + expected = [ + (str(key.lower()), str(value)) + for (key, value) in _get_parsed_msg().items()] + self.assertItemsEqual(_unpack_headers(msg.get_headers()), expected) + + def test_get_body_file(self): + d = self.get_inserted_msg(multi=True) + d.addCallback(self._test_get_body_file_cb) + return d + + def _test_get_body_file_cb(self, msg): + self.assertTrue(msg is not None) + orig = _get_parsed_msg(multi=True) + expected = orig.get_payload()[0].get_payload() + d = msg.get_body_file(self._soledad) + + def assert_body(fd): + self.assertTrue(fd is not None) + self.assertEqual(fd.read(), expected) + d.addCallback(assert_body) + return d + + def test_get_size(self): + d = self.get_inserted_msg() + d.addCallback(self._test_get_size_cb) + return d + + def _test_get_size_cb(self, msg): + self.assertTrue(msg is not None) + expected = len(_get_parsed_msg().as_string()) + self.assertEqual(msg.get_size(), expected) + + def test_is_multipart_no(self): + d = self.get_inserted_msg() + d.addCallback(self._test_is_multipart_no_cb) + return d + + def _test_is_multipart_no_cb(self, msg): + self.assertTrue(msg is not None) + expected = _get_parsed_msg().is_multipart() + self.assertEqual(msg.is_multipart(), expected) + + def test_is_multipart_yes(self): + d = self.get_inserted_msg(multi=True) + d.addCallback(self._test_is_multipart_yes_cb) + return d + + def _test_is_multipart_yes_cb(self, msg): + self.assertTrue(msg is not None) + expected = _get_parsed_msg(multi=True).is_multipart() + self.assertEqual(msg.is_multipart(), expected) + + def test_get_subpart(self): + d = self.get_inserted_msg(multi=True) + d.addCallback(self._test_get_subpart_cb) + return d + + def _test_get_subpart_cb(self, msg): + self.assertTrue(msg is not None) + + def test_get_tags(self): + d = self.get_inserted_msg() + d.addCallback(self._test_get_tags_cb) + return d + + def _test_get_tags_cb(self, msg): + self.assertTrue(msg is not None) + self.assertEquals(msg.get_tags(), self.msg_tags) + + +class MessageCollectionTestCase(SoledadTestMixin, CollectionMixin): + """ + Tests for the MessageCollection class. + """ + _mbox_uuid = None + + def assert_collection_count(self, _, expected): + def _assert_count(count): + self.assertEqual(count, expected) + + d = self.get_collection() + d.addCallback(lambda col: col.count()) + d.addCallback(_assert_count) + return d + + def add_msg_to_collection(self): + raw = _get_raw_msg() + + def add_msg_to_collection(collection): + # We keep the uuid in case we need to instantiate the same + # collection afterwards. + self._mbox_uuid = collection.mbox_uuid + d = collection.add_msg(raw, date=_get_msg_time()) + return d + + d = self.get_collection() + d.addCallback(add_msg_to_collection) + return d + + def test_is_mailbox_collection(self): + d = self.get_collection() + d.addCallback(self._test_is_mailbox_collection_cb) + return d + + def _test_is_mailbox_collection_cb(self, collection): + self.assertTrue(collection.is_mailbox_collection()) + + def test_get_uid_next(self): + d = self.add_msg_to_collection() + d.addCallback(lambda _: self.get_collection()) + d.addCallback(lambda col: col.get_uid_next()) + d.addCallback(self._test_get_uid_next_cb) + + def _test_get_uid_next_cb(self, next_uid): + self.assertEqual(next_uid, 2) + + def test_add_and_count_msg(self): + d = self.add_msg_to_collection() + d.addCallback(self._test_add_and_count_msg_cb) + return d + + def _test_add_and_count_msg_cb(self, _): + return partial(self.assert_collection_count, expected=1) + + def test_copy_msg(self): + # TODO ---- update when implementing messagecopier + # interface + pass + test_copy_msg.skip = "Not yet implemented" + + def test_delete_msg(self): + d = self.add_msg_to_collection() + + def del_msg(collection): + def _delete_it(msg): + self.assertTrue(msg is not None) + return collection.delete_msg(msg) + + d = collection.get_message_by_uid(1) + d.addCallback(_delete_it) + return d + + # We need to instantiate an mbox collection with the same uuid that + # the one in which we inserted the doc. + d.addCallback(lambda _: self.get_collection(mbox_uuid=self._mbox_uuid)) + d.addCallback(del_msg) + d.addCallback(self._test_delete_msg_cb) + return d + + def _test_delete_msg_cb(self, _): + return partial(self.assert_collection_count, expected=0) + + def test_update_flags(self): + d = self.add_msg_to_collection() + d.addCallback(self._test_update_flags_cb) + return d + + def _test_update_flags_cb(self, msg): + pass + + def test_update_tags(self): + d = self.add_msg_to_collection() + d.addCallback(self._test_update_tags_cb) + return d + + def _test_update_tags_cb(self, msg): + pass + + +class AccountTestCase(SoledadTestMixin): + """ + Tests for the Account class. + """ + def get_account(self, user_id): + store = self._soledad + return Account(store, user_id) + + def test_add_mailbox(self): + acc = self.get_account('some_user_id') + d = acc.callWhenReady(lambda _: acc.add_mailbox("TestMailbox")) + d.addCallback(lambda _: acc.list_all_mailbox_names()) + d.addCallback(self._test_add_mailbox_cb) + return d + + def _test_add_mailbox_cb(self, mboxes): + expected = ['INBOX', 'TestMailbox'] + self.assertItemsEqual(mboxes, expected) + + def test_delete_mailbox(self): + acc = self.get_account('some_user_id') + d = acc.callWhenReady(lambda _: acc.delete_mailbox("Inbox")) + d.addCallback(lambda _: acc.list_all_mailbox_names()) + d.addCallback(self._test_delete_mailbox_cb) + return d + + def _test_delete_mailbox_cb(self, mboxes): + expected = [] + self.assertItemsEqual(mboxes, expected) + + def test_rename_mailbox(self): + acc = self.get_account('some_user_id') + d = acc.callWhenReady(lambda _: acc.add_mailbox("OriginalMailbox")) + d.addCallback(lambda _: acc.rename_mailbox( + "OriginalMailbox", "RenamedMailbox")) + d.addCallback(lambda _: acc.list_all_mailbox_names()) + d.addCallback(self._test_rename_mailbox_cb) + return d + + def _test_rename_mailbox_cb(self, mboxes): + expected = ['INBOX', 'RenamedMailbox'] + self.assertItemsEqual(mboxes, expected) + + def test_get_all_mailboxes(self): + acc = self.get_account('some_user_id') + d = acc.callWhenReady(lambda _: acc.add_mailbox("OneMailbox")) + d.addCallback(lambda _: acc.add_mailbox("TwoMailbox")) + d.addCallback(lambda _: acc.add_mailbox("ThreeMailbox")) + d.addCallback(lambda _: acc.add_mailbox("anotherthing")) + d.addCallback(lambda _: acc.add_mailbox("anotherthing2")) + d.addCallback(lambda _: acc.get_all_mailboxes()) + d.addCallback(self._test_get_all_mailboxes_cb) + return d + + def _test_get_all_mailboxes_cb(self, mailboxes): + expected = ["INBOX", "OneMailbox", "TwoMailbox", "ThreeMailbox", + "anotherthing", "anotherthing2"] + names = [m.mbox for m in mailboxes] + self.assertItemsEqual(names, expected) + + def test_get_collection_by_mailbox(self): + acc = self.get_account('some_user_id') + d = acc.callWhenReady(lambda _: acc.get_collection_by_mailbox("INBOX")) + d.addCallback(self._test_get_collection_by_mailbox_cb) + return d + + def _test_get_collection_by_mailbox_cb(self, collection): + self.assertTrue(collection.is_mailbox_collection()) + + def assert_uid_next_empty_collection(uid): + self.assertEqual(uid, 1) + d = collection.get_uid_next() + d.addCallback(assert_uid_next_empty_collection) + return d + + def test_get_collection_by_docs(self): + pass + + test_get_collection_by_docs.skip = "Not yet implemented" + + def test_get_collection_by_tag(self): + pass + + test_get_collection_by_tag.skip = "Not yet implemented" diff --git a/src/leap/bitmask/mail/tests/test_mailbox_indexer.py b/src/leap/bitmask/mail/tests/test_mailbox_indexer.py new file mode 100644 index 0000000..5c1891d --- /dev/null +++ b/src/leap/bitmask/mail/tests/test_mailbox_indexer.py @@ -0,0 +1,250 @@ +# -*- coding: utf-8 -*- +# test_mailbox_indexer.py +# Copyright (C) 2014 LEAP +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . +""" +Tests for the mailbox_indexer module. +""" +import uuid +from functools import partial + +from leap.mail import mailbox_indexer as mi +from leap.mail.testing.common import SoledadTestMixin + +hash_test0 = '590c9f8430c7435807df8ba9a476e3f1295d46ef210f6efae2043a4c085a569e' +hash_test1 = '1b4f0e9851971998e732078544c96b36c3d01cedf7caa332359d6f1d83567014' +hash_test2 = '60303ae22b998861bce3b28f33eec1be758a213c86c93c076dbe9f558c11c752' +hash_test3 = 'fd61a03af4f77d870fc21e05e7e80678095c92d808cfb3b5c279ee04c74aca13' +hash_test4 = 'a4e624d686e03ed2767c0abd85c14426b0b1157d2ce81d27bb4fe4f6f01d688a' + + +def fmt_hash(mailbox_uuid, hash): + return "M-" + mailbox_uuid.replace('-', '_') + "-" + hash + +mbox_id = str(uuid.uuid4()) + + +class MailboxIndexerTestCase(SoledadTestMixin): + """ + Tests for the MailboxUID class. + """ + def get_mbox_uid(self): + m_uid = mi.MailboxIndexer(self._soledad) + return m_uid + + def list_mail_tables_cb(self, ignored): + def filter_mailuid_tables(tables): + filtered = [ + table[0] for table in tables if + table[0].startswith(mi.MailboxIndexer.table_preffix)] + return filtered + + sql = "SELECT name FROM sqlite_master WHERE type='table';" + d = self._soledad.raw_sqlcipher_query(sql) + d.addCallback(filter_mailuid_tables) + return d + + def select_uid_rows(self, mailbox): + sql = "SELECT * FROM %s%s;" % ( + mi.MailboxIndexer.table_preffix, mailbox.replace('-', '_')) + d = self._soledad.raw_sqlcipher_query(sql) + return d + + def test_create_table(self): + def assert_table_created(tables): + self.assertEqual( + tables, ["leapmail_uid_" + mbox_id.replace('-', '_')]) + + m_uid = self.get_mbox_uid() + d = m_uid.create_table(mbox_id) + d.addCallback(self.list_mail_tables_cb) + d.addCallback(assert_table_created) + return d + + def test_create_and_delete_table(self): + def assert_table_deleted(tables): + self.assertEqual(tables, []) + + m_uid = self.get_mbox_uid() + d = m_uid.create_table(mbox_id) + d.addCallback(lambda _: m_uid.delete_table(mbox_id)) + d.addCallback(self.list_mail_tables_cb) + d.addCallback(assert_table_deleted) + return d + + def test_insert_doc(self): + m_uid = self.get_mbox_uid() + + h1 = fmt_hash(mbox_id, hash_test0) + h2 = fmt_hash(mbox_id, hash_test1) + h3 = fmt_hash(mbox_id, hash_test2) + h4 = fmt_hash(mbox_id, hash_test3) + h5 = fmt_hash(mbox_id, hash_test4) + + def assert_uid_rows(rows): + expected = [(1, h1), (2, h2), (3, h3), (4, h4), (5, h5)] + self.assertEquals(rows, expected) + + d = m_uid.create_table(mbox_id) + d.addCallback(lambda _: m_uid.insert_doc(mbox_id, h1)) + d.addCallback(lambda _: m_uid.insert_doc(mbox_id, h2)) + d.addCallback(lambda _: m_uid.insert_doc(mbox_id, h3)) + d.addCallback(lambda _: m_uid.insert_doc(mbox_id, h4)) + d.addCallback(lambda _: m_uid.insert_doc(mbox_id, h5)) + d.addCallback(lambda _: self.select_uid_rows(mbox_id)) + d.addCallback(assert_uid_rows) + return d + + def test_insert_doc_return(self): + m_uid = self.get_mbox_uid() + + def assert_rowid(rowid, expected=None): + self.assertEqual(rowid, expected) + + h1 = fmt_hash(mbox_id, hash_test0) + h2 = fmt_hash(mbox_id, hash_test1) + h3 = fmt_hash(mbox_id, hash_test2) + + d = m_uid.create_table(mbox_id) + d.addCallback(lambda _: m_uid.insert_doc(mbox_id, h1)) + d.addCallback(partial(assert_rowid, expected=1)) + d.addCallback(lambda _: m_uid.insert_doc(mbox_id, h2)) + d.addCallback(partial(assert_rowid, expected=2)) + d.addCallback(lambda _: m_uid.insert_doc(mbox_id, h3)) + d.addCallback(partial(assert_rowid, expected=3)) + return d + + def test_delete_doc(self): + m_uid = self.get_mbox_uid() + + h1 = fmt_hash(mbox_id, hash_test0) + h2 = fmt_hash(mbox_id, hash_test1) + h3 = fmt_hash(mbox_id, hash_test2) + h4 = fmt_hash(mbox_id, hash_test3) + h5 = fmt_hash(mbox_id, hash_test4) + + def assert_uid_rows(rows): + expected = [(4, h4), (5, h5)] + self.assertEquals(rows, expected) + + d = m_uid.create_table(mbox_id) + d.addCallback(lambda _: m_uid.insert_doc(mbox_id, h1)) + d.addCallback(lambda _: m_uid.insert_doc(mbox_id, h2)) + d.addCallback(lambda _: m_uid.insert_doc(mbox_id, h3)) + d.addCallback(lambda _: m_uid.insert_doc(mbox_id, h4)) + d.addCallback(lambda _: m_uid.insert_doc(mbox_id, h5)) + + d.addCallbacks(lambda _: m_uid.delete_doc_by_uid(mbox_id, 1)) + d.addCallbacks(lambda _: m_uid.delete_doc_by_uid(mbox_id, 2)) + d.addCallbacks(lambda _: m_uid.delete_doc_by_hash(mbox_id, h3)) + + d.addCallback(lambda _: self.select_uid_rows(mbox_id)) + d.addCallback(assert_uid_rows) + return d + + def test_get_doc_id_from_uid(self): + m_uid = self.get_mbox_uid() + + h1 = fmt_hash(mbox_id, hash_test0) + + def assert_doc_hash(res): + self.assertEqual(res, h1) + + d = m_uid.create_table(mbox_id) + d.addCallback(lambda _: m_uid.insert_doc(mbox_id, h1)) + d.addCallback(lambda _: m_uid.get_doc_id_from_uid(mbox_id, 1)) + d.addCallback(assert_doc_hash) + return d + + def test_count(self): + m_uid = self.get_mbox_uid() + + h1 = fmt_hash(mbox_id, hash_test0) + h2 = fmt_hash(mbox_id, hash_test1) + h3 = fmt_hash(mbox_id, hash_test2) + h4 = fmt_hash(mbox_id, hash_test3) + h5 = fmt_hash(mbox_id, hash_test4) + + d = m_uid.create_table(mbox_id) + d.addCallback(lambda _: m_uid.insert_doc(mbox_id, h1)) + d.addCallback(lambda _: m_uid.insert_doc(mbox_id, h2)) + d.addCallback(lambda _: m_uid.insert_doc(mbox_id, h3)) + d.addCallback(lambda _: m_uid.insert_doc(mbox_id, h4)) + d.addCallback(lambda _: m_uid.insert_doc(mbox_id, h5)) + + def assert_count_after_inserts(count): + self.assertEquals(count, 5) + + d.addCallback(lambda _: m_uid.count(mbox_id)) + d.addCallback(assert_count_after_inserts) + + d.addCallbacks(lambda _: m_uid.delete_doc_by_uid(mbox_id, 1)) + d.addCallbacks(lambda _: m_uid.delete_doc_by_uid(mbox_id, 2)) + + def assert_count_after_deletions(count): + self.assertEquals(count, 3) + + d.addCallback(lambda _: m_uid.count(mbox_id)) + d.addCallback(assert_count_after_deletions) + return d + + def test_get_next_uid(self): + m_uid = self.get_mbox_uid() + + h1 = fmt_hash(mbox_id, hash_test0) + h2 = fmt_hash(mbox_id, hash_test1) + h3 = fmt_hash(mbox_id, hash_test2) + h4 = fmt_hash(mbox_id, hash_test3) + h5 = fmt_hash(mbox_id, hash_test4) + + d = m_uid.create_table(mbox_id) + d.addCallback(lambda _: m_uid.insert_doc(mbox_id, h1)) + d.addCallback(lambda _: m_uid.insert_doc(mbox_id, h2)) + d.addCallback(lambda _: m_uid.insert_doc(mbox_id, h3)) + d.addCallback(lambda _: m_uid.insert_doc(mbox_id, h4)) + d.addCallback(lambda _: m_uid.insert_doc(mbox_id, h5)) + + def assert_next_uid(result, expected=1): + self.assertEquals(result, expected) + + d.addCallback(lambda _: m_uid.get_next_uid(mbox_id)) + d.addCallback(partial(assert_next_uid, expected=6)) + return d + + def test_all_uid_iter(self): + + m_uid = self.get_mbox_uid() + + h1 = fmt_hash(mbox_id, hash_test0) + h2 = fmt_hash(mbox_id, hash_test1) + h3 = fmt_hash(mbox_id, hash_test2) + h4 = fmt_hash(mbox_id, hash_test3) + h5 = fmt_hash(mbox_id, hash_test4) + + d = m_uid.create_table(mbox_id) + d.addCallback(lambda _: m_uid.insert_doc(mbox_id, h1)) + d.addCallback(lambda _: m_uid.insert_doc(mbox_id, h2)) + d.addCallback(lambda _: m_uid.insert_doc(mbox_id, h3)) + d.addCallback(lambda _: m_uid.insert_doc(mbox_id, h4)) + d.addCallback(lambda _: m_uid.insert_doc(mbox_id, h5)) + d.addCallback(lambda _: m_uid.delete_doc_by_uid(mbox_id, 1)) + d.addCallback(lambda _: m_uid.delete_doc_by_uid(mbox_id, 4)) + + def assert_all_uid(result, expected=[2, 3, 5]): + self.assertEquals(result, expected) + + d.addCallback(lambda _: m_uid.all_uid_iter(mbox_id)) + d.addCallback(partial(assert_all_uid)) + return d diff --git a/src/leap/bitmask/mail/tests/test_walk.py b/src/leap/bitmask/mail/tests/test_walk.py new file mode 100644 index 0000000..826ec10 --- /dev/null +++ b/src/leap/bitmask/mail/tests/test_walk.py @@ -0,0 +1,81 @@ +""" +Tests for leap.mail.walk module +""" +import os.path +from email.parser import Parser + +from leap.mail import walk + +CORPUS = { + 'simple': 'rfc822.message', + 'multimin': 'rfc822.multi-minimal.message', + 'multisigned': 'rfc822.multi-signed.message', + 'bounced': 'rfc822.bounce.message', +} + +_here = os.path.dirname(__file__) +_parser = Parser() + + +# tests + + +def test_simple_mail(): + msg = _parse('simple') + tree = walk.get_tree(msg) + assert len(tree['part_map']) == 0 + assert tree['ctype'] == 'text/plain' + assert tree['multi'] is False + + +def test_multipart_minimal(): + msg = _parse('multimin') + tree = walk.get_tree(msg) + + assert tree['multi'] is True + assert len(tree['part_map']) == 1 + first = tree['part_map'][1] + assert first['multi'] is False + assert first['ctype'] == 'text/plain' + + +def test_multi_signed(): + msg = _parse('multisigned') + tree = walk.get_tree(msg) + assert tree['multi'] is True + assert len(tree['part_map']) == 2 + + _first = tree['part_map'][1] + _second = tree['part_map'][2] + assert len(_first['part_map']) == 3 + assert(_second['multi'] is False) + + +def test_bounce_mime(): + msg = _parse('bounced') + tree = walk.get_tree(msg) + + ctypes = [tree['part_map'][index]['ctype'] + for index in sorted(tree['part_map'].keys())] + third = tree['part_map'][3] + three_one_ctype = third['part_map'][1]['ctype'] + assert three_one_ctype == 'multipart/signed' + + assert ctypes == [ + 'text/plain', + 'message/delivery-status', + 'message/rfc822'] + + +# utils + +def _parse(name): + _str = _get_string_for_message(name) + return _parser.parsestr(_str) + + +def _get_string_for_message(name): + filename = os.path.join(_here, CORPUS[name]) + with open(filename) as f: + msgstr = f.read() + return msgstr diff --git a/src/leap/bitmask/mail/utils.py b/src/leap/bitmask/mail/utils.py new file mode 100644 index 0000000..64fca98 --- /dev/null +++ b/src/leap/bitmask/mail/utils.py @@ -0,0 +1,375 @@ +# -*- coding: utf-8 -*- +# utils.py +# Copyright (C) 2013 LEAP +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . +""" +Mail utilities. +""" +from email.utils import parseaddr +import json +import re +import traceback +import Queue + +from leap.soledad.common.document import SoledadDocument +from leap.common.check import leap_assert_type +from twisted.mail import smtp + + +CHARSET_PATTERN = r"""charset=([\w-]+)""" +CHARSET_RE = re.compile(CHARSET_PATTERN, re.IGNORECASE) + + +def first(things): + """ + Return the head of a collection. + """ + try: + return things[0] + except (IndexError, TypeError): + return None + + +def empty(thing): + """ + Return True if a thing is None or its length is zero. + If thing is a number (int, float, long), return False. + """ + if thing is None: + return True + if isinstance(thing, (int, float, long)): + return False + if isinstance(thing, SoledadDocument): + thing = thing.content + try: + return len(thing) == 0 + except (ReferenceError, TypeError): + return True + + +def maybe_call(thing): + """ + Return the same thing, or the result of its invocation if it is a + callable. + """ + return thing() if callable(thing) else thing + + +def find_charset(thing, default=None): + """ + Looks into the object 'thing' for a charset specification. + It searchs into the object's `repr`. + + :param thing: the object to look into. + :type thing: object + :param default: the dafault charset to return if no charset is found. + :type default: str + + :return: the charset or 'default' + :rtype: str or None + """ + charset = first(CHARSET_RE.findall(repr(thing))) + if charset is None: + charset = default + return charset + + +def lowerdict(_dict): + """ + Return a dict with the keys in lowercase. + + :param _dict: the dict to convert + :rtype: dict + """ + # TODO should properly implement a CaseInsensitive dict. + # Look into requests code. + return dict((key.lower(), value) + for key, value in _dict.items()) + + +PART_MAP = "part_map" +PHASH = "phash" + + +def _str_dict(d, k): + """ + Convert the dictionary key to string if it was a string. + + :param d: the dict + :type d: dict + :param k: the key + :type k: object + """ + if isinstance(k, int): + val = d[k] + d[str(k)] = val + del(d[k]) + + +def stringify_parts_map(d): + """ + Modify a dictionary making all the nested dicts under "part_map" keys + having strings as keys. + + :param d: the dictionary to modify + :type d: dictionary + :rtype: dictionary + """ + for k in d: + if k == PART_MAP: + pmap = d[k] + for kk in pmap.keys(): + _str_dict(d[k], kk) + for kk in pmap.keys(): + stringify_parts_map(d[k][str(kk)]) + return d + + +def phash_iter(d): + """ + A recursive generator that extracts all the payload-hashes + from an arbitrary nested parts-map dictionary. + + :param d: the dictionary to walk + :type d: dictionary + :return: a list of all the phashes found + :rtype: list + """ + if PHASH in d: + yield d[PHASH] + if PART_MAP in d: + for key in d[PART_MAP]: + for phash in phash_iter(d[PART_MAP][key]): + yield phash + + +def accumulator(fun, lim): + """ + A simple accumulator that uses a closure and a mutable + object to collect items. + When the count of items is greater than `lim`, the + collection is flushed after invoking a map of the function `fun` + over it. + + The returned accumulator can also be flushed at any moment + by passing a boolean as a second parameter. + + :param fun: the function to call over the collection + when its size is greater than `lim` + :type fun: callable + :param lim: the turning point for the collection + :type lim: int + :rtype: function + + >>> from pprint import pprint + >>> acc = accumulator(pprint, 2) + >>> acc(1) + >>> acc(2) + [1, 2] + >>> acc(3) + >>> acc(4) + [3, 4] + >>> acc = accumulator(pprint, 5) + >>> acc(1) + >>> acc(2) + >>> acc(3) + >>> acc(None, flush=True) + [1,2,3] + """ + KEY = "items" + _o = {KEY: []} + + def _accumulator(item, flush=False): + collection = _o[KEY] + collection.append(item) + if len(collection) >= lim or flush: + map(fun, filter(None, collection)) + _o[KEY] = [] + + return _accumulator + + +def accumulator_queue(fun, lim): + """ + A version of the accumulator that uses a queue. + + When the count of items is greater than `lim`, the + queue is flushed after invoking the function `fun` + over its items. + + The returned accumulator can also be flushed at any moment + by passing a boolean as a second parameter. + + :param fun: the function to call over the collection + when its size is greater than `lim` + :type fun: callable + :param lim: the turning point for the collection + :type lim: int + :rtype: function + """ + _q = Queue.Queue() + + def _accumulator(item, flush=False): + _q.put(item) + if _q.qsize() >= lim or flush: + collection = [_q.get() for i in range(_q.qsize())] + map(fun, filter(None, collection)) + + return _accumulator + + +def validate_address(address): + """ + Validate C{address} as defined in RFC 2822. + + :param address: The address to be validated. + :type address: str + + @return: A valid address. + @rtype: str + + @raise smtp.SMTPBadRcpt: Raised if C{address} is invalid. + """ + leap_assert_type(address, str) + # in the following, the address is parsed as described in RFC 2822 and + # ('', '') is returned if the parse fails. + _, address = parseaddr(address) + if address == '': + raise smtp.SMTPBadRcpt(address) + return address + +# +# String manipulation +# + + +class CustomJsonScanner(object): + """ + This class is a context manager definition used to monkey patch the default + json string parsing behavior. + The emails can have more than one encoding, so the `str` objects have more + than one encoding and json does not support direct work with `str` + (only `unicode`). + """ + + def _parse_string_str(self, s, idx, *args, **kwargs): + """ + Parses the string "s" starting at the point idx and returns an `str` + object. Which basically means it works exactly the same as the regular + JSON string parsing, except that it doesn't try to decode utf8. + We need this because mail raw strings might have bytes in multiple + encodings. + + :param s: the string we want to parse + :type s: str + :param idx: the starting point for parsing + :type idx: int + + :returns: the parsed string and the index where the + string ends. + :rtype: tuple (str, int) + """ + # NOTE: we just want to use this monkey patched version if we are + # calling the loads from our custom method. Otherwise, we use the + # json's default parser. + monkey_patched = False + for i in traceback.extract_stack(): + # look for json_loads method in the call stack + if i[2] == json_loads.__name__: + monkey_patched = True + break + + if not monkey_patched: + return self._orig_scanstring(s, idx, *args, **kwargs) + + # TODO profile to see if a compiled regex can get us some + # benefit here. + found = False + end = s.find("\"", idx) + while not found: + try: + if s[end - 1] != "\\": + found = True + else: + end = s.find("\"", end + 1) + except Exception: + found = True + return s[idx:end].decode("string-escape"), end + 1 + + def __enter__(self): + """ + Replace the json methods with the needed ones. + Also make a backup to restore them later. + """ + # backup original values + self._orig_make_scanner = json.scanner.make_scanner + self._orig_scanstring = json.decoder.scanstring + + # We need the make_scanner function to be the python one so we can + # monkey_patch the json string parsing + json.scanner.make_scanner = json.scanner.py_make_scanner + + # And now we monkey patch the money method + json.decoder.scanstring = self._parse_string_str + + def __exit__(self, exc_type, exc_value, traceback): + """ + Restores the backuped methods. + """ + # restore original values + json.scanner.make_scanner = self._orig_make_scanner + json.decoder.scanstring = self._orig_scanstring + + +def json_loads(data): + """ + It works as json.loads but supporting multiple encodings in the same + string and accepting an `str` parameter that won't be converted to unicode. + + :param data: the string to load the objects from + :type data: str + + :returns: the corresponding python object result of parsing 'data', this + behaves similarly as json.loads, with the exception of that + returns always `str` instead of `unicode`. + """ + obj = None + with CustomJsonScanner(): + # We need to use the cls parameter in order to trigger the code + # that will let us control the string parsing method. + obj = json.loads(data, cls=json.JSONDecoder) + + return obj + + +class CaseInsensitiveDict(dict): + """ + A dictionary subclass that will allow case-insenstive key lookups. + """ + def __init__(self, d=None): + if d is None: + d = [] + if isinstance(d, dict): + for key, value in d.items(): + self[key] = value + else: + for key, value in d: + self[key] = value + + def __setitem__(self, key, value): + super(CaseInsensitiveDict, self).__setitem__(key.lower(), value) + + def __getitem__(self, key): + return super(CaseInsensitiveDict, self).__getitem__(key.lower()) diff --git a/src/leap/bitmask/mail/walk.py b/src/leap/bitmask/mail/walk.py new file mode 100644 index 0000000..d143d61 --- /dev/null +++ b/src/leap/bitmask/mail/walk.py @@ -0,0 +1,107 @@ +# -*- coding: utf-8 -*- +# walk.py +# Copyright (C) 2013-2015 LEAP +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . +""" +Walk a message tree and generate documents that can be inserted in the backend +store. +""" +from email.parser import Parser + +from cryptography.hazmat.backends.multibackend import MultiBackend +from cryptography.hazmat.backends.openssl.backend import ( + Backend as OpenSSLBackend) +from cryptography.hazmat.primitives import hashes + +from leap.mail.utils import first + +crypto_backend = MultiBackend([OpenSSLBackend()]) + +_parser = Parser() + + +def get_tree(msg): + p = {} + p['ctype'] = msg.get_content_type() + p['headers'] = msg.items() + + payload = msg.get_payload() + is_multi = msg.is_multipart() + if is_multi: + p['part_map'] = dict( + [(idx, get_tree(part)) for idx, part in enumerate(payload, 1)]) + p['parts'] = len(payload) + p['phash'] = None + else: + p['parts'] = 0 + p['size'] = len(payload) + p['phash'] = get_hash(payload) + p['part_map'] = {} + p['multi'] = is_multi + return p + + +def get_tree_from_string(messagestr): + return get_tree(_parser.parsestr(messagestr)) + + +def get_body_phash(msg): + """ + Find the body payload-hash for this message. + """ + for part in msg.walk(): + # XXX what other ctypes should be considered body? + if part.get_content_type() in ("text/plain", "text/html"): + # XXX avoid hashing again + return get_hash(part.get_payload()) + + +def get_raw_docs(msg): + """ + We get also some of the headers to be able to + index the content. Here we remove any mutable part, as the the filename + in the content disposition. + """ + return ( + {'type': 'cnt', + 'raw': part.get_payload(), + 'phash': get_hash(part.get_payload()), + 'content-type': part.get_content_type(), + 'content-disposition': first(part.get( + 'content-disposition', '').split(';')), + 'content-transfer-encoding': part.get( + 'content-transfer-encoding', '') + } for part in msg.walk() if not isinstance(part.get_payload(), list)) + + +def get_hash(s): + digest = hashes.Hash(hashes.SHA256(), crypto_backend) + digest.update(s) + return digest.finalize().encode("hex").upper() + + +""" +Groucho Marx: Now pay particular attention to this first clause, because it's + most important. There's the party of the first part shall be + known in this contract as the party of the first part. How do you + like that, that's pretty neat eh? + +Chico Marx: No, that's no good. +Groucho Marx: What's the matter with it? + +Chico Marx: I don't know, let's hear it again. +Groucho Marx: So the party of the first part shall be known in this contract as + the party of the first part. +""" -- cgit v1.2.3