diff options
Diffstat (limited to 'mail')
126 files changed, 21710 insertions, 0 deletions
diff --git a/mail/.gitattributes b/mail/.gitattributes new file mode 100644 index 00000000..5041731c --- /dev/null +++ b/mail/.gitattributes @@ -0,0 +1 @@ +src/leap/mail/_version.py export-subst diff --git a/mail/.gitignore b/mail/.gitignore new file mode 100644 index 00000000..aafbdd13 --- /dev/null +++ b/mail/.gitignore @@ -0,0 +1,25 @@ +*.pyc +dropin.cache +build/ +dist/ +*.egg +*.egg-info +src/_trial_temp +*.swp +*.swo +.* +!.coveragerc +!.tx +!.gitignore +!.gitattributes +bin/ +core +docs/_build +include/ +lib/ +local/ +share/ +MANIFEST +twistd.pid +_trial_temp +_trial_temp.lock diff --git a/mail/AUTHORS b/mail/AUTHORS new file mode 100644 index 00000000..db070f4d --- /dev/null +++ b/mail/AUTHORS @@ -0,0 +1,8 @@ +Kali Kaneko <kali@leap.se> +Tomás Touceda <chiiph@leap.se> +drebs <drebs@leap.se> +Ivan Alejandro <ivanalejandro0@gmail.com> +Ruben Pollan <meskio@sindominio.net> +Bruno Wagner Goncalves <bwagner@thoughtworks.com> +Duda Dornelles <ddornell@thoughtworks.com> +Folker Bernitt <fbernitt@thoughtworks.com> diff --git a/mail/CHANGELOG.rst b/mail/CHANGELOG.rst new file mode 100644 index 00000000..22abf225 --- /dev/null +++ b/mail/CHANGELOG.rst @@ -0,0 +1,37 @@ +0.4.2 - 13 May, 2016 ++++++++++++++++++++++ + +Bugfixes +~~~~~~~~ +- `#8083 <https://leap.se/code/issues/8083>`_: Allow pixelated UA not interfere with Thunderbird operation. +- Cast local identity (version string) to bytes, avoid TLS transport raising exception. + + +0.4.1 - 18 Apr, 2016 ++++++++++++++++++++++ + +Features +~~~~~~~~ +- `#7656 <https://leap.se/code/issues/7656>`_: Emit multi-user aware events. +- `#4008 <https://leap.se/code/issues/4008>`_: Add token-based authentication to local IMAP/SMTP services. +- `#7889 <https://leap.se/code/issues/7889>`_: Use cryptography instead of pycryptopp to reduce dependencies. +- `#7263 <https://leap.se/code/issues/7263>`_: Implement local bounces to notify user of SMTP delivery errors. +- Use twisted.cred to authenticate IMAP/SMTP users. +- Verify plain text signed email. +- Validate signature with attachments. +- Use fingerprint instead of key_id to address keys. + + +Bugfixes +~~~~~~~~ +- `#7861 <https://leap.se/code/issues/7861>`_: Use the right succeed function for passthrough encrypted email. +- `#7898 <https://leap.se/code/issues/7898>`_: Fix IMAP fetch headers +- `#7977 <https://leap.se/code/issues/7977>`_: Decode attached keys so they are recognized by keymanager. +- `#7952 <https://leap.se/code/issues/7952>`_: Specify openssl backend explicitely. +- Fix the get_body logic for corner-cases in which body is None (yet-to-be synced docs, mainly). +- Let the inbox used in IncomingMail notify any subscribed Mailbox. +- Adds user_id to Account (fixes Pixelated mail leakage). + +Misc +~~~~ +- Change IMAPAccount signature, for consistency with a previous Account change. diff --git a/mail/HISTORY b/mail/HISTORY new file mode 100644 index 00000000..6ca54e7f --- /dev/null +++ b/mail/HISTORY @@ -0,0 +1,179 @@ +0.4.0 Oct 28, 2015: + o Expose generic and protocol-agnostic public mail API. + o Make use of the twisted-based, async soledad API. + o Create a OutgoingMail class that has the logic for encrypting, signing and + sending messages. Factors that logic out of EncryptedMessage so it can be + used by other clients. Closes: #6357. + o Refactor email fetching outside IMAP to its own independient IncomingMail + class. Closes: #6361. + o Adapt to new events api on leap.common. Related to #5359. + o Discover public keys via attachment. Closes: #5937. + o Add public key as attachment. Closes: #6617. + o Parse OpenPGP header and import keys from it. Closes: #3879. + o Don't add any footer to the emails. Closes: #4692. + o Add listener for each email added to inbox in IncomingMail. Closes: #6742. + o Ability to reindex local UIDs after a soledad sync. Closes: #6996. + o Feature: add very basic support for message sequence numbers. + o Send a BYE command to all open connections, so that the MUA is notified + when the server is shutted down. + o Fix nested multipart rendering. Closes: #7244 + o Update SMTP gateway docs. Closes #7169. + o Bugfix: fix keyerror when inserting msg on pending_inserts dict. + o Bugfix: Return the first cdoc if no body found + o Lots of style fixes and tests updates. + o If the auth token has expired signal the GUI to request her to log in again + (Closes: #7430) + o don't extract openpgp header if valid attached key (Closes: #7480) + o disable local only tcp bind on docker containers to allow access to IMAP + and SMTP. Related to #7471. + +0.3.10 Sept 26, 2014: + o MessageCollection iterator now creates the LeapMessage with the + collection reference, so setFlags will work properly. + o account#addMailbox can't allow empty mailbox names since it makes + it impossible to create it later (mailbox#__init__ will throw an + error), which makes it impossible to getMailbox or even delete it. + +0.3.9 Apr 4, 2014: + o Footer url shouldn't end in period. Closes #4791. + o Handle non-ascii headers. Closes #5021. + o Soledad writer consumes messages eagerly. Fixes failing + tests. Closes #4715. + o Convert unicode to str when raising exceptions in IMAP server. + Fixes #4830. + o Remove conversion of IMAP folder names to string. This makes the + IMAP server use twisted's transparent 7bit conversion. Fixes + #4830. + o Add a flag to be able to reset the session. Closes #4925. + o Check for none in payload detection. Closes #4933. + o Check for flags doc uniqueness before adding a message. Avoids + duplicates of a single message in the same mailbox while copying + or moving. Closes #4949. + o Correctly process attachments when signing. Fixes #5014. + o Fix bug in which destination folder sometimes was not showing + messages after copy/append. Closes #5167. + o Fix unread notifications to client UI. Only INBOX is + notified. Closes #5177. + o Fix bug in which deleted folder wouldn't show its messages + inside. Closes #5179. + o Keep processing after a decryption error. Closes #5307. + o Enqueue unsetting of recent flag. this was holding the new mails + from being displayed soonish. + o Properly parse emails crafted by Mail.app. Fixes #5013. + o Restrict adding outgoing footer to text/plain messages. + o Sanity check on last_uid setter. Avoids incomplete fetches. + o Stop providing hostname for helo in smtp gateway. Fixes #4335. + o Only try to fetch keys for multipart signed or encrypted emails. + Fixes #4671. + o Add a flag for offline mode in imap. Related to #4943. + o Flush IMAP data to disk when stopping. Closes #5095. + o Signal the client when auth token is invalid for syncing Soledad. + Fixes #5191. + o Ability to support SEARCH Commands, limited to HEADER Message-ID. + This is a quick workaround for avoiding duplicate saves in Drafts + Folder. Closes #4209. + o Use a memory store as write-buffer and read-cache. + o Implement IMAP4 non-synchronizing literals (rfc2088), so APPENDs + can be made in a single round-trip. Closes #5190. + o Defer costly operations to a pool of threads. + o Split the internal representation of messages into three distinct + documents: 1) Flags 2) Headers 3) Content. + o Make use of the Twisted MIME interface. + o Add deduplication ability to the save operation, for body and + attachments. + o Add IMessageCopier interface to mailbox implementation, so bulk + moves are costless. Closes #4654. + o Makes efficient use of indexes and count method. Closes #4616. + o Handle correctly unicode characters in emails. Closes #4838. + +0.3.8 Dec 6, 2013: + o Fail gracefully when failing to decrypt incoming messages. Closes + #4589. + o Fix a bug when adding a message with empty flags. Closes #4496 + o Allow to iterate in an empty mailbox during fetch. Closes #4603 + o Add 'signencrypt' preference to OpenPGP header on outgoing + email. Closes #3878. + o Add a header to incoming emails that reflects if a valid signature + was found when decrypting. Closes #4354. + o Add a footer to outgoing email pointing to the address where + sender keys can be fetched. Closes #4526. + o Serialize Soledad Writes for new messages. Fixes segmentation + fault when sqlcipher was been concurrently accessed from many + threads. Closes #4606 + o Set remote mail polling time to 60 seconds. Closes #4499 + +0.3.7 Nov 15, 2013: + o Uses deferToThread for sendMail. Closes #3937 + o Update pkey to allow multiple accounts. Solves: #4394 + o Change SMTP service name from "relay" to "gateway". Closes #4416. + o Identify ourselves with a fqdn, always. Closes: #4441 + o Remove 'multipart/encrypted' header after decrypting incoming + mail. Closes #4454. + o Fix several bugs with imap mailbox getUIDNext and notifiers that + were breaking the mail indexing after message deletion. This + solves also the perceived mismatch between the number of unread + mails reported by bitmask_client and the number reported by + MUAs. Closes: #4461 + o Check username in authentications. Closes: #4299 + o Reject senders that aren't the user that is currently logged + in. Fixes #3952. + o Prevent already encrypted outgoing messages from being encrypted + again. Closes #4324. + o Correctly handle email headers when gatewaying messages. Also add + OpenPGP header. Closes #4322 and #4447. + +0.3.6 Nov 1, 2013: + o Add support for non-ascii characters in emails. Closes #4000. + o Default to UTF-8 when there is no charset parsed from the mail + contents. + o Refactor get_email_charset to leap.common. + o Return the necessary references (factory, port) from IMAP4 launch + in order to be able to properly stop it. Related to #4199. + o Notify MUA of new mail, using IDLE as advertised. Closes: #3671 + o Use TLS wrapper mode instead of STARTTLS. Closes #3637. + +0.3.5 Oct 18, 2013: + o Do not log mail doc contents. + o Comply with RFC 3156. Closes #4029. + +0.3.4 Oct 4, 2013: + o Improve charset handling when exposing mails to the mail + client. Related to #3660. + o Return Twisted's smtp Port object to be able to stop listening to + it whenever we want. Related to #3873. + +0.3.3 Sep 20, 2013: + o Remove cleartext mail from logs. Closes: #3877. + +0.3.2 Sep 6, 2013: + o Make mail services bind to 127.0.0.1. Closes: #3627. + o Signal unread message to UI when message is saved locally. Closes: + #3654. + o Signal unread to UI when flag in message change. Closes: #3662. + o Use dirspec instead of plain xdg. Closes #3574. + o SMTP service invocation returns factory instance. + +0.3.1 Aug 23, 2013: + o Avoid logging dummy password on imap server. Closes: #3416 + o Do not fail while processing an empty mail, just skip it. Fixes + #3457. + o Notify of unread email explicitly every time the mailbox is + sync'ed. + o Fix signals to emit only string in the contents instead of bool or + int values. + o Improve unseen filter of email. + o Make default imap fetch period 5 minutes. Client can config it via + environment variable for debug. Closes: #3409 + o Refactor imap fetch code for better defer handling. Closes: #3423 + o Emit signals to notify UI for SMTP relay events. Closes #3464. + o Add events for notifications about imap activity. Closes: #3480 + o Update to new soledad package scheme (common, client and + server). Closes #3487. + o Improve packaging: add versioneer, parse_requirements, + classifiers. + +0.3.0 Aug 9, 2013: + o Add dependency for leap.keymanager. + o User 1984 default port for imap. + o Add client certificate authentication. Closes #3376. + o SMTP relay signs outgoing messages. diff --git a/mail/LICENSE b/mail/LICENSE new file mode 100644 index 00000000..bc08fe2e --- /dev/null +++ b/mail/LICENSE @@ -0,0 +1,619 @@ + GNU GENERAL PUBLIC LICENSE + Version 3, 29 June 2007 + + Copyright (C) 2007 Free Software Foundation, Inc. <http://fsf.org/> + Everyone is permitted to copy and distribute verbatim copies + of this license document, but changing it is not allowed. + + Preamble + + The GNU General Public License is a free, copyleft license for +software and other kinds of works. + + The licenses for most software and other practical works are designed +to take away your freedom to share and change the works. By contrast, +the GNU General Public License is intended to guarantee your freedom to +share and change all versions of a program--to make sure it remains free +software for all its users. We, the Free Software Foundation, use the +GNU General Public License for most of our software; it applies also to +any other work released this way by its authors. You can apply it to +your programs, too. + + When we speak of free software, we are referring to freedom, not +price. Our General Public Licenses are designed to make sure that you +have the freedom to distribute copies of free software (and charge for +them if you wish), that you receive source code or can get it if you +want it, that you can change the software or use pieces of it in new +free programs, and that you know you can do these things. + + To protect your rights, we need to prevent others from denying you +these rights or asking you to surrender the rights. Therefore, you have +certain responsibilities if you distribute copies of the software, or if +you modify it: responsibilities to respect the freedom of others. + + For example, if you distribute copies of such a program, whether +gratis or for a fee, you must pass on to the recipients the same +freedoms that you received. You must make sure that they, too, receive +or can get the source code. And you must show them these terms so they +know their rights. + + Developers that use the GNU GPL protect your rights with two steps: +(1) assert copyright on the software, and (2) offer you this License +giving you legal permission to copy, distribute and/or modify it. + + For the developers' and authors' protection, the GPL clearly explains +that there is no warranty for this free software. For both users' and +authors' sake, the GPL requires that modified versions be marked as +changed, so that their problems will not be attributed erroneously to +authors of previous versions. + + Some devices are designed to deny users access to install or run +modified versions of the software inside them, although the manufacturer +can do so. This is fundamentally incompatible with the aim of +protecting users' freedom to change the software. The systematic +pattern of such abuse occurs in the area of products for individuals to +use, which is precisely where it is most unacceptable. Therefore, we +have designed this version of the GPL to prohibit the practice for those +products. If such problems arise substantially in other domains, we +stand ready to extend this provision to those domains in future versions +of the GPL, as needed to protect the freedom of users. + + Finally, every program is threatened constantly by software patents. +States should not allow patents to restrict development and use of +software on general-purpose computers, but in those that do, we wish to +avoid the special danger that patents applied to a free program could +make it effectively proprietary. To prevent this, the GPL assures that +patents cannot be used to render the program non-free. + + The precise terms and conditions for copying, distribution and +modification follow. + + TERMS AND CONDITIONS + + 0. Definitions. + + "This License" refers to version 3 of the GNU General Public License. + + "Copyright" also means copyright-like laws that apply to other kinds of +works, such as semiconductor masks. + + "The Program" refers to any copyrightable work licensed under this +License. Each licensee is addressed as "you". "Licensees" and +"recipients" may be individuals or organizations. + + To "modify" a work means to copy from or adapt all or part of the work +in a fashion requiring copyright permission, other than the making of an +exact copy. The resulting work is called a "modified version" of the +earlier work or a work "based on" the earlier work. + + A "covered work" means either the unmodified Program or a work based +on the Program. + + To "propagate" a work means to do anything with it that, without +permission, would make you directly or secondarily liable for +infringement under applicable copyright law, except executing it on a +computer or modifying a private copy. Propagation includes copying, +distribution (with or without modification), making available to the +public, and in some countries other activities as well. + + To "convey" a work means any kind of propagation that enables other +parties to make or receive copies. Mere interaction with a user through +a computer network, with no transfer of a copy, is not conveying. + + An interactive user interface displays "Appropriate Legal Notices" +to the extent that it includes a convenient and prominently visible +feature that (1) displays an appropriate copyright notice, and (2) +tells the user that there is no warranty for the work (except to the +extent that warranties are provided), that licensees may convey the +work under this License, and how to view a copy of this License. If +the interface presents a list of user commands or options, such as a +menu, a prominent item in the list meets this criterion. + + 1. Source Code. + + The "source code" for a work means the preferred form of the work +for making modifications to it. "Object code" means any non-source +form of a work. + + A "Standard Interface" means an interface that either is an official +standard defined by a recognized standards body, or, in the case of +interfaces specified for a particular programming language, one that +is widely used among developers working in that language. + + The "System Libraries" of an executable work include anything, other +than the work as a whole, that (a) is included in the normal form of +packaging a Major Component, but which is not part of that Major +Component, and (b) serves only to enable use of the work with that +Major Component, or to implement a Standard Interface for which an +implementation is available to the public in source code form. A +"Major Component", in this context, means a major essential component +(kernel, window system, and so on) of the specific operating system +(if any) on which the executable work runs, or a compiler used to +produce the work, or an object code interpreter used to run it. + + The "Corresponding Source" for a work in object code form means all +the source code needed to generate, install, and (for an executable +work) run the object code and to modify the work, including scripts to +control those activities. However, it does not include the work's +System Libraries, or general-purpose tools or generally available free +programs which are used unmodified in performing those activities but +which are not part of the work. For example, Corresponding Source +includes interface definition files associated with source files for +the work, and the source code for shared libraries and dynamically +linked subprograms that the work is specifically designed to require, +such as by intimate data communication or control flow between those +subprograms and other parts of the work. + + The Corresponding Source need not include anything that users +can regenerate automatically from other parts of the Corresponding +Source. + + The Corresponding Source for a work in source code form is that +same work. + + 2. Basic Permissions. + + All rights granted under this License are granted for the term of +copyright on the Program, and are irrevocable provided the stated +conditions are met. This License explicitly affirms your unlimited +permission to run the unmodified Program. The output from running a +covered work is covered by this License only if the output, given its +content, constitutes a covered work. This License acknowledges your +rights of fair use or other equivalent, as provided by copyright law. + + You may make, run and propagate covered works that you do not +convey, without conditions so long as your license otherwise remains +in force. You may convey covered works to others for the sole purpose +of having them make modifications exclusively for you, or provide you +with facilities for running those works, provided that you comply with +the terms of this License in conveying all material for which you do +not control copyright. Those thus making or running the covered works +for you must do so exclusively on your behalf, under your direction +and control, on terms that prohibit them from making any copies of +your copyrighted material outside their relationship with you. + + Conveying under any other circumstances is permitted solely under +the conditions stated below. Sublicensing is not allowed; section 10 +makes it unnecessary. + + 3. Protecting Users' Legal Rights From Anti-Circumvention Law. + + No covered work shall be deemed part of an effective technological +measure under any applicable law fulfilling obligations under article +11 of the WIPO copyright treaty adopted on 20 December 1996, or +similar laws prohibiting or restricting circumvention of such +measures. + + When you convey a covered work, you waive any legal power to forbid +circumvention of technological measures to the extent such circumvention +is effected by exercising rights under this License with respect to +the covered work, and you disclaim any intention to limit operation or +modification of the work as a means of enforcing, against the work's +users, your or third parties' legal rights to forbid circumvention of +technological measures. + + 4. Conveying Verbatim Copies. + + You may convey verbatim copies of the Program's source code as you +receive it, in any medium, provided that you conspicuously and +appropriately publish on each copy an appropriate copyright notice; +keep intact all notices stating that this License and any +non-permissive terms added in accord with section 7 apply to the code; +keep intact all notices of the absence of any warranty; and give all +recipients a copy of this License along with the Program. + + You may charge any price or no price for each copy that you convey, +and you may offer support or warranty protection for a fee. + + 5. Conveying Modified Source Versions. + + You may convey a work based on the Program, or the modifications to +produce it from the Program, in the form of source code under the +terms of section 4, provided that you also meet all of these conditions: + + a) The work must carry prominent notices stating that you modified + it, and giving a relevant date. + + b) The work must carry prominent notices stating that it is + released under this License and any conditions added under section + 7. This requirement modifies the requirement in section 4 to + "keep intact all notices". + + c) You must license the entire work, as a whole, under this + License to anyone who comes into possession of a copy. This + License will therefore apply, along with any applicable section 7 + additional terms, to the whole of the work, and all its parts, + regardless of how they are packaged. This License gives no + permission to license the work in any other way, but it does not + invalidate such permission if you have separately received it. + + d) If the work has interactive user interfaces, each must display + Appropriate Legal Notices; however, if the Program has interactive + interfaces that do not display Appropriate Legal Notices, your + work need not make them do so. + + A compilation of a covered work with other separate and independent +works, which are not by their nature extensions of the covered work, +and which are not combined with it such as to form a larger program, +in or on a volume of a storage or distribution medium, is called an +"aggregate" if the compilation and its resulting copyright are not +used to limit the access or legal rights of the compilation's users +beyond what the individual works permit. Inclusion of a covered work +in an aggregate does not cause this License to apply to the other +parts of the aggregate. + + 6. Conveying Non-Source Forms. + + You may convey a covered work in object code form under the terms +of sections 4 and 5, provided that you also convey the +machine-readable Corresponding Source under the terms of this License, +in one of these ways: + + a) Convey the object code in, or embodied in, a physical product + (including a physical distribution medium), accompanied by the + Corresponding Source fixed on a durable physical medium + customarily used for software interchange. + + b) Convey the object code in, or embodied in, a physical product + (including a physical distribution medium), accompanied by a + written offer, valid for at least three years and valid for as + long as you offer spare parts or customer support for that product + model, to give anyone who possesses the object code either (1) a + copy of the Corresponding Source for all the software in the + product that is covered by this License, on a durable physical + medium customarily used for software interchange, for a price no + more than your reasonable cost of physically performing this + conveying of source, or (2) access to copy the + Corresponding Source from a network server at no charge. + + c) Convey individual copies of the object code with a copy of the + written offer to provide the Corresponding Source. This + alternative is allowed only occasionally and noncommercially, and + only if you received the object code with such an offer, in accord + with subsection 6b. + + d) Convey the object code by offering access from a designated + place (gratis or for a charge), and offer equivalent access to the + Corresponding Source in the same way through the same place at no + further charge. You need not require recipients to copy the + Corresponding Source along with the object code. If the place to + copy the object code is a network server, the Corresponding Source + may be on a different server (operated by you or a third party) + that supports equivalent copying facilities, provided you maintain + clear directions next to the object code saying where to find the + Corresponding Source. Regardless of what server hosts the + Corresponding Source, you remain obligated to ensure that it is + available for as long as needed to satisfy these requirements. + + e) Convey the object code using peer-to-peer transmission, provided + you inform other peers where the object code and Corresponding + Source of the work are being offered to the general public at no + charge under subsection 6d. + + A separable portion of the object code, whose source code is excluded +from the Corresponding Source as a System Library, need not be +included in conveying the object code work. + + A "User Product" is either (1) a "consumer product", which means any +tangible personal property which is normally used for personal, family, +or household purposes, or (2) anything designed or sold for incorporation +into a dwelling. In determining whether a product is a consumer product, +doubtful cases shall be resolved in favor of coverage. For a particular +product received by a particular user, "normally used" refers to a +typical or common use of that class of product, regardless of the status +of the particular user or of the way in which the particular user +actually uses, or expects or is expected to use, the product. A product +is a consumer product regardless of whether the product has substantial +commercial, industrial or non-consumer uses, unless such uses represent +the only significant mode of use of the product. + + "Installation Information" for a User Product means any methods, +procedures, authorization keys, or other information required to install +and execute modified versions of a covered work in that User Product from +a modified version of its Corresponding Source. The information must +suffice to ensure that the continued functioning of the modified object +code is in no case prevented or interfered with solely because +modification has been made. + + If you convey an object code work under this section in, or with, or +specifically for use in, a User Product, and the conveying occurs as +part of a transaction in which the right of possession and use of the +User Product is transferred to the recipient in perpetuity or for a +fixed term (regardless of how the transaction is characterized), the +Corresponding Source conveyed under this section must be accompanied +by the Installation Information. But this requirement does not apply +if neither you nor any third party retains the ability to install +modified object code on the User Product (for example, the work has +been installed in ROM). + + The requirement to provide Installation Information does not include a +requirement to continue to provide support service, warranty, or updates +for a work that has been modified or installed by the recipient, or for +the User Product in which it has been modified or installed. Access to a +network may be denied when the modification itself materially and +adversely affects the operation of the network or violates the rules and +protocols for communication across the network. + + Corresponding Source conveyed, and Installation Information provided, +in accord with this section must be in a format that is publicly +documented (and with an implementation available to the public in +source code form), and must require no special password or key for +unpacking, reading or copying. + + 7. Additional Terms. + + "Additional permissions" are terms that supplement the terms of this +License by making exceptions from one or more of its conditions. +Additional permissions that are applicable to the entire Program shall +be treated as though they were included in this License, to the extent +that they are valid under applicable law. If additional permissions +apply only to part of the Program, that part may be used separately +under those permissions, but the entire Program remains governed by +this License without regard to the additional permissions. + + When you convey a copy of a covered work, you may at your option +remove any additional permissions from that copy, or from any part of +it. (Additional permissions may be written to require their own +removal in certain cases when you modify the work.) You may place +additional permissions on material, added by you to a covered work, +for which you have or can give appropriate copyright permission. + + Notwithstanding any other provision of this License, for material you +add to a covered work, you may (if authorized by the copyright holders of +that material) supplement the terms of this License with terms: + + a) Disclaiming warranty or limiting liability differently from the + terms of sections 15 and 16 of this License; or + + b) Requiring preservation of specified reasonable legal notices or + author attributions in that material or in the Appropriate Legal + Notices displayed by works containing it; or + + c) Prohibiting misrepresentation of the origin of that material, or + requiring that modified versions of such material be marked in + reasonable ways as different from the original version; or + + d) Limiting the use for publicity purposes of names of licensors or + authors of the material; or + + e) Declining to grant rights under trademark law for use of some + trade names, trademarks, or service marks; or + + f) Requiring indemnification of licensors and authors of that + material by anyone who conveys the material (or modified versions of + it) with contractual assumptions of liability to the recipient, for + any liability that these contractual assumptions directly impose on + those licensors and authors. + + All other non-permissive additional terms are considered "further +restrictions" within the meaning of section 10. If the Program as you +received it, or any part of it, contains a notice stating that it is +governed by this License along with a term that is a further +restriction, you may remove that term. If a license document contains +a further restriction but permits relicensing or conveying under this +License, you may add to a covered work material governed by the terms +of that license document, provided that the further restriction does +not survive such relicensing or conveying. + + If you add terms to a covered work in accord with this section, you +must place, in the relevant source files, a statement of the +additional terms that apply to those files, or a notice indicating +where to find the applicable terms. + + Additional terms, permissive or non-permissive, may be stated in the +form of a separately written license, or stated as exceptions; +the above requirements apply either way. + + 8. Termination. + + You may not propagate or modify a covered work except as expressly +provided under this License. Any attempt otherwise to propagate or +modify it is void, and will automatically terminate your rights under +this License (including any patent licenses granted under the third +paragraph of section 11). + + However, if you cease all violation of this License, then your +license from a particular copyright holder is reinstated (a) +provisionally, unless and until the copyright holder explicitly and +finally terminates your license, and (b) permanently, if the copyright +holder fails to notify you of the violation by some reasonable means +prior to 60 days after the cessation. + + Moreover, your license from a particular copyright holder is +reinstated permanently if the copyright holder notifies you of the +violation by some reasonable means, this is the first time you have +received notice of violation of this License (for any work) from that +copyright holder, and you cure the violation prior to 30 days after +your receipt of the notice. + + Termination of your rights under this section does not terminate the +licenses of parties who have received copies or rights from you under +this License. If your rights have been terminated and not permanently +reinstated, you do not qualify to receive new licenses for the same +material under section 10. + + 9. Acceptance Not Required for Having Copies. + + You are not required to accept this License in order to receive or +run a copy of the Program. Ancillary propagation of a covered work +occurring solely as a consequence of using peer-to-peer transmission +to receive a copy likewise does not require acceptance. However, +nothing other than this License grants you permission to propagate or +modify any covered work. These actions infringe copyright if you do +not accept this License. Therefore, by modifying or propagating a +covered work, you indicate your acceptance of this License to do so. + + 10. Automatic Licensing of Downstream Recipients. + + Each time you convey a covered work, the recipient automatically +receives a license from the original licensors, to run, modify and +propagate that work, subject to this License. You are not responsible +for enforcing compliance by third parties with this License. + + An "entity transaction" is a transaction transferring control of an +organization, or substantially all assets of one, or subdividing an +organization, or merging organizations. If propagation of a covered +work results from an entity transaction, each party to that +transaction who receives a copy of the work also receives whatever +licenses to the work the party's predecessor in interest had or could +give under the previous paragraph, plus a right to possession of the +Corresponding Source of the work from the predecessor in interest, if +the predecessor has it or can get it with reasonable efforts. + + You may not impose any further restrictions on the exercise of the +rights granted or affirmed under this License. For example, you may +not impose a license fee, royalty, or other charge for exercise of +rights granted under this License, and you may not initiate litigation +(including a cross-claim or counterclaim in a lawsuit) alleging that +any patent claim is infringed by making, using, selling, offering for +sale, or importing the Program or any portion of it. + + 11. Patents. + + A "contributor" is a copyright holder who authorizes use under this +License of the Program or a work on which the Program is based. The +work thus licensed is called the contributor's "contributor version". + + A contributor's "essential patent claims" are all patent claims +owned or controlled by the contributor, whether already acquired or +hereafter acquired, that would be infringed by some manner, permitted +by this License, of making, using, or selling its contributor version, +but do not include claims that would be infringed only as a +consequence of further modification of the contributor version. For +purposes of this definition, "control" includes the right to grant +patent sublicenses in a manner consistent with the requirements of +this License. + + Each contributor grants you a non-exclusive, worldwide, royalty-free +patent license under the contributor's essential patent claims, to +make, use, sell, offer for sale, import and otherwise run, modify and +propagate the contents of its contributor version. + + In the following three paragraphs, a "patent license" is any express +agreement or commitment, however denominated, not to enforce a patent +(such as an express permission to practice a patent or covenant not to +sue for patent infringement). To "grant" such a patent license to a +party means to make such an agreement or commitment not to enforce a +patent against the party. + + If you convey a covered work, knowingly relying on a patent license, +and the Corresponding Source of the work is not available for anyone +to copy, free of charge and under the terms of this License, through a +publicly available network server or other readily accessible means, +then you must either (1) cause the Corresponding Source to be so +available, or (2) arrange to deprive yourself of the benefit of the +patent license for this particular work, or (3) arrange, in a manner +consistent with the requirements of this License, to extend the patent +license to downstream recipients. "Knowingly relying" means you have +actual knowledge that, but for the patent license, your conveying the +covered work in a country, or your recipient's use of the covered work +in a country, would infringe one or more identifiable patents in that +country that you have reason to believe are valid. + + If, pursuant to or in connection with a single transaction or +arrangement, you convey, or propagate by procuring conveyance of, a +covered work, and grant a patent license to some of the parties +receiving the covered work authorizing them to use, propagate, modify +or convey a specific copy of the covered work, then the patent license +you grant is automatically extended to all recipients of the covered +work and works based on it. + + A patent license is "discriminatory" if it does not include within +the scope of its coverage, prohibits the exercise of, or is +conditioned on the non-exercise of one or more of the rights that are +specifically granted under this License. You may not convey a covered +work if you are a party to an arrangement with a third party that is +in the business of distributing software, under which you make payment +to the third party based on the extent of your activity of conveying +the work, and under which the third party grants, to any of the +parties who would receive the covered work from you, a discriminatory +patent license (a) in connection with copies of the covered work +conveyed by you (or copies made from those copies), or (b) primarily +for and in connection with specific products or compilations that +contain the covered work, unless you entered into that arrangement, +or that patent license was granted, prior to 28 March 2007. + + Nothing in this License shall be construed as excluding or limiting +any implied license or other defenses to infringement that may +otherwise be available to you under applicable patent law. + + 12. No Surrender of Others' Freedom. + + If conditions are imposed on you (whether by court order, agreement or +otherwise) that contradict the conditions of this License, they do not +excuse you from the conditions of this License. If you cannot convey a +covered work so as to satisfy simultaneously your obligations under this +License and any other pertinent obligations, then as a consequence you may +not convey it at all. For example, if you agree to terms that obligate you +to collect a royalty for further conveying from those to whom you convey +the Program, the only way you could satisfy both those terms and this +License would be to refrain entirely from conveying the Program. + + 13. Use with the GNU Affero General Public License. + + Notwithstanding any other provision of this License, you have +permission to link or combine any covered work with a work licensed +under version 3 of the GNU Affero General Public License into a single +combined work, and to convey the resulting work. The terms of this +License will continue to apply to the part which is the covered work, +but the special requirements of the GNU Affero General Public License, +section 13, concerning interaction through a network will apply to the +combination as such. + + 14. Revised Versions of this License. + + The Free Software Foundation may publish revised and/or new versions of +the GNU General Public License from time to time. Such new versions will +be similar in spirit to the present version, but may differ in detail to +address new problems or concerns. + + Each version is given a distinguishing version number. If the +Program specifies that a certain numbered version of the GNU General +Public License "or any later version" applies to it, you have the +option of following the terms and conditions either of that numbered +version or of any later version published by the Free Software +Foundation. If the Program does not specify a version number of the +GNU General Public License, you may choose any version ever published +by the Free Software Foundation. + + If the Program specifies that a proxy can decide which future +versions of the GNU General Public License can be used, that proxy's +public statement of acceptance of a version permanently authorizes you +to choose that version for the Program. + + Later license versions may give you additional or different +permissions. However, no additional obligations are imposed on any +author or copyright holder as a result of your choosing to follow a +later version. + + 15. Disclaimer of Warranty. + + THERE IS NO WARRANTY FOR THE PROGRAM, TO THE EXTENT PERMITTED BY +APPLICABLE LAW. EXCEPT WHEN OTHERWISE STATED IN WRITING THE COPYRIGHT +HOLDERS AND/OR OTHER PARTIES PROVIDE THE PROGRAM "AS IS" WITHOUT WARRANTY +OF ANY KIND, EITHER EXPRESSED OR IMPLIED, INCLUDING, BUT NOT LIMITED TO, +THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR +PURPOSE. THE ENTIRE RISK AS TO THE QUALITY AND PERFORMANCE OF THE PROGRAM +IS WITH YOU. SHOULD THE PROGRAM PROVE DEFECTIVE, YOU ASSUME THE COST OF +ALL NECESSARY SERVICING, REPAIR OR CORRECTION. + + 16. Limitation of Liability. + + IN NO EVENT UNLESS REQUIRED BY APPLICABLE LAW OR AGREED TO IN WRITING +WILL ANY COPYRIGHT HOLDER, OR ANY OTHER PARTY WHO MODIFIES AND/OR CONVEYS +THE PROGRAM AS PERMITTED ABOVE, BE LIABLE TO YOU FOR DAMAGES, INCLUDING ANY +GENERAL, SPECIAL, INCIDENTAL OR CONSEQUENTIAL DAMAGES ARISING OUT OF THE +USE OR INABILITY TO USE THE PROGRAM (INCLUDING BUT NOT LIMITED TO LOSS OF +DATA OR DATA BEING RENDERED INACCURATE OR LOSSES SUSTAINED BY YOU OR THIRD +PARTIES OR A FAILURE OF THE PROGRAM TO OPERATE WITH ANY OTHER PROGRAMS), +EVEN IF SUCH HOLDER OR OTHER PARTY HAS BEEN ADVISED OF THE POSSIBILITY OF +SUCH DAMAGES. + + 17. Interpretation of Sections 15 and 16. + + If the disclaimer of warranty and limitation of liability provided +above cannot be given local legal effect according to their terms, +reviewing courts shall apply local law that most closely approximates +an absolute waiver of all civil liability in connection with the +Program, unless a warranty or assumption of liability accompanies a +copy of the Program in return for a fee. diff --git a/mail/MANIFEST.in b/mail/MANIFEST.in new file mode 100644 index 00000000..cf9cbd37 --- /dev/null +++ b/mail/MANIFEST.in @@ -0,0 +1,6 @@ +include pkg/* +include versioneer.py +include LICENSE +include CHANGELOG.rst +include README.rst +include src/leap/mail/_version.py diff --git a/mail/README.rst b/mail/README.rst new file mode 100644 index 00000000..f201baa1 --- /dev/null +++ b/mail/README.rst @@ -0,0 +1,37 @@ +leap.mail +========= +Mail services for the LEAP Client. + +.. image:: https://badge.fury.io/py/leap.mail.svg + :target: http://badge.fury.io/py/leap.mail + +.. image:: https://readthedocs.org/projects/leapmail/badge/?version=latest + :target: http://leapmail.readthedocs.org/en/latest/ + :alt: Documentation Status + +More info: https://leap.se + +running tests +------------- + +Use trial to run the test suite:: + + trial leap.mail + +... and all its goodies. To run all imap tests in a loop until some of them +fails:: + + trial -u leap.mail.imap + +Read the *trial* manpage for more options . + +imap regressions +---------------- + +For testing the IMAP server implementation, there are a couple of utilities. +From the ``leap.mail.imap.tests`` folder, and with an already initialized server +running:: + + ./regressions_mime_struct user@provider pass path_to_samples/ + +You can find several message samples in the ``leap/mail/tests`` folder. diff --git a/mail/changes/VERSION_COMPAT b/mail/changes/VERSION_COMPAT new file mode 100644 index 00000000..77a394f9 --- /dev/null +++ b/mail/changes/VERSION_COMPAT @@ -0,0 +1,11 @@ +################################################# +# This file keeps track of the recent changes +# introduced in internal leap dependencies. +# Add your changes here so we can properly update +# requirements.pip during the release process. +# (leave header when resetting) +################################################# +# +# BEGIN DEPENDENCY LIST ------------------------- +# leap.foo.bar>=x.y.z +leap.keymanager >= 0.5.2 diff --git a/mail/changes/next-changelog.rst b/mail/changes/next-changelog.rst new file mode 100644 index 00000000..21b10103 --- /dev/null +++ b/mail/changes/next-changelog.rst @@ -0,0 +1,30 @@ +0.4.3 - xxx ++++++++++++++++++++++++++++++++ + +Please add lines to this file, they will be moved to the CHANGELOG.rst during +the next release. + +There are two template lines for each category, use them as reference. + +I've added a new category `Misc` so we can track doc/style/packaging stuff. + +Features +~~~~~~~~ +- `#8031 <https://leap.se/code/issues/8031>`_: Adapt to the new KeyManager API without key types. + +- `#1234 <https://leap.se/code/issues/1234>`_: Description of the new feature corresponding with issue #1234. +- New feature without related issue number. + +Bugfixes +~~~~~~~~ +- `#1235 <https://leap.se/code/issues/1235>`_: Description for the fixed stuff corresponding with issue #1235. +- Bugfix without related issue number. + +Misc +~~~~ +- `#1236 <https://leap.se/code/issues/1236>`_: Description of the new feature corresponding with issue #1236. +- Some change without issue number. + +Known Issues +~~~~~~~~~~~~ +- `#1236 <https://leap.se/code/issues/1236>`_: Description of the known issue corresponding with issue #1236. diff --git a/mail/docs/Makefile b/mail/docs/Makefile new file mode 100644 index 00000000..388b6c65 --- /dev/null +++ b/mail/docs/Makefile @@ -0,0 +1,179 @@ +# Makefile for Sphinx documentation +# + +# You can set these variables from the command line. +SPHINXOPTS = +SPHINXBUILD = sphinx-build +PAPER = +BUILDDIR = _build + +# User-friendly check for sphinx-build +ifeq ($(shell which $(SPHINXBUILD) >/dev/null 2>&1; echo $$?), 1) +$(error The '$(SPHINXBUILD)' command was not found. Make sure you have Sphinx installed, then set the SPHINXBUILD environment variable to point to the full path of the '$(SPHINXBUILD)' executable. Alternatively you can add the directory with the executable to your PATH. If you don't have Sphinx installed, grab it from http://sphinx-doc.org/) +endif + +# Internal variables. +PAPEROPT_a4 = -D latex_paper_size=a4 +PAPEROPT_letter = -D latex_paper_size=letter +ALLSPHINXOPTS = -d $(BUILDDIR)/doctrees $(PAPEROPT_$(PAPER)) $(SPHINXOPTS) . +# the i18n builder cannot share the environment and doctrees with the others +I18NSPHINXOPTS = $(PAPEROPT_$(PAPER)) $(SPHINXOPTS) . + +.PHONY: help clean html dirhtml singlehtml pickle json htmlhelp qthelp devhelp epub latex latexpdf text man changes linkcheck doctest gettext + +help: + @echo "Please use \`make <target>' where <target> is one of" + @echo " html to make standalone HTML files" + @echo " dirhtml to make HTML files named index.html in directories" + @echo " singlehtml to make a single large HTML file" + @echo " pickle to make pickle files" + @echo " json to make JSON files" + @echo " htmlhelp to make HTML files and a HTML help project" + @echo " qthelp to make HTML files and a qthelp project" + @echo " devhelp to make HTML files and a Devhelp project" + @echo " epub to make an epub" + @echo " latex to make LaTeX files, you can set PAPER=a4 or PAPER=letter" + @echo " latexpdf to make LaTeX files and run them through pdflatex" + @echo " latexpdfja to make LaTeX files and run them through platex/dvipdfmx" + @echo " text to make text files" + @echo " man to make manual pages" + @echo " texinfo to make Texinfo files" + @echo " info to make Texinfo files and run them through makeinfo" + @echo " gettext to make PO message catalogs" + @echo " changes to make an overview of all changed/added/deprecated items" + @echo " xml to make Docutils-native XML files" + @echo " pseudoxml to make pseudoxml-XML files for display purposes" + @echo " linkcheck to check all external links for integrity" + @echo " doctest to run all doctests embedded in the documentation (if enabled)" + +clean: + rm -rf $(BUILDDIR)/* + +html: + $(SPHINXBUILD) -b html $(ALLSPHINXOPTS) $(BUILDDIR)/html + @echo + @echo "Build finished. The HTML pages are in $(BUILDDIR)/html." + +dirhtml: + $(SPHINXBUILD) -b dirhtml $(ALLSPHINXOPTS) $(BUILDDIR)/dirhtml + @echo + @echo "Build finished. The HTML pages are in $(BUILDDIR)/dirhtml." + +singlehtml: + $(SPHINXBUILD) -b singlehtml $(ALLSPHINXOPTS) $(BUILDDIR)/singlehtml + @echo + @echo "Build finished. The HTML page is in $(BUILDDIR)/singlehtml." + +pickle: + $(SPHINXBUILD) -b pickle $(ALLSPHINXOPTS) $(BUILDDIR)/pickle + @echo + @echo "Build finished; now you can process the pickle files." + +json: + $(SPHINXBUILD) -b json $(ALLSPHINXOPTS) $(BUILDDIR)/json + @echo + @echo "Build finished; now you can process the JSON files." + +htmlhelp: + $(SPHINXBUILD) -b htmlhelp $(ALLSPHINXOPTS) $(BUILDDIR)/htmlhelp + @echo + @echo "Build finished; now you can run HTML Help Workshop with the" \ + ".hhp project file in $(BUILDDIR)/htmlhelp." + +qthelp: + $(SPHINXBUILD) -b qthelp $(ALLSPHINXOPTS) $(BUILDDIR)/qthelp + @echo + @echo "Build finished; now you can run "qcollectiongenerator" with the" \ + ".qhcp project file in $(BUILDDIR)/qthelp, like this:" + @echo "# qcollectiongenerator $(BUILDDIR)/qthelp/leapmail.qhcp" + @echo "To view the help file:" + @echo "# assistant -collectionFile $(BUILDDIR)/qthelp/leapmail.qhc" + +devhelp: + $(SPHINXBUILD) -b devhelp $(ALLSPHINXOPTS) $(BUILDDIR)/devhelp + @echo + @echo "Build finished." + @echo "To view the help file:" + @echo "# mkdir -p $$HOME/.local/share/devhelp/leapmail" + @echo "# ln -s $(BUILDDIR)/devhelp $$HOME/.local/share/devhelp/leapmail" + @echo "# devhelp" + +epub: + $(SPHINXBUILD) -b epub $(ALLSPHINXOPTS) $(BUILDDIR)/epub + @echo + @echo "Build finished. The epub file is in $(BUILDDIR)/epub." + +latex: + $(SPHINXBUILD) -b latex $(ALLSPHINXOPTS) $(BUILDDIR)/latex + @echo + @echo "Build finished; the LaTeX files are in $(BUILDDIR)/latex." + @echo "Run \`make' in that directory to run these through (pdf)latex" \ + "(use \`make latexpdf' here to do that automatically)." + +latexpdf: + $(SPHINXBUILD) -b latex $(ALLSPHINXOPTS) $(BUILDDIR)/latex + @echo "Running LaTeX files through pdflatex..." + $(MAKE) -C $(BUILDDIR)/latex all-pdf + @echo "pdflatex finished; the PDF files are in $(BUILDDIR)/latex." + +latexpdfja: + $(SPHINXBUILD) -b latex $(ALLSPHINXOPTS) $(BUILDDIR)/latex + @echo "Running LaTeX files through platex and dvipdfmx..." + $(MAKE) -C $(BUILDDIR)/latex all-pdf-ja + @echo "pdflatex finished; the PDF files are in $(BUILDDIR)/latex." + +text: + $(SPHINXBUILD) -b text $(ALLSPHINXOPTS) $(BUILDDIR)/text + @echo + @echo "Build finished. The text files are in $(BUILDDIR)/text." + +man: + $(SPHINXBUILD) -b man $(ALLSPHINXOPTS) $(BUILDDIR)/man + @echo + @echo "Build finished. The manual pages are in $(BUILDDIR)/man." + +texinfo: + $(SPHINXBUILD) -b texinfo $(ALLSPHINXOPTS) $(BUILDDIR)/texinfo + @echo + @echo "Build finished. The Texinfo files are in $(BUILDDIR)/texinfo." + @echo "Run \`make' in that directory to run these through makeinfo" \ + "(use \`make info' here to do that automatically)." + +info: + $(SPHINXBUILD) -b texinfo $(ALLSPHINXOPTS) $(BUILDDIR)/texinfo + @echo "Running Texinfo files through makeinfo..." + make -C $(BUILDDIR)/texinfo info + @echo "makeinfo finished; the Info files are in $(BUILDDIR)/texinfo." + +gettext: + $(SPHINXBUILD) -b gettext $(I18NSPHINXOPTS) $(BUILDDIR)/locale + @echo + @echo "Build finished. The message catalogs are in $(BUILDDIR)/locale." + +changes: + $(SPHINXBUILD) -b changes $(ALLSPHINXOPTS) $(BUILDDIR)/changes + @echo + @echo "The overview file is in $(BUILDDIR)/changes." + +linkcheck: + $(SPHINXBUILD) -b linkcheck $(ALLSPHINXOPTS) $(BUILDDIR)/linkcheck + @echo + @echo "Link check complete; look for any errors in the above output " \ + "or in $(BUILDDIR)/linkcheck/output.txt." + +doctest: + $(SPHINXBUILD) -b doctest $(ALLSPHINXOPTS) $(BUILDDIR)/doctest + @echo "Testing of doctests in the sources finished, look at the " \ + "results in $(BUILDDIR)/doctest/output.txt." + +xml: + $(SPHINXBUILD) -b xml $(ALLSPHINXOPTS) $(BUILDDIR)/xml + @echo + @echo "Build finished. The XML files are in $(BUILDDIR)/xml." + +pseudoxml: + $(SPHINXBUILD) -b pseudoxml $(ALLSPHINXOPTS) $(BUILDDIR)/pseudoxml + @echo + @echo "Build finished. The pseudo-XML files are in $(BUILDDIR)/pseudoxml." +build_rtd: + curl -X POST http://readthedocs.org/build/leapmail diff --git a/mail/docs/api/leap.mail.adaptors.rst b/mail/docs/api/leap.mail.adaptors.rst new file mode 100644 index 00000000..472cadef --- /dev/null +++ b/mail/docs/api/leap.mail.adaptors.rst @@ -0,0 +1,43 @@ +mail.adaptors package +===================== + +.. automodule:: leap.mail.adaptors + :members: + :undoc-members: + :show-inheritance: + +Subpackages +----------- + +.. toctree:: + + leap.mail.adaptors.tests + +Submodules +---------- + +mail.adaptors.models module +--------------------------- + +.. automodule:: leap.mail.adaptors.models + :members: + :undoc-members: + :show-inheritance: + +mail.adaptors.soledad module +---------------------------- + +.. automodule:: leap.mail.adaptors.soledad + :members: + :undoc-members: + :show-inheritance: + +mail.adaptors.soledad_indexes module +------------------------------------ + +.. automodule:: leap.mail.adaptors.soledad_indexes + :members: + :undoc-members: + :show-inheritance: + + diff --git a/mail/docs/api/leap.mail.adaptors.tests.rst b/mail/docs/api/leap.mail.adaptors.tests.rst new file mode 100644 index 00000000..2ae76e8a --- /dev/null +++ b/mail/docs/api/leap.mail.adaptors.tests.rst @@ -0,0 +1,28 @@ +mail.adaptors.tests package +=========================== + +.. automodule:: leap.mail.adaptors.tests + :members: + :undoc-members: + :show-inheritance: + +Submodules +---------- + +mail.adaptors.tests.test_models module +-------------------------------------- + +.. automodule:: leap.mail.adaptors.tests.test_models + :members: + :undoc-members: + :show-inheritance: + +mail.adaptors.tests.test_soledad_adaptor module +----------------------------------------------- + +.. automodule:: leap.mail.adaptors.tests.test_soledad_adaptor + :members: + :undoc-members: + :show-inheritance: + + diff --git a/mail/docs/api/leap.mail.imap.rst b/mail/docs/api/leap.mail.imap.rst new file mode 100644 index 00000000..bfaa3fd7 --- /dev/null +++ b/mail/docs/api/leap.mail.imap.rst @@ -0,0 +1,52 @@ +leap.mail.imap package +======================= + +.. automodule:: leap.mail.imap + :members: + :undoc-members: + :show-inheritance: + +Subpackages +----------- + +.. toctree:: + + leap.mail.imap.service + leap.mail.imap.tests + +Submodules +---------- + +leap.mail.imap.account module +------------------------------ + +.. automodule:: leap.mail.imap.account + :members: + :undoc-members: + :show-inheritance: + +leap.mail.imap.mailbox module +------------------------------ + +.. automodule:: leap.mail.imap.mailbox + :members: + :undoc-members: + :show-inheritance: + +leap.mail.imap.messages module +------------------------------ + +.. automodule:: leap.mail.imap.messages + :members: + :undoc-members: + :show-inheritance: + +leap.mail.imap.server module +----------------------------- + +.. automodule:: leap.mail.imap.server + :members: + :undoc-members: + :show-inheritance: + + diff --git a/mail/docs/api/leap.mail.imap.service.rst b/mail/docs/api/leap.mail.imap.service.rst new file mode 100644 index 00000000..2f3ed4bb --- /dev/null +++ b/mail/docs/api/leap.mail.imap.service.rst @@ -0,0 +1,9 @@ +leap.mail.imap.service package +=============================== + +.. automodule:: leap.mail.imap.service + :members: + :undoc-members: + :show-inheritance: + + diff --git a/mail/docs/api/leap.mail.incoming.rst b/mail/docs/api/leap.mail.incoming.rst new file mode 100644 index 00000000..4bd16141 --- /dev/null +++ b/mail/docs/api/leap.mail.incoming.rst @@ -0,0 +1,20 @@ +leap.mail.incoming package +========================== + +.. automodule:: leap.mail.incoming + :members: + :undoc-members: + :show-inheritance: + +Submodules +---------- + +leap.mail.incoming.service module +---------------------------------- + +.. automodule:: leap.mail.incoming.service + :members: + :undoc-members: + :show-inheritance: + + diff --git a/mail/docs/api/leap.mail.outgoing.rst b/mail/docs/api/leap.mail.outgoing.rst new file mode 100644 index 00000000..af8c1734 --- /dev/null +++ b/mail/docs/api/leap.mail.outgoing.rst @@ -0,0 +1,21 @@ +leap.mail.outgoing package +========================== + +.. automodule:: leap.mail.outgoing + :members: + :undoc-members: + :show-inheritance: + + +Submodules +---------- + +mail.outgoing.service module +---------------------------- + +.. automodule:: leap.mail.outgoing.service + :members: + :undoc-members: + :show-inheritance: + + diff --git a/mail/docs/api/leap.mail.plugins.rst b/mail/docs/api/leap.mail.plugins.rst new file mode 100644 index 00000000..7a5d6b4d --- /dev/null +++ b/mail/docs/api/leap.mail.plugins.rst @@ -0,0 +1,20 @@ +leap.mail.plugins package +========================== + +.. automodule:: leap.mail.plugins + :members: + :undoc-members: + :show-inheritance: + +Submodules +---------- + +leap.mail.plugins.soledad_sync_hooks module +------------------------------------------- + +.. automodule:: leap.mail.plugins.soledad_sync_hooks + :members: + :undoc-members: + :show-inheritance: + + diff --git a/mail/docs/api/leap.mail.rst b/mail/docs/api/leap.mail.rst new file mode 100644 index 00000000..686e648e --- /dev/null +++ b/mail/docs/api/leap.mail.rst @@ -0,0 +1,105 @@ +leap.mail package +================= + +.. automodule:: leap.mail + :members: + :undoc-members: + :show-inheritance: + +Subpackages +----------- + +.. toctree:: + + leap.mail.adaptors + leap.mail.imap + leap.mail.incoming + leap.mail.outgoing + leap.mail.plugins + leap.mail.smtp + leap.mail.tests + +Submodules +---------- + +leap.mail.constants module +--------------------------- + +.. automodule:: leap.mail.constants + :members: + :undoc-members: + :show-inheritance: + +leap.mail.decorators module +--------------------------- + +.. automodule:: leap.mail.decorators + :members: + :undoc-members: + :show-inheritance: + +leap.mail.interfaces module +---------------------------- + +.. automodule:: leap.mail.interfaces + :members: + :undoc-members: + :show-inheritance: + +leap.mail.load_tests module +---------------------------- + +.. automodule:: leap.mail.load_tests + :members: + :undoc-members: + :show-inheritance: + +leap.mail.mail module +--------------------- + +.. automodule:: leap.mail.mail + :members: + :undoc-members: + :show-inheritance: + +leap.mail.mailbox_indexer module +--------------------------------- + +.. automodule:: leap.mail.mailbox_indexer + :members: + :undoc-members: + :show-inheritance: + +leap.mail.size module +---------------------- + +.. automodule:: leap.mail.size + :members: + :undoc-members: + :show-inheritance: + +leap.mail.sync_hooks module +---------------------------- + +.. automodule:: leap.mail.sync_hooks + :members: + :undoc-members: + :show-inheritance: + +leap.mail.utils module +----------------------- + +.. automodule:: leap.mail.utils + :members: + :undoc-members: + :show-inheritance: + +leap.mail.walk module +---------------------- + +.. automodule:: leap.mail.walk + :members: + :undoc-members: + :show-inheritance: + + diff --git a/mail/docs/api/leap.mail.smtp.rst b/mail/docs/api/leap.mail.smtp.rst new file mode 100644 index 00000000..f35d3f9d --- /dev/null +++ b/mail/docs/api/leap.mail.smtp.rst @@ -0,0 +1,19 @@ +leap.mail.smtp package +======================= + +.. automodule:: leap.mail.smtp + :members: + :undoc-members: + :show-inheritance: + +Submodules +---------- + +leap.mail.smtp.gateway module +----------------------------- + +.. automodule:: leap.mail.smtp.gateway + :members: + :undoc-members: + :show-inheritance: + diff --git a/mail/docs/api/modules.rst b/mail/docs/api/modules.rst new file mode 100644 index 00000000..78277797 --- /dev/null +++ b/mail/docs/api/modules.rst @@ -0,0 +1,7 @@ +mail +==== + +.. toctree:: + :maxdepth: 4 + + mail diff --git a/mail/docs/conf.py b/mail/docs/conf.py new file mode 100644 index 00000000..746f57cf --- /dev/null +++ b/mail/docs/conf.py @@ -0,0 +1,341 @@ +# -*- coding: utf-8 -*- +# +# leap.mail documentation build configuration file, created by +# sphinx-quickstart on Mon Aug 25 19:19:48 2014. +# +# This file is execfile()d with the current directory set to its +# containing dir. +# +# Note that not all possible configuration values are present in this +# autogenerated file. +# +# All configuration values have a default; values that are commented out +# serve to show the default. + +import sys +import os + +# If extensions (or modules to document with autodoc) are in another directory, +# add these directories to sys.path here. If the directory is relative to the +# documentation root, use os.path.abspath to make it absolute, like shown here. +#sys.path.insert(0, os.path.abspath('../src')) +#sys.path.insert(0, os.path.abspath('../src/leap')) +#sys.path.insert(0, os.path.abspath('../src/leap/mail')) +#sys.path.insert(0, os.path.abspath('../../leap_common/src/leap/common')) +#sys.path.insert(0, os.path.abspath('../../soledad/client/src/leap/soledad/client')) + +VENV_PATH = os.environ.get('VIRTUAL_ENV') +if VENV_PATH: + sys.path.insert(0, os.path.abspath(VENV_PATH + 'lib/python2.7/site-packages')) + + +# -- General configuration ------------------------------------------------ + +# If your documentation needs a minimal Sphinx version, state it here. +#needs_sphinx = '1.0' + +# Add any Sphinx extension module names here, as strings. They can be +# extensions coming with Sphinx (named 'sphinx.ext.*') or your custom +# ones. +extensions = [ + 'sphinx.ext.autodoc', + 'sphinx.ext.todo', + 'sphinx.ext.coverage', +] + +# Add any paths that contain templates here, relative to this directory. +templates_path = ['_templates'] + +# The suffix of source filenames. +source_suffix = '.rst' + +# The encoding of source files. +#source_encoding = 'utf-8-sig' + +# The master toctree document. +master_doc = 'index' + +# General information about the project. +project = u'leap.mail' +copyright = u'2014-2015, The LEAP Encryption Access Project' + +# The version info for the project you're documenting, acts as replacement for +# |version| and |release|, also used in various other places throughout the +# built documents. +# +# The short X.Y version. +version = '0.4.0rc2' +# The full version, including alpha/beta/rc tags. +release = '0.4.0' + +# The language for content autogenerated by Sphinx. Refer to documentation +# for a list of supported languages. +#language = None + +# There are two options for replacing |today|: either, you set today to some +# non-false value, then it is used: +#today = '' +# Else, today_fmt is used as the format for a strftime call. +#today_fmt = '%B %d, %Y' + +# List of patterns, relative to source directory, that match files and +# directories to ignore when looking for source files. +exclude_patterns = ['_build'] + +# The reST default role (used for this markup: `text`) to use for all +# documents. +#default_role = None + +# If true, '()' will be appended to :func: etc. cross-reference text. +#add_function_parentheses = True + +# If true, the current module name will be prepended to all description +# unit titles (such as .. function::). +#add_module_names = True + +# If true, sectionauthor and moduleauthor directives will be shown in the +# output. They are ignored by default. +#show_authors = False + +# The name of the Pygments (syntax highlighting) style to use. +pygments_style = 'sphinx' + +# A list of ignored prefixes for module index sorting. +#modindex_common_prefix = [] + +# If true, keep warnings as "system message" paragraphs in the built documents. +#keep_warnings = False + + +# -- Options for HTML output ---------------------------------------------- + +# The theme to use for HTML and HTML Help pages. See the documentation for +# a list of builtin themes. +html_theme = 'alabaster' + +# Theme options are theme-specific and customize the look and feel of a theme +# further. For a list of options available for each theme, see the +# documentation. +#html_theme_options = {} + +# Add any paths that contain custom themes here, relative to this directory. +#html_theme_path = [] + +# The name for this set of Sphinx documents. If None, it defaults to +# "<project> v<release> documentation". +#html_title = None + +# A shorter title for the navigation bar. Default is the same as html_title. +#html_short_title = None + +# The name of an image file (relative to this directory) to place at the top +# of the sidebar. +#html_logo = None + +# The name of an image file (within the static path) to use as favicon of the +# docs. This file should be a Windows icon file (.ico) being 16x16 or 32x32 +# pixels large. +#html_favicon = None + +# Add any paths that contain custom static files (such as style sheets) here, +# relative to this directory. They are copied after the builtin static files, +# so a file named "default.css" will overwrite the builtin "default.css". +html_static_path = ['_static'] + +# Add any extra paths that contain custom files (such as robots.txt or +# .htaccess) here, relative to this directory. These files are copied +# directly to the root of the documentation. +#html_extra_path = [] + +# If not '', a 'Last updated on:' timestamp is inserted at every page bottom, +# using the given strftime format. +#html_last_updated_fmt = '%b %d, %Y' + +# If true, SmartyPants will be used to convert quotes and dashes to +# typographically correct entities. +#html_use_smartypants = True + +# Custom sidebar templates, maps document names to template names. +#html_sidebars = {} + +# Additional templates that should be rendered to pages, maps page names to +# template names. +#html_additional_pages = {} + +# If false, no module index is generated. +#html_domain_indices = True + +# If false, no index is generated. +#html_use_index = True + +# If true, the index is split into individual pages for each letter. +#html_split_index = False + +# If true, links to the reST sources are added to the pages. +#html_show_sourcelink = True + +# If true, "Created using Sphinx" is shown in the HTML footer. Default is True. +#html_show_sphinx = True + +# If true, "(C) Copyright ..." is shown in the HTML footer. Default is True. +#html_show_copyright = True + +# If true, an OpenSearch description file will be output, and all pages will +# contain a <link> tag referring to it. The value of this option must be the +# base URL from which the finished HTML is served. +#html_use_opensearch = '' + +# This is the file name suffix for HTML files (e.g. ".xhtml"). +#html_file_suffix = None + +# Output file base name for HTML help builder. +htmlhelp_basename = 'leapmaildoc' + + +# -- Options for LaTeX output --------------------------------------------- + +latex_elements = { +# The paper size ('letterpaper' or 'a4paper'). +#'papersize': 'letterpaper', + +# The font size ('10pt', '11pt' or '12pt'). +#'pointsize': '10pt', + +# Additional stuff for the LaTeX preamble. +#'preamble': '', +} + +# Grouping the document tree into LaTeX files. List of tuples +# (source start file, target name, title, +# author, documentclass [howto, manual, or own class]). +latex_documents = [ + ('index', 'leapmail.tex', u'leap.mail Documentation', + u'Kali Kaneko', 'manual'), +] + +# The name of an image file (relative to this directory) to place at the top of +# the title page. +#latex_logo = None + +# For "manual" documents, if this is true, then toplevel headings are parts, +# not chapters. +#latex_use_parts = False + +# If true, show page references after internal links. +#latex_show_pagerefs = False + +# If true, show URL addresses after external links. +#latex_show_urls = False + +# Documents to append as an appendix to all manuals. +#latex_appendices = [] + +# If false, no module index is generated. +#latex_domain_indices = True + + +# -- Options for manual page output --------------------------------------- + +# One entry per manual page. List of tuples +# (source start file, name, description, authors, manual section). +man_pages = [ + ('index', 'leapmail', u'leap.mail Documentation', + [u'Kali Kaneko'], 1) +] + +# If true, show URL addresses after external links. +#man_show_urls = False + + +# -- Options for Texinfo output ------------------------------------------- + +# Grouping the document tree into Texinfo files. List of tuples +# (source start file, target name, title, author, +# dir menu entry, description, category) +texinfo_documents = [ + ('index', 'leapmail', u'leap.mail Documentation', + u'Kali Kaneko', 'leapmail', 'One line description of project.', + 'Miscellaneous'), +] + +# Documents to append as an appendix to all manuals. +#texinfo_appendices = [] + +# If false, no module index is generated. +#texinfo_domain_indices = True + +# How to display URL addresses: 'footnote', 'no', or 'inline'. +#texinfo_show_urls = 'footnote' + +# If true, do not generate a @detailmenu in the "Top" node's menu. +#texinfo_no_detailmenu = False + + +# -- Options for Epub output ---------------------------------------------- + +# Bibliographic Dublin Core info. +epub_title = u'leap.mail' +epub_author = u'Kali Kaneko' +epub_publisher = u'Kali Kaneko' +epub_copyright = u'2014, Kali Kaneko' + +# The basename for the epub file. It defaults to the project name. +#epub_basename = u'leap.mail' + +# The HTML theme for the epub output. Since the default themes are not optimized +# for small screen space, using the same theme for HTML and epub output is +# usually not wise. This defaults to 'epub', a theme designed to save visual +# space. +#epub_theme = 'epub' + +# The language of the text. It defaults to the language option +# or en if the language is not set. +#epub_language = '' + +# The scheme of the identifier. Typical schemes are ISBN or URL. +#epub_scheme = '' + +# The unique identifier of the text. This can be a ISBN number +# or the project homepage. +#epub_identifier = '' + +# A unique identification for the text. +#epub_uid = '' + +# A tuple containing the cover image and cover page html template filenames. +#epub_cover = () + +# A sequence of (type, uri, title) tuples for the guide element of content.opf. +#epub_guide = () + +# HTML files that should be inserted before the pages created by sphinx. +# The format is a list of tuples containing the path and title. +#epub_pre_files = [] + +# HTML files shat should be inserted after the pages created by sphinx. +# The format is a list of tuples containing the path and title. +#epub_post_files = [] + +# A list of files that should not be packed into the epub file. +epub_exclude_files = ['search.html'] + +# The depth of the table of contents in toc.ncx. +#epub_tocdepth = 3 + +# Allow duplicate toc entries. +#epub_tocdup = True + +# Choose between 'default' and 'includehidden'. +#epub_tocscope = 'default' + +# Fix unsupported image types using the PIL. +#epub_fix_images = False + +# Scale large images. +#epub_max_image_width = 0 + +# How to display URL addresses: 'footnote', 'no', or 'inline'. +#epub_show_urls = 'inline' + +# If false, no index is generated. +#epub_use_index = True diff --git a/mail/docs/git-commit-message.txt b/mail/docs/git-commit-message.txt new file mode 100644 index 00000000..1b28baf2 --- /dev/null +++ b/mail/docs/git-commit-message.txt @@ -0,0 +1,8 @@ +[bug|feature|docs|pkg] + +... + +Resolves: #XXX +Related: #XXX +Documentation: #XXX +Releases: XXX diff --git a/mail/docs/hacking.rst b/mail/docs/hacking.rst new file mode 100644 index 00000000..6c49c211 --- /dev/null +++ b/mail/docs/hacking.rst @@ -0,0 +1,195 @@ +.. _hacking: + +======== +Hacking +======== + +Some hints oriented to `leap.mail` hackers. These notes are mostly related to +the imap server, although they probably will be useful for other pieces too. + +Don't panic! Just manhole into it +================================= + +If you want to inspect the objects living in your application memory, in +realtime, you can manhole into it. + +First of all, check that the modules ``PyCrypto`` and ``pyasn1`` are installed +into your system, they are needed for it to work. + +You just have to pass the ``LEAP_MAIL_MANHOLE=1`` enviroment variable while +launching the client:: + + LEAP_MAIL_MANHOLE=1 bitmask --debug + +And then you can ssh into your application! (password is "leap"):: + + ssh boss@localhost -p 2222 + +Did I mention how *awesome* twisted is?? ``:)`` + + +Profiling +========= +If using ``twistd`` to launch the server, you can use twisted profiling +capabities:: + + LEAP_MAIL_CONFIG=~/.leapmailrc twistd --profile=/tmp/mail-profiling -n -y imap-server.tac + +``--profiler`` option allows you to select different profilers (default is +"hotshot"). + +You can also do profiling when using the ``bitmask`` client. Enable the +``LEAP_PROFILE_IMAPCMD`` environment flag to get profiling of certain IMAP +commands:: + + LEAP_PROFILE_IMAPCMD=1 bitmask --debug + +Offline mode +============ + +The client has an ``--offline`` flag that will make the Mail services (imap, +currently) not try to sync with remote replicas. Very useful during development, +although you need to login with the remote server at least once before being +able to use it. + +Mutt config +=========== + +You cannot live without mutt? You're lucky! Use the following minimal config +with the imap service:: + + set folder="imap://user@provider@localhost:1984" + set spoolfile="imap://user@provider@localhost:1984/INBOX" + set ssl_starttls = no + set ssl_force_tls = no + set imap_pass=MAHSIKRET + + +Running the service with twistd +=============================== + +In order to run the mail service (currently, the imap server only), you will +need a config with this info:: + + [leap_mail] + userid = "user@provider" + uuid = "deadbeefdeadabad" + passwd = "foobar" # Optional + +In the ``LEAP_MAIL_CONFIG`` enviroment variable. If you do not specify a password +parameter, you'll be prompted for it. + +In order to get the user uid (uuid), look into the +``~/.config/leap/leap-backend.conf`` file after you have logged in into your +provider at least once. + +Run the twisted service:: + + LEAP_MAIL_CONFIG=~/.leapmailrc twistd -n -y imap-server.tac + +Now you can telnet into your local IMAP server and read your mail like a real +programmer™:: + + % telnet localhost 1984 + Trying 127.0.0.1... + Connected to localhost. + Escape character is '^]'. + * OK [CAPABILITY IMAP4rev1 LITERAL+ IDLE NAMESPACE] Twisted IMAP4rev1 Ready + tag LOGIN me@myprovider.net mahsikret + tag OK LOGIN succeeded + tag SELECT Inbox + * 2 EXISTS + * 1 RECENT + * FLAGS (\Seen \Answered \Flagged \Deleted \Draft \Recent List) + * OK [UIDVALIDITY 1410453885932] UIDs valid + tag OK [READ-WRITE] SELECT successful + ^] + telnet> Connection closed. + + +Although you probably prefer to use ``offlineimap`` for tests:: + + offlineimap -c LEAPofflineimapRC-tests + + +Minimal offlineimap configuration +--------------------------------- + +You can use this as a sample offlineimap config file:: + + [general] + accounts = leap-local + + [Account leap-local] + localrepository = LocalLeap + remoterepository = RemoteLeap + + [Repository LocalLeap] + type = Maildir + localfolders = ~/LEAPMail/Mail + + [Repository RemoteLeap] + type = IMAP + ssl = no + remotehost = localhost + remoteport = 1984 + remoteuser = user + remotepass = pass + +Testing utilities +----------------- +There are a bunch of utilities to test IMAP delivery in ``imap/tests`` folder. +If looking for a quick way of inspecting mailboxes, have a look at ``getmail``:: + + ./getmail me@testprovider.net mahsikret + 1. Drafts + 2. INBOX + 3. Trash + Which mailbox? [1] 2 + 1 Subject: this is the time of the revolution + 2 Subject: ignore me + + Which message? [1] (Q quits) 1 + 1 X-Leap-Provenance: Thu, 11 Sep 2014 16:52:11 -0000; pubkey="C1F8DE10BD151F99" + Received: from mx1.testprovider.net(mx1.testprovider.net [198.197.196.195]) + (using TLSv1.2 with cipher ECDHE-RSA-AES256-GCM-SHA384 (256/256 bits)) + (Client CN "*.foobar.net", Issuer "Gandi Standard SSL CA" (not verified)) + by blackhole (Postfix) with ESMTPS id DEADBEEF + for <me@testprovider.net>; Thu, 11 Sep 2014 16:52:10 +0000 (UTC) + Delivered-To: 926d4915cfd42b6d96d38660c04613af@testprovider.net + Message-Id: <20140911165205.GB8054@samsara> + From: Kali <kali@leap.se> + + (snip) + +IMAP Message Rendering Regressions +---------------------------------- + +For testing the IMAP server implementation, there is a litte regressions script +that needs some manual work from your side. + +First of all, you need an already initialized account. Which for now basically +means you have created a new account with a provider that offers the Encrypted +Mail Service, using the Bitmask Client wizard. Then you need to log in with that +account, and let it generate the secrets and sync with the remote for a first +time. After this you can run the twistd server locally and offline. + +From the ``leap.mail.imap.tests`` folder, and with an already initialized server +running:: + + ./regressions_mime_struct user@provider pass path_to_samples/ + +You can find several message samples in the ``leap/mail/tests`` folder. + + +Debugging IMAP commands +======================= + +Use ``ngrep`` to obtain logs of the commands:: + + sudo ngrep -d lo -W byline port 1984 + +To get verbose output from thunderbird/icedove, set the following environment +variable:: + + NSPR_LOG_MODULES="imap:5" icedove diff --git a/mail/docs/index.rst b/mail/docs/index.rst new file mode 100644 index 00000000..a2133f48 --- /dev/null +++ b/mail/docs/index.rst @@ -0,0 +1,96 @@ +.. leap.mail documentation master file, created by + sphinx-quickstart on Mon Aug 25 19:19:48 2014. + You can adapt this file completely to your liking, but it should at least + contain the root `toctree` directive. + +leap.mail +========= + +*decentralized and secure mail delivery and synchronization* + +This is the documentation for the ``leap.mail`` module. It is a `twisted`_ +package that allows to receive, process, send and access existing messages using +the `LEAP`_ platform. + +One way to use this library is to let it launch two standard mail services, +``smtp`` and ``imap``, that run as local proxies and interact with a remote +``LEAP`` provider that offers *a soledad syncronization endpoint* and receives +the outgoing email. This is what `Bitmask`_ client does. + +From the release 0.4.0 on, it's also possible to use a protocol-agnostic email +public API, so that third party mail clients can manipulate the data layer. This +is what the awesome MUA in the `Pixelated`_ project is using. + +.. _`twisted`: https://twistedmatrix.com/trac/ +.. _`LEAP`: https://leap.se/en/docs +.. _`Bitmask`: https://bitmask.net/en/features#email +.. _`Pixelated`: https://pixelated-project.org/ + +How does this all work? +----------------------- + +All the underlying data storage and sync is handled by a library called +`soledad`_, which handles encryption, storage and sync. Based on `u1db`_, +documents are stored locally as local ``sqlcipher`` tables, and syncs against +the soledad sync service in the provider. + +OpenPGP key generation and keyring management are handled by another leap +python library: `keymanager`_. + +See :ref:`the life cycle of a leap email <mail_journey>` for an overview of the life cycle +of an email through ``LEAP`` providers. + +.. _`Soledad`: https://leap.se/en/docs/design/soledad +.. _`u1db`: https://en.wikipedia.org/wiki/U1DB +.. _`keymanager`: https://github.com/leapcode/keymanager/ + + +Data model +---------- +.. TODO clear document types documentation. + +The data model at the present moment consists of several *document types* that split email into +different documents that are stored in ``Soledad``. The idea behind this is to +keep clear the separation between *mutable* and *inmutable* parts, and still being able to +reconstruct arbitrarily nested email structures easily. + +Documentation index +=================== + +.. +.. Contents: +.. toctree:: + :maxdepth: 2 + + hacking + +.. intro +.. tutorial + + +API documentation +----------------- + +If you were looking for the documentation of the ``leap.mail`` module, you will +find it here. + +Of special interest is the `public mail api`_, which should remain relatively +stable across the next few releases. + +.. _`public mail api`: api/mail.html#module-mail + + +.. toctree:: + :maxdepth: 2 + + api/leap.mail + + + +Indices and tables +================== + +* :ref:`genindex` +* :ref:`modindex` +* :ref:`search` + diff --git a/mail/docs/mail_journey.rst b/mail/docs/mail_journey.rst new file mode 100644 index 00000000..7e64f186 --- /dev/null +++ b/mail/docs/mail_journey.rst @@ -0,0 +1,40 @@ +.. _mail_journey: + +The life cycle of a LEAP Email +============================== +The following are just some notes to facilitate the understanding of the +leap.mail internals to developers and collaborators. + +Server-side: receiving mail from the outside world +-------------------------------------------------- + +1. the mx server receives an email, it gets through all the postfix validation and it's written into disk +2. ``leap_mx`` gets that write in disk notification and encrypts the incoming mail to its intended recipient's pubkey +3. that encrypted blob gets written into couchdb in a way soledad will catch it in the next sync + + +Client-side: fetching and processing incoming email +--------------------------------------------------- +you have an imap and an smtp local server. For IMAP: + +1. soledad syncs +2. **fetch** module sees if there's new encrypted mail for the current user from leap_mx +3. if there is, the mail is decrypted using the user private key, and splitted + into several parts according to the internal mail data model (separating + mutable and inmutable email parts). Those documents it encrypts it properly + like other soledad documents and deletes the pubkey encrypted doc +4. desktop client is notified that there are N new mails +5. when a MUA connects to the **imap** local server, the different documents are glued + together and presented as response to the different imap commands. + + +Client side: sending email +-------------------------- + +1. you write an email to a recipient and hit send +2. the **smtp** local server gets that mail, it checks from whom it is and to whom it is for +3. it signs the mail with the ``From:``'s privkey +4. it retrieves ``To:``'s pubkey with the keymanager and if it finds it encrypts the mail to him/her +5. if it didn't find it and you don't have your client configured to "always encrypt", it goes out with just the signature +6. else, it fails out complaining about this conflict +7. that mail gets relayed to the provider's **smtp** server diff --git a/mail/docs/recreate_apidocs.sh b/mail/docs/recreate_apidocs.sh new file mode 100755 index 00000000..9a79d091 --- /dev/null +++ b/mail/docs/recreate_apidocs.sh @@ -0,0 +1,4 @@ +#!/bin/sh +# Watchout! this will need much manual touches +# to the generated apidocs. Mainly: s/mail/leap.mail/g +sphinx-apidoc -M -o api ../src/leap/mail diff --git a/mail/docs/tutorial.rst b/mail/docs/tutorial.rst new file mode 100644 index 00000000..5cf7d026 --- /dev/null +++ b/mail/docs/tutorial.rst @@ -0,0 +1,3 @@ +Tutorial +======== +`leap.mail` tutorial diff --git a/mail/pkg/__init__.py b/mail/pkg/__init__.py new file mode 100644 index 00000000..e69de29b --- /dev/null +++ b/mail/pkg/__init__.py diff --git a/mail/pkg/generate_wheels.sh b/mail/pkg/generate_wheels.sh new file mode 100755 index 00000000..6679d1d0 --- /dev/null +++ b/mail/pkg/generate_wheels.sh @@ -0,0 +1,14 @@ +#!/bin/sh +# Generate wheels for dependencies +# For convenience, u1db and dirspec are allowed with insecure flags enabled. +# Use at your own risk. + +if [ "$WHEELHOUSE" = "" ]; then + WHEELHOUSE=$HOME/wheelhouse +fi + +pip wheel --wheel-dir $WHEELHOUSE pip +pip wheel --wheel-dir $WHEELHOUSE -r pkg/requirements.pip +if [ -f pkg/requirements-testing.pip ]; then + pip wheel --wheel-dir $WHEELHOUSE -r pkg/requirements-testing.pip +fi diff --git a/mail/pkg/pip_install_requirements.sh b/mail/pkg/pip_install_requirements.sh new file mode 100755 index 00000000..57732e21 --- /dev/null +++ b/mail/pkg/pip_install_requirements.sh @@ -0,0 +1,86 @@ +#!/bin/bash +# Update pip and install LEAP base/testing requirements. +# For convenience, $insecure_packages are allowed with insecure flags enabled. +# Use at your own risk. +# See $usage for help + +insecure_packages="" +leap_wheelhouse=https://lizard.leap.se/wheels + +show_help() { + usage="Usage: $0 [--testing] [--use-leap-wheels]\n --testing\t\tInstall dependencies from requirements-testing.pip\n +\t\t\tOtherwise, it will install requirements.pip\n +--use-leap-wheels\tUse wheels from leap.se" + echo -e $usage + + exit 1 +} + +process_arguments() { + testing=false + use_leap_wheels=false + + while [ "$#" -gt 0 ]; do + # From http://stackoverflow.com/a/31443098 + case "$1" in + --help) show_help;; + --testing) testing=true; shift 1;; + --use-leap-wheels) use_leap_wheels=true; shift 1;; + + -h) show_help;; + -*) echo "unknown option: $1" >&2; exit 1;; + esac + done +} + +return_wheelhouse() { + if $use_leap_wheels ; then + WHEELHOUSE=$leap_wheelhouse + elif [ "$WHEELHOUSE" = "" ]; then + WHEELHOUSE=$HOME/wheelhouse + fi + + # Tested with bash and zsh + if [[ $WHEELHOUSE != http* && ! -d "$WHEELHOUSE" ]]; then + mkdir $WHEELHOUSE + fi + + echo "$WHEELHOUSE" +} + +return_install_options() { + wheelhouse=`return_wheelhouse` + install_options="-U --find-links=$wheelhouse" + if $use_leap_wheels ; then + install_options="$install_options --trusted-host lizard.leap.se" + fi + + echo $install_options +} + +return_insecure_flags() { + for insecure_package in $insecure_packages; do + flags="$flags --allow-external $insecure_package --allow-unverified $insecure_package" + done + + echo $flags +} + +return_packages() { + if $testing ; then + packages="-r pkg/requirements-testing.pip" + else + packages="-r pkg/requirements.pip" + fi + + echo $packages +} + +process_arguments $@ +install_options=`return_install_options` +insecure_flags=`return_insecure_flags` +packages=`return_packages` + +pip install -U wheel +pip install $install_options pip +pip install $install_options $insecure_flags $packages diff --git a/mail/pkg/requirements-dev.pip b/mail/pkg/requirements-dev.pip new file mode 100644 index 00000000..4bd76f64 --- /dev/null +++ b/mail/pkg/requirements-dev.pip @@ -0,0 +1,14 @@ +# --------------------------- +# -- external requirements -- +# -- during development -- +# --------------------------- +# +# For temporary work, you can point this to your developer repo. +# consolidated changes will be pushed to pypi and then added +# to the main requirements.pip +# +# NOTE: you have to run pip install -r pkg/requirements.pip for pip +# to install it. (do it after python setup.py develop and it +# will only install this) + +-e git+git://leap.se/soledad@develop#egg=leap.soledad diff --git a/mail/pkg/requirements-latest.pip b/mail/pkg/requirements-latest.pip new file mode 100644 index 00000000..58347b68 --- /dev/null +++ b/mail/pkg/requirements-latest.pip @@ -0,0 +1,9 @@ +--index-url https://pypi.python.org/simple/ + +https://launchpad.net/dirspec/stable-13-10/13.10/+download/dirspec-13.10.tar.gz +https://launchpad.net/ubuntu/+archive/primary/+files/u1db_13.09.orig.tar.bz2 +-e 'git+https://github.com/leapcode/leap_pycommon.git@develop#egg=leap.common' +-e 'git+https://github.com/leapcode/soledad.git@develop#egg=leap.soledad.common&subdirectory=common/' +-e 'git+https://github.com/leapcode/soledad.git@develop#egg=leap.soledad.client&subdirectory=client/' +-e 'git+https://github.com/leapcode/keymanager.git@develop#egg=leap.keymanager' +-e . diff --git a/mail/pkg/requirements-leap.pip b/mail/pkg/requirements-leap.pip new file mode 100644 index 00000000..134b7833 --- /dev/null +++ b/mail/pkg/requirements-leap.pip @@ -0,0 +1,3 @@ +leap.common>=0.5.1 +leap.soledad.client>=0.8.0 +leap.keymanager>=0.5.0 diff --git a/mail/pkg/requirements-testing.pip b/mail/pkg/requirements-testing.pip new file mode 100644 index 00000000..41222f70 --- /dev/null +++ b/mail/pkg/requirements-testing.pip @@ -0,0 +1,7 @@ +setuptools-trial +mock +nose +rednose +nose-progressive +coverage +pep8>=1.1 diff --git a/mail/pkg/requirements.pip b/mail/pkg/requirements.pip new file mode 100644 index 00000000..0caa66b0 --- /dev/null +++ b/mail/pkg/requirements.pip @@ -0,0 +1,4 @@ +zope.interface +twisted # >= 12.0.3 ?? +zope.proxy +service-identity diff --git a/mail/pkg/tools/get_authors.sh b/mail/pkg/tools/get_authors.sh new file mode 100755 index 00000000..0169bb17 --- /dev/null +++ b/mail/pkg/tools/get_authors.sh @@ -0,0 +1,2 @@ +#!/bin/sh +git log --format='%aN <%aE>' | awk '{arr[$0]++} END{for (i in arr){print arr[i], i;}}' | sort -rn | cut -d' ' -f2- diff --git a/mail/pkg/tools/with_venvwrapper.sh b/mail/pkg/tools/with_venvwrapper.sh new file mode 100755 index 00000000..693c0ac9 --- /dev/null +++ b/mail/pkg/tools/with_venvwrapper.sh @@ -0,0 +1,16 @@ +#!/bin/bash + +#Wraps a command in a virtualenwrapper passed as first argument. +#Example: +#with_virtualenvwrapper.sh leap-bitmask ./run_tests.sh + +wd=`pwd` +alias pyver='python -c "import $1;print $1.__path__[0]; print $1.__version__;"' + +source `which virtualenvwrapper.sh` +echo "Activating virtualenv " $1 +echo "------------------------------------" +workon $1 +cd $wd +echo "running version: " `pyver leap.bitmask` +$2 $3 $4 $5 diff --git a/mail/pkg/utils.py b/mail/pkg/utils.py new file mode 100644 index 00000000..d1680102 --- /dev/null +++ b/mail/pkg/utils.py @@ -0,0 +1,101 @@ +# -*- coding: utf-8 -*- +# utils.py +# Copyright (C) 2013 LEAP +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see <http://www.gnu.org/licenses/>. +""" +Utils to help in the setup process +""" +import os +import re +import sys + + +def is_develop_mode(): + """ + Returns True if we're calling the setup script using the argument for + setuptools development mode. + + This avoids messing up with dependency pinning and order, the + responsibility of installing the leap dependencies is left to the + developer. + """ + args = sys.argv + devflags = "setup.py", "develop" + if (args[0], args[1]) == devflags: + return True + return False + + +def get_reqs_from_files(reqfiles): + """ + Returns the contents of the top requirement file listed as a + string list with the lines. + + @param reqfiles: requirement files to parse + @type reqfiles: list of str + """ + for reqfile in reqfiles: + if os.path.isfile(reqfile): + return open(reqfile, 'r').read().split('\n') + + +def parse_requirements(reqfiles=['requirements.txt', + 'requirements.pip', + 'pkg/requirements.pip']): + """ + Parses the requirement files provided. + + The passed reqfiles list is a list of possible locations to try, the + function will return the contents of the first path found. + + Checks the value of LEAP_VENV_SKIP_PYSIDE to see if it should + return PySide as a dep or not. Don't set, or set to 0 if you want + to install it through pip. + + @param reqfiles: requirement files to parse + @type reqfiles: list of str + """ + + requirements = [] + skip_pyside = os.getenv("LEAP_VENV_SKIP_PYSIDE", "0") != "0" + for line in get_reqs_from_files(reqfiles): + # -e git://foo.bar/baz/master#egg=foobar + if re.match(r'\s*-e\s+', line): + pass + # do not try to do anything with externals on vcs + # requirements.append(re.sub(r'\s*-e\s+.*#egg=(.*)$', r'\1', + # line)) + # http://foo.bar/baz/foobar/zipball/master#egg=foobar + elif re.match(r'\s*https?:', line): + requirements.append(re.sub(r'\s*https?:.*#egg=(.*)$', r'\1', + line)) + # -f lines are for index locations, and don't get used here + elif re.match(r'\s*-f\s+', line): + pass + + # argparse is part of the standard library starting with 2.7 + # adding it to the requirements list screws distro installs + elif line == 'argparse' and sys.version_info >= (2, 7): + pass + elif line == 'PySide' and skip_pyside: + pass + # do not include comments + elif line.lstrip().startswith('#'): + pass + else: + if line != '': + requirements.append(line) + + return requirements diff --git a/mail/setup.cfg b/mail/setup.cfg new file mode 100644 index 00000000..a49fd352 --- /dev/null +++ b/mail/setup.cfg @@ -0,0 +1,24 @@ +[aliases] +test = trial + +[pep8] +exclude = versioneer.py,_version.py,*.egg,build,docs +ignore = E731 + +[flake8] +exclude = versioneer.py,_version.py,*.egg,build,docs +ignore = E731 + +[tool:pytest] +pep8ignore = + docs/conf.py ALL + versioneer.py ALL + _version.py ALL + *.egg ALL + +[versioneer] +VCS = git +style = pep440 +versionfile_source = src/leap/mail/_version.py +versionfile_build = leap/mail/_version.py +tag_prefix = diff --git a/mail/setup.py b/mail/setup.py new file mode 100644 index 00000000..ede620da --- /dev/null +++ b/mail/setup.py @@ -0,0 +1,156 @@ +# -*- coding: utf-8 -*- +# setup.py +# Copyright (C) 2013 LEAP +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see <http://www.gnu.org/licenses/>. +""" +Setup file for leap.mail +""" +import re +from setuptools import setup +from setuptools import find_packages +from setuptools import Command + +from pkg import utils + + +import versioneer + +trove_classifiers = [ + 'Development Status :: 4 - Beta', + 'Framework :: Twisted', + 'Intended Audience :: Developers', + 'License :: OSI Approved :: GNU General Public License ' + 'v3 (GPLv3)', + 'Operating System :: OS Independent', + 'Programming Language :: Python', + 'Programming Language :: Python :: 2.6', + 'Programming Language :: Python :: 2.7', + 'Topic :: Communications :: Email', + 'Topic :: Communications :: Email :: Post-Office :: IMAP', + 'Topic :: Communications :: Email :: Post-Office :: POP3', + 'Topic :: Internet', + 'Topic :: Security :: Cryptography', + 'Topic :: Software Development :: Libraries', +] + +DOWNLOAD_BASE = ('https://github.com/leapcode/leap_mail/' + 'archive/%s.tar.gz') +_versions = versioneer.get_versions() +VERSION = _versions['version'] +VERSION_REVISION = _versions['full-revisionid'] +DOWNLOAD_URL = "" + +# get the short version for the download url +_version_short = re.findall('\d+\.\d+\.\d+', VERSION) +if len(_version_short) > 0: + VERSION_SHORT = _version_short[0] + DOWNLOAD_URL = DOWNLOAD_BASE % VERSION_SHORT + +cmdclass = versioneer.get_cmdclass() + + +class freeze_debianver(Command): + + """ + Freezes the version in a debian branch. + To be used after merging the development branch onto the debian one. + """ + user_options = [] + template = r""" +# This file was generated by the `freeze_debianver` command in setup.py +# Using 'versioneer.py' (0.16) from +# revision-control system data, or from the parent directory name of an +# unpacked source archive. Distribution tarballs contain a pre-generated copy +# of this file. + +import json +import sys + +version_json = ''' +{ + "dirty": false, + "error": null, + "full-revisionid": "FULL_REVISIONID", + "version": "VERSION_STRING" +} +''' # END VERSION_JSON + +def get_versions(): + return json.loads(version_json) +""" + + def initialize_options(self): + pass + + def finalize_options(self): + pass + + def run(self): + proceed = str(raw_input( + "This will overwrite the file _version.py. Continue? [y/N] ")) + if proceed != "y": + print("He. You scared. Aborting.") + return + subst_template = self.template.replace( + 'VERSION_STRING', VERSION_SHORT).replace( + 'FULL_REVISIONID', VERSION_REVISION) + versioneer_cfg = versioneer.get_config_from_root('.') + with open(versioneer_cfg.versionfile_source, 'w') as f: + f.write(subst_template) + + +cmdclass["freeze_debianver"] = freeze_debianver + +# XXX add ref to docs + +requirements = utils.parse_requirements() + +if utils.is_develop_mode(): + print + print ("[WARNING] Skipping leap-specific dependencies " + "because development mode is detected.") + print ("[WARNING] You can install " + "the latest published versions with " + "'pip install -r pkg/requirements-leap.pip'") + print ("[WARNING] Or you can instead do 'python setup.py develop' " + "from the parent folder of each one of them.") + print +else: + requirements += utils.parse_requirements( + reqfiles=["pkg/requirements-leap.pip"]) + +setup( + name='leap.mail', + version=VERSION, + cmdclass=cmdclass, + url='https://leap.se/', + download_url=DOWNLOAD_URL, + license='GPLv3+', + author='The LEAP Encryption Access Project', + author_email='info@leap.se', + maintainer='Kali Kaneko', + maintainer_email='kali@leap.se', + description='Mail Services provided by Bitmask, the LEAP Client.', + long_description=open('README.rst').read() + '\n\n\n' + + open('CHANGELOG.rst').read(), + classifiers=trove_classifiers, + namespace_packages=["leap"], + package_dir={'': 'src'}, + packages=find_packages('src'), + test_suite='leap.mail.load_tests.load_tests', + install_requires=requirements, + tests_require=utils.parse_requirements( + reqfiles=['pkg/requirements-testing.pip']), +) diff --git a/mail/src/leap/__init__.py b/mail/src/leap/__init__.py new file mode 100644 index 00000000..f48ad105 --- /dev/null +++ b/mail/src/leap/__init__.py @@ -0,0 +1,6 @@ +# See http://peak.telecommunity.com/DevCenter/setuptools#namespace-packages +try: + __import__('pkg_resources').declare_namespace(__name__) +except ImportError: + from pkgutil import extend_path + __path__ = extend_path(__path__, __name__) diff --git a/mail/src/leap/mail/__init__.py b/mail/src/leap/mail/__init__.py new file mode 100644 index 00000000..4b25fe62 --- /dev/null +++ b/mail/src/leap/mail/__init__.py @@ -0,0 +1,26 @@ +# -*- coding: utf-8 -*- +# __init__.py +# Copyright (C) 2013 LEAP +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see <http://www.gnu.org/licenses/>. + + +""" +Client mail bits. +""" + + +from ._version import get_versions +__version__ = get_versions()['version'] +del get_versions diff --git a/mail/src/leap/mail/_version.py b/mail/src/leap/mail/_version.py new file mode 100644 index 00000000..954f4882 --- /dev/null +++ b/mail/src/leap/mail/_version.py @@ -0,0 +1,484 @@ + +# This file helps to compute a version number in source trees obtained from +# git-archive tarball (such as those provided by githubs download-from-tag +# feature). Distribution tarballs (built by setup.py sdist) and build +# directories (produced by setup.py build) will contain a much shorter file +# that just contains the computed version number. + +# This file is released into the public domain. Generated by +# versioneer-0.16 (https://github.com/warner/python-versioneer) + +"""Git implementation of _version.py.""" + +import errno +import os +import re +import subprocess +import sys + + +def get_keywords(): + """Get the keywords needed to look up the version information.""" + # these strings will be replaced by git during git-archive. + # setup.py/versioneer.py will grep for the variable names, so they must + # each be defined on a line of their own. _version.py will just call + # get_keywords(). + git_refnames = "$Format:%d$" + git_full = "$Format:%H$" + keywords = {"refnames": git_refnames, "full": git_full} + return keywords + + +class VersioneerConfig: + """Container for Versioneer configuration parameters.""" + + +def get_config(): + """Create, populate and return the VersioneerConfig() object.""" + # these strings are filled in when 'setup.py versioneer' creates + # _version.py + cfg = VersioneerConfig() + cfg.VCS = "git" + cfg.style = "pep440" + cfg.tag_prefix = "" + cfg.parentdir_prefix = "None" + cfg.versionfile_source = "src/leap/mail/_version.py" + cfg.verbose = False + return cfg + + +class NotThisMethod(Exception): + """Exception raised if a method is not valid for the current scenario.""" + + +LONG_VERSION_PY = {} +HANDLERS = {} + + +def register_vcs_handler(vcs, method): # decorator + """Decorator to mark a method as the handler for a particular VCS.""" + def decorate(f): + """Store f in HANDLERS[vcs][method].""" + if vcs not in HANDLERS: + HANDLERS[vcs] = {} + HANDLERS[vcs][method] = f + return f + return decorate + + +def run_command(commands, args, cwd=None, verbose=False, hide_stderr=False): + """Call the given command(s).""" + assert isinstance(commands, list) + p = None + for c in commands: + try: + dispcmd = str([c] + args) + # remember shell=False, so use git.cmd on windows, not just git + p = subprocess.Popen([c] + args, cwd=cwd, stdout=subprocess.PIPE, + stderr=(subprocess.PIPE if hide_stderr + else None)) + break + except EnvironmentError: + e = sys.exc_info()[1] + if e.errno == errno.ENOENT: + continue + if verbose: + print("unable to run %s" % dispcmd) + print(e) + return None + else: + if verbose: + print("unable to find command, tried %s" % (commands,)) + return None + stdout = p.communicate()[0].strip() + if sys.version_info[0] >= 3: + stdout = stdout.decode() + if p.returncode != 0: + if verbose: + print("unable to run %s (error)" % dispcmd) + return None + return stdout + + +def versions_from_parentdir(parentdir_prefix, root, verbose): + """Try to determine the version from the parent directory name. + + Source tarballs conventionally unpack into a directory that includes + both the project name and a version string. + """ + dirname = os.path.basename(root) + if not dirname.startswith(parentdir_prefix): + if verbose: + print("guessing rootdir is '%s', but '%s' doesn't start with " + "prefix '%s'" % (root, dirname, parentdir_prefix)) + raise NotThisMethod("rootdir doesn't start with parentdir_prefix") + return {"version": dirname[len(parentdir_prefix):], + "full-revisionid": None, + "dirty": False, "error": None} + + +@register_vcs_handler("git", "get_keywords") +def git_get_keywords(versionfile_abs): + """Extract version information from the given file.""" + # the code embedded in _version.py can just fetch the value of these + # keywords. When used from setup.py, we don't want to import _version.py, + # so we do it with a regexp instead. This function is not used from + # _version.py. + keywords = {} + try: + f = open(versionfile_abs, "r") + for line in f.readlines(): + if line.strip().startswith("git_refnames ="): + mo = re.search(r'=\s*"(.*)"', line) + if mo: + keywords["refnames"] = mo.group(1) + if line.strip().startswith("git_full ="): + mo = re.search(r'=\s*"(.*)"', line) + if mo: + keywords["full"] = mo.group(1) + f.close() + except EnvironmentError: + pass + return keywords + + +@register_vcs_handler("git", "keywords") +def git_versions_from_keywords(keywords, tag_prefix, verbose): + """Get version information from git keywords.""" + if not keywords: + raise NotThisMethod("no keywords at all, weird") + refnames = keywords["refnames"].strip() + if refnames.startswith("$Format"): + if verbose: + print("keywords are unexpanded, not using") + raise NotThisMethod("unexpanded keywords, not a git-archive tarball") + refs = set([r.strip() for r in refnames.strip("()").split(",")]) + # starting in git-1.8.3, tags are listed as "tag: foo-1.0" instead of + # just "foo-1.0". If we see a "tag: " prefix, prefer those. + TAG = "tag: " + tags = set([r[len(TAG):] for r in refs if r.startswith(TAG)]) + if not tags: + # Either we're using git < 1.8.3, or there really are no tags. We use + # a heuristic: assume all version tags have a digit. The old git %d + # expansion behaves like git log --decorate=short and strips out the + # refs/heads/ and refs/tags/ prefixes that would let us distinguish + # between branches and tags. By ignoring refnames without digits, we + # filter out many common branch names like "release" and + # "stabilization", as well as "HEAD" and "master". + tags = set([r for r in refs if re.search(r'\d', r)]) + if verbose: + print("discarding '%s', no digits" % ",".join(refs-tags)) + if verbose: + print("likely tags: %s" % ",".join(sorted(tags))) + for ref in sorted(tags): + # sorting will prefer e.g. "2.0" over "2.0rc1" + if ref.startswith(tag_prefix): + r = ref[len(tag_prefix):] + if verbose: + print("picking %s" % r) + return {"version": r, + "full-revisionid": keywords["full"].strip(), + "dirty": False, "error": None + } + # no suitable tags, so version is "0+unknown", but full hex is still there + if verbose: + print("no suitable tags, using unknown + full revision id") + return {"version": "0+unknown", + "full-revisionid": keywords["full"].strip(), + "dirty": False, "error": "no suitable tags"} + + +@register_vcs_handler("git", "pieces_from_vcs") +def git_pieces_from_vcs(tag_prefix, root, verbose, run_command=run_command): + """Get version from 'git describe' in the root of the source tree. + + This only gets called if the git-archive 'subst' keywords were *not* + expanded, and _version.py hasn't already been rewritten with a short + version string, meaning we're inside a checked out source tree. + """ + if not os.path.exists(os.path.join(root, ".git")): + if verbose: + print("no .git in %s" % root) + raise NotThisMethod("no .git directory") + + GITS = ["git"] + if sys.platform == "win32": + GITS = ["git.cmd", "git.exe"] + # if there is a tag matching tag_prefix, this yields TAG-NUM-gHEX[-dirty] + # if there isn't one, this yields HEX[-dirty] (no NUM) + describe_out = run_command(GITS, ["describe", "--tags", "--dirty", + "--always", "--long", + "--match", "%s*" % tag_prefix], + cwd=root) + # --long was added in git-1.5.5 + if describe_out is None: + raise NotThisMethod("'git describe' failed") + describe_out = describe_out.strip() + full_out = run_command(GITS, ["rev-parse", "HEAD"], cwd=root) + if full_out is None: + raise NotThisMethod("'git rev-parse' failed") + full_out = full_out.strip() + + pieces = {} + pieces["long"] = full_out + pieces["short"] = full_out[:7] # maybe improved later + pieces["error"] = None + + # parse describe_out. It will be like TAG-NUM-gHEX[-dirty] or HEX[-dirty] + # TAG might have hyphens. + git_describe = describe_out + + # look for -dirty suffix + dirty = git_describe.endswith("-dirty") + pieces["dirty"] = dirty + if dirty: + git_describe = git_describe[:git_describe.rindex("-dirty")] + + # now we have TAG-NUM-gHEX or HEX + + if "-" in git_describe: + # TAG-NUM-gHEX + mo = re.search(r'^(.+)-(\d+)-g([0-9a-f]+)$', git_describe) + if not mo: + # unparseable. Maybe git-describe is misbehaving? + pieces["error"] = ("unable to parse git-describe output: '%s'" + % describe_out) + return pieces + + # tag + full_tag = mo.group(1) + if not full_tag.startswith(tag_prefix): + if verbose: + fmt = "tag '%s' doesn't start with prefix '%s'" + print(fmt % (full_tag, tag_prefix)) + pieces["error"] = ("tag '%s' doesn't start with prefix '%s'" + % (full_tag, tag_prefix)) + return pieces + pieces["closest-tag"] = full_tag[len(tag_prefix):] + + # distance: number of commits since tag + pieces["distance"] = int(mo.group(2)) + + # commit: short hex revision ID + pieces["short"] = mo.group(3) + + else: + # HEX: no tags + pieces["closest-tag"] = None + count_out = run_command(GITS, ["rev-list", "HEAD", "--count"], + cwd=root) + pieces["distance"] = int(count_out) # total number of commits + + return pieces + + +def plus_or_dot(pieces): + """Return a + if we don't already have one, else return a .""" + if "+" in pieces.get("closest-tag", ""): + return "." + return "+" + + +def render_pep440(pieces): + """Build up version string, with post-release "local version identifier". + + Our goal: TAG[+DISTANCE.gHEX[.dirty]] . Note that if you + get a tagged build and then dirty it, you'll get TAG+0.gHEX.dirty + + Exceptions: + 1: no tags. git_describe was just HEX. 0+untagged.DISTANCE.gHEX[.dirty] + """ + if pieces["closest-tag"]: + rendered = pieces["closest-tag"] + if pieces["distance"] or pieces["dirty"]: + rendered += plus_or_dot(pieces) + rendered += "%d.g%s" % (pieces["distance"], pieces["short"]) + if pieces["dirty"]: + rendered += ".dirty" + else: + # exception #1 + rendered = "0+untagged.%d.g%s" % (pieces["distance"], + pieces["short"]) + if pieces["dirty"]: + rendered += ".dirty" + return rendered + + +def render_pep440_pre(pieces): + """TAG[.post.devDISTANCE] -- No -dirty. + + Exceptions: + 1: no tags. 0.post.devDISTANCE + """ + if pieces["closest-tag"]: + rendered = pieces["closest-tag"] + if pieces["distance"]: + rendered += ".post.dev%d" % pieces["distance"] + else: + # exception #1 + rendered = "0.post.dev%d" % pieces["distance"] + return rendered + + +def render_pep440_post(pieces): + """TAG[.postDISTANCE[.dev0]+gHEX] . + + The ".dev0" means dirty. Note that .dev0 sorts backwards + (a dirty tree will appear "older" than the corresponding clean one), + but you shouldn't be releasing software with -dirty anyways. + + Exceptions: + 1: no tags. 0.postDISTANCE[.dev0] + """ + if pieces["closest-tag"]: + rendered = pieces["closest-tag"] + if pieces["distance"] or pieces["dirty"]: + rendered += ".post%d" % pieces["distance"] + if pieces["dirty"]: + rendered += ".dev0" + rendered += plus_or_dot(pieces) + rendered += "g%s" % pieces["short"] + else: + # exception #1 + rendered = "0.post%d" % pieces["distance"] + if pieces["dirty"]: + rendered += ".dev0" + rendered += "+g%s" % pieces["short"] + return rendered + + +def render_pep440_old(pieces): + """TAG[.postDISTANCE[.dev0]] . + + The ".dev0" means dirty. + + Eexceptions: + 1: no tags. 0.postDISTANCE[.dev0] + """ + if pieces["closest-tag"]: + rendered = pieces["closest-tag"] + if pieces["distance"] or pieces["dirty"]: + rendered += ".post%d" % pieces["distance"] + if pieces["dirty"]: + rendered += ".dev0" + else: + # exception #1 + rendered = "0.post%d" % pieces["distance"] + if pieces["dirty"]: + rendered += ".dev0" + return rendered + + +def render_git_describe(pieces): + """TAG[-DISTANCE-gHEX][-dirty]. + + Like 'git describe --tags --dirty --always'. + + Exceptions: + 1: no tags. HEX[-dirty] (note: no 'g' prefix) + """ + if pieces["closest-tag"]: + rendered = pieces["closest-tag"] + if pieces["distance"]: + rendered += "-%d-g%s" % (pieces["distance"], pieces["short"]) + else: + # exception #1 + rendered = pieces["short"] + if pieces["dirty"]: + rendered += "-dirty" + return rendered + + +def render_git_describe_long(pieces): + """TAG-DISTANCE-gHEX[-dirty]. + + Like 'git describe --tags --dirty --always -long'. + The distance/hash is unconditional. + + Exceptions: + 1: no tags. HEX[-dirty] (note: no 'g' prefix) + """ + if pieces["closest-tag"]: + rendered = pieces["closest-tag"] + rendered += "-%d-g%s" % (pieces["distance"], pieces["short"]) + else: + # exception #1 + rendered = pieces["short"] + if pieces["dirty"]: + rendered += "-dirty" + return rendered + + +def render(pieces, style): + """Render the given version pieces into the requested style.""" + if pieces["error"]: + return {"version": "unknown", + "full-revisionid": pieces.get("long"), + "dirty": None, + "error": pieces["error"]} + + if not style or style == "default": + style = "pep440" # the default + + if style == "pep440": + rendered = render_pep440(pieces) + elif style == "pep440-pre": + rendered = render_pep440_pre(pieces) + elif style == "pep440-post": + rendered = render_pep440_post(pieces) + elif style == "pep440-old": + rendered = render_pep440_old(pieces) + elif style == "git-describe": + rendered = render_git_describe(pieces) + elif style == "git-describe-long": + rendered = render_git_describe_long(pieces) + else: + raise ValueError("unknown style '%s'" % style) + + return {"version": rendered, "full-revisionid": pieces["long"], + "dirty": pieces["dirty"], "error": None} + + +def get_versions(): + """Get version information or return default if unable to do so.""" + # I am in _version.py, which lives at ROOT/VERSIONFILE_SOURCE. If we have + # __file__, we can work backwards from there to the root. Some + # py2exe/bbfreeze/non-CPython implementations don't do __file__, in which + # case we can only use expanded keywords. + + cfg = get_config() + verbose = cfg.verbose + + try: + return git_versions_from_keywords(get_keywords(), cfg.tag_prefix, + verbose) + except NotThisMethod: + pass + + try: + root = os.path.realpath(__file__) + # versionfile_source is the relative path from the top of the source + # tree (where the .git directory might live) to this file. Invert + # this to find the root from __file__. + for i in cfg.versionfile_source.split('/'): + root = os.path.dirname(root) + except NameError: + return {"version": "0+unknown", "full-revisionid": None, + "dirty": None, + "error": "unable to find root of source tree"} + + try: + pieces = git_pieces_from_vcs(cfg.tag_prefix, root, verbose) + return render(pieces, cfg.style) + except NotThisMethod: + pass + + try: + if cfg.parentdir_prefix: + return versions_from_parentdir(cfg.parentdir_prefix, root, verbose) + except NotThisMethod: + pass + + return {"version": "0+unknown", "full-revisionid": None, + "dirty": None, + "error": "unable to compute version"} diff --git a/mail/src/leap/mail/adaptors/__init__.py b/mail/src/leap/mail/adaptors/__init__.py new file mode 100644 index 00000000..e69de29b --- /dev/null +++ b/mail/src/leap/mail/adaptors/__init__.py diff --git a/mail/src/leap/mail/adaptors/models.py b/mail/src/leap/mail/adaptors/models.py new file mode 100644 index 00000000..49460f74 --- /dev/null +++ b/mail/src/leap/mail/adaptors/models.py @@ -0,0 +1,123 @@ +# -*- coding: utf-8 -*- +# models.py +# Copyright (C) 2014 LEAP +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see <http://www.gnu.org/licenses/>. +""" +Generic Models to be used by the Document Adaptors. +""" +import copy + + +class SerializableModel(object): + """ + A Generic document model, that can be serialized into a dictionary. + + Subclasses of this `SerializableModel` are meant to be added as class + attributes of classes inheriting from DocumentWrapper. + + A subclass __meta__ of this SerializableModel might exist, and contain info + relative to particularities of this model. + + For instance, the use of `__meta__.index` marks the existence of a primary + index in the model, which will be used to do unique queries (in which case + all the other indexed fields in the underlying document will be filled with + the default info contained in the model definition). + """ + + @classmethod + def serialize(klass): + """ + Get a dictionary representation of the public attributes in the model + class. To avoid collisions with builtin functions, any occurrence of an + attribute ended in '_' (like 'type_') will be normalized by removing + the trailing underscore. + + This classmethod is used from within the serialized method of a + DocumentWrapper instance: it provides defaults for the + empty document. + """ + assert isinstance(klass, type) + return _normalize_dict(klass.__dict__) + + +class DocumentWrapper(object): + """ + A Wrapper object that can be manipulated, passed around, and serialized in + a format that the store understands. + It is related to a SerializableModel, which must be specified as the + ``model`` class attribute. The instance of this DocumentWrapper will not + allow any other *public* attributes than those defined in the corresponding + model. + """ + # TODO we could do some very basic type checking here + # TODO set a dirty flag (on __setattr__, whenever the value is != from + # before) + # TODO we could enforce the existence of a correct "model" attribute + # in some other way (other than in the initializer) + + def __init__(self, **kwargs): + if not getattr(self, 'model', None): + raise RuntimeError( + 'DocumentWrapper class needs a model attribute') + + defaults = self.model.serialize() + + if kwargs: + values = copy.deepcopy(defaults) + values.update(_normalize_dict(kwargs)) + else: + values = defaults + + for k, v in values.items(): + k = k.replace('-', '_') + setattr(self, k, v) + + def __setattr__(self, attr, value): + normalized = _normalize_dict(self.model.__dict__) + if not attr.startswith('_') and attr not in normalized: + raise RuntimeError( + "Cannot set attribute because it's not defined " + "in the model %s: %s" % (self.__class__, attr)) + object.__setattr__(self, attr, value) + + def serialize(self): + return _normalize_dict(self.__dict__) + + def create(self): + raise NotImplementedError() + + def update(self): + raise NotImplementedError() + + def delete(self): + raise NotImplementedError() + + @classmethod + def get_or_create(self): + raise NotImplementedError() + + @classmethod + def get_all(self): + raise NotImplementedError() + + +def _normalize_dict(_dict): + items = _dict.items() + items = filter(lambda (k, v): not callable(v), items) + items = filter(lambda (k, v): not k.startswith('_'), items) + items = [(k, v) if not k.endswith('_') else (k[:-1], v) + for (k, v) in items] + items = [(k.replace('-', '_'), v) for (k, v) in items] + return dict(items) diff --git a/mail/src/leap/mail/adaptors/soledad.py b/mail/src/leap/mail/adaptors/soledad.py new file mode 100644 index 00000000..ca8f741d --- /dev/null +++ b/mail/src/leap/mail/adaptors/soledad.py @@ -0,0 +1,1268 @@ +# soledad.py +# Copyright (C) 2014 LEAP +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see <http://www.gnu.org/licenses/>. +""" +Soledadad MailAdaptor module. +""" +import logging +import re + +from collections import defaultdict +from email import message_from_string + +from twisted.internet import defer +from twisted.python import log +from zope.interface import implements +from leap.soledad.common import l2db + +from leap.common.check import leap_assert, leap_assert_type + +from leap.mail import constants +from leap.mail import walk +from leap.mail.adaptors import soledad_indexes as indexes +from leap.mail.constants import INBOX_NAME +from leap.mail.adaptors import models +from leap.mail.imap.mailbox import normalize_mailbox +from leap.mail.utils import lowerdict, first +from leap.mail.utils import stringify_parts_map +from leap.mail.interfaces import IMailAdaptor, IMessageWrapper + +from leap.soledad.common.document import SoledadDocument + + +logger = logging.getLogger(__name__) + +# TODO +# [ ] Convenience function to create mail specifying subject, date, etc? + + +_MSGID_PATTERN = r"""<([\w@.]+)>""" +_MSGID_RE = re.compile(_MSGID_PATTERN) + + +class DuplicatedDocumentError(Exception): + """ + Raised when a duplicated document is detected. + """ + pass + + +def cleanup_deferred_locks(): + """ + Need to use this from within trial to cleanup the reactor before + each run. + """ + SoledadDocumentWrapper._k_locks = defaultdict(defer.DeferredLock) + + +class SoledadDocumentWrapper(models.DocumentWrapper): + """ + A Wrapper object that can be manipulated, passed around, and serialized in + a format that the Soledad Store understands. + + It ensures atomicity of the document operations on creation, update and + deletion. + """ + # TODO we could also use a _dirty flag (in models) + # TODO add a get_count() method ??? -- that is extended over l2db. + + # We keep a dictionary with DeferredLocks, that will be + # unique to every subclass of SoledadDocumentWrapper. + _k_locks = defaultdict(defer.DeferredLock) + + @classmethod + def _get_klass_lock(cls): + """ + Get a DeferredLock that is unique for this subclass name. + Used to lock the access to indexes in the `get_or_create` call + for a particular DocumentWrapper. + """ + return cls._k_locks[cls.__name__] + + def __init__(self, doc_id=None, future_doc_id=None, **kwargs): + self._doc_id = doc_id + self._future_doc_id = future_doc_id + self._lock = defer.DeferredLock() + super(SoledadDocumentWrapper, self).__init__(**kwargs) + + @property + def doc_id(self): + return self._doc_id + + @property + def future_doc_id(self): + return self._future_doc_id + + def set_future_doc_id(self, doc_id): + self._future_doc_id = doc_id + + def create(self, store, is_copy=False): + """ + Create the documents for this wrapper. + Since this method will not check for duplication, the + responsibility of avoiding duplicates is left to the caller. + + You might be interested in using `get_or_create` classmethod + instead (that's the preferred way of creating documents from + the wrapper object). + + :return: a deferred that will fire when the underlying + Soledad document has been created. + :rtype: Deferred + """ + leap_assert(self._doc_id is None, + "This document already has a doc_id!") + + def update_doc_id(doc): + self._doc_id = doc.doc_id + self.set_future_doc_id(None) + return doc + + def update_wrapper(failure): + # In the case of some copies (for instance, from one folder to + # another and back to the original folder), the document that we + # want to insert already exists. In this case, putting it + # and overwriting the document with that doc_id is the right thing + # to do. + failure.trap(l2db.errors.RevisionConflict) + self._doc_id = self.future_doc_id + self._future_doc_id = None + return self.update(store) + + if self.future_doc_id is None: + d = store.create_doc(self.serialize()) + else: + d = store.create_doc(self.serialize(), + doc_id=self.future_doc_id) + d.addCallback(update_doc_id) + + if is_copy: + d.addErrback(update_wrapper) + else: + d.addErrback(self._catch_revision_conflict, self.future_doc_id) + return d + + def update(self, store): + """ + Update the documents for this wrapper. + + :return: a deferred that will fire when the underlying + Soledad document has been updated. + :rtype: Deferred + """ + # the deferred lock guards against revision conflicts + return self._lock.run(self._update, store) + + def _update(self, store): + leap_assert(self._doc_id is not None, + "Need to create doc before updating") + + def update_and_put_doc(doc): + doc.content.update(self.serialize()) + d = store.put_doc(doc) + d.addErrback(self._catch_revision_conflict, doc.doc_id) + return d + + d = store.get_doc(self._doc_id) + d.addCallback(update_and_put_doc) + return d + + def _catch_revision_conflict(self, failure, doc_id): + # XXX We can have some RevisionConflicts if we try + # to put the docs that are already there. + # This can happen right now when creating/saving the cdocs + # during a copy. Instead of catching and ignoring this + # error, we should mark them in the copy so there is no attempt to + # create/update them. + failure.trap(l2db.errors.RevisionConflict) + logger.debug("Got conflict while putting %s" % doc_id) + + def delete(self, store): + """ + Delete the documents for this wrapper. + + :return: a deferred that will fire when the underlying + Soledad document has been deleted. + :rtype: Deferred + """ + # the deferred lock guards against conflicts while updating + return self._lock.run(self._delete, store) + + def _delete(self, store): + leap_assert(self._doc_id is not None, + "Need to create doc before deleting") + # XXX might want to flag this DocumentWrapper to avoid + # updating it by mistake. This could go in models.DocumentWrapper + + def delete_doc(doc): + return store.delete_doc(doc) + + d = store.get_doc(self._doc_id) + d.addCallback(delete_doc) + return d + + @classmethod + def get_or_create(cls, store, index, value): + """ + Get a unique DocumentWrapper by index, or create a new one if the + matching query does not exist. + + :param index: the primary index for the model. + :type index: str + :param value: the value to query the primary index. + :type value: str + + :return: a deferred that will be fired with the SoledadDocumentWrapper + matching the index query, either existing or just created. + :rtype: Deferred + """ + return cls._get_klass_lock().run( + cls._get_or_create, store, index, value) + + @classmethod + def _get_or_create(cls, store, index, value): + # TODO shorten this method. + assert store is not None + assert index is not None + assert value is not None + + def get_main_index(): + try: + return cls.model.__meta__.index + except AttributeError: + raise RuntimeError("The model is badly defined") + + # TODO separate into another method? + def try_to_get_doc_from_index(indexes): + values = [] + idx_def = dict(indexes)[index] + if len(idx_def) == 1: + values = [value] + else: + main_index = get_main_index() + fields = cls.model.serialize() + for field in idx_def: + if field == main_index: + values.append(value) + else: + values.append(fields[field]) + d = store.get_from_index(index, *values) + return d + + def get_first_doc_if_any(docs): + if not docs: + return None + if len(docs) > 1: + raise DuplicatedDocumentError + return docs[0] + + def wrap_existing_or_create_new(doc): + if doc: + return cls(doc_id=doc.doc_id, **doc.content) + else: + return create_and_wrap_new_doc() + + def create_and_wrap_new_doc(): + # XXX use closure to store indexes instead of + # querying for them again. + d = store.list_indexes() + d.addCallback(get_wrapper_instance_from_index) + d.addCallback(return_wrapper_when_created) + return d + + def get_wrapper_instance_from_index(indexes): + init_values = {} + idx_def = dict(indexes)[index] + if len(idx_def) == 1: + init_value = {idx_def[0]: value} + return cls(**init_value) + main_index = get_main_index() + fields = cls.model.serialize() + for field in idx_def: + if field == main_index: + init_values[field] = value + else: + init_values[field] = fields[field] + return cls(**init_values) + + def return_wrapper_when_created(wrapper): + d = wrapper.create(store) + d.addCallback(lambda doc: wrapper) + return d + + d = store.list_indexes() + d.addCallback(try_to_get_doc_from_index) + d.addCallback(get_first_doc_if_any) + d.addCallback(wrap_existing_or_create_new) + return d + + @classmethod + def get_all(cls, store): + """ + Get a collection of wrappers around all the documents belonging + to this kind. + + For this to work, the model.__meta__ needs to include a tuple with + the index to be used for listing purposes, and which is the field to be + used to query the index. + + Note that this method only supports indexes of a single field at the + moment. It also might be too expensive to return all the documents + matching the query, so handle with care. + + class __meta__(object): + index = "name" + list_index = ("by-type", "type_") + + :return: a deferred that will be fired with an iterable containing + as many SoledadDocumentWrapper are matching the index defined + in the model as the `list_index`. + :rtype: Deferred + """ + # TODO LIST (get_all) + # [ ] extend support to indexes with n-ples + # [ ] benchmark the cost of querying and returning indexes in a big + # database. This might badly need pagination before being put to + # serious use. + return cls._get_klass_lock().run(cls._get_all, store) + + @classmethod + def _get_all(cls, store): + try: + list_index, list_attr = cls.model.__meta__.list_index + except AttributeError: + raise RuntimeError("The model is badly defined: no list_index") + try: + index_value = getattr(cls.model, list_attr) + except AttributeError: + raise RuntimeError("The model is badly defined: " + "no attribute matching list_index") + + def wrap_docs(docs): + return (cls(doc_id=doc.doc_id, **doc.content) for doc in docs) + + d = store.get_from_index(list_index, index_value) + d.addCallback(wrap_docs) + return d + + def __repr__(self): + try: + idx = getattr(self, self.model.__meta__.index) + except AttributeError: + idx = "" + return "<%s: %s (%s)>" % (self.__class__.__name__, + idx, self._doc_id) + + +# +# Message documents +# + +class FlagsDocWrapper(SoledadDocumentWrapper): + + class model(models.SerializableModel): + type_ = "flags" + chash = "" + + mbox_uuid = "" + seen = False + deleted = False + recent = False + flags = [] + tags = [] + size = 0 + multi = False + + class __meta__(object): + index = "mbox" + + def set_mbox_uuid(self, mbox_uuid): + # XXX raise error if already created, should use copy instead + mbox_uuid = mbox_uuid.replace('-', '_') + new_id = constants.FDOCID.format(mbox_uuid=mbox_uuid, chash=self.chash) + self._future_doc_id = new_id + self.mbox_uuid = mbox_uuid + + def get_flags(self): + """ + Get the flags for this message (as a tuple of strings, not unicode). + """ + return map(str, self.flags) + + +class HeaderDocWrapper(SoledadDocumentWrapper): + + class model(models.SerializableModel): + type_ = "head" + chash = "" + + date = "" + subject = "" + headers = {} + part_map = {} + body = "" # link to phash of body + msgid = "" + multi = False + + class __meta__(object): + index = "chash" + + +class ContentDocWrapper(SoledadDocumentWrapper): + + class model(models.SerializableModel): + type_ = "cnt" + phash = "" + + ctype = "" # XXX index by ctype too? + lkf = [] # XXX not implemented yet! + raw = "" + + content_disposition = "" + content_transfer_encoding = "" + content_type = "" + + class __meta__(object): + index = "phash" + + +class MetaMsgDocWrapper(SoledadDocumentWrapper): + + class model(models.SerializableModel): + type_ = "meta" + fdoc = "" + hdoc = "" + cdocs = [] + + def set_mbox_uuid(self, mbox_uuid): + # XXX raise error if already created, should use copy instead + mbox_uuid = mbox_uuid.replace('-', '_') + chash = re.findall(constants.FDOCID_CHASH_RE, self.fdoc)[0] + new_id = constants.METAMSGID.format(mbox_uuid=mbox_uuid, chash=chash) + new_fdoc_id = constants.FDOCID.format(mbox_uuid=mbox_uuid, chash=chash) + self._future_doc_id = new_id + self.fdoc = new_fdoc_id + + +class MessageWrapper(object): + + # This could benefit of a DeferredLock to create/update all the + # documents at the same time maybe, and defend against concurrent updates? + + implements(IMessageWrapper) + + def __init__(self, mdoc, fdoc, hdoc, cdocs=None, is_copy=False): + """ + Need at least a metamsg-document, a flag-document and a header-document + to instantiate a MessageWrapper. Content-documents can be retrieved + lazily. + + cdocs, if any, should be a dictionary in which the keys are ascending + integers, beginning at one, and the values are dictionaries with the + content of the content-docs. + + is_copy, if set to True, will only attempt to create mdoc and fdoc + (because hdoc and cdocs are supposed to exist already) + """ + self._is_copy = is_copy + + def get_doc_wrapper(doc, cls): + if isinstance(doc, SoledadDocument): + doc_id = doc.doc_id + doc = doc.content + else: + doc_id = None + if not doc: + doc = {} + return cls(doc_id=doc_id, **doc) + + self.mdoc = get_doc_wrapper(mdoc, MetaMsgDocWrapper) + + self.fdoc = get_doc_wrapper(fdoc, FlagsDocWrapper) + self.fdoc.set_future_doc_id(self.mdoc.fdoc) + + self.hdoc = get_doc_wrapper(hdoc, HeaderDocWrapper) + self.hdoc.set_future_doc_id(self.mdoc.hdoc) + + if cdocs is None: + cdocs = {} + cdocs_keys = cdocs.keys() + assert sorted(cdocs_keys) == range(1, len(cdocs_keys) + 1) + self.cdocs = dict([ + (key, get_doc_wrapper(doc, ContentDocWrapper)) + for (key, doc) in cdocs.items()]) + for doc_id, cdoc in zip(self.mdoc.cdocs, self.cdocs.values()): + if cdoc.raw == "": + log.msg("Empty raw field in cdoc %s" % doc_id) + cdoc.set_future_doc_id(doc_id) + + def create(self, store, notify_just_mdoc=False, pending_inserts_dict=None): + """ + Create all the parts for this message in the store. + + :param store: an instance of Soledad + + :param notify_just_mdoc: + if set to True, this method will return *only* the deferred + corresponding to the creation of the meta-message document. + Be warned that in that case there will be no record of failures + when creating the other part-documents. + + Otherwise, this method will return a deferred that will wait for + the creation of all the part documents. + + Setting this flag to True is mostly a convenient workaround for the + fact that massive serial appends will take too much time, and in + most of the cases the MUA will only switch to the mailbox where the + appends have happened after a certain time, which in most of the + times will be enough to have all the queued insert operations + finished. + :type notify_just_mdoc: bool + :param pending_inserts_dict: + a dictionary with the pending inserts ids. + :type pending_inserts_dict: dict + + :return: a deferred whose callback will be called when either all the + part documents have been written, or just the metamsg-doc, + depending on the value of the notify_just_mdoc flag + :rtype: defer.Deferred + """ + if pending_inserts_dict is None: + pending_inserts_dict = {} + + leap_assert(self.cdocs, + "Need non empty cdocs to create the " + "MessageWrapper documents") + leap_assert(self.mdoc.doc_id is None, + "Cannot create: mdoc has a doc_id") + leap_assert(self.fdoc.doc_id is None, + "Cannot create: fdoc has a doc_id") + + def unblock_pending_insert(result): + if pending_inserts_dict: + ci_headers = lowerdict(self.hdoc.headers) + msgid = ci_headers.get('message-id', None) + try: + d = pending_inserts_dict[msgid] + d.callback(msgid) + except KeyError: + pass + return result + + # TODO check that the doc_ids in the mdoc are coherent + self.d = [] + + mdoc_created = self.mdoc.create(store, is_copy=self._is_copy) + fdoc_created = self.fdoc.create(store, is_copy=self._is_copy) + + mdoc_created.addErrback(lambda f: log.err(f)) + fdoc_created.addErrback(lambda f: log.err(f)) + + self.d.append(mdoc_created) + self.d.append(fdoc_created) + + if not self._is_copy: + if self.hdoc.doc_id is None: + self.d.append(self.hdoc.create(store)) + for cdoc in self.cdocs.values(): + if cdoc.doc_id is not None: + # we could be just linking to an existing + # content-doc. + continue + self.d.append(cdoc.create(store)) + + def log_all_inserted(result): + log.msg("All parts inserted for msg!") + return result + + self.all_inserted_d = defer.gatherResults(self.d, consumeErrors=True) + self.all_inserted_d.addCallback(log_all_inserted) + self.all_inserted_d.addCallback(unblock_pending_insert) + self.all_inserted_d.addErrback(lambda failure: log.err(failure)) + + if notify_just_mdoc: + return mdoc_created + else: + return self.all_inserted_d + + def update(self, store): + """ + Update the only mutable parts, which are within the flags document. + """ + return self.fdoc.update(store) + + def delete(self, store): + # TODO + # Eventually this would have to do the duplicate search or send for the + # garbage collector. At least mdoc and t the mdoc and fdoc can be + # unlinked. + d = [] + if self.mdoc.doc_id: + d.append(self.mdoc.delete(store)) + d.append(self.fdoc.delete(store)) + return defer.gatherResults(d) + + def copy(self, store, new_mbox_uuid): + """ + Return a copy of this MessageWrapper in a new mailbox. + + :param store: an instance of Soledad, or anything that behaves alike. + :param new_mbox_uuid: the uuid of the mailbox where we are copying this + message to. + :type new_mbox_uuid: str + :rtype: MessageWrapper + """ + new_mdoc = self.mdoc.serialize() + new_fdoc = self.fdoc.serialize() + + # the future doc_ids is properly set because we modified + # the pointers in mdoc, which has precedence. + new_wrapper = MessageWrapper(new_mdoc, new_fdoc, None, None, + is_copy=True) + new_wrapper.hdoc = self.hdoc + new_wrapper.cdocs = self.cdocs + new_wrapper.set_mbox_uuid(new_mbox_uuid) + + # XXX could flag so that it only creates mdoc/fdoc... + + d = new_wrapper.create(store) + d.addCallback(lambda result: new_wrapper) + d.addErrback(lambda failure: log.err(failure)) + return d + + def set_mbox_uuid(self, mbox_uuid): + """ + Set the mailbox for this wrapper. + This method should only be used before the Documents for the + MessageWrapper have been created, will raise otherwise. + """ + mbox_uuid = mbox_uuid.replace('-', '_') + self.mdoc.set_mbox_uuid(mbox_uuid) + self.fdoc.set_mbox_uuid(mbox_uuid) + + def set_flags(self, flags): + # TODO serialize the get + update + if flags is None: + flags = tuple() + leap_assert_type(flags, tuple) + self.fdoc.flags = list(flags) + self.fdoc.deleted = "\\Deleted" in flags + self.fdoc.seen = "\\Seen" in flags + self.fdoc.recent = "\\Recent" in flags + + def set_tags(self, tags): + # TODO serialize the get + update + if tags is None: + tags = tuple() + leap_assert_type(tags, tuple) + self.fdoc.tags = list(tags) + + def set_date(self, date): + # XXX assert valid date format + self.hdoc.date = date + + def get_subpart_dict(self, index): + """ + :param index: the part to lookup, 1-indexed + :type index: int + :rtype: dict + """ + return self.hdoc.part_map[str(index)] + + def get_subpart_indexes(self): + return self.hdoc.part_map.keys() + + def get_body(self, store): + """ + :rtype: deferred + """ + body_phash = self.hdoc.body + if body_phash: + d = store.get_doc('C-' + body_phash) + d.addCallback(lambda doc: ContentDocWrapper(**doc.content)) + return d + elif self.cdocs: + return self.cdocs[1] + else: + return '' + +# +# Mailboxes +# + + +class MailboxWrapper(SoledadDocumentWrapper): + + class model(models.SerializableModel): + type_ = "mbox" + mbox = INBOX_NAME + uuid = None + flags = [] + recent = [] + created = 1 + closed = False + subscribed = False + + class __meta__(object): + index = "mbox" + list_index = (indexes.TYPE_IDX, 'type_') + + +# +# Soledad Adaptor +# + +class SoledadIndexMixin(object): + """ + This will need a class attribute `indexes`, that is a dictionary containing + the index definitions for the underlying l2db store underlying soledad. + + It needs to be in the following format: + {'index-name': ['field1', 'field2']} + + You can also add a class attribute `wait_for_indexes` to any class + inheriting from this Mixin, that should be a list of strings representing + the methods that need to wait until the indexes have been initialized + before being able to work properly. + """ + # TODO move this mixin to soledad itself + # so that each application can pass a set of indexes for their data model. + + # TODO could have a wrapper class for indexes, supporting introspection + # and __getattr__ + + # TODO make this an interface? + + indexes = {} + wait_for_indexes = [] + store_ready = False + + def initialize_store(self, store): + """ + Initialize the indexes in the database. + + :param store: store + :returns: a Deferred that will fire when the store is correctly + initialized. + :rtype: deferred + """ + # TODO I think we *should* get another deferredLock in here, but + # global to the soledad namespace, to protect from several points + # initializing soledad indexes at the same time. + self._wait_for_indexes() + + d = self._init_indexes(store) + d.addCallback(self._restore_waiting_methods) + return d + + def _init_indexes(self, store): + """ + Initialize the database indexes. + """ + leap_assert(store, "Cannot init indexes with null soledad") + leap_assert_type(self.indexes, dict) + + def _create_index(name, expression): + return store.create_index(name, *expression) + + def init_idexes(indexes): + deferreds = [] + db_indexes = dict(indexes) + # Loop through the indexes we expect to find. + for name, expression in self.indexes.items(): + if name not in db_indexes: + # The index does not yet exist. + d = _create_index(name, expression) + deferreds.append(d) + elif expression != db_indexes[name]: + # The index exists but the definition is not what expected, + # so we delete it and add the proper index expression. + d = store.delete_index(name) + d.addCallback( + lambda _: _create_index(name, *expression)) + deferreds.append(d) + return defer.gatherResults(deferreds, consumeErrors=True) + + def store_ready(whatever): + self.store_ready = True + return whatever + + self.deferred_indexes = store.list_indexes() + self.deferred_indexes.addCallback(init_idexes) + self.deferred_indexes.addCallback(store_ready) + return self.deferred_indexes + + def _wait_for_indexes(self): + """ + Make the marked methods to wait for the indexes to be ready. + Heavily based on + http://blogs.fluidinfo.com/terry/2009/05/11/a-mixin-class-allowing-python-__init__-methods-to-work-with-twisted-deferreds/ + + :param methods: methods that need to wait for the indexes to be ready + :type methods: tuple(str) + """ + leap_assert_type(self.wait_for_indexes, list) + methods = self.wait_for_indexes + + self.waiting = [] + self.stored = {} + + def makeWrapper(method): + def wrapper(*args, **kw): + d = defer.Deferred() + d.addCallback(lambda _: self.stored[method](*args, **kw)) + self.waiting.append(d) + return d + return wrapper + + for method in methods: + self.stored[method] = getattr(self, method) + setattr(self, method, makeWrapper(method)) + + def _restore_waiting_methods(self, _): + for method in self.stored: + setattr(self, method, self.stored[method]) + for d in self.waiting: + d.callback(None) + + +class SoledadMailAdaptor(SoledadIndexMixin): + + implements(IMailAdaptor) + store = None + + indexes = indexes.MAIL_INDEXES + wait_for_indexes = ['get_or_create_mbox', 'update_mbox', 'get_all_mboxes'] + + mboxwrapper_klass = MailboxWrapper + + def __init__(self): + SoledadIndexMixin.__init__(self) + + # Message handling + + def get_msg_from_string(self, MessageClass, raw_msg): + """ + Get an instance of a MessageClass initialized with a MessageWrapper + that contains all the parts obtained from parsing the raw string for + the message. + + :param MessageClass: any Message class that can be initialized passing + an instance of an IMessageWrapper implementor. + :type MessageClass: type + :param raw_msg: a string containing the raw email message. + :type raw_msg: str + :rtype: MessageClass instance. + """ + assert(MessageClass is not None) + mdoc, fdoc, hdoc, cdocs = _split_into_parts(raw_msg) + return self.get_msg_from_docs( + MessageClass, mdoc, fdoc, hdoc, cdocs) + + def get_msg_from_docs(self, MessageClass, mdoc, fdoc, hdoc, cdocs=None, + uid=None): + """ + Get an instance of a MessageClass initialized with a MessageWrapper + that contains the passed part documents. + + This is not the recommended way of obtaining a message, unless you know + how to take care of ensuring the internal consistency between the part + documents, or unless you are glueing together the part documents that + have been previously generated by `get_msg_from_string`. + + :param MessageClass: any Message class that can be initialized passing + an instance of an IMessageWrapper implementor. + :type MessageClass: type + :param fdoc: a dictionary containing values from which a + FlagsDocWrapper can be initialized + :type fdoc: dict + :param hdoc: a dictionary containing values from which a + HeaderDocWrapper can be initialized + :type hdoc: dict + :param cdocs: None, or a dictionary mapping integers (1-indexed) to + dicts from where a ContentDocWrapper can be initialized. + :type cdocs: dict, or None + + :rtype: MessageClass instance. + """ + assert(MessageClass is not None) + return MessageClass(MessageWrapper(mdoc, fdoc, hdoc, cdocs), uid=uid) + + def get_msg_from_mdoc_id(self, MessageClass, store, mdoc_id, + uid=None, get_cdocs=False): + + def wrap_meta_doc(doc): + cls = MetaMsgDocWrapper + return cls(doc_id=doc.doc_id, **doc.content) + + def get_part_docs_from_mdoc_wrapper(wrapper): + d_docs = [] + d_docs.append(store.get_doc(wrapper.fdoc)) + d_docs.append(store.get_doc(wrapper.hdoc)) + for cdoc in wrapper.cdocs: + d_docs.append(store.get_doc(cdoc)) + + def add_mdoc(doc_list): + return [wrapper.serialize()] + doc_list + + d = defer.gatherResults(d_docs) + d.addCallback(add_mdoc) + return d + + def get_parts_doc_from_mdoc_id(): + mbox = re.findall(constants.METAMSGID_MBOX_RE, mdoc_id)[0] + chash = re.findall(constants.METAMSGID_CHASH_RE, mdoc_id)[0] + + def _get_fdoc_id_from_mdoc_id(): + return constants.FDOCID.format(mbox_uuid=mbox, chash=chash) + + def _get_hdoc_id_from_mdoc_id(): + return constants.HDOCID.format(mbox_uuid=mbox, chash=chash) + + d_docs = [] + fdoc_id = _get_fdoc_id_from_mdoc_id() + hdoc_id = _get_hdoc_id_from_mdoc_id() + + d_docs.append(store.get_doc(mdoc_id)) + d_docs.append(store.get_doc(fdoc_id)) + d_docs.append(store.get_doc(hdoc_id)) + + d = defer.gatherResults(d_docs) + return d + + def _err_log_failure_part_docs(failure): + # See https://leap.se/code/issues/7495. + # This avoids blocks, but the real cause still needs to be + # isolated (0.9.0rc3) -- kali + log.msg("BUG ---------------------------------------------------") + log.msg("BUG: Error while retrieving part docs for mdoc id %s" % + mdoc_id) + log.err(failure) + log.msg("BUG (please report above info) ------------------------") + return [] + + def _err_log_cannot_find_msg(failure): + log.msg("BUG: Error while getting msg (uid=%s)" % uid) + return None + + if get_cdocs: + d = store.get_doc(mdoc_id) + d.addCallback(wrap_meta_doc) + d.addCallback(get_part_docs_from_mdoc_wrapper) + d.addErrback(_err_log_failure_part_docs) + + else: + d = get_parts_doc_from_mdoc_id() + + d.addCallback(self._get_msg_from_variable_doc_list, + msg_class=MessageClass, uid=uid) + d.addErrback(_err_log_cannot_find_msg) + return d + + def _get_msg_from_variable_doc_list(self, doc_list, msg_class, uid=None): + if len(doc_list) == 3: + mdoc, fdoc, hdoc = doc_list + cdocs = None + elif len(doc_list) > 3: + # XXX is this case used? + mdoc, fdoc, hdoc = doc_list[:3] + cdocs = dict(enumerate(doc_list[3:], 1)) + return self.get_msg_from_docs( + msg_class, mdoc, fdoc, hdoc, cdocs, uid=uid) + + def get_flags_from_mdoc_id(self, store, mdoc_id): + """ + # XXX stuff here... + """ + mbox = re.findall(constants.METAMSGID_MBOX_RE, mdoc_id)[0] + chash = re.findall(constants.METAMSGID_CHASH_RE, mdoc_id)[0] + + def _get_fdoc_id_from_mdoc_id(): + return constants.FDOCID.format(mbox_uuid=mbox, chash=chash) + + fdoc_id = _get_fdoc_id_from_mdoc_id() + + def wrap_fdoc(doc): + if not doc: + return + cls = FlagsDocWrapper + return cls(doc_id=doc.doc_id, **doc.content) + + def get_flags(fdoc_wrapper): + if not fdoc_wrapper: + return [] + return fdoc_wrapper.get_flags() + + d = store.get_doc(fdoc_id) + d.addCallback(wrap_fdoc) + d.addCallback(get_flags) + return d + + def create_msg(self, store, msg): + """ + :param store: an instance of soledad, or anything that behaves alike + :param msg: a Message object. + + :return: a Deferred that is fired when all the underlying documents + have been created. + :rtype: defer.Deferred + """ + wrapper = msg.get_wrapper() + return wrapper.create(store) + + def update_msg(self, store, msg): + """ + :param msg: a Message object. + :param store: an instance of soledad, or anything that behaves alike + :return: a Deferred that is fired when all the underlying documents + have been updated (actually, it's only the fdoc that's allowed + to update). + :rtype: defer.Deferred + """ + wrapper = msg.get_wrapper() + return wrapper.update(store) + + # batch deletion + + def del_all_flagged_messages(self, store, mbox_uuid): + """ + Delete all messages flagged as deleted. + """ + def err(failure): + log.err(failure) + + def delete_fdoc_and_mdoc_flagged(fdocs): + # low level here, not using the wrappers... + # get meta doc ids from the flag doc ids + fdoc_ids = [doc.doc_id for doc in fdocs] + mdoc_ids = map(lambda s: "M" + s[1:], fdoc_ids) + + def delete_all_docs(mdocs, fdocs): + mdocs = list(mdocs) + doc_ids = [m.doc_id for m in mdocs] + _d = [] + docs = mdocs + fdocs + for doc in docs: + _d.append(store.delete_doc(doc)) + d = defer.gatherResults(_d) + # return the mdocs ids only + d.addCallback(lambda _: doc_ids) + return d + + d = store.get_docs(mdoc_ids) + d.addCallback(delete_all_docs, fdocs) + d.addErrback(err) + return d + + type_ = FlagsDocWrapper.model.type_ + uuid = mbox_uuid.replace('-', '_') + deleted_index = indexes.TYPE_MBOX_DEL_IDX + + d = store.get_from_index(deleted_index, type_, uuid, "1") + d.addCallbacks(delete_fdoc_and_mdoc_flagged, err) + return d + + # count messages + + def get_count_unseen(self, store, mbox_uuid): + """ + Get the number of unseen messages for a given mailbox. + + :param store: instance of Soledad. + :param mbox_uuid: the uuid for this mailbox. + :rtype: int + """ + type_ = FlagsDocWrapper.model.type_ + uuid = mbox_uuid.replace('-', '_') + + unseen_index = indexes.TYPE_MBOX_SEEN_IDX + + d = store.get_count_from_index(unseen_index, type_, uuid, "0") + d.addErrback(self._errback) + return d + + def get_count_recent(self, store, mbox_uuid): + """ + Get the number of recent messages for a given mailbox. + + :param store: instance of Soledad. + :param mbox_uuid: the uuid for this mailbox. + :rtype: int + """ + type_ = FlagsDocWrapper.model.type_ + uuid = mbox_uuid.replace('-', '_') + + recent_index = indexes.TYPE_MBOX_RECENT_IDX + + d = store.get_count_from_index(recent_index, type_, uuid, "1") + d.addErrback(self._errback) + return d + + # search api + + def get_mdoc_id_from_msgid(self, store, mbox_uuid, msgid): + """ + Get the UID for a message with the passed msgid (the one in the headers + msg-id). + This is used by the MUA to retrieve the recently saved draft. + """ + type_ = HeaderDocWrapper.model.type_ + uuid = mbox_uuid.replace('-', '_') + + msgid_index = indexes.TYPE_MSGID_IDX + + def get_mdoc_id(hdoc): + if not hdoc: + log.msg("Could not find a HDOC with MSGID %s" % msgid) + return None + hdoc = hdoc[0] + mdoc_id = hdoc.doc_id.replace("H-", "M-%s-" % uuid) + return mdoc_id + + d = store.get_from_index(msgid_index, type_, msgid) + d.addCallback(get_mdoc_id) + return d + + # Mailbox handling + + def get_or_create_mbox(self, store, name): + """ + Get the mailbox with the given name, or create one if it does not + exist. + + :param store: instance of Soledad + :param name: the name of the mailbox + :type name: str + """ + index = indexes.TYPE_MBOX_IDX + mbox = normalize_mailbox(name) + return MailboxWrapper.get_or_create(store, index, mbox) + + def update_mbox(self, store, mbox_wrapper): + """ + Update the documents for a given mailbox. + :param mbox_wrapper: MailboxWrapper instance + :type mbox_wrapper: MailboxWrapper + :return: a Deferred that will be fired when the mailbox documents + have been updated. + :rtype: defer.Deferred + """ + leap_assert_type(mbox_wrapper, SoledadDocumentWrapper) + return mbox_wrapper.update(store) + + def delete_mbox(self, store, mbox_wrapper): + leap_assert_type(mbox_wrapper, SoledadDocumentWrapper) + return mbox_wrapper.delete(store) + + def get_all_mboxes(self, store): + """ + Retrieve a list with wrappers for all the mailboxes. + + :return: a deferred that will be fired with a list of all the + MailboxWrappers found. + :rtype: defer.Deferred + """ + return MailboxWrapper.get_all(store) + + def _errback(self, failure): + log.err(failure) + + +def _split_into_parts(raw): + # TODO signal that we can delete the original message!----- + # when all the processing is done. + # TODO add the linked-from info ! + # TODO add reference to the original message? + # TODO populate Default FLAGS/TAGS (unseen?) + # TODO seed propely the content_docs with defaults?? + + msg, chash, multi = _parse_msg(raw) + size = len(msg.as_string()) + + parts_map = walk.get_tree(msg) + cdocs_list = list(walk.get_raw_docs(msg)) + cdocs_phashes = [c['phash'] for c in cdocs_list] + body_phash = walk.get_body_phash(msg) + + mdoc = _build_meta_doc(chash, cdocs_phashes) + fdoc = _build_flags_doc(chash, size, multi) + hdoc = _build_headers_doc(msg, chash, body_phash, parts_map) + + # The MessageWrapper expects a dict, one-indexed + cdocs = dict(enumerate(cdocs_list, 1)) + + return mdoc, fdoc, hdoc, cdocs + + +def _parse_msg(raw): + msg = message_from_string(raw) + chash = walk.get_hash(raw) + multi = msg.is_multipart() + return msg, chash, multi + + +def _build_meta_doc(chash, cdocs_phashes): + _mdoc = MetaMsgDocWrapper() + # FIXME passing the inbox name because we don't have the uuid at this + # point. + + _mdoc.fdoc = constants.FDOCID.format(mbox_uuid=INBOX_NAME, chash=chash) + _mdoc.hdoc = constants.HDOCID.format(chash=chash) + _mdoc.cdocs = [constants.CDOCID.format(phash=p) for p in cdocs_phashes] + + return _mdoc.serialize() + + +def _build_flags_doc(chash, size, multi): + _fdoc = FlagsDocWrapper(chash=chash, size=size, multi=multi) + return _fdoc.serialize() + + +def _build_headers_doc(msg, chash, body_phash, parts_map): + """ + Assemble a headers document from the original parsed message, the + content-hash, and the parts map. + + It takes into account possibly repeated headers. + """ + headers = defaultdict(list) + for k, v in msg.items(): + headers[k].append(v) + # "fix" for repeated headers (as in "Received:" + for k, v in headers.items(): + newline = "\n%s: " % (k.lower(),) + headers[k] = newline.join(v) + + lower_headers = lowerdict(dict(headers)) + msgid = first(_MSGID_RE.findall( + lower_headers.get('message-id', ''))) + + _hdoc = HeaderDocWrapper( + chash=chash, headers=headers, body=body_phash, + msgid=msgid) + + def copy_attr(headers, key, doc): + if key in headers: + setattr(doc, key, headers[key]) + + copy_attr(lower_headers, "subject", _hdoc) + copy_attr(lower_headers, "date", _hdoc) + + hdoc = _hdoc.serialize() + # add some of the attr from the parts map to header doc + for key in parts_map: + if key in ('body', 'multi', 'part_map'): + hdoc[key] = parts_map[key] + return stringify_parts_map(hdoc) diff --git a/mail/src/leap/mail/adaptors/soledad_indexes.py b/mail/src/leap/mail/adaptors/soledad_indexes.py new file mode 100644 index 00000000..eec7d286 --- /dev/null +++ b/mail/src/leap/mail/adaptors/soledad_indexes.py @@ -0,0 +1,106 @@ +# -*- coding: utf-8 -*- +# soledad_indexes.py +# Copyright (C) 2013, 2014 LEAP +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see <http://www.gnu.org/licenses/>. +""" +Soledad Indexes for Mail Documents. +""" + +# TODO +# [ ] hide most of the constants here + +# Document Type, for indexing + +TYPE = "type" +MBOX = "mbox" +MBOX_UUID = "mbox_uuid" +FLAGS = "flags" +HEADERS = "head" +CONTENT = "cnt" +RECENT = "rct" +HDOCS_SET = "hdocset" + +INCOMING_KEY = "incoming" +ERROR_DECRYPTING_KEY = "errdecr" + +# indexing keys +CONTENT_HASH = "chash" +PAYLOAD_HASH = "phash" +MSGID = "msgid" +UID = "uid" + + +# Index types +# -------------- + +TYPE_IDX = 'by-type' +TYPE_MBOX_IDX = 'by-type-and-mbox' +TYPE_MBOX_UUID_IDX = 'by-type-and-mbox-uuid' +TYPE_SUBS_IDX = 'by-type-and-subscribed' +TYPE_MSGID_IDX = 'by-type-and-message-id' +TYPE_MBOX_SEEN_IDX = 'by-type-and-mbox-and-seen' +TYPE_MBOX_RECENT_IDX = 'by-type-and-mbox-and-recent' +TYPE_MBOX_DEL_IDX = 'by-type-and-mbox-and-deleted' +TYPE_MBOX_C_HASH_IDX = 'by-type-and-mbox-and-contenthash' +TYPE_C_HASH_IDX = 'by-type-and-contenthash' +TYPE_C_HASH_PART_IDX = 'by-type-and-contenthash-and-partnumber' +TYPE_P_HASH_IDX = 'by-type-and-payloadhash' + +# Soledad index for incoming mail, without decrypting errors. +# and the backward-compatible index, will be deprecated at 0.7 +JUST_MAIL_IDX = "just-mail" +JUST_MAIL_COMPAT_IDX = "just-mail-compat" + + +# TODO +# it would be nice to measure the cost of indexing +# by many fields. + +# TODO +# make the indexes dict more readable! + +MAIL_INDEXES = { + # generic + TYPE_IDX: [TYPE], + TYPE_MBOX_IDX: [TYPE, MBOX], + TYPE_MBOX_UUID_IDX: [TYPE, MBOX_UUID], + + # XXX deprecate 0.4.0 + # TYPE_MBOX_UID_IDX: [TYPE, MBOX, UID], + + # mailboxes + TYPE_SUBS_IDX: [TYPE, 'bool(subscribed)'], + + # fdocs uniqueness + TYPE_MBOX_C_HASH_IDX: [TYPE, MBOX, CONTENT_HASH], + + # headers doc - search by msgid. + TYPE_MSGID_IDX: [TYPE, MSGID], + + # content, headers doc + TYPE_C_HASH_IDX: [TYPE, CONTENT_HASH], + + # attachment payload dedup + TYPE_P_HASH_IDX: [TYPE, PAYLOAD_HASH], + + # messages + TYPE_MBOX_SEEN_IDX: [TYPE, MBOX_UUID, 'bool(seen)'], + TYPE_MBOX_RECENT_IDX: [TYPE, MBOX_UUID, 'bool(recent)'], + TYPE_MBOX_DEL_IDX: [TYPE, MBOX_UUID, 'bool(deleted)'], + + # incoming queue + JUST_MAIL_IDX: ["bool(%s)" % (INCOMING_KEY,), + "bool(%s)" % (ERROR_DECRYPTING_KEY,)], +} diff --git a/mail/src/leap/mail/adaptors/tests/rfc822.message b/mail/src/leap/mail/adaptors/tests/rfc822.message new file mode 120000 index 00000000..b19cc280 --- /dev/null +++ b/mail/src/leap/mail/adaptors/tests/rfc822.message @@ -0,0 +1 @@ +../../tests/rfc822.message
\ No newline at end of file diff --git a/mail/src/leap/mail/adaptors/tests/test_models.py b/mail/src/leap/mail/adaptors/tests/test_models.py new file mode 100644 index 00000000..b82cfad0 --- /dev/null +++ b/mail/src/leap/mail/adaptors/tests/test_models.py @@ -0,0 +1,106 @@ +# -*- coding: utf-8 -*- +# test_models.py +# Copyright (C) 2014 LEAP +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see <http://www.gnu.org/licenses/>. +""" +Tests for the leap.mail.adaptors.models module. +""" +from twisted.trial import unittest + +from leap.mail.adaptors import models + + +class SerializableModelsTestCase(unittest.TestCase): + + def test_good_serialized_model(self): + + class M(models.SerializableModel): + foo = 42 + bar = 33 + baaz_ = None + _nope = 0 + __nope = 0 + + def not_today(self): + pass + + class IgnoreMe(object): + pass + + def killmeplease(x): + return x + + serialized = M.serialize() + expected = {'foo': 42, 'bar': 33, 'baaz': None} + self.assertEqual(serialized, expected) + + +class DocumentWrapperTestCase(unittest.TestCase): + + def test_wrapper_defaults(self): + + class Wrapper(models.DocumentWrapper): + class model(models.SerializableModel): + foo = 42 + bar = 11 + + wrapper = Wrapper() + wrapper._ignored = True + serialized = wrapper.serialize() + expected = {'foo': 42, 'bar': 11} + self.assertEqual(serialized, expected) + + def test_initialized_wrapper(self): + + class Wrapper(models.DocumentWrapper): + class model(models.SerializableModel): + foo = 42 + bar_ = 11 + + wrapper = Wrapper(foo=0, bar=-1) + serialized = wrapper.serialize() + expected = {'foo': 0, 'bar': -1} + self.assertEqual(serialized, expected) + + wrapper.foo = 23 + serialized = wrapper.serialize() + expected = {'foo': 23, 'bar': -1} + self.assertEqual(serialized, expected) + + wrapper = Wrapper(foo=0) + serialized = wrapper.serialize() + expected = {'foo': 0, 'bar': 11} + self.assertEqual(serialized, expected) + + def test_invalid_initialized_wrapper(self): + + class Wrapper(models.DocumentWrapper): + class model(models.SerializableModel): + foo = 42 + + def getwrapper(): + return Wrapper(bar=1) + self.assertRaises(RuntimeError, getwrapper) + + def test_no_model_wrapper(self): + + class Wrapper(models.DocumentWrapper): + pass + + def getwrapper(): + w = Wrapper() + w.foo = None + + self.assertRaises(RuntimeError, getwrapper) diff --git a/mail/src/leap/mail/adaptors/tests/test_soledad_adaptor.py b/mail/src/leap/mail/adaptors/tests/test_soledad_adaptor.py new file mode 100644 index 00000000..73eaf164 --- /dev/null +++ b/mail/src/leap/mail/adaptors/tests/test_soledad_adaptor.py @@ -0,0 +1,529 @@ +# -*- coding: utf-8 -*- +# test_soledad_adaptor.py +# Copyright (C) 2014 LEAP +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see <http://www.gnu.org/licenses/>. +""" +Tests for the Soledad Adaptor module - leap.mail.adaptors.soledad +""" +import os +from functools import partial + +from twisted.internet import defer + +from leap.mail.adaptors import models +from leap.mail.adaptors.soledad import SoledadDocumentWrapper +from leap.mail.adaptors.soledad import SoledadIndexMixin +from leap.mail.adaptors.soledad import SoledadMailAdaptor +from leap.mail.testing.common import SoledadTestMixin + +from email.MIMEMultipart import MIMEMultipart +from email.mime.text import MIMEText + +# DEBUG +# import logging +# logging.basicConfig(level=logging.DEBUG) + + +class CounterWrapper(SoledadDocumentWrapper): + class model(models.SerializableModel): + counter = 0 + flag = None + + +class CharacterWrapper(SoledadDocumentWrapper): + class model(models.SerializableModel): + name = "" + age = 20 + + +class ActorWrapper(SoledadDocumentWrapper): + class model(models.SerializableModel): + type_ = "actor" + name = None + + class __meta__(object): + index = "name" + list_index = ("by-type", "type_") + + +class TestAdaptor(SoledadIndexMixin): + indexes = {'by-name': ['name'], + 'by-type-and-name': ['type', 'name'], + 'by-type': ['type']} + + +class SoledadDocWrapperTestCase(SoledadTestMixin): + """ + Tests for the SoledadDocumentWrapper. + """ + def assert_num_docs(self, num, docs): + self.assertEqual(len(docs[1]), num) + + def test_create_single(self): + + store = self._soledad + wrapper = CounterWrapper() + + def assert_one_doc(docs): + self.assertEqual(docs[0], 1) + + d = wrapper.create(store) + d.addCallback(lambda _: store.get_all_docs()) + d.addCallback(assert_one_doc) + return d + + def test_create_many(self): + + store = self._soledad + w1 = CounterWrapper() + w2 = CounterWrapper(counter=1) + w3 = CounterWrapper(counter=2) + w4 = CounterWrapper(counter=3) + w5 = CounterWrapper(counter=4) + + d1 = [w1.create(store), + w2.create(store), + w3.create(store), + w4.create(store), + w5.create(store)] + + d = defer.gatherResults(d1) + d.addCallback(lambda _: store.get_all_docs()) + d.addCallback(partial(self.assert_num_docs, 5)) + return d + + def test_multiple_updates(self): + + store = self._soledad + wrapper = CounterWrapper(counter=1) + MAX = 100 + + def assert_doc_id(doc): + self.assertTrue(wrapper._doc_id is not None) + return doc + + def assert_counter_initial_ok(doc): + self.assertEqual(wrapper.counter, 1) + + def increment_counter(ignored): + d1 = [] + + def record_revision(revision): + rev = int(revision.split(':')[1]) + self.results.append(rev) + + for i in list(range(MAX)): + wrapper.counter += 1 + wrapper.flag = i % 2 == 0 + d = wrapper.update(store) + d.addCallback(record_revision) + d1.append(d) + + return defer.gatherResults(d1) + + def assert_counter_final_ok(doc): + self.assertEqual(doc.content['counter'], MAX + 1) + self.assertEqual(doc.content['flag'], False) + + def assert_results_ordered_list(ignored): + self.assertEqual(self.results, sorted(range(2, MAX + 2))) + + d = wrapper.create(store) + d.addCallback(assert_doc_id) + d.addCallback(assert_counter_initial_ok) + d.addCallback(increment_counter) + d.addCallback(lambda _: store.get_doc(wrapper._doc_id)) + d.addCallback(assert_counter_final_ok) + d.addCallback(assert_results_ordered_list) + return d + + def test_delete(self): + adaptor = TestAdaptor() + store = self._soledad + + wrapper_list = [] + + def get_or_create_bob(ignored): + def add_to_list(wrapper): + wrapper_list.append(wrapper) + return wrapper + wrapper = CharacterWrapper.get_or_create( + store, 'by-name', 'bob') + wrapper.addCallback(add_to_list) + return wrapper + + def delete_bob(ignored): + wrapper = wrapper_list[0] + return wrapper.delete(store) + + d = adaptor.initialize_store(store) + d.addCallback(lambda _: store.get_all_docs()) + d.addCallback(partial(self.assert_num_docs, 0)) + + # this should create bob document + d.addCallback(get_or_create_bob) + d.addCallback(lambda _: store.get_all_docs()) + d.addCallback(partial(self.assert_num_docs, 1)) + + d.addCallback(delete_bob) + d.addCallback(lambda _: store.get_all_docs()) + d.addCallback(partial(self.assert_num_docs, 0)) + return d + + def test_get_or_create(self): + adaptor = TestAdaptor() + store = self._soledad + + def get_or_create_bob(ignored): + wrapper = CharacterWrapper.get_or_create( + store, 'by-name', 'bob') + return wrapper + + d = adaptor.initialize_store(store) + d.addCallback(lambda _: store.get_all_docs()) + d.addCallback(partial(self.assert_num_docs, 0)) + + # this should create bob document + d.addCallback(get_or_create_bob) + d.addCallback(lambda _: store.get_all_docs()) + d.addCallback(partial(self.assert_num_docs, 1)) + + # this should get us bob document + d.addCallback(get_or_create_bob) + d.addCallback(lambda _: store.get_all_docs()) + d.addCallback(partial(self.assert_num_docs, 1)) + return d + + def test_get_or_create_multi_index(self): + adaptor = TestAdaptor() + store = self._soledad + + def get_or_create_actor_harry(ignored): + wrapper = ActorWrapper.get_or_create( + store, 'by-type-and-name', 'harrison') + return wrapper + + def create_director_harry(ignored): + wrapper = ActorWrapper(name="harrison", type="director") + return wrapper.create(store) + + d = adaptor.initialize_store(store) + d.addCallback(lambda _: store.get_all_docs()) + d.addCallback(partial(self.assert_num_docs, 0)) + + # this should create harrison document + d.addCallback(get_or_create_actor_harry) + d.addCallback(lambda _: store.get_all_docs()) + d.addCallback(partial(self.assert_num_docs, 1)) + + # this should get us harrison document + d.addCallback(get_or_create_actor_harry) + d.addCallback(lambda _: store.get_all_docs()) + d.addCallback(partial(self.assert_num_docs, 1)) + + # create director harry, should create new doc + d.addCallback(create_director_harry) + d.addCallback(lambda _: store.get_all_docs()) + d.addCallback(partial(self.assert_num_docs, 2)) + + # this should get us harrison document, still 2 docs + d.addCallback(get_or_create_actor_harry) + d.addCallback(lambda _: store.get_all_docs()) + d.addCallback(partial(self.assert_num_docs, 2)) + return d + + def test_get_all(self): + adaptor = TestAdaptor() + store = self._soledad + actor_names = ["harry", "carrie", "mark", "david"] + + def create_some_actors(ignored): + deferreds = [] + for name in actor_names: + dw = ActorWrapper.get_or_create( + store, 'by-type-and-name', name) + deferreds.append(dw) + return defer.gatherResults(deferreds) + + d = adaptor.initialize_store(store) + d.addCallback(lambda _: store.get_all_docs()) + d.addCallback(partial(self.assert_num_docs, 0)) + + d.addCallback(create_some_actors) + + d.addCallback(lambda _: store.get_all_docs()) + d.addCallback(partial(self.assert_num_docs, 4)) + + def assert_actor_list_is_expected(res): + got = set([actor.name for actor in res]) + expected = set(actor_names) + self.assertEqual(got, expected) + + d.addCallback(lambda _: ActorWrapper.get_all(store)) + d.addCallback(assert_actor_list_is_expected) + return d + +HERE = os.path.split(os.path.abspath(__file__))[0] + + +class MessageClass(object): + def __init__(self, wrapper, uid): + self.wrapper = wrapper + self.uid = uid + + def get_wrapper(self): + return self.wrapper + + +class SoledadMailAdaptorTestCase(SoledadTestMixin): + """ + Tests for the SoledadMailAdaptor. + """ + + def get_adaptor(self): + adaptor = SoledadMailAdaptor() + adaptor.store = self._soledad + return adaptor + + def assert_num_docs(self, num, docs): + self.assertEqual(len(docs[1]), num) + + def test_mail_adaptor_init(self): + adaptor = self.get_adaptor() + self.assertTrue(isinstance(adaptor.indexes, dict)) + self.assertTrue(len(adaptor.indexes) != 0) + + # Messages + + def test_get_msg_from_string(self): + adaptor = self.get_adaptor() + + with open(os.path.join(HERE, "rfc822.message")) as f: + raw = f.read() + + msg = adaptor.get_msg_from_string(MessageClass, raw) + + chash = ("D27B2771C0DCCDCB468EE65A4540438" + "09DBD11588E87E951545BE0CBC321C308") + phash = ("64934534C1C80E0D4FA04BE1CCBA104" + "F07BCA5F469C86E2C0ABE1D41310B7299") + subject = ("[Twisted-commits] rebuild now works on " + "python versions from 2.2.0 and up.") + self.assertTrue(msg.wrapper.fdoc is not None) + self.assertTrue(msg.wrapper.hdoc is not None) + self.assertTrue(msg.wrapper.cdocs is not None) + self.assertEquals(len(msg.wrapper.cdocs), 1) + self.assertEquals(msg.wrapper.fdoc.chash, chash) + self.assertEquals(msg.wrapper.fdoc.size, 3837) + self.assertEquals(msg.wrapper.hdoc.chash, chash) + self.assertEqual(dict(msg.wrapper.hdoc.headers)['Subject'], + subject) + self.assertEqual(msg.wrapper.hdoc.subject, subject) + self.assertEqual(msg.wrapper.cdocs[1].phash, phash) + + def test_get_msg_from_string_multipart(self): + msg = MIMEMultipart() + msg['Subject'] = 'Test multipart mail' + msg.attach(MIMEText(u'a utf8 message', _charset='utf-8')) + adaptor = self.get_adaptor() + + msg = adaptor.get_msg_from_string(MessageClass, msg.as_string()) + + self.assertEqual( + 'base64', msg.wrapper.cdocs[1].content_transfer_encoding) + self.assertEqual( + 'text/plain', msg.wrapper.cdocs[1].content_type) + self.assertEqual( + 'YSB1dGY4IG1lc3NhZ2U=\n', msg.wrapper.cdocs[1].raw) + + def test_get_msg_from_docs(self): + adaptor = self.get_adaptor() + mdoc = dict( + fdoc="F-Foobox-deadbeef", + hdoc="H-deadbeef", + cdocs=["C-deadabad"]) + fdoc = dict( + mbox_uuid="Foobox", + flags=('\Seen', '\Nice'), + tags=('Personal', 'TODO'), + seen=False, deleted=False, + recent=False, multi=False) + hdoc = dict( + chash="deadbeef", + subject="Test Msg") + cdocs = { + 1: dict( + raw='This is a test message')} + + msg = adaptor.get_msg_from_docs( + MessageClass, mdoc, fdoc, hdoc, cdocs=cdocs) + self.assertEqual(msg.wrapper.fdoc.flags, + ('\Seen', '\Nice')) + self.assertEqual(msg.wrapper.fdoc.tags, + ('Personal', 'TODO')) + self.assertEqual(msg.wrapper.fdoc.mbox_uuid, "Foobox") + self.assertEqual(msg.wrapper.hdoc.multi, False) + self.assertEqual(msg.wrapper.hdoc.subject, + "Test Msg") + self.assertEqual(msg.wrapper.cdocs[1].raw, + "This is a test message") + + def test_get_msg_from_metamsg_doc_id(self): + # TODO complete-me! + pass + + test_get_msg_from_metamsg_doc_id.skip = "Not yet implemented" + + def test_create_msg(self): + adaptor = self.get_adaptor() + + with open(os.path.join(HERE, "rfc822.message")) as f: + raw = f.read() + msg = adaptor.get_msg_from_string(MessageClass, raw) + + def check_create_result(created): + # that's one mdoc, one hdoc, one fdoc, one cdoc + self.assertEqual(len(created), 4) + for doc in created: + self.assertTrue( + doc.__class__.__name__, + "SoledadDocument") + + d = adaptor.create_msg(adaptor.store, msg) + d.addCallback(check_create_result) + return d + + def test_update_msg(self): + adaptor = self.get_adaptor() + with open(os.path.join(HERE, "rfc822.message")) as f: + raw = f.read() + + def assert_msg_has_doc_id(ignored, msg): + wrapper = msg.get_wrapper() + self.assertTrue(wrapper.fdoc.doc_id is not None) + + def assert_msg_has_no_flags(ignored, msg): + wrapper = msg.get_wrapper() + self.assertEqual(wrapper.fdoc.flags, []) + + def update_msg_flags(ignored, msg): + wrapper = msg.get_wrapper() + wrapper.fdoc.flags = ["This", "That"] + return wrapper.update(adaptor.store) + + def assert_msg_has_flags(ignored, msg): + wrapper = msg.get_wrapper() + self.assertEqual(wrapper.fdoc.flags, ["This", "That"]) + + def get_fdoc_and_check_flags(ignored): + def assert_doc_has_flags(doc): + self.assertEqual(doc.content['flags'], + ['This', 'That']) + wrapper = msg.get_wrapper() + d = adaptor.store.get_doc(wrapper.fdoc.doc_id) + d.addCallback(assert_doc_has_flags) + return d + + msg = adaptor.get_msg_from_string(MessageClass, raw) + d = adaptor.create_msg(adaptor.store, msg) + d.addCallback(lambda _: adaptor.store.get_all_docs()) + d.addCallback(partial(self.assert_num_docs, 4)) + d.addCallback(assert_msg_has_doc_id, msg) + d.addCallback(assert_msg_has_no_flags, msg) + + # update it! + d.addCallback(update_msg_flags, msg) + d.addCallback(assert_msg_has_flags, msg) + d.addCallback(get_fdoc_and_check_flags) + return d + + # Mailboxes + + def test_get_or_create_mbox(self): + adaptor = self.get_adaptor() + + def get_or_create_mbox(ignored): + d = adaptor.get_or_create_mbox(adaptor.store, "Trash") + return d + + def assert_good_doc(mbox_wrapper): + self.assertTrue(mbox_wrapper.doc_id is not None) + self.assertEqual(mbox_wrapper.mbox, "Trash") + self.assertEqual(mbox_wrapper.type, "mbox") + self.assertEqual(mbox_wrapper.closed, False) + self.assertEqual(mbox_wrapper.subscribed, False) + + d = adaptor.initialize_store(adaptor.store) + d.addCallback(get_or_create_mbox) + d.addCallback(assert_good_doc) + d.addCallback(lambda _: adaptor.store.get_all_docs()) + d.addCallback(partial(self.assert_num_docs, 1)) + return d + + def test_update_mbox(self): + adaptor = self.get_adaptor() + + wrapper_ref = [] + + def get_or_create_mbox(ignored): + d = adaptor.get_or_create_mbox(adaptor.store, "Trash") + return d + + def update_wrapper(wrapper, wrapper_ref): + wrapper_ref.append(wrapper) + wrapper.subscribed = True + wrapper.closed = True + d = adaptor.update_mbox(adaptor.store, wrapper) + return d + + def get_mbox_doc_and_check_flags(res, wrapper_ref): + wrapper = wrapper_ref[0] + + def assert_doc_has_flags(doc): + self.assertEqual(doc.content['subscribed'], True) + self.assertEqual(doc.content['closed'], True) + d = adaptor.store.get_doc(wrapper.doc_id) + d.addCallback(assert_doc_has_flags) + return d + + d = adaptor.initialize_store(adaptor.store) + d.addCallback(get_or_create_mbox) + d.addCallback(update_wrapper, wrapper_ref) + d.addCallback(get_mbox_doc_and_check_flags, wrapper_ref) + return d + + def test_get_all_mboxes(self): + adaptor = self.get_adaptor() + mboxes = ("Sent", "Trash", "Personal", "ListFoo") + + def get_or_create_mboxes(ignored): + d = [] + for mbox in mboxes: + d.append(adaptor.get_or_create_mbox( + adaptor.store, mbox)) + return defer.gatherResults(d) + + def get_all_mboxes(ignored): + return adaptor.get_all_mboxes(adaptor.store) + + def assert_mboxes_match_expected(wrappers): + names = [m.mbox for m in wrappers] + self.assertEqual(set(names), set(mboxes)) + + d = adaptor.initialize_store(adaptor.store) + d.addCallback(get_or_create_mboxes) + d.addCallback(get_all_mboxes) + d.addCallback(assert_mboxes_match_expected) + return d diff --git a/mail/src/leap/mail/constants.py b/mail/src/leap/mail/constants.py new file mode 100644 index 00000000..4ef42cbb --- /dev/null +++ b/mail/src/leap/mail/constants.py @@ -0,0 +1,52 @@ +# *- coding: utf-8 -*- +# constants.py +# Copyright (C) 2014 LEAP +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see <http://www.gnu.org/licenses/>. +""" +Constants for leap.mail. +""" + +INBOX_NAME = "INBOX" + +# Regular expressions for the identifiers to be used in the Message Data Layer. + +METAMSGID = "M-{mbox_uuid}-{chash}" +METAMSGID_RE = "M\-{mbox_uuid}\-[0-9a-fA-F]+" +METAMSGID_CHASH_RE = "M\-\w+\-([0-9a-fA-F]+)" +METAMSGID_MBOX_RE = "M\-(\w+)\-[0-9a-fA-F]+" + +FDOCID = "F-{mbox_uuid}-{chash}" +FDOCID_RE = "F\-{mbox_uuid}\-[0-9a-fA-F]+" +FDOCID_CHASH_RE = "F\-\w+\-([0-9a-fA-F]+)" + +HDOCID = "H-{chash}" +HDOCID_RE = "H\-[0-9a-fA-F]+" + +CDOCID = "C-{phash}" +CDOCID_RE = "C\-[0-9a-fA-F]+" + + +class MessageFlags(object): + """ + Flags used in Message and Mailbox. + """ + SEEN_FLAG = "\\Seen" + RECENT_FLAG = "\\Recent" + ANSWERED_FLAG = "\\Answered" + FLAGGED_FLAG = "\\Flagged" # yo dawg + DELETED_FLAG = "\\Deleted" + DRAFT_FLAG = "\\Draft" + NOSELECT_FLAG = "\\Noselect" + LIST_FLAG = "List" # is this OK? (no \. ie, no system flag) diff --git a/mail/src/leap/mail/cred.py b/mail/src/leap/mail/cred.py new file mode 100644 index 00000000..7eab1f04 --- /dev/null +++ b/mail/src/leap/mail/cred.py @@ -0,0 +1,80 @@ +# -*- coding: utf-8 -*- +# cred.py +# Copyright (C) 2015 LEAP +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see <http://www.gnu.org/licenses/>. +""" +Credentials handling. +""" + +from zope.interface import implementer +from twisted.cred.checkers import ICredentialsChecker +from twisted.cred.credentials import IUsernamePassword +from twisted.cred.error import UnauthorizedLogin +from twisted.internet import defer + + +@implementer(ICredentialsChecker) +class LocalSoledadTokenChecker(object): + + """ + A Credentials Checker for a LocalSoledad store. + + It checks that: + + 1) The Local SoledadStorage has been correctly unlocked for the given + user. This currently means that the right passphrase has been passed + to the Local SoledadStorage. + + 2) The password passed in the credentials matches whatever token has + been stored in the local encrypted SoledadStorage, associated to the + Protocol that is requesting the authentication. + """ + + credentialInterfaces = (IUsernamePassword,) + service = None + + def __init__(self, soledad_sessions): + """ + :param soledad_sessions: a dict-like object, containing instances + of a Store (soledad instances), indexed by + userid. + """ + self._soledad_sessions = soledad_sessions + + def requestAvatarId(self, credentials): + if self.service is None: + raise NotImplementedError( + "this checker has not defined its service name") + username, password = credentials.username, credentials.password + d = self.checkSoledadToken(username, password, self.service) + d.addErrback(lambda f: defer.fail(UnauthorizedLogin())) + return d + + def checkSoledadToken(self, username, password, service): + soledad = self._soledad_sessions.get(username) + if not soledad: + return defer.fail(Exception("No soledad")) + + def match_token(token): + if token is None: + raise RuntimeError('no token') + if token == password: + return username + else: + raise RuntimeError('bad token') + + d = soledad.get_or_create_service_token(service) + d.addCallback(match_token) + return d diff --git a/mail/src/leap/mail/decorators.py b/mail/src/leap/mail/decorators.py new file mode 100644 index 00000000..5105de92 --- /dev/null +++ b/mail/src/leap/mail/decorators.py @@ -0,0 +1,149 @@ +# -*- coding: utf-8 -*- +# decorators.py +# Copyright (C) 2013 LEAP +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see <http://www.gnu.org/licenses/>. +""" +Useful decorators for mail package. +""" +import logging +import os + +from functools import wraps + +from twisted.internet.threads import deferToThread + + +logger = logging.getLogger(__name__) + + +# TODO +# Should write a helper to be able to pass a timeout argument. +# See this answer: http://stackoverflow.com/a/19019648/1157664 +# And the notes by glyph and jpcalderone + +def deferred_to_thread(f): + """ + Decorator, for deferring methods to Threads. + + It will do a deferToThread of the decorated method + unless the environment variable LEAPMAIL_DEBUG is set. + + It uses a descriptor to delay the definition of the + method wrapper. + """ + class descript(object): + """ + The class to be used as decorator. + + It takes any method as the passed object. + """ + + def __init__(self, f): + """ + Initializes the decorator object. + + :param f: the decorated function + :type f: callable + """ + self.f = f + + def __get__(self, instance, klass): + """ + Descriptor implementation. + + At creation time, the decorated `method` is unbound. + + It will dispatch the make_unbound method if we still do not + have an instance available, and the make_bound method when the + method has already been bound to the instance. + + :param instance: the instance of the class, or None if not exist. + :type instance: instantiated class or None. + """ + if instance is None: + # Class method was requested + return self.make_unbound(klass) + return self.make_bound(instance) + + def _errback(self, failure): + """ + Errorback that logs the exception catched. + + :param failure: a twisted failure + :type failure: Failure + """ + logger.warning('Error in method: %s' % (self.f.__name__)) + logger.exception(failure.getTraceback()) + + def make_unbound(self, klass): + """ + Return a wrapped function with the unbound call, during the + early access to the decortad method. This gets passed + only the class (not the instance since it does not yet exist). + + :param klass: the class to which the still unbound method belongs + :type klass: type + """ + + @wraps(self.f) + def wrapper(*args, **kwargs): + """ + We're temporarily wrapping the decorated method, but this + should not be called, since our application should use + the bound-wrapped method after this decorator class has been + used. + + This documentation will vanish at runtime. + """ + raise TypeError( + 'unbound method {}() must be called with {} instance ' + 'as first argument (got nothing instead)'.format( + self.f.__name__, + klass.__name__) + ) + return wrapper + + def make_bound(self, instance): + """ + Return a function that wraps the bound method call, + after we are able to access the instance object. + + :param instance: an instance of the class the decorated method, + now bound, belongs to. + :type instance: object + """ + + @wraps(self.f) + def wrapper(*args, **kwargs): + """ + Do a proper function wrapper that defers the decorated method + call to a separated thread if the LEAPMAIL_DEBUG + environment variable is set. + + This documentation will vanish at runtime. + """ + if not os.environ.get('LEAPMAIL_DEBUG'): + d = deferToThread(self.f, instance, *args, **kwargs) + d.addErrback(self._errback) + return d + else: + return self.f(instance, *args, **kwargs) + + # This instance does not need the descriptor anymore, + # let it find the wrapper directly next time: + setattr(instance, self.f.__name__, wrapper) + return wrapper + + return descript(f) diff --git a/mail/src/leap/mail/errors.py b/mail/src/leap/mail/errors.py new file mode 100644 index 00000000..2f18e87f --- /dev/null +++ b/mail/src/leap/mail/errors.py @@ -0,0 +1,27 @@ +# -*- coding: utf-8 -*- +# errors.py +# Copyright (C) 2015 LEAP +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see <http://www.gnu.org/licenses/>. +""" +Exceptions for leap.mail +""" + + +class AuthenticationError(Exception): + pass + + +class ConfigurationError(Exception): + pass diff --git a/mail/src/leap/mail/generator.py b/mail/src/leap/mail/generator.py new file mode 100644 index 00000000..bb3f26e2 --- /dev/null +++ b/mail/src/leap/mail/generator.py @@ -0,0 +1,23 @@ +from email.generator import Generator as EmailGenerator + + +class Generator(EmailGenerator): + """ + Generates output from a Message object tree, keeping signatures. + + This code was extracted from Mailman.Generator.Generator, version 2.1.4: + + Most other Generator will be created not setting the foldheader flag, + as we do not overwrite clone(). The original clone() does not + set foldheaders. + + So you need to set foldheaders if you want the toplevel to fold headers + + TODO: Python 3.3 is patched against this problems. See issue 1590744 on + python bug tracker. + """ + def _write_headers(self, msg): + for h, v in msg.items(): + print >> self._fp, '%s:' % h, + print >> self._fp, v + print >> self._fp diff --git a/mail/src/leap/mail/imap/__init__.py b/mail/src/leap/mail/imap/__init__.py new file mode 100644 index 00000000..e69de29b --- /dev/null +++ b/mail/src/leap/mail/imap/__init__.py diff --git a/mail/src/leap/mail/imap/account.py b/mail/src/leap/mail/imap/account.py new file mode 100644 index 00000000..e795c1b9 --- /dev/null +++ b/mail/src/leap/mail/imap/account.py @@ -0,0 +1,498 @@ +# -*- coding: utf-8 -*- +# account.py +# Copyright (C) 2013-2015 LEAP +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see <http://www.gnu.org/licenses/>. +""" +Soledad Backed IMAP Account. +""" +import logging +import os +import time +from functools import partial + +from twisted.internet import defer +from twisted.mail import imap4 +from twisted.python import log +from zope.interface import implements + +from leap.common.check import leap_assert, leap_assert_type + +from leap.mail.constants import MessageFlags +from leap.mail.mail import Account +from leap.mail.imap.mailbox import IMAPMailbox, normalize_mailbox +from leap.soledad.client import Soledad + +logger = logging.getLogger(__name__) + +PROFILE_CMD = os.environ.get('LEAP_PROFILE_IMAPCMD', False) + +if PROFILE_CMD: + def _debugProfiling(result, cmdname, start): + took = (time.time() - start) * 1000 + log.msg("CMD " + cmdname + " TOOK: " + str(took) + " msec") + return result + + +####################################### +# Soledad IMAP Account +####################################### + + +class IMAPAccount(object): + """ + An implementation of an imap4 Account + that is backed by Soledad Encrypted Documents. + """ + + implements(imap4.IAccount, imap4.INamespacePresenter) + + selected = None + + def __init__(self, store, user_id, d=defer.Deferred()): + """ + Keeps track of the mailboxes and subscriptions handled by this account. + + The account is not ready to be used, since the store needs to be + initialized and we also need to do some initialization routines. + You can either pass a deferred to this constructor, or use + `callWhenReady` method. + + :param store: a Soledad instance. + :type store: Soledad + + :param user_id: The identifier of the user this account belongs to + (user id, in the form user@provider). + :type user_id: str + + + :param d: a deferred that will be fired with this IMAPAccount instance + when the account is ready to be used. + :type d: defer.Deferred + """ + leap_assert(store, "Need a store instance to initialize") + leap_assert_type(store, Soledad) + + # TODO assert too that the name matches the user/uuid with which + # soledad has been initialized. Although afaik soledad doesn't know + # about user_id, only the client backend. + + self.user_id = user_id + self.account = Account( + store, user_id, ready_cb=lambda: d.callback(self)) + + def end_session(self): + """ + Used to mark when the session has closed, and we should not allow any + more commands from the client. + + Right now it's called from the client backend. + """ + # TODO move its use to the service shutdown in leap.mail + self.account.end_session() + + @property + def session_ended(self): + return self.account.session_ended + + def callWhenReady(self, cb, *args, **kw): + """ + Execute callback when the account is ready to be used. + XXX note that this callback will be called with a first ignored + parameter. + """ + # TODO ignore the first parameter and change tests accordingly. + d = self.account.callWhenReady(cb, *args, **kw) + return d + + def getMailbox(self, name): + """ + Return a Mailbox with that name, without selecting it. + + :param name: name of the mailbox + :type name: str + + :returns: an IMAPMailbox instance + :rtype: IMAPMailbox + """ + name = normalize_mailbox(name) + + def check_it_exists(mailboxes): + if name not in mailboxes: + raise imap4.MailboxException("No such mailbox: %r" % name) + return True + + d = self.account.list_all_mailbox_names() + d.addCallback(check_it_exists) + d.addCallback(lambda _: self.account.get_collection_by_mailbox(name)) + d.addCallback(self._return_mailbox_from_collection) + return d + + def _return_mailbox_from_collection(self, collection, readwrite=1): + if collection is None: + return None + mbox = IMAPMailbox(collection, rw=readwrite) + return mbox + + # + # IAccount + # + + def addMailbox(self, name, creation_ts=None): + """ + Add a mailbox to the account. + + :param name: the name of the mailbox + :type name: str + + :param creation_ts: an optional creation timestamp to be used as + mailbox id. A timestamp will be used if no + one is provided. + :type creation_ts: int + + :returns: a Deferred that will contain the document if successful. + :rtype: defer.Deferred + """ + name = normalize_mailbox(name) + + # FIXME --- return failure instead of AssertionError + # See AccountTestCase... + leap_assert(name, "Need a mailbox name to create a mailbox") + + def check_it_does_not_exist(mailboxes): + if name in mailboxes: + raise imap4.MailboxCollision, repr(name) + return mailboxes + + d = self.account.list_all_mailbox_names() + d.addCallback(check_it_does_not_exist) + d.addCallback(lambda _: self.account.add_mailbox( + name, creation_ts=creation_ts)) + d.addCallback(lambda _: self.account.get_collection_by_mailbox(name)) + d.addCallback(self._return_mailbox_from_collection) + return d + + def create(self, pathspec): + """ + Create a new mailbox from the given hierarchical name. + + :param pathspec: + The full hierarchical name of a new mailbox to create. + If any of the inferior hierarchical names to this one + do not exist, they are created as well. + :type pathspec: str + + :return: + A deferred that will fire with a true value if the creation + succeeds. The deferred might fail with a MailboxException + if the mailbox cannot be added. + :rtype: Deferred + + """ + def pass_on_collision(failure): + failure.trap(imap4.MailboxCollision) + return True + + def handle_collision(failure): + failure.trap(imap4.MailboxCollision) + if not pathspec.endswith('/'): + return defer.succeed(False) + else: + return defer.succeed(True) + + def all_good(result): + return all(result) + + paths = filter(None, normalize_mailbox(pathspec).split('/')) + subs = [] + sep = '/' + + for accum in range(1, len(paths)): + partial_path = sep.join(paths[:accum]) + d = self.addMailbox(partial_path) + d.addErrback(pass_on_collision) + subs.append(d) + + df = self.addMailbox(sep.join(paths)) + df.addErrback(handle_collision) + subs.append(df) + + d1 = defer.gatherResults(subs) + d1.addCallback(all_good) + return d1 + + def select(self, name, readwrite=1): + """ + Selects a mailbox. + + :param name: the mailbox to select + :type name: str + + :param readwrite: 1 for readwrite permissions. + :type readwrite: int + + :rtype: IMAPMailbox + """ + name = normalize_mailbox(name) + + def check_it_exists(mailboxes): + if name not in mailboxes: + logger.warning("SELECT: No such mailbox!") + return None + return name + + def set_selected(_): + self.selected = name + + def get_collection(name): + if name is None: + return None + return self.account.get_collection_by_mailbox(name) + + d = self.account.list_all_mailbox_names() + d.addCallback(check_it_exists) + d.addCallback(get_collection) + d.addCallback(partial( + self._return_mailbox_from_collection, readwrite=readwrite)) + return d + + def delete(self, name, force=False): + """ + Deletes a mailbox. + + :param name: the mailbox to be deleted + :type name: str + + :param force: + if True, it will not check for noselect flag or inferior + names. use with care. + :type force: bool + :rtype: Deferred + """ + name = normalize_mailbox(name) + _mboxes = None + + def check_it_exists(mailboxes): + global _mboxes + _mboxes = mailboxes + if name not in mailboxes: + raise imap4.MailboxException("No such mailbox: %r" % name) + + def get_mailbox(_): + return self.getMailbox(name) + + def destroy_mailbox(mbox): + return mbox.destroy() + + def check_can_be_deleted(mbox): + global _mboxes + # See if this box is flagged \Noselect + mbox_flags = mbox.getFlags() + if MessageFlags.NOSELECT_FLAG in mbox_flags: + # Check for hierarchically inferior mailboxes with this one + # as part of their root. + for others in _mboxes: + if others != name and others.startswith(name): + raise imap4.MailboxException( + "Hierarchically inferior mailboxes " + "exist and \\Noselect is set") + return mbox + + d = self.account.list_all_mailbox_names() + d.addCallback(check_it_exists) + d.addCallback(get_mailbox) + if not force: + d.addCallback(check_can_be_deleted) + d.addCallback(destroy_mailbox) + return d + + # FIXME --- not honoring the inferior names... + # if there are no hierarchically inferior names, we will + # delete it from our ken. + # XXX is this right? + # if self._inferiorNames(name) > 1: + # self._index.removeMailbox(name) + + def rename(self, oldname, newname): + """ + Renames a mailbox. + + :param oldname: old name of the mailbox + :type oldname: str + + :param newname: new name of the mailbox + :type newname: str + """ + oldname = normalize_mailbox(oldname) + newname = normalize_mailbox(newname) + + def rename_inferiors((inferiors, mailboxes)): + rename_deferreds = [] + inferiors = [ + (o, o.replace(oldname, newname, 1)) for o in inferiors] + + for (old, new) in inferiors: + if new in mailboxes: + raise imap4.MailboxCollision(repr(new)) + + for (old, new) in inferiors: + d = self.account.rename_mailbox(old, new) + rename_deferreds.append(d) + + d1 = defer.gatherResults(rename_deferreds, consumeErrors=True) + return d1 + + d1 = self._inferiorNames(oldname) + d2 = self.account.list_all_mailbox_names() + + d = defer.gatherResults([d1, d2]) + d.addCallback(rename_inferiors) + return d + + def _inferiorNames(self, name): + """ + Return hierarchically inferior mailboxes. + + :param name: name of the mailbox + :rtype: list + """ + # XXX use wildcard query instead + def filter_inferiors(mailboxes): + inferiors = [] + for infname in mailboxes: + if infname.startswith(name): + inferiors.append(infname) + return inferiors + + d = self.account.list_all_mailbox_names() + d.addCallback(filter_inferiors) + return d + + def listMailboxes(self, ref, wildcard): + """ + List the mailboxes. + + from rfc 3501: + returns a subset of names from the complete set + of all names available to the client. Zero or more untagged LIST + replies are returned, containing the name attributes, hierarchy + delimiter, and name. + + :param ref: reference name + :type ref: str + + :param wildcard: mailbox name with possible wildcards + :type wildcard: str + """ + wildcard = imap4.wildcardToRegexp(wildcard, '/') + + def get_list(mboxes, mboxes_names): + return zip(mboxes_names, mboxes) + + def filter_inferiors(ref): + mboxes = [mbox for mbox in ref if wildcard.match(mbox)] + mbox_d = defer.gatherResults([self.getMailbox(m) for m in mboxes]) + + mbox_d.addCallback(get_list, mboxes) + return mbox_d + + d = self._inferiorNames(normalize_mailbox(ref)) + d.addCallback(filter_inferiors) + return d + + # + # The rest of the methods are specific for leap.mail.imap.account.Account + # + + def isSubscribed(self, name): + """ + Returns True if user is subscribed to this mailbox. + + :param name: the mailbox to be checked. + :type name: str + + :rtype: Deferred (will fire with bool) + """ + name = normalize_mailbox(name) + + def get_subscribed(mbox): + return mbox.collection.get_mbox_attr("subscribed") + + d = self.getMailbox(name) + d.addCallback(get_subscribed) + return d + + def subscribe(self, name): + """ + Subscribe to this mailbox if not already subscribed. + + :param name: name of the mailbox + :type name: str + :rtype: Deferred + """ + name = normalize_mailbox(name) + + def set_subscribed(mbox): + return mbox.collection.set_mbox_attr("subscribed", True) + + d = self.getMailbox(name) + d.addCallback(set_subscribed) + return d + + def unsubscribe(self, name): + """ + Unsubscribe from this mailbox + + :param name: name of the mailbox + :type name: str + :rtype: Deferred + """ + # TODO should raise MailboxException if attempted to unsubscribe + # from a mailbox that is not currently subscribed. + # TODO factor out with subscribe method. + name = normalize_mailbox(name) + + def set_unsubscribed(mbox): + return mbox.collection.set_mbox_attr("subscribed", False) + + d = self.getMailbox(name) + d.addCallback(set_unsubscribed) + return d + + def getSubscriptions(self): + def get_subscribed(mailboxes): + return [x.mbox for x in mailboxes if x.subscribed] + + d = self.account.get_all_mailboxes() + d.addCallback(get_subscribed) + return d + + # + # INamespacePresenter + # + + def getPersonalNamespaces(self): + return [["", "/"]] + + def getSharedNamespaces(self): + return None + + def getOtherNamespaces(self): + return None + + def __repr__(self): + """ + Representation string for this object. + """ + return "<IMAPAccount (%s)>" % self.user_id diff --git a/mail/src/leap/mail/imap/mailbox.py b/mail/src/leap/mail/imap/mailbox.py new file mode 100644 index 00000000..e70a1d80 --- /dev/null +++ b/mail/src/leap/mail/imap/mailbox.py @@ -0,0 +1,970 @@ +# *- coding: utf-8 -*- +# mailbox.py +# Copyright (C) 2013-2015 LEAP +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see <http://www.gnu.org/licenses/>. +""" +IMAP Mailbox. +""" +import re +import logging +import os +import cStringIO +import StringIO +import time + +from collections import defaultdict +from email.utils import formatdate + +from twisted.internet import defer +from twisted.internet import reactor +from twisted.python import log + +from twisted.mail import imap4 +from zope.interface import implements + +from leap.common.check import leap_assert +from leap.common.check import leap_assert_type +from leap.mail.constants import INBOX_NAME, MessageFlags +from leap.mail.imap.messages import IMAPMessage + +logger = logging.getLogger(__name__) + +# TODO LIST +# [ ] Restore profile_cmd instrumentation +# [ ] finish the implementation of IMailboxListener +# [ ] implement the rest of ISearchableMailbox + + +""" +If the environment variable `LEAP_SKIPNOTIFY` is set, we avoid +notifying clients of new messages. Use during stress tests. +""" +NOTIFY_NEW = not os.environ.get('LEAP_SKIPNOTIFY', False) +PROFILE_CMD = os.environ.get('LEAP_PROFILE_IMAPCMD', False) + +if PROFILE_CMD: + + def _debugProfiling(result, cmdname, start): + took = (time.time() - start) * 1000 + log.msg("CMD " + cmdname + " TOOK: " + str(took) + " msec") + return result + + def do_profile_cmd(d, name): + """ + Add the profiling debug to the passed callback. + :param d: deferred + :param name: name of the command + :type name: str + """ + d.addCallback(_debugProfiling, name, time.time()) + d.addErrback(lambda f: log.msg(f.getTraceback())) + +INIT_FLAGS = (MessageFlags.SEEN_FLAG, MessageFlags.ANSWERED_FLAG, + MessageFlags.FLAGGED_FLAG, MessageFlags.DELETED_FLAG, + MessageFlags.DRAFT_FLAG, MessageFlags.RECENT_FLAG, + MessageFlags.LIST_FLAG) + + +def make_collection_listener(mailbox): + """ + Wrap a mailbox in a class that can be hashed according to the mailbox name. + + This means that dicts or sets will use this new equality rule, so we won't + collect multiple instances of the same mailbox in collections like the + MessageCollection set where we keep track of listeners. + """ + + class HashableMailbox(object): + + def __init__(self, mbox): + self.mbox = mbox + + # See #8083, pixelated adaptor seems to be misusing this class. + self.mailbox_name = self.mbox.mbox_name + + def __hash__(self): + return hash(self.mbox.mbox_name) + + def __eq__(self, other): + return self.mbox.mbox_name == other.mbox.mbox_name + + def notify_new(self): + self.mbox.notify_new() + + return HashableMailbox(mailbox) + + +class IMAPMailbox(object): + """ + A Soledad-backed IMAP mailbox. + + Implements the high-level method needed for the Mailbox interfaces. + The low-level database methods are contained in the generic + MessageCollection class. We receive an instance of it and it is made + accessible in the `collection` attribute. + """ + implements( + imap4.IMailbox, + imap4.IMailboxInfo, + imap4.ISearchableMailbox, + # XXX I think we do not need to implement CloseableMailbox, do we? + # We could remove ourselves from the collectionListener, although I + # think it simply will be garbage collected. + # imap4.ICloseableMailbox + imap4.IMessageCopier) + + init_flags = INIT_FLAGS + + CMD_MSG = "MESSAGES" + CMD_RECENT = "RECENT" + CMD_UIDNEXT = "UIDNEXT" + CMD_UIDVALIDITY = "UIDVALIDITY" + CMD_UNSEEN = "UNSEEN" + + # TODO we should turn this into a datastructure with limited capacity + _listeners = defaultdict(set) + + def __init__(self, collection, rw=1): + """ + :param collection: instance of MessageCollection + :type collection: MessageCollection + + :param rw: read-and-write flag for this mailbox + :type rw: int + """ + self.rw = rw + self._uidvalidity = None + self.collection = collection + self.collection.addListener(make_collection_listener(self)) + + @property + def mbox_name(self): + return self.collection.mbox_name + + @property + def listeners(self): + """ + Returns listeners for this mbox. + + The server itself is a listener to the mailbox. + so we can notify it (and should!) after changes in flags + and number of messages. + + :rtype: set + """ + return self._listeners[self.mbox_name] + + def get_imap_message(self, message): + d = defer.Deferred() + IMAPMessage(message, store=self.collection.store, d=d) + return d + + # FIXME this grows too crazily when many instances are fired, like + # during imaptest stress testing. Should have a queue of limited size + # instead. + + def addListener(self, listener): + """ + Add a listener to the listeners queue. + The server adds itself as a listener when there is a SELECT, + so it can send EXIST commands. + + :param listener: listener to add + :type listener: an object that implements IMailboxListener + """ + if not NOTIFY_NEW: + return + + listeners = self.listeners + logger.debug('adding mailbox listener: %s. Total: %s' % ( + listener, len(listeners))) + listeners.add(listener) + + def removeListener(self, listener): + """ + Remove a listener from the listeners queue. + + :param listener: listener to remove + :type listener: an object that implements IMailboxListener + """ + self.listeners.remove(listener) + + def getFlags(self): + """ + Returns the flags defined for this mailbox. + + :returns: tuple of flags for this mailbox + :rtype: tuple of str + """ + flags = self.collection.mbox_wrapper.flags + if not flags: + flags = self.init_flags + flags_str = map(str, flags) + return flags_str + + def setFlags(self, flags): + """ + Sets flags for this mailbox. + + :param flags: a tuple with the flags + :type flags: tuple of str + """ + # XXX this is setting (overriding) old flags. + # Better pass a mode flag + leap_assert(isinstance(flags, tuple), + "flags expected to be a tuple") + return self.collection.set_mbox_attr("flags", flags) + + def getUIDValidity(self): + """ + Return the unique validity identifier for this mailbox. + + :return: unique validity identifier + :rtype: int + """ + return self.collection.get_mbox_attr("created") + + def getUID(self, message_number): + """ + Return the UID of a message in the mailbox + + .. note:: this implementation does not make much sense RIGHT NOW, + but in the future will be useful to get absolute UIDs from + message sequence numbers. + + + :param message: the message sequence number. + :type message: int + + :rtype: int + :return: the UID of the message. + + """ + # TODO support relative sequences. The (imap) message should + # receive a sequence number attribute: a deferred is not expected + return message_number + + def getUIDNext(self): + """ + Return the likely UID for the next message added to this + mailbox. Currently it returns the higher UID incremented by + one. + + :return: deferred with int + :rtype: Deferred + """ + d = self.collection.get_uid_next() + return d + + def getMessageCount(self): + """ + Returns the total count of messages in this mailbox. + + :return: deferred with int + :rtype: Deferred + """ + return self.collection.count() + + def getUnseenCount(self): + """ + Returns the number of messages with the 'Unseen' flag. + + :return: count of messages flagged `unseen` + :rtype: int + """ + return self.collection.count_unseen() + + def getRecentCount(self): + """ + Returns the number of messages with the 'Recent' flag. + + :return: count of messages flagged `recent` + :rtype: int + """ + return self.collection.count_recent() + + def isWriteable(self): + """ + Get the read/write status of the mailbox. + + :return: 1 if mailbox is read-writeable, 0 otherwise. + :rtype: int + """ + # XXX We don't need to store it in the mbox doc, do we? + # return int(self.collection.get_mbox_attr('rw')) + return self.rw + + def getHierarchicalDelimiter(self): + """ + Returns the character used to delimite hierarchies in mailboxes. + + :rtype: str + """ + return '/' + + def requestStatus(self, names): + """ + Handles a status request by gathering the output of the different + status commands. + + :param names: a list of strings containing the status commands + :type names: iter + """ + r = {} + maybe = defer.maybeDeferred + if self.CMD_MSG in names: + r[self.CMD_MSG] = maybe(self.getMessageCount) + if self.CMD_RECENT in names: + r[self.CMD_RECENT] = maybe(self.getRecentCount) + if self.CMD_UIDNEXT in names: + r[self.CMD_UIDNEXT] = maybe(self.getUIDNext) + if self.CMD_UIDVALIDITY in names: + r[self.CMD_UIDVALIDITY] = maybe(self.getUIDValidity) + if self.CMD_UNSEEN in names: + r[self.CMD_UNSEEN] = maybe(self.getUnseenCount) + + def as_a_dict(values): + return dict(zip(r.keys(), values)) + + d = defer.gatherResults(r.values()) + d.addCallback(as_a_dict) + return d + + def addMessage(self, message, flags, date=None, notify_just_mdoc=True): + """ + Adds a message to this mailbox. + + :param message: the raw message + :type message: str + + :param flags: flag list + :type flags: list of str + + :param date: timestamp + :type date: str, or None + + :param notify_just_mdoc: + boolean passed to the wrapper.create method, to indicate whether + we're insterested in being notified right after the mdoc has been + written (as it's the first doc to be written, and quite small, this + is faster, though potentially unsafe). + Setting it to True improves a *lot* the responsiveness of the + APPENDS: we just need to be notified when the mdoc is saved, and + let's just expect that the other parts are doing just fine. This + will not catch any errors when the inserts of the other parts + fail, but on the other hand allows us to return very quickly, + which seems a good compromise given that we have to serialize the + appends. + However, some operations like the saving of drafts need to wait for + all the parts to be saved, so if some heuristics are met down in + the call chain a Draft message will unconditionally set this flag + to False, and therefore ignoring the setting of this flag here. + :type notify_just_mdoc: bool + + :return: a deferred that will be triggered with the UID of the added + message. + """ + # TODO should raise ReadOnlyMailbox if not rw. + # TODO have a look at the cases for internal date in the rfc + # XXX we could treat the message as an IMessage from here + + # TODO change notify_just_mdoc to something more meaningful, like + # fast_insert_notify? + + # TODO notify_just_mdoc *sometimes* make the append tests fail. + # have to find a better solution for this. A workaround could probably + # be to have a list of the ongoing deferreds related to append, so that + # we queue for later all the requests having to do with these. + + # A better solution will probably involve implementing MULTIAPPEND + # extension or patching imap server to support pipelining. + + if isinstance(message, (cStringIO.OutputType, StringIO.StringIO)): + message = message.getvalue() + + leap_assert_type(message, basestring) + + if flags is None: + flags = tuple() + else: + flags = tuple(str(flag) for flag in flags) + + if date is None: + date = formatdate(time.time()) + + d = self.collection.add_msg(message, flags, date=date, + notify_just_mdoc=notify_just_mdoc) + d.addErrback(lambda failure: log.err(failure)) + return d + + def notify_new(self, *args): + """ + Notify of new messages to all the listeners. + + This will be called indirectly by the underlying collection, that will + notify this IMAPMailbox whenever there are changes in the number of + messages in the collection, since we have added ourselves to the + collection listeners. + + :param args: ignored. + """ + if not NOTIFY_NEW: + return + + def cbNotifyNew(result): + exists, recent = result + for listener in self.listeners: + listener.newMessages(exists, recent) + + d = self._get_notify_count() + d.addCallback(cbNotifyNew) + d.addCallback(self.collection.cb_signal_unread_to_ui) + d.addErrback(lambda failure: log.err(failure)) + + def _get_notify_count(self): + """ + Get message count and recent count for this mailbox. + + :return: a deferred that will fire with a tuple, with number of + messages and number of recent messages. + :rtype: Deferred + """ + # XXX this is way too expensive in cases like multiple APPENDS. + # We should have a way of keep a cache or do a self-increment for that + # kind of calls. + d_exists = defer.maybeDeferred(self.getMessageCount) + d_recent = defer.maybeDeferred(self.getRecentCount) + d_list = [d_exists, d_recent] + + def log_num_msg(result): + exists, recent = tuple(result) + logger.debug("NOTIFY (%r): there are %s messages, %s recent" % ( + self.mbox_name, exists, recent)) + return result + + d = defer.gatherResults(d_list) + d.addCallback(log_num_msg) + return d + + # commands, do not rename methods + + def destroy(self): + """ + Called before this mailbox is permanently deleted. + + Should cleanup resources, and set the \\Noselect flag + on the mailbox. + + """ + # XXX this will overwrite all the existing flags + # should better simply addFlag + self.setFlags((MessageFlags.NOSELECT_FLAG,)) + + def remove_mbox(_): + uuid = self.collection.mbox_uuid + d = self.collection.mbox_wrapper.delete(self.collection.store) + d.addCallback( + lambda _: self.collection.mbox_indexer.delete_table(uuid)) + return d + + d = self.deleteAllDocs() + d.addCallback(remove_mbox) + return d + + def expunge(self): + """ + Remove all messages flagged \\Deleted + """ + if not self.isWriteable(): + raise imap4.ReadOnlyMailbox + return self.collection.delete_all_flagged() + + def _get_message_fun(self, uid): + """ + Return the proper method to get a message for this mailbox, depending + on the passed uid flag. + + :param uid: If true, the IDs specified in the query are UIDs; + otherwise they are message sequence IDs. + :type uid: bool + :rtype: callable + """ + get_message_fun = [ + self.collection.get_message_by_sequence_number, + self.collection.get_message_by_uid][uid] + return get_message_fun + + def _get_messages_range(self, messages_asked, uid=True): + + def get_range(messages_asked): + return self._filter_msg_seq(messages_asked) + + d = self._bound_seq(messages_asked, uid) + if uid: + d.addCallback(get_range) + d.addErrback(lambda f: log.err(f)) + return d + + def _bound_seq(self, messages_asked, uid): + """ + Put an upper bound to a messages sequence if this is open. + + :param messages_asked: IDs of the messages. + :type messages_asked: MessageSet + :return: a Deferred that will fire with a MessageSet + """ + + def set_last_uid(last_uid): + messages_asked.last = last_uid + return messages_asked + + def set_last_seq(all_uid): + messages_asked.last = len(all_uid) + return messages_asked + + if not messages_asked.last: + try: + iter(messages_asked) + except TypeError: + # looks like we cannot iterate + if uid: + d = self.collection.get_last_uid() + d.addCallback(set_last_uid) + else: + d = self.collection.all_uid_iter() + d.addCallback(set_last_seq) + return d + return defer.succeed(messages_asked) + + def _filter_msg_seq(self, messages_asked): + """ + Filter a message sequence returning only the ones that do exist in the + collection. + + :param messages_asked: IDs of the messages. + :type messages_asked: MessageSet + :rtype: set + """ + # TODO we could pass the asked sequence to the indexer + # all_uid_iter, and bound the sql query instead. + def filter_by_asked(all_msg_uid): + set_asked = set(messages_asked) + set_exist = set(all_msg_uid) + return set_asked.intersection(set_exist) + + d = self.collection.all_uid_iter() + d.addCallback(filter_by_asked) + return d + + def fetch(self, messages_asked, uid): + """ + Retrieve one or more messages in this mailbox. + + from rfc 3501: The data items to be fetched can be either a single atom + or a parenthesized list. + + :param messages_asked: IDs of the messages to retrieve information + about + :type messages_asked: MessageSet + + :param uid: If true, the IDs are UIDs. They are message sequence IDs + otherwise. + :type uid: bool + + :rtype: deferred with a generator that yields... + """ + get_msg_fun = self._get_message_fun(uid) + getimapmsg = self.get_imap_message + + def get_imap_messages_for_range(msg_range): + + def _get_imap_msg(messages): + d_imapmsg = [] + # just in case we got bad data in here + for msg in filter(None, messages): + d_imapmsg.append(getimapmsg(msg)) + return defer.gatherResults(d_imapmsg, consumeErrors=True) + + def _zip_msgid(imap_messages): + zipped = zip( + list(msg_range), imap_messages) + return (item for item in zipped) + + # XXX not called?? + def _unset_recent(sequence): + reactor.callLater(0, self.unset_recent_flags, sequence) + return sequence + + d_msg = [] + for msgid in msg_range: + # XXX We want cdocs because we "probably" are asked for the + # body. We should be smarter at do_FETCH and pass a parameter + # to this method in order not to prefetch cdocs if they're not + # going to be used. + d_msg.append(get_msg_fun(msgid, get_cdocs=True)) + + d = defer.gatherResults(d_msg, consumeErrors=True) + d.addCallback(_get_imap_msg) + d.addCallback(_zip_msgid) + d.addErrback(lambda failure: log.err(failure)) + return d + + d = self._get_messages_range(messages_asked, uid) + d.addCallback(get_imap_messages_for_range) + d.addErrback(lambda failure: log.err(failure)) + return d + + def fetch_flags(self, messages_asked, uid): + """ + A fast method to fetch all flags, tricking just the + needed subset of the MIME interface that's needed to satisfy + a generic FLAGS query. + + Given how LEAP Mail is supposed to work without local cache, + this query is going to be quite common, and also we expect + it to be in the form 1:* at the beginning of a session, so + it's not bad to fetch all the FLAGS docs at once. + + :param messages_asked: IDs of the messages to retrieve information + about + :type messages_asked: MessageSet + + :param uid: If 1, the IDs are UIDs. They are message sequence IDs + otherwise. + :type uid: int + + :return: A tuple of two-tuples of message sequence numbers and + flagsPart, which is a only a partial implementation of + MessagePart. + :rtype: tuple + """ + # is_sequence = True if uid == 0 else False + # XXX FIXME ----------------------------------------------------- + # imap/tests, or muas like mutt, it will choke until we implement + # sequence numbers. This is an easy hack meanwhile. + is_sequence = False + # --------------------------------------------------------------- + + if is_sequence: + raise NotImplementedError( + "FETCH FLAGS NOT IMPLEMENTED FOR MESSAGE SEQUENCE NUMBERS YET") + + d = defer.Deferred() + reactor.callLater(0, self._do_fetch_flags, messages_asked, uid, d) + if PROFILE_CMD: + do_profile_cmd(d, "FETCH-ALL-FLAGS") + return d + + def _do_fetch_flags(self, messages_asked, uid, d): + """ + :param messages_asked: IDs of the messages to retrieve information + about + :type messages_asked: MessageSet + + :param uid: If 1, the IDs are UIDs. They are message sequence IDs + otherwise. + :type uid: int + :param d: deferred whose callback will be called with result. + :type d: Deferred + + :rtype: A generator that yields two-tuples of message sequence numbers + and flagsPart + """ + class flagsPart(object): + def __init__(self, uid, flags): + self.uid = uid + self.flags = flags + + def getUID(self): + return self.uid + + def getFlags(self): + return map(str, self.flags) + + def pack_flags(result): + _uid, _flags = result + return _uid, flagsPart(_uid, _flags) + + def get_flags_for_seq(sequence): + d_all_flags = [] + for msgid in sequence: + # TODO implement sequence numbers here too + d_flags_per_uid = self.collection.get_flags_by_uid(msgid) + d_flags_per_uid.addCallback(pack_flags) + d_all_flags.append(d_flags_per_uid) + gotflags = defer.gatherResults(d_all_flags) + gotflags.addCallback(get_uid_flag_generator) + return gotflags + + def get_uid_flag_generator(result): + generator = (item for item in result) + d.callback(generator) + + d_seq = self._get_messages_range(messages_asked, uid) + d_seq.addCallback(get_flags_for_seq) + return d_seq + + @defer.inlineCallbacks + def fetch_headers(self, messages_asked, uid): + """ + A fast method to fetch all headers, tricking just the + needed subset of the MIME interface that's needed to satisfy + a generic HEADERS query. + + Given how LEAP Mail is supposed to work without local cache, + this query is going to be quite common, and also we expect + it to be in the form 1:* at the beginning of a session, so + **MAYBE** it's not too bad to fetch all the HEADERS docs at once. + + :param messages_asked: IDs of the messages to retrieve information + about + :type messages_asked: MessageSet + + :param uid: If true, the IDs are UIDs. They are message sequence IDs + otherwise. + :type uid: bool + + :return: A tuple of two-tuples of message sequence numbers and + headersPart, which is a only a partial implementation of + MessagePart. + :rtype: tuple + """ + # TODO implement sequences + is_sequence = True if uid == 0 else False + if is_sequence: + raise NotImplementedError( + "FETCH HEADERS NOT IMPLEMENTED FOR SEQUENCE NUMBER YET") + + class headersPart(object): + def __init__(self, uid, headers): + self.uid = uid + self.headers = headers + + def getUID(self): + return self.uid + + def getHeaders(self, _): + return dict( + (str(key), str(value)) + for key, value in + self.headers.items()) + + messages_asked = yield self._bound_seq(messages_asked, uid) + seq_messg = yield self._filter_msg_seq(messages_asked) + + result = [] + for msgid in seq_messg: + msg = yield self.collection.get_message_by_uid(msgid) + headers = headersPart(msgid, msg.get_headers()) + result.append((msgid, headers)) + defer.returnValue(iter(result)) + + def store(self, messages_asked, flags, mode, uid): + """ + Sets the flags of one or more messages. + + :param messages: The identifiers of the messages to set the flags + :type messages: A MessageSet object with the list of messages requested + + :param flags: The flags to set, unset, or add. + :type flags: sequence of str + + :param mode: If mode is -1, these flags should be removed from the + specified messages. If mode is 1, these flags should be + added to the specified messages. If mode is 0, all + existing flags should be cleared and these flags should be + added. + :type mode: -1, 0, or 1 + + :param uid: If true, the IDs specified in the query are UIDs; + otherwise they are message sequence IDs. + :type uid: bool + + :return: A deferred, that will be called with a dict mapping message + sequence numbers to sequences of str representing the flags + set on the message after this operation has been performed. + :rtype: deferred + + :raise ReadOnlyMailbox: Raised if this mailbox is not open for + read-write. + """ + if not self.isWriteable(): + log.msg('read only mailbox!') + raise imap4.ReadOnlyMailbox + + d = defer.Deferred() + reactor.callLater(0, self._do_store, messages_asked, flags, + mode, uid, d) + if PROFILE_CMD: + do_profile_cmd(d, "STORE") + + d.addCallback(self.collection.cb_signal_unread_to_ui) + d.addErrback(lambda f: log.err(f)) + return d + + def _do_store(self, messages_asked, flags, mode, uid, observer): + """ + Helper method, invoke set_flags method in the IMAPMessageCollection. + + See the documentation for the `store` method for the parameters. + + :param observer: a deferred that will be called with the dictionary + mapping UIDs to flags after the operation has been + done. + :type observer: deferred + """ + # TODO we should prevent client from setting Recent flag + get_msg_fun = self._get_message_fun(uid) + leap_assert(not isinstance(flags, basestring), + "flags cannot be a string") + flags = tuple(flags) + + def set_flags_for_seq(sequence): + def return_result_dict(list_of_flags): + result = dict(zip(list(sequence), list_of_flags)) + observer.callback(result) + return result + + d_all_set = [] + for msgid in sequence: + d = get_msg_fun(msgid) + d.addCallback(lambda msg: self.collection.update_flags( + msg, flags, mode)) + d_all_set.append(d) + got_flags_setted = defer.gatherResults(d_all_set) + got_flags_setted.addCallback(return_result_dict) + return got_flags_setted + + d_seq = self._get_messages_range(messages_asked, uid) + d_seq.addCallback(set_flags_for_seq) + return d_seq + + # ISearchableMailbox + + def search(self, query, uid): + """ + Search for messages that meet the given query criteria. + + Warning: this is half-baked, and it might give problems since + it offers the SearchableInterface. + We'll be implementing it asap. + + :param query: The search criteria + :type query: list + + :param uid: If true, the IDs specified in the query are UIDs; + otherwise they are message sequence IDs. + :type uid: bool + + :return: A list of message sequence numbers or message UIDs which + match the search criteria or a C{Deferred} whose callback + will be invoked with such a list. + :rtype: C{list} or C{Deferred} + """ + # TODO see if we can raise w/o interrupting flow + # :raise IllegalQueryError: Raised when query is not valid. + # example query: + # ['UNDELETED', 'HEADER', 'Message-ID', + # XXX fixme, does not exist + # '52D44F11.9060107@dev.bitmask.net'] + + # TODO hardcoding for now! -- we'll support generic queries later on + # but doing a quickfix for avoiding duplicate saves in the draft + # folder. # See issue #4209 + + if len(query) > 2: + if query[1] == 'HEADER' and query[2].lower() == "message-id": + msgid = str(query[3]).strip() + logger.debug("Searching for %s" % (msgid,)) + + d = self.collection.get_uid_from_msgid(str(msgid)) + d.addCallback(lambda result: [result]) + return d + + # nothing implemented for any other query + logger.warning("Cannot process query: %s" % (query,)) + return [] + + # IMessageCopier + + def copy(self, message): + """ + Copy the given message object into this mailbox. + + :param message: an IMessage implementor + :type message: LeapMessage + :return: a deferred that will be fired with the message + uid when the copy succeed. + :rtype: Deferred + """ + # if PROFILE_CMD: + # do_profile_cmd(d, "COPY") + + # A better place for this would be the COPY/APPEND dispatcher + # in server.py, but qtreactor hangs when I do that, so this seems + # to work fine for now. + # d.addCallback(lambda r: self.reactor.callLater(0, self.notify_new)) + # deferLater(self.reactor, 0, self._do_copy, message, d) + # return d + + d = self.collection.copy_msg(message.message, + self.collection.mbox_uuid) + return d + + # convenience fun + + def deleteAllDocs(self): + """ + Delete all docs in this mailbox + """ + # FIXME not implemented + return self.collection.delete_all_docs() + + def unset_recent_flags(self, uid_seq): + """ + Unset Recent flag for a sequence of UIDs. + """ + # FIXME not implemented + return self.collection.unset_recent_flags(uid_seq) + + def __repr__(self): + """ + Representation string for this mailbox. + """ + return u"<IMAPMailbox: mbox '%s' (%s)>" % ( + self.mbox_name, self.collection.count()) + + +_INBOX_RE = re.compile(INBOX_NAME, re.IGNORECASE) + + +def normalize_mailbox(name): + """ + Return a normalized representation of the mailbox ``name``. + + This method ensures that an eventual initial 'inbox' part of a + mailbox name is made uppercase. + + :param name: the name of the mailbox + :type name: unicode + + :rtype: unicode + """ + # XXX maybe it would make sense to normalize common folders too: + # trash, sent, drafts, etc... + if _INBOX_RE.match(name): + # ensure inital INBOX is uppercase + return INBOX_NAME + name[len(INBOX_NAME):] + return name diff --git a/mail/src/leap/mail/imap/messages.py b/mail/src/leap/mail/imap/messages.py new file mode 100644 index 00000000..d1c7b93c --- /dev/null +++ b/mail/src/leap/mail/imap/messages.py @@ -0,0 +1,254 @@ +# -*- coding: utf-8 -*- +# imap/messages.py +# Copyright (C) 2013-2015 LEAP +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see <http://www.gnu.org/licenses/>. +""" +IMAPMessage implementation. +""" +import logging +from twisted.mail import imap4 +from twisted.internet import defer +from zope.interface import implements + +from leap.mail.utils import find_charset, CaseInsensitiveDict + + +logger = logging.getLogger(__name__) + +# TODO +# [ ] Add ref to incoming message during add_msg. + + +class IMAPMessage(object): + """ + The main representation of a message as seen by the IMAP Server. + This class implements the semantics specific to IMAP specification. + """ + implements(imap4.IMessage) + + def __init__(self, message, prefetch_body=True, + store=None, d=defer.Deferred()): + """ + Get an IMAPMessage. A mail.Message is needed, since many of the methods + are proxied to that object. + + + If you do not need to prefetch the body of the message, you can set + `prefetch_body` to False, but the current imap server implementation + expect the getBodyFile method to return inmediately. + + When the prefetch_body option is used, a deferred is also expected as a + parameter, and this will fire when the deferred initialization has + taken place, with this instance of IMAPMessage as a parameter. + + :param message: the abstract message + :type message: mail.Message + :param prefetch_body: Whether to prefetch the content doc for the body. + :type prefetch_body: bool + :param store: an instance of soledad, or anything that behaves like it. + :param d: an optional deferred, that will be fired with the instance of + the IMAPMessage being initialized + :type d: defer.Deferred + """ + # TODO substitute the use of the deferred initialization by a factory + # function, maybe. + + self.message = message + self.__body_fd = None + self.store = store + if prefetch_body: + gotbody = self.__prefetch_body_file() + gotbody.addCallback(lambda _: d.callback(self)) + + # IMessage implementation + + def getUID(self): + """ + Retrieve the unique identifier associated with this Message. + + :return: uid for this message + :rtype: int + """ + return self.message.get_uid() + + def getFlags(self): + """ + Retrieve the flags associated with this Message. + + :return: The flags, represented as strings + :rtype: tuple + """ + return self.message.get_flags() + + def getInternalDate(self): + """ + Retrieve the date internally associated with this message + + According to the spec, this is NOT the date and time in the + RFC-822 header, but rather a date and time that reflects when the + message was received. + + * In SMTP, date and time of final delivery. + * In COPY, internal date/time of the source message. + * In APPEND, date/time specified. + + :return: An RFC822-formatted date string. + :rtype: str + """ + return self.message.get_internal_date() + + # + # IMessagePart + # + + def getBodyFile(self, store=None): + """ + Retrieve a file object containing only the body of this message. + + :return: file-like object opened for reading + :rtype: a deferred that will fire with a StringIO object. + """ + if self.__body_fd is not None: + fd = self.__body_fd + fd.seek(0) + return fd + + if store is None: + store = self.store + return self.message.get_body_file(store) + + def getSize(self): + """ + Return the total size, in octets, of this message. + + :return: size of the message, in octets + :rtype: int + """ + return self.message.get_size() + + def getHeaders(self, negate, *names): + """ + Retrieve a group of message headers. + + :param names: The names of the headers to retrieve or omit. + :type names: tuple of str + + :param negate: If True, indicates that the headers listed in names + should be omitted from the return value, rather + than included. + :type negate: bool + + :return: A mapping of header field names to header field values + :rtype: dict + """ + headers = self.message.get_headers() + return _format_headers(headers, negate, *names) + + def isMultipart(self): + """ + Return True if this message is multipart. + """ + return self.message.is_multipart() + + def getSubPart(self, part): + """ + Retrieve a MIME submessage + + :type part: C{int} + :param part: The number of the part to retrieve, indexed from 0. + :raise IndexError: Raised if the specified part does not exist. + :raise TypeError: Raised if this message is not multipart. + :rtype: Any object implementing C{IMessagePart}. + :return: The specified sub-part. + """ + subpart = self.message.get_subpart(part + 1) + return IMAPMessagePart(subpart) + + def __prefetch_body_file(self): + def assign_body_fd(fd): + self.__body_fd = fd + return fd + d = self.getBodyFile() + d.addCallback(assign_body_fd) + return d + + +class IMAPMessagePart(object): + + def __init__(self, message_part): + self.message_part = message_part + + def getBodyFile(self, store=None): + return self.message_part.get_body_file() + + def getSize(self): + return self.message_part.get_size() + + def getHeaders(self, negate, *names): + headers = self.message_part.get_headers() + return _format_headers(headers, negate, *names) + + def isMultipart(self): + return self.message_part.is_multipart() + + def getSubPart(self, part): + subpart = self.message_part.get_subpart(part + 1) + return IMAPMessagePart(subpart) + + +def _format_headers(headers, negate, *names): + # current server impl. expects content-type to be present, so if for + # some reason we do not have headers, we have to return at least that + # one + if not headers: + logger.warning("No headers found") + return {str('content-type'): str('')} + + names = map(lambda s: s.upper(), names) + + if negate: + def cond(key): + return key.upper() not in names + else: + def cond(key): + return key.upper() in names + + if isinstance(headers, list): + headers = dict(headers) + + # default to most likely standard + charset = find_charset(headers, "utf-8") + + # We will return a copy of the headers dictionary that + # will allow case-insensitive lookups. In some parts of the twisted imap + # server code the keys are expected to be in lower case, and in this way + # we avoid having to convert them. + + _headers = CaseInsensitiveDict() + for key, value in headers.items(): + if not isinstance(key, str): + key = key.encode(charset, 'replace') + if not isinstance(value, str): + value = value.encode(charset, 'replace') + + if value.endswith(";"): + # bastards + value = value[:-1] + + # filter original dict by negate-condition + if cond(key): + _headers[key] = value + + return _headers diff --git a/mail/src/leap/mail/imap/server.py b/mail/src/leap/mail/imap/server.py new file mode 100644 index 00000000..5a63af01 --- /dev/null +++ b/mail/src/leap/mail/imap/server.py @@ -0,0 +1,693 @@ +# -*- coding: utf-8 -*- +# server.py +# Copyright (C) 2014 LEAP +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see <http://www.gnu.org/licenses/>. +""" +LEAP IMAP4 Server Implementation. +""" +import StringIO +from copy import copy + +from twisted.internet.defer import maybeDeferred +from twisted.mail import imap4 +from twisted.python import log + +# imports for LITERAL+ patch +from twisted.internet import defer, interfaces +from twisted.mail.imap4 import IllegalClientResponse +from twisted.mail.imap4 import LiteralString, LiteralFile + +from leap.common.events import emit_async, catalog + + +def _getContentType(msg): + """ + Return a two-tuple of the main and subtype of the given message. + """ + attrs = None + mm = msg.getHeaders(False, 'content-type').get('content-type', None) + if mm: + mm = ''.join(mm.splitlines()) + mimetype = mm.split(';') + if mimetype: + type = mimetype[0].split('/', 1) + if len(type) == 1: + major = type[0] + minor = None + elif len(type) == 2: + major, minor = type + else: + major = minor = None + # XXX patched --------------------------------------------- + attrs = dict(x.strip().split('=', 1) for x in mimetype[1:]) + # XXX patched --------------------------------------------- + else: + major = minor = None + else: + major = minor = None + return major, minor, attrs + +# Monkey-patch _getContentType to avoid bug that passes lower-case boundary in +# BODYSTRUCTURE response. +imap4._getContentType = _getContentType + + +class LEAPIMAPServer(imap4.IMAP4Server): + """ + An IMAP4 Server with a LEAP Storage Backend. + """ + + ############################################################# + # + # Twisted imap4 patch to workaround bad mime rendering in TB. + # See https://leap.se/code/issues/6773 + # and https://bugzilla.mozilla.org/show_bug.cgi?id=149771 + # Still unclear if this is a thunderbird bug. + # TODO send this patch upstream + # + ############################################################# + + def spew_body(self, part, id, msg, _w=None, _f=None): + if _w is None: + _w = self.transport.write + for p in part.part: + if msg.isMultipart(): + msg = msg.getSubPart(p) + elif p > 0: + # Non-multipart messages have an implicit first part but no + # other parts - reject any request for any other part. + raise TypeError("Requested subpart of non-multipart message") + + if part.header: + hdrs = msg.getHeaders(part.header.negate, *part.header.fields) + hdrs = imap4._formatHeaders(hdrs) + # PATCHED ########################################## + _w(str(part) + ' ' + imap4._literal(hdrs + "\r\n")) + # PATCHED ########################################## + elif part.text: + _w(str(part) + ' ') + _f() + return imap4.FileProducer( + msg.getBodyFile() + ).beginProducing(self.transport) + elif part.mime: + hdrs = imap4._formatHeaders(msg.getHeaders(True)) + + # PATCHED ########################################## + _w(str(part) + ' ' + imap4._literal(hdrs + "\r\n")) + # END PATCHED ###################################### + + elif part.empty: + _w(str(part) + ' ') + _f() + if part.part: + # PATCHED ############################################# + # implement partial FETCH + # TODO implement boundary checks + # TODO see if there's a more efficient way, without + # copying the original content into a new buffer. + fd = msg.getBodyFile() + begin = getattr(part, "partialBegin", None) + _len = getattr(part, "partialLength", None) + if begin is not None and _len is not None: + _fd = StringIO.StringIO() + fd.seek(part.partialBegin) + _fd.write(fd.read(part.partialLength)) + _fd.seek(0) + else: + _fd = fd + return imap4.FileProducer( + _fd + # END PATCHED #########################3 + ).beginProducing(self.transport) + else: + mf = imap4.IMessageFile(msg, None) + if mf is not None: + return imap4.FileProducer( + mf.open()).beginProducing(self.transport) + return imap4.MessageProducer( + msg, None, self._scheduler).beginProducing(self.transport) + + else: + _w('BODY ' + + imap4.collapseNestedLists([imap4.getBodyStructure(msg)])) + + ################################################################## + # + # END Twisted imap4 patch to workaround bad mime rendering in TB. + # #6773 + # + ################################################################## + + def lineReceived(self, line): + """ + Attempt to parse a single line from the server. + + :param line: the line from the server, without the line delimiter. + :type line: str + """ + if "login" in line.lower(): + # avoid to log the pass, even though we are using a dummy auth + # by now. + msg = line[:7] + " [...]" + else: + msg = copy(line) + log.msg('rcv (%s): %s' % (self.state, msg)) + imap4.IMAP4Server.lineReceived(self, line) + + def close_server_connection(self): + """ + Send a BYE command so that the MUA at least knows that we're closing + the connection. + """ + self.sendLine( + '* BYE LEAP IMAP Proxy is shutting down; ' + 'so long and thanks for all the fish') + self.transport.loseConnection() + if self.mbox: + self.mbox.removeListener(self) + self.mbox = None + self.state = 'unauth' + + def do_FETCH(self, tag, messages, query, uid=0): + """ + Overwritten fetch dispatcher to use the fast fetch_flags + method + """ + if not query: + self.sendPositiveResponse(tag, 'FETCH complete') + return + + cbFetch = self._IMAP4Server__cbFetch + ebFetch = self._IMAP4Server__ebFetch + + if len(query) == 1 and str(query[0]) == "flags": + self._oldTimeout = self.setTimeout(None) + # no need to call iter, we get a generator + maybeDeferred( + self.mbox.fetch_flags, messages, uid=uid + ).addCallback( + cbFetch, tag, query, uid + ).addErrback(ebFetch, tag) + + elif len(query) == 1 and str(query[0]) == "rfc822.header": + self._oldTimeout = self.setTimeout(None) + # no need to call iter, we get a generator + maybeDeferred( + self.mbox.fetch_headers, messages, uid=uid + ).addCallback( + cbFetch, tag, query, uid + ).addErrback(ebFetch, tag) + else: + self._oldTimeout = self.setTimeout(None) + # no need to call iter, we get a generator + maybeDeferred( + self.mbox.fetch, messages, uid=uid + ).addCallback( + cbFetch, tag, query, uid + ).addErrback( + ebFetch, tag) + + select_FETCH = (do_FETCH, imap4.IMAP4Server.arg_seqset, + imap4.IMAP4Server.arg_fetchatt) + + def _cbSelectWork(self, mbox, cmdName, tag): + """ + Callback for selectWork + + * patched to avoid conformance errors due to incomplete UIDVALIDITY + line. + * patched to accept deferreds for messagecount and recent count + """ + if mbox is None: + self.sendNegativeResponse(tag, 'No such mailbox') + return + if '\\noselect' in [s.lower() for s in mbox.getFlags()]: + self.sendNegativeResponse(tag, 'Mailbox cannot be selected') + return + + d1 = defer.maybeDeferred(mbox.getMessageCount) + d2 = defer.maybeDeferred(mbox.getRecentCount) + return defer.gatherResults([d1, d2]).addCallback( + self.__cbSelectWork, mbox, cmdName, tag) + + def __cbSelectWork(self, ((msg_count, recent_count)), mbox, cmdName, tag): + flags = mbox.getFlags() + self.sendUntaggedResponse('FLAGS (%s)' % ' '.join(flags)) + + # Patched ------------------------------------------------------- + # accept deferreds for the count + self.sendUntaggedResponse(str(msg_count) + ' EXISTS') + self.sendUntaggedResponse(str(recent_count) + ' RECENT') + # ---------------------------------------------------------------- + + # Patched ------------------------------------------------------- + # imaptest was complaining about the incomplete line, we're adding + # "UIDs valid" here. + self.sendPositiveResponse( + None, '[UIDVALIDITY %d] UIDs valid' % mbox.getUIDValidity()) + # ---------------------------------------------------------------- + + s = mbox.isWriteable() and 'READ-WRITE' or 'READ-ONLY' + mbox.addListener(self) + self.sendPositiveResponse(tag, '[%s] %s successful' % (s, cmdName)) + self.state = 'select' + self.mbox = mbox + + def checkpoint(self): + """ + Called when the client issues a CHECK command. + + This should perform any checkpoint operations required by the server. + It may be a long running operation, but may not block. If it returns + a deferred, the client will only be informed of success (or failure) + when the deferred's callback (or errback) is invoked. + """ + # TODO implement a collection of ongoing deferreds? + return None + + ############################################################# + # + # Twisted imap4 patch to support LITERAL+ extension + # TODO send this patch upstream asap! + # + ############################################################# + + def capabilities(self): + cap = {'AUTH': self.challengers.keys()} + if self.ctx and self.canStartTLS: + t = self.transport + ti = interfaces.ISSLTransport + if not self.startedTLS and ti(t, None) is None: + cap['LOGINDISABLED'] = None + cap['STARTTLS'] = None + cap['NAMESPACE'] = None + cap['IDLE'] = None + # patched ############ + cap['LITERAL+'] = None + ###################### + return cap + + def _stringLiteral(self, size, literal_plus=False): + if size > self._literalStringLimit: + raise IllegalClientResponse( + "Literal too long! I accept at most %d octets" % + (self._literalStringLimit,)) + d = defer.Deferred() + self.parseState = 'pending' + self._pendingLiteral = LiteralString(size, d) + # Patched ########################################################### + if not literal_plus: + self.sendContinuationRequest('Ready for %d octets of text' % size) + ##################################################################### + self.setRawMode() + return d + + def _fileLiteral(self, size, literal_plus=False): + d = defer.Deferred() + self.parseState = 'pending' + self._pendingLiteral = LiteralFile(size, d) + if not literal_plus: + self.sendContinuationRequest('Ready for %d octets of data' % size) + self.setRawMode() + return d + + def arg_astring(self, line): + """ + Parse an astring from the line, return (arg, rest), possibly + via a deferred (to handle literals) + """ + line = line.strip() + if not line: + raise IllegalClientResponse("Missing argument") + d = None + arg, rest = None, None + if line[0] == '"': + try: + spam, arg, rest = line.split('"', 2) + rest = rest[1:] # Strip space + except ValueError: + raise IllegalClientResponse("Unmatched quotes") + elif line[0] == '{': + # literal + if line[-1] != '}': + raise IllegalClientResponse("Malformed literal") + + # Patched ################ + if line[-2] == "+": + literalPlus = True + size_end = -2 + else: + literalPlus = False + size_end = -1 + + try: + size = int(line[1:size_end]) + except ValueError: + raise IllegalClientResponse( + "Bad literal size: " + line[1:size_end]) + d = self._stringLiteral(size, literalPlus) + ########################## + else: + arg = line.split(' ', 1) + if len(arg) == 1: + arg.append('') + arg, rest = arg + return d or (arg, rest) + + def arg_literal(self, line): + """ + Parse a literal from the line + """ + if not line: + raise IllegalClientResponse("Missing argument") + + if line[0] != '{': + raise IllegalClientResponse("Missing literal") + + if line[-1] != '}': + raise IllegalClientResponse("Malformed literal") + + # Patched ################## + if line[-2] == "+": + literalPlus = True + size_end = -2 + else: + literalPlus = False + size_end = -1 + + try: + size = int(line[1:size_end]) + except ValueError: + raise IllegalClientResponse( + "Bad literal size: " + line[1:size_end]) + + return self._fileLiteral(size, literalPlus) + ############################# + + # --------------------------------- isSubscribed patch + # TODO -- send patch upstream. + # There is a bug in twisted implementation: + # in cbListWork, it's assumed that account.isSubscribed IS a callable, + # although in the interface documentation it's stated that it can be + # a deferred. + + def _listWork(self, tag, ref, mbox, sub, cmdName): + mbox = self._parseMbox(mbox) + mailboxes = maybeDeferred(self.account.listMailboxes, ref, mbox) + mailboxes.addCallback(self._cbSubscribed) + mailboxes.addCallback( + self._cbListWork, tag, sub, cmdName, + ).addErrback(self._ebListWork, tag) + + def _cbSubscribed(self, mailboxes): + subscribed = [ + maybeDeferred(self.account.isSubscribed, name) + for (name, box) in mailboxes] + + def get_mailboxes_and_subs(result): + subscribed = [i[0] for i, yes in zip(mailboxes, result) if yes] + return mailboxes, subscribed + + d = defer.gatherResults(subscribed) + d.addCallback(get_mailboxes_and_subs) + return d + + def _cbListWork(self, mailboxes_subscribed, tag, sub, cmdName): + mailboxes, subscribed = mailboxes_subscribed + + for (name, box) in mailboxes: + if not sub or name in subscribed: + flags = box.getFlags() + delim = box.getHierarchicalDelimiter() + resp = (imap4.DontQuoteMe(cmdName), + map(imap4.DontQuoteMe, flags), + delim, name.encode('imap4-utf-7')) + self.sendUntaggedResponse( + imap4.collapseNestedLists(resp)) + self.sendPositiveResponse(tag, '%s completed' % (cmdName,)) + # -------------------- end isSubscribed patch ----------- + + # TODO subscribe method had also to be changed to accomodate deferred + def do_SUBSCRIBE(self, tag, name): + name = self._parseMbox(name) + + def _subscribeCb(_): + self.sendPositiveResponse(tag, 'Subscribed') + + def _subscribeEb(failure): + m = failure.value + log.err() + if failure.check(imap4.MailboxException): + self.sendNegativeResponse(tag, str(m)) + else: + self.sendBadResponse( + tag, + "Server error encountered while subscribing to mailbox") + + d = self.account.subscribe(name) + d.addCallbacks(_subscribeCb, _subscribeEb) + return d + + auth_SUBSCRIBE = (do_SUBSCRIBE, arg_astring) + select_SUBSCRIBE = auth_SUBSCRIBE + + def do_UNSUBSCRIBE(self, tag, name): + # unsubscribe method had also to be changed to accomodate + # deferred + name = self._parseMbox(name) + + def _unsubscribeCb(_): + self.sendPositiveResponse(tag, 'Unsubscribed') + + def _unsubscribeEb(failure): + m = failure.value + log.err() + if failure.check(imap4.MailboxException): + self.sendNegativeResponse(tag, str(m)) + else: + self.sendBadResponse( + tag, + "Server error encountered while unsubscribing " + "from mailbox") + + d = self.account.unsubscribe(name) + d.addCallbacks(_unsubscribeCb, _unsubscribeEb) + return d + + auth_UNSUBSCRIBE = (do_UNSUBSCRIBE, arg_astring) + select_UNSUBSCRIBE = auth_UNSUBSCRIBE + + def do_RENAME(self, tag, oldname, newname): + oldname, newname = [self._parseMbox(n) for n in oldname, newname] + if oldname.lower() == 'inbox' or newname.lower() == 'inbox': + self.sendNegativeResponse( + tag, + 'You cannot rename the inbox, or ' + 'rename another mailbox to inbox.') + return + + def _renameCb(_): + self.sendPositiveResponse(tag, 'Mailbox renamed') + + def _renameEb(failure): + m = failure.value + if failure.check(TypeError): + self.sendBadResponse(tag, 'Invalid command syntax') + elif failure.check(imap4.MailboxException): + self.sendNegativeResponse(tag, str(m)) + else: + log.err() + self.sendBadResponse( + tag, + "Server error encountered while " + "renaming mailbox") + + d = self.account.rename(oldname, newname) + d.addCallbacks(_renameCb, _renameEb) + return d + + auth_RENAME = (do_RENAME, arg_astring, arg_astring) + select_RENAME = auth_RENAME + + def do_CREATE(self, tag, name): + name = self._parseMbox(name) + + def _createCb(result): + if result: + self.sendPositiveResponse(tag, 'Mailbox created') + else: + self.sendNegativeResponse(tag, 'Mailbox not created') + + def _createEb(failure): + c = failure.value + if failure.check(imap4.MailboxException): + self.sendNegativeResponse(tag, str(c)) + else: + log.err() + self.sendBadResponse( + tag, "Server error encountered while creating mailbox") + + d = self.account.create(name) + d.addCallbacks(_createCb, _createEb) + return d + + auth_CREATE = (do_CREATE, arg_astring) + select_CREATE = auth_CREATE + + def do_DELETE(self, tag, name): + name = self._parseMbox(name) + if name.lower() == 'inbox': + self.sendNegativeResponse(tag, 'You cannot delete the inbox') + return + + def _deleteCb(result): + self.sendPositiveResponse(tag, 'Mailbox deleted') + + def _deleteEb(failure): + m = failure.value + if failure.check(imap4.MailboxException): + self.sendNegativeResponse(tag, str(m)) + else: + print "SERVER: other error" + log.err() + self.sendBadResponse( + tag, + "Server error encountered while deleting mailbox") + + d = self.account.delete(name) + d.addCallbacks(_deleteCb, _deleteEb) + return d + + auth_DELETE = (do_DELETE, arg_astring) + select_DELETE = auth_DELETE + + # ----------------------------------------------------------------------- + # Patched just to allow __cbAppend to receive a deferred from messageCount + # TODO format and send upstream. + def do_APPEND(self, tag, mailbox, flags, date, message): + mailbox = self._parseMbox(mailbox) + maybeDeferred(self.account.select, mailbox).addCallback( + self._cbAppendGotMailbox, tag, flags, date, message).addErrback( + self._ebAppendGotMailbox, tag) + + def __ebAppend(self, failure, tag): + self.sendBadResponse(tag, 'APPEND failed: ' + str(failure.value)) + + def _cbAppendGotMailbox(self, mbox, tag, flags, date, message): + if not mbox: + self.sendNegativeResponse(tag, '[TRYCREATE] No such mailbox') + return + + d = mbox.addMessage(message, flags, date) + d.addCallback(self.__cbAppend, tag, mbox) + d.addErrback(self.__ebAppend, tag) + + def _ebAppendGotMailbox(self, failure, tag): + self.sendBadResponse( + tag, "Server error encountered while opening mailbox.") + log.err(failure) + + def __cbAppend(self, result, tag, mbox): + + # XXX patched --------------------------------- + def send_response(count): + self.sendUntaggedResponse('%d EXISTS' % count) + self.sendPositiveResponse(tag, 'APPEND complete') + + d = mbox.getMessageCount() + d.addCallback(send_response) + return d + # XXX patched --------------------------------- + # ----------------------------------------------------------------------- + + auth_APPEND = (do_APPEND, arg_astring, imap4.IMAP4Server.opt_plist, + imap4.IMAP4Server.opt_datetime, arg_literal) + select_APPEND = auth_APPEND + + # Need to override the command table after patching + # arg_astring and arg_literal, except on the methods that we are already + # overriding. + + # TODO -------------------------------------------- + # Check if we really need to override these + # methods, or we can monkeypatch. + # do_DELETE = imap4.IMAP4Server.do_DELETE + # do_CREATE = imap4.IMAP4Server.do_CREATE + # do_RENAME = imap4.IMAP4Server.do_RENAME + # do_SUBSCRIBE = imap4.IMAP4Server.do_SUBSCRIBE + # do_UNSUBSCRIBE = imap4.IMAP4Server.do_UNSUBSCRIBE + # do_APPEND = imap4.IMAP4Server.do_APPEND + # ------------------------------------------------- + do_LOGIN = imap4.IMAP4Server.do_LOGIN + do_STATUS = imap4.IMAP4Server.do_STATUS + do_COPY = imap4.IMAP4Server.do_COPY + + _selectWork = imap4.IMAP4Server._selectWork + + arg_plist = imap4.IMAP4Server.arg_plist + arg_seqset = imap4.IMAP4Server.arg_seqset + opt_plist = imap4.IMAP4Server.opt_plist + opt_datetime = imap4.IMAP4Server.opt_datetime + + unauth_LOGIN = (do_LOGIN, arg_astring, arg_astring) + + auth_SELECT = (_selectWork, arg_astring, 1, 'SELECT') + select_SELECT = auth_SELECT + + auth_CREATE = (do_CREATE, arg_astring) + select_CREATE = auth_CREATE + + auth_EXAMINE = (_selectWork, arg_astring, 0, 'EXAMINE') + select_EXAMINE = auth_EXAMINE + + # TODO ----------------------------------------------- + # re-add if we stop overriding DELETE + # auth_DELETE = (do_DELETE, arg_astring) + # select_DELETE = auth_DELETE + # auth_APPEND = (do_APPEND, arg_astring, opt_plist, opt_datetime, + # arg_literal) + # select_APPEND = auth_APPEND + + # ---------------------------------------------------- + + auth_RENAME = (do_RENAME, arg_astring, arg_astring) + select_RENAME = auth_RENAME + + auth_SUBSCRIBE = (do_SUBSCRIBE, arg_astring) + select_SUBSCRIBE = auth_SUBSCRIBE + + auth_UNSUBSCRIBE = (do_UNSUBSCRIBE, arg_astring) + select_UNSUBSCRIBE = auth_UNSUBSCRIBE + + auth_LIST = (_listWork, arg_astring, arg_astring, 0, 'LIST') + select_LIST = auth_LIST + + auth_LSUB = (_listWork, arg_astring, arg_astring, 1, 'LSUB') + select_LSUB = auth_LSUB + + auth_STATUS = (do_STATUS, arg_astring, arg_plist) + select_STATUS = auth_STATUS + + select_COPY = (do_COPY, arg_seqset, arg_astring) + + ############################################################# + # END of Twisted imap4 patch to support LITERAL+ extension + ############################################################# + + def authenticateLogin(self, user, passwd): + result = imap4.IMAP4Server.authenticateLogin(self, user, passwd) + emit_async(catalog.IMAP_CLIENT_LOGIN, str(user)) + return result diff --git a/mail/src/leap/mail/imap/service/README.rst b/mail/src/leap/mail/imap/service/README.rst new file mode 100644 index 00000000..2cca9b38 --- /dev/null +++ b/mail/src/leap/mail/imap/service/README.rst @@ -0,0 +1,39 @@ +testing the service +=================== + +Run the twisted service:: + + twistd -n -y imap-server.tac + +And use offlineimap for tests:: + + offlineimap -c LEAPofflineimapRC-tests + +minimal offlineimap configuration +--------------------------------- + +[general] +accounts = leap-local + +[Account leap-local] +localrepository = LocalLeap +remoterepository = RemoteLeap + +[Repository LocalLeap] +type = Maildir +localfolders = ~/LEAPMail/Mail + +[Repository RemoteLeap] +type = IMAP +ssl = no +remotehost = localhost +remoteport = 9930 +remoteuser = user +remotepass = pass + +debugging +--------- + +Use ngrep to obtain logs of the sequences:: + + sudo ngrep -d lo -W byline port 9930 diff --git a/mail/src/leap/mail/imap/service/__init__.py b/mail/src/leap/mail/imap/service/__init__.py new file mode 100644 index 00000000..e69de29b --- /dev/null +++ b/mail/src/leap/mail/imap/service/__init__.py diff --git a/mail/src/leap/mail/imap/service/imap-server.tac b/mail/src/leap/mail/imap/service/imap-server.tac new file mode 100644 index 00000000..c4d602db --- /dev/null +++ b/mail/src/leap/mail/imap/service/imap-server.tac @@ -0,0 +1,145 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- +# imap-server.tac +# Copyright (C) 2013,2014 LEAP +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see <http://www.gnu.org/licenses/>. +""" +TAC file for initialization of the imap service using twistd. + +Use this for debugging and testing the imap server using a native reactor. + +For now, and for debugging/testing purposes, you need +to pass a config file with the following structure: + +[leap_mail] +userid = 'user@provider' +uuid = 'deadbeefdeadabad' +passwd = 'supersecret' # optional, will get prompted if not found. +""" + +# TODO -- this .tac file should be deprecated in favor of bitmask.core.bitmaskd + +import ConfigParser +import getpass +import os +import sys + +from leap.keymanager import KeyManager +from leap.mail.imap.service import imap +from leap.soledad.client import Soledad + +from twisted.application import service, internet + + +# TODO should get this initializers from some authoritative mocked source +# We might want to put them the soledad itself. + +def initialize_soledad(uuid, email, passwd, + secrets, localdb, + gnupg_home, tempdir): + """ + Initializes soledad by hand + + :param email: ID for the user + :param gnupg_home: path to home used by gnupg + :param tempdir: path to temporal dir + :rtype: Soledad instance + """ + server_url = "http://provider" + cert_file = "" + + soledad = Soledad( + uuid, + passwd, + secrets, + localdb, + server_url, + cert_file, + syncable=False) + + return soledad + +###################################################################### +# Remember to set your config files, see module documentation above! +###################################################################### + +print "[+] Running LEAP IMAP Service" + + +bmconf = os.environ.get("LEAP_MAIL_CONFIG", "") +if not bmconf: + print ("[-] Please set LEAP_MAIL_CONFIG environment variable " + "pointing to your config.") + sys.exit(1) + +SECTION = "leap_mail" +cp = ConfigParser.ConfigParser() +cp.read(bmconf) + +userid = cp.get(SECTION, "userid") +uuid = cp.get(SECTION, "uuid") +passwd = unicode(cp.get(SECTION, "passwd")) + +# XXX get this right from the environment variable !!! +port = 1984 + +if not userid or not uuid: + print "[-] Config file missing userid or uuid field" + sys.exit(1) + +if not passwd: + passwd = unicode(getpass.getpass("Soledad passphrase: ")) + + +secrets = os.path.expanduser("~/.config/leap/soledad/%s.secret" % (uuid,)) +localdb = os.path.expanduser("~/.config/leap/soledad/%s.db" % (uuid,)) + +# XXX Is this really used? Should point it to user var dirs defined in xdg? +gnupg_home = "/tmp/" +tempdir = "/tmp/" + +################################################### + +# Ad-hoc soledad/keymanager initialization. + +print "[~] user:", userid +soledad = initialize_soledad(uuid, userid, passwd, secrets, + localdb, gnupg_home, tempdir, userid=userid) +km_args = (userid, "https://localhost", soledad) +km_kwargs = { + "token": "", + "ca_cert_path": "", + "api_uri": "", + "api_version": "", + "uid": uuid, + "gpgbinary": "/usr/bin/gpg" +} +keymanager = KeyManager(*km_args, **km_kwargs) + +################################################## + +# Ok, let's expose the application object for the twistd application +# framework to pick up from here... + + +def getIMAPService(): + soledad_sessions = {userid: soledad} + factory = imap.LeapIMAPFactory(soledad_sessions) + return internet.TCPServer(port, factory, interface="localhost") + + +application = service.Application("LEAP IMAP Application") +service = getIMAPService() +service.setServiceParent(application) diff --git a/mail/src/leap/mail/imap/service/imap.py b/mail/src/leap/mail/imap/service/imap.py new file mode 100644 index 00000000..4663854a --- /dev/null +++ b/mail/src/leap/mail/imap/service/imap.py @@ -0,0 +1,208 @@ +# -*- coding: utf-8 -*- +# imap.py +# Copyright (C) 2013-2015 LEAP +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see <http://www.gnu.org/licenses/>. +""" +IMAP Service Initialization. +""" +import logging +import os + +from collections import defaultdict + +from twisted.cred.portal import Portal, IRealm +from twisted.mail.imap4 import IAccount +from twisted.internet import defer +from twisted.internet import reactor +from twisted.internet.error import CannotListenError +from twisted.internet.protocol import ServerFactory +from twisted.python import log +from zope.interface import implementer + +from leap.common.events import emit_async, catalog +from leap.mail.cred import LocalSoledadTokenChecker +from leap.mail.imap.account import IMAPAccount +from leap.mail.imap.server import LEAPIMAPServer + +# TODO: leave only an implementor of IService in here + +logger = logging.getLogger(__name__) + +DO_MANHOLE = os.environ.get("LEAP_MAIL_MANHOLE", None) +if DO_MANHOLE: + from leap.mail.imap.service import manhole + +# The default port in which imap service will run + +IMAP_PORT = 1984 + +# +# Credentials Handling +# + + +@implementer(IRealm) +class LocalSoledadIMAPRealm(object): + + _encoding = 'utf-8' + + def __init__(self, soledad_sessions): + """ + :param soledad_sessions: a dict-like object, containing instances + of a Store (soledad instances), indexed by + userid. + """ + self._soledad_sessions = soledad_sessions + + def requestAvatar(self, avatarId, mind, *interfaces): + if isinstance(avatarId, str): + avatarId = avatarId.decode(self._encoding) + + def gotSoledad(soledad): + for iface in interfaces: + if iface is IAccount: + avatar = IMAPAccount(soledad, avatarId) + return (IAccount, avatar, + getattr(avatar, 'logout', lambda: None)) + raise NotImplementedError(self, interfaces) + + return self.lookupSoledadInstance(avatarId).addCallback(gotSoledad) + + def lookupSoledadInstance(self, userid): + soledad = self._soledad_sessions[userid] + # XXX this should return the instance after whenReady callback + return defer.succeed(soledad) + + +class IMAPTokenChecker(LocalSoledadTokenChecker): + """A credentials checker that will lookup a token for the IMAP service. + For now it will be using the same identifier than SMTPTokenChecker""" + + service = 'mail_auth' + + +class LocalSoledadIMAPServer(LEAPIMAPServer): + + """ + An IMAP Server that authenticates against a LocalSoledad store. + """ + + def __init__(self, soledad_sessions, *args, **kw): + + LEAPIMAPServer.__init__(self, *args, **kw) + + realm = LocalSoledadIMAPRealm(soledad_sessions) + portal = Portal(realm) + checker = IMAPTokenChecker(soledad_sessions) + self.checker = checker + self.portal = portal + portal.registerChecker(checker) + + +class LeapIMAPFactory(ServerFactory): + + """ + Factory for a IMAP4 server with soledad remote sync and gpg-decryption + capabilities. + """ + + protocol = LocalSoledadIMAPServer + + def __init__(self, soledad_sessions): + """ + Initializes the server factory. + + :param soledad_sessions: a dict-like object, containing instances + of a Store (soledad instances), indexed by + userid. + """ + self._soledad_sessions = soledad_sessions + self._connections = defaultdict() + + def buildProtocol(self, addr): + """ + Return a protocol suitable for the job. + + :param addr: remote ip address + :type addr: str + """ + # TODO should reject anything from addr != localhost, + # just in case. + log.msg("Building protocol for connection %s" % addr) + imapProtocol = self.protocol(self._soledad_sessions) + self._connections[addr] = imapProtocol + return imapProtocol + + def stopFactory(self): + # say bye! + for conn, proto in self._connections.items(): + log.msg("Closing connections for %s" % conn) + proto.close_server_connection() + + def doStop(self): + """ + Stops imap service (fetcher, factory and port). + """ + return ServerFactory.doStop(self) + + +def run_service(soledad_sessions, port=IMAP_PORT): + """ + Main entry point to run the service from the client. + + :param soledad_sessions: a dict-like object, containing instances + of a Store (soledad instances), indexed by userid. + + :returns: the port as returned by the reactor when starts listening, and + the factory for the protocol. + :rtype: tuple + """ + factory = LeapIMAPFactory(soledad_sessions) + + try: + interface = "localhost" + # don't bind just to localhost if we are running on docker since we + # won't be able to access imap from the host + if os.environ.get("LEAP_DOCKERIZED"): + interface = '' + + # TODO use Endpoints !!! + tport = reactor.listenTCP(port, factory, + interface=interface) + except CannotListenError: + logger.error("IMAP Service failed to start: " + "cannot listen in port %s" % (port,)) + except Exception as exc: + logger.error("Error launching IMAP service: %r" % (exc,)) + else: + # all good. + + if DO_MANHOLE: + # TODO get pass from env var.too. + manhole_factory = manhole.getManholeFactory( + {'f': factory, + 'gm': factory.theAccount.getMailbox}, + "boss", "leap") + # TODO use Endpoints !!! + reactor.listenTCP(manhole.MANHOLE_PORT, manhole_factory, + interface="127.0.0.1") + logger.debug("IMAP4 Server is RUNNING in port %s" % (port,)) + emit_async(catalog.IMAP_SERVICE_STARTED, str(port)) + + # FIXME -- change service signature + return tport, factory + + # not ok, signal error. + emit_async(catalog.IMAP_SERVICE_FAILED_TO_START, str(port)) diff --git a/mail/src/leap/mail/imap/service/manhole.py b/mail/src/leap/mail/imap/service/manhole.py new file mode 100644 index 00000000..c83ae899 --- /dev/null +++ b/mail/src/leap/mail/imap/service/manhole.py @@ -0,0 +1,130 @@ +# -*- coding: utf-8 -*- +# manhole.py +# Copyright (C) 2014 LEAP +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see <http://www.gnu.org/licenses/>. +""" +Utilities for enabling the manhole administrative interface into the +LEAP Mail application. +""" +MANHOLE_PORT = 2222 + + +def getManholeFactory(namespace, user, secret): + """ + Get an administrative manhole into the application. + + :param namespace: the namespace to show in the manhole + :type namespace: dict + :param user: the user to authenticate into the administrative shell. + :type user: str + :param secret: pass for this manhole + :type secret: str + """ + import string + + from twisted.cred.portal import Portal + from twisted.conch import manhole, manhole_ssh + from twisted.conch.insults import insults + from twisted.cred.checkers import ( + InMemoryUsernamePasswordDatabaseDontUse as MemoryDB) + + from rlcompleter import Completer + + class EnhancedColoredManhole(manhole.ColoredManhole): + """ + A Manhole with some primitive autocomplete support. + """ + # TODO use introspection to make life easier + + def find_common(self, l): + """ + find common parts in thelist items + ex: 'ab' for ['abcd','abce','abf'] + requires an ordered list + """ + if len(l) == 1: + return l[0] + + init = l[0] + for item in l[1:]: + for i, (x, y) in enumerate(zip(init, item)): + if x != y: + init = "".join(init[:i]) + break + + if not init: + return None + return init + + def handle_TAB(self): + """ + Trap the TAB keystroke. + """ + necessarypart = "".join(self.lineBuffer).split(' ')[-1] + completer = Completer(globals()) + if completer.complete(necessarypart, 0): + matches = list(set(completer.matches)) # has multiples + + if len(matches) == 1: + length = len(necessarypart) + self.lineBuffer = self.lineBuffer[:-length] + self.lineBuffer.extend(matches[0]) + self.lineBufferIndex = len(self.lineBuffer) + else: + matches.sort() + commons = self.find_common(matches) + if commons: + length = len(necessarypart) + self.lineBuffer = self.lineBuffer[:-length] + self.lineBuffer.extend(commons) + self.lineBufferIndex = len(self.lineBuffer) + + self.terminal.nextLine() + while matches: + matches, part = matches[4:], matches[:4] + for item in part: + self.terminal.write('%s' % item.ljust(30)) + self.terminal.write('\n') + self.terminal.nextLine() + + self.terminal.eraseLine() + self.terminal.cursorBackward(self.lineBufferIndex + 5) + self.terminal.write("%s %s" % ( + self.ps[self.pn], "".join(self.lineBuffer))) + + def keystrokeReceived(self, keyID, modifier): + """ + Act upon any keystroke received. + """ + self.keyHandlers.update({'\b': self.handle_BACKSPACE}) + m = self.keyHandlers.get(keyID) + if m is not None: + m() + elif keyID in string.printable: + self.characterReceived(keyID, False) + + sshRealm = manhole_ssh.TerminalRealm() + + def chainedProtocolFactory(): + return insults.ServerProtocol(EnhancedColoredManhole, namespace) + + sshRealm = manhole_ssh.TerminalRealm() + sshRealm.chainedProtocolFactory = chainedProtocolFactory + + portal = Portal( + sshRealm, [MemoryDB(**{user: secret})]) + + f = manhole_ssh.ConchFactory(portal) + return f diff --git a/mail/src/leap/mail/imap/service/notes.txt b/mail/src/leap/mail/imap/service/notes.txt new file mode 100644 index 00000000..623e1224 --- /dev/null +++ b/mail/src/leap/mail/imap/service/notes.txt @@ -0,0 +1,81 @@ +T 127.0.0.1:9930 -> 127.0.0.1:42866 [AP] +* OK [CAPABILITY IMAP4rev1 IDLE NAMESPACE] Twisted IMAP4rev1 Ready. + +## +T 127.0.0.1:42866 -> 127.0.0.1:9930 [AP] +NCLJ1 CAPABILITY. + +## +T 127.0.0.1:9930 -> 127.0.0.1:42866 [AP] +* CAPABILITY IMAP4rev1 IDLE NAMESPACE. +NCLJ1 OK CAPABILITY completed. + +## +T 127.0.0.1:42866 -> 127.0.0.1:9930 [AP] +NCLJ2 LOGIN user "pass". + +# +T 127.0.0.1:9930 -> 127.0.0.1:42866 [AP] +NCLJ2 OK LOGIN succeeded. + +## +T 127.0.0.1:42866 -> 127.0.0.1:9930 [AP] +NCLJ3 CAPABILITY. + +# +T 127.0.0.1:9930 -> 127.0.0.1:42866 [AP] +* CAPABILITY IMAP4rev1 IDLE NAMESPACE. +NCLJ3 OK CAPABILITY completed. + +# +T 127.0.0.1:42866 -> 127.0.0.1:9930 [AP] +NCLJ4 LIST "" "". + +## +T 127.0.0.1:9930 -> 127.0.0.1:42866 [AP] +* LIST (\Seen \Answered \Flagged \Deleted \Draft \Recent List) "/" "INBOX". +NCLJ4 OK LIST completed. + +# +T 127.0.0.1:42866 -> 127.0.0.1:9930 [AP] +NCLJ5 LIST "" "*". + +## +T 127.0.0.1:9930 -> 127.0.0.1:42866 [AP] +* LIST (\Seen \Answered \Flagged \Deleted \Draft \Recent List) "/" "INBOX". +NCLJ5 OK LIST completed. + +# +T 127.0.0.1:42866 -> 127.0.0.1:9930 [AP] +NCLJ6 SELECT INBOX. + +# +T 127.0.0.1:9930 -> 127.0.0.1:42866 [AP] +* 0 EXISTS. +* 3 RECENT. +* FLAGS (\Seen \Answered \Flagged \Deleted \Draft \Recent List). +* OK [UIDVALIDITY 42]. +NCLJ6 OK [READ-WRITE] SELECT successful. + +# +T 127.0.0.1:42866 -> 127.0.0.1:9930 [AP] +NCLJ7 EXAMINE INBOX. + +## +T 127.0.0.1:9930 -> 127.0.0.1:42866 [AP] +* 0 EXISTS. +* 3 RECENT. +* FLAGS (\Seen \Answered \Flagged \Deleted \Draft \Recent List). +* OK [UIDVALIDITY 42]. +NCLJ7 OK [READ-ONLY] EXAMINE successful. + +# +T 127.0.0.1:42866 -> 127.0.0.1:9930 [AP] +NCLJ8 LOGOUT. + +## +T 127.0.0.1:9930 -> 127.0.0.1:42866 [AP] +* BYE Nice talking to you. +NCLJ8 OK LOGOUT successful. + + diff --git a/mail/src/leap/mail/imap/service/rfc822.message b/mail/src/leap/mail/imap/service/rfc822.message new file mode 100644 index 00000000..ee97ab92 --- /dev/null +++ b/mail/src/leap/mail/imap/service/rfc822.message @@ -0,0 +1,86 @@ +Return-Path: <twisted-commits-admin@twistedmatrix.com> +Delivered-To: exarkun@meson.dyndns.org +Received: from localhost [127.0.0.1] + by localhost with POP3 (fetchmail-6.2.1) + for exarkun@localhost (single-drop); Thu, 20 Mar 2003 14:50:20 -0500 (EST) +Received: from pyramid.twistedmatrix.com (adsl-64-123-27-105.dsl.austtx.swbell.net [64.123.27.105]) + by intarweb.us (Postfix) with ESMTP id 4A4A513EA4 + for <exarkun@meson.dyndns.org>; Thu, 20 Mar 2003 14:49:27 -0500 (EST) +Received: from localhost ([127.0.0.1] helo=pyramid.twistedmatrix.com) + by pyramid.twistedmatrix.com with esmtp (Exim 3.35 #1 (Debian)) + id 18w648-0007Vl-00; Thu, 20 Mar 2003 13:51:04 -0600 +Received: from acapnotic by pyramid.twistedmatrix.com with local (Exim 3.35 #1 (Debian)) + id 18w63j-0007VK-00 + for <twisted-commits@twistedmatrix.com>; Thu, 20 Mar 2003 13:50:39 -0600 +To: twisted-commits@twistedmatrix.com +From: etrepum CVS <etrepum@twistedmatrix.com> +Reply-To: twisted-python@twistedmatrix.com +X-Mailer: CVSToys +Message-Id: <E18w63j-0007VK-00@pyramid.twistedmatrix.com> +Subject: [Twisted-commits] rebuild now works on python versions from 2.2.0 and up. +Sender: twisted-commits-admin@twistedmatrix.com +Errors-To: twisted-commits-admin@twistedmatrix.com +X-BeenThere: twisted-commits@twistedmatrix.com +X-Mailman-Version: 2.0.11 +Precedence: bulk +List-Help: <mailto:twisted-commits-request@twistedmatrix.com?subject=help> +List-Post: <mailto:twisted-commits@twistedmatrix.com> +List-Subscribe: <http://twistedmatrix.com/cgi-bin/mailman/listinfo/twisted-commits>, + <mailto:twisted-commits-request@twistedmatrix.com?subject=subscribe> +List-Id: <twisted-commits.twistedmatrix.com> +List-Unsubscribe: <http://twistedmatrix.com/cgi-bin/mailman/listinfo/twisted-commits>, + <mailto:twisted-commits-request@twistedmatrix.com?subject=unsubscribe> +List-Archive: <http://twistedmatrix.com/pipermail/twisted-commits/> +Date: Thu, 20 Mar 2003 13:50:39 -0600 + +Modified files: +Twisted/twisted/python/rebuild.py 1.19 1.20 + +Log message: +rebuild now works on python versions from 2.2.0 and up. + + +ViewCVS links: +http://twistedmatrix.com/users/jh.twistd/viewcvs/cgi/viewcvs.cgi/twisted/python/rebuild.py.diff?r1=text&tr1=1.19&r2=text&tr2=1.20&cvsroot=Twisted + +Index: Twisted/twisted/python/rebuild.py +diff -u Twisted/twisted/python/rebuild.py:1.19 Twisted/twisted/python/rebuild.py:1.20 +--- Twisted/twisted/python/rebuild.py:1.19 Fri Jan 17 13:50:49 2003 ++++ Twisted/twisted/python/rebuild.py Thu Mar 20 11:50:08 2003 +@@ -206,15 +206,27 @@ + clazz.__dict__.clear() + clazz.__getattr__ = __getattr__ + clazz.__module__ = module.__name__ ++ if newclasses: ++ import gc ++ if (2, 2, 0) <= sys.version_info[:3] < (2, 2, 2): ++ hasBrokenRebuild = 1 ++ gc_objects = gc.get_objects() ++ else: ++ hasBrokenRebuild = 0 + for nclass in newclasses: + ga = getattr(module, nclass.__name__) + if ga is nclass: + log.msg("WARNING: new-class %s not replaced by reload!" % reflect.qual(nclass)) + else: +- import gc +- for r in gc.get_referrers(nclass): +- if isinstance(r, nclass): ++ if hasBrokenRebuild: ++ for r in gc_objects: ++ if not getattr(r, '__class__', None) is nclass: ++ continue + r.__class__ = ga ++ else: ++ for r in gc.get_referrers(nclass): ++ if getattr(r, '__class__', None) is nclass: ++ r.__class__ = ga + if doLog: + log.msg('') + log.msg(' (fixing %s): ' % str(module.__name__)) + + +_______________________________________________ +Twisted-commits mailing list +Twisted-commits@twistedmatrix.com +http://twistedmatrix.com/cgi-bin/mailman/listinfo/twisted-commits diff --git a/mail/src/leap/mail/imap/tests/.gitignore b/mail/src/leap/mail/imap/tests/.gitignore new file mode 100644 index 00000000..60baa9cb --- /dev/null +++ b/mail/src/leap/mail/imap/tests/.gitignore @@ -0,0 +1 @@ +data/* diff --git a/mail/src/leap/mail/imap/tests/getmail b/mail/src/leap/mail/imap/tests/getmail new file mode 100755 index 00000000..dd3fa0bb --- /dev/null +++ b/mail/src/leap/mail/imap/tests/getmail @@ -0,0 +1,344 @@ +#!/usr/bin/env python + +# Copyright (c) Twisted Matrix Laboratories. +# See LICENSE in twisted for details. + +# Modifications by LEAP Developers 2014 to fit +# Bitmask configuration settings. +""" +Simple IMAP4 client which displays the subjects of all messages in a +particular mailbox. +""" + +import os +import sys + +from twisted.internet import protocol +from twisted.internet import ssl +from twisted.internet import defer +from twisted.internet import stdio +from twisted.mail import imap4 +from twisted.protocols import basic +from twisted.python import log + +# Global options stored here from main +_opts = {} + + +class TrivialPrompter(basic.LineReceiver): + from os import linesep as delimiter + + promptDeferred = None + + def prompt(self, msg): + assert self.promptDeferred is None + self.display(msg) + self.promptDeferred = defer.Deferred() + return self.promptDeferred + + def display(self, msg): + self.transport.write(msg) + + def lineReceived(self, line): + if self.promptDeferred is None: + return + d, self.promptDeferred = self.promptDeferred, None + d.callback(line) + + +class SimpleIMAP4Client(imap4.IMAP4Client): + """ + A client with callbacks for greeting messages from an IMAP server. + """ + greetDeferred = None + + def serverGreeting(self, caps): + self.serverCapabilities = caps + if self.greetDeferred is not None: + d, self.greetDeferred = self.greetDeferred, None + d.callback(self) + + +class SimpleIMAP4ClientFactory(protocol.ClientFactory): + usedUp = False + + protocol = SimpleIMAP4Client + + def __init__(self, username, onConn): + self.ctx = ssl.ClientContextFactory() + + self.username = username + self.onConn = onConn + + def buildProtocol(self, addr): + """ + Initiate the protocol instance. Since we are building a simple IMAP + client, we don't bother checking what capabilities the server has. We + just add all the authenticators twisted.mail has. + """ + assert not self.usedUp + self.usedUp = True + + p = self.protocol(self.ctx) + p.factory = self + p.greetDeferred = self.onConn + + p.registerAuthenticator(imap4.PLAINAuthenticator(self.username)) + p.registerAuthenticator(imap4.LOGINAuthenticator(self.username)) + p.registerAuthenticator( + imap4.CramMD5ClientAuthenticator(self.username)) + + return p + + def clientConnectionFailed(self, connector, reason): + d, self.onConn = self.onConn, None + d.errback(reason) + + +def cbServerGreeting(proto, username, password): + """ + Initial callback - invoked after the server sends us its greet message. + """ + # Hook up stdio + tp = TrivialPrompter() + stdio.StandardIO(tp) + + # And make it easily accessible + proto.prompt = tp.prompt + proto.display = tp.display + + # Try to authenticate securely + return proto.authenticate( + password).addCallback( + cbAuthentication, + proto).addErrback( + ebAuthentication, proto, username, password + ) + + +def ebConnection(reason): + """ + Fallback error-handler. If anything goes wrong, log it and quit. + """ + log.startLogging(sys.stdout) + log.err(reason) + return reason + + +def cbAuthentication(result, proto): + """ + Callback after authentication has succeeded. + + Lists a bunch of mailboxes. + """ + return proto.list("", "*" + ).addCallback(cbMailboxList, proto + ) + + +def ebAuthentication(failure, proto, username, password): + """ + Errback invoked when authentication fails. + + If it failed because no SASL mechanisms match, offer the user the choice + of logging in insecurely. + + If you are trying to connect to your Gmail account, you will be here! + """ + failure.trap(imap4.NoSupportedAuthentication) + return InsecureLogin(proto, username, password) + + +def InsecureLogin(proto, username, password): + """ + insecure-login. + """ + return proto.login(username, password + ).addCallback(cbAuthentication, proto + ) + + +def cbMailboxList(result, proto): + """ + Callback invoked when a list of mailboxes has been retrieved. + If we have a selected mailbox in the global options, we directly pick it. + Otherwise, we offer a prompt to let user choose one. + """ + all_mbox_list = [e[2] for e in result] + s = '\n'.join(['%d. %s' % (n + 1, m) for (n, m) in zip(range(len(all_mbox_list)), all_mbox_list)]) + if not s: + return defer.fail(Exception("No mailboxes exist on server!")) + + selected_mailbox = _opts.get('mailbox') + + if not selected_mailbox: + return proto.prompt(s + "\nWhich mailbox? [1] " + ).addCallback(cbPickMailbox, proto, all_mbox_list + ) + else: + mboxes_lower = map(lambda s: s.lower(), all_mbox_list) + index = mboxes_lower.index(selected_mailbox.lower()) + 1 + return cbPickMailbox(index, proto, all_mbox_list) + + +def cbPickMailbox(result, proto, mboxes): + """ + When the user selects a mailbox, "examine" it. + """ + mbox = mboxes[int(result or '1') - 1] + return proto.examine(mbox + ).addCallback(cbExamineMbox, proto + ) + + +def cbExamineMbox(result, proto): + """ + Callback invoked when examine command completes. + + Retrieve the subject header of every message in the mailbox. + """ + return proto.fetchSpecific('1:*', + headerType='HEADER.FIELDS', + headerArgs=['SUBJECT'], + ).addCallback(cbFetch, proto, + ) + + +def cbFetch(result, proto): + """ + Display a listing of the messages in the mailbox, based on the collected + headers. + """ + selected_subject = _opts.get('subject', None) + index = None + + if result: + keys = result.keys() + keys.sort() + + if selected_subject: + for k in keys: + # remove 'Subject: ' preffix plus eol + subject = result[k][0][2][9:].rstrip('\r\n') + if subject.lower() == selected_subject.lower(): + index = k + break + else: + for k in keys: + proto.display('%s %s' % (k, result[k][0][2])) + else: + print "Hey, an empty mailbox!" + + if not index: + return proto.prompt("\nWhich message? [1] (Q quits) " + ).addCallback(cbPickMessage, proto) + else: + return cbPickMessage(index, proto) + + +def cbPickMessage(result, proto): + """ + Pick a message. + """ + if result == "Q": + print "Bye!" + return proto.logout() + + return proto.fetchSpecific( + '%s' % result, + headerType='', + headerArgs=['BODY.PEEK[]'], + ).addCallback(cbShowmessage, proto) + + +def cbShowmessage(result, proto): + """ + Display message. + """ + if result: + keys = result.keys() + keys.sort() + for k in keys: + proto.display('%s %s' % (k, result[k][0][2])) + else: + print "Hey, an empty message!" + + return proto.logout() + + +def cbClose(result): + """ + Close the connection when we finish everything. + """ + from twisted.internet import reactor + reactor.stop() + + +def main(): + import argparse + import ConfigParser + import sys + from twisted.internet import reactor + + description = ( + 'Get messages from a LEAP IMAP Proxy.\nThis is a ' + 'debugging tool, do not use this to retrieve any sensitive ' + 'information, or we will send ninjas to your house!') + epilog = ( + 'In case you want to automate the usage of this utility ' + 'you can place your credentials in a file pointed by ' + 'BITMASK_CREDENTIALS. You need to have a [Credentials] ' + 'section, with username=<user@provider> and password fields') + + parser = argparse.ArgumentParser(description=description, epilog=epilog) + credentials = os.environ.get('BITMASK_CREDENTIALS') + + if credentials: + try: + config = ConfigParser.ConfigParser() + config.read(credentials) + username = config.get('Credentials', 'username') + password = config.get('Credentials', 'password') + except Exception, e: + print "Error reading credentials file: {0}".format(e) + sys.exit() + else: + parser.add_argument('username', type=str) + parser.add_argument('password', type=str) + + parser.add_argument('--mailbox', dest='mailbox', default=None, + help='Which mailbox to retrieve. Empty for interactive prompt.') + parser.add_argument('--subject', dest='subject', default=None, + help='A subject for retrieve a mail that matches. Empty for interactive prompt.') + + ns = parser.parse_args() + + if not credentials: + username = ns.username + password = ns.password + + _opts['mailbox'] = ns.mailbox + _opts['subject'] = ns.subject + + hostname = "localhost" + port = "1984" + + onConn = defer.Deferred( + ).addCallback(cbServerGreeting, username, password + ).addErrback(ebConnection + ).addBoth(cbClose) + + factory = SimpleIMAP4ClientFactory(username, onConn) + + if port == '993': + reactor.connectSSL( + hostname, int(port), factory, ssl.ClientContextFactory()) + else: + if not port: + port = 143 + reactor.connectTCP(hostname, int(port), factory) + reactor.run() + + +if __name__ == '__main__': + main() diff --git a/mail/src/leap/mail/imap/tests/imapclient.py b/mail/src/leap/mail/imap/tests/imapclient.py new file mode 100755 index 00000000..c353ceed --- /dev/null +++ b/mail/src/leap/mail/imap/tests/imapclient.py @@ -0,0 +1,207 @@ +#!/usr/bin/env python + +# Copyright (c) Twisted Matrix Laboratories. +# See LICENSE for details. + + +""" +Simple IMAP4 client which connects to our custome +IMAP4 server: imapserver.py. +""" + +import sys + +from twisted.internet import protocol +from twisted.internet import defer +from twisted.internet import stdio +from twisted.mail import imap4 +from twisted.protocols import basic +from twisted.python import util +from twisted.python import log + + +class TrivialPrompter(basic.LineReceiver): + # from os import linesep as delimiter + + promptDeferred = None + + def prompt(self, msg): + assert self.promptDeferred is None + self.display(msg) + self.promptDeferred = defer.Deferred() + return self.promptDeferred + + def display(self, msg): + self.transport.write(msg) + + def lineReceived(self, line): + if self.promptDeferred is None: + return + d, self.promptDeferred = self.promptDeferred, None + d.callback(line) + + +class SimpleIMAP4Client(imap4.IMAP4Client): + + """ + Add callbacks when the client receives greeting messages from + an IMAP server. + """ + greetDeferred = None + + def serverGreeting(self, caps): + self.serverCapabilities = caps + if self.greetDeferred is not None: + d, self.greetDeferred = self.greetDeferred, None + d.callback(self) + + +class SimpleIMAP4ClientFactory(protocol.ClientFactory): + usedUp = False + protocol = SimpleIMAP4Client + + def __init__(self, username, onConn): + self.username = username + self.onConn = onConn + + def buildProtocol(self, addr): + assert not self.usedUp + self.usedUp = True + + p = self.protocol() + p.factory = self + p.greetDeferred = self.onConn + + p.registerAuthenticator(imap4.PLAINAuthenticator(self.username)) + p.registerAuthenticator(imap4.LOGINAuthenticator(self.username)) + p.registerAuthenticator( + imap4.CramMD5ClientAuthenticator(self.username)) + + return p + + def clientConnectionFailed(self, connector, reason): + d, self.onConn = self.onConn, None + d.errback(reason) + + +def cbServerGreeting(proto, username, password): + """ + Initial callback - invoked after the server sends us its greet message. + """ + # Hook up stdio + tp = TrivialPrompter() + stdio.StandardIO(tp) + + # And make it easily accessible + proto.prompt = tp.prompt + proto.display = tp.display + + # Try to authenticate securely + return proto.authenticate( + password).addCallback( + cbAuthentication, proto).addErrback( + ebAuthentication, proto, username, password) + + +def ebConnection(reason): + """ + Fallback error-handler. If anything goes wrong, log it and quit. + """ + log.startLogging(sys.stdout) + log.err(reason) + return reason + + +def cbAuthentication(result, proto): + """ + Callback after authentication has succeeded. + List a bunch of mailboxes. + """ + return proto.list("", "*" + ).addCallback(cbMailboxList, proto + ) + + +def ebAuthentication(failure, proto, username, password): + """ + Errback invoked when authentication fails. + If it failed because no SASL mechanisms match, offer the user the choice + of logging in insecurely. + If you are trying to connect to your Gmail account, you will be here! + """ + failure.trap(imap4.NoSupportedAuthentication) + return proto.prompt( + "No secure authentication available. Login insecurely? (y/N) " + ).addCallback(cbInsecureLogin, proto, username, password + ) + + +def cbInsecureLogin(result, proto, username, password): + """ + Callback for "insecure-login" prompt. + """ + if result.lower() == "y": + # If they said yes, do it. + return proto.login(username, password + ).addCallback(cbAuthentication, proto + ) + return defer.fail(Exception("Login failed for security reasons.")) + + +def cbMailboxList(result, proto): + """ + Callback invoked when a list of mailboxes has been retrieved. + """ + result = [e[2] for e in result] + s = '\n'.join( + ['%d. %s' % (n + 1, m) for (n, m) in zip(range(len(result)), result)]) + if not s: + return defer.fail(Exception("No mailboxes exist on server!")) + return proto.prompt(s + "\nWhich mailbox? [1] " + ).addCallback(cbPickMailbox, proto, result + ) + + +def cbPickMailbox(result, proto, mboxes): + """ + When the user selects a mailbox, "examine" it. + """ + mbox = mboxes[int(result or '1') - 1] + return proto.status(mbox, 'MESSAGES', 'UNSEEN' + ).addCallback(cbMboxStatus, proto) + + +def cbMboxStatus(result, proto): + print "You have %s messages (%s unseen)!" % ( + result['MESSAGES'], result['UNSEEN']) + return proto.logout() + + +def cbClose(result): + """ + Close the connection when we finish everything. + """ + from twisted.internet import reactor + reactor.stop() + + +def main(): + hostname = raw_input('IMAP4 Server Hostname: ') + port = raw_input('IMAP4 Server Port (the default is 143): ') + username = raw_input('IMAP4 Username: ') + password = util.getPassword('IMAP4 Password: ') + + onConn = defer.Deferred( + ).addCallback(cbServerGreeting, username, password + ).addErrback(ebConnection + ).addBoth(cbClose) + + factory = SimpleIMAP4ClientFactory(username, onConn) + + from twisted.internet import reactor + conn = reactor.connectTCP(hostname, int(port), factory) + reactor.run() + + +if __name__ == '__main__': + main() diff --git a/mail/src/leap/mail/imap/tests/regressions_mime_struct b/mail/src/leap/mail/imap/tests/regressions_mime_struct new file mode 100755 index 00000000..03326646 --- /dev/null +++ b/mail/src/leap/mail/imap/tests/regressions_mime_struct @@ -0,0 +1,461 @@ +#!/usr/bin/env python + +# -*- coding: utf-8 -*- +# regression_mime_struct +# Copyright (C) 2014 LEAP +# Copyright (c) Twisted Matrix Laboratories. +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see <http://www.gnu.org/licenses/>. +""" +Simple Regression Tests for checking MIME struct handling using IMAP4 client. + +Iterates trough all mails under a given folder and tries to APPEND them to +the server being tested. After FETCHING the pushed message, it compares +the received version with the one that was saved, and exits with an error +code if they do not match. +""" +import os +import StringIO +import sys + +from email.parser import Parser + +from twisted.internet import protocol +from twisted.internet import ssl +from twisted.internet import defer +from twisted.internet import stdio +from twisted.mail import imap4 +from twisted.protocols import basic +from twisted.python import log + + +REGRESSIONS_FOLDER = os.environ.get( + "REGRESSIONS_FOLDER", "regressions_test") +print "[+] Using regressions folder:", REGRESSIONS_FOLDER + +parser = Parser() + + +def get_msg_parts(raw): + """ + Return a representation of the parts of a message suitable for + comparison. + + :param raw: string for the message + :type raw: str + """ + m = parser.parsestr(raw) + return [dict(part.items()) + if part.is_multipart() + else part.get_payload() + for part in m.walk()] + + +def compare_msg_parts(a, b): + """ + Compare two sequences of parts of messages. + + :param a: part sequence for message a + :param b: part sequence for message b + + :return: True if both message sequences are equivalent. + :rtype: bool + """ + # XXX This could be smarter and show the differences in the + # different parts when/where they differ. + #import pprint; pprint.pprint(a[0]) + #import pprint; pprint.pprint(b[0]) + + def lowerkey(d): + return dict((k.lower(), v.replace('\r', '')) + for k, v in d.iteritems()) + + def eq(x, y): + # For dicts, we compare a variation with their keys + # in lowercase, and \r removed from their values + if all(map(lambda i: isinstance(i, dict), (x, y))): + x, y = map(lowerkey, (x, y)) + return x == y + + compare_vector = map(lambda tup: eq(tup[0], tup[1]), zip(a, b)) + all_match = all(compare_vector) + + if not all_match: + print "PARTS MISMATCH!" + print "vector: ", compare_vector + index = compare_vector.index(False) + from pprint import pprint + print "Expected:" + pprint(a[index]) + print ("***") + print "Found:" + pprint(b[index]) + print + + return all_match + + +def get_fd(string): + """ + Return a file descriptor with the passed string + as content. + """ + fd = StringIO.StringIO() + fd.write(string) + fd.seek(0) + return fd + + +class TrivialPrompter(basic.LineReceiver): + promptDeferred = None + + def prompt(self, msg): + assert self.promptDeferred is None + self.display(msg) + self.promptDeferred = defer.Deferred() + return self.promptDeferred + + def display(self, msg): + self.transport.write(msg) + + def lineReceived(self, line): + if self.promptDeferred is None: + return + d, self.promptDeferred = self.promptDeferred, None + d.callback(line) + + +class SimpleIMAP4Client(imap4.IMAP4Client): + """ + A client with callbacks for greeting messages from an IMAP server. + """ + greetDeferred = None + + def serverGreeting(self, caps): + self.serverCapabilities = caps + if self.greetDeferred is not None: + d, self.greetDeferred = self.greetDeferred, None + d.callback(self) + + +class SimpleIMAP4ClientFactory(protocol.ClientFactory): + usedUp = False + protocol = SimpleIMAP4Client + + def __init__(self, username, onConn): + self.ctx = ssl.ClientContextFactory() + + self.username = username + self.onConn = onConn + + def buildProtocol(self, addr): + """ + Initiate the protocol instance. Since we are building a simple IMAP + client, we don't bother checking what capabilities the server has. We + just add all the authenticators twisted.mail has. Note: Gmail no + longer uses any of the methods below, it's been using XOAUTH since + 2010. + """ + assert not self.usedUp + self.usedUp = True + + p = self.protocol(self.ctx) + p.factory = self + p.greetDeferred = self.onConn + + p.registerAuthenticator(imap4.PLAINAuthenticator(self.username)) + p.registerAuthenticator(imap4.LOGINAuthenticator(self.username)) + p.registerAuthenticator( + imap4.CramMD5ClientAuthenticator(self.username)) + + return p + + def clientConnectionFailed(self, connector, reason): + d, self.onConn = self.onConn, None + d.errback(reason) + + +def cbServerGreeting(proto, username, password): + """ + Initial callback - invoked after the server sends us its greet message. + """ + # Hook up stdio + tp = TrivialPrompter() + stdio.StandardIO(tp) + + # And make it easily accessible + proto.prompt = tp.prompt + proto.display = tp.display + + # Try to authenticate securely + return proto.authenticate( + password).addCallback( + cbAuthentication, + proto).addErrback( + ebAuthentication, proto, username, password + ) + + +def ebConnection(reason): + """ + Fallback error-handler. If anything goes wrong, log it and quit. + """ + log.startLogging(sys.stdout) + log.err(reason) + return reason + + +def cbAuthentication(result, proto): + """ + Callback after authentication has succeeded. + + Lists a bunch of mailboxes. + """ + return proto.select( + REGRESSIONS_FOLDER + ).addCallback( + cbSelectMbox, proto + ).addErrback( + ebSelectMbox, proto, REGRESSIONS_FOLDER) + + +def ebAuthentication(failure, proto, username, password): + """ + Errback invoked when authentication fails. + + If it failed because no SASL mechanisms match, offer the user the choice + of logging in insecurely. + + If you are trying to connect to your Gmail account, you will be here! + """ + failure.trap(imap4.NoSupportedAuthentication) + return InsecureLogin(proto, username, password) + + +def InsecureLogin(proto, username, password): + """ + Raise insecure-login error. + """ + return proto.login( + username, password + ).addCallback( + cbAuthentication, proto) + + +def cbSelectMbox(result, proto): + """ + Callback invoked when select command finishes successfully. + + If any message is in the test folder, it will flag them as deleted and + expunge. + If no messages found, it will start with the APPEND tests. + """ + print "SELECT: %s EXISTS " % result.get("EXISTS", "??") + + if result["EXISTS"] != 0: + # Flag as deleted, expunge, and do an examine again. + print "There is mail here, will delete..." + return cbDeleteAndExpungeTestFolder(proto) + + else: + return cbAppendNextMessage(proto) + + +def ebSelectMbox(failure, proto, folder): + """ + Errback invoked when the examine command fails. + + Creates the folder. + """ + log.err(failure) + log.msg("Folder %r does not exist. Creating..." % (folder,)) + return proto.create(folder).addCallback(cbAuthentication, proto) + + +def ebExpunge(failure): + log.err(failure) + + +def cbDeleteAndExpungeTestFolder(proto): + """ + Callback invoked fom cbExamineMbox when the number of messages in the + mailbox is not zero. It flags all messages as deleted and expunge the + mailbox. + """ + return proto.setFlags( + "1:*", ("\\Deleted",) + ).addCallback( + lambda r: proto.expunge() + ).addCallback( + cbExpunge, proto + ).addErrback( + ebExpunge) + + +def cbExpunge(result, proto): + return proto.select( + REGRESSIONS_FOLDER + ).addCallback( + cbSelectMbox, proto + ).addErrback(ebSettingDeleted, proto) + + +def ebSettingDeleted(failure, proto): + """ + Report errors during deletion of messages in the mailbox. + """ + print failure.getTraceback() + + +def cbAppendNextMessage(proto): + """ + Appends the next message in the global queue to the test folder. + """ + # 1. Get the next test message from global tuple. + try: + next_sample = SAMPLES.pop() + except IndexError: + # we're done! + return proto.logout() + + print "\nAPPEND %s" % (next_sample,) + raw = open(next_sample).read() + msg = get_fd(raw) + return proto.append( + REGRESSIONS_FOLDER, msg + ).addCallback( + lambda r: proto.select(REGRESSIONS_FOLDER) + ).addCallback( + cbAppend, proto, raw + ).addErrback( + ebAppend, proto, raw) + + +def cbAppend(result, proto, orig_msg): + """ + Fetches the message right after an append. + """ + # XXX keep account of highest UID + uid = "1:*" + + return proto.fetchSpecific( + '%s' % uid, + headerType='', + headerArgs=['BODY.PEEK[]'], + ).addCallback( + cbCompareMessage, proto, orig_msg + ).addErrback(ebAppend, proto, orig_msg) + + +def ebAppend(failure, proto, raw): + """ + Errorback for the append operation + """ + print "ERROR WHILE APPENDING!" + print failure.getTraceback() + + +def cbPickMessage(result, proto): + """ + Pick a message. + """ + return proto.fetchSpecific( + '%s' % result, + headerType='', + headerArgs=['BODY.PEEK[]'], + ).addCallback(cbCompareMessage, proto) + + +def cbCompareMessage(result, proto, raw): + """ + Display message and compare it with the original one. + """ + parts_orig = get_msg_parts(raw) + + if result: + keys = result.keys() + keys.sort() + else: + print "[-] GOT NO RESULT" + return proto.logout() + + latest = max(keys) + + fetched_msg = result[latest][0][2] + parts_fetched = get_msg_parts(fetched_msg) + + equal = compare_msg_parts( + parts_orig, + parts_fetched) + + if equal: + print "[+] MESSAGES MATCH" + return cbAppendNextMessage(proto) + else: + print "[-] ERROR: MESSAGES DO NOT MATCH !!!" + print " ABORTING COMPARISON..." + # FIXME logout and print the subject ... + return proto.logout() + + +def cbClose(result): + """ + Close the connection when we finish everything. + """ + from twisted.internet import reactor + reactor.stop() + + +def main(): + import glob + import sys + + if len(sys.argv) != 4: + print "Usage: regressions <user> <pass> <samples-folder>" + sys.exit() + + hostname = "localhost" + port = "1984" + username = sys.argv[1] + password = sys.argv[2] + + samplesdir = sys.argv[3] + + if not os.path.isdir(samplesdir): + print ("Could not find samples folder! " + "Make sure of copying mail_breaker contents there.") + sys.exit() + + samples = glob.glob(samplesdir + '/*') + + global SAMPLES + SAMPLES = [] + SAMPLES += samples + + onConn = defer.Deferred( + ).addCallback( + cbServerGreeting, username, password + ).addErrback( + ebConnection + ).addBoth(cbClose) + + factory = SimpleIMAP4ClientFactory(username, onConn) + + from twisted.internet import reactor + reactor.connectTCP(hostname, int(port), factory) + reactor.run() + + +if __name__ == '__main__': + main() diff --git a/mail/src/leap/mail/imap/tests/rfc822.message b/mail/src/leap/mail/imap/tests/rfc822.message new file mode 120000 index 00000000..b19cc280 --- /dev/null +++ b/mail/src/leap/mail/imap/tests/rfc822.message @@ -0,0 +1 @@ +../../tests/rfc822.message
\ No newline at end of file diff --git a/mail/src/leap/mail/imap/tests/rfc822.multi-minimal.message b/mail/src/leap/mail/imap/tests/rfc822.multi-minimal.message new file mode 120000 index 00000000..e0aa678b --- /dev/null +++ b/mail/src/leap/mail/imap/tests/rfc822.multi-minimal.message @@ -0,0 +1 @@ +../../tests/rfc822.multi-minimal.message
\ No newline at end of file diff --git a/mail/src/leap/mail/imap/tests/rfc822.multi-nested.message b/mail/src/leap/mail/imap/tests/rfc822.multi-nested.message new file mode 120000 index 00000000..306d0dec --- /dev/null +++ b/mail/src/leap/mail/imap/tests/rfc822.multi-nested.message @@ -0,0 +1 @@ +../../tests/rfc822.multi-nested.message
\ No newline at end of file diff --git a/mail/src/leap/mail/imap/tests/rfc822.multi-signed.message b/mail/src/leap/mail/imap/tests/rfc822.multi-signed.message new file mode 120000 index 00000000..4172244e --- /dev/null +++ b/mail/src/leap/mail/imap/tests/rfc822.multi-signed.message @@ -0,0 +1 @@ +../../tests/rfc822.multi-signed.message
\ No newline at end of file diff --git a/mail/src/leap/mail/imap/tests/rfc822.multi.message b/mail/src/leap/mail/imap/tests/rfc822.multi.message new file mode 120000 index 00000000..62057d20 --- /dev/null +++ b/mail/src/leap/mail/imap/tests/rfc822.multi.message @@ -0,0 +1 @@ +../../tests/rfc822.multi.message
\ No newline at end of file diff --git a/mail/src/leap/mail/imap/tests/rfc822.plain.message b/mail/src/leap/mail/imap/tests/rfc822.plain.message new file mode 120000 index 00000000..5bab0e8d --- /dev/null +++ b/mail/src/leap/mail/imap/tests/rfc822.plain.message @@ -0,0 +1 @@ +../../tests/rfc822.plain.message
\ No newline at end of file diff --git a/mail/src/leap/mail/imap/tests/stress_tests_imap.zsh b/mail/src/leap/mail/imap/tests/stress_tests_imap.zsh new file mode 100755 index 00000000..544facaa --- /dev/null +++ b/mail/src/leap/mail/imap/tests/stress_tests_imap.zsh @@ -0,0 +1,178 @@ +#!/bin/zsh +# BATCH STRESS TEST FOR IMAP ---------------------- +# http://imgs.xkcd.com/comics/science.jpg +# +# Run imaptest against a LEAP IMAP server +# for a fixed period of time, and collect output. +# +# Author: Kali Kaneko +# Date: 2014 01 26 +# +# To run, you need to have `imaptest` in your path. +# See: +# http://www.imapwiki.org/ImapTest/Installation +# +# For the tests, I'm using a 10MB file sample that +# can be downloaded from: +# http://www.dovecot.org/tmp/dovecot-crlf +# +# Want to contribute to benchmarking? +# +# 1. Create a pristine account in a bitmask provider. +# +# 2. Launch your bitmask client, with different flags +# if you desire. +# +# For example to try the nosync flag in sqlite: +# +# LEAP_SQLITE_NOSYNC=1 bitmask --debug -N --offline -l /tmp/leap.log +# +# 3. Run at several points in time (ie: just after +# launching the bitmask client. one minute after, +# ten minutes after) +# +# mkdir data +# cd data +# ../leap_tests_imap.zsh | tee sqlite_nosync_run2.log +# +# 4. Submit your results to: kali at leap dot se +# together with the logs of the bitmask run. +# +# Please provide also details about your system, and +# the type of hard disk setup you are running against. +# + +# ------------------------------------------------ +# Edit these variables if you are too lazy to pass +# the user and mbox as parameters. Like me. + +USER="test_f14@dev.bitmask.net" +MBOX="~/leap/imaptest/data/dovecot-crlf" + +HOST="localhost" +PORT="1984" + +# in case you have it aliased +GREP="/bin/grep" +IMAPTEST="imaptest" + +# ----------------------------------------------- +# +# These should be kept constant across benchmarking +# runs across different machines, for comparability. + +DURATION=200 +NUM_MSG=200 + + +# TODO add another function, and a cli flag, to be able +# to take several aggretates spaced in time, along a period +# of several minutes. + +imaptest_cmd() { + stdbuf -o0 ${IMAPTEST} user=${USER} pass=1234 host=${HOST} \ + port=${PORT} mbox=${MBOX} clients=1 msgs=${NUM_MSG} \ + no_pipelining 2>/dev/null +} + +stress_imap() { + mkfifo imap_pipe + cat imap_pipe | tee output & + imaptest_cmd >> imap_pipe +} + +wait_and_kill() { + while : + do + sleep $DURATION + pkill -2 imaptest + rm imap_pipe + break + done +} + +print_results() { + sleep 1 + echo + echo + echo "AGGREGATED RESULTS" + echo "----------------------" + echo "\tavg\tstdev" + $GREP "avg" ./output | sed -e 's/^ *//g' -e 's/ *$//g' | \ + gawk ' +function avg(data, count) { + sum=0; + for( x=0; x <= count-1; x++) { + sum += data[x]; + } + return sum/count; +} +function std_dev(data, count) { + sum=0; + for( x=0; x <= count-1; x++) { + sum += data[x]; + } + average = sum/count; + + sumsq=0; + for( x=0; x <= count-1; x++) { + sumsq += (data[x] - average)^2; + } + return sqrt(sumsq/count); +} +BEGIN { + cnt = 0 +} END { + +printf("LOGI:\t%04.2lf\t%04.2f\n", avg(array[1], NR), std_dev(array[1], NR)); +printf("LIST:\t%04.2lf\t%04.2f\n", avg(array[2], NR), std_dev(array[2], NR)); +printf("STAT:\t%04.2lf\t%04.2f\n", avg(array[3], NR), std_dev(array[3], NR)); +printf("SELE:\t%04.2lf\t%04.2f\n", avg(array[4], NR), std_dev(array[4], NR)); +printf("FETC:\t%04.2lf\t%04.2f\n", avg(array[5], NR), std_dev(array[5], NR)); +printf("FET2:\t%04.2lf\t%04.2f\n", avg(array[6], NR), std_dev(array[6], NR)); +printf("STOR:\t%04.2lf\t%04.2f\n", avg(array[7], NR), std_dev(array[7], NR)); +printf("DELE:\t%04.2lf\t%04.2f\n", avg(array[8], NR), std_dev(array[8], NR)); +printf("EXPU:\t%04.2lf\t%04.2f\n", avg(array[9], NR), std_dev(array[9], NR)); +printf("APPE:\t%04.2lf\t%04.2f\n", avg(array[10], NR), std_dev(array[10], NR)); +printf("LOGO:\t%04.2lf\t%04.2f\n", avg(array[11], NR), std_dev(array[11], NR)); + +print "" +print "TOT samples", NR; +} +{ + it = cnt++; + array[1][it] = $1; + array[2][it] = $2; + array[3][it] = $3; + array[4][it] = $4; + array[5][it] = $5; + array[6][it] = $6; + array[7][it] = $7; + array[8][it] = $8; + array[9][it] = $9; + array[10][it] = $10; + array[11][it] = $11; +}' +} + + +{ test $1 = "--help" } && { + echo "Usage: $0 [user@provider] [/path/to/sample.mbox]" + exit 0 +} + +# If the first parameter is passed, take it as the user +{ test $1 } && { + USER=$1 +} + +# If the second parameter is passed, take it as the mbox +{ test $2 } && { + MBOX=$2 +} + +echo "[+] LEAP IMAP TESTS" +echo "[+] Running imaptest for $DURATION seconds with $NUM_MSG messages" +wait_and_kill & +stress_imap +print_results diff --git a/mail/src/leap/mail/imap/tests/test_imap.py b/mail/src/leap/mail/imap/tests/test_imap.py new file mode 100644 index 00000000..9cca17ff --- /dev/null +++ b/mail/src/leap/mail/imap/tests/test_imap.py @@ -0,0 +1,1060 @@ +# -*- coding: utf-8 -*- +# test_imap.py +# Copyright (C) 2013 LEAP +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see <http://www.gnu.org/licenses/>. +""" +Test case for leap.email.imap.server +TestCases taken from twisted tests and modified to make them work +against our implementation of the IMAPAccount. + +@authors: Kali Kaneko, <kali@leap.se> +XXX add authors from the original twisted tests. + +@license: GPLv3, see included LICENSE file +""" +# XXX review license of the original tests!!! +import os +import string +import types + + +from twisted.mail import imap4 +from twisted.internet import defer +from twisted.python import util +from twisted.python import failure + +from twisted import cred + +from leap.mail.imap.mailbox import IMAPMailbox +from leap.mail.imap.messages import CaseInsensitiveDict +from leap.mail.testing.imap import IMAP4HelperMixin + + +TEST_USER = "testuser@leap.se" +TEST_PASSWD = "1234" + + +def strip(f): + return lambda result, f=f: f() + + +def sortNest(l): + l = l[:] + l.sort() + for i in range(len(l)): + if isinstance(l[i], types.ListType): + l[i] = sortNest(l[i]) + elif isinstance(l[i], types.TupleType): + l[i] = tuple(sortNest(list(l[i]))) + return l + + +class TestRealm: + """ + A minimal auth realm for testing purposes only + """ + theAccount = None + + def requestAvatar(self, avatarId, mind, *interfaces): + return imap4.IAccount, self.theAccount, lambda: None + +# +# TestCases +# + +# DEBUG --- +# from twisted.internet.base import DelayedCall +# DelayedCall.debug = True + + +class LEAPIMAP4ServerTestCase(IMAP4HelperMixin): + + """ + Tests for the generic behavior of the LEAPIMAP4Server + which, right now, it's just implemented in this test file as + LEAPIMAPServer. We will move the implementation, together with + authentication bits, to leap.mail.imap.server so it can be instantiated + from the tac file. + + Right now this TestCase tries to mimmick as close as possible the + organization from the twisted.mail.imap tests so we can achieve + a complete implementation. The order in which they appear reflect + the intended order of implementation. + """ + + # + # mailboxes operations + # + + def testCreate(self): + """ + Test whether we can create mailboxes + """ + succeed = ('testbox', 'test/box', 'test/', 'test/box/box', 'foobox') + fail = ('testbox', 'test/box') + acc = self.server.theAccount + + def cb(): + self.result.append(1) + + def eb(failure): + self.result.append(0) + + def login(): + return self.client.login(TEST_USER, TEST_PASSWD) + + def create(): + create_deferreds = [] + for name in succeed + fail: + d = self.client.create(name) + d.addCallback(strip(cb)).addErrback(eb) + create_deferreds.append(d) + dd = defer.gatherResults(create_deferreds) + dd.addCallbacks(self._cbStopClient, self._ebGeneral) + return dd + + self.result = [] + d1 = self.connected.addCallback(strip(login)) + d1.addCallback(strip(create)) + d2 = self.loopback() + d = defer.gatherResults([d1, d2], consumeErrors=True) + d.addCallback(lambda _: acc.account.list_all_mailbox_names()) + return d.addCallback(self._cbTestCreate, succeed, fail) + + def _cbTestCreate(self, mailboxes, succeed, fail): + self.assertEqual(self.result, [1] * len(succeed) + [0] * len(fail)) + + answers = ([u'INBOX', u'testbox', u'test/box', u'test', + u'test/box/box', 'foobox']) + self.assertEqual(sorted(mailboxes), sorted([a for a in answers])) + + def testDelete(self): + """ + Test whether we can delete mailboxes + """ + def add_mailbox(): + return self.server.theAccount.addMailbox('test-delete/me') + + def login(): + return self.client.login(TEST_USER, TEST_PASSWD) + + def delete(): + return self.client.delete('test-delete/me') + + acc = self.server.theAccount.account + + d1 = self.connected.addCallback(add_mailbox) + d1.addCallback(strip(login)) + d1.addCallbacks(strip(delete), self._ebGeneral) + d1.addCallbacks(self._cbStopClient, self._ebGeneral) + d2 = self.loopback() + d = defer.gatherResults([d1, d2]) + d.addCallback(lambda _: acc.list_all_mailbox_names()) + d.addCallback(lambda mboxes: self.assertEqual( + mboxes, ['INBOX'])) + return d + + def testIllegalInboxDelete(self): + """ + Test what happens if we try to delete the user Inbox. + We expect that operation to fail. + """ + self.stashed = None + + def login(): + return self.client.login(TEST_USER, TEST_PASSWD) + + def delete(): + return self.client.delete('inbox') + + def stash(result): + self.stashed = result + + d1 = self.connected.addCallback(strip(login)) + d1.addCallbacks(strip(delete), self._ebGeneral) + d1.addBoth(stash) + d1.addCallbacks(self._cbStopClient, self._ebGeneral) + d2 = self.loopback() + d = defer.gatherResults([d1, d2]) + d.addCallback(lambda _: self.failUnless(isinstance(self.stashed, + failure.Failure))) + return d + + def testNonExistentDelete(self): + """ + Test what happens if we try to delete a non-existent mailbox. + We expect an error raised stating 'No such mailbox' + """ + def login(): + return self.client.login(TEST_USER, TEST_PASSWD) + + def delete(): + return self.client.delete('delete/me') + self.failure = failure + + def deleteFailed(failure): + self.failure = failure + + self.failure = None + d1 = self.connected.addCallback(strip(login)) + d1.addCallback(strip(delete)).addErrback(deleteFailed) + d1.addCallbacks(self._cbStopClient, self._ebGeneral) + d2 = self.loopback() + d = defer.gatherResults([d1, d2]) + d.addCallback(lambda _: self.assertTrue( + str(self.failure.value).startswith('No such mailbox'))) + return d + + def testIllegalDelete(self): + """ + Try deleting a mailbox with sub-folders, and \NoSelect flag set. + An exception is expected. + """ + acc = self.server.theAccount + + def login(): + return self.client.login(TEST_USER, TEST_PASSWD) + + def create_mailboxes(): + d1 = acc.addMailbox('delete') + d2 = acc.addMailbox('delete/me') + d = defer.gatherResults([d1, d2]) + return d + + def get_noselect_mailbox(mboxes): + mbox = mboxes[0] + return mbox.setFlags((r'\Noselect',)) + + def delete_mbox(ignored): + return self.client.delete('delete') + + def deleteFailed(failure): + self.failure = failure + + self.failure = None + + d1 = self.connected.addCallback(strip(login)) + d1.addCallback(strip(create_mailboxes)) + d1.addCallback(get_noselect_mailbox) + + d1.addCallback(delete_mbox).addErrback(deleteFailed) + d1.addCallbacks(self._cbStopClient, self._ebGeneral) + d2 = self.loopback() + d = defer.gatherResults([d1, d2]) + expected = ("Hierarchically inferior mailboxes exist " + "and \\Noselect is set") + d.addCallback(lambda _: + self.assertTrue(self.failure is not None)) + d.addCallback(lambda _: + self.assertEqual(str(self.failure.value), expected)) + return d + + # FIXME --- this test sometimes FAILS (timing issue). + # Some of the deferreds used in the rename op is not waiting for the + # operations properly + def testRename(self): + """ + Test whether we can rename a mailbox + """ + def create_mbox(): + return self.server.theAccount.addMailbox('oldmbox') + + def login(): + return self.client.login(TEST_USER, TEST_PASSWD) + + def rename(): + return self.client.rename('oldmbox', 'newname') + + d1 = self.connected.addCallback(strip(create_mbox)) + d1.addCallback(strip(login)) + d1.addCallbacks(strip(rename), self._ebGeneral) + d1.addCallbacks(self._cbStopClient, self._ebGeneral) + d2 = self.loopback() + d = defer.gatherResults([d1, d2]) + d.addCallback(lambda _: + self.server.theAccount.account.list_all_mailbox_names()) + d.addCallback(lambda mboxes: + self.assertItemsEqual(mboxes, ['INBOX', 'newname'])) + return d + + def testIllegalInboxRename(self): + """ + Try to rename inbox. We expect it to fail. Then it would be not + an inbox anymore, would it? + """ + self.stashed = None + + def login(): + return self.client.login(TEST_USER, TEST_PASSWD) + + def rename(): + return self.client.rename('inbox', 'frotz') + + def stash(stuff): + self.stashed = stuff + + d1 = self.connected.addCallback(strip(login)) + d1.addCallbacks(strip(rename), self._ebGeneral) + d1.addBoth(stash) + d1.addCallbacks(self._cbStopClient, self._ebGeneral) + d2 = self.loopback() + d = defer.gatherResults([d1, d2]) + d.addCallback(lambda _: + self.failUnless(isinstance( + self.stashed, failure.Failure))) + return d + + def testHierarchicalRename(self): + """ + Try to rename hierarchical mailboxes + """ + acc = self.server.theAccount + + def add_mailboxes(): + return defer.gatherResults([ + acc.addMailbox('oldmbox/m1'), + acc.addMailbox('oldmbox/m2')]) + + def login(): + return self.client.login(TEST_USER, TEST_PASSWD) + + def rename(): + return self.client.rename('oldmbox', 'newname') + + d1 = self.connected.addCallback(strip(add_mailboxes)) + d1.addCallback(strip(login)) + d1.addCallbacks(strip(rename), self._ebGeneral) + d1.addCallbacks(self._cbStopClient, self._ebGeneral) + d2 = self.loopback() + d = defer.gatherResults([d1, d2]) + d.addCallback(lambda _: acc.account.list_all_mailbox_names()) + return d.addCallback(self._cbTestHierarchicalRename) + + def _cbTestHierarchicalRename(self, mailboxes): + expected = ['INBOX', 'newname/m1', 'newname/m2'] + self.assertEqual(sorted(mailboxes), sorted([s for s in expected])) + + def testSubscribe(self): + """ + Test whether we can mark a mailbox as subscribed to + """ + acc = self.server.theAccount + + def add_mailbox(): + return acc.addMailbox('this/mbox') + + def login(): + return self.client.login(TEST_USER, TEST_PASSWD) + + def subscribe(): + return self.client.subscribe('this/mbox') + + def get_subscriptions(ignored): + return self.server.theAccount.getSubscriptions() + + d1 = self.connected.addCallback(strip(add_mailbox)) + d1.addCallback(strip(login)) + d1.addCallbacks(strip(subscribe), self._ebGeneral) + d1.addCallbacks(self._cbStopClient, self._ebGeneral) + d2 = self.loopback() + d = defer.gatherResults([d1, d2]) + d.addCallback(get_subscriptions) + d.addCallback(lambda subscriptions: + self.assertEqual(subscriptions, + ['this/mbox'])) + return d + + def testUnsubscribe(self): + """ + Test whether we can unsubscribe from a set of mailboxes + """ + acc = self.server.theAccount + + def add_mailboxes(): + return defer.gatherResults([ + acc.addMailbox('this/mbox'), + acc.addMailbox('that/mbox')]) + + def dc1(): + return acc.subscribe('this/mbox') + + def dc2(): + return acc.subscribe('that/mbox') + + def login(): + return self.client.login(TEST_USER, TEST_PASSWD) + + def unsubscribe(): + return self.client.unsubscribe('this/mbox') + + def get_subscriptions(ignored): + return acc.getSubscriptions() + + d1 = self.connected.addCallback(strip(add_mailboxes)) + d1.addCallback(strip(login)) + d1.addCallback(strip(dc1)) + d1.addCallback(strip(dc2)) + d1.addCallbacks(strip(unsubscribe), self._ebGeneral) + d1.addCallbacks(self._cbStopClient, self._ebGeneral) + d2 = self.loopback() + d = defer.gatherResults([d1, d2]) + d.addCallback(get_subscriptions) + d.addCallback(lambda subscriptions: + self.assertEqual(subscriptions, + ['that/mbox'])) + return d + + def testSelect(self): + """ + Try to select a mailbox + """ + mbox_name = "TESTMAILBOXSELECT" + self.selectedArgs = None + + acc = self.server.theAccount + + def add_mailbox(): + return acc.addMailbox(mbox_name, creation_ts=42) + + def login(): + return self.client.login(TEST_USER, TEST_PASSWD) + + def select(): + def selected(args): + self.selectedArgs = args + self._cbStopClient(None) + d = self.client.select(mbox_name) + d.addCallback(selected) + return d + + d1 = self.connected.addCallback(strip(add_mailbox)) + d1.addCallback(strip(login)) + d1.addCallback(strip(select)) + # d1.addErrback(self._ebGeneral) + + d2 = self.loopback() + + d = defer.gatherResults([d1, d2]) + d.addCallback(self._cbTestSelect) + return d + + def _cbTestSelect(self, ignored): + self.assertTrue(self.selectedArgs is not None) + + self.assertEqual(self.selectedArgs, { + 'EXISTS': 0, 'RECENT': 0, 'UIDVALIDITY': 42, + 'FLAGS': ('\\Seen', '\\Answered', '\\Flagged', + '\\Deleted', '\\Draft', '\\Recent', 'List'), + 'READ-WRITE': True + }) + + # + # capabilities + # + + def testCapability(self): + caps = {} + + def getCaps(): + def gotCaps(c): + caps.update(c) + self.server.transport.loseConnection() + return self.client.getCapabilities().addCallback(gotCaps) + + d1 = self.connected + d1.addCallback( + strip(getCaps)).addErrback(self._ebGeneral) + + d = defer.gatherResults([self.loopback(), d1]) + expected = {'IMAP4rev1': None, 'NAMESPACE': None, 'LITERAL+': None, + 'IDLE': None} + d.addCallback(lambda _: self.assertEqual(expected, caps)) + return d + + def testCapabilityWithAuth(self): + caps = {} + self.server.challengers[ + 'CRAM-MD5'] = cred.credentials.CramMD5Credentials + + def getCaps(): + def gotCaps(c): + caps.update(c) + self.server.transport.loseConnection() + return self.client.getCapabilities().addCallback(gotCaps) + d1 = self.connected.addCallback( + strip(getCaps)).addErrback(self._ebGeneral) + + d = defer.gatherResults([self.loopback(), d1]) + + expCap = {'IMAP4rev1': None, 'NAMESPACE': None, + 'IDLE': None, 'LITERAL+': None, + 'AUTH': ['CRAM-MD5']} + + d.addCallback(lambda _: self.assertEqual(expCap, caps)) + return d + + # + # authentication + # + + def testLogout(self): + """ + Test log out + """ + self.loggedOut = 0 + + def logout(): + def setLoggedOut(): + self.loggedOut = 1 + self.client.logout().addCallback(strip(setLoggedOut)) + self.connected.addCallback(strip(logout)).addErrback(self._ebGeneral) + d = self.loopback() + return d.addCallback(lambda _: self.assertEqual(self.loggedOut, 1)) + + def testNoop(self): + """ + Test noop command + """ + self.responses = None + + def noop(): + def setResponses(responses): + self.responses = responses + self.server.transport.loseConnection() + self.client.noop().addCallback(setResponses) + self.connected.addCallback(strip(noop)).addErrback(self._ebGeneral) + d = self.loopback() + return d.addCallback(lambda _: self.assertEqual(self.responses, [])) + + def testLogin(self): + """ + Test login + """ + def login(): + d = self.client.login(TEST_USER, TEST_PASSWD) + d.addCallback(self._cbStopClient) + d1 = self.connected.addCallback( + strip(login)).addErrback(self._ebGeneral) + d = defer.gatherResults([d1, self.loopback()]) + return d.addCallback(self._cbTestLogin) + + def _cbTestLogin(self, ignored): + self.assertEqual(self.server.state, 'auth') + + def testFailedLogin(self): + """ + Test bad login + """ + def login(): + d = self.client.login("bad_user@leap.se", TEST_PASSWD) + d.addBoth(self._cbStopClient) + + d1 = self.connected.addCallback( + strip(login)).addErrback(self._ebGeneral) + d2 = self.loopback() + d = defer.gatherResults([d1, d2]) + return d.addCallback(self._cbTestFailedLogin) + + def _cbTestFailedLogin(self, ignored): + self.assertEqual(self.server.state, 'unauth') + self.assertEqual(self.server.account, None) + + def testLoginRequiringQuoting(self): + """ + Test login requiring quoting + """ + self.server.checker.userid = '{test}user@leap.se' + self.server.checker.password = '{test}password' + + def login(): + d = self.client.login('{test}user@leap.se', '{test}password') + d.addBoth(self._cbStopClient) + + d1 = self.connected.addCallback( + strip(login)).addErrback(self._ebGeneral) + d = defer.gatherResults([self.loopback(), d1]) + return d.addCallback(self._cbTestLoginRequiringQuoting) + + def _cbTestLoginRequiringQuoting(self, ignored): + self.assertEqual(self.server.state, 'auth') + + # + # Inspection + # + + def testNamespace(self): + """ + Test retrieving namespace + """ + self.namespaceArgs = None + + def login(): + return self.client.login(TEST_USER, TEST_PASSWD) + + def namespace(): + def gotNamespace(args): + self.namespaceArgs = args + self._cbStopClient(None) + return self.client.namespace().addCallback(gotNamespace) + + d1 = self.connected.addCallback(strip(login)) + d1.addCallback(strip(namespace)) + d1.addErrback(self._ebGeneral) + d2 = self.loopback() + d = defer.gatherResults([d1, d2]) + d.addCallback(lambda _: self.assertEqual(self.namespaceArgs, + [[['', '/']], [], []])) + return d + + def testExamine(self): + """ + L{IMAP4Client.examine} issues an I{EXAMINE} command to the server and + returns a L{Deferred} which fires with a C{dict} with as many of the + following keys as the server includes in its response: C{'FLAGS'}, + C{'EXISTS'}, C{'RECENT'}, C{'UNSEEN'}, C{'READ-WRITE'}, C{'READ-ONLY'}, + C{'UIDVALIDITY'}, and C{'PERMANENTFLAGS'}. + + Unfortunately the server doesn't generate all of these so it's hard to + test the client's handling of them here. See + L{IMAP4ClientExamineTests} below. + + See U{RFC 3501<http://www.faqs.org/rfcs/rfc3501.html>}, section 6.3.2, + for details. + """ + # TODO implement the IMAP4ClientExamineTests testcase. + mbox_name = "test_mailbox_e" + acc = self.server.theAccount + self.examinedArgs = None + + def add_mailbox(): + return acc.addMailbox(mbox_name, creation_ts=42) + + def login(): + return self.client.login(TEST_USER, TEST_PASSWD) + + def examine(): + def examined(args): + self.examinedArgs = args + self._cbStopClient(None) + d = self.client.examine(mbox_name) + d.addCallback(examined) + return d + + d1 = self.connected.addCallback(strip(add_mailbox)) + d1.addCallback(strip(login)) + d1.addCallback(strip(examine)) + d1.addErrback(self._ebGeneral) + d2 = self.loopback() + d = defer.gatherResults([d1, d2]) + return d.addCallback(self._cbTestExamine) + + def _cbTestExamine(self, ignored): + self.assertEqual(self.examinedArgs, { + 'EXISTS': 0, 'RECENT': 0, 'UIDVALIDITY': 42, + 'FLAGS': ('\\Seen', '\\Answered', '\\Flagged', + '\\Deleted', '\\Draft', '\\Recent', 'List'), + 'READ-WRITE': False}) + + def _listSetup(self, f, f2=None): + + acc = self.server.theAccount + + def dc1(): + return acc.addMailbox('root_subthing', creation_ts=42) + + def dc2(): + return acc.addMailbox('root_another_thing', creation_ts=42) + + def dc3(): + return acc.addMailbox('non_root_subthing', creation_ts=42) + + def login(): + return self.client.login(TEST_USER, TEST_PASSWD) + + def listed(answers): + self.listed = answers + + self.listed = None + d1 = self.connected.addCallback(strip(login)) + d1.addCallback(strip(dc1)) + d1.addCallback(strip(dc2)) + d1.addCallback(strip(dc3)) + + if f2 is not None: + d1.addCallback(f2) + + d1.addCallbacks(strip(f), self._ebGeneral) + d1.addCallbacks(listed, self._ebGeneral) + d1.addCallbacks(self._cbStopClient, self._ebGeneral) + d2 = self.loopback() + return defer.gatherResults([d1, d2]).addCallback(lambda _: self.listed) + + def testList(self): + """ + Test List command + """ + def list(): + return self.client.list('root', '%') + + d = self._listSetup(list) + d.addCallback(lambda listed: self.assertEqual( + sortNest(listed), + sortNest([ + (IMAPMailbox.init_flags, "/", "root_subthing"), + (IMAPMailbox.init_flags, "/", "root_another_thing") + ]) + )) + return d + + def testLSub(self): + """ + Test LSub command + """ + acc = self.server.theAccount + + def subs_mailbox(): + # why not client.subscribe instead? + return acc.subscribe('root_subthing') + + def lsub(): + return self.client.lsub('root', '%') + + d = self._listSetup(lsub, strip(subs_mailbox)) + d.addCallback(self.assertEqual, + [(IMAPMailbox.init_flags, "/", "root_subthing")]) + return d + + def testStatus(self): + """ + Test Status command + """ + acc = self.server.theAccount + + def add_mailbox(): + return acc.addMailbox('root_subthings') + + # XXX FIXME ---- should populate this a little bit, + # with unseen etc... + + def login(): + return self.client.login(TEST_USER, TEST_PASSWD) + + def status(): + return self.client.status( + 'root_subthings', 'MESSAGES', 'UIDNEXT', 'UNSEEN') + + def statused(result): + self.statused = result + + self.statused = None + + d1 = self.connected.addCallback(strip(add_mailbox)) + d1.addCallback(strip(login)) + d1.addCallbacks(strip(status), self._ebGeneral) + d1.addCallbacks(statused, self._ebGeneral) + d1.addCallbacks(self._cbStopClient, self._ebGeneral) + d2 = self.loopback() + d = defer.gatherResults([d1, d2]) + d.addCallback(lambda _: self.assertEqual( + self.statused, + {'MESSAGES': 0, 'UIDNEXT': '1', 'UNSEEN': 0} + )) + return d + + def testFailedStatus(self): + """ + Test failed status command with a non-existent mailbox + """ + def login(): + return self.client.login(TEST_USER, TEST_PASSWD) + + def status(): + return self.client.status( + 'root/nonexistent', 'MESSAGES', 'UIDNEXT', 'UNSEEN') + + def statused(result): + self.statused = result + + def failed(failure): + self.failure = failure + + self.statused = self.failure = None + d1 = self.connected.addCallback(strip(login)) + d1.addCallbacks(strip(status), self._ebGeneral) + d1.addCallbacks(statused, failed) + d1.addCallbacks(self._cbStopClient, self._ebGeneral) + d2 = self.loopback() + return defer.gatherResults([d1, d2]).addCallback( + self._cbTestFailedStatus) + + def _cbTestFailedStatus(self, ignored): + self.assertEqual( + self.statused, None + ) + self.assertEqual( + self.failure.value.args, + ('Could not open mailbox',) + ) + + # + # messages + # + + def testFullAppend(self): + """ + Test appending a full message to the mailbox + """ + infile = util.sibpath(__file__, 'rfc822.message') + message = open(infile) + acc = self.server.theAccount + mailbox_name = "appendmbox/subthing" + + def add_mailbox(): + return acc.addMailbox(mailbox_name) + + def login(): + return self.client.login(TEST_USER, TEST_PASSWD) + + def append(): + return self.client.append( + mailbox_name, message, + ('\\SEEN', '\\DELETED'), + 'Tue, 17 Jun 2003 11:22:16 -0600 (MDT)', + ) + + d1 = self.connected.addCallback(strip(add_mailbox)) + d1.addCallback(strip(login)) + d1.addCallbacks(strip(append), self._ebGeneral) + d1.addCallbacks(self._cbStopClient, self._ebGeneral) + d2 = self.loopback() + d = defer.gatherResults([d1, d2]) + + d.addCallback(lambda _: acc.getMailbox(mailbox_name)) + d.addCallback(lambda mb: mb.fetch(imap4.MessageSet(start=1), True)) + return d.addCallback(self._cbTestFullAppend, infile) + + def _cbTestFullAppend(self, fetched, infile): + fetched = list(fetched) + self.assertTrue(len(fetched) == 1) + self.assertTrue(len(fetched[0]) == 2) + uid, msg = fetched[0] + parsed = self.parser.parse(open(infile)) + expected_body = parsed.get_payload() + expected_headers = CaseInsensitiveDict(parsed.items()) + + def assert_flags(flags): + self.assertEqual( + set(('\\SEEN', '\\DELETED')), + set(flags)) + + def assert_date(date): + self.assertEqual( + 'Tue, 17 Jun 2003 11:22:16 -0600 (MDT)', + date) + + def assert_body(body): + gotbody = body.read() + self.assertEqual(expected_body, gotbody) + + def assert_headers(headers): + self.assertItemsEqual(map(string.lower, expected_headers), headers) + + d = defer.maybeDeferred(msg.getFlags) + d.addCallback(assert_flags) + + d.addCallback(lambda _: defer.maybeDeferred(msg.getInternalDate)) + d.addCallback(assert_date) + + d.addCallback( + lambda _: defer.maybeDeferred( + msg.getBodyFile, self._soledad)) + d.addCallback(assert_body) + + d.addCallback(lambda _: defer.maybeDeferred(msg.getHeaders, True)) + d.addCallback(assert_headers) + + return d + + def testPartialAppend(self): + """ + Test partially appending a message to the mailbox + """ + # TODO this test sometimes will fail because of the notify_just_mdoc + infile = util.sibpath(__file__, 'rfc822.message') + + acc = self.server.theAccount + + def add_mailbox(): + return acc.addMailbox('PARTIAL/SUBTHING') + + def login(): + return self.client.login(TEST_USER, TEST_PASSWD) + + def append(): + message = file(infile) + return self.client.sendCommand( + imap4.Command( + 'APPEND', + 'PARTIAL/SUBTHING (\\SEEN) "Right now" ' + '{%d}' % os.path.getsize(infile), + (), self.client._IMAP4Client__cbContinueAppend, message + ) + ) + d1 = self.connected.addCallback(strip(add_mailbox)) + d1.addCallback(strip(login)) + d1.addCallbacks(strip(append), self._ebGeneral) + d1.addCallbacks(self._cbStopClient, self._ebGeneral) + d2 = self.loopback() + d = defer.gatherResults([d1, d2]) + + d.addCallback(lambda _: acc.getMailbox("PARTIAL/SUBTHING")) + d.addCallback(lambda mb: mb.fetch(imap4.MessageSet(start=1), True)) + return d.addCallback( + self._cbTestPartialAppend, infile) + + def _cbTestPartialAppend(self, fetched, infile): + fetched = list(fetched) + self.assertTrue(len(fetched) == 1) + self.assertTrue(len(fetched[0]) == 2) + uid, msg = fetched[0] + parsed = self.parser.parse(open(infile)) + expected_body = parsed.get_payload() + + def assert_flags(flags): + self.assertEqual( + set((['\\SEEN'])), set(flags)) + + def assert_body(body): + gotbody = body.read() + self.assertEqual(expected_body, gotbody) + + d = defer.maybeDeferred(msg.getFlags) + d.addCallback(assert_flags) + + d.addCallback(lambda _: defer.maybeDeferred(msg.getBodyFile)) + d.addCallback(assert_body) + return d + + def testCheck(self): + """ + Test check command + """ + def add_mailbox(): + return self.server.theAccount.addMailbox('root/subthing') + + def login(): + return self.client.login(TEST_USER, TEST_PASSWD) + + def select(): + return self.client.select('root/subthing') + + def check(): + return self.client.check() + + d = self.connected.addCallbacks( + strip(add_mailbox), self._ebGeneral) + d.addCallbacks(lambda _: login(), self._ebGeneral) + d.addCallbacks(strip(select), self._ebGeneral) + d.addCallbacks(strip(check), self._ebGeneral) + d.addCallbacks(self._cbStopClient, self._ebGeneral) + d2 = self.loopback() + return defer.gatherResults([d, d2]) + + # Okay, that was much fun indeed + + def testExpunge(self): + """ + Test expunge command + """ + acc = self.server.theAccount + mailbox_name = 'mailboxexpunge' + + def add_mailbox(): + return acc.addMailbox(mailbox_name) + + def login(): + return self.client.login(TEST_USER, TEST_PASSWD) + + def select(): + return self.client.select(mailbox_name) + + def save_mailbox(mailbox): + self.mailbox = mailbox + + def get_mailbox(): + d = acc.getMailbox(mailbox_name) + d.addCallback(save_mailbox) + return d + + def add_messages(): + d = self.mailbox.addMessage( + 'test 1', flags=('\\Deleted', 'AnotherFlag'), + notify_just_mdoc=False) + d.addCallback(lambda _: self.mailbox.addMessage( + 'test 2', flags=('AnotherFlag',), + notify_just_mdoc=False)) + d.addCallback(lambda _: self.mailbox.addMessage( + 'test 3', flags=('\\Deleted',), + notify_just_mdoc=False)) + return d + + def expunge(): + return self.client.expunge() + + def expunged(results): + self.failIf(self.server.mbox is None) + self.results = results + + self.results = None + d1 = self.connected.addCallback(strip(add_mailbox)) + d1.addCallback(strip(login)) + d1.addCallback(strip(get_mailbox)) + d1.addCallbacks(strip(add_messages), self._ebGeneral) + d1.addCallbacks(strip(select), self._ebGeneral) + d1.addCallbacks(strip(expunge), self._ebGeneral) + d1.addCallbacks(expunged, self._ebGeneral) + d1.addCallbacks(self._cbStopClient, self._ebGeneral) + d2 = self.loopback() + d = defer.gatherResults([d1, d2]) + d.addCallback(lambda _: self.mailbox.getMessageCount()) + return d.addCallback(self._cbTestExpunge) + + def _cbTestExpunge(self, count): + # we only left 1 mssage with no deleted flag + self.assertEqual(count, 1) + # the uids of the deleted messages + self.assertItemsEqual(self.results, [1, 3]) + + +class AccountTestCase(IMAP4HelperMixin): + """ + Test the Account. + """ + def _create_empty_mailbox(self): + return self.server.theAccount.addMailbox('') + + def _create_one_mailbox(self): + return self.server.theAccount.addMailbox('one') + + def test_illegalMailboxCreate(self): + self.assertRaises(AssertionError, self._create_empty_mailbox) + + +class IMAP4ServerSearchTestCase(IMAP4HelperMixin): + """ + Tests for the behavior of the search_* functions in L{imap5.IMAP4Server}. + """ + # XXX coming soon to your screens! + pass diff --git a/mail/src/leap/mail/imap/tests/walktree.py b/mail/src/leap/mail/imap/tests/walktree.py new file mode 100644 index 00000000..f259a556 --- /dev/null +++ b/mail/src/leap/mail/imap/tests/walktree.py @@ -0,0 +1,127 @@ +# -*- coding: utf-8 -*- +# walktree.py +# Copyright (C) 2013 LEAP +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see <http://www.gnu.org/licenses/>. +""" +Tests for the walktree module. +""" +import os +import sys +import pprint +from email import parser + +from leap.mail import walk as W + +DEBUG = os.environ.get("BITMASK_MAIL_DEBUG") + + +p = parser.Parser() + +# TODO pass an argument of the type of message + +################################################## +# Input from hell + +if len(sys.argv) > 1: + FILENAME = sys.argv[1] +else: + FILENAME = "rfc822.multi-signed.message" + +""" +FILENAME = "rfc822.plain.message" +FILENAME = "rfc822.multi-minimal.message" +""" + +msg = p.parse(open(FILENAME)) +DO_CHECK = False +################################################# + +parts = W.get_parts(msg) + +if DEBUG: + def trim(item): + item = item[:10] + [trim(part["phash"]) for part in parts if part.get('phash', None)] + +raw_docs = list(W.get_raw_docs(msg, parts)) + +body_phash_fun = [W.get_body_phash_simple, + W.get_body_phash_multi][int(msg.is_multipart())] +body_phash = body_phash_fun(W.get_payloads(msg)) +parts_map = W.walk_msg_tree(parts, body_phash=body_phash) + + +# TODO add missing headers! +expected = { + 'body': '1ddfa80485', + 'multi': True, + 'part_map': { + 1: { + 'headers': {'Content-Disposition': 'inline', + 'Content-Type': 'multipart/mixed; ' + 'boundary="z0eOaCaDLjvTGF2l"'}, + 'multi': True, + 'part_map': {1: {'ctype': 'text/plain', + 'headers': [ + ('Content-Type', + 'text/plain; charset=utf-8'), + ('Content-Disposition', + 'inline'), + ('Content-Transfer-Encoding', + 'quoted-printable')], + 'multi': False, + 'parts': 1, + 'phash': '1ddfa80485', + 'size': 206}, + 2: {'ctype': 'text/plain', + 'headers': [('Content-Type', + 'text/plain; charset=us-ascii'), + ('Content-Disposition', + 'attachment; ' + 'filename="attach.txt"')], + 'multi': False, + 'parts': 1, + 'phash': '7a94e4d769', + 'size': 133}, + 3: {'ctype': 'application/octet-stream', + 'headers': [('Content-Type', + 'application/octet-stream'), + ('Content-Disposition', + 'attachment; filename="hack.ico"'), + ('Content-Transfer-Encoding', + 'base64')], + 'multi': False, + 'parts': 1, + 'phash': 'c42cccebbd', + 'size': 12736}}}, + 2: {'ctype': 'application/pgp-signature', + 'headers': [('Content-Type', 'application/pgp-signature')], + 'multi': False, + 'parts': 1, + 'phash': '8f49fbf749', + 'size': 877}}} + +if DEBUG and DO_CHECK: + # TODO turn this into a proper unittest + assert(parts_map == expected) + print "Structure: OK" + + +print +print "RAW DOCS" +pprint.pprint(raw_docs) +print +print "PARTS MAP" +pprint.pprint(parts_map) diff --git a/mail/src/leap/mail/incoming/__init__.py b/mail/src/leap/mail/incoming/__init__.py new file mode 100644 index 00000000..e69de29b --- /dev/null +++ b/mail/src/leap/mail/incoming/__init__.py diff --git a/mail/src/leap/mail/incoming/service.py b/mail/src/leap/mail/incoming/service.py new file mode 100644 index 00000000..1e20862b --- /dev/null +++ b/mail/src/leap/mail/incoming/service.py @@ -0,0 +1,844 @@ +# -*- coding: utf-8 -*- +# service.py +# Copyright (C) 2015 LEAP +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see <http://www.gnu.org/licenses/>. +""" +Incoming mail fetcher. +""" +import copy +import logging +import shlex +import time +import warnings + +from email.parser import Parser +from email.utils import parseaddr +from email.utils import formatdate +from StringIO import StringIO +from urlparse import urlparse + +from twisted.application.service import Service +from twisted.python import log +from twisted.python.failure import Failure +from twisted.internet import defer, reactor +from twisted.internet.task import LoopingCall +from twisted.internet.task import deferLater + +from leap.common.events import emit_async, catalog +from leap.common.check import leap_assert, leap_assert_type +from leap.common.mail import get_email_charset +from leap.keymanager import errors as keymanager_errors +from leap.mail.adaptors import soledad_indexes as fields +from leap.mail.generator import Generator +from leap.mail.utils import json_loads, empty +from leap.soledad.client import Soledad +from leap.soledad.common.crypto import ENC_SCHEME_KEY, ENC_JSON_KEY +from leap.soledad.common.errors import InvalidAuthTokenError + + +logger = logging.getLogger(__name__) + +MULTIPART_ENCRYPTED = "multipart/encrypted" +MULTIPART_SIGNED = "multipart/signed" +PGP_BEGIN = "-----BEGIN PGP MESSAGE-----" +PGP_END = "-----END PGP MESSAGE-----" + +# The period between succesive checks of the incoming mail +# queue (in seconds) +INCOMING_CHECK_PERIOD = 60 + + +class MalformedMessage(Exception): + """ + Raised when a given message is not well formed. + """ + pass + + +class IncomingMail(Service): + """ + Fetches and process mail from the incoming pool. + + This object implements IService interface, has public methods + startService and stopService that will actually initiate a + LoopingCall with check_period recurrency. + The LoopingCall itself will invoke the fetch method each time + that the check_period expires. + + This loop will sync the soledad db with the remote server and + process all the documents found tagged as incoming mail. + """ + # TODO implements IService? + + name = "IncomingMail" + + RECENT_FLAG = "\\Recent" + CONTENT_KEY = "content" + + LEAP_SIGNATURE_HEADER = 'X-Leap-Signature' + LEAP_ENCRYPTION_HEADER = 'X-Leap-Encryption' + """ + Header added to messages when they are decrypted by the fetcher, + which states the validity of an eventual signature that might be included + in the encrypted blob. + """ + LEAP_SIGNATURE_VALID = 'valid' + LEAP_SIGNATURE_INVALID = 'invalid' + LEAP_SIGNATURE_COULD_NOT_VERIFY = 'could not verify' + + LEAP_ENCRYPTION_DECRYPTED = 'decrypted' + + def __init__(self, keymanager, soledad, inbox, userid, + check_period=INCOMING_CHECK_PERIOD): + + """ + Initialize IncomingMail.. + + :param keymanager: a keymanager instance + :type keymanager: keymanager.KeyManager + + :param soledad: a soledad instance + :type soledad: Soledad + + :param inbox: the collection for the inbox where the new emails will be + stored + :type inbox: MessageCollection + + :param check_period: the period to fetch new mail, in seconds. + :type check_period: int + """ + leap_assert(keymanager, "need a keymanager to initialize") + leap_assert_type(soledad, Soledad) + leap_assert(check_period, "need a period to check incoming mail") + leap_assert_type(check_period, int) + leap_assert(userid, "need a userid to initialize") + + self._keymanager = keymanager + self._soledad = soledad + self._inbox_collection = inbox + self._userid = userid + + self._listeners = [] + self._loop = None + self._check_period = check_period + + # initialize a mail parser only once + self._parser = Parser() + + def add_listener(self, listener): + """ + Add a listener to inbox insertions. + + This listener function will be called for each message added to the + inbox with its uid as parameter. This function should not be blocking + or it will block the incoming queue. + + :param listener: the listener function + :type listener: callable + """ + self._listeners.append(listener) + + # + # Public API: fetch, start_loop, stop. + # + + def fetch(self): + """ + Fetch incoming mail, to be called periodically. + + Calls a deferred that will execute the fetch callback. + """ + def _sync_errback(failure): + log.err(failure) + + def syncSoledadCallback(_): + # XXX this should be moved to adaptors + d = self._soledad.get_from_index( + fields.JUST_MAIL_IDX, "1", "0") + d.addCallback(self._process_doclist) + d.addErrback(_sync_errback) + return d + + logger.debug("fetching mail for: %s %s" % ( + self._soledad.uuid, self._userid)) + d = self._sync_soledad() + d.addCallbacks(syncSoledadCallback, self._errback) + d.addCallbacks(self._signal_fetch_to_ui, self._errback) + return d + + def startService(self): + """ + Starts a loop to fetch mail. + + :returns: A Deferred whose callback will be invoked with + the LoopingCall instance when loop.stop is called, or + whose errback will be invoked when the function raises an + exception or returned a deferred that has its errback + invoked. + """ + Service.startService(self) + if self._loop is None: + self._loop = LoopingCall(self.fetch) + stop_deferred = self._loop.start(self._check_period) + return stop_deferred + else: + logger.warning("Tried to start an already running fetching loop.") + return defer.fail(Failure('Already running loop.')) + + def stopService(self): + """ + Stops the loop that fetches mail. + """ + if self._loop and self._loop.running is True: + self._loop.stop() + self._loop = None + Service.stopService(self) + + # + # Private methods. + # + + # synchronize incoming mail + + def _errback(self, failure): + log.err(failure) + + def _sync_soledad(self): + """ + Synchronize with remote soledad. + + :returns: a list of LeapDocuments, or None. + :rtype: iterable or None + """ + def _log_synced(result): + log.msg('FETCH soledad SYNCED.') + return result + + def _signal_invalid_auth(failure): + failure.trap(InvalidAuthTokenError) + # if the token is invalid, send an event so the GUI can + # disable mail and show an error message. + emit_async(catalog.SOLEDAD_INVALID_AUTH_TOKEN, self._userid) + + log.msg('FETCH: syncing soledad...') + d = self._soledad.sync() + d.addCallbacks(_log_synced, _signal_invalid_auth) + return d + + def _signal_fetch_to_ui(self, doclist): + """ + Send leap events to ui. + + :param doclist: iterable with msg documents. + :type doclist: iterable. + :returns: doclist + :rtype: iterable + """ + if doclist: + fetched_ts = time.mktime(time.gmtime()) + num_mails = len(doclist) if doclist is not None else 0 + if num_mails != 0: + log.msg("there are %s mails" % (num_mails,)) + emit_async(catalog.MAIL_FETCHED_INCOMING, self._userid, + str(num_mails), str(fetched_ts)) + return doclist + + def _signal_unread_to_ui(self, *args): + """ + Sends unread event to ui. + """ + emit_async(catalog.MAIL_UNREAD_MESSAGES, self._userid, + str(self._inbox_collection.count_unseen())) + + # process incoming mail. + + def _process_doclist(self, doclist): + """ + Iterates through the doclist, checks if each doc + looks like a message, and yields a deferred that will decrypt and + process the message. + + :param doclist: iterable with msg documents. + :type doclist: iterable. + :returns: a list of deferreds for individual messages. + """ + log.msg('processing doclist') + if not doclist: + logger.debug("no docs found") + return + num_mails = len(doclist) + + deferreds = [] + for index, doc in enumerate(doclist): + logger.debug("processing doc %d of %d" % (index + 1, num_mails)) + emit_async(catalog.MAIL_MSG_PROCESSING, self._userid, + str(index), str(num_mails)) + + keys = doc.content.keys() + + # TODO Compatibility check with the index in pre-0.6 mx + # that does not write the ERROR_DECRYPTING_KEY + # This should be removed in 0.7 + + has_errors = doc.content.get(fields.ERROR_DECRYPTING_KEY, None) + + if has_errors is None: + warnings.warn("JUST_MAIL_COMPAT_IDX will be deprecated!", + DeprecationWarning) + + if has_errors: + logger.debug("skipping msg with decrypting errors...") + elif self._is_msg(keys): + # TODO this pipeline is a bit obscure! + d = self._decrypt_doc(doc) + d.addCallback(self._maybe_extract_keys) + d.addCallbacks(self._add_message_locally, self._errback) + deferreds.append(d) + + d = defer.gatherResults(deferreds, consumeErrors=True) + d.addCallback(lambda _: doclist) + return d + + # + # operations on individual messages + # + + def _decrypt_doc(self, doc): + """ + Decrypt the contents of a document. + + :param doc: A document containing an encrypted message. + :type doc: SoledadDocument + + :return: A Deferred that will be fired with the document and the + decrypted message. + :rtype: SoledadDocument, str + """ + log.msg('decrypting msg') + + def process_decrypted(res): + if isinstance(res, tuple): + decrdata, _ = res + success = True + else: + decrdata = "" + success = False + + emit_async(catalog.MAIL_MSG_DECRYPTED, self._userid, + "1" if success else "0") + return self._process_decrypted_doc(doc, decrdata) + + d = self._keymanager.decrypt(doc.content[ENC_JSON_KEY], self._userid) + d.addErrback(self._errback) + d.addCallback(process_decrypted) + d.addCallback(lambda data: (doc, data)) + return d + + def _process_decrypted_doc(self, doc, data): + """ + Process a document containing a succesfully decrypted message. + + :param doc: the incoming message + :type doc: SoledadDocument + :param data: the json-encoded, decrypted content of the incoming + message + :type data: str + + :return: a Deferred that will be fired with an str of the proccessed + data. + :rtype: Deferred + """ + log.msg('processing decrypted doc') + + # XXX turn this into an errBack for each one of + # the deferreds that would process an individual document + try: + msg = json_loads(data) + except UnicodeError as exc: + logger.error("Error while decrypting %s" % (doc.doc_id,)) + logger.exception(exc) + + # we flag the message as "with decrypting errors", + # to avoid further decryption attempts during sync + # cycles until we're prepared to deal with that. + # What is the same, when Ivan deals with it... + # A new decrypting attempt event could be triggered by a + # future a library upgrade, or a cli flag to the client, + # we just `defer` that for now... :) + doc.content[fields.ERROR_DECRYPTING_KEY] = True + deferLater(reactor, 0, self._update_incoming_message, doc) + + # FIXME this is just a dirty hack to delay the proper + # deferred organization here... + # and remember, boys, do not do this at home. + return [] + + if not isinstance(msg, dict): + defer.returnValue(False) + if not msg.get(fields.INCOMING_KEY, False): + defer.returnValue(False) + + # ok, this is an incoming message + rawmsg = msg.get(self.CONTENT_KEY, None) + if rawmsg is None: + return "" + return self._maybe_decrypt_msg(rawmsg) + + def _update_incoming_message(self, doc): + """ + Do a put for a soledad document. This probably has been called only + in the case that we've needed to update the ERROR_DECRYPTING_KEY + flag in an incoming message, to get it out of the decrypting queue. + + :param doc: the SoledadDocument to update + :type doc: SoledadDocument + """ + log.msg("Updating Incoming MSG: SoledadDoc %s" % (doc.doc_id)) + return self._soledad.put_doc(doc) + + def _delete_incoming_message(self, doc): + """ + Delete document. + + :param doc: the SoledadDocument to delete + :type doc: SoledadDocument + """ + log.msg("Deleting Incoming message: %s" % (doc.doc_id,)) + return self._soledad.delete_doc(doc) + + def _maybe_decrypt_msg(self, data): + """ + Tries to decrypt a gpg message if data looks like one. + + :param data: the text to be decrypted. + :type data: str + + :return: a Deferred that will be fired with an str of data, possibly + decrypted. + :rtype: Deferred + """ + leap_assert_type(data, str) + log.msg('maybe decrypting doc') + + # parse the original message + encoding = get_email_charset(data) + msg = self._parser.parsestr(data) + + fromHeader = msg.get('from', None) + senderAddress = None + + if (fromHeader is not None and + (msg.get_content_type() == MULTIPART_ENCRYPTED or + msg.get_content_type() == MULTIPART_SIGNED)): + senderAddress = parseaddr(fromHeader)[1] + + def add_leap_header(ret): + decrmsg, signkey = ret + if (senderAddress is None or signkey is None or + isinstance(signkey, keymanager_errors.KeyNotFound)): + decrmsg.add_header( + self.LEAP_SIGNATURE_HEADER, + self.LEAP_SIGNATURE_COULD_NOT_VERIFY) + elif isinstance(signkey, keymanager_errors.InvalidSignature): + decrmsg.add_header( + self.LEAP_SIGNATURE_HEADER, + self.LEAP_SIGNATURE_INVALID) + else: + self._add_verified_signature_header(decrmsg, + signkey.fingerprint) + return decrmsg.as_string() + + if msg.get_content_type() == MULTIPART_ENCRYPTED: + d = self._decrypt_multipart_encrypted_msg( + msg, encoding, senderAddress) + elif msg.get_content_type() == MULTIPART_SIGNED: + d = self._verify_signature_not_encrypted_msg(msg, senderAddress) + else: + d = self._maybe_decrypt_inline_encrypted_msg( + msg, encoding, senderAddress) + d.addCallback(add_leap_header) + return d + + def _add_verified_signature_header(self, decrmsg, fingerprint): + decrmsg.add_header( + self.LEAP_SIGNATURE_HEADER, + self.LEAP_SIGNATURE_VALID, + pubkey=fingerprint) + + def _add_decrypted_header(self, msg): + msg.add_header(self.LEAP_ENCRYPTION_HEADER, + self.LEAP_ENCRYPTION_DECRYPTED) + + def _decrypt_multipart_encrypted_msg(self, msg, encoding, senderAddress): + """ + Decrypt a message with content-type 'multipart/encrypted'. + + :param msg: The original encrypted message. + :type msg: Message + :param encoding: The encoding of the email message. + :type encoding: str + :param senderAddress: The email address of the sender of the message. + :type senderAddress: str + + :return: A Deferred that will be fired with a tuple containing a + decrypted Message and the signing OpenPGPKey if the signature + is valid or InvalidSignature or KeyNotFound. + :rtype: Deferred + """ + log.msg('decrypting multipart encrypted msg') + msg = copy.deepcopy(msg) + self._msg_multipart_sanity_check(msg) + + # parse message and get encrypted content + pgpencmsg = msg.get_payload()[1] + encdata = pgpencmsg.get_payload() + + # decrypt or fail gracefully + def build_msg(res): + decrdata, signkey = res + + decrmsg = self._parser.parsestr(decrdata) + # remove original message's multipart/encrypted content-type + del(msg['content-type']) + + # replace headers back in original message + for hkey, hval in decrmsg.items(): + try: + # this will raise KeyError if header is not present + msg.replace_header(hkey, hval) + except KeyError: + msg[hkey] = hval + + # all ok, replace payload by unencrypted payload + msg.set_payload(decrmsg.get_payload()) + self._add_decrypted_header(msg) + return (msg, signkey) + + def verify_signature_after_decrypt_an_email(res): + decrdata, signkey = res + if decrdata.get_content_type() == MULTIPART_SIGNED: + res = self._verify_signature_not_encrypted_msg(decrdata, + senderAddress) + return res + + d = self._keymanager.decrypt( + encdata, self._userid, verify=senderAddress) + d.addCallbacks(build_msg, self._decryption_error, errbackArgs=(msg,)) + d.addCallbacks(verify_signature_after_decrypt_an_email) + return d + + def _maybe_decrypt_inline_encrypted_msg(self, origmsg, encoding, + senderAddress): + """ + Possibly decrypt an inline OpenPGP encrypted message. + + :param origmsg: The original, possibly encrypted message. + :type origmsg: Message + :param encoding: The encoding of the email message. + :type encoding: str + :param senderAddress: The email address of the sender of the message. + :type senderAddress: str + + :return: A Deferred that will be fired with a tuple containing a + decrypted Message and the signing OpenPGPKey if the signature + is valid or InvalidSignature or KeyNotFound. + :rtype: Deferred + """ + log.msg('maybe decrypting inline encrypted msg') + + data = self._serialize_msg(origmsg) + + def decrypted_data(res): + decrdata, signkey = res + replaced_data = data.replace(pgp_message, decrdata) + self._add_decrypted_header(origmsg) + return replaced_data, signkey + + def encode_and_return(res): + data, signkey = res + if isinstance(data, unicode): + data = data.encode(encoding, 'replace') + return (self._parser.parsestr(data), signkey) + + # handle exactly one inline PGP message + if PGP_BEGIN in data: + begin = data.find(PGP_BEGIN) + end = data.find(PGP_END) + pgp_message = data[begin:end + len(PGP_END)] + d = self._keymanager.decrypt( + pgp_message, self._userid, verify=senderAddress) + d.addCallbacks(decrypted_data, self._decryption_error, + errbackArgs=(data,)) + else: + d = defer.succeed((data, None)) + d.addCallback(encode_and_return) + return d + + def _verify_signature_not_encrypted_msg(self, origmsg, sender_address): + """ + Possibly decrypt an inline OpenPGP encrypted message. + + :param origmsg: The original, possibly encrypted message. + :type origmsg: Message + :param sender_address: The email address of the sender of the message. + :type sender_address: str + + :return: A Deferred that will be fired with a tuple containing a + signed Message and the signing OpenPGPKey if the signature + is valid or InvalidSignature. + :rtype: Deferred + """ + msg = copy.deepcopy(origmsg) + data = self._serialize_msg(msg.get_payload(0)) + detached_sig = self._extract_signature(msg) + d = self._keymanager.verify(data, sender_address, detached_sig) + + d.addCallback(lambda sign_key: (msg, sign_key)) + d.addErrback(lambda _: (msg, keymanager_errors.InvalidSignature())) + return d + + def _serialize_msg(self, origmsg): + buf = StringIO() + g = Generator(buf) + g.flatten(origmsg) + return buf.getvalue() + + def _extract_signature(self, msg): + body = msg.get_payload(0).get_payload() + + if isinstance(body, str): + body = msg.get_payload(0) + + detached_sig = msg.get_payload(1).get_payload() + msg.set_payload(body) + return detached_sig + + def _decryption_error(self, failure, msg): + """ + Check for known decryption errors + """ + if failure.check(keymanager_errors.DecryptError): + logger.warning('Failed to decrypt encrypted message (%s). ' + 'Storing message without modifications.' + % str(failure.value)) + return (msg, None) + elif failure.check(keymanager_errors.KeyNotFound): + logger.error('Failed to find private key for decryption (%s). ' + 'Storing message without modifications.' + % str(failure.value)) + return (msg, None) + else: + return failure + + @defer.inlineCallbacks + def _maybe_extract_keys(self, msgtuple): + """ + Retrieve attached keys to the mesage and parse message headers for an + *OpenPGP* header as described on the `IETF draft + <http://tools.ietf.org/html/draft-josefsson-openpgp-mailnews-header-06>` + only urls with https and the same hostname than the email are supported + for security reasons. + + :param msgtuple: a tuple consisting of a SoledadDocument + instance containing the incoming message + and data, the json-encoded, decrypted content of the + incoming message + :type msgtuple: (SoledadDocument, str) + + :return: A Deferred that will be fired with msgtuple when key + extraction finishes + :rtype: Deferred + """ + OpenPGP_HEADER = 'OpenPGP' + doc, data = msgtuple + + # XXX the parsing of the message is done in mailbox.addMessage, maybe + # we should do it in this module so we don't need to parse it again + # here + msg = self._parser.parsestr(data) + _, fromAddress = parseaddr(msg['from']) + + valid_attachment = False + if msg.is_multipart(): + valid_attachment = yield self._maybe_extract_attached_key( + msg.get_payload(), fromAddress) + + if not valid_attachment: + header = msg.get(OpenPGP_HEADER, None) + if header is not None: + yield self._maybe_extract_openpgp_header(header, fromAddress) + + defer.returnValue(msgtuple) + + def _maybe_extract_openpgp_header(self, header, address): + """ + Import keys from the OpenPGP header + + :param header: OpenPGP header string + :type header: str + :param address: email address in the from header + :type address: str + + :return: A Deferred that will be fired when header extraction is done + :rtype: Deferred + """ + d = defer.succeed(None) + fields = dict([f.strip(' ').split('=') for f in header.split(';')]) + if 'url' in fields: + url = shlex.split(fields['url'])[0] # remove quotations + urlparts = urlparse(url) + addressHostname = address.split('@')[1] + if ( + urlparts.scheme == 'https' and + urlparts.hostname == addressHostname + ): + def fetch_error(failure): + if failure.check(keymanager_errors.KeyNotFound): + logger.warning("Url from OpenPGP header %s failed" + % (url,)) + elif failure.check(keymanager_errors.KeyAttributesDiffer): + logger.warning("Key from OpenPGP header url %s didn't " + "match the from address %s" + % (url, address)) + else: + return failure + + d = self._keymanager.fetch_key(address, url) + d.addCallback( + lambda _: + logger.info("Imported key from header %s" % (url,))) + d.addErrback(fetch_error) + else: + logger.debug("No valid url on OpenPGP header %s" % (url,)) + else: + logger.debug("There is no url on the OpenPGP header: %s" + % (header,)) + return d + + def _maybe_extract_attached_key(self, attachments, address): + """ + Import keys from the attachments + + :param attachments: email attachment list + :type attachments: list(email.Message) + :param address: email address in the from header + :type address: str + + :return: A Deferred that will be fired when all the keys are stored + with a boolean: True if there was a valid key attached, or + False otherwise. + :rtype: Deferred + """ + MIME_KEY = "application/pgp-keys" + + def log_key_added(ignored): + logger.debug('Added key found in attachment for %s' % address) + return True + + def failed_put_key(failure): + logger.info("An error has ocurred adding attached key for %s: %s" + % (address, failure.getErrorMessage())) + return False + + deferreds = [] + for attachment in attachments: + if MIME_KEY == attachment.get_content_type(): + d = self._keymanager.put_raw_key( + attachment.get_payload(decode=True), address=address) + d.addCallbacks(log_key_added, failed_put_key) + deferreds.append(d) + d = defer.gatherResults(deferreds) + d.addCallback(lambda result: any(result)) + return d + + def _add_message_locally(self, msgtuple): + """ + Adds a message to local inbox and delete it from the incoming db + in soledad. + + :param msgtuple: a tuple consisting of a SoledadDocument + instance containing the incoming message + and data, the json-encoded, decrypted content of the + incoming message + :type msgtuple: (SoledadDocument, str) + + :return: A Deferred that will be fired when the messages is stored + :rtype: Defferred + """ + doc, raw_data = msgtuple + insertion_date = formatdate(time.time()) + log.msg('adding message %s to local db' % (doc.doc_id,)) + + def msgSavedCallback(result): + if empty(result): + return + + for listener in self._listeners: + listener(result) + + def signal_deleted(doc_id): + emit_async(catalog.MAIL_MSG_DELETED_INCOMING, + self._userid) + return doc_id + + emit_async(catalog.MAIL_MSG_SAVED_LOCALLY, self._userid) + d = self._delete_incoming_message(doc) + d.addCallback(signal_deleted) + return d + + d = self._inbox_collection.add_msg( + raw_data, (self.RECENT_FLAG,), date=insertion_date, + notify_just_mdoc=True) + d.addCallbacks(msgSavedCallback, self._errback) + return d + + # + # helpers + # + + def _msg_multipart_sanity_check(self, msg): + """ + Performs a sanity check against a multipart encrypted msg + + :param msg: The original encrypted message. + :type msg: Message + """ + # sanity check + payload = msg.get_payload() + if len(payload) != 2: + raise MalformedMessage( + 'Multipart/encrypted messages should have exactly 2 body ' + 'parts (instead of %d).' % len(payload)) + if payload[0].get_content_type() != 'application/pgp-encrypted': + raise MalformedMessage( + "Multipart/encrypted messages' first body part should " + "have content type equal to 'application/pgp-encrypted' " + "(instead of %s)." % payload[0].get_content_type()) + if payload[1].get_content_type() != 'application/octet-stream': + raise MalformedMessage( + "Multipart/encrypted messages' second body part should " + "have content type equal to 'octet-stream' (instead of " + "%s)." % payload[1].get_content_type()) + + def _is_msg(self, keys): + """ + Checks if the keys of a dictionary match the signature + of the document type we use for messages. + + :param keys: iterable containing the strings to match. + :type keys: iterable of strings. + :rtype: bool + """ + return ENC_SCHEME_KEY in keys and ENC_JSON_KEY in keys diff --git a/mail/src/leap/mail/incoming/tests/rfc822.multi-encrypt-signed.message b/mail/src/leap/mail/incoming/tests/rfc822.multi-encrypt-signed.message new file mode 100644 index 00000000..98304f24 --- /dev/null +++ b/mail/src/leap/mail/incoming/tests/rfc822.multi-encrypt-signed.message @@ -0,0 +1,61 @@ +Content-Type: multipart/encrypted; + boundary="Apple-Mail=_C01A1464-6C43-43BF-8F62-157335B7E25B"; + protocol="application/pgp-encrypted"; +Subject: Enc signed +Mime-Version: 1.0 (Mac OS X Mail 9.3 \(3124\)) +From: Leap Test Key <leap@leap.se> +Date: Tue, 24 May 2016 11:47:24 -0300 +Content-Description: OpenPGP encrypted message +To: leap@leap.se + +This is an OpenPGP/MIME encrypted message (RFC 2440 and 3156) +--Apple-Mail=_C01A1464-6C43-43BF-8F62-157335B7E25B +Content-Type: application/pgp-encrypted +Content-Description: PGP/MIME Versions Identification + +--Apple-Mail=_C01A1464-6C43-43BF-8F62-157335B7E25B +Content-Disposition: inline; + filename=encrypted.asc +Content-Type: application/octet-stream; + name=encrypted.asc +Content-Description: OpenPGP encrypted message + +-----BEGIN PGP MESSAGE----- +Version: GnuPG v2 + +hQIMAyj9aG/xtZOwAQ/9Gft0KmOpgzL6z4wmVlLm2aeAvHolXmxWb7N/ByL/dZ4n +YZd/GPRj42X3BwUrDEL5aO3Mcp+rqq8ACh9hsZXiau0Q9cs1K7Gr55Y06qLrIjom +2fLqwLFBxCL2sAX1dvClgStyfsRFk9Y/+5tX+IjWaD8dAoRdxCO8IbUDuYGnaKld +bB9h0NMfKVddCAvuQvX1Zc1Nx0Yb3Hd+ocDD7i9BVgX1BBiGu4/ElS3d32TAVCFs +Na3tjitWB2G472CYu1O6exY7h1F5V4FHfXH6iMRJSYnvV2Jr+oPZENzNdEEA5H/H +fUbpWrpKzPafjho9S5rJBBM/tqtmBQFBIdgFVcBVb+bXO6DJ8SMTLiiGcVUvvm1b +9N2VQIhsxtZ8DpcHHSqFVgT2Gt4UkSrEleSoReg36TzS1s8Uw0oU068PwTe3K0Gx +2pLMdT9NA6X/t7movpXP6tih1l6P5z62dxFl6W12J9OcegISCt0Q7gex1gk/a8zM +rzBJC3mVxRiFlvHPBgD6oUKarnTJPQx5f5dFXg8DXBWR1Eh/aFjPQIzhZBYpmOi8 +HqgjcAA+WhMQ7v5c0enJoJJS+8Xfai/MK2vTUGsfAT6HqHLw1HSIn6XQGEf4sQ/U +NfLeFHHbe9rTk8QhyjrSl2vvek2H4EBQVLF08/FUrAfPELUttOFtysQfC3+M0+PS +6QGyeIlUjKpBJG7HBd4ibuKMQ5vnA+ACsg/TySYeCO6P85xsN+Lmqlr8cAICn/hR +ezFSzlibaIelRgfDEDJdjVyCsa7qBMjhRCvGYBdkyTzIRq53qwD9pkhrQ6nwWQrv +bBzyLrl+NVR8CTEOwbeFLI6qf68kblojk3lwo3Qi3psmeMJdiaV9uevsHrgmEFTH +lZ3rFECPWzmrkMSfVjWu5d8jJqMcqa4lnGzFQKaB76I8BzGhCWrnuvHPB9c9SVhI +AnAwNw3gY5xgsbXMxZhnPgYeBSViPkQkgRCWl8Jz41eiAJ3Gtj8QSSFWGHpX+MgP +ohBaPHz6Fnkhz7Lok97e2AcuRZrDVKV6i28r8mizI3B2Mah6ZV0Yuv0EYNtzBv/v +yV3nu4DWuOOU0301CXBayxJGX0h07z1Ycv7jWD6LNiBXa1vahtbU4WSYNkF0OJaz +nf8O3CZy5twMq5kQYoPacdNNLregAmWquvE1nxqWbtHFMjtXitP7czxzUTU/DE+C +jr+irDoYEregEKg9xov91UCRPZgxL+TML71+tSYOMO3JG6lbGw77PQ8s2So7xore +8+FeDFPaaJqh6uhF5LETRSx8x/haZiXLd+WtO7wF8S3+Vz7AJIFIe8MUadZrYwnH +wfMAktQKbep3iHCeZ5jHYA461AOhnCca2y+GoyHZUDDFwS1pC1RN4lMkafSE1AgH +cmEcjLYsw1gqT0+DfqrvjbXmMjGgkgnkMybJH7df5TKu36Q0Nqvcbc2XLFkalr5V +Vk0SScqKYnKL+cJjabqA8rKkeAh22E2FBCpKPqxSS3te2bRb3XBX26bP0LshkJuy +GPu6LKvwmUn0obPKCnLJvb9ImIGZToXu6Fb/Cd2c3DG1IK5PptQz4f7ZRW98huPO +2w59Bswwt5q4lQqsMEzVRnIDH45MmnhEUeS4NaxqLTO7eJpMpb4VxT2u/Ac3XWKp +o2RE6CbqTyJ+n8tY9OwBRMKzdVd9RFAMqMHTzWTAuU4BgW2vT2sHYZdAsX8sktBr +5mo9P3MqvgdPNpg8+AOB03JlIv0dzrAFWCZxxLLGIIIz0eXsjghHzQ9QjGfr0xFH +Z79AKDjsoRisWyWCnadS2oM9fdAg4T/h1STnfxc44o7N1+ym7u58ODICFi+Kg8IR +JBHIp3CK02JLTLd/WFhUVyWgc6l8gn+oBK+r7Dw+FTWhqX2/ZHCO8qKK1ZK3NIMn +MBcSVvHSnTPtppb+oND5nk38xazVVHnwxNHaIh7g3NxDB4hl5rBhrWsgTNuqDDRU +w7ufvMYr1AOV+8e92cHCEKPM19nFKEgaBFECEptEObesGI3QZPAESlojzQ3cDeBa +=tEyc +-----END PGP MESSAGE----- + +--Apple-Mail=_C01A1464-6C43-43BF-8F62-157335B7E25B--
\ No newline at end of file diff --git a/mail/src/leap/mail/incoming/tests/test_incoming_mail.py b/mail/src/leap/mail/incoming/tests/test_incoming_mail.py new file mode 100644 index 00000000..29422ecc --- /dev/null +++ b/mail/src/leap/mail/incoming/tests/test_incoming_mail.py @@ -0,0 +1,391 @@ +# -*- coding: utf-8 -*- +# test_incoming_mail.py +# Copyright (C) 2015 LEAP +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see <http://www.gnu.org/licenses/>. +""" +Test case for leap.mail.incoming.service + +@authors: Ruben Pollan, <meskio@sindominio.net> + +@license: GPLv3, see included LICENSE file +""" + +import json +import os +import tempfile +import uuid + +from email.mime.application import MIMEApplication +from email.mime.multipart import MIMEMultipart +from email.parser import Parser +from mock import Mock + +from twisted.internet import defer +from twisted.python import log + +from leap.keymanager.errors import KeyAddressMismatch +from leap.mail.adaptors import soledad_indexes as fields +from leap.mail.adaptors.soledad import cleanup_deferred_locks +from leap.mail.adaptors.soledad import SoledadMailAdaptor +from leap.mail.mail import MessageCollection +from leap.mail.mailbox_indexer import MailboxIndexer + +from leap.mail.incoming.service import IncomingMail +from leap.mail.rfc3156 import MultipartEncrypted, PGPEncrypted +from leap.mail.testing import KeyManagerWithSoledadTestCase +from leap.mail.testing import ADDRESS, ADDRESS_2 +from leap.soledad.common.document import SoledadDocument +from leap.soledad.common.crypto import ( + EncryptionSchemes, + ENC_JSON_KEY, + ENC_SCHEME_KEY, +) + +HERE = os.path.split(os.path.abspath(__file__))[0] + +# TODO: add some tests for encrypted, unencrypted, signed and unsgined messages + + +class IncomingMailTestCase(KeyManagerWithSoledadTestCase): + """ + Tests for the incoming mail parser + """ + NICKSERVER = "http://domain" + BODY = """ +Governments of the Industrial World, you weary giants of flesh and steel, I +come from Cyberspace, the new home of Mind. On behalf of the future, I ask +you of the past to leave us alone. You are not welcome among us. You have +no sovereignty where we gather. + """ + EMAIL = """from: Test from SomeDomain <%(from)s> +to: %(to)s +subject: independence of cyberspace + +%(body)s + """ % { + "from": ADDRESS_2, + "to": ADDRESS, + "body": BODY + } + + def setUp(self): + cleanup_deferred_locks() + try: + del self._soledad + del self.km + except AttributeError: + pass + + # pytest handles correctly the setupEnv for the class, + # but trial ignores it. + if not getattr(self, 'tempdir', None): + self.tempdir = tempfile.mkdtemp() + + def getCollection(_): + adaptor = SoledadMailAdaptor() + store = self._soledad + adaptor.store = store + mbox_indexer = MailboxIndexer(store) + mbox_name = "INBOX" + mbox_uuid = str(uuid.uuid4()) + + def get_collection_from_mbox_wrapper(wrapper): + wrapper.uuid = mbox_uuid + return MessageCollection( + adaptor, store, + mbox_indexer=mbox_indexer, mbox_wrapper=wrapper) + + d = adaptor.initialize_store(store) + d.addCallback(lambda _: mbox_indexer.create_table(mbox_uuid)) + d.addCallback( + lambda _: adaptor.get_or_create_mbox(store, mbox_name)) + d.addCallback(get_collection_from_mbox_wrapper) + return d + + def setUpFetcher(inbox_collection): + self.fetcher = IncomingMail( + self.km, + self._soledad, + inbox_collection, + ADDRESS) + + # The messages don't exist on soledad will fail on deletion + self.fetcher._delete_incoming_message = Mock( + return_value=defer.succeed(None)) + + d = KeyManagerWithSoledadTestCase.setUp(self) + d.addCallback(getCollection) + d.addCallback(setUpFetcher) + d.addErrback(log.err) + return d + + def tearDown(self): + d = KeyManagerWithSoledadTestCase.tearDown(self) + return d + + def testExtractOpenPGPHeader(self): + """ + Test the OpenPGP header key extraction + """ + KEYURL = "https://leap.se/key.txt" + OpenPGP = "id=12345678; url=\"%s\"; preference=signencrypt" % (KEYURL,) + + message = Parser().parsestr(self.EMAIL) + message.add_header("OpenPGP", OpenPGP) + self.fetcher._keymanager.fetch_key = Mock( + return_value=defer.succeed(None)) + + def fetch_key_called(ret): + self.fetcher._keymanager.fetch_key.assert_called_once_with( + ADDRESS_2, KEYURL) + + d = self._create_incoming_email(message.as_string()) + d.addCallback( + lambda email: + self._mock_soledad_get_from_index(fields.JUST_MAIL_IDX, [email])) + d.addCallback(lambda _: self.fetcher.fetch()) + d.addCallback(fetch_key_called) + return d + + def testExtractOpenPGPHeaderInvalidUrl(self): + """ + Test the OpenPGP header key extraction + """ + KEYURL = "https://someotherdomain.com/key.txt" + OpenPGP = "id=12345678; url=\"%s\"; preference=signencrypt" % (KEYURL,) + + message = Parser().parsestr(self.EMAIL) + message.add_header("OpenPGP", OpenPGP) + self.fetcher._keymanager.fetch_key = Mock() + + def fetch_key_called(ret): + self.assertFalse(self.fetcher._keymanager.fetch_key.called) + + d = self._create_incoming_email(message.as_string()) + d.addCallback( + lambda email: + self._mock_soledad_get_from_index(fields.JUST_MAIL_IDX, [email])) + d.addCallback(lambda _: self.fetcher.fetch()) + d.addCallback(fetch_key_called) + return d + + def testExtractAttachedKey(self): + KEY = "-----BEGIN PGP PUBLIC KEY BLOCK-----\n..." + + message = MIMEMultipart() + message.add_header("from", ADDRESS_2) + key = MIMEApplication("", "pgp-keys") + key.set_payload(KEY) + message.attach(key) + self.fetcher._keymanager.put_raw_key = Mock( + return_value=defer.succeed(None)) + + def put_raw_key_called(_): + self.fetcher._keymanager.put_raw_key.assert_called_once_with( + KEY, address=ADDRESS_2) + + d = self._do_fetch(message.as_string()) + d.addCallback(put_raw_key_called) + return d + + def testExtractInvalidAttachedKey(self): + KEY = "-----BEGIN PGP PUBLIC KEY BLOCK-----\n..." + + message = MIMEMultipart() + message.add_header("from", ADDRESS_2) + key = MIMEApplication("", "pgp-keys") + key.set_payload(KEY) + message.attach(key) + self.fetcher._keymanager.put_raw_key = Mock( + return_value=defer.fail(KeyAddressMismatch())) + + def put_raw_key_called(_): + self.fetcher._keymanager.put_raw_key.assert_called_once_with( + KEY, address=ADDRESS_2) + + d = self._do_fetch(message.as_string()) + d.addCallback(put_raw_key_called) + d.addErrback(log.err) + return d + + def testExtractAttachedKeyAndNotOpenPGPHeader(self): + KEY = "-----BEGIN PGP PUBLIC KEY BLOCK-----\n..." + KEYURL = "https://leap.se/key.txt" + OpenPGP = "id=12345678; url=\"%s\"; preference=signencrypt" % (KEYURL,) + + message = MIMEMultipart() + message.add_header("from", ADDRESS_2) + message.add_header("OpenPGP", OpenPGP) + key = MIMEApplication("", "pgp-keys") + key.set_payload(KEY) + message.attach(key) + + self.fetcher._keymanager.put_raw_key = Mock( + return_value=defer.succeed(None)) + self.fetcher._keymanager.fetch_key = Mock() + + def put_raw_key_called(_): + self.fetcher._keymanager.put_raw_key.assert_called_once_with( + KEY, address=ADDRESS_2) + self.assertFalse(self.fetcher._keymanager.fetch_key.called) + + d = self._do_fetch(message.as_string()) + d.addCallback(put_raw_key_called) + return d + + def testExtractOpenPGPHeaderIfInvalidAttachedKey(self): + KEY = "-----BEGIN PGP PUBLIC KEY BLOCK-----\n..." + KEYURL = "https://leap.se/key.txt" + OpenPGP = "id=12345678; url=\"%s\"; preference=signencrypt" % (KEYURL,) + + message = MIMEMultipart() + message.add_header("from", ADDRESS_2) + message.add_header("OpenPGP", OpenPGP) + key = MIMEApplication("", "pgp-keys") + key.set_payload(KEY) + message.attach(key) + + self.fetcher._keymanager.put_raw_key = Mock( + return_value=defer.fail(KeyAddressMismatch())) + self.fetcher._keymanager.fetch_key = Mock() + + def put_raw_key_called(_): + self.fetcher._keymanager.put_raw_key.assert_called_once_with( + KEY, address=ADDRESS_2) + self.fetcher._keymanager.fetch_key.assert_called_once_with( + ADDRESS_2, KEYURL) + + d = self._do_fetch(message.as_string()) + d.addCallback(put_raw_key_called) + return d + + def testAddDecryptedHeader(self): + class DummyMsg(): + + def __init__(self): + self.headers = {} + + def add_header(self, k, v): + self.headers[k] = v + + msg = DummyMsg() + self.fetcher._add_decrypted_header(msg) + + self.assertEquals(msg.headers['X-Leap-Encryption'], 'decrypted') + + def testDecryptEmail(self): + + self.fetcher._decryption_error = Mock() + self.fetcher._add_decrypted_header = Mock() + + def create_encrypted_message(encstr): + message = Parser().parsestr(self.EMAIL) + newmsg = MultipartEncrypted('application/pgp-encrypted') + for hkey, hval in message.items(): + newmsg.add_header(hkey, hval) + + encmsg = MIMEApplication( + encstr, _subtype='octet-stream', _encoder=lambda x: x) + encmsg.add_header('content-disposition', 'attachment', + filename='msg.asc') + # create meta message + metamsg = PGPEncrypted() + metamsg.add_header('Content-Disposition', 'attachment') + # attach pgp message parts to new message + newmsg.attach(metamsg) + newmsg.attach(encmsg) + return newmsg + + def decryption_error_not_called(_): + self.assertFalse(self.fetcher._decryption_error.called, + "There was some errors with decryption") + + def add_decrypted_header_called(_): + self.assertTrue(self.fetcher._add_decrypted_header.called, + "There was some errors with decryption") + + d = self.km.encrypt(self.EMAIL, ADDRESS, sign=ADDRESS_2) + d.addCallback(create_encrypted_message) + d.addCallback( + lambda message: + self._do_fetch(message.as_string())) + d.addCallback(decryption_error_not_called) + d.addCallback(add_decrypted_header_called) + return d + + def testValidateSignatureFromEncryptedEmailFromAppleMail(self): + enc_signed_file = os.path.join( + HERE, 'rfc822.multi-encrypt-signed.message') + self.fetcher._add_verified_signature_header = Mock() + + def add_verified_signature_header_called(_): + self.assertTrue(self.fetcher._add_verified_signature_header.called, + "There was some errors verifying signature") + + with open(enc_signed_file) as f: + enc_signed_raw = f.read() + + d = self._do_fetch(enc_signed_raw) + d.addCallback(add_verified_signature_header_called) + return d + + def testListener(self): + self.called = False + + def listener(uid): + self.called = True + + def listener_called(_): + self.assertTrue(self.called) + + self.fetcher.add_listener(listener) + d = self._do_fetch(self.EMAIL) + d.addCallback(listener_called) + return d + + def _do_fetch(self, message): + d = self._create_incoming_email(message) + d.addCallback( + lambda email: + self._mock_soledad_get_from_index(fields.JUST_MAIL_IDX, [email])) + d.addCallback(lambda _: self.fetcher.fetch()) + return d + + def _create_incoming_email(self, email_str): + email = SoledadDocument() + data = json.dumps( + {"incoming": True, "content": email_str}, + ensure_ascii=False) + + def set_email_content(encr_data): + email.content = { + fields.INCOMING_KEY: True, + fields.ERROR_DECRYPTING_KEY: False, + ENC_SCHEME_KEY: EncryptionSchemes.PUBKEY, + ENC_JSON_KEY: encr_data + } + return email + d = self.km.encrypt(data, ADDRESS, fetch_remote=False) + d.addCallback(set_email_content) + return d + + def _mock_soledad_get_from_index(self, index_name, value): + get_from_index = self._soledad.get_from_index + + def soledad_mock(idx_name, *key_values): + if index_name == idx_name: + return defer.succeed(value) + return get_from_index(idx_name, *key_values) + self.fetcher._soledad.get_from_index = Mock(side_effect=soledad_mock) diff --git a/mail/src/leap/mail/interfaces.py b/mail/src/leap/mail/interfaces.py new file mode 100644 index 00000000..10f51237 --- /dev/null +++ b/mail/src/leap/mail/interfaces.py @@ -0,0 +1,215 @@ +# -*- coding: utf-8 -*- +# interfaces.py +# Copyright (C) 2014,2015 LEAP +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see <http://www.gnu.org/licenses/>. +""" +Interfaces for the leap.mail module. +""" +from zope.interface import Interface, Attribute + + +class IMessageWrapper(Interface): + """ + I know how to access the different parts into which a given message is + splitted into. + + :ivar fdoc: dict with flag document. + :ivar hdoc: dict with flag document. + :ivar cdocs: dict with content-documents, one-indexed. + """ + + fdoc = Attribute('A dictionaly-like containing the flags document ' + '(mutable)') + hdoc = Attribute('A dictionary-like containing the headers document ' + '(immutable)') + cdocs = Attribute('A dictionary with the content-docs, one-indexed') + + def create(self, store, notify_just_mdoc=False, pending_inserts_dict={}): + """ + Create the underlying wrapper. + """ + + def update(self, store): + """ + Update the only mutable parts, which are within the flags document. + """ + + def delete(self, store): + """ + Delete the parts for this wrapper that are not referenced from anywhere + else. + """ + + def copy(self, store, new_mbox_uuid): + """ + Return a copy of this IMessageWrapper in a new mailbox. + """ + + def set_mbox_uuid(self, mbox_uuid): + """ + Set the mailbox for this wrapper. + """ + + def set_flags(self, flags): + """ + """ + + def set_tags(self, tags): + """ + """ + + def set_date(self, date): + """ + """ + + def get_subpart_dict(self, index): + """ + :param index: the part to lookup, 1-indexed + """ + + def get_subpart_indexes(self): + """ + """ + + def get_body(self, store): + """ + """ + + +# TODO -- split into smaller interfaces? separate mailbox interface at least? + +class IMailAdaptor(Interface): + """ + I know how to store the standard representation for messages and mailboxes, + and how to update the relevant mutable parts when needed. + """ + + def initialize_store(self, store): + """ + Performs whatever initialization is needed before the store can be + used (creating indexes, sanity checks, etc). + + :param store: store + :returns: a Deferred that will fire when the store is correctly + initialized. + :rtype: deferred + """ + + def get_msg_from_string(self, MessageClass, raw_msg): + """ + Get an instance of a MessageClass initialized with a MessageWrapper + that contains all the parts obtained from parsing the raw string for + the message. + + :param MessageClass: an implementor of IMessage + :type raw_msg: str + :rtype: implementor of leap.mail.IMessage + """ + + def get_msg_from_docs(self, MessageClass, mdoc, fdoc, hdoc, cdocs=None, + uid=None): + """ + Get an instance of a MessageClass initialized with a MessageWrapper + that contains the passed part documents. + + This is not the recommended way of obtaining a message, unless you know + how to take care of ensuring the internal consistency between the part + documents, or unless you are glueing together the part documents that + have been previously generated by `get_msg_from_string`. + """ + + def get_flags_from_mdoc_id(self, store, mdoc_id): + """ + """ + + def create_msg(self, store, msg): + """ + :param store: an instance of soledad, or anything that behaves alike + :param msg: a Message object. + + :return: a Deferred that is fired when all the underlying documents + have been created. + :rtype: defer.Deferred + """ + + def update_msg(self, store, msg): + """ + :param msg: a Message object. + :param store: an instance of soledad, or anything that behaves alike + :return: a Deferred that is fired when all the underlying documents + have been updated (actually, it's only the fdoc that's allowed + to update). + :rtype: defer.Deferred + """ + + def get_count_unseen(self, store, mbox_uuid): + """ + Get the number of unseen messages for a given mailbox. + + :param store: instance of Soledad. + :param mbox_uuid: the uuid for this mailbox. + :rtype: int + """ + + def get_count_recent(self, store, mbox_uuid): + """ + Get the number of recent messages for a given mailbox. + + :param store: instance of Soledad. + :param mbox_uuid: the uuid for this mailbox. + :rtype: int + """ + + def get_mdoc_id_from_msgid(self, store, mbox_uuid, msgid): + """ + Get the UID for a message with the passed msgid (the one in the headers + msg-id). + This is used by the MUA to retrieve the recently saved draft. + """ + + # mbox handling + + def get_or_create_mbox(self, store, name): + """ + Get the mailbox with the given name, or create one if it does not + exist. + + :param store: instance of Soledad + :param name: the name of the mailbox + :type name: str + """ + + def update_mbox(self, store, mbox_wrapper): + """ + Update the documents for a given mailbox. + :param mbox_wrapper: MailboxWrapper instance + :type mbox_wrapper: MailboxWrapper + :return: a Deferred that will be fired when the mailbox documents + have been updated. + :rtype: defer.Deferred + """ + + def delete_mbox(self, store, mbox_wrapper): + """ + """ + + def get_all_mboxes(self, store): + """ + Retrieve a list with wrappers for all the mailboxes. + + :return: a deferred that will be fired with a list of all the + MailboxWrappers found. + :rtype: defer.Deferred + """ diff --git a/mail/src/leap/mail/load_tests.py b/mail/src/leap/mail/load_tests.py new file mode 100644 index 00000000..be65b8d9 --- /dev/null +++ b/mail/src/leap/mail/load_tests.py @@ -0,0 +1,29 @@ +# -*- coding: utf-8 -*- +# tests.py +# Copyright (C) 2013 LEAP +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see <http://www.gnu.org/licenses/>. +""" +Provide a function for loading tests. +""" +import unittest + + +def load_tests(): + suite = unittest.TestSuite() + for test in unittest.defaultTestLoader.discover( + './src/leap/mail/', + top_level_dir='./src/'): + suite.addTest(test) + return suite diff --git a/mail/src/leap/mail/mail.py b/mail/src/leap/mail/mail.py new file mode 100644 index 00000000..2fde3a1b --- /dev/null +++ b/mail/src/leap/mail/mail.py @@ -0,0 +1,1070 @@ +# -*- coding: utf-8 -*- +# mail.py +# Copyright (C) 2014,2015 LEAP +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see <http://www.gnu.org/licenses/>. +""" +Generic Access to Mail objects. + +This module holds the public LEAP Mail API, which should be viewed as the main +entry point for message and account manipulation, in a protocol-agnostic way. + +In the future, pluggable transports will expose this generic API. +""" +import itertools +import uuid +import logging +import StringIO +import time +import weakref + +from collections import defaultdict + +from twisted.internet import defer +from twisted.python import log + +from leap.common.check import leap_assert_type +from leap.common.events import emit_async, catalog + +from leap.mail.adaptors.soledad import SoledadMailAdaptor +from leap.mail.constants import INBOX_NAME +from leap.mail.constants import MessageFlags +from leap.mail.mailbox_indexer import MailboxIndexer +from leap.mail.plugins import soledad_sync_hooks +from leap.mail.utils import find_charset, CaseInsensitiveDict +from leap.mail.utils import lowerdict + +logger = logging.getLogger(name=__name__) + + +# TODO LIST +# [ ] Probably change the name of this module to "api" or "account", mail is +# too generic (there's also IncomingMail, and OutgoingMail +# [ ] Profile add_msg. + +def _get_mdoc_id(mbox, chash): + """ + Get the doc_id for the metamsg document. + """ + return "M+{mbox}+{chash}".format(mbox=mbox, chash=chash) + + +def _write_and_rewind(payload): + fd = StringIO.StringIO() + fd.write(payload) + fd.seek(0) + return fd + + +def _encode_payload(payload, ctype=""): + """ + Properly encode an unicode payload (which can be string or unicode) as a + string. + + :param payload: the payload to encode. currently soledad returns unicode + strings. + :type payload: basestring + :param ctype: optional, the content of the content-type header for this + payload. + :type ctype: str + :rtype: str + """ + # TODO Related, it's proposed that we're able to pass + # the encoding to the soledad documents. Better to store the charset there? + # FIXME ----------------------------------------------- + # this need a dedicated test-suite + charset = find_charset(ctype) + + # XXX get from mail headers if not multipart! + # Beware also that we should pass the proper encoding to + # soledad when it's creating the documents. + # if not charset: + # charset = get_email_charset(payload) + # ----------------------------------------------------- + + if not charset: + charset = "utf-8" + + try: + if isinstance(payload, unicode): + payload = payload.encode(charset) + except UnicodeError as exc: + logger.error( + "Unicode error, using 'replace'. {0!r}".format(exc)) + payload = payload.encode(charset, 'replace') + return payload + + +def _unpack_headers(headers_dict): + """ + Take a "packed" dict containing headers (with repeated keys represented as + line breaks inside each value, preceded by the header key) and return a + list of tuples in which each repeated key has a different tuple. + """ + headers_l = headers_dict.items() + for i, (k, v) in enumerate(headers_l): + splitted = v.split(k.lower() + ": ") + if len(splitted) != 1: + inner = zip( + itertools.cycle([k]), + map(lambda l: l.rstrip('\n'), splitted)) + headers_l = headers_l[:i] + inner + headers_l[i + 1:] + return headers_l + + +class MessagePart(object): + # TODO This class should be better abstracted from the data model. + # TODO support arbitrarily nested multiparts (right now we only support + # the trivial case) + """ + Represents a part of a multipart MIME Message. + """ + + def __init__(self, part_map, cdocs=None, nested=False): + """ + :param part_map: a dictionary mapping the subparts for + this MessagePart (1-indexed). + :type part_map: dict + + The format for the part_map is as follows: + + {u'ctype': u'text/plain', + u'headers': [[u'Content-Type', u'text/plain; charset="utf-8"'], + [u'Content-Transfer-Encoding', u'8bit']], + u'multi': False, + u'parts': 1, + u'phash': u'02D82B29F6BB0C8612D1C', + u'size': 132} + + :param cdocs: optional, a reference to the top-level dict of wrappers + for content-docs (1-indexed). + """ + if cdocs is None: + cdocs = {} + self._pmap = part_map + self._cdocs = cdocs + self._nested = nested + + def get_size(self): + """ + Size of the body, in octets. + """ + total = self._pmap['size'] + _h = self.get_headers() + headers = len( + '\n'.join(["%s: %s" % (k, v) for k, v in dict(_h).items()])) + # have to subtract 2 blank lines + return total - headers - 2 + + def get_body_file(self): + payload = "" + pmap = self._pmap + + multi = pmap.get('multi') + if not multi: + payload = self._get_payload(pmap.get('phash')) + if payload: + payload = _encode_payload(payload) + + return _write_and_rewind(payload) + + def get_headers(self): + return CaseInsensitiveDict(self._pmap.get("headers", [])) + + def is_multipart(self): + return self._pmap.get("multi", False) + + def get_subpart(self, part): + if not self.is_multipart(): + raise TypeError + sub_pmap = self._pmap.get("part_map", {}) + + try: + part_map = sub_pmap[str(part)] + except KeyError: + log.msg("getSubpart for %s: KeyError" % (part,)) + raise IndexError + return MessagePart(part_map, cdocs=self._cdocs, nested=True) + + def _get_payload(self, phash): + for cdocw in self._cdocs.values(): + if cdocw.phash == phash: + return cdocw.raw + return "" + + +class Message(object): + """ + Represents a single message, and gives access to all its attributes. + """ + + def __init__(self, wrapper, uid=None): + """ + :param wrapper: an instance of an implementor of IMessageWrapper + :param uid: + :type uid: int + """ + self._wrapper = wrapper + self._uid = uid + + def get_wrapper(self): + """ + Get the wrapper for this message. + """ + return self._wrapper + + def get_uid(self): + """ + Get the (optional) UID. + """ + return self._uid + + # imap.IMessage methods + + def get_flags(self): + """ + Get flags for this message. + :rtype: tuple + """ + return self._wrapper.fdoc.get_flags() + + def get_internal_date(self): + """ + Retrieve the date internally associated with this message + + According to the spec, this is NOT the date and time in the + RFC-822 header, but rather a date and time that reflects when the + message was received. + + * In SMTP, date and time of final delivery. + * In COPY, internal date/time of the source message. + * In APPEND, date/time specified. + + :return: An RFC822-formatted date string. + :rtype: str + """ + return self._wrapper.hdoc.date + + # imap.IMessageParts + + def get_headers(self): + """ + Get the raw headers document. + """ + return CaseInsensitiveDict(self._wrapper.hdoc.headers) + + def get_body_file(self, store): + """ + Get a file descriptor with the body content. + """ + def write_and_rewind_if_found(cdoc): + payload = cdoc.raw if cdoc else "" + # XXX pass ctype from headers if not multipart? + if payload: + payload = _encode_payload(payload, ctype=cdoc.content_type) + return _write_and_rewind(payload) + + d = defer.maybeDeferred(self._wrapper.get_body, store) + d.addCallback(write_and_rewind_if_found) + return d + + def get_size(self): + """ + Size of the whole message, in octets (including headers). + """ + total = self._wrapper.fdoc.size + return total + + def is_multipart(self): + """ + Return True if this message is multipart. + """ + return self._wrapper.fdoc.multi + + def get_subpart(self, part): + """ + :param part: The number of the part to retrieve, indexed from 1. + :type part: int + :rtype: MessagePart + """ + if not self.is_multipart(): + raise TypeError + try: + subpart_dict = self._wrapper.get_subpart_dict(part) + except KeyError: + raise IndexError + + return MessagePart( + subpart_dict, cdocs=self._wrapper.cdocs) + + # Custom methods. + + def get_tags(self): + """ + Get the tags for this message. + """ + return tuple(self._wrapper.fdoc.tags) + + +class Flagsmode(object): + """ + Modes for setting the flags/tags. + """ + APPEND = 1 + REMOVE = -1 + SET = 0 + + +class MessageCollection(object): + """ + A generic collection of messages. It can be messages sharing the same + mailbox, tag, the result of a given query, or just a bunch of ids for + master documents. + + Since LEAP Mail is primarily oriented to store mail in Soledad, the default + (and, so far, only) implementation of the store is contained in the + Soledad Mail Adaptor, which is passed to every collection on creation by + the root Account object. If you need to use a different adaptor, change the + adaptor class attribute in your Account object. + + Store is a reference to a particular instance of the message store (soledad + instance or proxy, for instance). + """ + + # TODO LIST + # [ ] look at IMessageSet methods + # [ ] make constructor with a per-instance deferredLock to use on + # creation/deletion? + # [ ] instead of a mailbox, we could pass an arbitrary container with + # pointers to different doc_ids (type: foo) + # [ ] To guarantee synchronicity of the documents sent together during a + # sync, we could get hold of a deferredLock that inhibits + # synchronization while we are updating (think more about this!) + # [ ] review the serveral count_ methods. I think it's better to patch + # server to accept deferreds. + # [ ] Use inheritance for the mailbox-collection instead of handling the + # special cases everywhere? + # [ ] or maybe a mailbox_only decorator... + + # Account should provide an adaptor instance when creating this collection. + adaptor = None + store = None + messageklass = Message + + _pending_inserts = dict() + + def __init__(self, adaptor, store, mbox_indexer=None, mbox_wrapper=None): + """ + Constructor for a MessageCollection. + """ + self.adaptor = adaptor + self.store = store + + # XXX think about what to do when there is no mbox passed to + # the initialization. We could still get the MetaMsg by index, instead + # of by doc_id. See get_message_by_content_hash + self.mbox_indexer = mbox_indexer + self.mbox_wrapper = mbox_wrapper + self._listeners = set([]) + + def is_mailbox_collection(self): + """ + Return True if this collection represents a Mailbox. + :rtype: bool + """ + return bool(self.mbox_wrapper) + + @property + def mbox_name(self): + # TODO raise instead? + if self.mbox_wrapper is None: + return None + return self.mbox_wrapper.mbox + + @property + def mbox_uuid(self): + # TODO raise instead? + if self.mbox_wrapper is None: + return None + return self.mbox_wrapper.uuid + + def get_mbox_attr(self, attr): + if self.mbox_wrapper is None: + raise RuntimeError("This is not a mailbox collection") + return getattr(self.mbox_wrapper, attr) + + def set_mbox_attr(self, attr, value): + if self.mbox_wrapper is None: + raise RuntimeError("This is not a mailbox collection") + setattr(self.mbox_wrapper, attr, value) + return self.mbox_wrapper.update(self.store) + + # Get messages + + def get_message_by_content_hash(self, chash, get_cdocs=False): + """ + Retrieve a message by its content hash. + :rtype: Deferred + """ + if not self.is_mailbox_collection(): + # TODO instead of getting the metamsg by chash, in this case we + # should query by (meta) index or use the internal collection of + # pointers-to-docs. + raise NotImplementedError() + + metamsg_id = _get_mdoc_id(self.mbox_name, chash) + + return self.adaptor.get_msg_from_mdoc_id( + self.messageklass, self.store, + metamsg_id, get_cdocs=get_cdocs) + + def get_message_by_sequence_number(self, msn, get_cdocs=False): + """ + Retrieve a message by its Message Sequence Number. + :rtype: Deferred + """ + def get_uid_for_msn(all_uid): + return all_uid[msn - 1] + d = self.all_uid_iter() + d.addCallback(get_uid_for_msn) + d.addCallback( + lambda uid: self.get_message_by_uid( + uid, get_cdocs=get_cdocs)) + d.addErrback(lambda f: log.err(f)) + return d + + def get_message_by_uid(self, uid, absolute=True, get_cdocs=False): + """ + Retrieve a message by its Unique Identifier. + + If this is a Mailbox collection, that is the message UID, unique for a + given mailbox, or a relative sequence number depending on the absolute + flag. For now, only absolute identifiers are supported. + :rtype: Deferred + """ + # TODO deprecate absolute flag, it doesn't make sense UID and + # !absolute. use _by_sequence_number instead. + if not absolute: + raise NotImplementedError("Does not support relative ids yet") + + get_doc_fun = self.mbox_indexer.get_doc_id_from_uid + + def get_msg_from_mdoc_id(doc_id): + if doc_id is None: + return None + return self.adaptor.get_msg_from_mdoc_id( + self.messageklass, self.store, + doc_id, uid=uid, get_cdocs=get_cdocs) + + def cleanup_and_get_doc_after_pending_insert(result): + for key in result: + self._pending_inserts.pop(key, None) + return get_doc_fun(self.mbox_uuid, uid) + + if not self._pending_inserts: + d = get_doc_fun(self.mbox_uuid, uid) + else: + d = defer.gatherResults(self._pending_inserts.values()) + d.addCallback(cleanup_and_get_doc_after_pending_insert) + d.addCallback(get_msg_from_mdoc_id) + return d + + def get_flags_by_uid(self, uid, absolute=True): + # TODO use sequence numbers + if not absolute: + raise NotImplementedError("Does not support relative ids yet") + + def get_flags_from_mdoc_id(doc_id): + if doc_id is None: # XXX needed? or bug? + return None + return self.adaptor.get_flags_from_mdoc_id( + self.store, doc_id) + + def wrap_in_tuple(flags): + return (uid, flags) + + d = self.mbox_indexer.get_doc_id_from_uid(self.mbox_uuid, uid) + d.addCallback(get_flags_from_mdoc_id) + d.addCallback(wrap_in_tuple) + return d + + def count(self): + """ + Count the messages in this collection. + :return: a Deferred that will fire with the integer for the count. + :rtype: Deferred + """ + if not self.is_mailbox_collection(): + raise NotImplementedError() + + d = self.mbox_indexer.count(self.mbox_uuid) + return d + + def count_recent(self): + """ + Count the recent messages in this collection. + :return: a Deferred that will fire with the integer for the count. + :rtype: Deferred + """ + if not self.is_mailbox_collection(): + raise NotImplementedError() + return self.adaptor.get_count_recent(self.store, self.mbox_uuid) + + def count_unseen(self): + """ + Count the unseen messages in this collection. + :return: a Deferred that will fire with the integer for the count. + :rtype: Deferred + """ + if not self.is_mailbox_collection(): + raise NotImplementedError() + return self.adaptor.get_count_unseen(self.store, self.mbox_uuid) + + def get_uid_next(self): + """ + Get the next integer beyond the highest UID count for this mailbox. + + :return: a Deferred that will fire with the integer for the next uid. + :rtype: Deferred + """ + return self.mbox_indexer.get_next_uid(self.mbox_uuid) + + def get_last_uid(self): + """ + Get the last UID for this mailbox. + """ + return self.mbox_indexer.get_last_uid(self.mbox_uuid) + + def all_uid_iter(self): + """ + Iterator through all the uids for this collection. + """ + return self.mbox_indexer.all_uid_iter(self.mbox_uuid) + + def get_uid_from_msgid(self, msgid): + """ + Return the UID(s) of the matching msg-ids for this mailbox collection. + """ + if not self.is_mailbox_collection(): + raise NotImplementedError() + + def get_uid(mdoc_id): + if not mdoc_id: + return None + d = self.mbox_indexer.get_uid_from_doc_id( + self.mbox_uuid, mdoc_id) + return d + + d = self.adaptor.get_mdoc_id_from_msgid( + self.store, self.mbox_uuid, msgid) + d.addCallback(get_uid) + return d + + # Manipulate messages + + def add_msg(self, raw_msg, flags=tuple(), tags=tuple(), date="", + notify_just_mdoc=False): + """ + Add a message to this collection. + + :param raw_msg: the raw message + :param flags: tuple of flags for this message + :param tags: tuple of tags for this message + :param date: + formatted date, it will be used to retrieve the internal + date for this message. According to the spec, this is NOT the date + and time in the RFC-822 header, but rather a date and time that + reflects when the message was received. + :type date: str + :param notify_just_mdoc: + boolean passed to the wrapper.create method, to indicate whether + we're insterested in being notified right after the mdoc has been + written (as it's the first doc to be written, and quite small, this + is faster, though potentially unsafe), or on the contrary we want + to wait untill all the parts have been written. + Used by the imap mailbox implementation to get faster responses. + This will be ignored (and set to False) if a heuristic for a Draft + message is met, which currently is a specific mozilla header. + :type notify_just_mdoc: bool + + :returns: a deferred that will fire with the UID of the inserted + message. + :rtype: deferred + """ + # TODO watch out if the use of this method in IMAP COPY/APPEND is + # passing the right date. + # XXX mdoc ref is a leaky abstraction here. generalize. + leap_assert_type(flags, tuple) + leap_assert_type(date, str) + + msg = self.adaptor.get_msg_from_string(Message, raw_msg) + wrapper = msg.get_wrapper() + + headers = lowerdict(msg.get_headers()) + moz_draft_hdr = "X-Mozilla-Draft-Info" + if moz_draft_hdr.lower() in headers: + log.msg("Setting fast notify to False, Draft detected") + notify_just_mdoc = False + + if notify_just_mdoc: + msgid = headers.get('message-id') + if msgid: + self._pending_inserts[msgid] = defer.Deferred() + + if not self.is_mailbox_collection(): + raise NotImplementedError() + + else: + mbox_id = self.mbox_uuid + wrapper.set_mbox_uuid(mbox_id) + wrapper.set_flags(flags) + wrapper.set_tags(tags) + wrapper.set_date(date) + + def insert_mdoc_id(_, wrapper): + doc_id = wrapper.mdoc.doc_id + if not doc_id: + # --- BUG ----------------------------------------- + # XXX watch out, sometimes mdoc doesn't have doc_id + # but it has future_id. Should be solved already. + logger.error("BUG: (please report) Null doc_id for " + "document %s" % + (wrapper.mdoc.serialize(),)) + return defer.succeed("mdoc_id not inserted") + # XXX BUG ----------------------------------------- + + # XXX BUG sometimes the table is not yet created, + # so workaround is to make sure we always check for it before + # inserting the doc. I should debug into the real cause. + d = self.mbox_indexer.create_table(self.mbox_uuid) + d.addBoth(lambda _: self.mbox_indexer.insert_doc( + self.mbox_uuid, doc_id)) + return d + + d = wrapper.create( + self.store, + notify_just_mdoc=notify_just_mdoc, + pending_inserts_dict=self._pending_inserts) + d.addCallback(insert_mdoc_id, wrapper) + d.addCallback(self.cb_signal_unread_to_ui) + d.addCallback(self.notify_new_to_listeners) + d.addErrback(lambda failure: log.err(failure)) + + return d + + # Listeners + + def addListener(self, listener): + self._listeners.add(listener) + + def removeListener(self, listener): + self._listeners.remove(listener) + + def notify_new_to_listeners(self, result): + for listener in self._listeners: + listener.notify_new() + return result + + def cb_signal_unread_to_ui(self, result): + """ + Sends an unread event to ui, passing *only* the number of unread + messages if *this* is the inbox. This event is catched, for instance, + in the Bitmask client that displays a message with the number of unread + mails in the INBOX. + + Used as a callback in several commands. + + :param result: ignored + """ + # TODO it might make sense to modify the event so that + # it receives both the mailbox name AND the number of unread messages. + if self.mbox_name.lower() == "inbox": + d = defer.maybeDeferred(self.count_unseen) + d.addCallback(self.__cb_signal_unread_to_ui) + return result + + def __cb_signal_unread_to_ui(self, unseen): + """ + Send the unread signal to UI. + :param unseen: number of unseen messages. + :type unseen: int + """ + emit_async(catalog.MAIL_UNREAD_MESSAGES, self.store.uuid, str(unseen)) + + def copy_msg(self, msg, new_mbox_uuid): + """ + Copy the message to another collection. (it only makes sense for + mailbox collections) + """ + # TODO should CHECK first if the mdoc is present in the mailbox + # WITH a Deleted flag... and just simply remove the flag... + # Another option is to delete the previous mdoc if it already exists + # (so we get a new UID) + + if not self.is_mailbox_collection(): + raise NotImplementedError() + + def delete_mdoc_entry_and_insert(failure, mbox_uuid, doc_id): + d = self.mbox_indexer.delete_doc_by_hash(mbox_uuid, doc_id) + d.addCallback(lambda _: self.mbox_indexer.insert_doc( + new_mbox_uuid, doc_id)) + return d + + def insert_copied_mdoc_id(wrapper_new_msg): + # XXX FIXME -- since this is already saved, the future_doc_id + # should be already copied into the doc_id! + # Investigate why we are not receiving the already saved doc_id + doc_id = wrapper_new_msg.mdoc.doc_id + if not doc_id: + doc_id = wrapper_new_msg.mdoc._future_doc_id + + def insert_conditionally(uid, mbox_uuid, doc_id): + indexer = self.mbox_indexer + if uid: + d = indexer.delete_doc_by_hash(mbox_uuid, doc_id) + d.addCallback(lambda _: indexer.insert_doc( + new_mbox_uuid, doc_id)) + return d + else: + d = indexer.insert_doc(mbox_uuid, doc_id) + return d + + def log_result(result): + return result + + def insert_doc(_, mbox_uuid, doc_id): + d = self.mbox_indexer.get_uid_from_doc_id(mbox_uuid, doc_id) + d.addCallback(insert_conditionally, mbox_uuid, doc_id) + d.addErrback(lambda err: log.failure(err)) + d.addCallback(log_result) + return d + + d = self.mbox_indexer.create_table(new_mbox_uuid) + d.addBoth(insert_doc, new_mbox_uuid, doc_id) + return d + + wrapper = msg.get_wrapper() + + d = wrapper.copy(self.store, new_mbox_uuid) + d.addCallback(insert_copied_mdoc_id) + d.addCallback(self.notify_new_to_listeners) + return d + + def delete_msg(self, msg): + """ + Delete this message. + """ + wrapper = msg.get_wrapper() + + def delete_mdoc_id(_, wrapper): + doc_id = wrapper.mdoc.doc_id + return self.mbox_indexer.delete_doc_by_hash( + self.mbox_uuid, doc_id) + d = wrapper.delete(self.store) + d.addCallback(delete_mdoc_id, wrapper) + return d + + def delete_all_flagged(self): + """ + Delete all messages flagged as \\Deleted. + Used from IMAPMailbox.expunge() + """ + def get_uid_list(hashes): + d = [] + for h in hashes: + d.append(self.mbox_indexer.get_uid_from_doc_id( + self.mbox_uuid, h)) + return defer.gatherResults(d), hashes + + def delete_uid_entries((uids, hashes)): + d = [] + for h in hashes: + d.append(self.mbox_indexer.delete_doc_by_hash( + self.mbox_uuid, h)) + + def return_uids_when_deleted(ignored): + return uids + + all_deleted = defer.gatherResults(d).addCallback( + return_uids_when_deleted) + return all_deleted + + mdocs_deleted = self.adaptor.del_all_flagged_messages( + self.store, self.mbox_uuid) + mdocs_deleted.addCallback(get_uid_list) + mdocs_deleted.addCallback(delete_uid_entries) + mdocs_deleted.addErrback(lambda f: log.err(f)) + return mdocs_deleted + + # TODO should add a delete-by-uid to collection? + + def delete_all_docs(self): + def del_all_uid(uid_list): + deferreds = [] + for uid in uid_list: + d = self.get_message_by_uid(uid) + d.addCallback(lambda msg: msg.delete()) + deferreds.append(d) + return defer.gatherResults(deferreds) + + d = self.all_uid_iter() + d.addCallback(del_all_uid) + return d + + def update_flags(self, msg, flags, mode): + """ + Update flags for a given message. + """ + wrapper = msg.get_wrapper() + current = wrapper.fdoc.flags + newflags = map(str, self._update_flags_or_tags(current, flags, mode)) + wrapper.fdoc.flags = newflags + + wrapper.fdoc.seen = MessageFlags.SEEN_FLAG in newflags + wrapper.fdoc.deleted = MessageFlags.DELETED_FLAG in newflags + + d = self.adaptor.update_msg(self.store, msg) + d.addCallback(lambda _: newflags) + return d + + def update_tags(self, msg, tags, mode): + """ + Update tags for a given message. + """ + wrapper = msg.get_wrapper() + current = wrapper.fdoc.tags + newtags = self._update_flags_or_tags(current, tags, mode) + + wrapper.fdoc.tags = newtags + d = self.adaptor.update_msg(self.store, msg) + d.addCallback(newtags) + return d + + def _update_flags_or_tags(self, old, new, mode): + if mode == Flagsmode.APPEND: + final = list((set(tuple(old) + new))) + elif mode == Flagsmode.REMOVE: + final = list(set(old).difference(set(new))) + elif mode == Flagsmode.SET: + final = new + return final + + +class Account(object): + """ + Account is the top level abstraction to access collections of messages + associated with a LEAP Mail Account. + + It primarily handles creation and access of Mailboxes, which will be the + basic collection handled by traditional MUAs, but it can also handle other + types of Collections (tag based, for instance). + + leap.mail.imap.IMAPAccount partially proxies methods in this + class. + """ + + # Adaptor is passed to the returned MessageCollections, so if you want to + # use a different adaptor this is the place to change it, by subclassing + # the Account class. + + adaptor_class = SoledadMailAdaptor + + # this is a defaultdict, indexed by userid, that returns a + # WeakValueDictionary mapping to collection instances so that we always + # return a reference to them instead of creating new ones. however, + # being a dictionary of weakrefs values, they automagically vanish + # from the dict when no hard refs is left to them (so they can be + # garbage collected) this is important because the different wrappers + # rely on several kinds of deferredlocks that are kept as class or + # instance variables. + + # We need it to be a class property because we create more than one Account + # object in the current usage pattern (ie, one in the mail service, and + # another one in the IncomingMailService). When we move to a proper service + # tree we can let it be an instance attribute. + _collection_mapping = defaultdict(weakref.WeakValueDictionary) + + def __init__(self, store, user_id, ready_cb=None): + self.store = store + self.user_id = user_id + self.adaptor = self.adaptor_class() + + self.mbox_indexer = MailboxIndexer(self.store) + + # This flag is only used from the imap service for the moment. + # In the future, we should prevent any public method to continue if + # this is set to True. Also, it would be good to plug to the + # authentication layer. + self.session_ended = False + + self.deferred_initialization = defer.Deferred() + self._ready_cb = ready_cb + + self._init_d = self._initialize_storage() + self._initialize_sync_hooks() + + def _initialize_storage(self): + + def add_mailbox_if_none(mboxes): + if not mboxes: + return self.add_mailbox(INBOX_NAME) + + def finish_initialization(result): + self.deferred_initialization.callback(None) + if self._ready_cb is not None: + self._ready_cb() + + d = self.adaptor.initialize_store(self.store) + d.addCallback(lambda _: self.list_all_mailbox_names()) + d.addCallback(add_mailbox_if_none) + d.addCallback(finish_initialization) + return d + + def callWhenReady(self, cb, *args, **kw): + """ + Execute the callback when the initialization of the Account is ready. + Note that the callback will receive a first meaningless parameter. + """ + # TODO this should ignore the first parameter explicitely + # lambda _: cb(*args, **kw) + self.deferred_initialization.addCallback(cb, *args, **kw) + return self.deferred_initialization + + # Sync hooks + + def _initialize_sync_hooks(self): + soledad_sync_hooks.post_sync_uid_reindexer.set_account(self) + + def _teardown_sync_hooks(self): + soledad_sync_hooks.post_sync_uid_reindexer.set_account(None) + + # + # Public API Starts + # + + def list_all_mailbox_names(self): + + def filter_names(mboxes): + return [m.mbox for m in mboxes] + + d = self.get_all_mailboxes() + d.addCallback(filter_names) + return d + + def get_all_mailboxes(self): + d = self.adaptor.get_all_mboxes(self.store) + return d + + def add_mailbox(self, name, creation_ts=None): + + if creation_ts is None: + # by default, we pass an int value + # taken from the current time + # we make sure to take enough decimals to get a unique + # mailbox-uidvalidity. + creation_ts = int(time.time() * 10E2) + + def set_creation_ts(wrapper): + wrapper.created = creation_ts + d = wrapper.update(self.store) + d.addCallback(lambda _: wrapper) + return d + + def create_uuid(wrapper): + if not wrapper.uuid: + wrapper.uuid = str(uuid.uuid4()) + d = wrapper.update(self.store) + d.addCallback(lambda _: wrapper) + return d + return wrapper + + def create_uid_table_cb(wrapper): + d = self.mbox_indexer.create_table(wrapper.uuid) + d.addCallback(lambda _: wrapper) + return d + + d = self.adaptor.get_or_create_mbox(self.store, name) + d.addCallback(set_creation_ts) + d.addCallback(create_uuid) + d.addCallback(create_uid_table_cb) + return d + + def delete_mailbox(self, name): + + def delete_uid_table_cb(wrapper): + d = self.mbox_indexer.delete_table(wrapper.uuid) + d.addCallback(lambda _: wrapper) + return d + + d = self.adaptor.get_or_create_mbox(self.store, name) + d.addCallback(delete_uid_table_cb) + d.addCallback( + lambda wrapper: self.adaptor.delete_mbox(self.store, wrapper)) + return d + + def rename_mailbox(self, oldname, newname): + + def _rename_mbox(wrapper): + wrapper.mbox = newname + d = wrapper.update(self.store) + d.addCallback(lambda result: wrapper) + return d + + d = self.adaptor.get_or_create_mbox(self.store, oldname) + d.addCallback(_rename_mbox) + return d + + # Get Collections + + def get_collection_by_mailbox(self, name): + """ + :rtype: deferred + :return: a deferred that will fire with a MessageCollection + """ + collection = self._collection_mapping[self.user_id].get( + name, None) + if collection: + return defer.succeed(collection) + + # imap select will use this, passing the collection to SoledadMailbox + def get_collection_for_mailbox(mbox_wrapper): + collection = MessageCollection( + self.adaptor, self.store, self.mbox_indexer, mbox_wrapper) + self._collection_mapping[self.user_id][name] = collection + return collection + + d = self.adaptor.get_or_create_mbox(self.store, name) + d.addCallback(get_collection_for_mailbox) + return d + + def get_collection_by_docs(self, docs): + """ + :rtype: MessageCollection + """ + # get a collection of docs by a list of doc_id + # get.docs(...) --> it should be a generator. does it behave in the + # threadpool? + raise NotImplementedError() + + def get_collection_by_tag(self, tag): + """ + :rtype: MessageCollection + """ + raise NotImplementedError() + + # Session handling + + def end_session(self): + self._teardown_sync_hooks() + self.session_ended = True diff --git a/mail/src/leap/mail/mailbox_indexer.py b/mail/src/leap/mail/mailbox_indexer.py new file mode 100644 index 00000000..c49f808b --- /dev/null +++ b/mail/src/leap/mail/mailbox_indexer.py @@ -0,0 +1,327 @@ +# -*- coding: utf-8 -*- +# mailbox_indexer.py +# Copyright (C) 2014 LEAP +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see <http://www.gnu.org/licenses/>. +""" +.. :py:module::mailbox_indexer + +Local tables to store the message Unique Identifiers for a given mailbox. +""" +import re +import uuid + +from leap.mail.constants import METAMSGID_RE + + +def _maybe_first_query_item(thing): + """ + Return the first item the returned query result, or None + if empty. + """ + try: + return thing[0][0] + except (TypeError, IndexError): + return None + + +class WrongMetaDocIDError(Exception): + pass + + +def sanitize(mailbox_uuid): + return mailbox_uuid.replace("-", "_") + + +def check_good_uuid(mailbox_uuid): + """ + Check that the passed mailbox identifier is a valid UUID. + :param mailbox_uuid: the uuid to check + :type mailbox_uuid: str + :return: None + :raises: AssertionError if a wrong uuid was passed. + """ + try: + uuid.UUID(str(mailbox_uuid)) + except (AttributeError, ValueError): + raise AssertionError( + "the mbox_id is not a valid uuid: %s" % mailbox_uuid) + + +class MailboxIndexer(object): + """ + This class contains the commands needed to create, modify and alter the + local-only UID tables for a given mailbox. + + Its purpouse is to keep a local-only index with the messages in each + mailbox, mainly to satisfy the demands of the IMAP specification, but + useful too for any effective listing of the messages in a mailbox. + + Since the incoming mail can be processed at any time in any replica, it's + preferred not to attempt to maintain a global chronological global index. + + These indexes are Message Attributes needed for the IMAP specification (rfc + 3501), although they can be useful for other non-imap store + implementations. + + """ + # The uids are expected to be 32-bits values, but the ROWIDs in sqlite + # are 64-bit values. I *don't* think it really matters for any + # practical use, but it's good to remember we've got that difference going + # on. + + store = None + table_preffix = "leapmail_uid_" + + def __init__(self, store): + self.store = store + + def _query(self, *args, **kw): + assert self.store is not None + return self.store.raw_sqlcipher_query(*args, **kw) + + def _operation(self, *args, **kw): + assert self.store is not None + return self.store.raw_sqlcipher_operation(*args, **kw) + + def create_table(self, mailbox_uuid): + """ + Create the UID table for a given mailbox. + :param mailbox: the mailbox identifier. + :type mailbox: str + :rtype: Deferred + """ + check_good_uuid(mailbox_uuid) + sql = ("CREATE TABLE if not exists {preffix}{name}( " + "uid INTEGER PRIMARY KEY AUTOINCREMENT, " + "hash TEXT UNIQUE NOT NULL)".format( + preffix=self.table_preffix, name=sanitize(mailbox_uuid))) + return self._operation(sql) + + def delete_table(self, mailbox_uuid): + """ + Delete the UID table for a given mailbox. + :param mailbox: the mailbox name + :type mailbox: str + :rtype: Deferred + """ + check_good_uuid(mailbox_uuid) + sql = ("DROP TABLE if exists {preffix}{name}".format( + preffix=self.table_preffix, name=sanitize(mailbox_uuid))) + return self._operation(sql) + + def insert_doc(self, mailbox_uuid, doc_id): + """ + Insert the doc_id for a MetaMsg in the UID table for a given mailbox. + + The doc_id must be in the format: + + M-<mailbox>-<content-hash-of-the-message> + + :param mailbox: the mailbox name + :type mailbox: str + :param doc_id: the doc_id for the MetaMsg + :type doc_id: str + :return: a deferred that will fire with the uid of the newly inserted + document. + :rtype: Deferred + """ + check_good_uuid(mailbox_uuid) + assert doc_id + mailbox_uuid = mailbox_uuid.replace('-', '_') + + if not re.findall(METAMSGID_RE.format(mbox_uuid=mailbox_uuid), doc_id): + raise WrongMetaDocIDError("Wrong format for the MetaMsg doc_id") + + def get_rowid(result): + return _maybe_first_query_item(result) + + sql = ("INSERT INTO {preffix}{name} VALUES (" + "NULL, ?)".format( + preffix=self.table_preffix, name=sanitize(mailbox_uuid))) + values = (doc_id,) + + sql_last = ("SELECT MAX(rowid) FROM {preffix}{name} " + "LIMIT 1;").format( + preffix=self.table_preffix, name=sanitize(mailbox_uuid)) + + d = self._operation(sql, values) + d.addCallback(lambda _: self._query(sql_last)) + d.addCallback(get_rowid) + d.addErrback(lambda f: f.printTraceback()) + return d + + def delete_doc_by_uid(self, mailbox_uuid, uid): + """ + Delete the entry for a MetaMsg in the UID table for a given mailbox. + + :param mailbox_uuid: the mailbox uuid + :type mailbox: str + :param uid: the UID of the message. + :type uid: int + :rtype: Deferred + """ + check_good_uuid(mailbox_uuid) + assert uid + sql = ("DELETE FROM {preffix}{name} " + "WHERE uid=?".format( + preffix=self.table_preffix, name=sanitize(mailbox_uuid))) + values = (uid,) + return self._query(sql, values) + + def delete_doc_by_hash(self, mailbox_uuid, doc_id): + """ + Delete the entry for a MetaMsg in the UID table for a given mailbox. + + The doc_id must be in the format: + + M-<mailbox_uuid>-<content-hash-of-the-message> + + :param mailbox_uuid: the mailbox uuid + :type mailbox: str + :param doc_id: the doc_id for the MetaMsg + :type doc_id: str + :return: a deferred that will fire when the deletion has succed. + :rtype: Deferred + """ + check_good_uuid(mailbox_uuid) + assert doc_id + sql = ("DELETE FROM {preffix}{name} " + "WHERE hash=?".format( + preffix=self.table_preffix, name=sanitize(mailbox_uuid))) + values = (doc_id,) + return self._query(sql, values) + + def get_doc_id_from_uid(self, mailbox_uuid, uid): + """ + Get the doc_id for a MetaMsg in the UID table for a given mailbox. + + :param mailbox_uuid: the mailbox uuid + :type mailbox: str + :param uid: the uid for the MetaMsg for this mailbox + :type uid: int + :rtype: Deferred + """ + check_good_uuid(mailbox_uuid) + mailbox_uuid = mailbox_uuid.replace('-', '_') + + def get_hash(result): + return _maybe_first_query_item(result) + + sql = ("SELECT hash from {preffix}{name} " + "WHERE uid=?".format( + preffix=self.table_preffix, name=sanitize(mailbox_uuid))) + values = (uid,) + d = self._query(sql, values) + d.addCallback(get_hash) + return d + + def get_uid_from_doc_id(self, mailbox_uuid, doc_id): + check_good_uuid(mailbox_uuid) + mailbox_uuid = mailbox_uuid.replace('-', '_') + + def get_uid(result): + return _maybe_first_query_item(result) + + sql = ("SELECT uid from {preffix}{name} " + "WHERE hash=?".format( + preffix=self.table_preffix, name=sanitize(mailbox_uuid))) + values = (doc_id,) + d = self._query(sql, values) + d.addCallback(get_uid) + return d + + def get_doc_ids_from_uids(self, mailbox_uuid, uids): + # For IMAP relative numbering /sequences. + # XXX dereference the range (n,*) + raise NotImplementedError() + + def count(self, mailbox_uuid): + """ + Get the number of entries in the UID table for a given mailbox. + + :param mailbox_uuid: the mailbox uuid + :type mailbox_uuid: str + :return: a deferred that will fire with an integer returning the count. + :rtype: Deferred + """ + check_good_uuid(mailbox_uuid) + + def get_count(result): + return _maybe_first_query_item(result) + + sql = ("SELECT Count(*) FROM {preffix}{name};".format( + preffix=self.table_preffix, name=sanitize(mailbox_uuid))) + d = self._query(sql) + d.addCallback(get_count) + d.addErrback(lambda _: 0) + return d + + def get_next_uid(self, mailbox_uuid): + """ + Get the next integer beyond the highest UID count for a given mailbox. + + This is expected by the IMAP implementation. There are no guarantees + that a document to be inserted in the future gets the returned UID: the + only thing that can be assured is that it will be equal or greater than + the value returned. + + :param mailbox_uuid: the mailbox uuid + :type mailbox: str + :return: a deferred that will fire with an integer returning the next + uid. + :rtype: Deferred + """ + check_good_uuid(mailbox_uuid) + d = self.get_last_uid(mailbox_uuid) + d.addCallback(lambda uid: uid + 1) + return d + + def get_last_uid(self, mailbox_uuid): + """ + Get the highest UID for a given mailbox. + """ + check_good_uuid(mailbox_uuid) + sql = ("SELECT MAX(rowid) FROM {preffix}{name} " + "LIMIT 1;").format( + preffix=self.table_preffix, name=sanitize(mailbox_uuid)) + + def getit(result): + rowid = _maybe_first_query_item(result) + if not rowid: + rowid = 0 + return rowid + + d = self._query(sql) + d.addCallback(getit) + return d + + def all_uid_iter(self, mailbox_uuid): + """ + Get a sequence of all the uids in this mailbox. + + :param mailbox_uuid: the mailbox uuid + :type mailbox_uuid: str + """ + check_good_uuid(mailbox_uuid) + + sql = ("SELECT uid from {preffix}{name} ").format( + preffix=self.table_preffix, name=sanitize(mailbox_uuid)) + + def get_results(result): + return [x[0] for x in result] + + d = self._query(sql) + d.addCallback(get_results) + return d diff --git a/mail/src/leap/mail/outgoing/__init__.py b/mail/src/leap/mail/outgoing/__init__.py new file mode 100644 index 00000000..e69de29b --- /dev/null +++ b/mail/src/leap/mail/outgoing/__init__.py diff --git a/mail/src/leap/mail/outgoing/service.py b/mail/src/leap/mail/outgoing/service.py new file mode 100644 index 00000000..8b02f2e4 --- /dev/null +++ b/mail/src/leap/mail/outgoing/service.py @@ -0,0 +1,518 @@ +# -*- coding: utf-8 -*- +# outgoing/service.py +# Copyright (C) 2013-2015 LEAP +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see <http://www.gnu.org/licenses/>. + +""" +OutgoingMail module. + +The OutgoingMail class allows to send mail, and encrypts/signs it if needed. +""" + +import os.path +import re +from StringIO import StringIO +from copy import deepcopy +from email.parser import Parser +from email.mime.application import MIMEApplication +from email.mime.multipart import MIMEMultipart +from email.mime.text import MIMEText + +from OpenSSL import SSL + +from twisted.mail import smtp +from twisted.internet import reactor +from twisted.internet import defer +from twisted.protocols.amp import ssl +from twisted.python import log + +from leap.common.check import leap_assert_type, leap_assert +from leap.common.events import emit_async, catalog +from leap.keymanager.errors import KeyNotFound, KeyAddressMismatch +from leap.mail import __version__ +from leap.mail import errors +from leap.mail.utils import validate_address +from leap.mail.rfc3156 import MultipartEncrypted +from leap.mail.rfc3156 import MultipartSigned +from leap.mail.rfc3156 import encode_base64_rec +from leap.mail.rfc3156 import RFC3156CompliantGenerator +from leap.mail.rfc3156 import PGPSignature +from leap.mail.rfc3156 import PGPEncrypted + +# TODO +# [ ] rename this module to something else, service should be the implementor +# of IService + + +class SSLContextFactory(ssl.ClientContextFactory): + def __init__(self, cert, key): + self.cert = cert + self.key = key + + def getContext(self): + # FIXME -- we should use sslv23 to allow for tlsv1.2 + # and, if possible, explicitely disable sslv3 clientside. + # Servers should avoid sslv3 + self.method = SSL.TLSv1_METHOD # SSLv23_METHOD + ctx = ssl.ClientContextFactory.getContext(self) + ctx.use_certificate_file(self.cert) + ctx.use_privatekey_file(self.key) + return ctx + + +def outgoingFactory(userid, keymanager, opts, check_cert=True, bouncer=None): + + cert = unicode(opts.cert) + key = unicode(opts.key) + hostname = str(opts.hostname) + port = opts.port + + if check_cert: + if not os.path.isfile(cert): + raise errors.ConfigurationError( + 'No valid SMTP certificate could be found for %s!' % userid) + + return OutgoingMail( + str(userid), keymanager, cert, key, hostname, port, + bouncer) + + +class OutgoingMail(object): + """ + Sends Outgoing Mail, encrypting and signing if needed. + """ + + def __init__(self, from_address, keymanager, cert, key, host, port, + bouncer=None): + """ + Initialize the outgoing mail service. + + :param from_address: The sender address. + :type from_address: str + :param keymanager: A KeyManager for retrieving recipient's keys. + :type keymanager: leap.common.keymanager.KeyManager + :param cert: The client certificate for SSL authentication. + :type cert: str + :param key: The client private key for SSL authentication. + :type key: str + :param host: The hostname of the remote SMTP server. + :type host: str + :param port: The port of the remote SMTP server. + :type port: int + """ + + # assert params + leap_assert_type(from_address, str) + leap_assert('@' in from_address) + + # XXX it can be a zope.proxy too + # leap_assert_type(keymanager, KeyManager) + + leap_assert_type(host, str) + leap_assert(host != '') + leap_assert_type(port, int) + leap_assert(port is not 0) + leap_assert_type(cert, unicode) + leap_assert(cert != '') + leap_assert_type(key, unicode) + leap_assert(key != '') + + self._port = port + self._host = host + self._key = key + self._cert = cert + self._from_address = from_address + self._keymanager = keymanager + self._bouncer = bouncer + + def send_message(self, raw, recipient): + """ + Sends a message to a recipient. Maybe encrypts and signs. + + :param raw: The raw message + :type raw: str + :param recipient: The recipient for the message + :type recipient: smtp.User + :return: a deferred which delivers the message when fired + """ + d = self._maybe_encrypt_and_sign(raw, recipient) + d.addCallback(self._route_msg, raw) + d.addErrback(self.sendError, raw) + return d + + def sendSuccess(self, smtp_sender_result): + """ + Callback for a successful send. + + :param smtp_sender_result: The result from the ESMTPSender from + _route_msg + :type smtp_sender_result: tuple(int, list(tuple)) + """ + dest_addrstr = smtp_sender_result[1][0][0] + fromaddr = self._from_address + log.msg('Message sent from %s to %s' % (fromaddr, dest_addrstr)) + emit_async(catalog.SMTP_SEND_MESSAGE_SUCCESS, + fromaddr, dest_addrstr) + + def sendError(self, failure, origmsg): + """ + Callback for an unsuccessfull send. + + :param failure: The result from the last errback. + :type failure: anything + :param origmsg: the original, unencrypted, raw message, to be passed to + the bouncer. + :type origmsg: str + """ + # XXX: need to get the address from the original message to send signal + # emit_async(catalog.SMTP_SEND_MESSAGE_ERROR, self._from_address, + # self._user.dest.addrstr) + + # TODO when we implement outgoing queues/long-term-retries, we could + # examine the error *here* and delay the notification if it's just a + # temporal error. We might want to notify the permanent errors + # differently. + + err = failure.value + log.err(err) + + if self._bouncer: + self._bouncer.bounce_message( + err.message, to=self._from_address, + orig=origmsg) + else: + raise err + + def _route_msg(self, encrypt_and_sign_result, raw): + """ + Sends the msg using the ESMTPSenderFactory. + + :param encrypt_and_sign_result: A tuple containing the 'maybe' + encrypted message and the recipient + :type encrypt_and_sign_result: tuple + """ + message, recipient = encrypt_and_sign_result + log.msg("Connecting to SMTP server %s:%s" % (self._host, self._port)) + msg = message.as_string(False) + + # we construct a defer to pass to the ESMTPSenderFactory + d = defer.Deferred() + d.addCallback(self.sendSuccess) + d.addErrback(self.sendError, raw) + # we don't pass an ssl context factory to the ESMTPSenderFactory + # because ssl will be handled by reactor.connectSSL() below. + factory = smtp.ESMTPSenderFactory( + "", # username is blank, no client auth here + "", # password is blank, no client auth here + self._from_address, + recipient.dest.addrstr, + StringIO(msg), + d, + heloFallback=True, + requireAuthentication=False, + requireTransportSecurity=True) + factory.domain = bytes('leap.mail-' + __version__) + emit_async(catalog.SMTP_SEND_MESSAGE_START, + self._from_address, recipient.dest.addrstr) + reactor.connectSSL( + self._host, self._port, factory, + contextFactory=SSLContextFactory(self._cert, self._key)) + + def _maybe_encrypt_and_sign(self, raw, recipient, fetch_remote=True): + """ + Attempt to encrypt and sign the outgoing message. + + The behaviour of this method depends on: + + 1. the original message's content-type, and + 2. the availability of the recipient's public key. + + If the original message's content-type is "multipart/encrypted", then + the original message is not altered. For any other content-type, the + method attempts to fetch the recipient's public key. If the + recipient's public key is available, the message is encrypted and + signed; otherwise it is only signed. + + Note that, if the C{encrypted_only} configuration is set to True and + the recipient's public key is not available, then the recipient + address would have been rejected in SMTPDelivery.validateTo(). + + The following table summarizes the overall behaviour of the gateway: + + +---------------------------------------------------+----------------+ + | content-type | rcpt pubkey | enforce encr. | action | + +---------------------+-------------+---------------+----------------+ + | multipart/encrypted | any | any | pass | + | other | available | any | encrypt + sign | + | other | unavailable | yes | reject | + | other | unavailable | no | sign | + +---------------------+-------------+---------------+----------------+ + + :param raw: The raw message + :type raw: str + :param recipient: The recipient for the message + :type: recipient: smtp.User + + :return: A Deferred that will be fired with a MIMEMultipart message + and the original recipient Message + :rtype: Deferred + """ + # pass if the original message's content-type is "multipart/encrypted" + origmsg = Parser().parsestr(raw) + + if origmsg.get_content_type() == 'multipart/encrypted': + return defer.succeed((origmsg, recipient)) + + from_address = validate_address(self._from_address) + username, domain = from_address.split('@') + to_address = validate_address(recipient.dest.addrstr) + + def maybe_encrypt_and_sign(message): + d = self._encrypt_and_sign( + message, to_address, from_address, + fetch_remote=fetch_remote) + d.addCallbacks(signal_encrypt_sign, + if_key_not_found_send_unencrypted, + errbackArgs=(message,)) + return d + + def signal_encrypt_sign(newmsg): + emit_async(catalog.SMTP_END_ENCRYPT_AND_SIGN, + self._from_address, + "%s,%s" % (self._from_address, to_address)) + return newmsg, recipient + + def if_key_not_found_send_unencrypted(failure, message): + failure.trap(KeyNotFound, KeyAddressMismatch) + + log.msg('Will send unencrypted message to %s.' % to_address) + emit_async(catalog.SMTP_START_SIGN, self._from_address, to_address) + d = self._sign(message, from_address) + d.addCallback(signal_sign) + return d + + def signal_sign(newmsg): + emit_async(catalog.SMTP_END_SIGN, self._from_address) + return newmsg, recipient + + log.msg("Will encrypt the message with %s and sign with %s." + % (to_address, from_address)) + emit_async(catalog.SMTP_START_ENCRYPT_AND_SIGN, + self._from_address, + "%s,%s" % (self._from_address, to_address)) + d = self._maybe_attach_key(origmsg, from_address, to_address) + d.addCallback(maybe_encrypt_and_sign) + return d + + def _maybe_attach_key(self, origmsg, from_address, to_address): + filename = "%s-email-key.asc" % (from_address,) + + def attach_if_address_hasnt_encrypted(to_key): + # if the sign_used flag is true that means that we got an encrypted + # email from this address, because we conly check signatures on + # encrypted emails. In this case we don't attach. + # XXX: this might not be true some time in the future + if to_key.sign_used: + return origmsg + return get_key_and_attach(None) + + def get_key_and_attach(_): + d = self._keymanager.get_key(from_address, fetch_remote=False) + d.addCallback(attach_key) + return d + + def attach_key(from_key): + msg = origmsg + if not origmsg.is_multipart(): + msg = MIMEMultipart() + for h, v in origmsg.items(): + msg.add_header(h, v) + msg.attach(MIMEText(origmsg.get_payload())) + + keymsg = MIMEApplication(from_key.key_data, _subtype='pgp-keys', + _encoder=lambda x: x) + keymsg.add_header('content-disposition', 'attachment', + filename=filename) + msg.attach(keymsg) + return msg + + d = self._keymanager.get_key(to_address, fetch_remote=False) + d.addCallbacks(attach_if_address_hasnt_encrypted, get_key_and_attach) + d.addErrback(lambda _: origmsg) + return d + + def _encrypt_and_sign(self, origmsg, encrypt_address, sign_address, + fetch_remote=True): + """ + Create an RFC 3156 compliang PGP encrypted and signed message using + C{encrypt_address} to encrypt and C{sign_address} to sign. + + :param origmsg: The original message + :type origmsg: email.message.Message + :param encrypt_address: The address used to encrypt the message. + :type encrypt_address: str + :param sign_address: The address used to sign the message. + :type sign_address: str + + :return: A Deferred with the MultipartEncrypted message + :rtype: Deferred + """ + # create new multipart/encrypted message with 'pgp-encrypted' protocol + + def encrypt(res): + newmsg, origmsg = res + d = self._keymanager.encrypt( + origmsg.as_string(unixfrom=False), + encrypt_address, sign=sign_address, + fetch_remote=fetch_remote) + d.addCallback(lambda encstr: (newmsg, encstr)) + return d + + def create_encrypted_message(res): + newmsg, encstr = res + encmsg = MIMEApplication( + encstr, _subtype='octet-stream', _encoder=lambda x: x) + encmsg.add_header('content-disposition', 'attachment', + filename='msg.asc') + # create meta message + metamsg = PGPEncrypted() + metamsg.add_header('Content-Disposition', 'attachment') + # attach pgp message parts to new message + newmsg.attach(metamsg) + newmsg.attach(encmsg) + return newmsg + + d = self._fix_headers( + origmsg, + MultipartEncrypted('application/pgp-encrypted'), + sign_address) + d.addCallback(encrypt) + d.addCallback(create_encrypted_message) + return d + + def _sign(self, origmsg, sign_address): + """ + Create an RFC 3156 compliant PGP signed MIME message using + C{sign_address}. + + :param origmsg: The original message + :type origmsg: email.message.Message + :param sign_address: The address used to sign the message. + :type sign_address: str + + :return: A Deferred with the MultipartSigned message. + :rtype: Deferred + """ + # apply base64 content-transfer-encoding + encode_base64_rec(origmsg) + # get message text with headers and replace \n for \r\n + fp = StringIO() + g = RFC3156CompliantGenerator( + fp, mangle_from_=False, maxheaderlen=76) + g.flatten(origmsg) + msgtext = re.sub('\r?\n', '\r\n', fp.getvalue()) + # make sure signed message ends with \r\n as per OpenPGP stantard. + if origmsg.is_multipart(): + if not msgtext.endswith("\r\n"): + msgtext += "\r\n" + + def create_signed_message(res): + (msg, _), signature = res + sigmsg = PGPSignature(signature) + # attach original message and signature to new message + msg.attach(origmsg) + msg.attach(sigmsg) + return msg + + dh = self._fix_headers( + origmsg, + MultipartSigned('application/pgp-signature', 'pgp-sha512'), + sign_address) + ds = self._keymanager.sign( + msgtext, sign_address, digest_algo='SHA512', + clearsign=False, detach=True, binary=False) + d = defer.gatherResults([dh, ds]) + d.addCallback(create_signed_message) + return d + + def _fix_headers(self, msg, newmsg, sign_address): + """ + Move some headers from C{origmsg} to C{newmsg}, delete unwanted + headers from C{origmsg} and add new headers to C{newms}. + + Outgoing messages are either encrypted and signed or just signed + before being sent. Because of that, they are packed inside new + messages and some manipulation has to be made on their headers. + + Allowed headers for passing through: + + - From + - Date + - To + - Subject + - Reply-To + - References + - In-Reply-To + - Cc + + Headers to be added: + + - Message-ID (i.e. should not use origmsg's Message-Id) + - Received (this is added automatically by twisted smtp API) + - OpenPGP (see #4447) + + Headers to be deleted: + + - User-Agent + + :param msg: The original message. + :type msg: email.message.Message + :param newmsg: The new message being created. + :type newmsg: email.message.Message + :param sign_address: The address used to sign C{newmsg} + :type sign_address: str + + :return: A Deferred with a touple: + (new Message with the unencrypted headers, + original Message with headers removed) + :rtype: Deferred + """ + origmsg = deepcopy(msg) + # move headers from origmsg to newmsg + headers = origmsg.items() + passthrough = [ + 'from', 'date', 'to', 'subject', 'reply-to', 'references', + 'in-reply-to', 'cc' + ] + headers = filter(lambda x: x[0].lower() in passthrough, headers) + for hkey, hval in headers: + newmsg.add_header(hkey, hval) + del (origmsg[hkey]) + # add a new message-id to newmsg + newmsg.add_header('Message-Id', smtp.messageid()) + # delete user-agent from origmsg + del (origmsg['user-agent']) + + def add_openpgp_header(signkey): + username, domain = sign_address.split('@') + newmsg.add_header( + 'OpenPGP', 'id=%s' % signkey.fingerprint, + url='https://%s/key/%s' % (domain, username), + preference='signencrypt') + return newmsg, origmsg + + d = self._keymanager.get_key(sign_address, private=True) + d.addCallback(add_openpgp_header) + return d diff --git a/mail/src/leap/mail/outgoing/tests/test_outgoing.py b/mail/src/leap/mail/outgoing/tests/test_outgoing.py new file mode 100644 index 00000000..dd053c15 --- /dev/null +++ b/mail/src/leap/mail/outgoing/tests/test_outgoing.py @@ -0,0 +1,263 @@ +# -*- coding: utf-8 -*- +# test_gateway.py +# Copyright (C) 2013 LEAP +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see <http://www.gnu.org/licenses/>. + + +""" +SMTP gateway tests. +""" +import re +from copy import deepcopy +from StringIO import StringIO +from email.parser import Parser +from datetime import datetime +from twisted.internet.defer import fail +from twisted.mail.smtp import User +from twisted.python import log + +from mock import Mock + +from leap.mail.rfc3156 import RFC3156CompliantGenerator +from leap.mail.outgoing.service import OutgoingMail +from leap.mail.testing import ADDRESS, ADDRESS_2, PUBLIC_KEY_2 +from leap.mail.testing import KeyManagerWithSoledadTestCase +from leap.mail.testing.smtp import getSMTPFactory +from leap.keymanager import errors + + +BEGIN_PUBLIC_KEY = "-----BEGIN PGP PUBLIC KEY BLOCK-----" + +TEST_USER = u'anotheruser@leap.se' + + +class TestOutgoingMail(KeyManagerWithSoledadTestCase): + EMAIL_DATA = ['HELO gateway.leap.se', + 'MAIL FROM: <%s>' % ADDRESS_2, + 'RCPT TO: <%s>' % ADDRESS, + 'DATA', + 'From: User <%s>' % ADDRESS_2, + 'To: Leap <%s>' % ADDRESS, + 'Date: ' + datetime.now().strftime('%c'), + 'Subject: test message', + '', + 'This is a secret message.', + 'Yours,', + 'A.', + '', + '.', + 'QUIT'] + + def setUp(self): + self.lines = [line for line in self.EMAIL_DATA[4:12]] + self.lines.append('') # add a trailing newline + self.raw = '\r\n'.join(self.lines) + self.expected_body = '\r\n'.join(self.EMAIL_DATA[9:12]) + "\r\n" + self.fromAddr = ADDRESS_2 + + class opts: + cert = u'/tmp/cert' + key = u'/tmp/cert' + hostname = 'remote' + port = 666 + self.opts = opts + + def init_outgoing_and_proto(_): + self.outgoing_mail = OutgoingMail( + self.fromAddr, self.km, opts.cert, + opts.key, opts.hostname, opts.port) + + user = TEST_USER + + # TODO -- this shouldn't need SMTP to be tested!? or does it? + self.proto = getSMTPFactory( + {user: None}, {user: self.km}, {user: None}) + self.dest = User(ADDRESS, 'gateway.leap.se', self.proto, ADDRESS_2) + + d = KeyManagerWithSoledadTestCase.setUp(self) + d.addCallback(init_outgoing_and_proto) + return d + + def test_message_encrypt(self): + """ + Test if message gets encrypted to destination email. + """ + def check_decryption(res): + decrypted, _ = res + self.assertEqual( + '\n' + self.expected_body, + decrypted, + 'Decrypted text differs from plaintext.') + + d = self._set_sign_used(ADDRESS) + d.addCallback( + lambda _: + self.outgoing_mail._maybe_encrypt_and_sign(self.raw, self.dest)) + d.addCallback(self._assert_encrypted) + d.addCallback(lambda message: self.km.decrypt( + message.get_payload(1).get_payload(), ADDRESS)) + d.addCallback(check_decryption) + return d + + def test_message_encrypt_sign(self): + """ + Test if message gets encrypted to destination email and signed with + sender key. + '""" + def check_decryption_and_verify(res): + decrypted, signkey = res + self.assertEqual( + '\n' + self.expected_body, + decrypted, + 'Decrypted text differs from plaintext.') + self.assertTrue(ADDRESS_2 in signkey.address, + "Verification failed") + + d = self._set_sign_used(ADDRESS) + d.addCallback( + lambda _: + self.outgoing_mail._maybe_encrypt_and_sign(self.raw, self.dest)) + d.addCallback(self._assert_encrypted) + d.addCallback(lambda message: self.km.decrypt( + message.get_payload(1).get_payload(), ADDRESS, verify=ADDRESS_2)) + d.addCallback(check_decryption_and_verify) + return d + + def test_message_sign(self): + """ + Test if message is signed with sender key. + """ + # mock the key fetching + self.km._fetch_keys_from_server = Mock( + return_value=fail(errors.KeyNotFound())) + recipient = User('ihavenopubkey@nonleap.se', + 'gateway.leap.se', self.proto, ADDRESS) + self.outgoing_mail = OutgoingMail( + self.fromAddr, self.km, self.opts.cert, self.opts.key, + self.opts.hostname, self.opts.port) + + def check_signed(res): + message, _ = res + self.assertTrue('Content-Type' in message) + self.assertEqual('multipart/signed', message.get_content_type()) + self.assertEqual('application/pgp-signature', + message.get_param('protocol')) + self.assertEqual('pgp-sha512', message.get_param('micalg')) + # assert content of message + body = (message.get_payload(0) + .get_payload(0) + .get_payload(decode=True)) + self.assertEqual(self.expected_body, + body) + # assert content of signature + self.assertTrue( + message.get_payload(1).get_payload().startswith( + '-----BEGIN PGP SIGNATURE-----\n'), + 'Message does not start with signature header.') + self.assertTrue( + message.get_payload(1).get_payload().endswith( + '-----END PGP SIGNATURE-----\n'), + 'Message does not end with signature footer.') + return message + + def verify(message): + # replace EOL before verifying (according to rfc3156) + fp = StringIO() + g = RFC3156CompliantGenerator( + fp, mangle_from_=False, maxheaderlen=76) + g.flatten(message.get_payload(0)) + signed_text = re.sub('\r?\n', '\r\n', + fp.getvalue()) + + def assert_verify(key): + self.assertTrue(ADDRESS_2 in key.address, + 'Signature could not be verified.') + + d = self.km.verify( + signed_text, ADDRESS_2, + detached_sig=message.get_payload(1).get_payload()) + d.addCallback(assert_verify) + return d + + d = self.outgoing_mail._maybe_encrypt_and_sign(self.raw, recipient) + d.addCallback(check_signed) + d.addCallback(verify) + return d + + def test_attach_key(self): + d = self.outgoing_mail._maybe_encrypt_and_sign(self.raw, self.dest) + d.addCallback(self._assert_encrypted) + d.addCallback(self._check_headers, self.lines[:4]) + d.addCallback(lambda message: self.km.decrypt( + message.get_payload(1).get_payload(), ADDRESS)) + d.addCallback(lambda (decrypted, _): + self._check_key_attachment(Parser().parsestr(decrypted))) + return d + + def test_attach_key_not_known(self): + unknown_address = "someunknownaddress@somewhere.com" + lines = deepcopy(self.lines) + lines[1] = "To: <%s>" % (unknown_address,) + raw = '\r\n'.join(lines) + dest = User(unknown_address, 'gateway.leap.se', self.proto, ADDRESS_2) + + d = self.outgoing_mail._maybe_encrypt_and_sign( + raw, dest, fetch_remote=False) + d.addCallback(lambda (message, _): + self._check_headers(message, lines[:4])) + d.addCallback(self._check_key_attachment) + d.addErrback(log.err) + return d + + def _check_headers(self, message, headers): + msgstr = message.as_string(unixfrom=False) + for header in headers: + self.assertTrue(header in msgstr, + "Missing header: %s" % (header,)) + return message + + def _check_key_attachment(self, message): + for payload in message.get_payload(): + if payload.is_multipart(): + return self._check_key_attachment(payload) + if 'application/pgp-keys' == payload.get_content_type(): + keylines = PUBLIC_KEY_2.split('\n') + key = BEGIN_PUBLIC_KEY + '\n\n' + '\n'.join(keylines[4:-1]) + self.assertTrue(key in payload.get_payload(decode=True), + "Key attachment don't match") + return + self.fail("No public key attachment found") + + def _set_sign_used(self, address): + def set_sign(key): + key.sign_used = True + return self.km.put_key(key) + + d = self.km.get_key(address, fetch_remote=False) + d.addCallback(set_sign) + return d + + def _assert_encrypted(self, res): + message, _ = res + self.assertTrue('Content-Type' in message) + self.assertEqual('multipart/encrypted', message.get_content_type()) + self.assertEqual('application/pgp-encrypted', + message.get_param('protocol')) + self.assertEqual(2, len(message.get_payload())) + self.assertEqual('application/pgp-encrypted', + message.get_payload(0).get_content_type()) + self.assertEqual('application/octet-stream', + message.get_payload(1).get_content_type()) + return message diff --git a/mail/src/leap/mail/plugins/__init__.py b/mail/src/leap/mail/plugins/__init__.py new file mode 100644 index 00000000..ddb86917 --- /dev/null +++ b/mail/src/leap/mail/plugins/__init__.py @@ -0,0 +1,3 @@ +from twisted.plugin import pluginPackagePaths +__path__.extend(pluginPackagePaths(__name__)) +__all__ = [] diff --git a/mail/src/leap/mail/plugins/soledad_sync_hooks.py b/mail/src/leap/mail/plugins/soledad_sync_hooks.py new file mode 100644 index 00000000..9d48126e --- /dev/null +++ b/mail/src/leap/mail/plugins/soledad_sync_hooks.py @@ -0,0 +1,19 @@ +# -*- coding: utf-8 -*- +# soledad_sync_hooks.py +# Copyright (C) 2015 LEAP +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see <http://www.gnu.org/licenses/>. + +from leap.mail.sync_hooks import MailProcessingPostSyncHook +post_sync_uid_reindexer = MailProcessingPostSyncHook() diff --git a/mail/src/leap/mail/rfc3156.py b/mail/src/leap/mail/rfc3156.py new file mode 100644 index 00000000..7d7bc0f0 --- /dev/null +++ b/mail/src/leap/mail/rfc3156.py @@ -0,0 +1,390 @@ +# -*- coding: utf-8 -*- +# rfc3156.py +# Copyright (C) 2013 LEAP +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see <http://www.gnu.org/licenses/>. + +""" +Implements RFC 3156: MIME Security with OpenPGP. +""" + +import base64 +from StringIO import StringIO + +from twisted.python import log +from email.mime.application import MIMEApplication +from email.mime.multipart import MIMEMultipart +from email import errors +from email.generator import ( + Generator, + fcre, + NL, + _make_boundary, +) + + +# +# A generator that solves http://bugs.python.org/issue14983 +# + +class RFC3156CompliantGenerator(Generator): + """ + An email generator that addresses Python's issue #14983 for multipart + messages. + + This is just a copy of email.generator.Generator which fixes the following + bug: http://bugs.python.org/issue14983 + """ + + def _handle_multipart(self, msg): + """ + A multipart handling implementation that addresses issue #14983. + + This is just a copy of the parent's method which fixes the following + bug: http://bugs.python.org/issue14983 (see the line marked with + "(***)"). + + :param msg: The multipart message to be handled. + :type msg: email.message.Message + """ + # The trick here is to write out each part separately, merge them all + # together, and then make sure that the boundary we've chosen isn't + # present in the payload. + msgtexts = [] + subparts = msg.get_payload() + if subparts is None: + subparts = [] + elif isinstance(subparts, basestring): + # e.g. a non-strict parse of a message with no starting boundary. + self._fp.write(subparts) + return + elif not isinstance(subparts, list): + # Scalar payload + subparts = [subparts] + for part in subparts: + s = StringIO() + g = self.clone(s) + g.flatten(part, unixfrom=False) + msgtexts.append(s.getvalue()) + # BAW: What about boundaries that are wrapped in double-quotes? + boundary = msg.get_boundary() + if not boundary: + # Create a boundary that doesn't appear in any of the + # message texts. + alltext = NL.join(msgtexts) + boundary = _make_boundary(alltext) + msg.set_boundary(boundary) + # If there's a preamble, write it out, with a trailing CRLF + if msg.preamble is not None: + preamble = msg.preamble + if self._mangle_from_: + preamble = fcre.sub('>From ', msg.preamble) + self._fp.write(preamble + '\n') + # dash-boundary transport-padding CRLF + self._fp.write('--' + boundary + '\n') + # body-part + if msgtexts: + self._fp.write(msgtexts.pop(0)) + # *encapsulation + # --> delimiter transport-padding + # --> CRLF body-part + for body_part in msgtexts: + # delimiter transport-padding CRLF + self._fp.write('\n--' + boundary + '\n') + # body-part + self._fp.write(body_part) + # close-delimiter transport-padding + self._fp.write('\n--' + boundary + '--' + '\n') # (***) Solve #14983 + if msg.epilogue is not None: + self._fp.write('\n') + epilogue = msg.epilogue + if self._mangle_from_: + epilogue = fcre.sub('>From ', msg.epilogue) + self._fp.write(epilogue) + + +# +# Base64 encoding: these are almost the same as python's email.encoder +# solution, but a bit modified. +# + +def _bencode(s): + """ + Encode C{s} in base64. + + :param s: The string to be encoded. + :type s: str + """ + # We can't quite use base64.encodestring() since it tacks on a "courtesy + # newline". Blech! + if not s: + return s + value = base64.encodestring(s) + return value[:-1] + + +def encode_base64(msg): + """ + Encode a non-multipart message's payload in Base64 (in place). + + This method modifies the message contents in place and adds or replaces an + appropriate Content-Transfer-Encoding header. + + :param msg: The non-multipart message to be encoded. + :type msg: email.message.Message + """ + encoding = msg.get('Content-Transfer-Encoding', None) + if encoding is not None: + encoding = encoding.lower() + # XXX Python's email module can only decode quoted-printable, base64 and + # uuencoded data, so we might have to implement other decoding schemes in + # order to support RFC 3156 properly and correctly calculate signatures + # for multipart attachments (eg. 7bit or 8bit encoded attachments). For + # now, if content is already encoded as base64 or if it is encoded with + # some unknown encoding, we just pass. + if encoding in [None, 'quoted-printable', 'x-uuencode', 'uue', 'x-uue']: + orig = msg.get_payload(decode=True) + encdata = _bencode(orig) + msg.set_payload(encdata) + # replace or set the Content-Transfer-Encoding header. + try: + msg.replace_header('Content-Transfer-Encoding', 'base64') + except KeyError: + msg['Content-Transfer-Encoding'] = 'base64' + elif encoding is not 'base64': + log.err('Unknown content-transfer-encoding: %s' % encoding) + + +def encode_base64_rec(msg): + """ + Encode (possibly multipart) messages in base64 (in place). + + This method modifies the message contents in place. + + :param msg: The non-multipart message to be encoded. + :type msg: email.message.Message + """ + if not msg.is_multipart(): + encode_base64(msg) + else: + for sub in msg.get_payload(): + encode_base64_rec(sub) + + +# +# RFC 1847: multipart/signed and multipart/encrypted +# + +class MultipartSigned(MIMEMultipart): + """ + Multipart/Signed MIME message according to RFC 1847. + + 2.1. Definition of Multipart/Signed + + (1) MIME type name: multipart + (2) MIME subtype name: signed + (3) Required parameters: boundary, protocol, and micalg + (4) Optional parameters: none + (5) Security considerations: Must be treated as opaque while in + transit + + The multipart/signed content type contains exactly two body parts. + The first body part is the body part over which the digital signature + was created, including its MIME headers. The second body part + contains the control information necessary to verify the digital + signature. The first body part may contain any valid MIME content + type, labeled accordingly. The second body part is labeled according + to the value of the protocol parameter. + + When the OpenPGP digital signature is generated: + + (1) The data to be signed MUST first be converted to its content- + type specific canonical form. For text/plain, this means + conversion to an appropriate character set and conversion of + line endings to the canonical <CR><LF> sequence. + + (2) An appropriate Content-Transfer-Encoding is then applied; see + section 3. In particular, line endings in the encoded data + MUST use the canonical <CR><LF> sequence where appropriate + (note that the canonical line ending may or may not be present + on the last line of encoded data and MUST NOT be included in + the signature if absent). + + (3) MIME content headers are then added to the body, each ending + with the canonical <CR><LF> sequence. + + (4) As described in section 3 of this document, any trailing + whitespace MUST then be removed from the signed material. + + (5) As described in [2], the digital signature MUST be calculated + over both the data to be signed and its set of content headers. + + (6) The signature MUST be generated detached from the signed data + so that the process does not alter the signed data in any way. + """ + + def __init__(self, protocol, micalg, boundary=None, _subparts=None): + """ + Initialize the multipart/signed message. + + :param boundary: the multipart boundary string. By default it is + calculated as needed. + :type boundary: str + :param _subparts: a sequence of initial subparts for the payload. It + must be an iterable object, such as a list. You can always + attach new subparts to the message by using the attach() method. + :type _subparts: iterable + """ + MIMEMultipart.__init__( + self, _subtype='signed', boundary=boundary, + _subparts=_subparts) + self.set_param('protocol', protocol) + self.set_param('micalg', micalg) + + def attach(self, payload): + """ + Add the C{payload} to the current payload list. + + Also prevent from adding payloads with wrong Content-Type and from + exceeding a maximum of 2 payloads. + + :param payload: The payload to be attached. + :type payload: email.message.Message + """ + # second payload's content type must be equal to the protocol + # parameter given on object creation + if len(self.get_payload()) == 1: + if payload.get_content_type() != self.get_param('protocol'): + raise errors.MultipartConversionError( + 'Wrong content type %s.' % payload.get_content_type) + # prevent from adding more payloads + if len(self._payload) == 2: + raise errors.MultipartConversionError( + 'Cannot have more than two subparts.') + MIMEMultipart.attach(self, payload) + + +class MultipartEncrypted(MIMEMultipart): + """ + Multipart/encrypted MIME message according to RFC 1847. + + 2.2. Definition of Multipart/Encrypted + + (1) MIME type name: multipart + (2) MIME subtype name: encrypted + (3) Required parameters: boundary, protocol + (4) Optional parameters: none + (5) Security considerations: none + + The multipart/encrypted content type contains exactly two body parts. + The first body part contains the control information necessary to + decrypt the data in the second body part and is labeled according to + the value of the protocol parameter. The second body part contains + the data which was encrypted and is always labeled + application/octet-stream. + """ + + def __init__(self, protocol, boundary=None, _subparts=None): + """ + :param protocol: The encryption protocol to be added as a parameter to + the Content-Type header. + :type protocol: str + :param boundary: the multipart boundary string. By default it is + calculated as needed. + :type boundary: str + :param _subparts: a sequence of initial subparts for the payload. It + must be an iterable object, such as a list. You can always + attach new subparts to the message by using the attach() method. + :type _subparts: iterable + """ + MIMEMultipart.__init__( + self, _subtype='encrypted', boundary=boundary, + _subparts=_subparts) + self.set_param('protocol', protocol) + + def attach(self, payload): + """ + Add the C{payload} to the current payload list. + + Also prevent from adding payloads with wrong Content-Type and from + exceeding a maximum of 2 payloads. + + :param payload: The payload to be attached. + :type payload: email.message.Message + """ + # first payload's content type must be equal to the protocol parameter + # given on object creation + if len(self._payload) == 0: + if payload.get_content_type() != self.get_param('protocol'): + raise errors.MultipartConversionError( + 'Wrong content type.') + # second payload is always application/octet-stream + if len(self._payload) == 1: + if payload.get_content_type() != 'application/octet-stream': + raise errors.MultipartConversionError( + 'Wrong content type %s.' % payload.get_content_type) + # prevent from adding more payloads + if len(self._payload) == 2: + raise errors.MultipartConversionError( + 'Cannot have more than two subparts.') + MIMEMultipart.attach(self, payload) + + +# +# RFC 3156: application/pgp-encrypted, application/pgp-signed and +# application-pgp-signature. +# + +class PGPEncrypted(MIMEApplication): + """ + Application/pgp-encrypted MIME media type according to RFC 3156. + + * MIME media type name: application + * MIME subtype name: pgp-encrypted + * Required parameters: none + * Optional parameters: none + """ + + def __init__(self, version=1): + data = "Version: %d" % version + MIMEApplication.__init__(self, data, 'pgp-encrypted') + + +class PGPSignature(MIMEApplication): + """ + Application/pgp-signature MIME media type according to RFC 3156. + + * MIME media type name: application + * MIME subtype name: pgp-signature + * Required parameters: none + * Optional parameters: none + """ + def __init__(self, _data, name='signature.asc'): + MIMEApplication.__init__(self, _data, 'pgp-signature', + _encoder=lambda x: x, name=name) + self.add_header('Content-Description', 'OpenPGP Digital Signature') + + +class PGPKeys(MIMEApplication): + """ + Application/pgp-keys MIME media type according to RFC 3156. + + * MIME media type name: application + * MIME subtype name: pgp-keys + * Required parameters: none + * Optional parameters: none + """ + + def __init__(self, _data): + MIMEApplication.__init__(self, _data, 'pgp-keys') diff --git a/mail/src/leap/mail/size.py b/mail/src/leap/mail/size.py new file mode 100644 index 00000000..c9eaabd3 --- /dev/null +++ b/mail/src/leap/mail/size.py @@ -0,0 +1,57 @@ +# -*- coding: utf-8 -*- +# size.py +# Copyright (C) 2014 LEAP +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see <http://www.gnu.org/licenses/>. +""" +Recursively get size of objects. +""" +from gc import collect +from itertools import chain +from sys import getsizeof + + +def _get_size(item, seen): + known_types = {dict: lambda d: chain.from_iterable(d.items())} + default_size = getsizeof(0) + + def size_walk(item): + if id(item) in seen: + return 0 + seen.add(id(item)) + s = getsizeof(item, default_size) + for _type, fun in known_types.iteritems(): + if isinstance(item, _type): + s += sum(map(size_walk, fun(item))) + break + return s + + return size_walk(item) + + +def get_size(item): + """ + Return the cumulative size of a given object. + + Currently it supports only dictionaries, and seemingly leaks + some memory, so use with care. + + :param item: the item which size wants to be computed + :rtype: int + """ + seen = set() + size = _get_size(item, seen) + del seen + collect() + return size diff --git a/mail/src/leap/mail/smtp/README.rst b/mail/src/leap/mail/smtp/README.rst new file mode 100644 index 00000000..1d3a9038 --- /dev/null +++ b/mail/src/leap/mail/smtp/README.rst @@ -0,0 +1,44 @@ +Leap SMTP Gateway +================= + +The Bitmask Client runs a thin SMTP gateway on the user's device, which +intends to encrypt and sign outgoing messages to achieve point to point +encryption. + +The gateway is bound to localhost and the user's MUA should be configured to +send messages to it. After doing its thing, the gateway will relay the +messages to the remote SMTP server. + +Outgoing mail workflow: + + * SMTP gateway receives a message from the MUA. + + * SMTP gateway queries Key Manager for the user's private key. + + * For each recipient (including addresses in "To", "Cc" anc "Bcc" fields), + the following happens: + + - The recipient's address is validated against RFC2822. + + - An attempt is made to fetch the recipient's public PGP key. + + - If key is not found: + + - If the gateway is configured to only send encrypted messages the + recipient is rejected. + + - Otherwise, the message is signed and sent as plain text. + + - If the key is found, the message is encrypted to the recipient and + signed with the sender's private PGP key. + + * Finally, one message for each recipient is gatewayed to provider's SMTP + server. + + +Running tests +------------- + +Tests are run using Twisted's Trial API, like this:: + + python setup.py test -s leap.mail.gateway.tests diff --git a/mail/src/leap/mail/smtp/__init__.py b/mail/src/leap/mail/smtp/__init__.py new file mode 100644 index 00000000..9fab70a7 --- /dev/null +++ b/mail/src/leap/mail/smtp/__init__.py @@ -0,0 +1,73 @@ +# -*- coding: utf-8 -*- +# __init__.py +# Copyright (C) 2013 LEAP +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see <http://www.gnu.org/licenses/>. + +""" +SMTP gateway helper function. +""" +import logging +import os + +from twisted.internet import reactor +from twisted.internet.error import CannotListenError + +from leap.common.events import emit_async, catalog + +from leap.mail.smtp.gateway import SMTPFactory + +logger = logging.getLogger(__name__) + + +SMTP_PORT = 2013 + + +def run_service(soledad_sessions, keymanager_sessions, sendmail_opts, + port=SMTP_PORT): + """ + Main entry point to run the service from the client. + + :param soledad_sessions: a dict-like object, containing instances + of a Store (soledad instances), indexed by userid. + :param keymanager_sessions: a dict-like object, containing instances + of Keymanager, indexed by userid. + :param sendmail_opts: a dict-like object of sendmailOptions. + + :returns: the port as returned by the reactor when starts listening, and + the factory for the protocol. + :rtype: tuple + """ + factory = SMTPFactory(soledad_sessions, keymanager_sessions, + sendmail_opts) + + try: + interface = "localhost" + # don't bind just to localhost if we are running on docker since we + # won't be able to access smtp from the host + if os.environ.get("LEAP_DOCKERIZED"): + interface = '' + + # TODO Use Endpoints instead -------------------------------- + tport = reactor.listenTCP(port, factory, interface=interface) + emit_async(catalog.SMTP_SERVICE_STARTED, str(port)) + + return factory, tport + except CannotListenError: + logger.error("STMP Service failed to start: " + "cannot listen in port %s" % port) + emit_async(catalog.SMTP_SERVICE_FAILED_TO_START, str(port)) + except Exception as exc: + logger.error("Unhandled error while launching smtp gateway service") + logger.exception(exc) diff --git a/mail/src/leap/mail/smtp/bounces.py b/mail/src/leap/mail/smtp/bounces.py new file mode 100644 index 00000000..7a4674b9 --- /dev/null +++ b/mail/src/leap/mail/smtp/bounces.py @@ -0,0 +1,90 @@ +# -*- coding: utf-8 -*- +# bounces.py +# Copyright (C) 2016 LEAP +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see <http://www.gnu.org/licenses/>. +""" +Deliver bounces to the user Inbox. +""" +import time +from email.message import Message +from email.utils import formatdate + +from leap.mail.constants import INBOX_NAME +from leap.mail.mail import Account + + +# TODO implement localization for this template. + +BOUNCE_TEMPLATE = """This is your local Bitmask Mail Agent running at localhost. + +I'm sorry to have to inform you that your message could not be delivered to one +or more recipients. + +The reasons I got for the error are: + +{raw_error} + +If the problem persists and it's not a network connectivity issue, you might +want to contact your provider ({provider}) with this information (remove any +sensitive data before). + +--- Original message (*before* it was encrypted by bitmask) below ----: + +{orig}""" + + +class Bouncer(object): + """ + Implements a mechanism to deliver bounces to user inbox. + """ + # TODO this should follow RFC 6522, and compose a correct multipart + # attaching the report and the original message. Leaving this for a future + # iteration. + + def __init__(self, inbox_collection): + self._inbox_collection = inbox_collection + + def bounce_message(self, error_data, to, date=None, orig=''): + if not date: + date = formatdate(time.time()) + + raw_data = self._format_msg(error_data, to, date, orig) + d = self._inbox_collection.add_msg( + raw_data, ('\\Recent',), date=date) + return d + + def _format_msg(self, error_data, to, date, orig): + provider = to.split('@')[1] + + msg = Message() + msg.add_header( + 'From', 'bitmask-bouncer@localhost (Bitmask Local Agent)') + msg.add_header('To', to) + msg.add_header('Subject', 'Undelivered Message') + msg.add_header('Date', date) + msg.set_payload(BOUNCE_TEMPLATE.format( + raw_error=error_data, + provider=provider, + orig=orig)) + + return msg.as_string() + + +def bouncerFactory(soledad): + user_id = soledad.uuid + acc = Account(soledad, user_id) + d = acc.callWhenReady(lambda _: acc.get_collection_by_mailbox(INBOX_NAME)) + d.addCallback(lambda inbox: Bouncer(inbox)) + return d diff --git a/mail/src/leap/mail/smtp/gateway.py b/mail/src/leap/mail/smtp/gateway.py new file mode 100644 index 00000000..e49bbe82 --- /dev/null +++ b/mail/src/leap/mail/smtp/gateway.py @@ -0,0 +1,413 @@ +# -*- coding: utf-8 -*- +# gateway.py +# Copyright (C) 2013 LEAP +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see <http://www.gnu.org/licenses/>. +""" +LEAP SMTP encrypted gateway. + +The following classes comprise the SMTP gateway service: + + * SMTPFactory - A twisted.internet.protocol.ServerFactory that provides + the SMTPDelivery protocol. + + * SMTPDelivery - A twisted.mail.smtp.IMessageDelivery implementation. It + knows how to validate sender and receiver of messages and it generates + an EncryptedMessage for each recipient. + + * EncryptedMessage - An implementation of twisted.mail.smtp.IMessage that + knows how to encrypt/sign itself before sending. +""" +from email.Header import Header + +from zope.interface import implements +from zope.interface import implementer + +from twisted.cred.portal import Portal, IRealm +from twisted.mail import smtp +from twisted.mail.imap4 import LOGINCredentials, PLAINCredentials +from twisted.internet import defer, protocol +from twisted.python import log + +from leap.common.check import leap_assert_type +from leap.common.events import emit_async, catalog +from leap.mail import errors +from leap.mail.cred import LocalSoledadTokenChecker +from leap.mail.utils import validate_address +from leap.mail.rfc3156 import RFC3156CompliantGenerator +from leap.mail.outgoing.service import outgoingFactory +from leap.mail.smtp.bounces import bouncerFactory +from leap.keymanager.errors import KeyNotFound + +# replace email generator with a RFC 3156 compliant one. +from email import generator + +generator.Generator = RFC3156CompliantGenerator + + +LOCAL_FQDN = "bitmask.local" + + +@implementer(IRealm) +class LocalSMTPRealm(object): + + _encoding = 'utf-8' + + def __init__(self, keymanager_sessions, soledad_sessions, sendmail_opts, + encrypted_only=False): + """ + :param keymanager_sessions: a dict-like object, containing instances + of a Keymanager objects, indexed by + userid. + """ + self._keymanager_sessions = keymanager_sessions + self._soledad_sessions = soledad_sessions + self._sendmail_opts = sendmail_opts + self.encrypted_only = encrypted_only + + def requestAvatar(self, avatarId, mind, *interfaces): + + if isinstance(avatarId, str): + avatarId = avatarId.decode(self._encoding) + + def gotKeymanagerAndSoledad(result): + keymanager, soledad = result + d = bouncerFactory(soledad) + d.addCallback(lambda bouncer: (keymanager, soledad, bouncer)) + return d + + def getMessageDelivery(result): + keymanager, soledad, bouncer = result + # TODO use IMessageDeliveryFactory instead ? + # it could reuse the connections. + if smtp.IMessageDelivery in interfaces: + userid = avatarId + opts = self.getSendingOpts(userid) + + outgoing = outgoingFactory( + userid, keymanager, opts, bouncer=bouncer) + avatar = SMTPDelivery(userid, keymanager, self.encrypted_only, + outgoing) + + return (smtp.IMessageDelivery, avatar, + getattr(avatar, 'logout', lambda: None)) + + raise NotImplementedError(self, interfaces) + + d1 = self.lookupKeymanagerInstance(avatarId) + d2 = self.lookupSoledadInstance(avatarId) + d = defer.gatherResults([d1, d2]) + d.addCallback(gotKeymanagerAndSoledad) + d.addCallback(getMessageDelivery) + return d + + def lookupKeymanagerInstance(self, userid): + try: + keymanager = self._keymanager_sessions[userid] + except: + raise errors.AuthenticationError( + 'No keymanager session found for user %s. Is it authenticated?' + % userid) + # XXX this should return the instance after whenReady callback + return defer.succeed(keymanager) + + def lookupSoledadInstance(self, userid): + try: + soledad = self._soledad_sessions[userid] + except: + raise errors.AuthenticationError( + 'No soledad session found for user %s. Is it authenticated?' + % userid) + # XXX this should return the instance after whenReady callback + return defer.succeed(soledad) + + def getSendingOpts(self, userid): + try: + opts = self._sendmail_opts[userid] + except KeyError: + raise errors.ConfigurationError( + 'No sendingMail options found for user %s' % userid) + return opts + + +class SMTPTokenChecker(LocalSoledadTokenChecker): + """A credentials checker that will lookup a token for the SMTP service. + For now it will be using the same identifier than IMAPTokenChecker""" + + service = 'mail_auth' + + # TODO besides checking for token credential, + # we could also verify the certificate here. + + +class LEAPInitMixin(object): + + """ + A Mixin that takes care of initialization of all the data needed to access + LEAP sessions. + """ + def __init__(self, soledad_sessions, keymanager_sessions, sendmail_opts, + encrypted_only=False): + realm = LocalSMTPRealm( + keymanager_sessions, soledad_sessions, sendmail_opts, + encrypted_only) + portal = Portal(realm) + + checker = SMTPTokenChecker(soledad_sessions) + self.checker = checker + self.portal = portal + portal.registerChecker(checker) + + +class LocalSMTPServer(smtp.ESMTP, LEAPInitMixin): + """ + The Production ESMTP Server: Authentication Needed. + Authenticates against SMTP Token stored in Local Soledad instance. + The Realm will produce a Delivery Object that handles encryption/signing. + """ + + # TODO: implement Queue using twisted.mail.mail.MailService + + def __init__(self, soledads, keyms, sendmailopts, *args, **kw): + encrypted_only = kw.pop('encrypted_only', False) + + LEAPInitMixin.__init__(self, soledads, keyms, sendmailopts, + encrypted_only) + smtp.ESMTP.__init__(self, *args, **kw) + + +# TODO implement retries -- see smtp.SenderMixin +class SMTPFactory(protocol.ServerFactory): + """ + Factory for an SMTP server with encrypted gatewaying capabilities. + """ + + protocol = LocalSMTPServer + domain = LOCAL_FQDN + timeout = 600 + encrypted_only = False + + def __init__(self, soledad_sessions, keymanager_sessions, sendmail_opts, + deferred=None, retries=3): + + self._soledad_sessions = soledad_sessions + self._keymanager_sessions = keymanager_sessions + self._sendmail_opts = sendmail_opts + + def buildProtocol(self, addr): + p = self.protocol( + self._soledad_sessions, self._keymanager_sessions, + self._sendmail_opts, encrypted_only=self.encrypted_only) + p.factory = self + p.host = LOCAL_FQDN + p.challengers = {"LOGIN": LOGINCredentials, "PLAIN": PLAINCredentials} + return p + + +# +# SMTPDelivery +# + +@implementer(smtp.IMessageDelivery) +class SMTPDelivery(object): + """ + Validate email addresses and handle message delivery. + """ + + def __init__(self, userid, keymanager, encrypted_only, outgoing_mail): + """ + Initialize the SMTP delivery object. + + :param userid: The user currently logged in + :type userid: unicode + :param keymanager: A Key Manager from where to get recipients' public + keys. + :param encrypted_only: Whether the SMTP gateway should send unencrypted + mail or not. + :type encrypted_only: bool + :param outgoing_mail: The outgoing mail to send the message + :type outgoing_mail: leap.mail.outgoing.service.OutgoingMail + """ + self._userid = userid + self._outgoing_mail = outgoing_mail + self._km = keymanager + self._encrypted_only = encrypted_only + self._origin = None + + def receivedHeader(self, helo, origin, recipients): + """ + Generate the 'Received:' header for a message. + + :param helo: The argument to the HELO command and the client's IP + address. + :type helo: (str, str) + :param origin: The address the message is from. + :type origin: twisted.mail.smtp.Address + :param recipients: A list of the addresses for which this message is + bound. + :type: list of twisted.mail.smtp.User + + @return: The full "Received" header string. + :type: str + """ + myHostname, clientIP = helo + headerValue = "by bitmask.local from %s with ESMTP ; %s" % ( + clientIP, smtp.rfc822date()) + # email.Header.Header used for automatic wrapping of long lines + return "Received: %s" % Header(s=headerValue, header_name='Received') + + def validateTo(self, user): + """ + Validate the address of a recipient of the message, possibly + rejecting it if the recipient key is not available. + + This method is called once for each recipient, i.e. for each SMTP + protocol line beginning with "RCPT TO:", which includes all addresses + in "To", "Cc" and "Bcc" MUA fields. + + The recipient's address is validated against the RFC 2822 definition. + If self._encrypted_only is True and no key is found for a recipient, + then that recipient is rejected. + + The method returns an encrypted message object that is able to send + itself to the user's address. + + :param user: The user whose address we wish to validate. + :type: twisted.mail.smtp.User + + @return: A callable which takes no arguments and returns an + encryptedMessage. + @rtype: no-argument callable + + @raise SMTPBadRcpt: Raised if messages to the address are not to be + accepted. + """ + # try to find recipient's public key + address = validate_address(user.dest.addrstr) + + # verify if recipient key is available in keyring + def found(_): + log.msg("Accepting mail for %s..." % user.dest.addrstr) + emit_async(catalog.SMTP_RECIPIENT_ACCEPTED_ENCRYPTED, + self._userid, user.dest.addrstr) + + def not_found(failure): + failure.trap(KeyNotFound) + + # if key was not found, check config to see if will send anyway + if self._encrypted_only: + emit_async(catalog.SMTP_RECIPIENT_REJECTED, self._userid, + user.dest.addrstr) + raise smtp.SMTPBadRcpt(user.dest.addrstr) + log.msg("Warning: will send an unencrypted message (because " + "encrypted_only' is set to False).") + emit_async( + catalog.SMTP_RECIPIENT_ACCEPTED_UNENCRYPTED, + self._userid, user.dest.addrstr) + + def encrypt_func(_): + return lambda: EncryptedMessage(user, self._outgoing_mail) + + d = self._km.get_key(address) + d.addCallbacks(found, not_found) + d.addCallback(encrypt_func) + return d + + def validateFrom(self, helo, origin): + """ + Validate the address from which the message originates. + + :param helo: The argument to the HELO command and the client's IP + address. + :type: (str, str) + :param origin: The address the message is from. + :type origin: twisted.mail.smtp.Address + + @return: origin or a Deferred whose callback will be passed origin. + @rtype: Deferred or Address + + @raise twisted.mail.smtp.SMTPBadSender: Raised if messages from this + address are not to be accepted. + """ + # accept mail from anywhere. To reject an address, raise + # smtp.SMTPBadSender here. + if str(origin) != str(self._userid): + log.msg("Rejecting sender {0}, expected {1}".format(origin, + self._userid)) + raise smtp.SMTPBadSender(origin) + self._origin = origin + return origin + + +# +# EncryptedMessage +# + +class EncryptedMessage(object): + """ + Receive plaintext from client, encrypt it and send message to a + recipient. + """ + implements(smtp.IMessage) + + def __init__(self, user, outgoing_mail): + """ + Initialize the encrypted message. + + :param user: The recipient of this message. + :type user: twisted.mail.smtp.User + :param outgoing_mail: The outgoing mail to send the message + :type outgoing_mail: leap.mail.outgoing.service.OutgoingMail + """ + # assert params + leap_assert_type(user, smtp.User) + + self._user = user + self._lines = [] + self._outgoing_mail = outgoing_mail + + def lineReceived(self, line): + """ + Handle another line. + + :param line: The received line. + :type line: str + """ + self._lines.append(line) + + def eomReceived(self): + """ + Handle end of message. + + This method will encrypt and send the message. + + :returns: a deferred + """ + log.msg("Message data complete.") + self._lines.append('') # add a trailing newline + raw_mail = '\r\n'.join(self._lines) + + return self._outgoing_mail.send_message(raw_mail, self._user) + + def connectionLost(self): + """ + Log an error when the connection is lost. + """ + log.msg("Connection lost unexpectedly!") + log.err() + emit_async(catalog.SMTP_CONNECTION_LOST, self._userid, + self._user.dest.addrstr) + # unexpected loss of connection; don't save + + self._lines = [] diff --git a/mail/src/leap/mail/smtp/tests/185CA770.key b/mail/src/leap/mail/smtp/tests/185CA770.key new file mode 100644 index 00000000..587b4164 --- /dev/null +++ b/mail/src/leap/mail/smtp/tests/185CA770.key @@ -0,0 +1,79 @@ +-----BEGIN PGP PRIVATE KEY BLOCK----- +Version: GnuPG v1.4.10 (GNU/Linux) + +lQIVBFCJNL4BEADFsI1TCD4yq7ZqL7VhdVviTuX6JUps8/mVEhRVOZhojLcTYaqQ +gs6T6WabRxcK7ymOnf4K8NhYdz6HFoJN46BT87etokx7J/Sl2OhpiqBQEY+jW8Rp ++3MSGrGmvFw0s1lGrz/cXzM7UNgWSTOnYZ5nJS1veMhy0jseZOUK7ekp2oEDjGZh +pzgd3zICCR2SvlpLIXB2Nr/CUcuRWTcc5LlKmbjMybu0E/uuY14st3JL+7qI6QX0 +atFm0VhFVpagOl0vWKxakUx4hC7j1wH2ADlCvSZPG0StSLUyHkJx3UPsmYxOZFao +ATED3Okjwga6E7PJEbzyqAkvzw/M973kaZCUSH75ZV0cQnpdgXV3DK1gSa3d3gug +W1lE0V7pwnN2NTOYfBMi+WloCs/bp4iZSr4QP1duZ3IqKraeBDCk7MoFo4A9Wk07 +kvqPwF9IBgatu62WVEZIzwyViN+asFUGfgp+8D7gtnlWAw0V6y/lSTzyl+dnLP98 +Hfr2eLBylFs+Kl3Pivpg2uHw09LLCrjeLEN3dj9SfBbA9jDIo9Zhs1voiIK/7Shx +E0BRJaBgG3C4QaytYEu7RFFOKuvBai9w2Y5OfsKFo8rA7v4dxFFDvzKGujCtNnwf +oyaGlZmMBU5MUmHUNiG8ON21COZBtK5oMScuY1VC9CQonj3OClg3IbU9SQARAQAB +/gNlAkdOVQG0JGRyZWJzIChncGcgdGVzdCBrZXkpIDxkcmVic0BsZWFwLnNlPokC +OAQTAQIAIgUCUIk0vgIbAwYLCQgHAwIGFQgCCQoLBBYCAwECHgECF4AACgkQty9e +xhhcp3Bdhw//bdPUNbp6rgIjRRuwYvGJ6IuiFuFWJQ0m3iAuuAoZo5GHAPqZAuGk +dMVYu0dtCtZ68MJ/QpjBCT9RRL+mgIgfLfUSj2ZknP4nb6baiG5u28l0KId/e5IC +iQKBnIsjxKxhLBVHSzRaS1P+vZeF2C2R9XyNy0eCnAwyCMcD0R8TVROGQ7i4ZQsM +bMj1LPpOwhV/EGp23nD+upWOVbn/wQHOYV2kMiA/8fizmWRIWsV4/68uMA+WDP4L +40AnJ0fcs04f9deM9P6pjlm00VD7qklYEGw6Mpr2g/M73kGh1nlAv+ImQBGlLMle +RXyzHY3WAhzmRKWO4koFuKeR9Q0EMzk2R4/kuagdWEpM+bhwE4xPV1tPZhn9qFTz +pQD4p/VT4qNQKOD0+aTFWre65Rt2cFFMLI7UmEHNLi0NB9JCIAi4+l+b9WQNlmaO +C8EhOGwRzmehUyHmXM3BNW28MnyKFJ7bBFMd7uJz+vAPOrr6OzuNvVCv2I2ICkTs +ihIj/zw5GXxkPO7YbMu9rKG0nKF1N3JB1gUJ78DHmhbjeaGSvHw85sPD0/1dPZK4 +8Gig8i62aCxf8OlJPlt8ZhBBolzs6ITUNa75Rw9fJsj3UWuv2VFaIuR57bFWmY3s +A9KPgdf7jVQlAZKlVyli7IkyaZmxDZNFQoTdIC9uo0aggIDP8zKv0n2dBz4EUIk0 +vgEQAOO8BAR7sBdqj2RRMRNeWSA4S9GuHfV3YQARnqYsbITs1jRgAo7jx9Z5C80c +ZOxOUVK7CJjtTqU0JB9QP/zwV9hk5i6y6aQTysclQyTNN10aXu/3zJla5Duhz+Cs ++5UcVAmNJX9FgTMVvhKDEIY/LNmb9MoBLMut1CkDx+WPCV45WOIBCDdj2HpIjie4 +phs0/65SWjPiVg3WsFZljVxpJCGXP48Eet2bf8afYH1lx3sQMcNbyJACIPtz+YKz +c7jIKwKSWzg1VyYikbk9eWCxcz6VKNJKi94YH9c7U8X3TdZ8G0kGYUldjYDvesyl +nuQlcGCtSGKOAhrN/Bu2R0gpFgYl247u79CmjotefMdv8BGUDW6u9/Sep9xN3dW8 +S87h6M/tvs0ChlkDDpJedzCd7ThdikGvFRJfW/8sT/+qoTKskySQaDIeNJnxZuyK +wELLMBvCZGpamwmnkEGhvuZWq0h/DwyTs4QAE8OVHXJSM3UN7hM4lJIUh+sRKJ1F +AXXTdSY4cUNaS+OKtj2LJ85zFqhfAZ4pFwLCgYbJtU5hej2LnMJNbYcSkjxbk+c5 +IjkoZRF+ExjZlc0VLYNT57ZriwZ/pX42ofjOyMR/dkHQuFik/4K7v1ZemfaTdm07 +SEMBknR6OZsy/5+viEtXiih3ptTMaT9row+g+cFoxdXkisKvABEBAAH+AwMCIlVK +Xs3x0Slgwx03cTNIoWXmishkPCJlEEdcjldz2VyQF9hjdp1VIe+npI26chKwCZqm +U8yYbJh4UBrugUUzKKd4EfnmKfu+/BsJciFRVKwBtiolIiUImzcHPWktYLwo9yzX +W42teShXXVgWmsJN1/6FqJdsLg8dxWesXMKoaNF4n1P7zx6vKBmDHTRz7PToaI/d +5/nKrjED7ZT1h+qR5i9UUgbvF0ySp8mlqk/KNqHUSLDB9kf/JDg4XVtPHGGd9Ik/ +60UJ7aDfohi4Z0VgwWmfLBwcQ3It+ENtnPFufH3WHW8c1UA4wVku9tOTqyrRG6tP +TZGiRfuwsv7Hq3pWT6rntbDkTiVgESM4C1fiZblc98iWUKGXSHqm+te1TwXOUCci +J/gryXcjQFM8A0rwA/m+EvsoWuzoqIl3x++p3/3/mGux6UD4O7OhJNRVRz+8Mhq1 +ksrR9XkQzpq3Yv3ulTHz7l+WCRRXxw5+XWAkRHHF47Vf/na38NJQHcsCBbRIuLYR +wBzS48cYzYkF6VejKThdQmdYJ0/fUrlUBCAJWgrfqCihFLDa1s4jJ16/fqi8a97Y +4raVy2hrF2vFc/wet13hsaddVn4rPRAMDEGdgEmJX7MmU1emT/yaIG9lvjMpI2c5 +ADXGF2yYYa7H8zPIFyHU1RSavlT0S/K9yzIZvv+jA5KbNeGp+WWFT8MLZs0IhoCZ +d1EgLUYAt7LPUSm2lBy1w/IL+VtYuyn/UVFo2xWiHd1ABiNWl1ji3X9Ki5613QqH +bvn4z46voCzdZ02rYkAwrdqDr92fiBR8ctwA0AudaG6nf2ztmFKtM3E/RPMkPgKF +8NHYc7QxS2jruJxXBtjRBMtoIaZ0+AXUO6WuEJrDLDHWaM08WKByQMm808xNCbRr +CpiK8qyR3SwkfaOMCp22mqViirQ2KfuVvBpBT2pBYlgDKs50nE+stDjUMv+FDKAo +5NtiyPfNtaBOYnXAEQb/hjjW5bKq7JxHSxIWAYKbNKIWgftJ3ACZAsBMHfaOCFNH ++XLojAoxOI+0zbN6FtjN+YMU1XrLd6K49v7GEiJQZVQSfLCecVDhDU9paNROA/Xq +/3nDCTKhd3stTPnc8ymLAwhTP0bSoFh/KtU96D9ZMC2cu9XZ+UcSQYES/ncZWcLw +wTKrt+VwBG1z3DbV2O0ruUiXTLcZMsrwbUSDx1RVhmKZ0i42AttMdauFQ9JaX2CS +2ddqFBS1b4X6+VCy44KkpdXsmp0NWMgm/PM3PTisCxrha7bI5/LqfXG0b+GuIFb4 +h/lEA0Ae0gMgkzm3ePAPPVlRj7kFl5Osjxm3YVRW23WWGDRF5ywIROlBjbdozA0a +MyMgXlG9hhJseIpFveoiwqenNE5Wxg0yQbnhMUTKeCQ0xskG82P+c9bvDsevAQUR +uv1JAGGxDd1/4nk0M5m9/Gf4Bn0uLAz29LdMg0FFUvAm2ol3U3uChm7OISU8dqFy +JdCFACKBMzAREiXfgH2TrTxAhpy5uVcUSQV8x5J8qJ/mUoTF1WE3meXEm9CIvIAF +Mz49KKebLS3zGFixMcKLAOKA+s/tUWO7ZZoJyQjvQVerLyDo6UixVb11LQUJQOXb +ZIuSKV7deCgBDQ26C42SpF3rHfEQa7XH7j7tl1IIW/9DfYJYVQHaz1NTq6zcjWS2 +e+cUexBPhxbadGn0zelXr6DLJqQT7kaVeYOHlkYUHkZXdHE4CWoHqOboeB02uM/A +e7nge1rDi57ySrsF4AVl59QJYBPR43AOVbCJAh8EGAECAAkFAlCJNL4CGwwACgkQ +ty9exhhcp3DetA/8D/IscSBlWY3TjCD2P7t3+X34USK8EFD3QJse9dnCWOLcskFQ +IoIfhRM752evFu2W9owEvxSQdG+otQAOqL72k1EH2g7LsADuV8I4LOYOnLyeIE9I +b+CFPBkmzTEzrdYp6ITUU7qqgkhcgnltKGHoektIjxE8gtxCKEdyxkzazum6nCQQ +kSBZOXVU3ezm+A2QHHP6XT1GEbdKbJ0tIuJR8ADu08pBx2c/LDBBreVStrrt1Dbz +uR+U8MJsfLVcYX/Rw3V+KA24oLRzg91y3cfi3sNU/kmd5Cw42Tj00B+FXQny51Mq +s4KyqHobj62II68eL5HRB2pcGsoaedQyxu2cYSeVyarBOiUPNYkoGDJoKdDyZRIB +NNK0W+ASTf0zeHhrY/okt1ybTVtvbt6wkTEbKVePUaYmNmhre1cAj4uNwFzYjkzJ +cm+8XWftD+TV8cE5DyVdnF00SPDuPzodRAPXaGpQUMLkE4RPr1TAwcuoPH9aFHZ/ +se6rw6TQHLd0vMk0U/DocikXpSJ1N6caE3lRwI/+nGfXNiCr8MIdofgkBeO86+G7 +k0UXS4v5FKk1nwTyt4PkFJDvAJX6rZPxIZ9NmtA5ao5vyu1DT5IhoXgDzwurAe8+ +R+y6gtA324hXIweFNt7SzYPfI4SAjunlmm8PIBf3owBrk3j+w6EQoaCreK4= +=6HcJ +-----END PGP PRIVATE KEY BLOCK----- diff --git a/mail/src/leap/mail/smtp/tests/185CA770.pub b/mail/src/leap/mail/smtp/tests/185CA770.pub new file mode 100644 index 00000000..38af19f8 --- /dev/null +++ b/mail/src/leap/mail/smtp/tests/185CA770.pub @@ -0,0 +1,52 @@ +-----BEGIN PGP PUBLIC KEY BLOCK----- +Version: GnuPG v1.4.10 (GNU/Linux) + +mQINBFCJNL4BEADFsI1TCD4yq7ZqL7VhdVviTuX6JUps8/mVEhRVOZhojLcTYaqQ +gs6T6WabRxcK7ymOnf4K8NhYdz6HFoJN46BT87etokx7J/Sl2OhpiqBQEY+jW8Rp ++3MSGrGmvFw0s1lGrz/cXzM7UNgWSTOnYZ5nJS1veMhy0jseZOUK7ekp2oEDjGZh +pzgd3zICCR2SvlpLIXB2Nr/CUcuRWTcc5LlKmbjMybu0E/uuY14st3JL+7qI6QX0 +atFm0VhFVpagOl0vWKxakUx4hC7j1wH2ADlCvSZPG0StSLUyHkJx3UPsmYxOZFao +ATED3Okjwga6E7PJEbzyqAkvzw/M973kaZCUSH75ZV0cQnpdgXV3DK1gSa3d3gug +W1lE0V7pwnN2NTOYfBMi+WloCs/bp4iZSr4QP1duZ3IqKraeBDCk7MoFo4A9Wk07 +kvqPwF9IBgatu62WVEZIzwyViN+asFUGfgp+8D7gtnlWAw0V6y/lSTzyl+dnLP98 +Hfr2eLBylFs+Kl3Pivpg2uHw09LLCrjeLEN3dj9SfBbA9jDIo9Zhs1voiIK/7Shx +E0BRJaBgG3C4QaytYEu7RFFOKuvBai9w2Y5OfsKFo8rA7v4dxFFDvzKGujCtNnwf +oyaGlZmMBU5MUmHUNiG8ON21COZBtK5oMScuY1VC9CQonj3OClg3IbU9SQARAQAB +tCRkcmVicyAoZ3BnIHRlc3Qga2V5KSA8ZHJlYnNAbGVhcC5zZT6JAjgEEwECACIF +AlCJNL4CGwMGCwkIBwMCBhUIAgkKCwQWAgMBAh4BAheAAAoJELcvXsYYXKdwXYcP +/23T1DW6eq4CI0UbsGLxieiLohbhViUNJt4gLrgKGaORhwD6mQLhpHTFWLtHbQrW +evDCf0KYwQk/UUS/poCIHy31Eo9mZJz+J2+m2ohubtvJdCiHf3uSAokCgZyLI8Ss +YSwVR0s0WktT/r2XhdgtkfV8jctHgpwMMgjHA9EfE1UThkO4uGULDGzI9Sz6TsIV +fxBqdt5w/rqVjlW5/8EBzmFdpDIgP/H4s5lkSFrFeP+vLjAPlgz+C+NAJydH3LNO +H/XXjPT+qY5ZtNFQ+6pJWBBsOjKa9oPzO95BodZ5QL/iJkARpSzJXkV8sx2N1gIc +5kSljuJKBbinkfUNBDM5NkeP5LmoHVhKTPm4cBOMT1dbT2YZ/ahU86UA+Kf1U+Kj +UCjg9PmkxVq3uuUbdnBRTCyO1JhBzS4tDQfSQiAIuPpfm/VkDZZmjgvBIThsEc5n +oVMh5lzNwTVtvDJ8ihSe2wRTHe7ic/rwDzq6+js7jb1Qr9iNiApE7IoSI/88ORl8 +ZDzu2GzLvayhtJyhdTdyQdYFCe/Ax5oW43mhkrx8PObDw9P9XT2SuPBooPIutmgs +X/DpST5bfGYQQaJc7OiE1DWu+UcPXybI91Frr9lRWiLkee2xVpmN7APSj4HX+41U +JQGSpVcpYuyJMmmZsQ2TRUKE3SAvbqNGoICAz/Myr9J9uQINBFCJNL4BEADjvAQE +e7AXao9kUTETXlkgOEvRrh31d2EAEZ6mLGyE7NY0YAKO48fWeQvNHGTsTlFSuwiY +7U6lNCQfUD/88FfYZOYusumkE8rHJUMkzTddGl7v98yZWuQ7oc/grPuVHFQJjSV/ +RYEzFb4SgxCGPyzZm/TKASzLrdQpA8fljwleOVjiAQg3Y9h6SI4nuKYbNP+uUloz +4lYN1rBWZY1caSQhlz+PBHrdm3/Gn2B9Zcd7EDHDW8iQAiD7c/mCs3O4yCsCkls4 +NVcmIpG5PXlgsXM+lSjSSoveGB/XO1PF903WfBtJBmFJXY2A73rMpZ7kJXBgrUhi +jgIazfwbtkdIKRYGJduO7u/Qpo6LXnzHb/ARlA1urvf0nqfcTd3VvEvO4ejP7b7N +AoZZAw6SXncwne04XYpBrxUSX1v/LE//qqEyrJMkkGgyHjSZ8WbsisBCyzAbwmRq +WpsJp5BBob7mVqtIfw8Mk7OEABPDlR1yUjN1De4TOJSSFIfrESidRQF103UmOHFD +WkvjirY9iyfOcxaoXwGeKRcCwoGGybVOYXo9i5zCTW2HEpI8W5PnOSI5KGURfhMY +2ZXNFS2DU+e2a4sGf6V+NqH4zsjEf3ZB0LhYpP+Cu79WXpn2k3ZtO0hDAZJ0ejmb +Mv+fr4hLV4ood6bUzGk/a6MPoPnBaMXV5IrCrwARAQABiQIfBBgBAgAJBQJQiTS+ +AhsMAAoJELcvXsYYXKdw3rQP/A/yLHEgZVmN04wg9j+7d/l9+FEivBBQ90CbHvXZ +wlji3LJBUCKCH4UTO+dnrxbtlvaMBL8UkHRvqLUADqi+9pNRB9oOy7AA7lfCOCzm +Dpy8niBPSG/ghTwZJs0xM63WKeiE1FO6qoJIXIJ5bShh6HpLSI8RPILcQihHcsZM +2s7pupwkEJEgWTl1VN3s5vgNkBxz+l09RhG3SmydLSLiUfAA7tPKQcdnPywwQa3l +Ura67dQ287kflPDCbHy1XGF/0cN1figNuKC0c4Pdct3H4t7DVP5JneQsONk49NAf +hV0J8udTKrOCsqh6G4+tiCOvHi+R0QdqXBrKGnnUMsbtnGEnlcmqwTolDzWJKBgy +aCnQ8mUSATTStFvgEk39M3h4a2P6JLdcm01bb27esJExGylXj1GmJjZoa3tXAI+L +jcBc2I5MyXJvvF1n7Q/k1fHBOQ8lXZxdNEjw7j86HUQD12hqUFDC5BOET69UwMHL +qDx/WhR2f7Huq8Ok0By3dLzJNFPw6HIpF6UidTenGhN5UcCP/pxn1zYgq/DCHaH4 +JAXjvOvhu5NFF0uL+RSpNZ8E8reD5BSQ7wCV+q2T8SGfTZrQOWqOb8rtQ0+SIaF4 +A88LqwHvPkfsuoLQN9uIVyMHhTbe0s2D3yOEgI7p5ZpvDyAX96MAa5N4/sOhEKGg +q3iu +=RChS +-----END PGP PUBLIC KEY BLOCK----- diff --git a/mail/src/leap/mail/smtp/tests/cert/server.crt b/mail/src/leap/mail/smtp/tests/cert/server.crt new file mode 100644 index 00000000..a27391c2 --- /dev/null +++ b/mail/src/leap/mail/smtp/tests/cert/server.crt @@ -0,0 +1,29 @@ +-----BEGIN CERTIFICATE----- +MIIFBjCCAu4CCQCWn3oMoQrDJTANBgkqhkiG9w0BAQUFADBFMQswCQYDVQQGEwJV +UzETMBEGA1UECAwKU29tZS1TdGF0ZTEhMB8GA1UECgwYSW50ZXJuZXQgV2lkZ2l0 +cyBQdHkgTHRkMB4XDTEzMTAyMzE0NDUwNFoXDTE2MDcxOTE0NDUwNFowRTELMAkG +A1UEBhMCVVMxEzARBgNVBAgMClNvbWUtU3RhdGUxITAfBgNVBAoMGEludGVybmV0 +IFdpZGdpdHMgUHR5IEx0ZDCCAiIwDQYJKoZIhvcNAQEBBQADggIPADCCAgoCggIB +APexTvEvG7cSmZdAERHt9TB11cSor54Y/F7NmYMdSOJNi4Y0kwkSslpdfipi+mt/ +NFg/uGKi1mcgvuXdVbVPZ9rCgVpIzMncO8RAP7a5+I2zKUzqMCCbLH16sYpo/rDk +VQ5V15TwLsTzOFGG8Cgp68TR8zHuZ4Edf2zMGC1IaiJ6W38LTnJgsowYOCFDAF3z +L36kxMO5gNGEUYV6tjltx+rAcXka3po+xiAgvW6q65UUgDHcIdEGG2dc9bkxxPl7 +RkprF2RwwADNzYS7Tn+Hpmjy06pfYZHNME+Iw515bCRF3GQFUU4BpGnY7EO+h4P9 +Kb1h948gUT9/oswXG+q2Kwk8AoggMJkUOWDFiCa5UjW1GBoxxb7VtZ+QTJXxlFWc +M2VzT7M/HX+P4b05vY4MXJjxPAFKrAGS7J8DKW8WJNUnXa9XSDBHg5qijDzZ/zGm +HTdG6iADnJLmOHBQgFQ12a/n9mYV2GPVC6FlgDzG9f0/SUPBUCafyWYz1LwKY4VM +2NLx/iwYMQsNIMSZQfNmufNDBr70+BShe3ZpbmKB/J33d87AuJd2HjnsThTEAAr+ +6CejyYmwFutoDUCF8IaKGJEp7OGP2//ub4nt5WwW8DYLRi8EqtzEnxPo5ZiayHMY +GHR1jpX1O5JVJFUE79bZCFFHKmtJc4kVZS4m4rTLsk83AgMBAAEwDQYJKoZIhvcN +AQEFBQADggIBAEt4PIRqVuALQSdgZ+GiZYuvEVjxoDVtMSc/ym93Gi8R7DDivFH9 +4suQc5QUiuEF8lpEtkmh+PZ+oFdQkjhBH80h7p4BUSyBy5Yi6dy7ATTlBAqwzCYZ +4wzHeJzu1SI6FinZLksoULbcw04n410aGHkLa6I9O3vCC4kXSnBlwU1sUsJphxM2 +3pkHBpvv79XYf5kFqZPzF16aO7rxFuVvqgXLyzwuyP9kH5zMA21Kioxs/pNyg1lm +5h0VinpHLPse+4tYih1L1WLMpEZiSwZgFhoRtlcdIVXokZPaX4G2EkdrMmSQruWg +Uz8Av6LEYHmRfbYwYM2kEX/+AF8thpTQDbvxjqYk5oyGX4wpKGpih1ac/jYu3O8B +VLhbxZlBYcLxCqqNsGJrWaiHj2Jf4GhUB0O9hXfaZDMqEGXT9GzOz0yF6b3pDQVy +H0lKIBb+kQzB/jhZKu4vrTAowXtt/av5d7D+rpAU1SxfUhBOPNSRoJUI5NSBbokp +a7u4azdB2IQETX3d2rhDk09EbG1XmMi5Vg1oa8nxfMOWXZnDMusJoZClKjrthmwd +rtR5et44XYhX6p217RBkYMDOVFT7aZpu4SaFeqZIuarVYodSmgXToOFXPsrLppRQ +adOT0FpU64RPNrQz5NF1bSIjqrHSaRVacf8yr7qqxNnpMsrtkDJzsMBz +-----END CERTIFICATE----- diff --git a/mail/src/leap/mail/smtp/tests/cert/server.key b/mail/src/leap/mail/smtp/tests/cert/server.key new file mode 100644 index 00000000..197a4496 --- /dev/null +++ b/mail/src/leap/mail/smtp/tests/cert/server.key @@ -0,0 +1,51 @@ +-----BEGIN RSA PRIVATE KEY----- +MIIJKQIBAAKCAgEA97FO8S8btxKZl0AREe31MHXVxKivnhj8Xs2Zgx1I4k2LhjST +CRKyWl1+KmL6a380WD+4YqLWZyC+5d1VtU9n2sKBWkjMydw7xEA/trn4jbMpTOow +IJssfXqximj+sORVDlXXlPAuxPM4UYbwKCnrxNHzMe5ngR1/bMwYLUhqInpbfwtO +cmCyjBg4IUMAXfMvfqTEw7mA0YRRhXq2OW3H6sBxeRremj7GICC9bqrrlRSAMdwh +0QYbZ1z1uTHE+XtGSmsXZHDAAM3NhLtOf4emaPLTql9hkc0wT4jDnXlsJEXcZAVR +TgGkadjsQ76Hg/0pvWH3jyBRP3+izBcb6rYrCTwCiCAwmRQ5YMWIJrlSNbUYGjHF +vtW1n5BMlfGUVZwzZXNPsz8df4/hvTm9jgxcmPE8AUqsAZLsnwMpbxYk1Sddr1dI +MEeDmqKMPNn/MaYdN0bqIAOckuY4cFCAVDXZr+f2ZhXYY9ULoWWAPMb1/T9JQ8FQ +Jp/JZjPUvApjhUzY0vH+LBgxCw0gxJlB82a580MGvvT4FKF7dmluYoH8nfd3zsC4 +l3YeOexOFMQACv7oJ6PJibAW62gNQIXwhooYkSns4Y/b/+5vie3lbBbwNgtGLwSq +3MSfE+jlmJrIcxgYdHWOlfU7klUkVQTv1tkIUUcqa0lziRVlLibitMuyTzcCAwEA +AQKCAgAFQdcqGVTeQt/NrQdvuPw+RhH+dZIcqe0ZWgXLGaEFZJ30gEMqqyHr9xYJ +ckZcZ7vFr7yLI2enkrNaj6MVERVkOEKkluz5z9nY5YA0safL4iSbRFE3L/P2ydbg +2C+ns4D2p+3GdH6ZoYvtdw6723/skoQ16Bh8ThL5TS+qLmJKTwyIGsZUeSbxAEaY +tiJY3COC7Z5bhSFt0QAl9B/QAjt/CQyfhGl7Hp/36Jn8slYDuQariD+TfyyvufJh +NuQ2Y15vj+xULmx01+lnys30uP1YNuc1M4cPoCpJVd7JBd28u1rdKJu8Kx7BPGBv +Y6jerU3ofh7SA96VmXDsIgVuquUo51Oklspe6a9VaDmzLvjYqJsBKQ7BH3J2f07x +NiOob56CGXykX51Ig3WBK1wKn+pA69FL62DbkEa6SykGCqdZPdgBF/kiMc0TESsl +867Em63Yx/2hq+mG3Dknnq8jWXf+Es/zZSSak6N4154IxPOD3m1hzuUq73PP7Ptt +KFe6NfU0DmAuTJL3FqNli8F8lFfvJfuwmW2qk5iTMfwPxybSd8FPbGxi7aRgoZdh +7fIbTFJ0X2f83/SO+9rCzV+B091+d7TM8AaOJ4dEoS74rlRZg53EgmAU0phVnE+l +taMNKGHy2kpJrv9IHX3w5Gm6CjNJj5t4ccS0J18NFFJ+j077eQKCAQEA/RJNRUBS +mI5l0eirl78Q9uDPh1usChZpQiLsvscIJITWQ1vtXSRCvP0hVQRRv8+4CtrZr2rX +v0afkzg/3HNFaNsjYT6aHjgnombFqfpyS/NZN/p3gOzi2h+1Sujzz5fBUGhNLVgZ +F2GLnJbiIHnM1BmKA6597pHpXcRMh1E3DSjDMQAEEsBgF6MyS+MT9WfNwHvJukii +k028tNzR4wRq3Xo3WTfvXZRjbX54Ew9Zy3+TFiu19j2FmuOoqyj+ZvMic4EYmTaY +BWm7viDff4dW34dR9sYCuTWWehLtMJGroA38e7lTLfNOHNDGaUZWkfxs4uJCsxvP +0fPp3xlbU3NUGwKCAQEA+o8SeHwEN+VN2dZvC3wFvbnRvWLc1aLnNcndRE9QLVcC +B4LMRuQMpxaNYRiSQPppoPTNq6zWbo6FEjUO5Md7R8I8dbg1vHo4PzuHOu2wXNcm +DEicocCpSKShSS27NCK6uoSsTqTIlG4u+1x9/R2gJEjlTqjeIkOQkPv7PbWhrUyt +XqvzPy4bewOz9Brmd6ryi8ZLtNbUSNwMyd64s9b1V4A6JRlYZrMDOQ6kXEZo+mbL +ynet0vuj7lYxsAZvxoPIq+Gi5i0CrDYtze6JCg+kGahjMX0zXRjXrYh/YID8NWYT +0GXr2+a0V5pXg86YCDp/jpr3lq75HJJ+vIvm2VHLFQKCAQATEm0GWgmfe6PKxPkh +j4GsyVZ6gfseK4A1PsKOwhsn/WbUXrotuczZx03axV+P0AyzrLiZErk9rgnao3OU +no9Njq5E5t3ghyTdhVdCLyCr/qPrpxGYgsG55IfaJGIzc+FauPGQCEKj03MdEvXp +sqQwG9id3GmbMB3hNij6TbGTaU4EhFbKPvs+7Mqek3dumCsWZX3Xbx/pcANXsgiT +TkLrfAltzNxaNhOkLdLIxPBkeLHSCutEqnBGMwAEHivGAG7JO6Jp8YZVahl/A6U0 +TDPM1rrjmRqdcJ9thb2gWmoPvt4XSOku3lY1r7o0NtvRVq+yDZEvRFpOHU6zxIpw +aJGfAoIBAQDiTvvF62379pc8nJwr6VdeKEozHuqL49mmEbBTFLg8W4wvsIpFtZFg +EdSc0I65NfTWNobV+wSrUvsKmPXc2fiVtfDZ+wo+NL49Ds10Al/7WzC4g5VF3DiK +rngnGrEtw/iYo2Dmn5uzxVmWG9KIHowYeeb0Bz6sAA7BhXdGI5nmZ41oJzNL659S +muOdJfboO3Vbnj2fFzMio+7BHvQBK7Tp1Z2vCJd6G1Jb5Me7uLT1BognVbWhDTzh +9uRmM0oeKcXEycZS1HDHjyAMEtmgRsRXkGoXtxf/jIKx8MnsJlSm/o4C+yvvsQ9O +2M8W9DEJrZys93eNmHjUv9TNBCf8Pg6JAoIBAQDDItnQPLntCUgd7dy0dDjQYBGN +4wVRJNINpgjqwJj0hVjB/dmvrcxkXcOG4VAH+iNH8A25qLU+RTDcNipuL3uEFKbF +O4DSjFih3qL1Y8otTXSrPeqZOMvYpY8dXS5uyI7DSWQQZyZ9bMpeWbxgx4LHqPPH +rdcVJy9Egw1ZIOA7JBFM02uGn9TVwFzNUJk0G/3xwVHzDxYNbJ98vDfflc2vD4CH +OAN6un0pOuol2h200F6zFgc5mbETWHCPIom+ZMXIX3bq7g341c/cgqIELPTk8DLS +s+AgrZ4qYmskrFaD0PHakWsQNHGC8yOh80lgE3Gl4nxSGAvkcR7dkSmsIQFL +-----END RSA PRIVATE KEY----- diff --git a/mail/src/leap/mail/smtp/tests/mail.txt b/mail/src/leap/mail/smtp/tests/mail.txt new file mode 100644 index 00000000..95420470 --- /dev/null +++ b/mail/src/leap/mail/smtp/tests/mail.txt @@ -0,0 +1,10 @@ +HELO drebs@riseup.net +MAIL FROM: drebs@riseup.net +RCPT TO: drebs@riseup.net +RCPT TO: drebs@leap.se +DATA +Subject: leap test + +Hello world! +. +QUIT diff --git a/mail/src/leap/mail/smtp/tests/test_gateway.py b/mail/src/leap/mail/smtp/tests/test_gateway.py new file mode 100644 index 00000000..9d88afb8 --- /dev/null +++ b/mail/src/leap/mail/smtp/tests/test_gateway.py @@ -0,0 +1,181 @@ +# -*- coding: utf-8 -*- +# test_gateway.py +# Copyright (C) 2013 LEAP +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see <http://www.gnu.org/licenses/>. + +""" +SMTP gateway tests. +""" +import re +import tempfile +from datetime import datetime + +from twisted.internet import reactor +from twisted.internet.defer import inlineCallbacks, fail, succeed, Deferred +from twisted.test import proto_helpers + +from mock import Mock + +from leap.keymanager import openpgp, errors +from leap.mail.testing import KeyManagerWithSoledadTestCase +from leap.mail.testing import ADDRESS, ADDRESS_2 +from leap.mail.testing.smtp import getSMTPFactory, TEST_USER + + +# some regexps +IP_REGEX = "(([0-9]|[1-9][0-9]|1[0-9]{2}|2[0-4][0-9]|25[0-5])\.){3}" + \ + "([0-9]|[1-9][0-9]|1[0-9]{2}|2[0-4][0-9]|25[0-5])" +HOSTNAME_REGEX = "(([a-zA-Z0-9]|[a-zA-Z0-9][a-zA-Z0-9\-]*[a-zA-Z0-9])\.)*" + \ + "([A-Za-z0-9]|[A-Za-z0-9][A-Za-z0-9\-]*[A-Za-z0-9])" +IP_OR_HOST_REGEX = '(' + IP_REGEX + '|' + HOSTNAME_REGEX + ')' + + +class TestSmtpGateway(KeyManagerWithSoledadTestCase): + + EMAIL_DATA = ['HELO gateway.leap.se', + 'MAIL FROM: <%s>' % ADDRESS_2, + 'RCPT TO: <%s>' % ADDRESS, + 'DATA', + 'From: User <%s>' % ADDRESS_2, + 'To: Leap <%s>' % ADDRESS, + 'Date: ' + datetime.now().strftime('%c'), + 'Subject: test message', + '', + 'This is a secret message.', + 'Yours,', + 'A.', + '', + '.', + 'QUIT'] + + def setUp(self): + # pytest handles correctly the setupEnv for the class, + # but trial ignores it. + if not getattr(self, 'tempdir', None): + self.tempdir = tempfile.mkdtemp() + return KeyManagerWithSoledadTestCase.setUp(self) + + def tearDown(self): + return KeyManagerWithSoledadTestCase.tearDown(self) + + def assertMatch(self, string, pattern, msg=None): + if not re.match(pattern, string): + msg = self._formatMessage(msg, '"%s" does not match pattern "%s".' + % (string, pattern)) + raise self.failureException(msg) + + @inlineCallbacks + def test_gateway_accepts_valid_email(self): + """ + Test if SMTP server responds correctly for valid interaction. + """ + + SMTP_ANSWERS = ['220 ' + IP_OR_HOST_REGEX + + ' NO UCE NO UBE NO RELAY PROBES', + '250 ' + IP_OR_HOST_REGEX + ' Hello ' + + IP_OR_HOST_REGEX + ', nice to meet you', + '250 Sender address accepted', + '250 Recipient address accepted', + '354 Continue'] + + user = TEST_USER + proto = getSMTPFactory({user: None}, {user: self.km}, {user: None}) + transport = proto_helpers.StringTransport() + proto.makeConnection(transport) + reply = "" + for i, line in enumerate(self.EMAIL_DATA): + reply += yield self.getReply(line + '\r\n', proto, transport) + self.assertMatch(reply, '\r\n'.join(SMTP_ANSWERS), + 'Did not get expected answer from gateway.') + proto.setTimeout(None) + + @inlineCallbacks + def test_missing_key_rejects_address(self): + """ + Test if server rejects to send unencrypted when 'encrypted_only' is + True. + """ + # remove key from key manager + pubkey = yield self.km.get_key(ADDRESS) + pgp = openpgp.OpenPGPScheme( + self._soledad, gpgbinary=self.gpg_binary_path) + yield pgp.delete_key(pubkey) + # mock the key fetching + self.km._fetch_keys_from_server = Mock( + return_value=fail(errors.KeyNotFound())) + user = TEST_USER + proto = getSMTPFactory( + {user: None}, {user: self.km}, {user: None}, + encrypted_only=True) + transport = proto_helpers.StringTransport() + proto.makeConnection(transport) + yield self.getReply(self.EMAIL_DATA[0] + '\r\n', proto, transport) + yield self.getReply(self.EMAIL_DATA[1] + '\r\n', proto, transport) + reply = yield self.getReply(self.EMAIL_DATA[2] + '\r\n', + proto, transport) + # ensure the address was rejected + self.assertEqual( + '550 Cannot receive for specified address\r\n', + reply, + 'Address should have been rejected with appropriate message.') + proto.setTimeout(None) + + @inlineCallbacks + def test_missing_key_accepts_address(self): + """ + Test if server accepts to send unencrypted when 'encrypted_only' is + False. + """ + # remove key from key manager + pubkey = yield self.km.get_key(ADDRESS) + pgp = openpgp.OpenPGPScheme( + self._soledad, gpgbinary=self.gpg_binary_path) + yield pgp.delete_key(pubkey) + # mock the key fetching + self.km._fetch_keys_from_server = Mock( + return_value=fail(errors.KeyNotFound())) + user = TEST_USER + proto = getSMTPFactory({user: None}, {user: self.km}, {user: None}) + transport = proto_helpers.StringTransport() + proto.makeConnection(transport) + yield self.getReply(self.EMAIL_DATA[0] + '\r\n', proto, transport) + yield self.getReply(self.EMAIL_DATA[1] + '\r\n', proto, transport) + reply = yield self.getReply(self.EMAIL_DATA[2] + '\r\n', + proto, transport) + # ensure the address was accepted + self.assertEqual( + '250 Recipient address accepted\r\n', + reply, + 'Address should have been accepted with appropriate message.') + proto.setTimeout(None) + + def getReply(self, line, proto, transport): + proto.lineReceived(line) + + if line[:4] not in ['HELO', 'MAIL', 'RCPT', 'DATA']: + return succeed("") + + def check_transport(_): + reply = transport.value() + if reply: + transport.clear() + return succeed(reply) + + d = Deferred() + d.addCallback(check_transport) + reactor.callLater(0, lambda: d.callback(None)) + return d + + return check_transport(None) diff --git a/mail/src/leap/mail/sync_hooks.py b/mail/src/leap/mail/sync_hooks.py new file mode 100644 index 00000000..8efbb7ce --- /dev/null +++ b/mail/src/leap/mail/sync_hooks.py @@ -0,0 +1,120 @@ +# -*- coding: utf-8 -*- +# sync_hooks.py +# Copyright (C) 2015 LEAP +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see <http://www.gnu.org/licenses/>. +""" +Soledad PostSync Hooks. + +Process every new document of interest after every soledad synchronization, +using the hooks that soledad exposes via plugins. +""" +import logging + +from re import compile as regex_compile + +from zope.interface import implements +from twisted.internet import defer +from twisted.plugin import IPlugin +from twisted.python import log + +from leap.soledad.client.interfaces import ISoledadPostSyncPlugin +from leap.mail import constants + +logger = logging.getLogger(__name__) + + +def _get_doc_type_preffix(s): + return s[:2] + + +class MailProcessingPostSyncHook(object): + implements(IPlugin, ISoledadPostSyncPlugin) + + META_DOC_PREFFIX = _get_doc_type_preffix(constants.METAMSGID) + watched_doc_types = (META_DOC_PREFFIX, ) + + _account = None + _pending_docs = [] + _processing_deferreds = [] + + def process_received_docs(self, doc_id_list): + if self._has_configured_account(): + process_fun = self._make_uid_index + else: + self._processing_deferreds = [] + process_fun = self._queue_doc_id + + for doc_id in doc_id_list: + if _get_doc_type_preffix(doc_id) in self.watched_doc_types: + log.msg("Mail post-sync hook: processing %s" % doc_id) + process_fun(doc_id) + + return defer.gatherResults(self._processing_deferreds) + + def set_account(self, account): + self._account = account + if account: + self._process_queued_docs() + + def _has_configured_account(self): + return self._account is not None + + def _queue_doc_id(self, doc_id): + self._pending_docs.append(doc_id) + + def _make_uid_index(self, mdoc_id): + indexer = self._account.mbox_indexer + mbox_uuid = _get_mbox_uuid(mdoc_id) + if mbox_uuid: + chash = _get_chash_from_mdoc(mdoc_id) + logger.debug("Making index table for %s:%s" % (mbox_uuid, chash)) + index_docid = constants.METAMSGID.format( + mbox_uuid=mbox_uuid.replace('-', '_'), + chash=chash) + # XXX could avoid creating table if I track which ones I already + # have seen -- but make sure *it's already created* before + # inserting the index entry!. + d = indexer.create_table(mbox_uuid) + d.addBoth(lambda _: indexer.insert_doc(mbox_uuid, index_docid)) + self._processing_deferreds.append(d) + + def _process_queued_docs(self): + assert(self._has_configured_account()) + pending = self._pending_docs + log.msg("Mail post-sync hook: processing queued docs") + + def remove_pending_docs(res): + self._pending_docs = [] + return res + + d = self.process_received_docs(pending) + d.addCallback(remove_pending_docs) + return d + + +_mbox_uuid_regex = regex_compile(constants.METAMSGID_MBOX_RE) +_mdoc_chash_regex = regex_compile(constants.METAMSGID_CHASH_RE) + + +def _get_mbox_uuid(doc_id): + matches = _mbox_uuid_regex.findall(doc_id) + if matches: + return matches[0].replace('_', '-') + + +def _get_chash_from_mdoc(doc_id): + matches = _mdoc_chash_regex.findall(doc_id) + if matches: + return matches[0] diff --git a/mail/src/leap/mail/testing/__init__.py b/mail/src/leap/mail/testing/__init__.py new file mode 100644 index 00000000..8822a5c7 --- /dev/null +++ b/mail/src/leap/mail/testing/__init__.py @@ -0,0 +1,353 @@ +# -*- coding: utf-8 -*- +# __init__.py +# Copyright (C) 2013 LEAP +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see <http://www.gnu.org/licenses/>. +""" +Base classes and keys for leap.mail tests. +""" +import os +import distutils.spawn +from mock import Mock +from twisted.internet.defer import gatherResults +from twisted.trial import unittest +from twisted.internet import defer + + +from leap.soledad.client import Soledad +from leap.keymanager import KeyManager + + +from leap.common.testing.basetest import BaseLeapTest + +ADDRESS = 'leap@leap.se' +ADDRESS_2 = 'anotheruser@leap.se' + + +class defaultMockSharedDB(object): + get_doc = Mock(return_value=None) + put_doc = Mock(side_effect=None) + open = Mock(return_value=None) + close = Mock(return_value=None) + syncable = True + + def __call__(self): + return self + + +class KeyManagerWithSoledadTestCase(unittest.TestCase, BaseLeapTest): + + def setUp(self): + self.gpg_binary_path = self._find_gpg() + + self._soledad = Soledad( + u"leap@leap.se", + u"123456", + secrets_path=self.tempdir + "/secret.gpg", + local_db_path=self.tempdir + "/soledad.u1db", + server_url='', + cert_file=None, + auth_token=None, + shared_db=defaultMockSharedDB(), + syncable=False) + + self.km = self._key_manager() + + class Response(object): + code = 200 + phrase = '' + + def deliverBody(self, x): + return '' + + self.km._async_client_pinned.request = Mock( + return_value=defer.succeed(Response())) + + d1 = self.km.put_raw_key(PRIVATE_KEY, ADDRESS) + d2 = self.km.put_raw_key(PRIVATE_KEY_2, ADDRESS_2) + return gatherResults([d1, d2]) + + def tearDown(self): + km = self._key_manager() + # wait for the indexes to be ready for the tear down + d = km._openpgp.deferred_init + d.addCallback(lambda _: self.delete_all_keys(km)) + d.addCallback(lambda _: self._soledad.close()) + return d + + def delete_all_keys(self, km): + def delete_keys(keys): + deferreds = [] + for key in keys: + d = km._openpgp.delete_key(key) + deferreds.append(d) + return gatherResults(deferreds) + + def check_deleted(_, private): + d = km.get_all_keys(private=private) + d.addCallback(lambda keys: self.assertEqual(keys, [])) + return d + + deferreds = [] + for private in [True, False]: + d = km.get_all_keys(private=private) + d.addCallback(delete_keys) + d.addCallback(check_deleted, private) + deferreds.append(d) + return gatherResults(deferreds) + + def _key_manager(self, user=ADDRESS, url='', token=None, + ca_cert_path=None): + return KeyManager(user, url, self._soledad, token=token, + gpgbinary=self.gpg_binary_path, + ca_cert_path=ca_cert_path) + + def _find_gpg(self): + gpg_path = distutils.spawn.find_executable('gpg') + if gpg_path is not None: + return os.path.realpath(gpg_path) + else: + return "/usr/bin/gpg" + + def get_public_binary_key(self): + with open(PATH + '/fixtures/public_key.bin', 'r') as binary_public_key: + return binary_public_key.read() + + def get_private_binary_key(self): + with open( + PATH + '/fixtures/private_key.bin', 'r') as binary_private_key: + return binary_private_key.read() + + +# key 24D18DDF: public key "Leap Test Key <leap@leap.se>" +KEY_FINGERPRINT = "E36E738D69173C13D709E44F2F455E2824D18DDF" +PUBLIC_KEY = """ +-----BEGIN PGP PUBLIC KEY BLOCK----- +Version: GnuPG v1.4.10 (GNU/Linux) + +mQINBFC9+dkBEADNRfwV23TWEoGc/x0wWH1P7PlXt8MnC2Z1kKaKKmfnglVrpOiz +iLWoiU58sfZ0L5vHkzXHXCBf6Eiy/EtUIvdiWAn+yASJ1mk5jZTBKO/WMAHD8wTO +zpMsFmWyg3xc4DkmFa9KQ5EVU0o/nqPeyQxNMQN7px5pPwrJtJFmPxnxm+aDkPYx +irDmz/4DeDNqXliazGJKw7efqBdlwTHkl9Akw2gwy178pmsKwHHEMOBOFFvX61AT +huKqHYmlCGSliwbrJppTG7jc1/ls3itrK+CWTg4txREkSpEVmfcASvw/ZqLbjgfs +d/INMwXnR9U81O8+7LT6yw/ca4ppcFoJD7/XJbkRiML6+bJ4Dakiy6i727BzV17g +wI1zqNvm5rAhtALKfACha6YO43aJzairO4II1wxVHvRDHZn2IuKDDephQ3Ii7/vb +hUOf6XCSmchkAcpKXUOvbxm1yfB1LRa64mMc2RcZxf4mW7KQkulBsdV5QG2276lv +U2UUy2IutXcGP5nXC+f6sJJGJeEToKJ57yiO/VWJFjKN8SvP+7AYsQSqINUuEf6H +T5gCPCraGMkTUTPXrREvu7NOohU78q6zZNaL3GW8ai7eSeANSuQ8Vzffx7Wd8Y7i +Pw9sYj0SMFs1UgjbuL6pO5ueHh+qyumbtAq2K0Bci0kqOcU4E9fNtdiovQARAQAB +tBxMZWFwIFRlc3QgS2V5IDxsZWFwQGxlYXAuc2U+iQI3BBMBCAAhBQJQvfnZAhsD +BQsJCAcDBRUKCQgLBRYCAwEAAh4BAheAAAoJEC9FXigk0Y3fT7EQAKH3IuRniOpb +T/DDIgwwjz3oxB/W0DDMyPXowlhSOuM0rgGfntBpBb3boezEXwL86NPQxNGGruF5 +hkmecSiuPSvOmQlqlS95NGQp6hNG0YaKColh+Q5NTspFXCAkFch9oqUje0LdxfSP +QfV9UpeEvGyPmk1I9EJV/YDmZ4+Djge1d7qhVZInz4Rx1NrSyF/Tc2EC0VpjQFsU +Y9Kb2YBBR7ivG6DBc8ty0jJXi7B4WjkFcUEJviQpMF2dCLdonCehYs1PqsN1N7j+ +eFjQd+hqVMJgYuSGKjvuAEfClM6MQw7+FmFwMyLgK/Ew/DttHEDCri77SPSkOGSI +txCzhTg6798f6mJr7WcXmHX1w1Vcib5FfZ8vTDFVhz/XgAgArdhPo9V6/1dgSSiB +KPQ/spsco6u5imdOhckERE0lnAYvVT6KE81TKuhF/b23u7x+Wdew6kK0EQhYA7wy +7LmlaNXc7rMBQJ9Z60CJ4JDtatBWZ0kNrt2VfdDHVdqBTOpl0CraNUjWE5YMDasr +K2dF5IX8D3uuYtpZnxqg0KzyLg0tzL0tvOL1C2iudgZUISZNPKbS0z0v+afuAAnx +2pTC3uezbh2Jt8SWTLhll4i0P4Ps5kZ6HQUO56O+/Z1cWovX+mQekYFmERySDR9n +3k1uAwLilJmRmepGmvYbB8HloV8HqwgguQINBFC9+dkBEAC0I/xn1uborMgDvBtf +H0sEhwnXBC849/32zic6udB6/3Efk9nzbSpL3FSOuXITZsZgCHPkKarnoQ2ztMcS +sh1ke1C5gQGms75UVmM/nS+2YI4vY8OX/GC/on2vUyncqdH+bR6xH5hx4NbWpfTs +iQHmz5C6zzS/kuabGdZyKRaZHt23WQ7JX/4zpjqbC99DjHcP9BSk7tJ8wI4bkMYD +uFVQdT9O6HwyKGYwUU4sAQRAj7XCTGvVbT0dpgJwH4RmrEtJoHAx4Whg8mJ710E0 +GCmzf2jqkNuOw76ivgk27Kge+Hw00jmJjQhHY0yVbiaoJwcRrPKzaSjEVNgrpgP3 +lXPRGQArgESsIOTeVVHQ8fhK2YtTeCY9rIiO+L0OX2xo9HK7hfHZZWL6rqymXdyS +fhzh/f6IPyHFWnvj7Brl7DR8heMikygcJqv+ed2yx7iLyCUJ10g12I48+aEj1aLe +dP7lna32iY8/Z0SHQLNH6PXO9SlPcq2aFUgKqE75A/0FMk7CunzU1OWr2ZtTLNO1 +WT/13LfOhhuEq9jTyTosn0WxBjJKq18lnhzCXlaw6EAtbA7CUwsD3CTPR56aAXFK +3I7KXOVAqggrvMe5Tpdg5drfYpI8hZovL5aAgb+7Y5ta10TcJdUhS5K3kFAWe/td +U0cmWUMDP1UMSQ5Jg6JIQVWhSwARAQABiQIfBBgBCAAJBQJQvfnZAhsMAAoJEC9F +Xigk0Y3fRwsP/i0ElYCyxeLpWJTwo1iCLkMKz2yX1lFVa9nT1BVTPOQwr/IAc5OX +NdtbJ14fUsKL5pWgW8OmrXtwZm1y4euI1RPWWubG01ouzwnGzv26UcuHeqC5orZj +cOnKtL40y8VGMm8LoicVkRJH8blPORCnaLjdOtmA3rx/v2EXrJpSa3AhOy0ZSRXk +ZSrK68AVNwamHRoBSYyo0AtaXnkPX4+tmO8X8BPfj125IljubvwZPIW9VWR9UqCE +VPfDR1XKegVb6VStIywF7kmrknM1C5qUY28rdZYWgKorw01hBGV4jTW0cqde3N51 +XT1jnIAa+NoXUM9uQoGYMiwrL7vNsLlyyiW5ayDyV92H/rIuiqhFgbJsHTlsm7I8 +oGheR784BagAA1NIKD1qEO9T6Kz9lzlDaeWS5AUKeXrb7ZJLI1TTCIZx5/DxjLqM +Tt/RFBpVo9geZQrvLUqLAMwdaUvDXC2c6DaCPXTh65oCZj/hqzlJHH+RoTWWzKI+ +BjXxgUWF9EmZUBrg68DSmI+9wuDFsjZ51BcqvJwxyfxtTaWhdoYqH/UQS+D1FP3/ +diZHHlzwVwPICzM9ooNTgbrcDzyxRkIVqsVwBq7EtzcvgYUyX53yG25Giy6YQaQ2 +ZtQ/VymwFL3XdUWV6B/hU4PVAFvO3qlOtdJ6TpE+nEWgcWjCv5g7RjXX +=MuOY +-----END PGP PUBLIC KEY BLOCK----- +""" +PRIVATE_KEY = """ +-----BEGIN PGP PRIVATE KEY BLOCK----- +Version: GnuPG v1.4.10 (GNU/Linux) + +lQcYBFC9+dkBEADNRfwV23TWEoGc/x0wWH1P7PlXt8MnC2Z1kKaKKmfnglVrpOiz +iLWoiU58sfZ0L5vHkzXHXCBf6Eiy/EtUIvdiWAn+yASJ1mk5jZTBKO/WMAHD8wTO +zpMsFmWyg3xc4DkmFa9KQ5EVU0o/nqPeyQxNMQN7px5pPwrJtJFmPxnxm+aDkPYx +irDmz/4DeDNqXliazGJKw7efqBdlwTHkl9Akw2gwy178pmsKwHHEMOBOFFvX61AT +huKqHYmlCGSliwbrJppTG7jc1/ls3itrK+CWTg4txREkSpEVmfcASvw/ZqLbjgfs +d/INMwXnR9U81O8+7LT6yw/ca4ppcFoJD7/XJbkRiML6+bJ4Dakiy6i727BzV17g +wI1zqNvm5rAhtALKfACha6YO43aJzairO4II1wxVHvRDHZn2IuKDDephQ3Ii7/vb +hUOf6XCSmchkAcpKXUOvbxm1yfB1LRa64mMc2RcZxf4mW7KQkulBsdV5QG2276lv +U2UUy2IutXcGP5nXC+f6sJJGJeEToKJ57yiO/VWJFjKN8SvP+7AYsQSqINUuEf6H +T5gCPCraGMkTUTPXrREvu7NOohU78q6zZNaL3GW8ai7eSeANSuQ8Vzffx7Wd8Y7i +Pw9sYj0SMFs1UgjbuL6pO5ueHh+qyumbtAq2K0Bci0kqOcU4E9fNtdiovQARAQAB +AA/+JHtlL39G1wsH9R6UEfUQJGXR9MiIiwZoKcnRB2o8+DS+OLjg0JOh8XehtuCs +E/8oGQKtQqa5bEIstX7IZoYmYFiUQi9LOzIblmp2vxOm+HKkxa4JszWci2/ZmC3t +KtaA4adl9XVnshoQ7pijuCMUKB3naBEOAxd8s9d/JeReGIYkJErdrnVfNk5N71Ds +FmH5Ll3XtEDvgBUQP3nkA6QFjpsaB94FHjL3gDwum/cxzj6pCglcvHOzEhfY0Ddb +J967FozQTaf2JW3O+w3LOqtcKWpq87B7+O61tVidQPSSuzPjCtFF0D2LC9R/Hpky +KTMQ6CaKja4MPhjwywd4QPcHGYSqjMpflvJqi+kYIt8psUK/YswWjnr3r4fbuqVY +VhtiHvnBHQjz135lUqWvEz4hM3Xpnxydx7aRlv5NlevK8+YIO5oFbWbGNTWsPZI5 +jpoFBpSsnR1Q5tnvtNHauvoWV+XN2qAOBTG+/nEbDYH6Ak3aaE9jrpTdYh0CotYF +q7csANsDy3JvkAzeU6WnYpsHHaAjqOGyiZGsLej1UcXPFMosE/aUo4WQhiS8Zx2c +zOVKOi/X5vQ2GdNT9Qolz8AriwzsvFR+bxPzyd8V6ALwDsoXvwEYinYBKK8j0OPv +OOihSR6HVsuP9NUZNU9ewiGzte/+/r6pNXHvR7wTQ8EWLcEIAN6Zyrb0bHZTIlxt +VWur/Ht2mIZrBaO50qmM5RD3T5oXzWXi/pjLrIpBMfeZR9DWfwQwjYzwqi7pxtYx +nJvbMuY505rfnMoYxb4J+cpRXV8MS7Dr1vjjLVUC9KiwSbM3gg6emfd2yuA93ihv +Pe3mffzLIiQa4mRE3wtGcioC43nWuV2K2e1KjxeFg07JhrezA/1Cak505ab/tmvP +4YmjR5c44+yL/YcQ3HdFgs4mV+nVbptRXvRcPpolJsgxPccGNdvHhsoR4gwXMS3F +RRPD2z6x8xeN73Q4KH3bm01swQdwFBZbWVfmUGLxvN7leCdfs9+iFJyqHiCIB6Iv +mQfp8F0IAOwSo8JhWN+V1dwML4EkIrM8wUb4yecNLkyR6TpPH/qXx4PxVMC+vy6x +sCtjeHIwKE+9vqnlhd5zOYh7qYXEJtYwdeDDmDbL8oks1LFfd+FyAuZXY33DLwn0 +cRYsr2OEZmaajqUB3NVmj3H4uJBN9+paFHyFSXrH68K1Fk2o3n+RSf2EiX+eICwI +L6rqoF5sSVUghBWdNegV7qfy4anwTQwrIMGjgU5S6PKW0Dr/3iO5z3qQpGPAj5OW +ATqPWkDICLbObPxD5cJlyyNE2wCA9VVc6/1d6w4EVwSq9h3/WTpATEreXXxTGptd +LNiTA1nmakBYNO2Iyo3djhaqBdWjk+EIAKtVEnJH9FAVwWOvaj1RoZMA5DnDMo7e +SnhrCXl8AL7Z1WInEaybasTJXn1uQ8xY52Ua4b8cbuEKRKzw/70NesFRoMLYoHTO +dyeszvhoDHberpGRTciVmpMu7Hyi33rM31K9epA4ib6QbbCHnxkWOZB+Bhgj1hJ8 +xb4RBYWiWpAYcg0+DAC3w9gfxQhtUlZPIbmbrBmrVkO2GVGUj8kH6k4UV6kUHEGY +HQWQR0HcbKcXW81ZXCCD0l7ROuEWQtTe5Jw7dJ4/QFuqZnPutXVRNOZqpl6eRShw +7X2/a29VXBpmHA95a88rSQsL+qm7Fb3prqRmuMCtrUZgFz7HLSTuUMR867QcTGVh +cCBUZXN0IEtleSA8bGVhcEBsZWFwLnNlPokCNwQTAQgAIQUCUL352QIbAwULCQgH +AwUVCgkICwUWAgMBAAIeAQIXgAAKCRAvRV4oJNGN30+xEACh9yLkZ4jqW0/wwyIM +MI896MQf1tAwzMj16MJYUjrjNK4Bn57QaQW926HsxF8C/OjT0MTRhq7heYZJnnEo +rj0rzpkJapUveTRkKeoTRtGGigqJYfkOTU7KRVwgJBXIfaKlI3tC3cX0j0H1fVKX +hLxsj5pNSPRCVf2A5mePg44HtXe6oVWSJ8+EcdTa0shf03NhAtFaY0BbFGPSm9mA +QUe4rxugwXPLctIyV4uweFo5BXFBCb4kKTBdnQi3aJwnoWLNT6rDdTe4/nhY0Hfo +alTCYGLkhio77gBHwpTOjEMO/hZhcDMi4CvxMPw7bRxAwq4u+0j0pDhkiLcQs4U4 +Ou/fH+pia+1nF5h19cNVXIm+RX2fL0wxVYc/14AIAK3YT6PVev9XYEkogSj0P7Kb +HKOruYpnToXJBERNJZwGL1U+ihPNUyroRf29t7u8flnXsOpCtBEIWAO8Muy5pWjV +3O6zAUCfWetAieCQ7WrQVmdJDa7dlX3Qx1XagUzqZdAq2jVI1hOWDA2rKytnReSF +/A97rmLaWZ8aoNCs8i4NLcy9Lbzi9QtornYGVCEmTTym0tM9L/mn7gAJ8dqUwt7n +s24dibfElky4ZZeItD+D7OZGeh0FDuejvv2dXFqL1/pkHpGBZhEckg0fZ95NbgMC +4pSZkZnqRpr2GwfB5aFfB6sIIJ0HGARQvfnZARAAtCP8Z9bm6KzIA7wbXx9LBIcJ +1wQvOPf99s4nOrnQev9xH5PZ820qS9xUjrlyE2bGYAhz5Cmq56ENs7THErIdZHtQ +uYEBprO+VFZjP50vtmCOL2PDl/xgv6J9r1Mp3KnR/m0esR+YceDW1qX07IkB5s+Q +us80v5LmmxnWcikWmR7dt1kOyV/+M6Y6mwvfQ4x3D/QUpO7SfMCOG5DGA7hVUHU/ +Tuh8MihmMFFOLAEEQI+1wkxr1W09HaYCcB+EZqxLSaBwMeFoYPJie9dBNBgps39o +6pDbjsO+or4JNuyoHvh8NNI5iY0IR2NMlW4mqCcHEazys2koxFTYK6YD95Vz0RkA +K4BErCDk3lVR0PH4StmLU3gmPayIjvi9Dl9saPRyu4Xx2WVi+q6spl3ckn4c4f3+ +iD8hxVp74+wa5ew0fIXjIpMoHCar/nndsse4i8glCddINdiOPPmhI9Wi3nT+5Z2t +9omPP2dEh0CzR+j1zvUpT3KtmhVICqhO+QP9BTJOwrp81NTlq9mbUyzTtVk/9dy3 +zoYbhKvY08k6LJ9FsQYySqtfJZ4cwl5WsOhALWwOwlMLA9wkz0eemgFxStyOylzl +QKoIK7zHuU6XYOXa32KSPIWaLy+WgIG/u2ObWtdE3CXVIUuSt5BQFnv7XVNHJllD +Az9VDEkOSYOiSEFVoUsAEQEAAQAP/1AagnZQZyzHDEgw4QELAspYHCWLXE5aZInX +wTUJhK31IgIXNn9bJ0hFiSpQR2xeMs9oYtRuPOu0P8oOFMn4/z374fkjZy8QVY3e +PlL+3EUeqYtkMwlGNmVw5a/NbNuNfm5Darb7pEfbYd1gPcni4MAYw7R2SG/57GbC +9gucvspHIfOSfBNLBthDzmK8xEKe1yD2eimfc2T7IRYb6hmkYfeds5GsqvGI6mwI +85h4uUHWRc5JOlhVM6yX8hSWx0L60Z3DZLChmc8maWnFXd7C8eQ6P1azJJbW71Ih +7CoK0XW4LE82vlQurSRFgTwfl7wFYszW2bOzCuhHDDtYnwH86Nsu0DC78ZVRnvxn +E8Ke/AJgrdhIOo4UAyR+aZD2+2mKd7/waOUTUrUtTzc7i8N3YXGi/EIaNReBXaq+ +ZNOp24BlFzRp+FCF/pptDW9HjPdiV09x0DgICmeZS4Gq/4vFFIahWctg52NGebT0 +Idxngjj+xDtLaZlLQoOz0n5ByjO/Wi0ANmMv1sMKCHhGvdaSws2/PbMR2r4caj8m +KXpIgdinM/wUzHJ5pZyF2U/qejsRj8Kw8KH/tfX4JCLhiaP/mgeTuWGDHeZQERAT +xPmRFHaLP9/ZhvGNh6okIYtrKjWTLGoXvKLHcrKNisBLSq+P2WeFrlme1vjvJMo/ +jPwLT5o9CADQmcbKZ+QQ1ZM9v99iDZol7SAMZX43JC019sx6GK0u6xouJBcLfeB4 +OXacTgmSYdTa9RM9fbfVpti01tJ84LV2SyL/VJq/enJF4XQPSynT/tFTn1PAor6o +tEAAd8fjKdJ6LnD5wb92SPHfQfXqI84rFEO8rUNIE/1ErT6DYifDzVCbfD2KZdoF +cOSp7TpD77sY1bs74ocBX5ejKtd+aH99D78bJSMM4pSDZsIEwnomkBHTziubPwJb +OwnATy0LmSMAWOw5rKbsh5nfwCiUTM20xp0t5JeXd+wPVWbpWqI2EnkCEN+RJr9i +7dp/ymDQ+Yt5wrsN3NwoyiexPOG91WQVCADdErHsnglVZZq9Z8Wx7KwecGCUurJ2 +H6lKudv5YOxPnAzqZS5HbpZd/nRTMZh2rdXCr5m2YOuewyYjvM757AkmUpM09zJX +MQ1S67/UX2y8/74TcRF97Ncx9HeELs92innBRXoFitnNguvcO6Esx4BTe1OdU6qR +ER3zAmVf22Le9ciXbu24DN4mleOH+OmBx7X2PqJSYW9GAMTsRB081R6EWKH7romQ +waxFrZ4DJzZ9ltyosEJn5F32StyLrFxpcrdLUoEaclZCv2qka7sZvi0EvovDVEBU +e10jOx9AOwf8Gj2ufhquQ6qgVYCzbP+YrodtkFrXRS3IsljIchj1M2ffB/0bfoUs +rtER9pLvYzCjBPg8IfGLw0o754Qbhh/ReplCRTusP/fQMybvCvfxreS3oyEriu/G +GufRomjewZ8EMHDIgUsLcYo2UHZsfF7tcazgxMGmMvazp4r8vpgrvW/8fIN/6Adu +tF+WjWDTvJLFJCe6O+BFJOWrssNrrra1zGtLC1s8s+Wfpe+bGPL5zpHeebGTwH1U +22eqgJArlEKxrfarz7W5+uHZJHSjF/K9ZvunLGD0n9GOPMpji3UO3zeM8IYoWn7E +/EWK1XbjnssNemeeTZ+sDh+qrD7BOi+vCX1IyBxbfqnQfJZvmcPWpruy1UsO+aIC +0GY8Jr3OL69dDQ21jueJAh8EGAEIAAkFAlC9+dkCGwwACgkQL0VeKCTRjd9HCw/+ +LQSVgLLF4ulYlPCjWIIuQwrPbJfWUVVr2dPUFVM85DCv8gBzk5c121snXh9Swovm +laBbw6ate3BmbXLh64jVE9Za5sbTWi7PCcbO/bpRy4d6oLmitmNw6cq0vjTLxUYy +bwuiJxWREkfxuU85EKdouN062YDevH+/YResmlJrcCE7LRlJFeRlKsrrwBU3BqYd +GgFJjKjQC1peeQ9fj62Y7xfwE9+PXbkiWO5u/Bk8hb1VZH1SoIRU98NHVcp6BVvp +VK0jLAXuSauSczULmpRjbyt1lhaAqivDTWEEZXiNNbRyp17c3nVdPWOcgBr42hdQ +z25CgZgyLCsvu82wuXLKJblrIPJX3Yf+si6KqEWBsmwdOWybsjygaF5HvzgFqAAD +U0goPWoQ71PorP2XOUNp5ZLkBQp5etvtkksjVNMIhnHn8PGMuoxO39EUGlWj2B5l +Cu8tSosAzB1pS8NcLZzoNoI9dOHrmgJmP+GrOUkcf5GhNZbMoj4GNfGBRYX0SZlQ +GuDrwNKYj73C4MWyNnnUFyq8nDHJ/G1NpaF2hiof9RBL4PUU/f92JkceXPBXA8gL +Mz2ig1OButwPPLFGQhWqxXAGrsS3Ny+BhTJfnfIbbkaLLphBpDZm1D9XKbAUvdd1 +RZXoH+FTg9UAW87eqU610npOkT6cRaBxaMK/mDtGNdc= +=JTFu +-----END PGP PRIVATE KEY BLOCK----- +""" + +# key 7FEE575A: public key "anotheruser <anotheruser@leap.se>" +PUBLIC_KEY_2 = """ +-----BEGIN PGP PUBLIC KEY BLOCK----- +Version: GnuPG v1.4.10 (GNU/Linux) + +mI0EUYwJXgEEAMbTKHuPJ5/Gk34l9Z06f+0WCXTDXdte1UBoDtZ1erAbudgC4MOR +gquKqoj3Hhw0/ILqJ88GcOJmKK/bEoIAuKaqlzDF7UAYpOsPZZYmtRfPC2pTCnXq +Z1vdeqLwTbUspqXflkCkFtfhGKMq5rH8GV5a3tXZkRWZhdNwhVXZagC3ABEBAAG0 +IWFub3RoZXJ1c2VyIDxhbm90aGVydXNlckBsZWFwLnNlPoi4BBMBAgAiBQJRjAle +AhsDBgsJCAcDAgYVCAIJCgsEFgIDAQIeAQIXgAAKCRB/nfpof+5XWotuA/4tLN4E +gUr7IfLy2HkHAxzw7A4rqfMN92DIM9mZrDGaWRrOn3aVF7VU1UG7MDkHfPvp/cFw +ezoCw4s4IoHVc/pVlOkcHSyt4/Rfh248tYEJmFCJXGHpkK83VIKYJAithNccJ6Q4 +JE/o06Mtf4uh/cA1HUL4a4ceqUhtpLJULLeKo7iNBFGMCV4BBADsyQI7GR0wSAxz +VayLjuPzgT+bjbFeymIhjuxKIEwnIKwYkovztW+4bbOcQs785k3Lp6RzvigTpQQt +Z/hwcLOqZbZw8t/24+D+Pq9mMP2uUvCFFqLlVvA6D3vKSQ/XNN+YB919WQ04jh63 +yuRe94WenT1RJd6xU1aaUff4rKizuQARAQABiJ8EGAECAAkFAlGMCV4CGwwACgkQ +f536aH/uV1rPZQQAqCzRysOlu8ez7PuiBD4SebgRqWlxa1TF1ujzfLmuPivROZ2X +Kw5aQstxgGSjoB7tac49s0huh4X8XK+BtJBfU84JS8Jc2satlfwoyZ35LH6sDZck +I+RS/3we6zpMfHs3vvp9xgca6ZupQxivGtxlJs294TpJorx+mFFqbV17AzQ= +=Thdu +-----END PGP PUBLIC KEY BLOCK----- +""" + +PRIVATE_KEY_2 = """ +-----BEGIN PGP PRIVATE KEY BLOCK----- +Version: GnuPG v1.4.10 (GNU/Linux) + +lQHYBFGMCV4BBADG0yh7jyefxpN+JfWdOn/tFgl0w13bXtVAaA7WdXqwG7nYAuDD +kYKriqqI9x4cNPyC6ifPBnDiZiiv2xKCALimqpcwxe1AGKTrD2WWJrUXzwtqUwp1 +6mdb3Xqi8E21LKal35ZApBbX4RijKuax/BleWt7V2ZEVmYXTcIVV2WoAtwARAQAB +AAP7BLuSAx7tOohnimEs74ks8l/L6dOcsFQZj2bqs4AoY3jFe7bV0tHr4llypb/8 +H3/DYvpf6DWnCjyUS1tTnXSW8JXtx01BUKaAufSmMNg9blKV6GGHlT/Whe9uVyks +7XHk/+9mebVMNJ/kNlqq2k+uWqJohzC8WWLRK+d1tBeqDsECANZmzltPaqUsGV5X +C3zszE3tUBgptV/mKnBtopKi+VH+t7K6fudGcG+bAcZDUoH/QVde52mIIjjIdLje +uajJuHUCAO1mqh+vPoGv4eBLV7iBo3XrunyGXiys4a39eomhxTy3YktQanjjx+ty +GltAGCs5PbWGO6/IRjjvd46wh53kzvsCAO0J97gsWhzLuFnkxFAJSPk7RRlyl7lI +1XS/x0Og6j9XHCyY1OYkfBm0to3UlCfkgirzCYlTYObCofzdKFIPDmSqHbQhYW5v +dGhlcnVzZXIgPGFub3RoZXJ1c2VyQGxlYXAuc2U+iLgEEwECACIFAlGMCV4CGwMG +CwkIBwMCBhUIAgkKCwQWAgMBAh4BAheAAAoJEH+d+mh/7ldai24D/i0s3gSBSvsh +8vLYeQcDHPDsDiup8w33YMgz2ZmsMZpZGs6fdpUXtVTVQbswOQd8++n9wXB7OgLD +izgigdVz+lWU6RwdLK3j9F+Hbjy1gQmYUIlcYemQrzdUgpgkCK2E1xwnpDgkT+jT +oy1/i6H9wDUdQvhrhx6pSG2kslQst4qjnQHYBFGMCV4BBADsyQI7GR0wSAxzVayL +juPzgT+bjbFeymIhjuxKIEwnIKwYkovztW+4bbOcQs785k3Lp6RzvigTpQQtZ/hw +cLOqZbZw8t/24+D+Pq9mMP2uUvCFFqLlVvA6D3vKSQ/XNN+YB919WQ04jh63yuRe +94WenT1RJd6xU1aaUff4rKizuQARAQABAAP9EyElqJ3dq3EErXwwT4mMnbd1SrVC +rUJrNWQZL59mm5oigS00uIyR0SvusOr+UzTtd8ysRuwHy5d/LAZsbjQStaOMBILx +77TJveOel0a1QK0YSMF2ywZMCKvquvjli4hAtWYz/EwfuzQN3t23jc5ny+GqmqD2 +3FUxLJosFUfLNmECAO9KhVmJi+L9dswIs+2Dkjd1eiRQzNOEVffvYkGYZyKxNiXF +UA5kvyZcB4iAN9sWCybE4WHZ9jd4myGB0MPDGxkCAP1RsXJbbuD6zS7BXe5gwunO +2q4q7ptdSl/sJYQuTe1KNP5d/uGsvlcFfsYjpsopasPjFBIncc/2QThMKlhoEaEB +/0mVAxpT6SrEvUbJ18z7kna24SgMPr3OnPMxPGfvNLJY/Xv/A17YfoqjmByCvsKE +JCDjopXtmbcrZyoEZbEht9mko4ifBBgBAgAJBQJRjAleAhsMAAoJEH+d+mh/7lda +z2UEAKgs0crDpbvHs+z7ogQ+Enm4EalpcWtUxdbo83y5rj4r0TmdlysOWkLLcYBk +o6Ae7WnOPbNIboeF/FyvgbSQX1POCUvCXNrGrZX8KMmd+Sx+rA2XJCPkUv98Hus6 +THx7N776fcYHGumbqUMYrxrcZSbNveE6SaK8fphRam1dewM0 +=a5gs +-----END PGP PRIVATE KEY BLOCK----- +""" diff --git a/mail/src/leap/mail/testing/__init__.py~ b/mail/src/leap/mail/testing/__init__.py~ new file mode 100644 index 00000000..ffaadd88 --- /dev/null +++ b/mail/src/leap/mail/testing/__init__.py~ @@ -0,0 +1,358 @@ +# -*- coding: utf-8 -*- +# __init__.py +# Copyright (C) 2013 LEAP +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see <http://www.gnu.org/licenses/>. +""" +Base classes and keys for leap.mail tests. +""" +import os +import distutils.spawn +from mock import Mock +from twisted.internet.defer import gatherResults, succeed +from twisted.trial import unittest +from twisted.web.client import Response +from twisted.internet import defer +from twisted.python import log + + +from leap.soledad.client import Soledad +from leap.keymanager import KeyManager + + +from leap.common.testing.basetest import BaseLeapTest + +ADDRESS = 'leap@leap.se' +ADDRESS_2 = 'anotheruser@leap.se' + +class defaultMockSharedDB(object): + get_doc = Mock(return_value=None) + put_doc = Mock(side_effect=None) + open = Mock(return_value=None) + close = Mock(return_value=None) + syncable = True + + def __call__(self): + return self + + +class KeyManagerWithSoledadTestCase(unittest.TestCase, BaseLeapTest): + + def setUp(self): + self.gpg_binary_path = self._find_gpg() + + self._soledad = Soledad( + u"leap@leap.se", + u"123456", + secrets_path=self.tempdir + "/secret.gpg", + local_db_path=self.tempdir + "/soledad.u1db", + server_url='', + cert_file=None, + auth_token=None, + shared_db=defaultMockSharedDB(), + syncable=False) + + self.km = self._key_manager() + + class Response(object): + code = 200 + phrase = '' + def deliverBody(self, x): + return + + # XXX why the fuck is this needed? ------------------------ + self.km._async_client_pinned.request = Mock( + return_value=defer.succeed(Response())) + #self.km._async_client.request = Mock(return_value='') + #self.km._async_client_pinned.request = Mock( + #return_value='') + # ------------------------------------------------------- + + d1 = self.km.put_raw_key(PRIVATE_KEY, ADDRESS) + d2 = self.km.put_raw_key(PRIVATE_KEY_2, ADDRESS_2) + return gatherResults([d1, d2]) + + def tearDown(self): + km = self._key_manager() + # wait for the indexes to be ready for the tear down + d = km._openpgp.deferred_init + d.addCallback(lambda _: self.delete_all_keys(km)) + d.addCallback(lambda _: self._soledad.close()) + return d + + def delete_all_keys(self, km): + def delete_keys(keys): + deferreds = [] + for key in keys: + d = km._openpgp.delete_key(key) + deferreds.append(d) + return gatherResults(deferreds) + + def check_deleted(_, private): + d = km.get_all_keys(private=private) + d.addCallback(lambda keys: self.assertEqual(keys, [])) + return d + + deferreds = [] + for private in [True, False]: + d = km.get_all_keys(private=private) + d.addCallback(delete_keys) + d.addCallback(check_deleted, private) + deferreds.append(d) + return gatherResults(deferreds) + + def _key_manager(self, user=ADDRESS, url='', token=None, + ca_cert_path=None): + return KeyManager(user, url, self._soledad, token=token, + gpgbinary=self.gpg_binary_path, + ca_cert_path=ca_cert_path) + + def _find_gpg(self): + gpg_path = distutils.spawn.find_executable('gpg') + if gpg_path is not None: + return os.path.realpath(gpg_path) + else: + return "/usr/bin/gpg" + + def get_public_binary_key(self): + with open(PATH + '/fixtures/public_key.bin', 'r') as binary_public_key: + return binary_public_key.read() + + def get_private_binary_key(self): + with open( + PATH + '/fixtures/private_key.bin', 'r') as binary_private_key: + return binary_private_key.read() + + +# key 24D18DDF: public key "Leap Test Key <leap@leap.se>" +KEY_FINGERPRINT = "E36E738D69173C13D709E44F2F455E2824D18DDF" +PUBLIC_KEY = """ +-----BEGIN PGP PUBLIC KEY BLOCK----- +Version: GnuPG v1.4.10 (GNU/Linux) + +mQINBFC9+dkBEADNRfwV23TWEoGc/x0wWH1P7PlXt8MnC2Z1kKaKKmfnglVrpOiz +iLWoiU58sfZ0L5vHkzXHXCBf6Eiy/EtUIvdiWAn+yASJ1mk5jZTBKO/WMAHD8wTO +zpMsFmWyg3xc4DkmFa9KQ5EVU0o/nqPeyQxNMQN7px5pPwrJtJFmPxnxm+aDkPYx +irDmz/4DeDNqXliazGJKw7efqBdlwTHkl9Akw2gwy178pmsKwHHEMOBOFFvX61AT +huKqHYmlCGSliwbrJppTG7jc1/ls3itrK+CWTg4txREkSpEVmfcASvw/ZqLbjgfs +d/INMwXnR9U81O8+7LT6yw/ca4ppcFoJD7/XJbkRiML6+bJ4Dakiy6i727BzV17g +wI1zqNvm5rAhtALKfACha6YO43aJzairO4II1wxVHvRDHZn2IuKDDephQ3Ii7/vb +hUOf6XCSmchkAcpKXUOvbxm1yfB1LRa64mMc2RcZxf4mW7KQkulBsdV5QG2276lv +U2UUy2IutXcGP5nXC+f6sJJGJeEToKJ57yiO/VWJFjKN8SvP+7AYsQSqINUuEf6H +T5gCPCraGMkTUTPXrREvu7NOohU78q6zZNaL3GW8ai7eSeANSuQ8Vzffx7Wd8Y7i +Pw9sYj0SMFs1UgjbuL6pO5ueHh+qyumbtAq2K0Bci0kqOcU4E9fNtdiovQARAQAB +tBxMZWFwIFRlc3QgS2V5IDxsZWFwQGxlYXAuc2U+iQI3BBMBCAAhBQJQvfnZAhsD +BQsJCAcDBRUKCQgLBRYCAwEAAh4BAheAAAoJEC9FXigk0Y3fT7EQAKH3IuRniOpb +T/DDIgwwjz3oxB/W0DDMyPXowlhSOuM0rgGfntBpBb3boezEXwL86NPQxNGGruF5 +hkmecSiuPSvOmQlqlS95NGQp6hNG0YaKColh+Q5NTspFXCAkFch9oqUje0LdxfSP +QfV9UpeEvGyPmk1I9EJV/YDmZ4+Djge1d7qhVZInz4Rx1NrSyF/Tc2EC0VpjQFsU +Y9Kb2YBBR7ivG6DBc8ty0jJXi7B4WjkFcUEJviQpMF2dCLdonCehYs1PqsN1N7j+ +eFjQd+hqVMJgYuSGKjvuAEfClM6MQw7+FmFwMyLgK/Ew/DttHEDCri77SPSkOGSI +txCzhTg6798f6mJr7WcXmHX1w1Vcib5FfZ8vTDFVhz/XgAgArdhPo9V6/1dgSSiB +KPQ/spsco6u5imdOhckERE0lnAYvVT6KE81TKuhF/b23u7x+Wdew6kK0EQhYA7wy +7LmlaNXc7rMBQJ9Z60CJ4JDtatBWZ0kNrt2VfdDHVdqBTOpl0CraNUjWE5YMDasr +K2dF5IX8D3uuYtpZnxqg0KzyLg0tzL0tvOL1C2iudgZUISZNPKbS0z0v+afuAAnx +2pTC3uezbh2Jt8SWTLhll4i0P4Ps5kZ6HQUO56O+/Z1cWovX+mQekYFmERySDR9n +3k1uAwLilJmRmepGmvYbB8HloV8HqwgguQINBFC9+dkBEAC0I/xn1uborMgDvBtf +H0sEhwnXBC849/32zic6udB6/3Efk9nzbSpL3FSOuXITZsZgCHPkKarnoQ2ztMcS +sh1ke1C5gQGms75UVmM/nS+2YI4vY8OX/GC/on2vUyncqdH+bR6xH5hx4NbWpfTs +iQHmz5C6zzS/kuabGdZyKRaZHt23WQ7JX/4zpjqbC99DjHcP9BSk7tJ8wI4bkMYD +uFVQdT9O6HwyKGYwUU4sAQRAj7XCTGvVbT0dpgJwH4RmrEtJoHAx4Whg8mJ710E0 +GCmzf2jqkNuOw76ivgk27Kge+Hw00jmJjQhHY0yVbiaoJwcRrPKzaSjEVNgrpgP3 +lXPRGQArgESsIOTeVVHQ8fhK2YtTeCY9rIiO+L0OX2xo9HK7hfHZZWL6rqymXdyS +fhzh/f6IPyHFWnvj7Brl7DR8heMikygcJqv+ed2yx7iLyCUJ10g12I48+aEj1aLe +dP7lna32iY8/Z0SHQLNH6PXO9SlPcq2aFUgKqE75A/0FMk7CunzU1OWr2ZtTLNO1 +WT/13LfOhhuEq9jTyTosn0WxBjJKq18lnhzCXlaw6EAtbA7CUwsD3CTPR56aAXFK +3I7KXOVAqggrvMe5Tpdg5drfYpI8hZovL5aAgb+7Y5ta10TcJdUhS5K3kFAWe/td +U0cmWUMDP1UMSQ5Jg6JIQVWhSwARAQABiQIfBBgBCAAJBQJQvfnZAhsMAAoJEC9F +Xigk0Y3fRwsP/i0ElYCyxeLpWJTwo1iCLkMKz2yX1lFVa9nT1BVTPOQwr/IAc5OX +NdtbJ14fUsKL5pWgW8OmrXtwZm1y4euI1RPWWubG01ouzwnGzv26UcuHeqC5orZj +cOnKtL40y8VGMm8LoicVkRJH8blPORCnaLjdOtmA3rx/v2EXrJpSa3AhOy0ZSRXk +ZSrK68AVNwamHRoBSYyo0AtaXnkPX4+tmO8X8BPfj125IljubvwZPIW9VWR9UqCE +VPfDR1XKegVb6VStIywF7kmrknM1C5qUY28rdZYWgKorw01hBGV4jTW0cqde3N51 +XT1jnIAa+NoXUM9uQoGYMiwrL7vNsLlyyiW5ayDyV92H/rIuiqhFgbJsHTlsm7I8 +oGheR784BagAA1NIKD1qEO9T6Kz9lzlDaeWS5AUKeXrb7ZJLI1TTCIZx5/DxjLqM +Tt/RFBpVo9geZQrvLUqLAMwdaUvDXC2c6DaCPXTh65oCZj/hqzlJHH+RoTWWzKI+ +BjXxgUWF9EmZUBrg68DSmI+9wuDFsjZ51BcqvJwxyfxtTaWhdoYqH/UQS+D1FP3/ +diZHHlzwVwPICzM9ooNTgbrcDzyxRkIVqsVwBq7EtzcvgYUyX53yG25Giy6YQaQ2 +ZtQ/VymwFL3XdUWV6B/hU4PVAFvO3qlOtdJ6TpE+nEWgcWjCv5g7RjXX +=MuOY +-----END PGP PUBLIC KEY BLOCK----- +""" +PRIVATE_KEY = """ +-----BEGIN PGP PRIVATE KEY BLOCK----- +Version: GnuPG v1.4.10 (GNU/Linux) + +lQcYBFC9+dkBEADNRfwV23TWEoGc/x0wWH1P7PlXt8MnC2Z1kKaKKmfnglVrpOiz +iLWoiU58sfZ0L5vHkzXHXCBf6Eiy/EtUIvdiWAn+yASJ1mk5jZTBKO/WMAHD8wTO +zpMsFmWyg3xc4DkmFa9KQ5EVU0o/nqPeyQxNMQN7px5pPwrJtJFmPxnxm+aDkPYx +irDmz/4DeDNqXliazGJKw7efqBdlwTHkl9Akw2gwy178pmsKwHHEMOBOFFvX61AT +huKqHYmlCGSliwbrJppTG7jc1/ls3itrK+CWTg4txREkSpEVmfcASvw/ZqLbjgfs +d/INMwXnR9U81O8+7LT6yw/ca4ppcFoJD7/XJbkRiML6+bJ4Dakiy6i727BzV17g +wI1zqNvm5rAhtALKfACha6YO43aJzairO4II1wxVHvRDHZn2IuKDDephQ3Ii7/vb +hUOf6XCSmchkAcpKXUOvbxm1yfB1LRa64mMc2RcZxf4mW7KQkulBsdV5QG2276lv +U2UUy2IutXcGP5nXC+f6sJJGJeEToKJ57yiO/VWJFjKN8SvP+7AYsQSqINUuEf6H +T5gCPCraGMkTUTPXrREvu7NOohU78q6zZNaL3GW8ai7eSeANSuQ8Vzffx7Wd8Y7i +Pw9sYj0SMFs1UgjbuL6pO5ueHh+qyumbtAq2K0Bci0kqOcU4E9fNtdiovQARAQAB +AA/+JHtlL39G1wsH9R6UEfUQJGXR9MiIiwZoKcnRB2o8+DS+OLjg0JOh8XehtuCs +E/8oGQKtQqa5bEIstX7IZoYmYFiUQi9LOzIblmp2vxOm+HKkxa4JszWci2/ZmC3t +KtaA4adl9XVnshoQ7pijuCMUKB3naBEOAxd8s9d/JeReGIYkJErdrnVfNk5N71Ds +FmH5Ll3XtEDvgBUQP3nkA6QFjpsaB94FHjL3gDwum/cxzj6pCglcvHOzEhfY0Ddb +J967FozQTaf2JW3O+w3LOqtcKWpq87B7+O61tVidQPSSuzPjCtFF0D2LC9R/Hpky +KTMQ6CaKja4MPhjwywd4QPcHGYSqjMpflvJqi+kYIt8psUK/YswWjnr3r4fbuqVY +VhtiHvnBHQjz135lUqWvEz4hM3Xpnxydx7aRlv5NlevK8+YIO5oFbWbGNTWsPZI5 +jpoFBpSsnR1Q5tnvtNHauvoWV+XN2qAOBTG+/nEbDYH6Ak3aaE9jrpTdYh0CotYF +q7csANsDy3JvkAzeU6WnYpsHHaAjqOGyiZGsLej1UcXPFMosE/aUo4WQhiS8Zx2c +zOVKOi/X5vQ2GdNT9Qolz8AriwzsvFR+bxPzyd8V6ALwDsoXvwEYinYBKK8j0OPv +OOihSR6HVsuP9NUZNU9ewiGzte/+/r6pNXHvR7wTQ8EWLcEIAN6Zyrb0bHZTIlxt +VWur/Ht2mIZrBaO50qmM5RD3T5oXzWXi/pjLrIpBMfeZR9DWfwQwjYzwqi7pxtYx +nJvbMuY505rfnMoYxb4J+cpRXV8MS7Dr1vjjLVUC9KiwSbM3gg6emfd2yuA93ihv +Pe3mffzLIiQa4mRE3wtGcioC43nWuV2K2e1KjxeFg07JhrezA/1Cak505ab/tmvP +4YmjR5c44+yL/YcQ3HdFgs4mV+nVbptRXvRcPpolJsgxPccGNdvHhsoR4gwXMS3F +RRPD2z6x8xeN73Q4KH3bm01swQdwFBZbWVfmUGLxvN7leCdfs9+iFJyqHiCIB6Iv +mQfp8F0IAOwSo8JhWN+V1dwML4EkIrM8wUb4yecNLkyR6TpPH/qXx4PxVMC+vy6x +sCtjeHIwKE+9vqnlhd5zOYh7qYXEJtYwdeDDmDbL8oks1LFfd+FyAuZXY33DLwn0 +cRYsr2OEZmaajqUB3NVmj3H4uJBN9+paFHyFSXrH68K1Fk2o3n+RSf2EiX+eICwI +L6rqoF5sSVUghBWdNegV7qfy4anwTQwrIMGjgU5S6PKW0Dr/3iO5z3qQpGPAj5OW +ATqPWkDICLbObPxD5cJlyyNE2wCA9VVc6/1d6w4EVwSq9h3/WTpATEreXXxTGptd +LNiTA1nmakBYNO2Iyo3djhaqBdWjk+EIAKtVEnJH9FAVwWOvaj1RoZMA5DnDMo7e +SnhrCXl8AL7Z1WInEaybasTJXn1uQ8xY52Ua4b8cbuEKRKzw/70NesFRoMLYoHTO +dyeszvhoDHberpGRTciVmpMu7Hyi33rM31K9epA4ib6QbbCHnxkWOZB+Bhgj1hJ8 +xb4RBYWiWpAYcg0+DAC3w9gfxQhtUlZPIbmbrBmrVkO2GVGUj8kH6k4UV6kUHEGY +HQWQR0HcbKcXW81ZXCCD0l7ROuEWQtTe5Jw7dJ4/QFuqZnPutXVRNOZqpl6eRShw +7X2/a29VXBpmHA95a88rSQsL+qm7Fb3prqRmuMCtrUZgFz7HLSTuUMR867QcTGVh +cCBUZXN0IEtleSA8bGVhcEBsZWFwLnNlPokCNwQTAQgAIQUCUL352QIbAwULCQgH +AwUVCgkICwUWAgMBAAIeAQIXgAAKCRAvRV4oJNGN30+xEACh9yLkZ4jqW0/wwyIM +MI896MQf1tAwzMj16MJYUjrjNK4Bn57QaQW926HsxF8C/OjT0MTRhq7heYZJnnEo +rj0rzpkJapUveTRkKeoTRtGGigqJYfkOTU7KRVwgJBXIfaKlI3tC3cX0j0H1fVKX +hLxsj5pNSPRCVf2A5mePg44HtXe6oVWSJ8+EcdTa0shf03NhAtFaY0BbFGPSm9mA +QUe4rxugwXPLctIyV4uweFo5BXFBCb4kKTBdnQi3aJwnoWLNT6rDdTe4/nhY0Hfo +alTCYGLkhio77gBHwpTOjEMO/hZhcDMi4CvxMPw7bRxAwq4u+0j0pDhkiLcQs4U4 +Ou/fH+pia+1nF5h19cNVXIm+RX2fL0wxVYc/14AIAK3YT6PVev9XYEkogSj0P7Kb +HKOruYpnToXJBERNJZwGL1U+ihPNUyroRf29t7u8flnXsOpCtBEIWAO8Muy5pWjV +3O6zAUCfWetAieCQ7WrQVmdJDa7dlX3Qx1XagUzqZdAq2jVI1hOWDA2rKytnReSF +/A97rmLaWZ8aoNCs8i4NLcy9Lbzi9QtornYGVCEmTTym0tM9L/mn7gAJ8dqUwt7n +s24dibfElky4ZZeItD+D7OZGeh0FDuejvv2dXFqL1/pkHpGBZhEckg0fZ95NbgMC +4pSZkZnqRpr2GwfB5aFfB6sIIJ0HGARQvfnZARAAtCP8Z9bm6KzIA7wbXx9LBIcJ +1wQvOPf99s4nOrnQev9xH5PZ820qS9xUjrlyE2bGYAhz5Cmq56ENs7THErIdZHtQ +uYEBprO+VFZjP50vtmCOL2PDl/xgv6J9r1Mp3KnR/m0esR+YceDW1qX07IkB5s+Q +us80v5LmmxnWcikWmR7dt1kOyV/+M6Y6mwvfQ4x3D/QUpO7SfMCOG5DGA7hVUHU/ +Tuh8MihmMFFOLAEEQI+1wkxr1W09HaYCcB+EZqxLSaBwMeFoYPJie9dBNBgps39o +6pDbjsO+or4JNuyoHvh8NNI5iY0IR2NMlW4mqCcHEazys2koxFTYK6YD95Vz0RkA +K4BErCDk3lVR0PH4StmLU3gmPayIjvi9Dl9saPRyu4Xx2WVi+q6spl3ckn4c4f3+ +iD8hxVp74+wa5ew0fIXjIpMoHCar/nndsse4i8glCddINdiOPPmhI9Wi3nT+5Z2t +9omPP2dEh0CzR+j1zvUpT3KtmhVICqhO+QP9BTJOwrp81NTlq9mbUyzTtVk/9dy3 +zoYbhKvY08k6LJ9FsQYySqtfJZ4cwl5WsOhALWwOwlMLA9wkz0eemgFxStyOylzl +QKoIK7zHuU6XYOXa32KSPIWaLy+WgIG/u2ObWtdE3CXVIUuSt5BQFnv7XVNHJllD +Az9VDEkOSYOiSEFVoUsAEQEAAQAP/1AagnZQZyzHDEgw4QELAspYHCWLXE5aZInX +wTUJhK31IgIXNn9bJ0hFiSpQR2xeMs9oYtRuPOu0P8oOFMn4/z374fkjZy8QVY3e +PlL+3EUeqYtkMwlGNmVw5a/NbNuNfm5Darb7pEfbYd1gPcni4MAYw7R2SG/57GbC +9gucvspHIfOSfBNLBthDzmK8xEKe1yD2eimfc2T7IRYb6hmkYfeds5GsqvGI6mwI +85h4uUHWRc5JOlhVM6yX8hSWx0L60Z3DZLChmc8maWnFXd7C8eQ6P1azJJbW71Ih +7CoK0XW4LE82vlQurSRFgTwfl7wFYszW2bOzCuhHDDtYnwH86Nsu0DC78ZVRnvxn +E8Ke/AJgrdhIOo4UAyR+aZD2+2mKd7/waOUTUrUtTzc7i8N3YXGi/EIaNReBXaq+ +ZNOp24BlFzRp+FCF/pptDW9HjPdiV09x0DgICmeZS4Gq/4vFFIahWctg52NGebT0 +Idxngjj+xDtLaZlLQoOz0n5ByjO/Wi0ANmMv1sMKCHhGvdaSws2/PbMR2r4caj8m +KXpIgdinM/wUzHJ5pZyF2U/qejsRj8Kw8KH/tfX4JCLhiaP/mgeTuWGDHeZQERAT +xPmRFHaLP9/ZhvGNh6okIYtrKjWTLGoXvKLHcrKNisBLSq+P2WeFrlme1vjvJMo/ +jPwLT5o9CADQmcbKZ+QQ1ZM9v99iDZol7SAMZX43JC019sx6GK0u6xouJBcLfeB4 +OXacTgmSYdTa9RM9fbfVpti01tJ84LV2SyL/VJq/enJF4XQPSynT/tFTn1PAor6o +tEAAd8fjKdJ6LnD5wb92SPHfQfXqI84rFEO8rUNIE/1ErT6DYifDzVCbfD2KZdoF +cOSp7TpD77sY1bs74ocBX5ejKtd+aH99D78bJSMM4pSDZsIEwnomkBHTziubPwJb +OwnATy0LmSMAWOw5rKbsh5nfwCiUTM20xp0t5JeXd+wPVWbpWqI2EnkCEN+RJr9i +7dp/ymDQ+Yt5wrsN3NwoyiexPOG91WQVCADdErHsnglVZZq9Z8Wx7KwecGCUurJ2 +H6lKudv5YOxPnAzqZS5HbpZd/nRTMZh2rdXCr5m2YOuewyYjvM757AkmUpM09zJX +MQ1S67/UX2y8/74TcRF97Ncx9HeELs92innBRXoFitnNguvcO6Esx4BTe1OdU6qR +ER3zAmVf22Le9ciXbu24DN4mleOH+OmBx7X2PqJSYW9GAMTsRB081R6EWKH7romQ +waxFrZ4DJzZ9ltyosEJn5F32StyLrFxpcrdLUoEaclZCv2qka7sZvi0EvovDVEBU +e10jOx9AOwf8Gj2ufhquQ6qgVYCzbP+YrodtkFrXRS3IsljIchj1M2ffB/0bfoUs +rtER9pLvYzCjBPg8IfGLw0o754Qbhh/ReplCRTusP/fQMybvCvfxreS3oyEriu/G +GufRomjewZ8EMHDIgUsLcYo2UHZsfF7tcazgxMGmMvazp4r8vpgrvW/8fIN/6Adu +tF+WjWDTvJLFJCe6O+BFJOWrssNrrra1zGtLC1s8s+Wfpe+bGPL5zpHeebGTwH1U +22eqgJArlEKxrfarz7W5+uHZJHSjF/K9ZvunLGD0n9GOPMpji3UO3zeM8IYoWn7E +/EWK1XbjnssNemeeTZ+sDh+qrD7BOi+vCX1IyBxbfqnQfJZvmcPWpruy1UsO+aIC +0GY8Jr3OL69dDQ21jueJAh8EGAEIAAkFAlC9+dkCGwwACgkQL0VeKCTRjd9HCw/+ +LQSVgLLF4ulYlPCjWIIuQwrPbJfWUVVr2dPUFVM85DCv8gBzk5c121snXh9Swovm +laBbw6ate3BmbXLh64jVE9Za5sbTWi7PCcbO/bpRy4d6oLmitmNw6cq0vjTLxUYy +bwuiJxWREkfxuU85EKdouN062YDevH+/YResmlJrcCE7LRlJFeRlKsrrwBU3BqYd +GgFJjKjQC1peeQ9fj62Y7xfwE9+PXbkiWO5u/Bk8hb1VZH1SoIRU98NHVcp6BVvp +VK0jLAXuSauSczULmpRjbyt1lhaAqivDTWEEZXiNNbRyp17c3nVdPWOcgBr42hdQ +z25CgZgyLCsvu82wuXLKJblrIPJX3Yf+si6KqEWBsmwdOWybsjygaF5HvzgFqAAD +U0goPWoQ71PorP2XOUNp5ZLkBQp5etvtkksjVNMIhnHn8PGMuoxO39EUGlWj2B5l +Cu8tSosAzB1pS8NcLZzoNoI9dOHrmgJmP+GrOUkcf5GhNZbMoj4GNfGBRYX0SZlQ +GuDrwNKYj73C4MWyNnnUFyq8nDHJ/G1NpaF2hiof9RBL4PUU/f92JkceXPBXA8gL +Mz2ig1OButwPPLFGQhWqxXAGrsS3Ny+BhTJfnfIbbkaLLphBpDZm1D9XKbAUvdd1 +RZXoH+FTg9UAW87eqU610npOkT6cRaBxaMK/mDtGNdc= +=JTFu +-----END PGP PRIVATE KEY BLOCK----- +""" + +# key 7FEE575A: public key "anotheruser <anotheruser@leap.se>" +PUBLIC_KEY_2 = """ +-----BEGIN PGP PUBLIC KEY BLOCK----- +Version: GnuPG v1.4.10 (GNU/Linux) + +mI0EUYwJXgEEAMbTKHuPJ5/Gk34l9Z06f+0WCXTDXdte1UBoDtZ1erAbudgC4MOR +gquKqoj3Hhw0/ILqJ88GcOJmKK/bEoIAuKaqlzDF7UAYpOsPZZYmtRfPC2pTCnXq +Z1vdeqLwTbUspqXflkCkFtfhGKMq5rH8GV5a3tXZkRWZhdNwhVXZagC3ABEBAAG0 +IWFub3RoZXJ1c2VyIDxhbm90aGVydXNlckBsZWFwLnNlPoi4BBMBAgAiBQJRjAle +AhsDBgsJCAcDAgYVCAIJCgsEFgIDAQIeAQIXgAAKCRB/nfpof+5XWotuA/4tLN4E +gUr7IfLy2HkHAxzw7A4rqfMN92DIM9mZrDGaWRrOn3aVF7VU1UG7MDkHfPvp/cFw +ezoCw4s4IoHVc/pVlOkcHSyt4/Rfh248tYEJmFCJXGHpkK83VIKYJAithNccJ6Q4 +JE/o06Mtf4uh/cA1HUL4a4ceqUhtpLJULLeKo7iNBFGMCV4BBADsyQI7GR0wSAxz +VayLjuPzgT+bjbFeymIhjuxKIEwnIKwYkovztW+4bbOcQs785k3Lp6RzvigTpQQt +Z/hwcLOqZbZw8t/24+D+Pq9mMP2uUvCFFqLlVvA6D3vKSQ/XNN+YB919WQ04jh63 +yuRe94WenT1RJd6xU1aaUff4rKizuQARAQABiJ8EGAECAAkFAlGMCV4CGwwACgkQ +f536aH/uV1rPZQQAqCzRysOlu8ez7PuiBD4SebgRqWlxa1TF1ujzfLmuPivROZ2X +Kw5aQstxgGSjoB7tac49s0huh4X8XK+BtJBfU84JS8Jc2satlfwoyZ35LH6sDZck +I+RS/3we6zpMfHs3vvp9xgca6ZupQxivGtxlJs294TpJorx+mFFqbV17AzQ= +=Thdu +-----END PGP PUBLIC KEY BLOCK----- +""" + +PRIVATE_KEY_2 = """ +-----BEGIN PGP PRIVATE KEY BLOCK----- +Version: GnuPG v1.4.10 (GNU/Linux) + +lQHYBFGMCV4BBADG0yh7jyefxpN+JfWdOn/tFgl0w13bXtVAaA7WdXqwG7nYAuDD +kYKriqqI9x4cNPyC6ifPBnDiZiiv2xKCALimqpcwxe1AGKTrD2WWJrUXzwtqUwp1 +6mdb3Xqi8E21LKal35ZApBbX4RijKuax/BleWt7V2ZEVmYXTcIVV2WoAtwARAQAB +AAP7BLuSAx7tOohnimEs74ks8l/L6dOcsFQZj2bqs4AoY3jFe7bV0tHr4llypb/8 +H3/DYvpf6DWnCjyUS1tTnXSW8JXtx01BUKaAufSmMNg9blKV6GGHlT/Whe9uVyks +7XHk/+9mebVMNJ/kNlqq2k+uWqJohzC8WWLRK+d1tBeqDsECANZmzltPaqUsGV5X +C3zszE3tUBgptV/mKnBtopKi+VH+t7K6fudGcG+bAcZDUoH/QVde52mIIjjIdLje +uajJuHUCAO1mqh+vPoGv4eBLV7iBo3XrunyGXiys4a39eomhxTy3YktQanjjx+ty +GltAGCs5PbWGO6/IRjjvd46wh53kzvsCAO0J97gsWhzLuFnkxFAJSPk7RRlyl7lI +1XS/x0Og6j9XHCyY1OYkfBm0to3UlCfkgirzCYlTYObCofzdKFIPDmSqHbQhYW5v +dGhlcnVzZXIgPGFub3RoZXJ1c2VyQGxlYXAuc2U+iLgEEwECACIFAlGMCV4CGwMG +CwkIBwMCBhUIAgkKCwQWAgMBAh4BAheAAAoJEH+d+mh/7ldai24D/i0s3gSBSvsh +8vLYeQcDHPDsDiup8w33YMgz2ZmsMZpZGs6fdpUXtVTVQbswOQd8++n9wXB7OgLD +izgigdVz+lWU6RwdLK3j9F+Hbjy1gQmYUIlcYemQrzdUgpgkCK2E1xwnpDgkT+jT +oy1/i6H9wDUdQvhrhx6pSG2kslQst4qjnQHYBFGMCV4BBADsyQI7GR0wSAxzVayL +juPzgT+bjbFeymIhjuxKIEwnIKwYkovztW+4bbOcQs785k3Lp6RzvigTpQQtZ/hw +cLOqZbZw8t/24+D+Pq9mMP2uUvCFFqLlVvA6D3vKSQ/XNN+YB919WQ04jh63yuRe +94WenT1RJd6xU1aaUff4rKizuQARAQABAAP9EyElqJ3dq3EErXwwT4mMnbd1SrVC +rUJrNWQZL59mm5oigS00uIyR0SvusOr+UzTtd8ysRuwHy5d/LAZsbjQStaOMBILx +77TJveOel0a1QK0YSMF2ywZMCKvquvjli4hAtWYz/EwfuzQN3t23jc5ny+GqmqD2 +3FUxLJosFUfLNmECAO9KhVmJi+L9dswIs+2Dkjd1eiRQzNOEVffvYkGYZyKxNiXF +UA5kvyZcB4iAN9sWCybE4WHZ9jd4myGB0MPDGxkCAP1RsXJbbuD6zS7BXe5gwunO +2q4q7ptdSl/sJYQuTe1KNP5d/uGsvlcFfsYjpsopasPjFBIncc/2QThMKlhoEaEB +/0mVAxpT6SrEvUbJ18z7kna24SgMPr3OnPMxPGfvNLJY/Xv/A17YfoqjmByCvsKE +JCDjopXtmbcrZyoEZbEht9mko4ifBBgBAgAJBQJRjAleAhsMAAoJEH+d+mh/7lda +z2UEAKgs0crDpbvHs+z7ogQ+Enm4EalpcWtUxdbo83y5rj4r0TmdlysOWkLLcYBk +o6Ae7WnOPbNIboeF/FyvgbSQX1POCUvCXNrGrZX8KMmd+Sx+rA2XJCPkUv98Hus6 +THx7N776fcYHGumbqUMYrxrcZSbNveE6SaK8fphRam1dewM0 +=a5gs +-----END PGP PRIVATE KEY BLOCK----- +""" diff --git a/mail/src/leap/mail/testing/common.py b/mail/src/leap/mail/testing/common.py new file mode 100644 index 00000000..1bf1de22 --- /dev/null +++ b/mail/src/leap/mail/testing/common.py @@ -0,0 +1,109 @@ +# -*- coding: utf-8 -*- +# common.py +# Copyright (C) 2014 LEAP +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see <http://www.gnu.org/licenses/>. +""" +Common utilities for testing Soledad. +""" +import os +import tempfile + +from twisted.internet import defer +from twisted.trial import unittest + +from leap.common.testing.basetest import BaseLeapTest +from leap.soledad.client import Soledad + +# TODO move to common module, or Soledad itself +# XXX remove duplication + +TEST_USER = "testuser@leap.se" +TEST_PASSWD = "1234" + + +def _initialize_soledad(email, gnupg_home, tempdir): + """ + Initializes soledad by hand + + :param email: ID for the user + :param gnupg_home: path to home used by gnupg + :param tempdir: path to temporal dir + :rtype: Soledad instance + """ + + uuid = "foobar-uuid" + passphrase = u"verysecretpassphrase" + secret_path = os.path.join(tempdir, "secret.gpg") + local_db_path = os.path.join(tempdir, "soledad.u1db") + server_url = "https://provider" + cert_file = "" + + soledad = Soledad( + uuid, + passphrase, + secret_path, + local_db_path, + server_url, + cert_file, + syncable=False) + + return soledad + + +class SoledadTestMixin(unittest.TestCase, BaseLeapTest): + """ + It is **VERY** important that this base is added *AFTER* unittest.TestCase + """ + + def setUp(self): + self.results = [] + self.setUpEnv() + + # pytest handles correctly the setupEnv for the class, + # but trial ignores it. + if not getattr(self, 'tempdir', None): + self.tempdir = tempfile.gettempdir() + + # Soledad: config info + self.gnupg_home = "%s/gnupg" % self.tempdir + self.email = 'leap@leap.se' + + # initialize soledad by hand so we can control keys + self._soledad = _initialize_soledad( + self.email, + self.gnupg_home, + self.tempdir) + + return defer.succeed(True) + + def tearDown(self): + """ + tearDown method called after each test. + """ + self.results = [] + try: + self._soledad.close() + except Exception: + print "ERROR WHILE CLOSING SOLEDAD" + # logging.exception(exc) + self.tearDownEnv() + + @classmethod + def setUpClass(self): + pass + + @classmethod + def tearDownClass(self): + pass diff --git a/mail/src/leap/mail/testing/imap.py b/mail/src/leap/mail/testing/imap.py new file mode 100644 index 00000000..72acbf2e --- /dev/null +++ b/mail/src/leap/mail/testing/imap.py @@ -0,0 +1,186 @@ +# -*- coding: utf-8 -*- +# utils.py +# Copyright (C) 2014, 2015 LEAP +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see <http://www.gnu.org/licenses/>. +""" +Common utilities for testing Soledad IMAP Server. +""" +from email import parser + +from mock import Mock +from twisted.cred.checkers import ICredentialsChecker +from twisted.cred.credentials import IUsernamePassword +from twisted.cred.error import UnauthorizedLogin +from twisted.cred.portal import Portal, IRealm +from twisted.mail import imap4 +from twisted.internet import defer +from twisted.protocols import loopback +from twisted.python import log +from zope.interface import implementer + +from leap.mail.adaptors import soledad as soledad_adaptor +from leap.mail.imap.account import IMAPAccount +from leap.mail.imap.server import LEAPIMAPServer +from leap.mail.testing.common import SoledadTestMixin + +TEST_USER = "testuser@leap.se" +TEST_PASSWD = "1234" + + +# +# Simple IMAP4 Client for testing +# + +class SimpleClient(imap4.IMAP4Client): + """ + A Simple IMAP4 Client to test our + Soledad-LEAPServer + """ + + def __init__(self, deferred, contextFactory=None): + imap4.IMAP4Client.__init__(self, contextFactory) + self.deferred = deferred + self.events = [] + + def serverGreeting(self, caps): + self.deferred.callback(None) + + def modeChanged(self, writeable): + self.events.append(['modeChanged', writeable]) + self.transport.loseConnection() + + def flagsChanged(self, newFlags): + self.events.append(['flagsChanged', newFlags]) + self.transport.loseConnection() + + def newMessages(self, exists, recent): + self.events.append(['newMessages', exists, recent]) + self.transport.loseConnection() + +# +# Dummy credentials for tests +# + + +@implementer(IRealm) +class TestRealm(object): + + def __init__(self, account): + self._account = account + + def requestAvatar(self, avatarId, mind, *interfaces): + avatar = self._account + return (imap4.IAccount, avatar, + getattr(avatar, 'logout', lambda: None)) + + +@implementer(ICredentialsChecker) +class TestCredentialsChecker(object): + + credentialInterfaces = (IUsernamePassword,) + + userid = TEST_USER + password = TEST_PASSWD + + def requestAvatarId(self, credentials): + username, password = credentials.username, credentials.password + d = self.checkTestCredentials(username, password) + d.addErrback(lambda f: defer.fail(UnauthorizedLogin())) + return d + + def checkTestCredentials(self, username, password): + if username == self.userid and password == self.password: + return defer.succeed(username) + else: + return defer.fail(Exception("Wrong credentials")) + + +class TestSoledadIMAPServer(LEAPIMAPServer): + + def __init__(self, account, *args, **kw): + + LEAPIMAPServer.__init__(self, *args, **kw) + + realm = TestRealm(account) + portal = Portal(realm) + checker = TestCredentialsChecker() + self.checker = checker + self.portal = portal + portal.registerChecker(checker) + + +class IMAP4HelperMixin(SoledadTestMixin): + """ + MixIn containing several utilities to be shared across + different TestCases + """ + serverCTX = None + clientCTX = None + + def setUp(self): + + soledad_adaptor.cleanup_deferred_locks() + + USERID = TEST_USER + + def setup_server(account): + self.server = TestSoledadIMAPServer( + account=account, + contextFactory=self.serverCTX) + self.server.theAccount = account + + d_server_ready = defer.Deferred() + self.client = SimpleClient( + d_server_ready, contextFactory=self.clientCTX) + self.connected = d_server_ready + + def setup_account(_): + self.parser = parser.Parser() + + # XXX this should be fixed in soledad. + # Soledad sync makes trial block forever. The sync it's mocked to + # fix this problem. _mock_soledad_get_from_index can be used from + # the tests to provide documents. + # TODO see here, possibly related? + # -- http://www.pythoneye.com/83_20424875/ + self._soledad.sync = Mock() + + d = defer.Deferred() + self.acc = IMAPAccount(self._soledad, USERID, d=d) + return d + + d = super(IMAP4HelperMixin, self).setUp() + d.addCallback(setup_account) + d.addCallback(setup_server) + return d + + def tearDown(self): + SoledadTestMixin.tearDown(self) + del self._soledad + del self.client + del self.server + del self.connected + + def _cbStopClient(self, ignore): + self.client.transport.loseConnection() + + def _ebGeneral(self, failure): + self.client.transport.loseConnection() + self.server.transport.loseConnection() + if hasattr(self, 'function'): + log.err(failure, "Problem with %r" % (self.function,)) + + def loopback(self): + return loopback.loopbackAsync(self.server, self.client) diff --git a/mail/src/leap/mail/testing/smtp.py b/mail/src/leap/mail/testing/smtp.py new file mode 100644 index 00000000..d8690f12 --- /dev/null +++ b/mail/src/leap/mail/testing/smtp.py @@ -0,0 +1,51 @@ +from twisted.mail import smtp + +from leap.mail.smtp.gateway import SMTPFactory, LOCAL_FQDN +from leap.mail.smtp.gateway import SMTPDelivery +from leap.mail.outgoing.service import outgoingFactory + +TEST_USER = u'anotheruser@leap.se' + + +class UnauthenticatedSMTPServer(smtp.SMTP): + + encrypted_only = False + + def __init__(self, soledads, keyms, opts, encrypted_only=False): + smtp.SMTP.__init__(self) + + userid = TEST_USER + keym = keyms[userid] + + class Opts: + cert = '/tmp/cert' + key = '/tmp/cert' + hostname = 'remote' + port = 666 + + outgoing = outgoingFactory( + userid, keym, Opts, check_cert=False) + avatar = SMTPDelivery(userid, keym, encrypted_only, outgoing) + self.delivery = avatar + + def validateFrom(self, helo, origin): + return origin + + +class UnauthenticatedSMTPFactory(SMTPFactory): + """ + A Factory that produces a SMTP server that does not authenticate user. + Only for tests! + """ + protocol = UnauthenticatedSMTPServer + domain = LOCAL_FQDN + encrypted_only = False + + +def getSMTPFactory(soledad_s, keymanager_s, sendmail_opts, + encrypted_only=False): + factory = UnauthenticatedSMTPFactory + factory.encrypted_only = encrypted_only + proto = factory( + soledad_s, keymanager_s, sendmail_opts).buildProtocol(('127.0.0.1', 0)) + return proto diff --git a/mail/src/leap/mail/tests/rfc822.bounce.message b/mail/src/leap/mail/tests/rfc822.bounce.message new file mode 100644 index 00000000..7a51ac04 --- /dev/null +++ b/mail/src/leap/mail/tests/rfc822.bounce.message @@ -0,0 +1,152 @@ +Return-Path: <> +X-Original-To: yoyo@dev.pixelated-project.org +Delivered-To: a6973ec1af0a6d1e2a1e4db4ff85f6c2@deliver.local +Received: by dev1.dev.pixelated-project.org (Postfix) + id 92CEA83164; Thu, 16 Jun 2016 14:53:34 +0200 (CEST) +Date: Thu, 16 Jun 2016 14:53:34 +0200 (CEST) +From: MAILER-DAEMON@dev1.dev.pixelated-project.org (Mail Delivery System) +Subject: Undelivered Mail Returned to Sender +To: yoyo@dev.pixelated-project.org +Auto-Submitted: auto-replied +MIME-Version: 1.0 +Content-Type: multipart/report; report-type=delivery-status; + boundary="8F60183010.1466081614/dev1.dev.pixelated-project.org" +Message-Id: <20160616125334.92CEA83164@dev1.dev.pixelated-project.org> + +This is a MIME-encapsulated message. + +--8F60183010.1466081614/dev1.dev.pixelated-project.org +Content-Description: Notification +Content-Type: text/plain; charset=us-ascii + +This is the mail system at host dev1.dev.pixelated-project.org. + +I'm sorry to have to inform you that your message could not +be delivered to one or more recipients. It's attached below. + +For further assistance, please send mail to postmaster. + +If you do so, please include this problem report. You can +delete your own text from the attached returned message. + + The mail system + +<nobody@leap.se>: host caribou.leap.se[176.53.69.122] said: 550 5.1.1 + <nobody@leap.se>: Recipient address rejected: User unknown in virtual alias + table (in reply to RCPT TO command) + +--8F60183010.1466081614/dev1.dev.pixelated-project.org +Content-Description: Delivery report +Content-Type: message/delivery-status + +Reporting-MTA: dns; dev1.dev.pixelated-project.org +X-Postfix-Queue-ID: 8F60183010 +X-Postfix-Sender: rfc822; yoyo@dev.pixelated-project.org +Arrival-Date: Thu, 16 Jun 2016 14:53:33 +0200 (CEST) + +Final-Recipient: rfc822; nobody@leap.se +Original-Recipient: rfc822;nobody@leap.se +Action: failed +Status: 5.1.1 +Remote-MTA: dns; caribou.leap.se +Diagnostic-Code: smtp; 550 5.1.1 <nobody@leap.se>: Recipient address rejected: + User unknown in virtual alias table + +--8F60183010.1466081614/dev1.dev.pixelated-project.org +Content-Description: Undelivered Message +Content-Type: message/rfc822 + +Return-Path: <yoyo@dev.pixelated-project.org> +Received: from leap.mail-0.4.0rc1+111.g736ea86 (localhost [127.0.0.1]) + (using TLSv1 with cipher ECDHE-RSA-AES128-SHA (128/128 bits)) + (Client CN "yoyo@dev.pixelated-project.org", Issuer "Pixelated Project Root CA (client certificates only!)" (verified OK)) + by dev1.dev.pixelated-project.org (Postfix) with ESMTPS id 8F60183010 + for <nobody@leap.se>; Thu, 16 Jun 2016 14:53:33 +0200 (CEST) +MIME-Version: 1.0 +Content-Type: multipart/signed; protocol="application/pgp-signature"; + micalg="pgp-sha512"; boundary="===============7598747164910592838==" +To: nobody@leap.se +Subject: vrgg +From: yoyo@dev.pixelated-project.org +Date: Thu, 16 Jun 2016 13:53:32 -0000 +Message-Id: <20160616125332.16961.677041909.5@dev1.dev.pixelated-project.org> +OpenPGP: id=CB546109E857BC34DFF2BCB3288870B39C400C24; + url="https://dev.pixelated-project.org/key/yoyo"; preference="signencrypt" + +--===============7598747164910592838== +Content-Type: multipart/mixed; boundary="===============3737055506052708210==" +MIME-Version: 1.0 +To: nobody@leap.se +Subject: vrgg +From: yoyo@dev.pixelated-project.org +Date: Thu, 16 Jun 2016 13:53:32 -0000 + +--===============3737055506052708210== +Content-Type: text/plain; charset="utf-8" +MIME-Version: 1.0 +Content-Transfer-Encoding: base64 + + +--===============3737055506052708210== +Content-Type: application/pgp-keys +MIME-Version: 1.0 +content-disposition: attachment; filename="yoyo@dev.pixelated-project.org-email-key.asc" +Content-Transfer-Encoding: base64 + +LS0tLS1CRUdJTiBQR1AgUFVCTElDIEtFWSBCTE9DSy0tLS0tCgptUUlOQkZkZ01BZ0JFQURIWWpU +T20wcTdOT0lYVUpoTmlHVXg2S05OZ1M0Q0I2VlMvbGtab2UvYjZuRjdCSENmCkFnRVkxeFlxMkIv +MzA3YzBtNTZWMEZvOWt2ZmZCUWhQckU5WG9rckI5blRlN1RsSDZUNTdiV09LSWMyMHhNSy8KSlVU +djZ3UEpybjdLN0VyNEdxbzdrUmpWcFVBcWlBbGFxMkhVYllGd2NEMnBIb0VENmU2L01CZDBVUTFX +b2s4QQpPNURDc2ZmeWhBZ0NFU1poK2w2VHlsVEJXYTJDTmJvUTl0SWtPZ0ZWTk9kTW9uWkxoTk1N +Y0tIeU54dmF5bUdCCjhjQlRISVE2UWhGRThvR2JDRTdvczdZWWhyTmNmcUsyMzJJQllzTHNXN3Vk +QmdwRTA0YkpwQWlvbW1zTHBCYmwKV0pCSjdqeEhwWmhJR3JGL1ltejNsSXpkbm9Mb3BSSWJyS0pC +MmxaVDhIUHBlTVVJdVE2eHErd3RhQXFJVzlPTgo5U29uZWYyVU5BL3VseW1LeDRkOFhxbEwxY3hE +aDFQU1E5YVlPcVg0RDlrMklmOXZmR2hET0xVMzR2Y2VFOC8vCnM1WGdTY2ZFbHg2SWlEVWZHdGx2 +aE5zQUM4TmhhUU1sOHJjUXVoRDA2RFdvSUowMVhkeFJVM2JSVVZkc0I1NWMKcXRWSHJMbVBVb256 +NU13MGFURzlTZzZudUlQcU1QOVNKRlBzbVpzR3ZYVnZWbCtSNzl1SFBlc25yWkoyTjZqOQpNaUth +S045NFBhL1dJUnRoYWdzVnpHeHNtd2orTVZCRkZKRmh0TUtnNlFzYUsvbzRLNGJFR1ZLdWNXQk1i +MnNxCldmd0o0SndTcHcrOHgyS3p6aXhWTllTZXhRdm9oMkc3RDRmRXdISDJzazNST3k3dTlldjhs +bEVqUFFBUkFRQUIKdEQ5NWIzbHZRR1JsZGk1d2FYaGxiR0YwWldRdGNISnZhbVZqZEM1dmNtY2dQ +SGx2ZVc5QVpHVjJMbkJwZUdWcwpZWFJsWkMxd2NtOXFaV04wTG05eVp6NkpBajRFRXdFQ0FDZ0ZB +bGRnTUFnQ0d5OEZDUUhnTUZnR0N3a0lCd01DCkJoVUlBZ2tLQ3dRV0FnTUJBaDRCQWhlQUFBb0pF +Q2lJY0xPY1FBd2s4djBQL2o2MmNyNjRUMlZPMVNKdHp1RlEKWjVpeVJsVFVHSGN2NW5hQjlUSDdI +VVB3cTVwekZiTkg5SnhNRjVFRWtvZjdvV0hWeldWVTFBM1NDdzVNZ2FFbwppWTk5ZFBGNzdHazJ4 +ZEczNXZlWmIwWkg2WkVLdks1S042VXBucG5IeStxaVZVc1FLcE9DdUZKNkF0UlVEOTRJClJ2YnUv +S1hsMHdORDlzVXFlYkJZN1BBSlRNY1RjLzVEdWpIT1Erd3VlSkFtaFZZbEozVnpZK1lBS2t5U05B +QVoKZ3VVenNyUm5xQWU5SmU5TGgrcERpcVpHT2tEK1Z3b2kvRlVPQXJwbWFnNzZONTVjR3hiK2VG +QUlzRHYrM1NNOQpjUDFyQkFON2lEaGgvdkdJeHgzMFlrYUlpMmpmcXg3VXUydnNwSXh6K0NsWWdi +dm1wZm1CWmFqVzYzR0FsK3YvCngrby92eFZmVTMraTZ3alFjRS8vRTBTR2pvY3lQdUw0ZTZLNERy +S3k2SHQycjBQckdHVFZ0dUZPaWU2dnVzbVcKL09sdVB1dGszU3o1S1BmRDFpRXBobmpPQ0pNRkZx +Z2xRM1pPa3MweG00WGdwWW1ycnpQcXc1WWlzK1NEVjhobwp6anlrSzRWUlcrcC9IcUVzU29GQm5a +MG5XSmg2Q1pZOExIeVNiMVJwaFlMRFpWd21JRXd1OW12Vm1ISVIyWUZVCllNZEx4UExiOFZNei9t +QWpMb2Q0OGNSSzdSTzBSZ1RoMTUyK0VieXRGR3k5Y2tiS3VzRmJzVTFCQjN2MFJyUlUKenozTTcx +T3hjcFhVQ0tpWlI0MEVYZnErSnVtZVFudm1wSWdZdUNaQkh5MzJwQUJuOHNDdUlrMStyQnp4bXdt +bgp0WGh0K0RvNlExYXYyVjZYR00xV2xoKzEKPU8zaHEKLS0tLS1FTkQgUEdQIFBVQkxJQyBLRVkg +QkxPQ0stLS0tLQo= +--===============3737055506052708210==-- + +--===============7598747164910592838== +Content-Type: application/pgp-signature; name="signature.asc" +MIME-Version: 1.0 +Content-Description: OpenPGP Digital Signature + +-----BEGIN PGP SIGNATURE----- + +iQIcBAABCgAGBQJXYqFNAAoJECiIcLOcQAwkDEIQAL67/XJXDv+lusoy18jr7Ony +WQEP0pIRLp4GywGpH3dAITFAkuamO4VX3QEdVGjOHNoaT8VkSVWf9mnsYLl+Mh2v +1OIwMv0u8WyVtrcxyXijIznnJv8X1RgyCzpUJcmOh04VZcDyxKbnFHWSDMfJ4Jtq +qnXDONcfEeT8pwrGjP5qzTgcF/irG3w5svyQjEtj6kycddYtqUc9Hx3cMaRIzsHg +kuUzznSzU/6P0Z345q/kXyYvU9rlcsP9vogrsqL2ueLwYSipxUJQUrRWG82FYoCo +PAKNdGIt0xl2gEW+xWZkJqFarPiUFCx//+bVBelKrqj6rjwbj+E7mHJW318JYVHQ +en3Smv7pEWlT4hZHXnoe8ng6TAvKzQjf7/bUxq2JpKSycp2hDO3Qz3Tv+kc+jC/r +5UDWe/flR+syq8lAQTRSn6057g3BgDG2RtAwsjedg1aTFSrljSxbKlK4vsj5Muek +Olq9+MUdMFSE3Jj/JC2COcS3rlt/Qt+JLDYXKahU3CodaSgF2dobikDe1bW0/QNS +7O4Ng2PK0pA416RCFRUgPXerUnMGiWAiq7BoRHeym9y7fkHYhIYGpPVKXJ6t67y5 +JjvuzwfwG8SZTp4Wy2pg1Mr6znm6uVBxUDxTHyP3BjciI1zpEigOIg9UwJ9nCDxL +uUGz4VqipNKbkpRkjLLW +=3IaF +-----END PGP SIGNATURE----- + +--===============7598747164910592838==-- + +--8F60183010.1466081614/dev1.dev.pixelated-project.org-- diff --git a/mail/src/leap/mail/tests/rfc822.message b/mail/src/leap/mail/tests/rfc822.message new file mode 100644 index 00000000..ee97ab92 --- /dev/null +++ b/mail/src/leap/mail/tests/rfc822.message @@ -0,0 +1,86 @@ +Return-Path: <twisted-commits-admin@twistedmatrix.com> +Delivered-To: exarkun@meson.dyndns.org +Received: from localhost [127.0.0.1] + by localhost with POP3 (fetchmail-6.2.1) + for exarkun@localhost (single-drop); Thu, 20 Mar 2003 14:50:20 -0500 (EST) +Received: from pyramid.twistedmatrix.com (adsl-64-123-27-105.dsl.austtx.swbell.net [64.123.27.105]) + by intarweb.us (Postfix) with ESMTP id 4A4A513EA4 + for <exarkun@meson.dyndns.org>; Thu, 20 Mar 2003 14:49:27 -0500 (EST) +Received: from localhost ([127.0.0.1] helo=pyramid.twistedmatrix.com) + by pyramid.twistedmatrix.com with esmtp (Exim 3.35 #1 (Debian)) + id 18w648-0007Vl-00; Thu, 20 Mar 2003 13:51:04 -0600 +Received: from acapnotic by pyramid.twistedmatrix.com with local (Exim 3.35 #1 (Debian)) + id 18w63j-0007VK-00 + for <twisted-commits@twistedmatrix.com>; Thu, 20 Mar 2003 13:50:39 -0600 +To: twisted-commits@twistedmatrix.com +From: etrepum CVS <etrepum@twistedmatrix.com> +Reply-To: twisted-python@twistedmatrix.com +X-Mailer: CVSToys +Message-Id: <E18w63j-0007VK-00@pyramid.twistedmatrix.com> +Subject: [Twisted-commits] rebuild now works on python versions from 2.2.0 and up. +Sender: twisted-commits-admin@twistedmatrix.com +Errors-To: twisted-commits-admin@twistedmatrix.com +X-BeenThere: twisted-commits@twistedmatrix.com +X-Mailman-Version: 2.0.11 +Precedence: bulk +List-Help: <mailto:twisted-commits-request@twistedmatrix.com?subject=help> +List-Post: <mailto:twisted-commits@twistedmatrix.com> +List-Subscribe: <http://twistedmatrix.com/cgi-bin/mailman/listinfo/twisted-commits>, + <mailto:twisted-commits-request@twistedmatrix.com?subject=subscribe> +List-Id: <twisted-commits.twistedmatrix.com> +List-Unsubscribe: <http://twistedmatrix.com/cgi-bin/mailman/listinfo/twisted-commits>, + <mailto:twisted-commits-request@twistedmatrix.com?subject=unsubscribe> +List-Archive: <http://twistedmatrix.com/pipermail/twisted-commits/> +Date: Thu, 20 Mar 2003 13:50:39 -0600 + +Modified files: +Twisted/twisted/python/rebuild.py 1.19 1.20 + +Log message: +rebuild now works on python versions from 2.2.0 and up. + + +ViewCVS links: +http://twistedmatrix.com/users/jh.twistd/viewcvs/cgi/viewcvs.cgi/twisted/python/rebuild.py.diff?r1=text&tr1=1.19&r2=text&tr2=1.20&cvsroot=Twisted + +Index: Twisted/twisted/python/rebuild.py +diff -u Twisted/twisted/python/rebuild.py:1.19 Twisted/twisted/python/rebuild.py:1.20 +--- Twisted/twisted/python/rebuild.py:1.19 Fri Jan 17 13:50:49 2003 ++++ Twisted/twisted/python/rebuild.py Thu Mar 20 11:50:08 2003 +@@ -206,15 +206,27 @@ + clazz.__dict__.clear() + clazz.__getattr__ = __getattr__ + clazz.__module__ = module.__name__ ++ if newclasses: ++ import gc ++ if (2, 2, 0) <= sys.version_info[:3] < (2, 2, 2): ++ hasBrokenRebuild = 1 ++ gc_objects = gc.get_objects() ++ else: ++ hasBrokenRebuild = 0 + for nclass in newclasses: + ga = getattr(module, nclass.__name__) + if ga is nclass: + log.msg("WARNING: new-class %s not replaced by reload!" % reflect.qual(nclass)) + else: +- import gc +- for r in gc.get_referrers(nclass): +- if isinstance(r, nclass): ++ if hasBrokenRebuild: ++ for r in gc_objects: ++ if not getattr(r, '__class__', None) is nclass: ++ continue + r.__class__ = ga ++ else: ++ for r in gc.get_referrers(nclass): ++ if getattr(r, '__class__', None) is nclass: ++ r.__class__ = ga + if doLog: + log.msg('') + log.msg(' (fixing %s): ' % str(module.__name__)) + + +_______________________________________________ +Twisted-commits mailing list +Twisted-commits@twistedmatrix.com +http://twistedmatrix.com/cgi-bin/mailman/listinfo/twisted-commits diff --git a/mail/src/leap/mail/tests/rfc822.multi-minimal.message b/mail/src/leap/mail/tests/rfc822.multi-minimal.message new file mode 100644 index 00000000..582297c6 --- /dev/null +++ b/mail/src/leap/mail/tests/rfc822.multi-minimal.message @@ -0,0 +1,16 @@ +Content-Type: multipart/mixed; boundary="===============6203542367371144092==" +MIME-Version: 1.0 +Subject: [TEST] 010 - Inceptos cum lorem risus congue +From: testmailbitmaskspam@gmail.com +To: test_c5@dev.bitmask.net + +--===============6203542367371144092== +Content-Type: text/plain; charset="us-ascii" +MIME-Version: 1.0 +Content-Transfer-Encoding: 7bit + +Howdy from python! +The subject: [TEST] 010 - Inceptos cum lorem risus congue +Current date & time: Wed Jan 8 16:36:21 2014 +Trying to attach: [] +--===============6203542367371144092==-- diff --git a/mail/src/leap/mail/tests/rfc822.multi-nested.message b/mail/src/leap/mail/tests/rfc822.multi-nested.message new file mode 100644 index 00000000..694bef59 --- /dev/null +++ b/mail/src/leap/mail/tests/rfc822.multi-nested.message @@ -0,0 +1,619 @@ +From: TEST <test.bitmask@example.com> +Content-Type: multipart/alternative; + boundary="Apple-Mail=_F4EF9C8E-2E66-4FC6-8840-F435ADBED5C8" +X-Smtp-Server: smtp.example.com:test.bitmask +Subject: test simple attachment +X-Universally-Unique-Identifier: 0ea1b4b2-cdb8-43c3-b54c-dc88a19c6e0a +Date: Wed, 8 Jul 2015 04:25:56 +0900 +Message-Id: <47278179-628A-43F5-95C9-BC7E1753C521@example.com> +To: test_alpha14_001@dev.bitmask.net +Mime-Version: 1.0 (Apple Message framework v1251.1) + + +--Apple-Mail=_F4EF9C8E-2E66-4FC6-8840-F435ADBED5C8 +Content-Transfer-Encoding: 7bit +Content-Type: text/plain; + charset=us-ascii + +this is a simple attachment +--Apple-Mail=_F4EF9C8E-2E66-4FC6-8840-F435ADBED5C8 +Content-Type: multipart/related; + type="text/html"; + boundary="Apple-Mail=_C7D5288F-B043-4A7F-AF3F-1EDF1A78438B" + + +--Apple-Mail=_C7D5288F-B043-4A7F-AF3F-1EDF1A78438B +Content-Transfer-Encoding: 7bit +Content-Type: text/html; + charset=us-ascii + +<html><head></head><body style="word-wrap: break-word; -webkit-nbsp-mode: space; -webkit-line-break: after-white-space; ">this is a simple attachment<img height="286" width="300" apple-width="yes" apple-height="yes" id="fd3d0c89-709d-419f-b293-a6827f75c8d4" src="cid:163B7957-4342-485F-8FD6-D46A4A53A2C1"></body></html> +--Apple-Mail=_C7D5288F-B043-4A7F-AF3F-1EDF1A78438B +Content-Transfer-Encoding: base64 +Content-Disposition: inline; + filename="saing_ergol.jpg" +Content-Type: image/jpg; + x-mac-hide-extension=yes; + x-unix-mode=0600; + name="saint_ergol.jpg" +Content-Id: <163B7957-4342-485F-8FD6-D46A4A53A2C1> + +/9j/4AAQSkZJRgABAQEAYABgAAD/4QCURXhpZgAASUkqAAgAAAACADEBAgALAAAAJgAAAGmHBAAB +AAAAMgAAAAAAAABQaWNhc2EgMy4wAAAEAAKgBAABAAAALAEAAAOgBAABAAAAHgEAAACQBwAEAAAA +MDIxMAWgBAABAAAAaAAAAAAAAAACAAEAAgAFAAAAhgAAAAIABwAEAAAAMDEwMAAAAAAgICAgAAD/ +7QAcUGhvdG9zaG9wIDMuMAA4QklNBAQAAAAAAAD//gAmRmlsZSB3cml0dGVuIGJ5IEFkb2JlIFBo +b3Rvc2hvcKggNS4y/9sAQwAFAwQEBAMFBAQEBQUFBgcMCAcHBwcPCwsJDBEPEhIRDxERExYcFxMU +GhURERghGBodHR8fHxMXIiQiHiQcHh8e/9sAQwEFBQUHBgcOCAgOHhQRFB4eHh4eHh4eHh4eHh4e +Hh4eHh4eHh4eHh4eHh4eHh4eHh4eHh4eHh4eHh4eHh4eHh4e/8AAEQgBHgEsAwEiAAIRAQMRAf/E +AB0AAAEFAQEBAQAAAAAAAAAAAAUCAwQGBwgAAQn/xABQEAABAwIEAwUFBQUGBAQFAgcCAwQFARIA +BhETFCEiBzEyQVEVI0JSYSQzYnGBCBZDcqElNIKRkqJTscHRRGOy0jVUc4PhF2SkwsPT4vDy/8QA +FAEBAAAAAAAAAAAAAAAAAAAAAP/EABQRAQAAAAAAAAAAAAAAAAAAAAD/2gAMAwEAAhEDEQA/ANW2 +3RMhJzJOiUK4up+4Lv8A4JUt/rgjBZfmN1JRBsaDchEklXfSaQ15GkdfFX6YteR4tNtDiuuCC7gh +6iHpt89O+vLFg+ARv8WAQ0bizSSaD1CkNo4fuL58J2/BdhXSJ+PAex64sL+PCDTEjK47vw4D1qnz +49b1+Pw4bMrf+IOFWj/NgPdV5Wn/AIceSUuMus/5sKMRw2Y+Lo6i8OAV4kiETtL5sMGpaACofUXh +/F+WPN1vvei0R/1YV7tRLpC4fxYBvcT2hJMztIenxXD/AP76YVvCICQnd+G/COpMNwvu/iMvh/Om +Bkgs63QXYtk+IIv4h9FuAJm4+0XCdpdN4+Lp+v8A3xGBZYXBJ3mVt1nzW/T1w6qmperb0koNoeG8 +f+44Emo6QBJN9YoSnVv+FJUvr5p4CVxBKXEmsdxdNvwCPpT5S+mIvGSBN1d4zbOkytuLqEgH1p/1 +phDtZwg1VIt+0LbwEKXjr6j/ABKfWmGGjhQmSvE/CVqVp3Xaf8Mq+Gv0rgLEyUW4JJVT4vx3D+mv +w4dN04IVyRv6Oq6y4fr3YQ0IXMeLkQMRtutIOr/LAqY4VNJqMgYJo2/iG3/TgGJDMyyCS5JuQEh6 +UhctlQH9a24EuJbjnFzGShyISuV3HiqRidPpUfBhSpEoBptjfKIp+7AUXgFdp8WhjiM3FYd1y5fr +kj/+7igI0h/FZXATmSz7dIkW0omSfUYIyQGCo18x6sNHMSjFVVZ28mEESK1tcwFURL5a2692BgLM +RBJMloRe4iIbQWQJL8Q99448kzjyVVJNtFqXCJf/ABJUAX+nOnTgGpiWeO0l0F5K5NTqVFRFZsqg +VO46Wj4cRop8pxC7FN4+EhSEr05gDScj+C8qFStPyxMWEuldBaUUWTIhDZeJKkkVP4daENLh9MMG +Kj5IWOy6ZFaSi7ZaNAyIfmoQlTAS28lME0/vkpvUuFuuTluYpDX/AIlL+/DDSQmkFVV1JibJqmRE +ZKMwIkD866hWuqeGHCjdNp7SF4xJO0UUC9lGIL+VqlKH5YdSKNjY8XJLNdkfERM1ht15W1pd4cA0 +bjMSkO1TXkl7R98u54ZUCS6unwjzGuGnr6U2lVBklEBUEb7XLojSLyU0t8OBTh8oSSCaaLVNqur0 +ELlZK3q0tKlRLpri6/um4FoRIGgIiQ2JEe6QpU/g+XTrgKe79rPrU1JWR4xMbiJs8dW20LkuOo86 +euLXkyPnEGntKSknwrOfAgm5JUEh7q8lNO/lXBiJg4tsDYkGa+8gO4Jl/wCmnV3YK+8USVLZ6vh6 +PX9cA3/aCaVvtJdRa3wiil1f7sKbuH3FluP1yH+QLS8vmx401ib7lnw22eHCVk1t0RFY+rqO0BL/ +AK+eAc4p0nu778y+XoDp/rhYLOCMS4ldNNPqUuttLUfzxBj94jESbGSY9VpIj1f7sPpKOBcEPiR8 +I22l3eeAfCSJNv8AfXeK0yWG78PnhJksropxT1O6mtorDp/zwhuKyiRD4VBL8FuHiJ0VdRqadPl6 +cBToqUKPcIFwx2q9JG5tAiL5SKnQWL5GPEXzIXYgpaXzYzA0U2J7DZYLrSIjK9iRad415bZV/wAW +Lbld4oLtBMUdtuokV1yJCV/0INRL/PAWsxHCTG48KuIvD1Y8ZF04BIF1+D/Vh3wn4MNH81/+rHkl +FLyuDAKtuAui7DG5aYiQdP8A6cP9VmEqjcA3dJCXTbgHbv8Adho07rh/04i++Td9R/Z/iH4h/LEq +4f5R8WAauIlbbOr5sNJIrClcXVb8v/TEhZMd3cI/F0/LiPcKYffGPwl8PV8v0r9cB5VToLr/AAl/ +3w2bckw93Z7zxB4bvy+WuEGPujITu6rb7P8AbXDpimolaQWl8v4cBFSWTLpUW3xTK0yILSSLy19M +J4cve9ZkXSN/iK38VPOmJjtPwiSxipb0mPUQ/wDfDTdmSYbdgW9V3i/1U/PADDTWEFSQC7cK4Osv +Ll7s/g/LFbnSeEqKaaNyYn9qVIPCXounT/1ji7pR6nUQncRdRJWdKunnWnrhjg7URVH3dtxXl4kv +yKvfT6VwDGWnlzUWy5nb4UNzquGnfYfxYmSo7bQiT8SfUIpmIjZX+bWlcDJXejW5Wsw2SIulO4ki +9Knp1BX60w5BOJJTpK9dmRWmC3jQ/wDudyg4AYlH+9XJyz30yt6lGHr8VydKcsNfYU2/EpmDZQSF +Mi49VLp9DpUsWl6mok4FZAA92Nto3D0V/Lz9MQ9kSbiShuhu6SEljLqr9K0wFb2RJXiWjl7t/Akm +5SPq86p1KlS/TDSqJCkkx3nztFUSWICYIkJD8vIeRUxMerJppCg5O1MS6QWATEvxDU9LcDHCm84J +sotHEiRD0u0UhEiqPK6oFUqVwC3DFumqlILhHDtESaS7mNJIh8tK1DTDjRSPdpW3tUFhuG4X5pEk +XzUpUudK4S9jfY2X9xcDQapl9pVbPFRJIfmr088KbpuvZiqyDx0u3QtUQJN4KpD+KnPUh591cAwH +Ctm5Nr0C3SuMBmCucl8wal34YVki4dqgUk6+0+AikgK4e+xTX4sSjGYUZIERrkiJEoR3pWkdS5VD +UuRU+XEVVGSkHpFs76atyZbkakQql5jqBVtL64B9wimQCuKxjtD1A5lfuh8k66d419cXBJwptIKc +GaC20IkIgJj4eY618WBOX4V42PcI1+DBINseGAC8VemulcWAy6/dhaJXdI2Db/XAI6VQLYDbLwkJ +AIkWvnzwwkTwXApkshaVvQNtwjTyxISIdolFPDbdaVn/AErjLc5zXs3tryVEoOTLjUFxVSTBK0Q6 +a3F1YDVne8QCQrAh8XgHqH64jKl96oRgIj1FcAlb+euHHAjYkmR3DddulZ4qfrjAP2jc/SBOEsoZ +QACcO1RRVGwS4nXltAdK8vxYAjCdpEl2g9qbWCyS5XbQMWRLSL4Ww3LlTltalTQRrz6sbU3RtutC +34hGwBIv9uM77Gsh/uLk9nHqAHHEJOHRpoo+Ov47uqlMXreWvJRS8fitLa9/08q63eeAmfcJWqBc +RfDYHh/yw8KgFTVR0FS8/By/pga7HfAVCcroDd0j7rw/L4sRyXSGtqbghEeVKJrJW/8AqwAqHb9D +wRBBNFUusGjwXIJLXaXCmoOtLq9/PEHedRbhrw14iToU3IIgQKiXz7ZXUrSv81MTNtuILryiICok +vtkT1htEvr8eoUpX/Fgrl8dj3fGLrt0uoy4wHNw1/PUsBZEpBO9dN2sCbhLqMRO60fLyxNPwfNis +N3DUnBKNt9QkyK4rFi6q/StcSmT5RNJK5suSZdV4omQ3fStfLAGrbvEGEqjaZW9JW3Ya4gbBu3BI +urqAsKbum6hlaYEVvhE7iup5aUwDqRDtdXi+LCjttx4BEgust+a3Dd2302GWAcMRsu+L/wBOGNxM +XBCXT8mJXwXYQqnuWlZ1D4S+IcBD3hTPpDcu6enqtH6YdNNEjJT4bbbvFcP5YYVuG5AQuIuoPhu9 +dPr9MeScEKQWmChEPSRBbd6/kXrgPJKfZ/uTEfiEerp/7Y8BCQDaBl8tvi/w4TasQEW9bcQiN3hH +/tXD7RNTiOo9serpEOkvzwDrdEhD357iheIvw4UZWmPXaNuFH8OB0hxCjhL7xAfDfvANuAlGp8Im +Al8JYGPVhbRi4qLIcQp8qxCJfrpX/liY7uF2P2k0xttHrH/rgE4UFRJUk3LohTIRsTchd/lgCPHN +xSVInIbJJCVhH0W07x8OJzQkRbjsBanaJdJ9PPu0xXjUUct0k0wXXJUesSRAwVG3zqI9OFRjxxGp +L3AuLceoS4AhtKnLnpTSuAKO+OT30yNclFLRSuciJF+tB6cQnpPBb2+/Hbt97xI3fkXThYSm4xIn +exxCdqgincHTXnd1/wBcQwkmq6tzaYBe64TMTStH/wAutNP64CO4cOC3RHilRTESEBcpF0111tuH +nXEF66TFUSdn94NoGswEwIbddCqBU0KmIOY2/wDYSqbYFyRQIVC4ZEF10Du8VtB1KnPvwwkxWsJy +ViZKiKaqAmq2FyFP4yfVpgCDRwjcP2xqoip4hK9Akh/lOh0KnpiU4FYj8Cgo23CAtgXD86VAg/yx +GbxrjhLWz90797uAuRi5EUqFrs1Guv6Urh9uiixVEk2xoN1BJR4AomgSRU7ipTy78AObrM7+GUbM +dsi3PdrEBEVC5EmBjpQ6fniL7NeNJAlF5VdMiPbQ3rhSXHv+8Ctaa0/lwb2SFwkmJm7akO8oCljw +efcVKVpUqaYRw6KIDcFpeIQbLKobpV53CmdbaF9MBKhJoU24i5WXXT+Ix97bp6+Gv+3BoH0W52iY +uQUEhu6ekuX0rirTCKZHuF7sRG5VdRtZ/wDxCendiGqo3kFUl00eGFoQ3P8AeErht15Kj02187sB +fEU00EiTU6iH8HixgvbLJIse1iFmbwIotdoN42ltCtuUK6nTrroPnjaoeWTctBtPc6RHqWSEhL8v ++WOY/wBo1qnJZgmpCPA3b5SRZxjVIQEw3QuKpnpy+LAbJ2wdozHKGWnjRMFymFUi4YSRG0ip6dVe +eMr/AGX8llO5gddpE62kRTTVL2ULvrAhLxndypr+mM37SE3Ehn1DK+aJv3iSqZTrps2E0m2gD9zQ +afJprSvcXPHQX/6pdnOR4GHyvl8150U0uGQZsAWclcI8x8VeeA15X3jgSTR27vFuIjaWn+L+mBzg +nC73qZqKIh4fcjbb6eLA7JMo+norjpbKr2H1+6auESuKlfi/pixNExELtlBsiXhSURtL/wDGAU4J +QgEU23u+nw22jiNSig1KgpbdNfD0f+3D52iGwIAJfONvhr9MOJN0doOIVQUU05lshz/pgKvk8lFz +VUFmCagpEo2BQ1hEdf4ZXjTSmJwC+GPXJjv9VvUpduj1cxpy50wJdSDx63SU3uJJUhECUDqJGnMh +U0KminLBSPTeNGQoILe5V6QcE2L3WpcwL3uAfVFwrH+Bds8EiEDELhtr8NfUa4fjGJF9rXRQFS0h +JIWxDcXxFrTCQZkTRW6xsI3dSiJ9I0/I+6uByKg2L75tbum0RWVSIS+bnrgH3AppgK6CKaZJ+7Ax +WVQO30PpxKSdPOLSUQDdTtLdEdpX8+dCuL/LEe5w2cWpuXyjhRK4myblJcSD56ajTnhi4SbjuGgT +dQvdEo2JIhKnKo3AXTWn9cBa4yQRcpGRGA+ERHqD+h0piZ8H8uK9GLcc3VUFYE9vwCR7/VTz0rQS +wdaERNxu6VLbiEfDgFbnXb4RHqLouwoyHpK/xYYcbZJXXn09XThO4NgkXhU8VuAfcJpqBtl1CXVg +c4TtcESnURl/qKnd+VfriVdtq2j4bekvl/8A8cIaKIuTVtv3E+kx/wCX+H64BIbJN+pbcISJMiIL +bvw1/wC+GnBOlHbXYM00RLqIfEJW/HTzH64mXeL8PUY/DhaRJkrb0XD8vit9MA4Y9YXdXy/zeuIb +jb4hLrtHq6Cs/wAu/E8y6Pw4iqkmVwqI7lxeGzAQTFwKvUZkn/wyAOn/AHYrr1PfkEF1D21Ej3Ac +popWkXop1f0xaXu8JpJoLbZXeKwekfm78AnCaKDhXrAhckN4rGAi5DuqQ/iwEG14mqSbYNxQUh4p +mTYRBXXvIK0r54UDdmpHtXKCIL7dyYlsmNpV+A7aV7sLcTBNIx8pe64dLpEhRG9IaFpb4ueKVlLO +0LmLMBRbaSapzyAkso2EDteI15Xjofl50wF1cOkWjRIWPFfMYC5VAhGneVNR8OuIAOnTlVBBiG4S +hWqr3pGC4V7+8qdWBj3bF17QjX/92LZJVS4dg689sqVu7/LD/tR0xkHTYTNNuyS60hWASI+/3fuu +dNO/ALVUYk0JymsCaKDrhh3GYXCXmFbCr04cBTgXDoUwPbQEVthNYwND0LQhp04Y4ckHaBDYo82F +FhDZExV10r16EPOmEx7MY+PSUXcroKLqk4VeEBCSQV52c7u/y9MAo3SjaQFpH3uZJ21EnI+6MHI9 +1fi13KUx5u+dNgERcgTe4UwdKXttoqeJI7ht/rgK92ZSP33Ky/Bqq7zoLBI0BEukkzGwvFjzdq+9 +sJJrtgFq594qwJyQk5DyWoJUPAGHr4V27qZURNOPRIiEyRFU0iEtPEFa1tuxFip5u+BUXZuouQbf +AS3uhKo6X0opbdTzxYjhWq6RIIHc12OHFIvAqN1dddK06sAnDUk5Uk2KwXDcmVywmIhQvuq0IfP+ +bATm7N8pGKrw7BBR4ncLpSOW2CVV7+jToLX88VtkNsgKd5xcgr4mrkCZkRVHUxu+7V/K7ErMUoSG +TYpPLrnhm6zpcXBX7H2ig60Cteq3XyxnPY/2mDmjKUxF9qMkbtvHKkK/EIjutgqdaCpRYdNa0r0+ +H4cBfXs5H5Fy4q9/u0ekVtt5ILqq17kaWUIVDrXlTSuMjnc0PMkuFc3yDMHeYph4LlJqp9+x1HSq +NCDWu5pt8jtwf7WG62TnCGYM2vAkW7RcigOH6mz4qhXkuPz2/wAQe6uAfYpl192n57LtEzeCCka0 +K1mwcrEHWOltdaD1269/ngCGTOynOHaQD7NGf5J1Cx8ouKhxgomKq4jyEj0Hl3Y3jKnZ/k3LcU1a +QmXmrZG7cMyRI1SK3kWteeDEYs3UcERGC/iESTvLp9MSDWETSQFsdviG5Eur6eLAIdtWqloi29yn +aXUF135YlJC3La2D6bbQ9zb03eHDUg3UXO0um23q2S6S/wBWFJDa48B3CN1+zaN3n8WAYVaqL+Hq +IS8JMwL/AA8ywpOrlMKDukNfOgtktNf9WPo7yO+oXSSnh9z1D/uwtFOhJ0qoNCL1FHl/zwFO9k7j +Rdjw32pdUiExRRISGnKleQ+PTExu1TbGQuUTURFIRO6NMBL/AEfH+WJziFRcu1UyYIdXUJcMBjdT +vLmV2EybNmRigIG2IhG4RBUbtC+YRrTAQ0nEgs0SYNHMcSbsis23JpKiPkHVXvpg+CaijcUxcuvl +L3wL2/QvXXENgK3tAVOMkRuK1UU3IKj/AJV6ta1w6qXRwyjZdQSVtAl2Y2l5W9+v64CL7LT6uJZg +O3aKSpMLbTu8N4UpqGPJM2qavwJ7he9Fs5VSC6heLSpYdVcC2aJJp8K2RHpvT3Q7tenSg1078Smj +wrBUI/fWimKAvBu8XfoemAkt2ai0vxithDZaIltKXfioVPDgoBEoFyVlvV1YQyIiAeu64viMSIfp +ywsFveql4huEREeq71wDlvw9H83iwwbVQTAkztG7q/CWF2rCJEO2RfD0fDhpJwN6qafwlcfuS/7Y +BXD7Ye7C3DAD767wlaQl8wj5a/hxOBQSDx4S4RIvuz27vDb8vy4BhJQiNXcC23wl4ht+anqP0wwk +jaZEhfb/AD+H8q+mJwC3TtH/AIf4+kdcJuTEBEQtTLp6Qu/5YBzq2hEvF8VuEqkXUJeH/EX/ACwO +OQtVFAjQtUO26+20qfniUagqAuh19Nt1wWjbXz1wA6QTH3q5AHu/CWzcVv5V78cO/tF5mkn3aLIx +t5sGrQxRsTAgAituvT07u/yx3ce2SVrYNzb8I+L+uMy7YOzGBzxDnxKJoPkklCQFFYrhOvxDbSv6 +0rgOaA7ZMwNsqOstyyISSyrNNuMin0iQcrhP1L8eAHYllOSzb2kRLaJM40RLiFV0zMSQAS1v1pXz +7sCc8ZHlss5lGJcrA9HwpKiZCJaDqVNC0rTBPsizxIZCzGM3HxpySl3CGkodo7NedRGlO8qaYDtF +2PAzo8MiabpISUVQUNUUnw28iHnbWtMQZNRvMmkPEgW6uK1yh2mx0LXqofw64g5S7RIHOzRV9ls9 +xqyt3+g+JYlUdbtKU8OJztZRo9Vkhc2oqJCNxbtr4bfS3kWAYVUTkGk0SgbbpRcW6pJgkREAc6KJ +20uw5Jk4kIol48zUtS2WwiCw8SlQeevPSheuILJaQTj1Xz4EE1I+5RmgIB039wqXFStcDknDpCPQ +UFtt2tVFJHoIeDMir1UqF3Td3YCHNvHSdqjFYF3X90YCpYXeXUioFerlXBaCJYdpdofDJqK7Krlb +dI0CHxDSuultcQ527LcU1cvgaqEggTl0k5edSqp9G8G5b82AEnIZils1rwmVI105FNJNpvpgCSQg +Y31PfCtev1wF3kM8ZbYvUkJZ5HNm5CooIEsiJidDLQuXPAKMzZE5tMUI+YQHiVxbGxTvG0R67yvr +pSug4qmcOyOaQdrzcyftJwo1TRSjE1ituoWlDVLTrGmObZhHMmX5NWQ3jjnC6pJhwhiJX09RGtef +0wHZMK+Yy3tDJahoC+kEifxxJgFhKp1tr4KaXW4wrKLHJ8tHv3c+/Xi3kI+X4tspq3SeJEqVqJqj +omZUPXvxXYftokGhtXCDBBo+ZvE3aZprHtlaNqgCnSmlx0xdf2pURm8nwXaDle9zluYK1dIQtJo4 +Itba0p6nry+bAEOzDN2X88LSPZZma9eHWIih3ghyaHXwhUx8yH0xdVe0px2ayC+Uhio4W8Paja/m +EkDVRt8YBWt35YwXKSOXYl6xEWexMKrp2MFrVV17kipRbcqVBS6vEHiHG3ZwJrm3ITybUh8uzWds +otxbvOJuVbJJFzopS2lbyHTADFf2hnhNFZlTIy4sbiEFxniSBUaFp0jUuf1wYaftCErk8syM8mOj +jWx7bhdaVtJL1rTq1PT6Y5omHHD5cXbOcvQ7aWc+8J4udV3TwVCvpVNGn3I2lrz8sPqsYtOPZsYJ +zl17JCQtjBPdJVyZBWhFQ1BEQGn/AKsB1UH7RWU49ukhmCEn4VZVIVrVmxq9Fe4/5cSm/b92XruN +v95F7i6R/s1a0tO7TXGB5oymcW+y3HlA5eczCqVq8EymCNddGo8ycOK0tAR05DrimRXGRsxPxooy +jBNcdkmcY5ScpEN1dBWXu0Cn1wHbDTPGVZtuxdxs3HKIyglwt3SatvfpSv8AXBU3xt6AndbS2laU +2fLHBEfGtyBiIg1zHFtECWkQjjJsbYSLXQlDtuL8sEY6NynJJG6mO0CQg16qVoDAk1XBIp/CNVB5 +FXTzwHfLgkWjvcXNBMi94S5ImIEXy60LH33K6pKILXCPVamsQld81taYHAoiVqHGAmoQ7hEm8MRI +bvQywg953aKDyR2WxkIriiC5CVPhry10rrgJircrxUUsLxdZIiYjr9QqOE3Ji3LrNdr0ioCZkYjp +3cq1oVP88BxcOFAJBNsxSTH3hoCCzUyG7xJlXQcE2ixEsQrhvolbwypbS+6NvrTrp+WAD5gtTNJB +8bHqVJS29Ud0KDyGvf7z0w/Dvickk5E91MhEWwkYGqOg8xr7umLAYpkySacS6QvK3pMw+vnzw+yj +yJ6q7XM/utkTvuEgu+SvxU+bADuIkuNSQbAZIiO4vaA+6O3y00xMZJuCtc8MumpaX3iJDb/u88Fg +FuhbaiFw/wCrCjcCNokZ/wCgsBW5XjEw3E0UFOq0bgMSEfTShYU3U2zFSwLbeohckI8h76UrSuLE +bpMQIusbvwFiCq8TUZELkwHc8XwiOndzr64Ac3dPHIAKJmV5WiqVq4/49KDg+BEKQ7h+8+IhC0eQ +/rgY3dMyblw2wmsXjJMwEh/PElJ8iqAknYSalw2isFpFTv0/LAJSj1hcKqqLApu23iSJF/W7DUgJ +IBxIo9Q9NxLEI/0w+ahDcuJmQlb4QEsOt3gk4Ftfa4EbiEukrfXTABVpJqmaSDkHS6aolaQgBj/W +lK64aSISMUxNBQh6htvSIR/FrUqYLTEaMk32uhD8RIiZf1pgA7FRNVJB3Zw4+7ESvQ3fw6F0lXAH +G6xEyEhO0RPrAguL+bkXdj0rJM4mHXdvjBNMEiUIhC278sVlw8GEaOncksxaIppXWFYNofmGla4w +ftI7YGso1eSEei6QZsEPdEsaoiurUtEqbZV08XxYDMO0gX072tlGtHi7twuuLQFyMeo1CtU8uXfi +29tfYf8Auy4/eDI58SzSFNNdgiZqqpHQaX6d3u6+eGP2QstupntT/eBf7SnG3EuQheIrGPLn3c+/ +HXOaBTFwTlN+u2WttIk3Ipd/cVR88Bz7+y/2fuo/L6+ZnYBFupJ1cglsqkIpCX3ZUup01r640aYb +uIsCtA1FFnQqGxTckIthp3Kp1IS11xfmjxFCPtF4gXT1qk8G0jp5VpSvLFWNw8KbfO1DBRNNJNMQ +RkrDH8SZGXd64BjbUdtHRXoLqSFok5JyBJKhTnbzGnOnngBl9jISUfLLsWdzVR4mmBuQNIeFEaCp +Sp39Y8vTAztleKZd7MppQTTXFJmVwEaSlxqFyup81PWmMo7D83ZwhJOAybNrIO4GSSEUHKhkQpDX +4aV88A7+0ApJZdmIwpts1dkquo4SapuSVQ2e8RKtaa/pjqPscnonNeQY6Zi0UGyKo2mgj0ikdOVR +rypzw1nXIcPmnK/AkCBN10veKiFxEGt1LP8AFTXAWHZw/Zrld409sAgN3ELmodypDXl+dacsAW7T +WakzCSMM2kgbFaKjxfwkklXlqNaefpjjjtDheEy5ttg2BTeJqM2ajkbyCpW7ltRvIirXW6qn6Yum +YO1bicvyKDGedEirIkntCwVAHI1PSiZrH4Nfp4cV6dzQ6YuDXm8vQ79FdcU3C8iZqAkr82zSvvCp +T1wGPzYqcaF3uyEtnYTRt2redvfX19caP2F9o0XFoyOQ88Gf7nTvTVVPxR61e5UK110/6Yoz2aUX +lXT5tY2RUXEngCijYJ9VKbIUHSg24BSCwru1VEkQTTU8IfKNOXOn1wGvZ17OEsmZ9QUmlgex7tcS +iiYPNpWSRrTQVRUMSpuXVG/F3/Z4lEYvtVQj1DfPWsw1KHkzWAOHFal9QBNQdL6UpXTw4pnY92kw +q8Gl2adosX7VywfUwclaS8atQdakNa/DjXshZByu2ze17SkM+Qj/ACwxVTcoCmAJEJAJDQbB00Pn +gMwzaz9jTs1lJMPZKjZ0omIRkUTlyulQtRGq6p17vSnw4qTduK/Zk8gpCbmxeRZE4OKbQgECVvxE +4pW6nV3/AFxrL2Jku0bND6ZUWQfw71cmDAE5Uw4NZQ613KgBeQ1540fIWT4WJhyaZLeLuU5Z+LAT +dnckg1bnQlST/CVuAweT7Jc9LysO0fxuVoseA41BVBZUeJCtLy3CqV1a8vw+LFn7MuyEnmeGK+b4 +eOUhZ2HN+q2ZOTQQZhTW3purUtdO+tca6xcOs2vppKQZoMlJmTGFaq9RAqxQG6ulfmrrin5tfOGs +rcxm2rkZaWHLisWwAiXYx6HKtEbeq+lOoq4AZlfsty64exzR8xdTjqSQcroA593wzIPBubdt1alb +1Y+9k+R8szGQo54tm4YdX3iZs0yREUqioVNNFBIu6lO+uCrJ44TcTXsebeu2+ZHiUPliTICEWyQc +1EK+VtLfF8WM17Ycms8ydoEh7IynGsvZdsc7obsU91wnSlTUpQOWlb6c68+WA6bRfOmwqsf7bQIx +EjQcMBck0qXylSvUOHYq7qTj0Ytyool71dRFVrv6fCVLeWKeqs6THbTMF1F3hNxXcrOGptv/ACi6 +i6dfPGgx4vBjxXXNdk4JAhVaovANJUacrqEY19cApwTxNuPArGKdvUkm8EiSKny6+VcCnEgiLjfc +tnW2V3VwAqiqXdcFUyraWBLuQlhbuicg+bR6CRJ8U/ZgqIl+aZDy0x5kUW5cPHaxxaCLZAViFNZZ +C3T4/FUcBeoov7PSTEwIfhtMgMf8J0pzwRVU21RJQzES8IiZdReflilQT58o3auRcunYqD0E2eIr +3fWuoULzwa450SpIyRmKIW2kszIBE/xFQ68/0wDUxnyFi8wDCSRmm66Ss2VS3Rr4Oqg201+tcFHc +oJASiBh0/ESN4/y10xmmeGL6NnWeeMttuPUYCXtEOPvBVGg89AIfHTvxYsnrZfncuK5kykYOWrtX +cVSUM7R08dnV3YC0gXFt99dHY6rvd33Fp5jW3COOInFwrIWq9NxLGQ+mmhDTA5u6auWQin/cfF1X +gaQ/NTqrityEw6d2pwjlBR9cRAYvBFJUKFzHqTr7yuAtjhZZFVVS8BTSStEiMCLn5U58+7EdJ069 +n7gguTciFQSEEiG36UvxBh3Cz4NwQfcOkXUKjMCJA6fDrrzHnhTh43TVdXbnvEulsTAhFUafxB0L +ASqqLKEIkTJdYri3SbKpDbQta30pSumPJSTxA9tA7vmBFyKto9/TQ7f6YAKuBT4VpxgCsqraDlNY +wt0/grUK7nicZEo3ErHTZNoqQ3+6MxV/QaahgLTCSRPmiBR5oLpkBEBiBiH+LpwnMbgULXK6xijt +FYArWkRelOWmAkYTxNwKhM0EFPESXUhvj849RDg/GCpIe8Jb7OmIiKBAJdX1rp5YAAlDpzKTOSkr +yRH7pJRG01R8r/yxzN+0b2d5mj5hVzEw67mJekSxigAiKStPSl2vdjtAExEOoLrR6R8Q/pgBnMm4 +tEt899S0iSQ2b7it9KVp/wA8ByX2BZ4cdnMezYzsI6Qg5R4XErleKu7UtKFSlR6gpT0x1bMPk0AF +2ma5AraILtgAejyqQHWmuOUs6tXGf+0OMy/lmKek6YPEyVMXO77q64zKlelIraaUpTHU1roWl3vy +RQHbHbZ9XL1oZVwCm6inDoJke24ckRWkYbRDTnzqNa86jirTcojFgRPeOTZgqo7XSJZI7RpzpVJS +71+HFkBR8lH742JkRdR8MApfnXn008sZF+0BnDKsW7jsuyjldRRJVNZVsmAEKQ99C6fFTz0wGV5t +lJbtYzB/Z8lKxuX36pCqCzYEiV+QaUu67dCrX0xecqdk8S0ewqEXNryjGJSInK5GRJb1eeqdKd/P +/LFGzX+7ck9SnSmDUg2F1qVhWrq9OvMCGzv/AFxfMpdsWUVAVhFwNo3SakRk2RtSSSoOpDTn0cu/ +xYC7uMxPISPKSQzICEKhamSFhHb1adWlOWvlijdrbeazXkyRmcszzGShU0hLYK1QV7br+fwlTXGI +NO1DM0FNSycO5aqRMo6JTg3CO5tbnOhCNa89MDgzpPSkYWX2zlrGxqxC2EUQ4YCGpaqFWvO3u5+m +Ahw6cauaRKImg3JISSIXIiVwDfUqHWvRWpVwSz5KrLxJi9Bom6cEmk5QTtqu5NOnPW2taAkJfXqr +gN9jbOjbQz81yXXFDg+GEwVSE61oW4X/AD0prgY72U4z3bb7Urt2rl1EI0HU7K00+L1wEN6s4ILn +bnfWuuG4yIh+tNKaYYVFGwRTO4h8Zjdb+mEGI2EQmA+HovuLnh1u3cOd3hkbhRSJQ7flp8WAmZcj +XktNNYuLZqPZBc+hvYJbpU56aVrTWnLHRvYR2Px9PZ55ohJh0+cy224aqGKTBAgG60wqVx9/y4uP +7PuScsw+So6WGNXQlm1sobpw2LfVK21MNdem6teQ4uj2cLLpyDZ2ALy0XGKSjm463KOnJWpiH5UH +AO5XynFxMPc0ZsY1x7RkH5N0UbRIRIgDWvcGlBxFhk28fmjJrRg/BdbL0Au5etRutVSLpvp611rg +J+/yJRAkig6e5fb5d2J5cQ0SbOz16OfVUr/Fz5YrQR/7vvc0sZDM5/vZGwiDbL6qP3pN1g5IV+E9 +DtHXAWPJiR25IqJIL5eeuXz1q5QO3gVfx6+nriN2eos5jMTB9Hnt5uj2cgslxN1r65wqNSv0tPp8 +6d2Bzh1IQTKamYTLfARLCJbR00xerW2rLFosSI87CprStfmw48FpFu5WJeLOXsfHtm0Pld41C0UH +CydK1Cp0+MrvFgG27zJ8S4YxCiK7iHlIBzahfVUo18Neszr/AA9a9xYsuRs8ZZylkuHh84ZffNpm +jWii+raq9VKVKtp1UGlaFWo0pz1wEcNXjYcxSy8UyZOsrw4wr6OI7ReAuH3255/TpxSs3yfbBD0h +o9nmJoTdOKR2qptrumtSrSla611rSmlPLu7sBvbt8+cuECEHRFdtoLtJVJUFQt8KiZlUe/Dhk8JI +k3ca+TbkVrls5ihMufxBtjz7sDAUWlgJyTYyUEi2gUYAok5C3ysOtL8SmTpSNbk2QBD3hEiYuWyo +GzuHWzpErqcu/AOgUeobVsoiDS4x2leDcJA5G3S2tKcrsDpNNiWTJaPmcyNYlEhuamKytyHVqGoK +V5h64hw75mu7dMXbbhk0FxUPZfqggudvIk7g78DO2BqtLZXVzAxlTFZpc4bKjJFYI0GtaCsnb318 +sBRcuZuyeLtqnm0GsTNOUuIbzsUAqtXmhW9QV1prXTGzwTN49iUH0bmaONS4C3VDMe8uXh8q4xMH +2RZmCVhs2g+y88FJO102MTYuSMdR7vXSvfpg/lLsxybGuI6Wgu1ddMVCTWXbk5EgeCP3gn1eHywG +pTWR1loCWDMkws9QfhYItEbditfQ/FX9cRf2Ymfs/Ji7NS/cbPlUwuAQ6KaacqcrsaQ9WJONFZsC +bkSEbB+EhqXi/LzxWcnt1ISdXaKAvsu7VErrdsjqVddPxYAFNsXmWc1ul2xmqzfiSiW4ssZpKkVd +wk9K+HTyxBBFqMZ7PXW3ESElHKQuQsXGni27+odMXDPEgm5BWJFyug4X+ztl0ekxPzK7T4fPHKXb +xmDPmUJ5rCN8yLv492gL1AXaIkZABeKulfiwHQ0Y6byTIXaDMxjR6W7rgw9+VOVito8tMOpN9twL +tzY2WuEgbE5cIKpH3ahQi58vLHJeR+2iciZVV7NxsdmGHdl9sakiQFb8JUrTkOnljoLJXadl/MWw +5aPzTdbu2k248hNmVvLVMw5j+VeWA0GScPlNpBj9pkF+o9t+BJLjTxaUUr4qUx6VYltJLoLL7KA2 +++jRMf5S0p1054CK5wYtJAWhTDFzIF7x0kmsiaRacq2VMxtKnfpgjl99EzqROWhoOxUIrz4MhK6n +rQa1pWuAlMkxFLcFHY2BJQGye8kQ+pJ0+WvlgrCPm8fxUlIPNtqQ9a6jkiAdPz+L1xmHaB2jR8I0 +XGC2J2SESFBg2MhVZlXoutrS4hrWnOmKKGX8zdoOWkpbP8qom1IRFqzvIWbnq5ipUepuf5jgNjnu +37s1i+JTGVXeqIEQ/ZESMVSpbyvp0/FjIe2Dt2azbjhMshIpkfu26qjbbAXHwa106ufr04Mfui3h +FUigoEPbEb769yF75AKjz5U1Byly9dcYZnXPk1JNF3cgzjmUW9ebiDFozAAfLInaRLalcn3c+WA2 +f9kKHY/uo6za+eIO5JdUkxBNYjVEaFpW6lOd1fTF67UO17KOR/7NlpV0TxVDcBq2bGSpfz7nhpXH +Ks3mqcyFmh0hlSSdRryQa/2ogQBtidedBCytR6acqYpntK5w6cyRunssSorA/JzcSRfNpXkWA1yT +z52idqvthpDOYHLkftCmux4xJInNLuVOqvj/AJeWMsk4V4KpLiEiusIiiLn70COnKo0UT5VpTFp2 +ZCZOJzFLZk9pLO91RfeZkqDFEen3tQ5ipWlOXLBaCy/ltSVdS2zl1eLuFFmKjxaMNIg5VqI2V6/M +q1wGWu3jxoyViRld9mv7xVIQtuOnffSvPliG0UIkiTIAVRFJQhSUOwRGo+Kn5Y3HOGR4UocY8Tn4 +VwLVNZq1ethkGdlSOu6LhMq1oPfrdTFNmMhtUuMXbSUc7GPFInIRRmuNhANd7W3/AP5wFbh414Nz +xSNB2KAisYCAmJXDoI1rT8+dPLD6pNU4f7YCiaaaBWJJ3g2XVqXhPX4x89Pw4GMotQnrxEXOwLIr +lTLUkr/DQLw1pTXXTDTtq3QBVcVkLSG5AbyIyKvLly0rpgFhMELjdFEy6RtVvK4Sp3fQvpriZmCU +ZKSRC2hWsOqKRJqVTvExr53p16afp04rpiJK2+IvDh9JqSgJbBm5WXK1JBMCI1dO/XAJSTUduEEU +kffEIpjthbd389Mat2FZRy7mTtaistzbAHLUUFBXJot0kdOoiMq+n4MGOx/9n/MmavZUtOouo6De +qkiJDaS6Zc9CINaVGmOlezzJbHsnyk6ThIcHLrj9gVbxVUckQ0ChVrXTb+o4CzTcKtINI6ICSuj3 +MiLlxsrEJi2R60xCvfpcI0xW+0KPWfNYxy9Uatvt6s0u4W16EkBtBHT0rUsW2RB86kpMmtlVl004 +9sSev2bUdVO/TFGmG6a6XCRLw5ZrJFwBLisQBHx6ZarK93jvwAyCasZDK8i3m2a6DMUlJybjhvHd +MyIkqDr/AA7benFYzXKFsZNicxN4tpLOf7TQnWhiRJNkAqYifxeVnpgwjF5oLMDObjX+1JPZEhbe ++/v0S15UQ/mK27FSexKcsD7NCiKjaJzvIkJ8Mjc5jGrfrM9NenWzq0wBNlNZmkHpOXbM085ZgXT9 +pxdg8OvFJalRdPX0pT/dh/8AeaDcxr6LYsJVCBzfOpezJMj2towEAU7uoLSHzxGSdSGa3sYxkIqR +TzJmMTRh5RsskkaESjpStdLuV/xD54hqrM4admFN6HdumSow8dBJgXDJAQCBuhGlLjL5vTAOx7VH +MGeJoUZVf2lJZiQQYKuLwGTatOag1qPir04NS6mR835kmXzp08iF2j0mSjZJZK0STEedPpzwOzA3 +9l5ShYqAePZBjDvCSy/NsrRVKQMap0Rt+USLmVfhw8lm7KGXboOeyvVWdZ125VROy1RzpSplzrr5 +0wF93G+0kmobFoR3CbMkVkFROvKpp2lbSuFw6nCSqQsTi3LghJvxSMqqBl3aCompfzxYYyBFKMSE +pV8wFAyWbf2qKto+mt3ViUrEpqPVXKDlBRwukIqmQNy3dPOtMBXQUWdyYx6iMoTdNcidCLlurtad +xDTb18WDBIqOyXeoIqJoubhtIEVUFdS+MaDStP8APHm+XxXMehiKyZWmqLNvfb3+VMTEoFu0t3Gy +GyXSYjGjYJU8+kfPAYfmPs/GNz7wmXXMim4ciT1WMJmBtbrdOYkXxelMBIpx2hRMgxhk8kwD9R28 +EbhZ2mkHmRj8AjjXs4ZNg52P2FzQTkBVuYP2TZUV0NC8NCGndTASVynnZKPXaKdp0qmiqI/afZR8 +SOheEdBwFszB2qZTyo9Z5Zm8yRzKSFr78bCsEre7kX+WKjm3t4yfHxSrmLmAeqJEO+QtiP6Uspd3 +fXBPLnZTktSMtfM46aklPeOZNbeJUir8+vg/lxXnH7PvZm5kN9BHhritJDiXCSRF52lWuAfjO1SN +mYwVCB1tkKYkkQCJFryBKnmF5eIsYZ+0tLN5bPqUPlt+g76dl4qif8YyoO3rXwpjrpTSuBna7leW +7O84Euxm+LR3SJqSLm5XaEdLSrrqWg8sVbs5j3WYsykXsEJFmQkifRtpCZgVA+l2tfPAezQKKb2O +TEweikgmjJs4zwCYEVKdfdzuwhxl2ej99ZaBXbLXEInxNpoBQdyvPypVMhxenr5EjSj3MlHQTNyS +aJtmACRiksJDXSvdVShtx10+fEyQRZzcqzhG0VIuyJBJy+VcrEkIk36V6VofxVSEBwGTTcMoi0Xe +iKI7a4p1ETIlSM6X86/Fbpprg12aZyzNBSDptDTxsE3KRe9WvMUir00KmhUsrz01xq+aHzFskWX3 +LZi9JOHGJJVkYkZGoO4ianyaENA5/NjHcrxu7mhncBshUMbEBDirQp86ddTNO5Pn6YDcOzRNjDO3 +xJooKOhJJRVq/WudLkI0uVZOaeIty/UK40Z7m4W0Yk+bLG/uVJuq+FsIgqVDsseofBS6um5TGTSq +y0M9YxLZm1i/aCqb8mbtYBSfXnrvtFf/AA31HpxufZ+xh42Q9koLA/lhako8NZEQkVUT53KDXk4p ++LqwFbeymfvaboU8sAnHtvdpNifikbYrdaG1cW9YFr4SxzvnXL+ZI126zM59lcUkqIvEiMLxSUHT +3iPOlda95jXG+dsuSZ7NbJ0ojmpjCw6g/wDzJhHLkPcCievu1PzxzhP5fy+0Vaw0PmF1IzYobNjo +w4MdeeqKlOXKvngKjsuLCT3gUtEVCK/15aa151LBXK+XXkzvu7DQh2FvtN+IXi2AuVK1p54smX+y +uWdsvaksDpNmLVJ+SrSxURaVKmpfnTGjZHy3FyjKMF2TLbRVOOkWTJYox+Il1N11R6dz9cBJ7H8k +wPBCmq2ay3GkqwIhWVbLktdUm6hBdb1jUNNaYub/ACyyWn2LsmLpszIVHLKPk2YEgnICFRVQUUrT +uUpdgnGR/wDZls3x0pIKEhGOR6UHzZUC0buK1pp8NupYNSTpxKNVY1u8jicSlrd7Ey5+8QkAG4Kh +9K24DJ89rReRTdT2yvBOJJLiIl02WNdiuPKizJVK7QfpjOZicarsmMfHsGMOiW6/XVYLbAuWVRvJ +Dc5jUrqlTmOLX2kSxLzSUAhJOsqx7ISJzHSLO9ik+AeoAOtNOuleWMHcCK6S8goANk1CUJqkmBEF +1S1qI1+HT64B+VdKIGTRia7SPJIREbLCco18BKUp01LT6YgmoS6pKEdpfD8o6fTyx5K5D3lgESvu +QBQLum3S4a+tMWfsvyS+z1mVrl2JWQTeK9RG4uEREe8dfWtMBEyFk+Yz1m1jleEBEnTu4h3j0AQp +3kX5Y7M7GuyNnkuRy8+JmnGzqrddJ6isjRcOXdtH8PPqw32UdmMHkn2dmRFgg7e0kybqul7gNsld +bSgU/mxrPGf3xBis6TUiVd9QVEamSiNR1tCnf+WAQxW/shjKe4lnTZUhVVZdA866EWnxF4cNyDNw +LSTJP3CzR1xIiytM1xt8VRr8WI6ScW9YrwSPu0ZZInbBJFEkLRoVKkN3z3Yr2Zc3IoO2eZLHQoxZ +cPOt2R3Ls9O5Q6U5kl64AxlREVz2FJUHMogahDcZCSorc7lA8unAfPDyJj5VjDMXhsnUoJQ7VBEA +2hAupVxX8tNNcVHtrzY6guz95M5dkoQnQvCYEq5RsV2lPB1U7ra89ccoXN1pOVLMWZJFzKMmYptV +WixOTcuCLUhEi15UH5eWA6NzpmrIeW8iTSbSadFwy4xOX0kD3HKAB94oPnpU7tcNuM4QMzl11nRs +8h2QzTwYdVqssQWsg++UEKF0GQ0xzetGx7kJCbgeEg04ZqgtsOn9CdOVa0pzRqNddfit+HDrLJst +JBKyTZG5jCtUnr5eVAhItwh6qJ+Ihrry+mA6BdosSyFO5oGeapov0k2WXWvEikUQ1qVaVWurS+6u +3gV2VZ6iZSS9sNm7V3N5SjnKEYigjti+bWaGssZa2mXixgjd4pPCvJLM3eYZZBdN65cXltC2DkQE +Ff8ABg2bNnKRM7MNHy45sdr7iELCI3ICy8St9Q5VGg4DpKEgY8v3Zy+pKoKwbRgvmtd8hcKSCxDX +poQlTpuK+mtPhxKyoz7O3eWmEp2hMWimY5BMnLpRzXQ1KVMqAX5VARxyzJZmzMmq8KWmHTIphJsg +5jhbVTJWP0upbpTQE6DTw0w52lTMzmzNKkzlaKzA3hTRSSZhUSPoAaDrr+dK4D9ApCWJiyLaYOnq +hdSQJswC4fUNS0rpiuhmBRR0cWg/fETAvtS/sfrQ6dbS1rS6ldfhwFcIpuZB45TCHdsUepVqUqtc +JULkaVdeQ/TFdZTDOJZcSQLisuvw7V40myJQdda2qApStMBdWWbCd/aX1jISVJFsKkOZJPPTQh1I +f8sSm8wzT+1oItd63qFQFmyo6d9eQldSlcU521kBSHL/ALKn2zh2qLm1OVAkHOo8ttTTo/TuwWXW +nRcKtiRznwrUwHifcmq0ER500s1MK4C6qyDdy0QIuolREhtcpEJF5UG4hLX9MPgoo7aAgu2Xeoiu +PSVwqj9dNNP64zSkhHrv15Jw+uV2hG+RhCSQIOeg1qNelX8sfcvyDcW68g2coKOLiEUE3izZygNC +58lCOh0p/wAsBa8xqFDuFU09gXRKiQkW7db+KohWluH1U1CbquU1uGLxEk2kiEVfOpDcHf8ATDTR +wT5uKC7ldRZyIqCScqBDbT01D/OmDG2m7DYvejcXSSeyuIlTurXppgMK/a6h2st2coTblF9vMVd5 +i6TciokQmPvEz7rK4yPs3i4Fj2eJTLk5h64epL77NtcIbyJXgOunSW2JV1xrn7UBFIJZdyTIOV4l +u5kUydOiAUklUSOy7l08tNe7GV51ecXxOWxeTD9wgWyDZojsAubfpqXLuubbutcBSMvqAUw2YkjH +QqTtW0V1A3XHWV4FStNddCT0/wAWLRHqSzTtIdZiXigcyyAlIuuPeCkKoh/eAER18VbqaYp+Wk48 +p15KO3nCNU1U0RQ+/dJDXWoKBbpQbTAf9WFuH3t2VkXyhxyDwRUckq5c3kSwD7wenT7wufy4A/Hv +Nr2jHtph1JRMgJNBZsg9+KtfeJjQq/CKop0rprgJ2SIqFnJdBjxzZ8kNzVdMBJdJahiFp6102yuO +hYH5aWJpxX9pOkEbVEySaBcrbQrxKlfl3KBrUbcK7N3gx+ZUnLnikyXVUTMiuFBfUhrVI606uf5+ +K3AbBmBYWPaEu8QRdRqkA63HTUmwqoRBVAQuOl1dxuoVPLw4tv7OrhTM2aprMzaN2ExXJRy2HxNt +OSazMu+o28iHFEzFmx9GlxUWunuHus2b9yFoKpU6jYOhrrU1OfKuNA/Z49m5bhH0zvPmEeO0JKrg +QlFLEOpAQ+betcBG/aVzYo2cEhGrINBXSJQ3hdTCVAdKVTrSn8al+KB2aN27Ro6Fszjm0kyQF7w0 +rdYuSB1WBIFCGmlwVHp88Fs+PNJ/jXzYHca7XEn8ZZ0lvXgLhkPwhWtBxZM0Q7okozJqEkg2cEIk +8GcRvJJ23SHh6XjbbQxoA4CoquHDRw1YyEIu2br7sm6dZfWuAopyNtR56fdEY91MXfs4bt5dornS +YRa5pRjx9mLk3bbTwUaF9ncc/FpjM4SQKC35RMJSHlFXigsSaBxjYmv/AIlqVK8h07+WNpaM3CLS +MaOVvaaKTUl20xCmIE8ji+83EqfEkWAkPXQiAuZtZeWRXL2TLNlLUpBsdVSq2W1u59NRxTMwZsWY +wjopQ/a0optxzmOeo8LIC48Ld6Ned5YBSeYo+Seus3yxxeZYWHEYU3LTdSeKjX+7uqhd7whrXTux +nmaFn0obGSlJv94czSCXALtnNwPmKqJe75fXAO9pco6bQ6GW1ZibXkl7VJ2MmkRJVs6HWgkB+hCX +PFFNwomqJCz2G4qipwal2wqQ9J6curu54Ovvbk3uqLzwSUo7AnEik7MRXSNDptur9K8vXEFJvJKR +6DGQW2BTtKOZufEqKneSf4fPvwEGQInyqTZMzQUVX22bMjHYSAy+f0u79cdr9kmXW/Zdk9dRys64 +qNeJrvBRWFcVxMRpdTlcKfP/AG45O7KomPlM4MfaUachDpltyZKARglr4CrQK07yx2Tm6Sh04JnP +Ciu7y3LsxinhRAEa/LUU+XoPPAEc2yEXCCUI9mzGPn19yMdJ9RtnNSvp/h53UxVp7tBl2GRoLNAt +XUhLtnCLBddvaSbsqlQVRVTpXWnTqeAvahFtXfZ6/aSje5jlkmhQuYm95GkFo6b4UrQqVDzLAU8v +i+zxNRb7NQNnDtdKYi5ODAuFuT6bFh50Cta1wF0zlmRGLOai5J++lYtyz41g4ikfexi1fhM6V6aa +088AJWJmJdWMc5sc8W4ncuiizmIY6bguEhqr9oSpXQtRIe6pYlzEg0lX2Xczt1nuUnk4t7HeOxQB +WOkkqULmr8I6lTliM4y2xyFlxBo5eKQE5HyhLDNtDM2LYFz01KhVrZ0aUtwEbMCeYMwZMSzt/Zyj +p/lhRo8S4Yb1VrqAIop61tP1+XGJw8C+Ty+8mYsGLB5lUuLfSah7rpUlx29nQ7R93p/vxsTLNGYB +ZDFwkOcopkiT2ym2RgQKokXLdCtPFXvrjPc1SUPlXtNGJ7Tma7tEnjmYdNmH3BGuAcONRpzOmoc+ +eAo80WRYn2EtEwq889YoEU05WWIW6rg/uu6lcMrs4mUCFfPsyS87NS6SqktHNek0ACmoCVSrSnLT +E6cdZjdrA9zB7Ljsv5hoMsuzRAU/ctjqFltKUK/o8OEPVE5ZaThcrZGWi/az8ZFs+vO9CPoNaV59 +9vO6tcBEn2843gIzMDSUQbp5qQUYKRsZpv7KZU0Egp6+eD2SW2didx9MloxeXRzKkWXyoJ9R7I+9 +VurTp566liI3DKGVo6dpAS9ZPNNJFJpCl4bUbfen+HW7TAIo9mWT3j9bMjptKRcmLCLiU1qmaomV +yxCdNPMsBbcrzUSxkJ+GY5Y9pSU1Gex2JktxXDOKdBERlbSgkPUOnhwf7LU89SOTmwU7Vo/L6DEi +Zt2pElStQTrpd1VpWutalT9MZ+oxdJuJvNGR33sGHi3gNEkH61AWBRYbS6dK92terHyQkMt5Wl30 +MyZ5WnWyS1yb18moZqaiNS0rSult12mA7DNN5HtEsuj7bXTQuWXFaHFddtqWoEnURqJjSvpgPmBm +3lJVBd2jCKN0iTUGRe5eVDfGnKqahCGlOde/BBW5o99qWIOUzuJs5aT1y7ET9aENOnCTmJCLV4Rd +nMP5J2kVzZvMImk5C74btOfPAAHZQqcg8kpuEgItq2XHhWqm8kSpAFK7iJ/T5aYkwjNP2Oq9YzDX +7SqNkxHTxJLoalcKawKFTXniwcGsukqJHnMmLa29JZZFVdidPi7/AE7sfFW71eTWkicSiZCIi0X9 +ipCguJDySOgnW7TzrgIgN5InfsYQmBcOUBXcoNJ5IgXG7QySKpa3euJKQvnzhBym5zEMfH7ggus2 +SeGloOm2ppqZDy8WAcemom3XjVNgnjkRJditCGkqzCpFqaZUKtCHl3a4NOPZ6IthjVoQniZdbwQc +IJK6d4LDbWndgHeMkkDVc8HuLLq9LUoEiSc6fxk6iOtC/PFhb8PxYkojHDtDcuqpFKpXF/NSnI8V +ZwnEuXvu20cm1SH+6+0lkhJXvGqNahp+uuJDJ0QsiUiXi/FOfdgqUr1JFT/jCYUr+umAxDtozEjJ +dpEnIJv0EGrRrwy4E2NURSIdFNsa06NB6q/NjK2TrMk+9VHem3LwldlquTmwCWt5VrrW2nuNylLc +aD2sSikhnV9BSG+yJVUbVeJCwnVtEwUqpTuDp0trTnjNcryTFtmiRTlDAWLv7ETp2ZEbYiL77p77 +aYATGL1bTSbdoxRXBYT3UB0I7CLQx3S7q0s8WJSojHZ0S99FIikKayRJt6rp6UGhpjSg0rW6vdcO +JKrVEnHtBi/aqMRdWr3e6bX07+impbZ0AefrgZm4kyV4hoZizVMVG6QtiSFIi6zS1r5BrywHm6hI +OF10DkXqKqRCBCj0Kj3q9/OlKV54GR6zy+1s5feLcEEzLxVIaUrWtPX1xYnyLcR4lRm63thJFwo5 +eD0u1OtRYaU52EFD8sV+0mL0hQcmg4Iegkwt7++mtfywGsnAySm++Js6fouUFU3gSqxIARgiPQI1 +57g/Cf642Psqdey4p+io8azDN2km2YvnVojIDZpwLsK/dq0+b4sc3qziLmPXkBB9LS3UJryPS3sF +uFKGNNdKKhp3fFTGrZKzV+7qpMc5Ngko2WYIO3XAWkC6VdKcYGuhbwVr5DgLRIMY9lmhq8LLAPYt +sRsF2r1baeR5kI6CN3eI1rqGBWcPbhOUsly0q19sTqqbB+3mg90SVOTVwmqPiIvPFuSnC9poCJx2 +bCbMFVHL7wjJxVSGlNSry309e/GaZ9nEcr8Y2Xee2m67UWiDCaO41WKhVNFVJYdbKpVPTATGUG6T +zQ1IY11khNgqJbROQOOKVQHw6a9AKjQufnhrMEx7GhUF20bw00/JWRy2/wAvgRilcVjhsaXy+fpi +hOMzoRPDskMxyUS8oahzKVnFIKukepuv1acjrQaVwGLNUtLO5HNrlm+bzSio+y3EZ0oIOP4nR+Mf +TAJza42XCEWo8ayLWLX3OPadLwt3romXrtlX9K4qzh06dqqvnyxvXSokQqEt1pFTuxMbuikm6SCs +Ugu8VdKkR+A1yW5ac+XIu6uGFkXDSNFpwaI75itvuQtVuDpMB58g19cARkBg+I/styuqzMUCQ40N +pVU6iV4606dKV88QpBRQjHccm5ERtDr3REaF4Rr8uLY3kvZKAIRbUBZlHJXk6bA5tCp13FKlSvRT +ny88V0eFbTcj7Jcg/R3VU25KI2gqlXuKnp9MBpX7N6hITGZE05hdg4SYC9bG2WEhVNEqHQSGvjHl +jeLJRGCmBWkv3cTlzTmI542MQbLq23KN6VpzDWzz/FjmHsceN4ftCh5CQeMWylyZMzco7qCtxUEk +ldK9PKv+rG/59mE4KTfISBuk0Y+fQdoZbegRpPEVAqFKNFadPVUi6a4A1O5kWzQ4gnbZFTKo5mYq +tnKoo8S1eOiDRMDMdR0qXLXxYEQgyWX5PIexsZRlnoqxM1J7KRtnJgNSpSpV+OpiPixAcC4HImZk +xZh+7sPmJJwOXn5iLpijS2qg6ULw693PDqrhQssZ1yc0bIv2MS+SlIyJkUSSdEjUqKKbR879K8qY +CBmuPfQ3ZnP5eeNnXtKDmE13oogarGZGp3cw/g9+Ck6pmb94HMIKzXKMHmyEJZIXv25qurbzAj51 +R0HFhZTyjTtA/d/Ksr7DFeCJ4ULNh7pVxXTp3C5Dp81LsDoThUHGSJIkTym8FVdF+T1EVY5yag21 +06rdCLuPXAAIx06j8ykxUikMoxecWqTYXzY0nMe5cIh7ylacxompSleunVgZnvslmJvMTV3lvY3H +48RIvCMV0A2PuxbnX1592Dcgj+6mZZFynMBDx6UiJTCorA8jBEuVABCuhBXnz0xVu2ftOfZOWLL+ +T34Nm7Yk1EnEc/FdmSRcxGg94FzU19MBTc0Ry6E4zRztFryjuj1N4s4aHurizMddqiQdA3V/33YI +TD6ed5Ris7DmAIKJeoFltmyaGRuSYpczI/P4fLniY3F1m+elY3LL+Oyr7QgN6VQTWE2zkadd4HWu +t3Pn8pYocetAxuWpbjo2RZZiUJsUOQnupII/xSqfdrXAWTPcejKGxzBknJgQuW5BgMcz9yO+5JMr +lnFNPBWlKeIsR0pDJLGZnmuQ8vTWYTfQ4NIp4sFSUFyQ6qr2fMOvl54EKzTeSaRkNJP5TM4x5KIx +0S0MgSED51PcpS4i9enGqZSedpTRxlYWmQzjW8ag5btn1gG5JqQ8ysu8XPxYCi5Fjch/vBIscySr +rMyyrBMWrBgiQC5kFBtoHT4qhr48VrMyT5F2gxcFGRq7JCjdVAUTcHQhIvvCpStKHpWlK08tMaOB +M8s/u9JZL9lu57LMiqnJuXrbbN46cFYnTbrWlS8XfTw4nNx7JYWUmG3bQo6d5xVkFF3SzQ67ZCYj +UdLa6cudP0wGuKqChNryintHeepE3bLrZeAmznpuHWzTQtKaYVKy0a0AicsIddRTbI2ZRSqTlnr0 +kQaF5fTEFWPdOXqUbBAaiLYSKTim2aritoNdCT1PlWpYfaFMCDWQcts4D0kjFPhnkVdoq/w1dD0L +n5lgGnsxEoSrWEYv4FZqVpFKbzpK4qcqAp161/zxKCeatlXke0Wjk49ICJdqnNmW6NedNute7XHy +MTzZDuV2BLZi42UAVXDNyi3XQVqPItsufV8VuPswM9Ht9tocw/jUVRJUnOXgI2fT4k+nw09MBLSz +ADY2uYCRevUySFNur7bAjZiWtLVKWeHWmFqyE4g7SiSZzacw7SJwaCazdVBcPirrZyLTuwFcOhey +ass59iEi2ajtG9y2qCDnUtLS6PFr6YFm4btAdSC+W8tMnDkdvg1GbgVxO7TcR1HX/LAXBxCyD7g4 +ti2zEUS2IiXFQG97Y6lr0UIK6j64nyrgSMpbhnwuCLgmrrg0bS/CtSo8qcsZzIJsWyQxZOYTe3QT +VmE+ItEhLWgLcvp59+CMgMWM3aoGT2BIKpkK5IuCZvLh8VSoNl1f88BVO2XsXzBMoFmiFZqLTF32 +xi/NIAIBLWlUraUvprz54wfMEwjJH7Pkoq2aL7O5VTtARVDpT0HT079fFjrmKeRqkZIoKMMrSQq3 +DwDdsqqrf57dbeY6YfZQMfKSaCacCxU4JL3qrSBEDEq+FMtwacsBwsKaiZrjvdKfjHwj4vT9MWmM +i56ZS4ZSBkZLc8KiIGZIFUtaFprbppjrbL/ZjklebSevssQ6klvq8UKiwqhcXKmqSetpaeXdjbMu +Q8XCMhj4Zgg0appDdw1oiRUHTupgPzwkMn5mUh0F2mT3TZ0kuqJqkiRXCXLnUq1w7lTshzxmRVqK +cDIpt1PdkuQFakXrrXH6ImLMVeqwiIbSut8PrzwFdvBbKkV/DJ/GIgj1D5FgPz87QMt5iydmBLLu +YpUEiFcXAfZvddIUDd5/y24C5XfQLFwTl88kRkGzW6OVQ6gQdAXKlde9M8dS/tYKZTd5KJOWcoFM +L/3EhBIlyOncPRztr544/epuG32ZREB96XWPVu6eh/FpgLFIZ0mJaH9kqbDZvvquQBqsSQiR2XjX +8FbfDiLOzDPi0PZvFLs21nDIPwuFKlQpen81vlTnit/BcOJyRLPnou11kFFE+ohWP4RHkPVy+mmA +eScLGyXUbImInci/dKdfiOlac64fVlH14rtlgYNytUBJsZWoaDZXQfIipiKrIJ7Qi0RNp0+8AViI +FSEumtRr6YigoRKkSl5EoJdXxXYAjKyTqQcM0yWNZuxSFBsSiIgQpUK6mttKeuHQFaQcWoXuXjlV +NMWwgV6pl3cyrXTA5kLh2rsJ76rhQSTSAQvIit8OLXlqHFSMeO5JHaTZLtnLqTJYd1BIh+6Ee+4v +pgBSRNWaRfaXrZbhyFdLZ92q4A6e50+WnmXdh+6UzA3eSSbACFV0ncCIW9Zcuke7EVuIyQLlGooE +4SarkrvWiWzQx0PWvTfzwYbtYeNy+845hNoShPGiiSpBYLZvaNaqV0+f4cAFJ83hHsY+i0VxfNhL +juJREhJW7mOleVeWOqnqjfMTeTbNv7UazWVknqCEi5sEVUb/AO6lXuqNfixyS4TubpOxeAv94RId +RbAVLTXn82N67GnEpmjskKClIr2s3glSdteJbFdwVeSopqVppWlO/TAFuzpixksuvGwt32cBzJl0 +Xj5LwPkHCI2VoB/xPB4a9+CeXJaNzJnDJ6c68DNyhZbUbjwVyD9odCGhjUKF1HSnljzSNFePkSyL +MLuYPLLpB7HKkCoyKDUwAlE076aKDXXSuuD8bkWPXjJqZerLryEbKJSscbREGb4gr4xPXTTv5lTA +AuzprOSErCy04f72Novi4VzDPURSfM0SrcJqXa1ryHywWzbnDsvjcqHE+23T+BQfisvlh3aJilQu +Yp6jdaJfDriozXadNPms6xhMsLy0jv8AEnJulhB4KNO/bpTn+WmMwtZ5kz3HfY805jTlECHbfvwS +dEdvIQOlfDb83iwE/tDksvvpqRyyhnBCRy6RcawkWsbU1xVLvTUoNaXUEddSrgBk94WW2SuYIRtx +Lxirw66i8UK7HaLXqLXwF3Yiw7GacxMq+bMFxdQlopq8eKSrEKd47d1x0t86YPum76GyvGJMniDR +bMKW88eJzBKtnwU1rtLBr7uuAhw8HMSVkWTORaSUy+J22YiwERVRMfeKJqU500+X0xbofsb7VM1p +IZfs4CFZJFsG/tAUgvpqNdOZFrjeuwfIMDl/KrHOM62XXkE0AdM1yuV2hMeQI0r+HlSmLa4zMzUA +UYmEdPZK3ebwSYWigZeFRwfhDXvrrgAHZ12R9n/ZvDoZqYm6UkBSFMnjm0jVKvIhANOm4saek3ap +gMo5A2zzhR+MbmKVvhDlypjOWWallXq6Aj+9WZWyopvHg+7i4oq940KvLpp3+uCj3MzqfdIM8uma +yKCoDIyyaIiTmtC+5R9da95YCH2gdmMHnhJdyoi1bSnGEXEpo3CIiWtDPTQqVqGONu1ns+Xgc8vm +yKEo9ZrVos0Xq1JSppV5Urrr60L9NMfoI3XEW679K9Nsklc+e7Ni65o/h+LliUlFNn1KvFxRMV9F +ERJtzBOtOQ9351/XAc4KvGKEUggo/hFuLfbiqS0Csg6Yifg8OvL6YNRieX3b5JoLbJNW6DYhVETc +bTvSvK2tR92VOeuJTdOYFIZltmSY2XLoRjnnEt3JFdztPVPX9a4RBNZxoy9ksUZsX3FEtJoOY1uY +9WupI9HdXANmOWXbRV65hMrOWaH3V0qsKrYu4j8OttPpgg3j30Tlwtixyo7K0XyeYbiEO8KKVqGv +08OJUhHvHMm2T490KLZIrn37tpCCRDzqKnT50xOVUkrFXYo2qCltpAnAgKDz0KuvxeWArIfvI0ky +hkwP2SkhvOmaOYQVVV1+JO4KD/XA5xIE+SdTMstnMYOPIeFdFJN+JbFQdK0qFNbqa4LN2cgLImy4 +bjp+JEbN3l4RXQC6nhIK0rbTHnELJC4axK4MXe0qKhOW0CYqCdC1DcGp6FSlMBHis1QLFw+kpLMM +3x0jtJtSIESQc/L0UrprX64ek3m1lr2WutmVRR2vuJMHINSP/wC2d+mnpiTKotUHCssu24Zumqm2 +SQTYNwbKn4daiYkQ1wYgOzdZ26Xkm0ko33xK1s7Z3igRd9lfl/LAAIwVE3DVsL/MUkiySIiSUNFq +5E7uQ7lCrfph2PGWeZt4ko18om/Q2Wy8m8V3UtPhrYnbbjSIrJsShFey5BsDsU7bjURtK6n4/FSn +pixshUTZC3HpTQLpAjIitwETLUe4Yx+w5Bim6ErjVZdN3prrTXBVuoV/u/eD8REf/wCMeNNbiPdg +FvxdHw4C59zdA5Jy/wC1Jl+1ZCRbaQqHbcfpywBN3am3VUc2J9f8Y7REfm/7Ywft77dMt5UinkFl +8wfzSokmBtDEhQP8Rf8AbGNdrfbpmbPCLlWCWOJjQZi2ftVFh96VVR0IK1pz1+mMphGbds3XknzA +1xds1eBBPq97Qg6i86efVgDUw3fOeGls1LBLPljbEKDsyBVdIiurRLXw059RYgoFFCs8vP2SzVB8 +idzbfa601NNFLnrdrTv9cWY0XE2ynXb6eZSyiUSzJAVkS3VxHuRR8wGnd+LA9pl+YUzR7ETZrxb5 +R4V4uzE2LNJYCpXWtaeLngIGc4FjHhwblgEXLHwzlO1YlEEm5BXWqqlKeKpUp5YoppkPUQeLqEv8 +WmNVPJok0SUdrOmzM4xTbXJapjLum6vhEfIba4gTuVXQtBdiwXF4kgD8WyZiSAtFOZWU8Vw93fgK +GbNZOMGQJseyuqSaCvTaWneNfQqYYBMt3bs6h+H5S+uDRpyGW3aqCgfZVCFbach0rh5KDTEOQ4N2 +quszPgk9q4m6nivqXhCtO/ASsqPvZc2lLKIggmJKJgQhdadvlz5fzYdm5BZ8bVGbBASFq2TA23SC +QXU6lPmU078QFfajaQJfxOEEvEICQiNAp5d3Lzw1vfZ9hC9sQkmRJEF9x0Hxa+VPpgLVGuIeHNmU +lCBmGPTSckyTTebZKGVvvlNBrWg0qPIcCfbkwpHrrqTy6lzxJbaU6iVOgjShV5fBgOCniU+7U6iE +hDqIq/DyxaOyrIcpnrMCUXG/ZkSuFd4QXiloPdpy54B3svyXIZ6zGQqAunHkW4+eJtrtrq1rp9cd +rNI2DhMvw4sWz1BNgQpgkiz2isqNtSPnXWldcTsg5Dj8r5cSho+NBNNMbVSEFR3T8y8Xng29TTE/ +cHcNvSF5kQjXl1dXdgMj7bZLL+VJOMkk4tkLiLFRsmXH7S6iRDckfSNaaUKuMbb5+zJn2Pmoly2a +yxKx1zM1HgoOUjEqU8ddKH/LjSO3rs0eyT2MkYiHZKVaIW+8BUkypdeSapVPwVuxhKrMouP2pRaO +FGJV4lJiszK1cy5ElRelaVLTXlgI2SVGse6Z5km4prMRLBXhnzEnmw6VMrhpTlXqHXDsJDqzaM62 +iYqIYrR6nugUf7D5OlSqVoqV5KaeHy6cSwJvCNEkF4Fk/LNDW09yNVA4y4unZO7qLl54Hyb7jp2H +uABbwtqLp8jFWroJUOoe+GuoqcvpgLHOw8lG5fytmhRbKzlvbsk+bPLzVEipW50jTqP5K4v/AGP9 +n/HTbXPso5jkItgqSjX2UwMhkyPlUNs/CI6emKbkLIcfnTOEwSh7EHHvE1imGzYWYiZF/wAM6F5d +w46Imm5Rskll85J6yjbLQbNgEpGXoI/wRGlNoK69ReeAUWYM0T+bXgxsWCbOFMUWJJrDwLEqhS41 +y7jUp5CPSPrhcwxfPY8WSaMiMKo6tLhFhSfTzmveVS+BCmGcsCDU0GkkzNZ6venTLrJaxlGpVKtb +3KnmVvfdj7K5iTkJ1WSQcGhGtEuCKWEOi6v/AIdgn/Er6n5YATIZZnHMOllltEx1rZUVCjG7kgjG +ZXV63a3iWOnyYvWS4OLy62XzNMZgB8T8dk3BaoJICHKiLVH6lit8Uqmm1QUy86KTL3kTlNNb3hF5 +Onp/154cjmLpzmBjISrxjPvodUl5KRdnawjTr/CQTpoJGNPPAXqBRmpucOdlHBobqRDEw5XACaWv +NZTlzMv6YsbOOQvcGUu/MjWqRUFwIiFdKUtpT05YqXtQZlkC3tVeJy+ouHDPCP7U+K7XopXwp17u +7FoZLSJUV32LFjopWgJ1UEiqOlNCKvrXAY81hyeSrqUdsItYRLbBq4gSFquN3QVwjh2dFq0ZE5Jm +xJ4oW2ZDGuAXZiXnrQLraaU0xbmmT5BtFJM1odld1EXCPzQAi8iIKiWnP64aOBzQ2VSubSKjdsO5 +sJzAkV1O4R1T6+/zwFKjLVEmKe9DruFCIVVUXjhAlypypuh82nzYOyDxi2aIRbT92kEdpNZRs5kj +FIhoVKdOnT3+uDUfH5gUk0lXLbMSYgO8ZXo+KvwkPnh1JrKIKkmUVNrtV7uhZsir3/W6lmArLJrF +rSZPiZwim1btF7buXEq/8Ot3hwVBu3eq8W7YRyjpPcTQBY1lXP16qUrdyxKDLc4+ZcIoEi0R/FtJ +GI/Qqa4Iw+VyX2l5s3xOGCv2beciqRB566UHvwEDK+V49y3SUUbAmIqkQkLDaMht0tKhUofd54vk +ZFsY1kLZszaotx8ACFo3fNhYEJOBuMC2/hs8P9cSXCyKDcl3KwIopjcRKdIiP64BsGqI22+7EfDb +04CZwzhlfKrdJXMkq1ZCors3KdQ3/Wvl+uMP7bf2gCi3CENkB+1XcLlauvZeaWpWXJ07j7vXHOeb +fbE7Ie0p2YMnkhxKxGpcW6QiVBCiX8MqeHAbr2n/ALSBOUfZ+Q2N9FW5bij3pMeu2m2P8TX6YwqT +zVOPpvjp+SueJCuoJubrBKgWW0T8QFyx5WLko8yTFyDSUTdNNpmXU8G0CrQkuWnnzw1KyCbF2l7Q +bNXMhIEqLxUjuVLUq6mY91K4CrsmrVeSSduXMdw5Ok0yS6hEurxfgH1wfgWbVs8QbvtlNFzxjUhY +LbpuR6SEbKfw/rg7GZJzZmtJ5+6jNr7PX94e5YBkKfeI869NK4fylBvhmGcpEgBKKLk0YbgAIKnQ +feCfV0/T5sAdaTUe5jF49oaDuWd5fSZXqAIixMfgS+ZSvzYGLTzt8/22MPwUcLhBw3YqBfVy+Qro +e8VfWl3PBplkFu2cbjQAbJyjwmvtXZtOKc+SNKXcta4s2X+zNGQSJzKNuGar7kU6atnJBwL2pa0X +rqOtplTAQ4x9JL5rGZTZxyEwqkT+FZtFr2ot6jY5Tsryv88QTZtWjRr7lrJCwLi4VsSI3yqJnWpp +qUDyArun0xO/cPgpj7cC6cpvpsnD5Y7eDdpiRUUpp8JjbTXEbKgxvtNrLQz/AG5Rd4rtIEHTGSNC +qNe+v3a1KcqeHAVHtgyaOXYJi7UeLu30gkLtjt3ihwneTbq77O//AA4yllBvnaW8j8TVR2l0F1BQ +qUqNMbj2zuCmYVrleJWUXYOTN2wF17so94A+/QEvTv6cUd7CxKcYl7LeLprPYziYzbPoScCWi6P+ +L0wFGjykHYKimG41ZDvKpdNgjbpdZXl5YgKrEoBXABLKdRFf4vp6UwYZOIlSFdIqImMom6FZILOg +kf4iZUwHIRILrwtUIrh+L9aeWA1LsJ7I1s9SyTmQ308vp/emmsKSpfldWmOx4Ls/g8utEm0TDhti +kmIri2b320HlWpeeAH7N7MlOw+AklLxJVAi6Tuu0LTutxp+yRJCShmoXhH4em3lgBMeooQKioYEp +4blASErfXDCTVuglamG2REVpfZ+r/diUyRZiqruHt7YkJeL/ANuFAioNxJublPh6y8/8GAGO2qKa +SqCh9KgkJCoCJCQenf3YBPct5ZdtxbO42LdtytIgUbN7breVdK4siorJ3Fx7obRtG4yL/wDp4fbv +FkwFteBCI9RqGV3h/wDp4DDO1XsxlJSHVGGzCC8agO41jHLkEmzMR8k6J186+WMuyZ2eyUpMR0yM +r7YRkFfZ0mLB+YvGPwkoY16yHHWM7KLR7QXMe2XfqFamkzZHcapV/nClMZTKs5RDMq8s7W/dxw7X +QEkhfibldKpaWLogGlOX4sBcm7f3oxOXZVDjGyWzNZtdoiRFp3UD4VFfrTw4nTEe3jWDh+pmF1E+ +0rWwSNm/KPq/8NOlfAPpbgTCzAk+blMIoZjzImkS0XARCJA3Zh3UJQq0pS6vzFiaxlHrycSJq2az +mbDoXGPCPWOgk6d9KFWmhFT6YAItDrNpZNqUPxpF/cctIreI7uTqQV7v01w7QZAsxKsvaUc9lIdC +59IpgKUZlwa+KidK+NbT/wDOLTl5ipJUk0YSXdA1clbJz3/iXytOVjf0GndriBG5Ti0zGCXhduBa +WrNYfeI3L5xXvXdf/nAU9FwLZBzKISL5CHfq7DP45bMqtfkrXrTRxNeJOXqjPKqjBjIzTfbXVjiO +nszLqPfQ3H/FV+nfi0N2b5jJlEi/QTneFNxKTBJ0MIxvTkCSNO4a6d2IMSxGG9mNIlguUK5dXJC4 +6nU47IruIWLv2x1uwEl0+i4J6rmTMDx09sVSQZyaiP36tf8Aw7VtT17rtMfMy9oDJtKmMtnqDyg5 +KlD9mORE1gGvcSnylXzp5aYmzb5rmJ4q1i6pruI8th5NlaLWP1p7wUa15bn1piBl/KkO+jRdQOSc +vSTAyLbfTRVq5d1pXSqta176VrrpX6YDXwEbLS+XxEeEq2l1D1KeEcO3dFw//wA2Gj3PFYHi/FgE +AJCqQkd13gIrbhH/ACwkFFB3RK8tzq8Y4dVJbduTC78N/ThwfD1B4huL+bANgO4A3I2/zW4UaY+G +z+Yit/7Y9cVnTjztwLRkq5UPpSSJQrvoOAizchGwUUvLyCiCCKAXEfTdbT4afnjj/tY7UMydoO+c +a/OHy+oquixFYOle0R1oWnxVu8OC3aln5x2oybZoPHQ+X2zxsnvj1GkqZa0qQ08X0xT27OUQyuxi +WiKAtXa7695s3hICJB0080y5d+AqPCx8M9SaLsF01kFWZElfc6Z6JAdTTpT4da64am5xRtGcCpwI +iuKqgPy6l1xu1tU+QvTBLMqkXDZoEXMa+iU1CbOQVW63iWgD90pX4fTngc4h2Mw9JVOeais53CJy +7uA0FS50FXTzL8WAcAph89Zk0Dgk3rpJxGLufeurwHnRNWml3f4cWmPa5dTzw6cyFijctwvaKkad +yUhXuQWCtbaUr+WJmR41wnl9BWSilFY1svsNUhf+8inVPCuA1r93XXvxdY/Lco7jJYXz90TpchRz +OzIAO5Kpe6dpV/64Co5HiWbmMllF544laQdJovB2SD2eY/dq8u5Mq4ubjLcWo0XdqMGrIk9uOlTT +/wDDOqc0nQ0p30LzLFyy4izjY9djIPPanBCKD5WwbnzQ/u1/xKD54aOLdNJsmy7YHLdJAWD4iMbX +LI+aS9KfgrgIcOzbpwjxSZRNgL0uAndsyIUFqfcvE9ddNaW1rXFgaLSTmPMpY2q6g2x06SICJX0/ +u7sdfTENoxR4dWPmQXUKNH2TLGJ3AqxMdUXP6a484fC0VSaO1gtu9kzu4HXtGOjdz+etvPAQ8xrM +ZBwqpOgumJF7JlgLq2lbfszmmnrz54ocnDzCfHbiLGJREW0C/LhupdWhe6c0L4aW1HqwYm5R02zG ++bSxtSWSIYN+YnYJBUa1QdemtO7ATOE5INGW5KOd/wCx+xZYSO4L6fcrjX0+uAxjPDhOUzGSD43X +tbqTctm11qr6hWgVfS7D+bszNV4J+LbLy8W1XVTUY/CTOQSGgrf6vp+uEvk55zJ8FLP2UWm9dcI6 +kekPfD1pl6iNbcCc4IqJxQu5A13pOVRJR4oY3cSF1DEtPmGod+ATldrLIO2slHooPbhVeiFlxKkA ++8E9fyuwCklouxsMWC+4Ilvmp/F1LlXT1xZmmXXybRBzEzCDYlIdV6QLdKgiJFRZL866d2KbduJJ +IJtrrbi6QuIvzwHf37NXDl2KZbubOl0xQIkzTAwHxV/F/wAsaM4TFc/44ppj4CA7i05992M1/ZtE +WnYrllMg3CTQ8Sm0J21IuWla3UxfnCJO3Y7DY7R6bx2THn69WAUrxgpWkBlb4rWxl8XL4v8APEF2 +T5NkXCNjXIS6REFfF9OrE5JqimoTUQREfAoRbOhV8vPHm7cW1qCbbcT8IkIIj1V9OrAAOOmuHLiU +V7i6rU0Vv/fgnxT7h0hFgvcVolcBj3fWpYHvWMb1DwwJl1e9JFv0+XLUsUrtdz84yhkfcI0BcKCK +LYljbkQiRcyENdTpQeelMAMzXOReZMxlH/urMOU0FdkZVOSVj0t2pa7Y9XUVbcXlkms5fICKOWoA +nKRNyal9seKlbpT3tPTGLdh80nmIlynQi5SS4pQhKefkG6rSz3gt+9Pp+mN/yoo1afZoRyyeorqq +LPHTQBJBmVB16Cry78AJyu1a5SW9jqNuARcpWqmosSr+QK3S0R8VBp64hybOSXSVjVofeRXG5llh +qtRsCAUr986Wpzur8uBaVUWs8+kMpyXFunStkjmd/XiFbA6qos0ud3d4R6cSHD54WWnSctFO2jV8 +vti2Jzuy0qVfD3fdAX+2mAJtX0+vGkjl32VIyLRUkwep+4jIilRoJCH/ABa0wHZS0fCZYeKx8w+Q +Ypq2yeaX4Fvyat2m22pXx615Ut6Rw5KQryPjmLaWigfrIICMblSIMkmaF3eTlSug10/F+mBqKcxO +5iZu2z+HIou4eJRbVKOgQoNfubh0WXr9e7ASlt5y4atHeXl4mDVtcDE8SRScu6/h73O4E6aa1uw5 +MOHjl5JySz+2XaIbLldH+4wLb+IIF3EqQ4XBQ8e2buZBT2w5GSVJFWWX6X0qsXcknp90l04OSbXL +cW1jovMjsItND7WhCMlqlugFNffW+Pn54CgS66KjWIqpFGqyUSJPLGUbDSVXr/8AOOufSHxVu8se +aZeknqW+5j5rOrita0VkWL7hWol5ooAI6bYd1K+ddcWdim4nMyqi5jVGj2SSFebfkdvBtO5JkHnQ +i9MWFyhKt1OFr2iMcqgjTbTjWzZJWiYU8NSIud9ad+A0S7o6f92PGJCHT4vhwsOrqLHumy7ANXKX +9QBh0LvF1/y4909JeL8OIsrJRsM04uUfoNG9wp7qx2jfXuHAPqrN2jddd2tsopBcqqXTaP1xyd28 +dpUhmuaKPgni/sVk6UT2mi1irnQBPeEudCHn4cQO3DtWzFnGUk8ops30XBpAuK7ZoFzxWqdn31PJ +Hq15YpeT/wB3Y2PZsXKzV6m0dXLoIn7q1RELTFbv5VwEuM9sJpKyCCyHGNl4/hpEfumwW8hcBX4K +V7ywluxnotk6Xj3JucxNl11pMr72pN1NOpGmJ7eDnJ1psSDZAnTlqm2iXLBawSNDuSXoWtD105XY +tsJAymWYLLPDHvsydKEPEgIL8R8bRTX+GXlgK2yiZJ20jkHkU6kpJsKai43iYLtO8FU/xjixR+X1 +pR2qWyu/WU95MJEzFMXKNC+9T/8ANDzpg7lSBFtIIPmKK8a1dvCWauiMrox3TxM1PLbKvKmLarvL +yCD4nPsclXRJpJEdvBvh7hrp/CUwAVWDZsTXknwb7xghbZ8ErH106qUp8YYkg6WJug7iVkFFmCW4 +1VLxScZX7xOvzEHP8sRjavH1qaiJoOifEo1uMvsb2njQ5fw1R7sCXrV05kEJCEZrobW64Ztr+lm4 +Aq8S109DpgDsS4KNkEnLFmo7YtPtbE7x+0x6w+8RL+StcO5iUaxMggu7M3Ps9Ir/AISXjl//AO2W +AWYpD2EkzGL3yaikMtGAPgVRPku3H8NNcGDnE/Zi6ikbxIxoj0F1krHr6a86/KWAlQ7cRaf2g5BR +u0ujpW7+OxU5orV/LXA7OdwskmLta0Ykhjp0B61V2Rfdq0L6d+HTT9lhdIOTKNQSJg8EQuEminNB +YvPp1wEe5mFNk1kE4037xJJSDnUr7Ssr0pql6/8AtwFIzatGzOVxfKIrk+hyKOmBUMhNVqoXuF66 +U6qU08WKlmJ019scDNyQRaIoJxMml4iVtG5F1X0pzxa3ZFDO9vMD9BAd32LLeEt1oQ3pkNKd9Brj +KMyyCJSAtBhEHakfuMF3ThYrXIGdaIKV9NBt0wDEISMlNi0bLHLOH6oo77joFBa+m2pTv1qWJXaL +Le1M0JFGw4JyAkIrtkw6Rdj0qDSndXURphrL7N5JSr4ZQ+CZiPBKumwWg2dAOqJV09a0w12XrOCz +AkxXRBckiVepHeQqbolS8qV118NC/PAEcwQaL4J9SEeAm3YIJvT3ukiKoU32wV9Br5UxRW7wmwKq +JommRNdki3v0qXd3YueblmMpDz60XvoRaboHMWneJFtLlot+XPXFAMiFIrunpK7owH6F9ggvG3Y/ +lklzBRRRn4yW+G6v4MXlkRNktxy52/is3iMbv9FMVbs0ZjE9lWW2xAuntsExIRMrefPnp+eLYkQp +pCK5r2l1ERXXf564BSq3UShIoEPwnvEJFr/gww7cLJnsCFpEXwrFcI/6MRXr5HdVJB509IhcZdJe +fnhbclF26qm8Cl3hK9W39a0LuwENInBXFfcJDcFrkrrfyqljC/2k5hqpJsYld41TcJtTUFV6iSrY +RqFeWlBoVFKeWN7abiCVxH74riG01vDXy8WOeP2hXzdt2ipLprXOkWe4OyiRkzGvKp1E61osPPQw +8sBmGTM0I5fVffukaDZm5QEXLmTbCb4deSotT1tP6a2+LG65dTcZhyoxi4394l0XbUU2sY90bhaJ +dSq6g08Gn+rHO7Ru1ThxdxcJxqLR0oL6Vd//AA/r+Krf+F5Y6c7AlE22V44bZV28XIRbILPN3fSp +zKz0RGv9MBdDkIlhJpREAim9ogCiiT0QAWcMNB0rqp5lX6YGKz2V4dAW8XIKXL+/eTJBU1V/LRHl +1FX6eHE+byOnKNI6LFxttWC6qx8MG0xsv02lBHxFQcGsj5TiY9v7SFb2kS9ynHOQEdoe6gop06QH +8sAHcJvJ8UOJbLtotIRIIdM/tj7XkNXBfAniZDwck9Fq+mWbVJREi4CHa/3ZidvKq1afeH+mLO4T +Yxrfi0TQZboihxxdRkVS0Eda864oWcpKU4h4CbNeFy+SpdTc7X004qPgQD4KfiwBGWnFI9RVPLyL +aazOqkImO9QWbOo69VS0rQKU154pq3s32euUlnY2CJXDIyzQPtMg5IetJvqNS2h+lMQMvwLpiqMM +5hDkZd2qLn9245ztM4xLnYTtenUodfOhV6sX7LUGaL5WZfBHyuYU1RaJJNAHh4YflH09a4DNQWYx +7eJkI2NmxFddRbL8E5O59Juy8Ltf0SHy1xbzn5ZgXCQ/Zw6zTt/3ySMtdxzXmoNK6c6DWumv0xKk +6lAZxUXYM/3gnHSFs2/UPrQC3QG6FPgIq/DTy6sD3uXidEkpMzWZYtxt0oLGDoVW7UNa1onWuvUf +PWtfrTAbKAiPhDHjtEPBh0cJ54BCRDZ8v+3HIn7X+eJJ9m9vHwj/AG42NQVRXVbnRcCcW3+AdbCH +TlWuOoe0CY/d/IsxLCdqiDVTbH5jqPTSlPPH55cGIq8c7cuiuJDiVWx2GruJFeFKV1offpdgFhNL +RJiS5rpPFCUHaRW+03LAHXv/ABjy501xfsiZX4FoSko23EyY2yO2Hul0ajaRp/8AmI6eKmKj2Pps +SzR7ScogvHtEhTcoKASpIJEZUorr5EJbeNGOQcZZnXjknJi4UdEmLYgtSSdW6VoeuugLBbX/ABYC +5wWWYdQxfSD/AH3iAppya49IkNf7q/T07q911aYuQRayce+kJ1mu9UVIUZZIfB/5b5Gnd8uumK63 +mm8f7H9nw7XhVWpC1QUP79uX94Yl+MPEP8mLFFPkRkGrSPeLuSZNVHUYfVZIR9fvGta1+NOtMAAz +Q4UbOHSEzKmmmQi2mCT6Rsr/AHd8nr6fHiBmCQUIFXMle9UbCnHTZtguEh/gPaUph3tAkEyaFLII +oO24sxJIVg/vkeZe9RrT50ueA/taNy+qzj2J7jNNgLeTVL7pePXKgp6V56qBd+mAsTTNBNGSskoz +MXSiqbKRJQ7iSMeaLmn83dXFPk86EvmNedbb7Rwp0qpdIglIJ+L/AAqDhjNCgwj1JjJb66jYSaPC +E/vWp9SC3Lx26YxrNeYiJukuu/4lwJbLwU/wF7lan5jb1f0wGmZ1zNIcQKmX2G3FtBJ+ld1ESS3Q +sKf0pf8Ay4Hg6zBlJwqNhvWLBnauPzNHPhLX4rS8sCXc08Uyo6fQiJiMIqRBudRKslx0P/TWuGof +MWZlAdLyFijVkw9mOTI7t1FYtUlO74dPF/TAbPl8XBZMZtJkAXcdMLLFvWmLVQbm6nPnpS7vxVJV +ZZSPVUlpLbRSX9iyzMeki2x90ty/FQerzxKyEsop7MY5k4UhmGqkK+citdaYf3danp5YVmt0oSTU +iZsWjp+gpCzB32kg6H7tauvzad+AyTNuaFk5OMF2w2pJkko0kFVgtuKv3alafNQfXFRVcM5J2zUk +FnTl5ukm5JG7qSHkJU/Fj02soU2q5mX/ALSkt9Ru5SHp8OggV2PQTV8nxiHBgSyoi0uWO0kDIuRU +wBuCJOLyEqTZzxac6KjV01K33TgCuSP+mFdnMaQpSc2Jmo6gkk3CDZQ7CVC73g8++nP8sO9ocgmh +FRgkHDSRCSb8bB6XCHSBUp5aj/niz5S3I3syatpBFihIEqMixfEdxLslCqCwV9a0rpgKVnNq1aC6 +9pGbSUcr8RtJ2kgk1UAVUxoNO4+vFbj2sgvbGsWy6ijshEEk/ER+WlMWHtAGFjZD932hnJCwFQUn +wncTkSG5On+DXEFks8dyApxZoNk+P4hql1AQqiOtREtPDgP0PyOm+QyVGIPjuecGnviRq/CNKacx +xOcKLIbRCB/CJW7vn+mIMOs+KHY76IIqEgmoqJIjddZTWv3mH5AR4clLLiHwkmHi/wB+A89WWFUh +URX2x+IuI8X6Dhpo6dIOCHZ8NtvvlhHn60IaYSyTRdgP2YBISttUAh/xfe1xM4URMkyR2PiExDw/ +n11wDr1ZYUkiEF1BIrSFO8/r6Yxz9p3LLybyuxzEga6AxKvEFts1SXFKheVbf89cbKkis23U11kC +HpISJEh/y6sCcxwrObhHkavYui5SK3oIhIajz/iUwHJLtwhOspydJJbMq9jZyMwij9lZgNw2vGvn +Wmvi0xrv7IScehDzDSNmGr94Dol1XDICJszRr1WDfSnfjnzMrdbLshLQSDORJG3h0AFyKBiND199 +WmoKUGvdSunjxrH7Ms9MFnV004N6uioSRSL5EAbEkNB5I1R+Lq+KneOA6rboorAguPS13dxqCNwd +487vXHgWcJ1QTd7ZSRCptCjdtW0+avliSipaqQie6oRCQoeHYCuAc81nFEiicuufZokW44k1g3bQ +r3inTl1f8sBHzFmJvDAlx6KktMKpgSUSyDdLdpzoX4efxFgFlaJzNNSCWZMyOWSEskqSfAo+8CMb +lSvRSvmtXzLFnh4FrHtVUYu8U10ivkSW+0qHd3c6d2IuYJRuhHruXL4IWJtEvaN9qqytK06aBWnn +TAMQuVYeGZPo6CNRkzXIl3stxNy+7QqV0I689MB5iaWcpKqZbftcvQZFacmLa9d8t3e5D4/5sU9X +NTrPk8xi2gcBCgqSoQSlyDiQAf4y5/w0fw18WA/aHna12upk1YJaatJgxVbo3JCr/wAFmn4en41M +A7mDNEbEGuXttCAcJiQpM1LyXvU6auXNaU5KWVKwPXuwmAkO015FIuGGf4TIUeVPscZIMAJ0ol5L +q386Edda6emmEdmXZqUb/a2dFo6Smo10Lt+g7P3DMqjeSypV++UpTu+EcFs3Z8yCi8bKTEPGzyq7 +aiyT1+6FJVRKpFQenypyrp9MBvvx+PCvFj4Ajf3Y+15a6eXdgOef21c1DE5XjMvqMzXbyhksRitt +EFnPpr82OSMzPh3UOEeLuUWQ8MAEY2pWleOnzjpTHQ37Y7mQLPgtmpN1ExiVK2OQuFIfnT5clPrj +niPTkpmBbMGz6gIryQAILDr11Gtda179K178BrHZkKcRldisq5ZOU3KSj10aNu4qxUKgLJ10+JMq +AXLCpVmom7k4l3MOnLe1NGVc2CuRMq9TR0H8o1pTEuDi3ySqpRps29VGwv0E6JWp0EfdronSneJ0 +pStC76YrmYiFmo4BZRfaibQUogdRq6jVOfDnX5g1roXPXzwB/J8g8sdRcoZthTVFTitkruKoXuXQ +6+GivhP+fFuj5SWKCeC2cgyJdUlIwyusZvh+8b//AE1B54xxw+UXhXHBSkkiq1dIxyaihUKtY5au +qadfUxrSnfyp5YvPZ1OSUoJxrrbMFqVaODLqqLlKlyTkOXKunIqf1wBSVzA8kG8SMbGmLfj+JEiO +4Gbug/aGqn/ln8OKHnucYje0y2AezySUWSbEdxIJGNaKtS1+IddaY88zlKZlaydFUko9OQUFNxwZ +VCou0g04gPSpUpTWnLXz1xWmsWwWjk0UKrIqOo83Jr/GDgB5lSmulRLzp/TAWOQJ1Fx4zruVCWeQ +opCaSJiO+xUGtKcq/LjPOIasZtBRyw9z7wjKwT3UTLpLTw8sG7G45faS3CpaRZJgonSmm+gdNSCt +fLn5c6Yqjt0g+lG7ZFExZm6Mm4qncVAIuQl66YDR27daJj3S7ZY3rFgW2qF5CRMl/ARUr36VrTAV +V1IIN3jZewk0Ek2D74QVSMtUVOXkOuLhInXJCLriCJ8ghRSMdDTkSqRjWo6etaVpTWtef1xnjxN8 +3kF2ajlMkkwSarjZdQklK6hSmvnSvr+mAs2Qmb5tJycfKSX2hRUY4x6fcEI02luXw/XF57VWbeWj +Gs2+bLuXDlLgpNsn4kpABtTUqP1xnmX8ssOMRZLLOarlMnErGBaCoppqB189KV8+/F6gZprIQ1ZJ +03VApAVI2QBMqVEl0qag4CleV2vnXngMMlUVGxl9j4Ikvsy4F4t2nOpYNQ8SUkySlnkkajcnQsnl +txEgdvuir6jgVmhJYaJyLxyaq7xRSqtfFrUK6U7/AKYumSE2yOUgUqlU208ktHvEa+RhzSVGvrSv +f3fngKzm6SWlpMnLttwyxCKLnc6iVWEbalWlPDrX1xa80Ok2zTK0XHtvbUD1P2LYfvRG2lFkSLxV +0t10xRYtwqEok4TbtlXJGK4KLUuqKqZUMq93dXu07vWmLmbqSWziM3Bk3bK0aDKKJLhqFCK4STHS +ngrTy0wAKbavF5D3fAsmuwLlnVQxDfRqdSTEa+RU8PLvxFhE5Sbm2DHe6XbwVitttEqnShF0+HDe +amjhGdfbrmqttU6lry0pULqCP0p5YtHYvEm/zxEhHPFUttRBZxRbnQq7g00HT6euA7n2xQh2aZLI +ESaSfSRpCPIdPMce4xum3SQJsxIVCuG1yl/1HywaogKBkSKCQKiVhaEVtPoP0xJSZvCVvMkan8VL +yt/TlgBDckb7d5raPyrI+L5uQ+eHTdEmqNphvEVxANl1tPXpxPJo/TQNusadEKd20udC8X5YivnH +AK1SJV2e5TSlarkVtP1wCjeEuZKEBqD8IkiJdXp3YfBMRuTFHqHwe5G23/viIaTUgEbVKlTuqVcR +hdtA02xWpeNbeVOnXv8APAczftC5VTY9oTqQbNkH60ta7bRg3CkSoDWil4dxdNdenFB7NZ5qnmZJ +9LM15tZBAukZIkgbABVITTMa0qoIj01Avlx1J2y5ZaZ5yO9hUXKzV8nQKtHJDTRM7uWunPT1xyqG +XvbMs0bOHCQzbt3wIUogPB1KgiJGY9+te+6lNcB3BkWW9t5fYzKjYGzqQapuJECuE0BqHSP+WCqo +qKKhILouk02R2t0kTuFca6UuKmOUexfPS2Ss8o5RclJvSqqSEuuo8qsDkxPQSAD0oNKUx0a+zgqW +X46Rat9l5JOeDaUrXVNGl2lxU8+XlgDrgniDhmgKPFut9Qb07tpsFfn/AE7sZznXNULxaSe9FT2Y +I5JTiXqxiTOPIfGVAryJT8NOeLJPrPqSq8G2XpHgtRKi7lr0rKLHTS7XlbSlPTHOud8tqJyuVhlp +g0Ip46dxyKUYzTTWbGJXblD5VIiKlK1KvPAXTPce1aZCnXzGSN63XFstNkK21KvBIuYqKV+5S08v +lxj7+YmMoTb7NeX5iIQ9hPmzZiggtvpNmiojoIadFtteo9NSrimzWa4R07NzKUnnaziKqksqbqh7 +6gqHQLwrXSyg0py9cQ6ZkZSUll1lBZajIWWZtiaruRHdTdFdX3pDWnIvrzr9cAfzrmyYlM2lDz2f ++JbkZIqP2QUJrw6/OtKhpcVvpX8sQYma7I6sE0815azFNSiWqRvkX1oLCNdBIRrzGmmnLywh5x/a +T2oUbxMbE5bfE1rRbhiOiKhohdVTSlNbq9/59+KPmSUSeTTlZywboL3UFQWw2p1KlKUrWlPLXTX8 +64D/2Q== + +--Apple-Mail=_C7D5288F-B043-4A7F-AF3F-1EDF1A78438B-- + +--Apple-Mail=_F4EF9C8E-2E66-4FC6-8840-F435ADBED5C8-- diff --git a/mail/src/leap/mail/tests/rfc822.multi-signed.message b/mail/src/leap/mail/tests/rfc822.multi-signed.message new file mode 100644 index 00000000..9907c2de --- /dev/null +++ b/mail/src/leap/mail/tests/rfc822.multi-signed.message @@ -0,0 +1,238 @@ +Date: Mon, 6 Jan 2014 04:40:47 -0400 +From: Kali Kaneko <kali@leap.se> +To: penguin@example.com +Subject: signed message +Message-ID: <20140106084047.GA21317@samsara.lan> +MIME-Version: 1.0 +Content-Type: multipart/signed; micalg=pgp-sha1; + protocol="application/pgp-signature"; boundary="z9ECzHErBrwFF8sy" +Content-Disposition: inline +User-Agent: Mutt/1.5.21 (2012-12-30) + + +--z9ECzHErBrwFF8sy +Content-Type: multipart/mixed; boundary="z0eOaCaDLjvTGF2l" +Content-Disposition: inline + + +--z0eOaCaDLjvTGF2l +Content-Type: text/plain; charset=utf-8 +Content-Disposition: inline +Content-Transfer-Encoding: quoted-printable + +This is an example of a signed message, +with attachments. + + +--=20 +Nihil sine chao! =E2=88=B4 + +--z0eOaCaDLjvTGF2l +Content-Type: text/plain; charset=us-ascii +Content-Disposition: attachment; filename="attach.txt" + +this is attachment in plain text. + +--z0eOaCaDLjvTGF2l +Content-Type: application/octet-stream +Content-Disposition: attachment; filename="hack.ico" +Content-Transfer-Encoding: base64 + +AAABAAMAEBAAAAAAAABoBQAANgAAACAgAAAAAAAAqAgAAJ4FAABAQAAAAAAAACgWAABGDgAA +KAAAABAAAAAgAAAAAQAIAAAAAABAAQAAAAAAAAAAAAAAAAAAAAAAAAAAAAD///8Ai4uLAEZG +RgDDw8MAJCQkAGVlZQDh4eEApqamADQ0NADw8PAADw8PAFVVVQDT09MAtLS0AJmZmQAaGhoA +PT09AMvLywAsLCwA+Pj4AAgICADp6ekA2traALy8vABeXl4An5+fAJOTkwAfHx8A9PT0AOXl +5QA4ODgAuLi4ALCwsACPj48ABQUFAPv7+wDt7e0AJycnADExMQDe3t4A0NDQAL+/vwCcnJwA +/f39ACkpKQDy8vIA6+vrADY2NgDn5+cAOjo6AOPj4wDc3NwASEhIANjY2ADV1dUAU1NTAMnJ +yQC6uroApKSkAAEBAQAGBgYAICAgAP7+/gD6+voA+fn5AC0tLQD19fUA8/PzAPHx8QDv7+8A +Pj4+AO7u7gDs7OwA6urqAOjo6ADk5OQAVFRUAODg4ADf398A3d3dANvb2wBfX18A2dnZAMrK +ygDCwsIAu7u7ALm5uQC3t7cAs7OzAKWlpQCdnZ0AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA +AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA +AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA +AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA +AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA +AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA +AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA +AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA +AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA +AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA +AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA +AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA +AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAABKRC5ESDRELi4uNEUhIhcK +LgEBAUEeAQEBAUYCAAATNC4BPwEUMwE/PwFOQgAAACsuAQEBQUwBAQEBSk0AABVWSCwBP0RP +QEFBFDNTUkdbLk4eOg0xEh5MTEw5RlEqLgdKTQAcGEYBAQEBJQ4QPBklWwAAAANKAT8/AUwy +AAAAOxoAAAA1LwE/PwEeEQAAAFpJGT0mVUgBAQE/SVYFFQZIKEtVNjFUJR4eSTlIKARET0gs +AT8dS1kJH1dINzgnGy5EAQEBASk+AAAtUAwAACNYLgE/AQEYFQAAC1UwAAAAW0QBAQEkMRkA +AAZDGwAAME8WRC5EJU4lOwhIT0UgD08KAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA +AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAACgAAAAgAAAAQAAAAAEACAAAAAAA +gAQAAAAAAAAAAAAAAAAAAAAAAAAAAAAA////AH9/fwC/v78APz8/AN/f3wBfX18An5+fAB0d +HQAuLi4A7+/vAM/PzwCvr68Ab29vAE5OTgAPDw8AkZGRAPf39wDn5+cAJiYmANfX1wA3NzcA +x8fHAFdXVwC3t7cAh4eHAAcHBwAWFhYAaGhoAEhISAClpaUAmZmZAHl5eQCMjIwAdHR0APv7 ++wALCwsA8/PzAOvr6wDj4+MAKioqANvb2wDT09MAy8vLAMPDwwBTU1MAu7u7AFtbWwBjY2MA +AwMDABkZGQAjIyMANDQ0ADw8PABCQkIAtLS0AEtLSwCioqIAnJycAGxsbAD9/f0ABQUFAPn5 ++QAJCQkA9fX1AA0NDQDx8fEAERERAO3t7QDp6ekA5eXlAOHh4QAsLCwA3d3dADAwMADZ2dkA +OTk5ANHR0QDNzc0AycnJAMXFxQDBwcEAUVFRAL29vQBZWVkAXV1dALKysgBycnIAk5OTAIqK +igABAQEABgYGAAwMDAD+/v4A/Pz8APr6+gAXFxcA+Pj4APb29gD09PQA8vLyACQkJADw8PAA +JycnAOzs7AApKSkA6urqAOjo6AAvLy8A5ubmAOTk5ADi4uIAODg4AODg4ADe3t4A3NzcANra +2gDY2NgA1tbWANTU1ABNTU0A0tLSANDQ0ABUVFQAzs7OAMzMzABYWFgAysrKAMjIyABcXFwA +xsbGAF5eXgDExMQAYGBgAMDAwABkZGQAuLi4AG1tbQC2trYAtbW1ALCwsACurq4Aenp6AKOj +owChoaEAoKCgAJ6engCdnZ0AmpqaAI2NjQCSkpIAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA +AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA +AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA +AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA +AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA +AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA +AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA +AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA +AAAAAAAAAAAAAAAAAAAAAAAAFHFvR3Fvb0dHJ1F0R0dHR29HR0YLf28nJkVraGtHBXMnAQEB +AQEBAQEBCxEBAQEBAQEBASdzASOMHHsZSQEBcnEBAV1dXV1dXQFOJQEBXV1dXV0BR0kBOwAA +AAAIUAFyJwFdXV1dXV1dAU4lAV1dXV1dXQFHbVgAAAAAAAAoaG5xAV1dXV1dXV0BfSUBXV1d +XV1dASd2HQAAAAAAAFoMEkcBXV1dXV1dXQFOZAEBXV1dXV0BbU8TAAAAAAAAAFkmcQFdXV1d +XV1dAU4lAV1dXV1dXQEnSzgAAAAAAABaN2tHAV1dXV1dXV0BTiUBXV1dXV1dAUdtHwAAAAAA +AEpEJycBXV1dXV1dAQFOJQFdAV1dAV0BRykBIgAAAABlfAFzJwEBAQEBAQEBAQtAAQEBAQEB +AQFuSQE8iFeBEG8BXUeGTn0LdnR3fH0LOYR8Tk5OTnxOeouNTQspJ0YFd30rgCljIwpTlCxm +X2KERWMlJSUlJSURFE1hPEYMBysRYSV0RwF3NT0AGjYpAQtjAQEBAQEBAQFvKQGKMzEAP4dC +AXESEmcAAAAAAEpEKiUBXV1dXV1dAUduLEEAAAAAAIFdcUSWAAAAAAAAADp1ZAFdXV1dXV0B +bwVVAAAAAAAAW4Jta34AAAAAAAAAhRQlAV1dXV1dAQFtK0gAAAAAAAAAEGtFhwAAAAAAAACJ +S2QBXV1dXV1dAW5NFQAAAAAAAACTa2geAAAAAAAAAAx0ZAFdXV1dXV0BR0YNAAAAAAAADxRu +J14tAAAAAAAvXQslAV1dXV1dXQFHcW4JAAAAAAAhAXFuAWMgbBsJAhEBTWIBAQEBAQEBAW5y +AW+DZWBwkQEBcQtHbWh2hnZEbm6LFG9HR21uR3FGgFFGa2oqFgVob3FNf0t0dAUncnR0SY1N +KW5xK01ucUlRLklyRksqR250S3pGAQEBAQEBAQEBeWIBUFRINA1uAUYFAQqOTGlSiAEBb0cB +XV1dAQFdAQF9I4pcAAAAABNHEnIKBAAAAAA9kAFJJwFdXV1dXV1dAXptZwAAAAAAAAZqbY4A +AAAAAAAbcm5HAV1dXV1dXV0BFFZbAAAAAAAAZ3pLNQAAAAAAAACPa0cBXV1dXV1dXQEpkgAA +AAAAAAAygHppAAAAAAAAAJVrcQFdXV1dXV1dAXl9QwAAAAAAADZxcRcAAAAAAAA9UW1vAV1d +XV1dXV0BC2EwAAAAAAAAkmhGGD0AAAAAAHg+cW8BAV1dAV1dAQFOESWBAAAAJJUBJykBkEMA +AAAOJgFzRwE8AV1dXV1dAX0lAV8WEDp1AQFxSwEBBTkhAxEBPHJzSXEFcnJJcnFyFnRycRJr +RW5ycXl8cXJuRSYScQVJcQAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA +AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA +AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAKAAAAEAAAACAAAAAAQAIAAAA +AAAAEgAAAAAAAAAAAAAAAAAAAAAAAAAAAAD///8Af39/AL+/vwA/Pz8A39/fAF9fXwCfn58A +Hx8fAO/v7wAvLy8Ab29vAI+PjwAPDw8A0NDQALCwsABQUFAA9/f3ABcXFwDn5+cAJycnAMjI +yABHR0cAqKioAGdnZwCXl5cAd3d3AIeHhwAHBwcA2NjYALi4uABXV1cANTU1ADo6OgD7+/sA +CwsLAPPz8wATExMA6+vrABsbGwDj4+MAIyMjANTU1AArKysAzMzMAMTExABLS0sAtLS0AKys +rABbW1sApKSkAGNjYwCbm5sAa2trAJOTkwBzc3MAi4uLAHt7ewCDg4MAAwMDANzc3AAyMjIA +vLy8AFNTUwD9/f0ABQUFAPn5+QAJCQkADQ0NAPHx8QDt7e0AFRUVAOnp6QAZGRkA5eXlAB0d +HQDh4eEAISEhACUlJQDa2toAKSkpANbW1gDS0tIAysrKADw8PADGxsYAwsLCAEVFRQBJSUkA +urq6ALa2tgCysrIArq6uAFlZWQCqqqoAXV1dAKampgBlZWUAoqKiAJ2dnQBtbW0AmZmZAHFx +cQCVlZUAeXl5AH19fQCJiYkAhYWFAAEBAQACAgIABAQEAP7+/gAGBgYA/Pz8AAgICAD6+voA +CgoKAPj4+AAMDAwA9vb2APT09AASEhIA8vLyABQUFADu7u4AFhYWAOzs7AAYGBgA6urqAOjo +6AAeHh4AICAgAOTk5AAiIiIA4uLiACQkJADg4OAAJiYmAN7e3gDd3d0AKCgoANvb2wAqKioA +2dnZACwsLADX19cALi4uANXV1QAxMTEA09PTADMzMwDR0dEANDQ0AM3NzQA5OTkAy8vLADs7 +OwDJyckAPT09AMfHxwBAQEAAxcXFAMPDwwDBwcEAwMDAAL6+vgBKSkoAvb29ALu7uwC5ubkA +UVFRALe3twBSUlIAtbW1AFRUVACzs7MAVlZWAFhYWABaWloAra2tAFxcXACrq6sAXl5eAKmp +qQCnp6cAZGRkAKOjowChoaEAaGhoAKCgoACenp4AnJycAG5ubgCampoAcHBwAJiYmABycnIA +lpaWAJSUlAB2dnYAkpKSAHh4eACQkJAAenp6AI6OjgB8fHwAjIyMAIiIiACCgoIAhISEAAAA +AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA +AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA +AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA +AAAAAAAAAAAAAAAAAAAAAAAAAAC1WlpaWlpaWlpaWlpaWlpaWlpaHjAHr6taWlpaWlpaWlpa +WlpaWlpaq68HMB5aWlpap6KlWzBaA6KoWlpaWlq1WgEBAQEBAQEBAQEBAQEBAQEBAQGXNkUB +AQEBAQEBAQEBAQEBAQEBAQFFNpcBAQEBASg4EI6HPa5lfgEBAQEBWloBAQEBAQEBAQEBAQEB +AQEBAQEBlzZFAQEBAQEBAQEBAQEBAQEBAQEBRTaXAQEBETpEAAAAAAAAAH/FbwEBAVpaAQEB +AQEBAQEBAQEBAQEBAQEBAZc2RQEBAQEBAQEBAQEBAQEBAQEBAUU2lwEBhFQAAAAAAAAAAAAA +ALJCAQFaWgEBAQEBAQEBAQEBAQEBAQEBAQGXNkUBAQEBAQEBAQEBAQEBAQEBAQFFNpcBeJoA +AAAAAAAAAAAAAAAAMQEBWloBAQEBAQEBAQEBAQEBAQEBAQEBlzZFAQEBAQEBAQEBAQEBAQEB +AQEBRTZSATUAAAAAAAAAAAAAAAAAAABnAVpaAQEBAQEBAQEBAQEBAQEBAQEBAZc2RQEBAQEB +AQEBAQEBAQEBAQEBAUU2Tx1wAAAAAAAAAAAAAAAAAAAAgkaoWgEBAQEBAQEBAQEBAQEBAQEB +AQGXNkUBAQEBAQEBAQEBAQEBAQEBAQFFNgVrAAAAAAAAAAAAAAAAAAAAAABioloBAQEBAQEB +AQEBAQEBAQEBAQEBlzZFAQEBAQEBAQEBAQEBAQEBAQEBRWcqngAAAAAAAAAAAAAAAAAAAAAA +tANaAQEBAQEBAQEBAQEBAQEBAQEBAZc2RQEBAQEBAQEBAQEBAQEBAQEBAUXDpIcAAAAAAAAA +AAAAAAAAAAAAAJRaWgEBAQEBAQEBAQEBAQEBAQEBAQGXNkUBAQEBAQEBAQEBAQEBAQEBAQFF +wa9HAAAAAAAAAAAAAAAAAAAAAABOMFoBAQEBAQEBAQEBAQEBAQEBAQEBlzZFAQEBAQEBAQEB +AQEBAQEBAQEBRWVZggAAAAAAAAAAAAAAAAAAAAAAjltaAQEBAQEBAQEBAQEBAQEBAQEBAZc2 +RQEBAQEBAQEBAQEBAQEBAQEBAUXFmZYAAAAAAAAAAAAAAAAAAAAAAKqlWgEBAQEBAQEBAQEB +AQEBAQEBAQGXNkUBAQEBAQEBAQEBAQEBAQEBAQFFNorHAAAAAAAAAAAAAAAAAAAAAABloloB +AQEBAQEBAQEBAQEBAQEBAQEBlzZFAQEBAQEBAQEBAQEBAQEBAQEBRTY8UwAAAAAAAAAAAAAA +AAAAAAASEz5aAQEBAQEBAQEBAQEBAQEBAQEBAZc2RQEBAQEBAQEBAQEBAQEBAQEBAUU2lQFd +AAAAAAAAAAAAAAAAAAAA0AFaWgEBAQEBAQEBAQEBAQEBAQEBAQGXNkUBAQEBAQEBAQEBAQEB +AQEBAQFFNpcBhoUAAAAAAAAAAAAAAAAAVxEBWloBAQEBAQEBAQEBAQEBAQEBAQEBlzZFAQEB +AQEBAQEBAQEBAQEBAQEBRTaXAQGXTQAAAAAAAAAAAAAAnCgBAVpaAQEBAQEBAQEBAQEBAQEB +AQEBAZc2RQEBAQEBAQEBAQEBAQEBAQEBAUU2lwEBASiwAAAAAAAAAAAcwncBAQFaWgEBAQEB +AQEBAQEBAQEBAQEBAQGXNkUBAQEBAQEBAQEBAQEBAQEBAQFFNpcBAQEBASy8khINgiFojQEB +AQEBWjCVl5eXl5eXl5dSUpeXl5eXl5eTHsWdlZeXl5eXl5eXl5eXl5eXl5eVncUek5eXl1I8 +ipsvs6iVBU9Sl5eXlTAHNjY2NjY2Zb1ivbtiY2c2NjY2NsVlxjY2NjY2NjY2NjY2NjY2NjY2 +NsZlxTY2NjY2xr8yFxcXusHGNjY2NjYHW3hFRUURAY8HC7Jh0ahFb3pFRRGdxkp4RUVFRUVF +RUVFRUVFRUVFRXhKxp0RRUVFIkKhDLkxwMiXInNFRUV4W1oBAQEBCcclAAAAAAAAnK0BAQEB +lzZFAQEBAQEBAQEBAQEBAQEBAQEBRTaXAQEBAQ4ucAAAAAAAdAaNAQEBAVpaAQEBpYMAAAAA +AAAAAAAAGHUBAZc2RQEBAQEBAQEBAQEBAQEBAQEBAUU2lwEBAWtwAAAAAAAAAAAADboBAQFa +WgEBHnIAAAAAAAAAAAAAAACxcwGXNkUBAQEBAQEBAQEBAQEBAQEBAQFFNpcBAcQAAAAAAAAA +AAAAAABtwQEBWloBiCcAAAAAAAAAAAAAAAAAAM0BUjZFAQEBAQEBAQEBAQEBAQEBAQEBRTaX +AbsAAAAAAAAAAAAAAAAAAHCiAVpaAQYAAAAAAAAAAAAAAAAAAAAck082RQEBAQEBAQEBAQEB +AQEBAQEBAUU2UUVLAAAAAAAAAAAAAAAAAAAAIQEePkoNAAAAAAAAAAAAAAAAAAAAAMCLxkUB +AQEBAQEBAQEBAQEBAQEBAQFFNgViAAAAAAAAAAAAAAAAAAAAAACppKK9AAAAAAAAAAAAAAAA +AAAAAACQnxlFAQEBAQEBAQEBAQEBAQEBAQEBRcZPrAAAAAAAAAAAAAAAAAAAAAAAZqOjCwAA +AAAAAAAAAAAAAAAAAAAAQ7i/RQEBAQEBAQEBAQEBAQEBAQEBAUUZVSsAAAAAAAAAAAAAAAAA +AAAAAFRZpT8AAAAAAAAAAAAAAAAAAAAAAADKvkUBAQEBAQEBAQEBAQEBAQEBAQFFZVpJAAAA +AAAAAAAAAAAAAAAAAAAUXKU/AAAAAAAAAAAAAAAAAAAAAAAAyr5FAQEBAQEBAQEBAQEBAQEB +AQEBRWVaSQAAAAAAAAAAAAAAAAAAAAAAFFyjCwAAAAAAAAAAAAAAAAAAAAAAdl40RQEBAQEB +AQEBAQEBAQEBAQEBAUUZVSsAAAAAAAAAAAAAAAAAAAAAAKCoVrcAAAAAAAAAAAAAAAAAAAAA +ACCZxUUBAQEBAQEBAQEBAQEBAQEBAQFFxo1fAAAAAAAAAAAAAAAAAAAAAABpVqh+fQAAAAAA +AAAAAAAAAAAAAADRijZFAQEBAQEBAQEBAQEBAQEBAQEBRTaKXAAAAAAAAAAAAAAAAAAAAAA7 +LANaAWgAAAAAAAAAAAAAAAAAAABJSJE2RQEBAQEBAQEBAQEBAQEBAQEBAUU2KgEKAAAAAAAA +AAAAAAAAAAAAHwGrWgF8kAAAAAAAAAAAAAAAAAAAZQGXNkUBAQEBAQEBAQEBAQEBAQEBAQFF +NpcBHm0AAAAAAAAAAAAAAAAAEk8BWloBAZVLAAAAAAAAAAAAAAAANwEBlzZFAQEBAQEBAQEB +AQEBAQEBAQEBRTaXAQHFAAAAAAAAAAAAAAAAQx4BAVpaAQEBj1QAAAAAAAAAAAByGQEBAZc2 +RQEBAQEBAQEBAQEBAQEBAQEBAUU2lwEBARcSAAAAAAAAAAAAjJkBAQFaWgEBAQFxuphuAAAA +ABK8jwEBAQGXNkUBAQEBAQEBAQEBAQEBAQEBAQFFNpcBAQEBSMlLAAAAAG0rDEUBAQEBWlt4 +RUVFeAFFLWU6DC8FcXNFRUURncZKeEVFRUVFRUVFRUVFRUVFRUV4SsadEUVFRXUBhC8MOmWi +JgF3RUVFeFsHNjY2NjY2Z7+9Yru+wzY2NjY2NsVlxjY2NjY2NsU0vr6/wzY2NjY2NsZlxTY2 +NjY2NmUytbO3Yhk2NjY2NjYHMJWXl5eXl5eXl5eXl5eXl5eXl5MexZ2Vl5eXHQWdXgwMYKKK +T5eXl5WdxR6Tl5eXKgWVrWfOvquPipWXl5eVMFoBAQEBAQEBAQEBAQEBAQEBAQEBlzZFAQEB +AYE5kHYAAEMpvJEBAQEBRTaXAQEBAXFiBEcAAG4Spi8BAQEBAVpaAQEBAQEBAQEBAQEBAQEB +AQEBAZc2RQEBAcF7AAAAAAAAAABBaUIBAUU2lwEBAZsgAAAAAAAAAAAAFooBAQFaWgEBAQEB +AQEBAQEBAQEBAQEBAQGXNkUBAQsAAAAAAAAAAAAAAACxcwFFNpcBAQ92AAAAAAAAAAAAAABN +UQEBWloBAQEBAQEBAQEBAQEBAQEBAQEBlzZFAcwAAAAAAAAAAAAAAAAAABgBejaXAZd5AAAA +AAAAAAAAAAAAAImAAVpaAQEBAQEBAQEBAQEBAQEBAQEBAZc2c1JDAAAAAAAAAAAAAAAAAAAA +W3E2KgGeAAAAAAAAAAAAAAAAAAAAMwGrWgEBAQEBAQEBAQEBAQEBAQEBAQGXNm9kAAAAAAAA +AAAAAAAAAAAAAAQJZ4ukAAAAAAAAAAAAAAAAAAAAAHKVpVoBAQEBAQEBAQEBAQEBAQEBAQEB +l8OGKQAAAAAAAAAAAAAAAAAAAAAcor+LNQAAAAAAAAAAAAAAAAAAAAAAaqJaAQEBAQEBAQEB +AQEBAQEBAQEBAZdjHmwAAAAAAAAAAAAAAAAAAAAAAM8ymT0AAAAAAAAAAAAAAAAAAAAAAFg+ +WgEBAQEBAQEBAQEBAQEBAQEBAQGXvWUAAAAAAAAAAAAAAAAAAAAAAABhuFmCAAAAAAAAAAAA +AAAAAAAAAACOW1oBAQEBAQEBAQEBAQEBAQEBAQEBl7vOAAAAAAAAAAAAAAAAAAAAAAAAtGCv +RwAAAAAAAAAAAAAAAAAAAAAATjBaAQEBAQEBAQEBAQEBAQEBAQEBAZcHYgAAAAAAAAAAAAAA +AAAAAAAAAAu4pIcAAAAAAAAAAAAAAAAAAAAAAD1aWgEBAQEBAQEBAQEBAQEBAQEBAQGXNBUj +AAAAAAAAAAAAAAAAAAAAAAAyvSpXAAAAAAAAAAAAAAAAAAAAAAAYpFoBAQEBAQEBAQEBAQEB +AQEBAQEBl2ckVAAAAAAAAAAAAAAAAAAAAACDiMMFzAAAAAAAAAAAAAAAAAAAAAAAr6NaAQEB +AQEBAQEBAQEBAQEBAQEBAZc2b7sAAAAAAAAAAAAAAAAAAAAAaW82HRMlAAAAAAAAAAAAAAAA +AAAAlECpWgEBAQEBAQEBAQEBAQEBAQEBAQGXNngBBAAAAAAAAAAAAAAAAAAAKUZ3NpcBzwAA +AAAAAAAAAAAAAAAAAA8BWloBAQEBAQEBAQEBAQEBAQEBAQEBlzZFAZGCAAAAAAAAAAAAAAAA +dC0BRTaXAXGwAAAAAAAAAAAAAAAAAAIBAVpaAQEBAQEBAQEBAQEBAQEBAQEBAZc2RQEBlY4A +AAAAAAAAAAAACD4BAUU2lwEBd7YAAAAAAAAAAAAAbmtvAQFaWgEBAQEBAQEBAQEBAQEBAQEB +AQGXNkUBAQEJyw0AAAAAAAB0M0wBAQFFNpcBAQEBF1AAAAAAAAAAVD4BAQEBWloBAQEBAQEB +AQEBAQEBAQEBAQEBlzZFAQEBAQETB7ymprxliwEBAQEBRTaXAQEBAQF1qxqsV7QbVXEBAQEB +AVq1WlpaWlpaWlpaWlpaWlpaWlpaHjAHr6taWlpaPqKkPj6kLadaWlpaq68HMB5aWlpaqaNW +pz4DLaQeWlpaWlq1AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA +AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA +AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA +AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA +AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA +AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA +AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA +AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA +AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA +AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA= + +--z0eOaCaDLjvTGF2l-- + +--z9ECzHErBrwFF8sy +Content-Type: application/pgp-signature + +-----BEGIN PGP SIGNATURE----- +Version: GnuPG v1.4.15 (GNU/Linux) + +iQIcBAEBAgAGBQJSymwPAAoJECNji/csWTvBhtcP/2AKF0uk6ljrfMWhNBSFwDqv +kYng3slREnF/pxnIGOpR2GAxPBPjRipZOuUU8QL+pXBwk5kWzb9RYpr26xMYWRtl +vXdVbob5NolNEYrqTkkQ1kejERQGFyescsUJDcEDXJl024czKWbxHTYYN4vlYJMK +PZ5mPSdADFn970PnVXfNix3Rjvv7SFQGammDBGjQzyROkoiDKPZcomp6dzm6zEXC +w8i42WfHU8GkyVVNvXZI52Xw3LUXiXsJ58B1V1O5U42facepG6S+S0DC/PWptqPw +sAM9/YGkvBNWrsJA/BavXPRLE1gVpu+hZZEsOqRvs244k7JTrVo54xDbdeOT2nTr +BDk4e88vmCVKGgE9MZjDbjgOHDZhmsxNQm4DBGRH2huF0noUc/8Sm4KhSO49S2mN +QjIT5QrPerQNiP5QtShHZRJX7ElXYZWX1SG/c9jQjfd0W1XK/cGtwClICe+lpprt +mLC2607yalbRhCxV9bQlVUnd2tY3NY4UgIKgCEiEwb1hf/k9jQDvpk16VuNWSZQJ +jFeg9F2WdNjQMp79cyvnayyhjS9o/K2LbSIgJi7KdlQcVZ/2DQfbMjCwByR7P9g8 +gcAKh8V7E6IpAu1mnvs4FDagipppK6hOTRj2s/I3xZzneprSK1WaVro/8LAWZe9X +sSdfcAhT7Tno7PB/Acoh +=+okv +-----END PGP SIGNATURE----- + +--z9ECzHErBrwFF8sy-- diff --git a/mail/src/leap/mail/tests/rfc822.multi.message b/mail/src/leap/mail/tests/rfc822.multi.message new file mode 100644 index 00000000..30f74e52 --- /dev/null +++ b/mail/src/leap/mail/tests/rfc822.multi.message @@ -0,0 +1,96 @@ +Date: Fri, 19 May 2000 09:55:48 -0400 (EDT)
+From: Doug Sauder <doug@penguin.example.com>
+To: Joe Blow <blow@example.com>
+Subject: Test message from PINE
+Message-ID: <Pine.LNX.4.21.0005190951410.8452-102000@penguin.example.com>
+MIME-Version: 1.0
+Content-Type: MULTIPART/MIXED; BOUNDARY="-1463757054-952513540-958744548=:8452"
+
+ This message is in MIME format. The first part should be readable text,
+ while the remaining parts are likely unreadable without MIME-aware tools.
+ Send mail to mime@docserver.cac.washington.edu for more info.
+
+---1463757054-952513540-958744548=:8452
+Content-Type: TEXT/PLAIN; charset=US-ASCII
+
+This is a test message from PINE MUA.
+
+
+---1463757054-952513540-958744548=:8452
+Content-Type: APPLICATION/octet-stream; name="redball.png"
+Content-Transfer-Encoding: BASE64
+Content-ID: <Pine.LNX.4.21.0005190955480.8452@penguin.example.com>
+Content-Description: A PNG graphic file
+Content-Disposition: attachment; filename="redball.png"
+
+iVBORw0KGgoAAAANSUhEUgAAABsAAAAbCAMAAAC6CgRnAAADAFBMVEX///8A
+AAABAAALAAAVAAAaAAAXAAARAAAKAAADAAAcAAAyAABEAABNAABIAAA9AAAj
+AAAWAAAmAABhAAB7AACGAACHAAB9AAB0AABgAAA5AAAUAAAGAAAnAABLAABv
+AACQAAClAAC7AAC/AACrAAChAACMAABzAABbAAAuAAAIAABMAAB3AACZAAC0
+GRnKODjVPT3bKSndBQW4AACoAAB5AAAxAAAYAAAEAABFAACaAAC7JCTRYWHf
+hITmf3/mVlbqHx/SAAC5AACjAABdAABCAAAoAAAJAABnAAC6Dw/QVFTek5Pl
+rKzpmZntZWXvJSXXAADBAACxAACcAABtAABTAAA2AAAbAAAFAABKAACBAADL
+ICDdZ2fonJzrpqbtiorvUVHvFBTRAADDAAC2AAB4AABeAABAAAAiAABXAACS
+AADCAADaGxvoVVXseHjveHjvV1fvJibhAADOAAC3AACnAACVAABHAAArAAAP
+AACdAADFAADhBQXrKCjvPDzvNTXvGxvjAADQAADJAAC1AACXAACEAABsAABP
+AAASAAACAABiAADpAADvAgLnAADYAADLAAC6AACwAABwAAATAAAkAABYAADI
+AADTAADNAACzAACDAABuAAAeAAB+AADAAACkAACNAAB/AABpAABQAAAwAACR
+AACpAAC8AACqAACbAABlAABJAAAqAAAOAAA0AACsAACvAACtAACmAACJAAB6
+AABrAABaAAA+AAApAABqAACCAACfAACeAACWAACPAAB8AAAZAAAHAABVAACO
+AACKAAA4AAAQAAA/AAByAACAAABcAAA3AAAsAABmAABDAABWAAAgAAAzAAA8
+AAA6AAAfAAAMAAAdAAANAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAD8
+LtlFAAAAAXRSTlMAQObYZgAAABZ0RVh0U29mdHdhcmUAZ2lmMnBuZyAyLjAu
+MT1evmgAAAIISURBVHicY2CAg/8QwIABmJhZWFnZ2Dk4MaU5uLh5eHn5+LkF
+BDlQJf8zC/EIi4iKiUtI8koJScsgyf5nlpWTV1BUUlZRVVPX4NFk1UJIyghp
+6+jq6RsYGhmbKJgK85mZW8Dk/rNaSlhZ29ja2Ts4Ojkr6Li4urFDNf53N/Ow
+8vTy9vH18w8IDAoWDQkNC4+ASP5ni4wKio6JjYtPSExKTnFWSE1LF4A69n9G
+ZlZ2Tm5efkFhUXFySWlZlEd5RSVY7j+TkGRVdU1tXX1DY1Ozcktpa1t7h2Yn
+OAj+d7l1tyo79vT29SdNSJ44SbFVdHIo9xSIHNPUaWqTpifNSJrZnK00S0U1
+a/acUG5piNz/uXLzVJ2qm6dXz584S2WB1cJFi5cshZr539xVftnyFKUVTi2T
+VjqvyhJLXb1m7TqoHPt6F/HW0g0bN63crGqVtWXrtu07BJihcsw71+zanRW8
+Z89eq337RQ/Ip60xO3gIElX/LbikDm8T36KwbNmRo7O3zpHkPSZwHBqL//8f
+lz1x2OOkyKJTi7aqbzutfUZI2gIuF8F2lr/D5dw2+fZdwpl8YVOlI+CJ4/9/
+joOyYed5QzMvhGqnm2V0WiClm///D0lfXHtJ6vLlK9w7rx7vQk5SQJbFtSms
+1y9evXid7QZacgOxmSxktNzdtSwwU+J/VICaCPFIYU3XAJhIOtjf5sfyAAAA
+JXRFWHRDb21tZW50AGNsaXAyZ2lmIHYuMC42IGJ5IFl2ZXMgUGlndWV0NnM7
+vAAAAABJRU5ErkJggg==
+---1463757054-952513540-958744548=:8452
+Content-Type: APPLICATION/octet-stream; name="blueball.png"
+Content-Transfer-Encoding: BASE64
+Content-ID: <Pine.LNX.4.21.0005190955481.8452@penguin.example.com>
+Content-Description: A PNG graphic file
+Content-Disposition: attachment; filename="blueball.png"
+
+iVBORw0KGgoAAAANSUhEUgAAABsAAAAbCAMAAAC6CgRnAAADAFBMVEX///8A
+AAgAABAAABgAAAAACCkAEEIAEEoACDEAEFIIIXMIKXsIKYQIIWsAGFoACDkI
+IWMQOZwYQqUYQq0YQrUQOaUQMZQAGFIQMYwpUrU5Y8Y5Y84pWs4YSs4YQs4Y
+Qr1Ca8Z7nNacvd6Mtd5jlOcxa94hUt4YStYYQsYQMaUAACHO5+/n7++cxu9S
+hO8pWucQOa1Ke86tzt6lzu9ajO8QMZxahNat1ufO7++Mve9Ke+8YOaUYSsaM
+vee15++Uve8AAClajOdzpe9rnO8IKYwxY+8pWu8IIXsAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAADB
+Mg1VAAAAAXRSTlMAQObYZgAAABZ0RVh0U29mdHdhcmUAZ2lmMnBuZyAyLjAu
+MT1evmgAAAGISURBVHicddJtV5swGAbgEk6AJhBSk4bMCUynBSLaqovbrG/b
+fPn/vyh70lbsscebL5xznTsh5BmNhgQoRChwo50EOIohUYLDj4zHhKYQkrEo
+Qdvock4ne0IKMVUpKZLQDeqSTIsv+18PyqqWUw2IBsRM7307PPp+fDJrWtnp
+LDJvewYxnewfnvanZ+fzpmwXijC8KbqEa3Fx2ff91Y95U9XCUpaDeQwiMpHX
+P/v+1++bWVPWQoGFawtjury9vru/f/C1Vi7ezT0WWpQHf/7+u/G71aLThK/M
+jRxmT6KdzZ9fGk9yatMsTgZLl3XVgFRAC6spj/13enssqJVtWVa3NdBSacL8
++VZmYqKmdd1CSYoOiMOSGwtzlqqlFFIuOqv0a1ZEZrUkWICLLFW266y1KvWE
+1zV/iDAH1EopnVLCiygZCIomH3NCKX0lnI+B1iuuzCGTxwXjnDO4d7NpbX42
+YJJHkBwmAm2TxwAZg40J3+Xtbv1rgOAZwG0NxW62p+lT+Yi747sD/wEUVMzY
+mWkOvwAAACV0RVh0Q29tbWVudABjbGlwMmdpZiB2LjAuNiBieSBZdmVzIFBp
+Z3VldDZzO7wAAAAASUVORK5CYII=
+---1463757054-952513540-958744548=:8452--
diff --git a/mail/src/leap/mail/tests/rfc822.plain.message b/mail/src/leap/mail/tests/rfc822.plain.message new file mode 100644 index 00000000..fc627c3a --- /dev/null +++ b/mail/src/leap/mail/tests/rfc822.plain.message @@ -0,0 +1,66 @@ +From pyar-bounces@python.org.ar Wed Jan 8 14:46:02 2014 +Return-Path: <pyar-bounces@python.org.ar> +X-Spam-Checker-Version: SpamAssassin 3.3.2 (2011-06-06) on spamd2.riseup.net +X-Spam-Level: ** +X-Spam-Pyzor: Reported 0 times. +X-Spam-Status: No, score=2.1 required=8.0 tests=AM_TRUNCATED,CK_419SIZE, + CK_NAIVER_NO_DNS,CK_NAIVE_NO_DNS,ENV_FROM_DIFF0,HAS_REPLY_TO,LINK_NR_TOP, + NO_REAL_NAME,RDNS_NONE,RISEUP_SPEAR_C shortcircuit=no autolearn=disabled + version=3.3.2 +Delivered-To: kali@leap.se +Received: from mx1.riseup.net (mx1-pn.riseup.net [10.0.1.33]) + (using TLSv1 with cipher DHE-RSA-AES256-SHA (256/256 bits)) + (Client CN "*.riseup.net", Issuer "Gandi Standard SSL CA" (not verified)) + by vireo.riseup.net (Postfix) with ESMTPS id 6C39A8F + for <kali@leap.se>; Wed, 8 Jan 2014 18:46:02 +0000 (UTC) +Received: from pyar.usla.org.ar (unknown [190.228.30.157]) + by mx1.riseup.net (Postfix) with ESMTP id F244C533F4 + for <kali@leap.se>; Wed, 8 Jan 2014 10:46:01 -0800 (PST) +Received: from [127.0.0.1] (localhost [127.0.0.1]) + by pyar.usla.org.ar (Postfix) with ESMTP id CC51D26A4F + for <kali@leap.se>; Wed, 8 Jan 2014 15:46:00 -0300 (ART) +MIME-Version: 1.0 +Content-Type: text/plain; charset="iso-8859-1" +Content-Transfer-Encoding: quoted-printable +From: pyar-request@python.org.ar +To: kali@leap.se +Subject: confirm 0e47e4342e4d42508e8c283175b05b3377148ac2 +Reply-To: pyar-request@python.org.ar +Auto-Submitted: auto-replied +Message-ID: <mailman.245.1389206759.1579.pyar@python.org.ar> +Date: Wed, 08 Jan 2014 15:45:59 -0300 +Precedence: bulk +X-BeenThere: pyar@python.org.ar +X-Mailman-Version: 2.1.15 +List-Id: Python Argentina <pyar.python.org.ar> +X-List-Administrivia: yes +Errors-To: pyar-bounces@python.org.ar +Sender: "pyar" <pyar-bounces@python.org.ar> +X-Virus-Scanned: clamav-milter 0.97.8 at mx1 +X-Virus-Status: Clean + +Mailing list subscription confirmation notice for mailing list pyar + +We have received a request de kaliyuga@riseup.net for subscription of +your email address, "kaliyuga@riseup.net", to the pyar@python.org.ar +mailing list. To confirm that you want to be added to this mailing +list, simply reply to this message, keeping the Subject: header +intact. Or visit this web page: + + http://listas.python.org.ar/confirm/pyar/0e47e4342e4d42508e8c283175b05b= +3377148ac2 + + +Or include the following line -- and only the following line -- in a +message to pyar-request@python.org.ar: + + confirm 0e47e4342e4d42508e8c283175b05b3377148ac2 + +Note that simply sending a `reply' to this message should work from +most mail readers, since that usually leaves the Subject: line in the +right form (additional "Re:" text in the Subject: is okay). + +If you do not wish to be subscribed to this list, please simply +disregard this message. If you think you are being maliciously +subscribed to the list, or have any other questions, send them to +pyar-owner@python.org.ar. diff --git a/mail/src/leap/mail/tests/test_mail.py b/mail/src/leap/mail/tests/test_mail.py new file mode 100644 index 00000000..f9cded29 --- /dev/null +++ b/mail/src/leap/mail/tests/test_mail.py @@ -0,0 +1,399 @@ +# -*- coding: utf-8 -*- +# test_mail.py +# Copyright (C) 2014 LEAP +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see <http://www.gnu.org/licenses/>. +""" +Tests for the mail module. +""" +import os +import time +import uuid + +from functools import partial +from email.parser import Parser +from email.Utils import formatdate + +from leap.mail.adaptors.soledad import SoledadMailAdaptor +from leap.mail.mail import MessageCollection, Account, _unpack_headers +from leap.mail.mailbox_indexer import MailboxIndexer +from leap.mail.testing.common import SoledadTestMixin + +HERE = os.path.split(os.path.abspath(__file__))[0] + + +def _get_raw_msg(multi=False): + if multi: + sample = "rfc822.multi.message" + else: + sample = "rfc822.message" + with open(os.path.join(HERE, sample)) as f: + raw = f.read() + return raw + + +def _get_parsed_msg(multi=False): + mail_parser = Parser() + raw = _get_raw_msg(multi=multi) + return mail_parser.parsestr(raw) + + +def _get_msg_time(): + timestamp = time.mktime((2010, 12, 12, 1, 1, 1, 1, 1, 1)) + return formatdate(timestamp) + + +class CollectionMixin(object): + + def get_collection(self, mbox_collection=True, mbox_name=None, + mbox_uuid=None): + """ + Get a collection for tests. + """ + adaptor = SoledadMailAdaptor() + store = self._soledad + adaptor.store = store + + if mbox_collection: + mbox_indexer = MailboxIndexer(store) + mbox_name = mbox_name or "TestMbox" + mbox_uuid = mbox_uuid or str(uuid.uuid4()) + else: + mbox_indexer = mbox_name = None + + def get_collection_from_mbox_wrapper(wrapper): + wrapper.uuid = mbox_uuid + return MessageCollection( + adaptor, store, + mbox_indexer=mbox_indexer, mbox_wrapper=wrapper) + + d = adaptor.initialize_store(store) + if mbox_collection: + d.addCallback(lambda _: mbox_indexer.create_table(mbox_uuid)) + d.addCallback(lambda _: adaptor.get_or_create_mbox(store, mbox_name)) + d.addCallback(get_collection_from_mbox_wrapper) + return d + + +# TODO profile add_msg. Why are these tests so SLOW??! +class MessageTestCase(SoledadTestMixin, CollectionMixin): + """ + Tests for the Message class. + """ + msg_flags = ('\Recent', '\Unseen', '\TestFlag') + msg_tags = ('important', 'todo', 'wonderful') + internal_date = "19-Mar-2015 19:22:21 -0500" + + maxDiff = None + + def _do_insert_msg(self, multi=False): + """ + Inserts and return a regular message, for tests. + """ + def insert_message(collection): + self._mbox_uuid = collection.mbox_uuid + return collection.add_msg( + raw, flags=self.msg_flags, tags=self.msg_tags, + date=self.internal_date) + + raw = _get_raw_msg(multi=multi) + + d = self.get_collection() + d.addCallback(insert_message) + return d + + def get_inserted_msg(self, multi=False): + d = self._do_insert_msg(multi=multi) + d.addCallback(lambda _: self.get_collection(mbox_uuid=self._mbox_uuid)) + d.addCallback(lambda col: col.get_message_by_uid(1)) + return d + + def test_get_flags(self): + d = self.get_inserted_msg() + d.addCallback(self._test_get_flags_cb) + return d + + def _test_get_flags_cb(self, msg): + self.assertTrue(msg is not None) + self.assertEquals(tuple(msg.get_flags()), self.msg_flags) + + def test_get_internal_date(self): + d = self.get_inserted_msg() + d.addCallback(self._test_get_internal_date_cb) + + def _test_get_internal_date_cb(self, msg): + self.assertTrue(msg is not None) + self.assertDictEqual(msg.get_internal_date(), + self.internal_date) + + def test_get_headers(self): + d = self.get_inserted_msg() + d.addCallback(self._test_get_headers_cb) + return d + + def _test_get_headers_cb(self, msg): + self.assertTrue(msg is not None) + expected = [ + (str(key.lower()), str(value)) + for (key, value) in _get_parsed_msg().items()] + self.assertItemsEqual(_unpack_headers(msg.get_headers()), expected) + + def test_get_body_file(self): + d = self.get_inserted_msg(multi=True) + d.addCallback(self._test_get_body_file_cb) + return d + + def _test_get_body_file_cb(self, msg): + self.assertTrue(msg is not None) + orig = _get_parsed_msg(multi=True) + expected = orig.get_payload()[0].get_payload() + d = msg.get_body_file(self._soledad) + + def assert_body(fd): + self.assertTrue(fd is not None) + self.assertEqual(fd.read(), expected) + d.addCallback(assert_body) + return d + + def test_get_size(self): + d = self.get_inserted_msg() + d.addCallback(self._test_get_size_cb) + return d + + def _test_get_size_cb(self, msg): + self.assertTrue(msg is not None) + expected = len(_get_parsed_msg().as_string()) + self.assertEqual(msg.get_size(), expected) + + def test_is_multipart_no(self): + d = self.get_inserted_msg() + d.addCallback(self._test_is_multipart_no_cb) + return d + + def _test_is_multipart_no_cb(self, msg): + self.assertTrue(msg is not None) + expected = _get_parsed_msg().is_multipart() + self.assertEqual(msg.is_multipart(), expected) + + def test_is_multipart_yes(self): + d = self.get_inserted_msg(multi=True) + d.addCallback(self._test_is_multipart_yes_cb) + return d + + def _test_is_multipart_yes_cb(self, msg): + self.assertTrue(msg is not None) + expected = _get_parsed_msg(multi=True).is_multipart() + self.assertEqual(msg.is_multipart(), expected) + + def test_get_subpart(self): + d = self.get_inserted_msg(multi=True) + d.addCallback(self._test_get_subpart_cb) + return d + + def _test_get_subpart_cb(self, msg): + self.assertTrue(msg is not None) + + def test_get_tags(self): + d = self.get_inserted_msg() + d.addCallback(self._test_get_tags_cb) + return d + + def _test_get_tags_cb(self, msg): + self.assertTrue(msg is not None) + self.assertEquals(msg.get_tags(), self.msg_tags) + + +class MessageCollectionTestCase(SoledadTestMixin, CollectionMixin): + """ + Tests for the MessageCollection class. + """ + _mbox_uuid = None + + def assert_collection_count(self, _, expected): + def _assert_count(count): + self.assertEqual(count, expected) + + d = self.get_collection() + d.addCallback(lambda col: col.count()) + d.addCallback(_assert_count) + return d + + def add_msg_to_collection(self): + raw = _get_raw_msg() + + def add_msg_to_collection(collection): + # We keep the uuid in case we need to instantiate the same + # collection afterwards. + self._mbox_uuid = collection.mbox_uuid + d = collection.add_msg(raw, date=_get_msg_time()) + return d + + d = self.get_collection() + d.addCallback(add_msg_to_collection) + return d + + def test_is_mailbox_collection(self): + d = self.get_collection() + d.addCallback(self._test_is_mailbox_collection_cb) + return d + + def _test_is_mailbox_collection_cb(self, collection): + self.assertTrue(collection.is_mailbox_collection()) + + def test_get_uid_next(self): + d = self.add_msg_to_collection() + d.addCallback(lambda _: self.get_collection()) + d.addCallback(lambda col: col.get_uid_next()) + d.addCallback(self._test_get_uid_next_cb) + + def _test_get_uid_next_cb(self, next_uid): + self.assertEqual(next_uid, 2) + + def test_add_and_count_msg(self): + d = self.add_msg_to_collection() + d.addCallback(self._test_add_and_count_msg_cb) + return d + + def _test_add_and_count_msg_cb(self, _): + return partial(self.assert_collection_count, expected=1) + + def test_copy_msg(self): + # TODO ---- update when implementing messagecopier + # interface + pass + test_copy_msg.skip = "Not yet implemented" + + def test_delete_msg(self): + d = self.add_msg_to_collection() + + def del_msg(collection): + def _delete_it(msg): + self.assertTrue(msg is not None) + return collection.delete_msg(msg) + + d = collection.get_message_by_uid(1) + d.addCallback(_delete_it) + return d + + # We need to instantiate an mbox collection with the same uuid that + # the one in which we inserted the doc. + d.addCallback(lambda _: self.get_collection(mbox_uuid=self._mbox_uuid)) + d.addCallback(del_msg) + d.addCallback(self._test_delete_msg_cb) + return d + + def _test_delete_msg_cb(self, _): + return partial(self.assert_collection_count, expected=0) + + def test_update_flags(self): + d = self.add_msg_to_collection() + d.addCallback(self._test_update_flags_cb) + return d + + def _test_update_flags_cb(self, msg): + pass + + def test_update_tags(self): + d = self.add_msg_to_collection() + d.addCallback(self._test_update_tags_cb) + return d + + def _test_update_tags_cb(self, msg): + pass + + +class AccountTestCase(SoledadTestMixin): + """ + Tests for the Account class. + """ + def get_account(self, user_id): + store = self._soledad + return Account(store, user_id) + + def test_add_mailbox(self): + acc = self.get_account('some_user_id') + d = acc.callWhenReady(lambda _: acc.add_mailbox("TestMailbox")) + d.addCallback(lambda _: acc.list_all_mailbox_names()) + d.addCallback(self._test_add_mailbox_cb) + return d + + def _test_add_mailbox_cb(self, mboxes): + expected = ['INBOX', 'TestMailbox'] + self.assertItemsEqual(mboxes, expected) + + def test_delete_mailbox(self): + acc = self.get_account('some_user_id') + d = acc.callWhenReady(lambda _: acc.delete_mailbox("Inbox")) + d.addCallback(lambda _: acc.list_all_mailbox_names()) + d.addCallback(self._test_delete_mailbox_cb) + return d + + def _test_delete_mailbox_cb(self, mboxes): + expected = [] + self.assertItemsEqual(mboxes, expected) + + def test_rename_mailbox(self): + acc = self.get_account('some_user_id') + d = acc.callWhenReady(lambda _: acc.add_mailbox("OriginalMailbox")) + d.addCallback(lambda _: acc.rename_mailbox( + "OriginalMailbox", "RenamedMailbox")) + d.addCallback(lambda _: acc.list_all_mailbox_names()) + d.addCallback(self._test_rename_mailbox_cb) + return d + + def _test_rename_mailbox_cb(self, mboxes): + expected = ['INBOX', 'RenamedMailbox'] + self.assertItemsEqual(mboxes, expected) + + def test_get_all_mailboxes(self): + acc = self.get_account('some_user_id') + d = acc.callWhenReady(lambda _: acc.add_mailbox("OneMailbox")) + d.addCallback(lambda _: acc.add_mailbox("TwoMailbox")) + d.addCallback(lambda _: acc.add_mailbox("ThreeMailbox")) + d.addCallback(lambda _: acc.add_mailbox("anotherthing")) + d.addCallback(lambda _: acc.add_mailbox("anotherthing2")) + d.addCallback(lambda _: acc.get_all_mailboxes()) + d.addCallback(self._test_get_all_mailboxes_cb) + return d + + def _test_get_all_mailboxes_cb(self, mailboxes): + expected = ["INBOX", "OneMailbox", "TwoMailbox", "ThreeMailbox", + "anotherthing", "anotherthing2"] + names = [m.mbox for m in mailboxes] + self.assertItemsEqual(names, expected) + + def test_get_collection_by_mailbox(self): + acc = self.get_account('some_user_id') + d = acc.callWhenReady(lambda _: acc.get_collection_by_mailbox("INBOX")) + d.addCallback(self._test_get_collection_by_mailbox_cb) + return d + + def _test_get_collection_by_mailbox_cb(self, collection): + self.assertTrue(collection.is_mailbox_collection()) + + def assert_uid_next_empty_collection(uid): + self.assertEqual(uid, 1) + d = collection.get_uid_next() + d.addCallback(assert_uid_next_empty_collection) + return d + + def test_get_collection_by_docs(self): + pass + + test_get_collection_by_docs.skip = "Not yet implemented" + + def test_get_collection_by_tag(self): + pass + + test_get_collection_by_tag.skip = "Not yet implemented" diff --git a/mail/src/leap/mail/tests/test_mailbox_indexer.py b/mail/src/leap/mail/tests/test_mailbox_indexer.py new file mode 100644 index 00000000..5c1891d5 --- /dev/null +++ b/mail/src/leap/mail/tests/test_mailbox_indexer.py @@ -0,0 +1,250 @@ +# -*- coding: utf-8 -*- +# test_mailbox_indexer.py +# Copyright (C) 2014 LEAP +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see <http://www.gnu.org/licenses/>. +""" +Tests for the mailbox_indexer module. +""" +import uuid +from functools import partial + +from leap.mail import mailbox_indexer as mi +from leap.mail.testing.common import SoledadTestMixin + +hash_test0 = '590c9f8430c7435807df8ba9a476e3f1295d46ef210f6efae2043a4c085a569e' +hash_test1 = '1b4f0e9851971998e732078544c96b36c3d01cedf7caa332359d6f1d83567014' +hash_test2 = '60303ae22b998861bce3b28f33eec1be758a213c86c93c076dbe9f558c11c752' +hash_test3 = 'fd61a03af4f77d870fc21e05e7e80678095c92d808cfb3b5c279ee04c74aca13' +hash_test4 = 'a4e624d686e03ed2767c0abd85c14426b0b1157d2ce81d27bb4fe4f6f01d688a' + + +def fmt_hash(mailbox_uuid, hash): + return "M-" + mailbox_uuid.replace('-', '_') + "-" + hash + +mbox_id = str(uuid.uuid4()) + + +class MailboxIndexerTestCase(SoledadTestMixin): + """ + Tests for the MailboxUID class. + """ + def get_mbox_uid(self): + m_uid = mi.MailboxIndexer(self._soledad) + return m_uid + + def list_mail_tables_cb(self, ignored): + def filter_mailuid_tables(tables): + filtered = [ + table[0] for table in tables if + table[0].startswith(mi.MailboxIndexer.table_preffix)] + return filtered + + sql = "SELECT name FROM sqlite_master WHERE type='table';" + d = self._soledad.raw_sqlcipher_query(sql) + d.addCallback(filter_mailuid_tables) + return d + + def select_uid_rows(self, mailbox): + sql = "SELECT * FROM %s%s;" % ( + mi.MailboxIndexer.table_preffix, mailbox.replace('-', '_')) + d = self._soledad.raw_sqlcipher_query(sql) + return d + + def test_create_table(self): + def assert_table_created(tables): + self.assertEqual( + tables, ["leapmail_uid_" + mbox_id.replace('-', '_')]) + + m_uid = self.get_mbox_uid() + d = m_uid.create_table(mbox_id) + d.addCallback(self.list_mail_tables_cb) + d.addCallback(assert_table_created) + return d + + def test_create_and_delete_table(self): + def assert_table_deleted(tables): + self.assertEqual(tables, []) + + m_uid = self.get_mbox_uid() + d = m_uid.create_table(mbox_id) + d.addCallback(lambda _: m_uid.delete_table(mbox_id)) + d.addCallback(self.list_mail_tables_cb) + d.addCallback(assert_table_deleted) + return d + + def test_insert_doc(self): + m_uid = self.get_mbox_uid() + + h1 = fmt_hash(mbox_id, hash_test0) + h2 = fmt_hash(mbox_id, hash_test1) + h3 = fmt_hash(mbox_id, hash_test2) + h4 = fmt_hash(mbox_id, hash_test3) + h5 = fmt_hash(mbox_id, hash_test4) + + def assert_uid_rows(rows): + expected = [(1, h1), (2, h2), (3, h3), (4, h4), (5, h5)] + self.assertEquals(rows, expected) + + d = m_uid.create_table(mbox_id) + d.addCallback(lambda _: m_uid.insert_doc(mbox_id, h1)) + d.addCallback(lambda _: m_uid.insert_doc(mbox_id, h2)) + d.addCallback(lambda _: m_uid.insert_doc(mbox_id, h3)) + d.addCallback(lambda _: m_uid.insert_doc(mbox_id, h4)) + d.addCallback(lambda _: m_uid.insert_doc(mbox_id, h5)) + d.addCallback(lambda _: self.select_uid_rows(mbox_id)) + d.addCallback(assert_uid_rows) + return d + + def test_insert_doc_return(self): + m_uid = self.get_mbox_uid() + + def assert_rowid(rowid, expected=None): + self.assertEqual(rowid, expected) + + h1 = fmt_hash(mbox_id, hash_test0) + h2 = fmt_hash(mbox_id, hash_test1) + h3 = fmt_hash(mbox_id, hash_test2) + + d = m_uid.create_table(mbox_id) + d.addCallback(lambda _: m_uid.insert_doc(mbox_id, h1)) + d.addCallback(partial(assert_rowid, expected=1)) + d.addCallback(lambda _: m_uid.insert_doc(mbox_id, h2)) + d.addCallback(partial(assert_rowid, expected=2)) + d.addCallback(lambda _: m_uid.insert_doc(mbox_id, h3)) + d.addCallback(partial(assert_rowid, expected=3)) + return d + + def test_delete_doc(self): + m_uid = self.get_mbox_uid() + + h1 = fmt_hash(mbox_id, hash_test0) + h2 = fmt_hash(mbox_id, hash_test1) + h3 = fmt_hash(mbox_id, hash_test2) + h4 = fmt_hash(mbox_id, hash_test3) + h5 = fmt_hash(mbox_id, hash_test4) + + def assert_uid_rows(rows): + expected = [(4, h4), (5, h5)] + self.assertEquals(rows, expected) + + d = m_uid.create_table(mbox_id) + d.addCallback(lambda _: m_uid.insert_doc(mbox_id, h1)) + d.addCallback(lambda _: m_uid.insert_doc(mbox_id, h2)) + d.addCallback(lambda _: m_uid.insert_doc(mbox_id, h3)) + d.addCallback(lambda _: m_uid.insert_doc(mbox_id, h4)) + d.addCallback(lambda _: m_uid.insert_doc(mbox_id, h5)) + + d.addCallbacks(lambda _: m_uid.delete_doc_by_uid(mbox_id, 1)) + d.addCallbacks(lambda _: m_uid.delete_doc_by_uid(mbox_id, 2)) + d.addCallbacks(lambda _: m_uid.delete_doc_by_hash(mbox_id, h3)) + + d.addCallback(lambda _: self.select_uid_rows(mbox_id)) + d.addCallback(assert_uid_rows) + return d + + def test_get_doc_id_from_uid(self): + m_uid = self.get_mbox_uid() + + h1 = fmt_hash(mbox_id, hash_test0) + + def assert_doc_hash(res): + self.assertEqual(res, h1) + + d = m_uid.create_table(mbox_id) + d.addCallback(lambda _: m_uid.insert_doc(mbox_id, h1)) + d.addCallback(lambda _: m_uid.get_doc_id_from_uid(mbox_id, 1)) + d.addCallback(assert_doc_hash) + return d + + def test_count(self): + m_uid = self.get_mbox_uid() + + h1 = fmt_hash(mbox_id, hash_test0) + h2 = fmt_hash(mbox_id, hash_test1) + h3 = fmt_hash(mbox_id, hash_test2) + h4 = fmt_hash(mbox_id, hash_test3) + h5 = fmt_hash(mbox_id, hash_test4) + + d = m_uid.create_table(mbox_id) + d.addCallback(lambda _: m_uid.insert_doc(mbox_id, h1)) + d.addCallback(lambda _: m_uid.insert_doc(mbox_id, h2)) + d.addCallback(lambda _: m_uid.insert_doc(mbox_id, h3)) + d.addCallback(lambda _: m_uid.insert_doc(mbox_id, h4)) + d.addCallback(lambda _: m_uid.insert_doc(mbox_id, h5)) + + def assert_count_after_inserts(count): + self.assertEquals(count, 5) + + d.addCallback(lambda _: m_uid.count(mbox_id)) + d.addCallback(assert_count_after_inserts) + + d.addCallbacks(lambda _: m_uid.delete_doc_by_uid(mbox_id, 1)) + d.addCallbacks(lambda _: m_uid.delete_doc_by_uid(mbox_id, 2)) + + def assert_count_after_deletions(count): + self.assertEquals(count, 3) + + d.addCallback(lambda _: m_uid.count(mbox_id)) + d.addCallback(assert_count_after_deletions) + return d + + def test_get_next_uid(self): + m_uid = self.get_mbox_uid() + + h1 = fmt_hash(mbox_id, hash_test0) + h2 = fmt_hash(mbox_id, hash_test1) + h3 = fmt_hash(mbox_id, hash_test2) + h4 = fmt_hash(mbox_id, hash_test3) + h5 = fmt_hash(mbox_id, hash_test4) + + d = m_uid.create_table(mbox_id) + d.addCallback(lambda _: m_uid.insert_doc(mbox_id, h1)) + d.addCallback(lambda _: m_uid.insert_doc(mbox_id, h2)) + d.addCallback(lambda _: m_uid.insert_doc(mbox_id, h3)) + d.addCallback(lambda _: m_uid.insert_doc(mbox_id, h4)) + d.addCallback(lambda _: m_uid.insert_doc(mbox_id, h5)) + + def assert_next_uid(result, expected=1): + self.assertEquals(result, expected) + + d.addCallback(lambda _: m_uid.get_next_uid(mbox_id)) + d.addCallback(partial(assert_next_uid, expected=6)) + return d + + def test_all_uid_iter(self): + + m_uid = self.get_mbox_uid() + + h1 = fmt_hash(mbox_id, hash_test0) + h2 = fmt_hash(mbox_id, hash_test1) + h3 = fmt_hash(mbox_id, hash_test2) + h4 = fmt_hash(mbox_id, hash_test3) + h5 = fmt_hash(mbox_id, hash_test4) + + d = m_uid.create_table(mbox_id) + d.addCallback(lambda _: m_uid.insert_doc(mbox_id, h1)) + d.addCallback(lambda _: m_uid.insert_doc(mbox_id, h2)) + d.addCallback(lambda _: m_uid.insert_doc(mbox_id, h3)) + d.addCallback(lambda _: m_uid.insert_doc(mbox_id, h4)) + d.addCallback(lambda _: m_uid.insert_doc(mbox_id, h5)) + d.addCallback(lambda _: m_uid.delete_doc_by_uid(mbox_id, 1)) + d.addCallback(lambda _: m_uid.delete_doc_by_uid(mbox_id, 4)) + + def assert_all_uid(result, expected=[2, 3, 5]): + self.assertEquals(result, expected) + + d.addCallback(lambda _: m_uid.all_uid_iter(mbox_id)) + d.addCallback(partial(assert_all_uid)) + return d diff --git a/mail/src/leap/mail/tests/test_walk.py b/mail/src/leap/mail/tests/test_walk.py new file mode 100644 index 00000000..826ec10c --- /dev/null +++ b/mail/src/leap/mail/tests/test_walk.py @@ -0,0 +1,81 @@ +""" +Tests for leap.mail.walk module +""" +import os.path +from email.parser import Parser + +from leap.mail import walk + +CORPUS = { + 'simple': 'rfc822.message', + 'multimin': 'rfc822.multi-minimal.message', + 'multisigned': 'rfc822.multi-signed.message', + 'bounced': 'rfc822.bounce.message', +} + +_here = os.path.dirname(__file__) +_parser = Parser() + + +# tests + + +def test_simple_mail(): + msg = _parse('simple') + tree = walk.get_tree(msg) + assert len(tree['part_map']) == 0 + assert tree['ctype'] == 'text/plain' + assert tree['multi'] is False + + +def test_multipart_minimal(): + msg = _parse('multimin') + tree = walk.get_tree(msg) + + assert tree['multi'] is True + assert len(tree['part_map']) == 1 + first = tree['part_map'][1] + assert first['multi'] is False + assert first['ctype'] == 'text/plain' + + +def test_multi_signed(): + msg = _parse('multisigned') + tree = walk.get_tree(msg) + assert tree['multi'] is True + assert len(tree['part_map']) == 2 + + _first = tree['part_map'][1] + _second = tree['part_map'][2] + assert len(_first['part_map']) == 3 + assert(_second['multi'] is False) + + +def test_bounce_mime(): + msg = _parse('bounced') + tree = walk.get_tree(msg) + + ctypes = [tree['part_map'][index]['ctype'] + for index in sorted(tree['part_map'].keys())] + third = tree['part_map'][3] + three_one_ctype = third['part_map'][1]['ctype'] + assert three_one_ctype == 'multipart/signed' + + assert ctypes == [ + 'text/plain', + 'message/delivery-status', + 'message/rfc822'] + + +# utils + +def _parse(name): + _str = _get_string_for_message(name) + return _parser.parsestr(_str) + + +def _get_string_for_message(name): + filename = os.path.join(_here, CORPUS[name]) + with open(filename) as f: + msgstr = f.read() + return msgstr diff --git a/mail/src/leap/mail/utils.py b/mail/src/leap/mail/utils.py new file mode 100644 index 00000000..64fca981 --- /dev/null +++ b/mail/src/leap/mail/utils.py @@ -0,0 +1,375 @@ +# -*- coding: utf-8 -*- +# utils.py +# Copyright (C) 2013 LEAP +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see <http://www.gnu.org/licenses/>. +""" +Mail utilities. +""" +from email.utils import parseaddr +import json +import re +import traceback +import Queue + +from leap.soledad.common.document import SoledadDocument +from leap.common.check import leap_assert_type +from twisted.mail import smtp + + +CHARSET_PATTERN = r"""charset=([\w-]+)""" +CHARSET_RE = re.compile(CHARSET_PATTERN, re.IGNORECASE) + + +def first(things): + """ + Return the head of a collection. + """ + try: + return things[0] + except (IndexError, TypeError): + return None + + +def empty(thing): + """ + Return True if a thing is None or its length is zero. + If thing is a number (int, float, long), return False. + """ + if thing is None: + return True + if isinstance(thing, (int, float, long)): + return False + if isinstance(thing, SoledadDocument): + thing = thing.content + try: + return len(thing) == 0 + except (ReferenceError, TypeError): + return True + + +def maybe_call(thing): + """ + Return the same thing, or the result of its invocation if it is a + callable. + """ + return thing() if callable(thing) else thing + + +def find_charset(thing, default=None): + """ + Looks into the object 'thing' for a charset specification. + It searchs into the object's `repr`. + + :param thing: the object to look into. + :type thing: object + :param default: the dafault charset to return if no charset is found. + :type default: str + + :return: the charset or 'default' + :rtype: str or None + """ + charset = first(CHARSET_RE.findall(repr(thing))) + if charset is None: + charset = default + return charset + + +def lowerdict(_dict): + """ + Return a dict with the keys in lowercase. + + :param _dict: the dict to convert + :rtype: dict + """ + # TODO should properly implement a CaseInsensitive dict. + # Look into requests code. + return dict((key.lower(), value) + for key, value in _dict.items()) + + +PART_MAP = "part_map" +PHASH = "phash" + + +def _str_dict(d, k): + """ + Convert the dictionary key to string if it was a string. + + :param d: the dict + :type d: dict + :param k: the key + :type k: object + """ + if isinstance(k, int): + val = d[k] + d[str(k)] = val + del(d[k]) + + +def stringify_parts_map(d): + """ + Modify a dictionary making all the nested dicts under "part_map" keys + having strings as keys. + + :param d: the dictionary to modify + :type d: dictionary + :rtype: dictionary + """ + for k in d: + if k == PART_MAP: + pmap = d[k] + for kk in pmap.keys(): + _str_dict(d[k], kk) + for kk in pmap.keys(): + stringify_parts_map(d[k][str(kk)]) + return d + + +def phash_iter(d): + """ + A recursive generator that extracts all the payload-hashes + from an arbitrary nested parts-map dictionary. + + :param d: the dictionary to walk + :type d: dictionary + :return: a list of all the phashes found + :rtype: list + """ + if PHASH in d: + yield d[PHASH] + if PART_MAP in d: + for key in d[PART_MAP]: + for phash in phash_iter(d[PART_MAP][key]): + yield phash + + +def accumulator(fun, lim): + """ + A simple accumulator that uses a closure and a mutable + object to collect items. + When the count of items is greater than `lim`, the + collection is flushed after invoking a map of the function `fun` + over it. + + The returned accumulator can also be flushed at any moment + by passing a boolean as a second parameter. + + :param fun: the function to call over the collection + when its size is greater than `lim` + :type fun: callable + :param lim: the turning point for the collection + :type lim: int + :rtype: function + + >>> from pprint import pprint + >>> acc = accumulator(pprint, 2) + >>> acc(1) + >>> acc(2) + [1, 2] + >>> acc(3) + >>> acc(4) + [3, 4] + >>> acc = accumulator(pprint, 5) + >>> acc(1) + >>> acc(2) + >>> acc(3) + >>> acc(None, flush=True) + [1,2,3] + """ + KEY = "items" + _o = {KEY: []} + + def _accumulator(item, flush=False): + collection = _o[KEY] + collection.append(item) + if len(collection) >= lim or flush: + map(fun, filter(None, collection)) + _o[KEY] = [] + + return _accumulator + + +def accumulator_queue(fun, lim): + """ + A version of the accumulator that uses a queue. + + When the count of items is greater than `lim`, the + queue is flushed after invoking the function `fun` + over its items. + + The returned accumulator can also be flushed at any moment + by passing a boolean as a second parameter. + + :param fun: the function to call over the collection + when its size is greater than `lim` + :type fun: callable + :param lim: the turning point for the collection + :type lim: int + :rtype: function + """ + _q = Queue.Queue() + + def _accumulator(item, flush=False): + _q.put(item) + if _q.qsize() >= lim or flush: + collection = [_q.get() for i in range(_q.qsize())] + map(fun, filter(None, collection)) + + return _accumulator + + +def validate_address(address): + """ + Validate C{address} as defined in RFC 2822. + + :param address: The address to be validated. + :type address: str + + @return: A valid address. + @rtype: str + + @raise smtp.SMTPBadRcpt: Raised if C{address} is invalid. + """ + leap_assert_type(address, str) + # in the following, the address is parsed as described in RFC 2822 and + # ('', '') is returned if the parse fails. + _, address = parseaddr(address) + if address == '': + raise smtp.SMTPBadRcpt(address) + return address + +# +# String manipulation +# + + +class CustomJsonScanner(object): + """ + This class is a context manager definition used to monkey patch the default + json string parsing behavior. + The emails can have more than one encoding, so the `str` objects have more + than one encoding and json does not support direct work with `str` + (only `unicode`). + """ + + def _parse_string_str(self, s, idx, *args, **kwargs): + """ + Parses the string "s" starting at the point idx and returns an `str` + object. Which basically means it works exactly the same as the regular + JSON string parsing, except that it doesn't try to decode utf8. + We need this because mail raw strings might have bytes in multiple + encodings. + + :param s: the string we want to parse + :type s: str + :param idx: the starting point for parsing + :type idx: int + + :returns: the parsed string and the index where the + string ends. + :rtype: tuple (str, int) + """ + # NOTE: we just want to use this monkey patched version if we are + # calling the loads from our custom method. Otherwise, we use the + # json's default parser. + monkey_patched = False + for i in traceback.extract_stack(): + # look for json_loads method in the call stack + if i[2] == json_loads.__name__: + monkey_patched = True + break + + if not monkey_patched: + return self._orig_scanstring(s, idx, *args, **kwargs) + + # TODO profile to see if a compiled regex can get us some + # benefit here. + found = False + end = s.find("\"", idx) + while not found: + try: + if s[end - 1] != "\\": + found = True + else: + end = s.find("\"", end + 1) + except Exception: + found = True + return s[idx:end].decode("string-escape"), end + 1 + + def __enter__(self): + """ + Replace the json methods with the needed ones. + Also make a backup to restore them later. + """ + # backup original values + self._orig_make_scanner = json.scanner.make_scanner + self._orig_scanstring = json.decoder.scanstring + + # We need the make_scanner function to be the python one so we can + # monkey_patch the json string parsing + json.scanner.make_scanner = json.scanner.py_make_scanner + + # And now we monkey patch the money method + json.decoder.scanstring = self._parse_string_str + + def __exit__(self, exc_type, exc_value, traceback): + """ + Restores the backuped methods. + """ + # restore original values + json.scanner.make_scanner = self._orig_make_scanner + json.decoder.scanstring = self._orig_scanstring + + +def json_loads(data): + """ + It works as json.loads but supporting multiple encodings in the same + string and accepting an `str` parameter that won't be converted to unicode. + + :param data: the string to load the objects from + :type data: str + + :returns: the corresponding python object result of parsing 'data', this + behaves similarly as json.loads, with the exception of that + returns always `str` instead of `unicode`. + """ + obj = None + with CustomJsonScanner(): + # We need to use the cls parameter in order to trigger the code + # that will let us control the string parsing method. + obj = json.loads(data, cls=json.JSONDecoder) + + return obj + + +class CaseInsensitiveDict(dict): + """ + A dictionary subclass that will allow case-insenstive key lookups. + """ + def __init__(self, d=None): + if d is None: + d = [] + if isinstance(d, dict): + for key, value in d.items(): + self[key] = value + else: + for key, value in d: + self[key] = value + + def __setitem__(self, key, value): + super(CaseInsensitiveDict, self).__setitem__(key.lower(), value) + + def __getitem__(self, key): + return super(CaseInsensitiveDict, self).__getitem__(key.lower()) diff --git a/mail/src/leap/mail/walk.py b/mail/src/leap/mail/walk.py new file mode 100644 index 00000000..d143d61e --- /dev/null +++ b/mail/src/leap/mail/walk.py @@ -0,0 +1,107 @@ +# -*- coding: utf-8 -*- +# walk.py +# Copyright (C) 2013-2015 LEAP +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see <http://www.gnu.org/licenses/>. +""" +Walk a message tree and generate documents that can be inserted in the backend +store. +""" +from email.parser import Parser + +from cryptography.hazmat.backends.multibackend import MultiBackend +from cryptography.hazmat.backends.openssl.backend import ( + Backend as OpenSSLBackend) +from cryptography.hazmat.primitives import hashes + +from leap.mail.utils import first + +crypto_backend = MultiBackend([OpenSSLBackend()]) + +_parser = Parser() + + +def get_tree(msg): + p = {} + p['ctype'] = msg.get_content_type() + p['headers'] = msg.items() + + payload = msg.get_payload() + is_multi = msg.is_multipart() + if is_multi: + p['part_map'] = dict( + [(idx, get_tree(part)) for idx, part in enumerate(payload, 1)]) + p['parts'] = len(payload) + p['phash'] = None + else: + p['parts'] = 0 + p['size'] = len(payload) + p['phash'] = get_hash(payload) + p['part_map'] = {} + p['multi'] = is_multi + return p + + +def get_tree_from_string(messagestr): + return get_tree(_parser.parsestr(messagestr)) + + +def get_body_phash(msg): + """ + Find the body payload-hash for this message. + """ + for part in msg.walk(): + # XXX what other ctypes should be considered body? + if part.get_content_type() in ("text/plain", "text/html"): + # XXX avoid hashing again + return get_hash(part.get_payload()) + + +def get_raw_docs(msg): + """ + We get also some of the headers to be able to + index the content. Here we remove any mutable part, as the the filename + in the content disposition. + """ + return ( + {'type': 'cnt', + 'raw': part.get_payload(), + 'phash': get_hash(part.get_payload()), + 'content-type': part.get_content_type(), + 'content-disposition': first(part.get( + 'content-disposition', '').split(';')), + 'content-transfer-encoding': part.get( + 'content-transfer-encoding', '') + } for part in msg.walk() if not isinstance(part.get_payload(), list)) + + +def get_hash(s): + digest = hashes.Hash(hashes.SHA256(), crypto_backend) + digest.update(s) + return digest.finalize().encode("hex").upper() + + +""" +Groucho Marx: Now pay particular attention to this first clause, because it's + most important. There's the party of the first part shall be + known in this contract as the party of the first part. How do you + like that, that's pretty neat eh? + +Chico Marx: No, that's no good. +Groucho Marx: What's the matter with it? + +Chico Marx: I don't know, let's hear it again. +Groucho Marx: So the party of the first part shall be known in this contract as + the party of the first part. +""" diff --git a/mail/tox.ini b/mail/tox.ini new file mode 100644 index 00000000..e2393f74 --- /dev/null +++ b/mail/tox.ini @@ -0,0 +1,18 @@ +[tox] +envlist = py27 + +[testenv] +commands = py.test --pep8 {posargs} +deps = + mock + pep8 + pytest + pytest-pep8 + pdbpp + setuptools-trial + -e../soledad/common + -e../soledad/client + -e../keymanager + -e. +setenv = + HOME=/tmp diff --git a/mail/versioneer.py b/mail/versioneer.py new file mode 100644 index 00000000..7ed2a21d --- /dev/null +++ b/mail/versioneer.py @@ -0,0 +1,1774 @@ + +# Version: 0.16 + +"""The Versioneer - like a rocketeer, but for versions. + +The Versioneer +============== + +* like a rocketeer, but for versions! +* https://github.com/warner/python-versioneer +* Brian Warner +* License: Public Domain +* Compatible With: python2.6, 2.7, 3.3, 3.4, 3.5, and pypy +* [![Latest Version] +(https://pypip.in/version/versioneer/badge.svg?style=flat) +](https://pypi.python.org/pypi/versioneer/) +* [![Build Status] +(https://travis-ci.org/warner/python-versioneer.png?branch=master) +](https://travis-ci.org/warner/python-versioneer) + +This is a tool for managing a recorded version number in distutils-based +python projects. The goal is to remove the tedious and error-prone "update +the embedded version string" step from your release process. Making a new +release should be as easy as recording a new tag in your version-control +system, and maybe making new tarballs. + + +## Quick Install + +* `pip install versioneer` to somewhere to your $PATH +* add a `[versioneer]` section to your setup.cfg (see below) +* run `versioneer install` in your source tree, commit the results + +## Version Identifiers + +Source trees come from a variety of places: + +* a version-control system checkout (mostly used by developers) +* a nightly tarball, produced by build automation +* a snapshot tarball, produced by a web-based VCS browser, like github's + "tarball from tag" feature +* a release tarball, produced by "setup.py sdist", distributed through PyPI + +Within each source tree, the version identifier (either a string or a number, +this tool is format-agnostic) can come from a variety of places: + +* ask the VCS tool itself, e.g. "git describe" (for checkouts), which knows + about recent "tags" and an absolute revision-id +* the name of the directory into which the tarball was unpacked +* an expanded VCS keyword ($Id$, etc) +* a `_version.py` created by some earlier build step + +For released software, the version identifier is closely related to a VCS +tag. Some projects use tag names that include more than just the version +string (e.g. "myproject-1.2" instead of just "1.2"), in which case the tool +needs to strip the tag prefix to extract the version identifier. For +unreleased software (between tags), the version identifier should provide +enough information to help developers recreate the same tree, while also +giving them an idea of roughly how old the tree is (after version 1.2, before +version 1.3). Many VCS systems can report a description that captures this, +for example `git describe --tags --dirty --always` reports things like +"0.7-1-g574ab98-dirty" to indicate that the checkout is one revision past the +0.7 tag, has a unique revision id of "574ab98", and is "dirty" (it has +uncommitted changes. + +The version identifier is used for multiple purposes: + +* to allow the module to self-identify its version: `myproject.__version__` +* to choose a name and prefix for a 'setup.py sdist' tarball + +## Theory of Operation + +Versioneer works by adding a special `_version.py` file into your source +tree, where your `__init__.py` can import it. This `_version.py` knows how to +dynamically ask the VCS tool for version information at import time. + +`_version.py` also contains `$Revision$` markers, and the installation +process marks `_version.py` to have this marker rewritten with a tag name +during the `git archive` command. As a result, generated tarballs will +contain enough information to get the proper version. + +To allow `setup.py` to compute a version too, a `versioneer.py` is added to +the top level of your source tree, next to `setup.py` and the `setup.cfg` +that configures it. This overrides several distutils/setuptools commands to +compute the version when invoked, and changes `setup.py build` and `setup.py +sdist` to replace `_version.py` with a small static file that contains just +the generated version data. + +## Installation + +First, decide on values for the following configuration variables: + +* `VCS`: the version control system you use. Currently accepts "git". + +* `style`: the style of version string to be produced. See "Styles" below for + details. Defaults to "pep440", which looks like + `TAG[+DISTANCE.gSHORTHASH[.dirty]]`. + +* `versionfile_source`: + + A project-relative pathname into which the generated version strings should + be written. This is usually a `_version.py` next to your project's main + `__init__.py` file, so it can be imported at runtime. If your project uses + `src/myproject/__init__.py`, this should be `src/myproject/_version.py`. + This file should be checked in to your VCS as usual: the copy created below + by `setup.py setup_versioneer` will include code that parses expanded VCS + keywords in generated tarballs. The 'build' and 'sdist' commands will + replace it with a copy that has just the calculated version string. + + This must be set even if your project does not have any modules (and will + therefore never import `_version.py`), since "setup.py sdist" -based trees + still need somewhere to record the pre-calculated version strings. Anywhere + in the source tree should do. If there is a `__init__.py` next to your + `_version.py`, the `setup.py setup_versioneer` command (described below) + will append some `__version__`-setting assignments, if they aren't already + present. + +* `versionfile_build`: + + Like `versionfile_source`, but relative to the build directory instead of + the source directory. These will differ when your setup.py uses + 'package_dir='. If you have `package_dir={'myproject': 'src/myproject'}`, + then you will probably have `versionfile_build='myproject/_version.py'` and + `versionfile_source='src/myproject/_version.py'`. + + If this is set to None, then `setup.py build` will not attempt to rewrite + any `_version.py` in the built tree. If your project does not have any + libraries (e.g. if it only builds a script), then you should use + `versionfile_build = None`. To actually use the computed version string, + your `setup.py` will need to override `distutils.command.build_scripts` + with a subclass that explicitly inserts a copy of + `versioneer.get_version()` into your script file. See + `test/demoapp-script-only/setup.py` for an example. + +* `tag_prefix`: + + a string, like 'PROJECTNAME-', which appears at the start of all VCS tags. + If your tags look like 'myproject-1.2.0', then you should use + tag_prefix='myproject-'. If you use unprefixed tags like '1.2.0', this + should be an empty string, using either `tag_prefix=` or `tag_prefix=''`. + +* `parentdir_prefix`: + + a optional string, frequently the same as tag_prefix, which appears at the + start of all unpacked tarball filenames. If your tarball unpacks into + 'myproject-1.2.0', this should be 'myproject-'. To disable this feature, + just omit the field from your `setup.cfg`. + +This tool provides one script, named `versioneer`. That script has one mode, +"install", which writes a copy of `versioneer.py` into the current directory +and runs `versioneer.py setup` to finish the installation. + +To versioneer-enable your project: + +* 1: Modify your `setup.cfg`, adding a section named `[versioneer]` and + populating it with the configuration values you decided earlier (note that + the option names are not case-sensitive): + + ```` + [versioneer] + VCS = git + style = pep440 + versionfile_source = src/myproject/_version.py + versionfile_build = myproject/_version.py + tag_prefix = + parentdir_prefix = myproject- + ```` + +* 2: Run `versioneer install`. This will do the following: + + * copy `versioneer.py` into the top of your source tree + * create `_version.py` in the right place (`versionfile_source`) + * modify your `__init__.py` (if one exists next to `_version.py`) to define + `__version__` (by calling a function from `_version.py`) + * modify your `MANIFEST.in` to include both `versioneer.py` and the + generated `_version.py` in sdist tarballs + + `versioneer install` will complain about any problems it finds with your + `setup.py` or `setup.cfg`. Run it multiple times until you have fixed all + the problems. + +* 3: add a `import versioneer` to your setup.py, and add the following + arguments to the setup() call: + + version=versioneer.get_version(), + cmdclass=versioneer.get_cmdclass(), + +* 4: commit these changes to your VCS. To make sure you won't forget, + `versioneer install` will mark everything it touched for addition using + `git add`. Don't forget to add `setup.py` and `setup.cfg` too. + +## Post-Installation Usage + +Once established, all uses of your tree from a VCS checkout should get the +current version string. All generated tarballs should include an embedded +version string (so users who unpack them will not need a VCS tool installed). + +If you distribute your project through PyPI, then the release process should +boil down to two steps: + +* 1: git tag 1.0 +* 2: python setup.py register sdist upload + +If you distribute it through github (i.e. users use github to generate +tarballs with `git archive`), the process is: + +* 1: git tag 1.0 +* 2: git push; git push --tags + +Versioneer will report "0+untagged.NUMCOMMITS.gHASH" until your tree has at +least one tag in its history. + +## Version-String Flavors + +Code which uses Versioneer can learn about its version string at runtime by +importing `_version` from your main `__init__.py` file and running the +`get_versions()` function. From the "outside" (e.g. in `setup.py`), you can +import the top-level `versioneer.py` and run `get_versions()`. + +Both functions return a dictionary with different flavors of version +information: + +* `['version']`: A condensed version string, rendered using the selected + style. This is the most commonly used value for the project's version + string. The default "pep440" style yields strings like `0.11`, + `0.11+2.g1076c97`, or `0.11+2.g1076c97.dirty`. See the "Styles" section + below for alternative styles. + +* `['full-revisionid']`: detailed revision identifier. For Git, this is the + full SHA1 commit id, e.g. "1076c978a8d3cfc70f408fe5974aa6c092c949ac". + +* `['dirty']`: a boolean, True if the tree has uncommitted changes. Note that + this is only accurate if run in a VCS checkout, otherwise it is likely to + be False or None + +* `['error']`: if the version string could not be computed, this will be set + to a string describing the problem, otherwise it will be None. It may be + useful to throw an exception in setup.py if this is set, to avoid e.g. + creating tarballs with a version string of "unknown". + +Some variants are more useful than others. Including `full-revisionid` in a +bug report should allow developers to reconstruct the exact code being tested +(or indicate the presence of local changes that should be shared with the +developers). `version` is suitable for display in an "about" box or a CLI +`--version` output: it can be easily compared against release notes and lists +of bugs fixed in various releases. + +The installer adds the following text to your `__init__.py` to place a basic +version in `YOURPROJECT.__version__`: + + from ._version import get_versions + __version__ = get_versions()['version'] + del get_versions + +## Styles + +The setup.cfg `style=` configuration controls how the VCS information is +rendered into a version string. + +The default style, "pep440", produces a PEP440-compliant string, equal to the +un-prefixed tag name for actual releases, and containing an additional "local +version" section with more detail for in-between builds. For Git, this is +TAG[+DISTANCE.gHEX[.dirty]] , using information from `git describe --tags +--dirty --always`. For example "0.11+2.g1076c97.dirty" indicates that the +tree is like the "1076c97" commit but has uncommitted changes (".dirty"), and +that this commit is two revisions ("+2") beyond the "0.11" tag. For released +software (exactly equal to a known tag), the identifier will only contain the +stripped tag, e.g. "0.11". + +Other styles are available. See details.md in the Versioneer source tree for +descriptions. + +## Debugging + +Versioneer tries to avoid fatal errors: if something goes wrong, it will tend +to return a version of "0+unknown". To investigate the problem, run `setup.py +version`, which will run the version-lookup code in a verbose mode, and will +display the full contents of `get_versions()` (including the `error` string, +which may help identify what went wrong). + +## Updating Versioneer + +To upgrade your project to a new release of Versioneer, do the following: + +* install the new Versioneer (`pip install -U versioneer` or equivalent) +* edit `setup.cfg`, if necessary, to include any new configuration settings + indicated by the release notes +* re-run `versioneer install` in your source tree, to replace + `SRC/_version.py` +* commit any changed files + +### Upgrading to 0.16 + +Nothing special. + +### Upgrading to 0.15 + +Starting with this version, Versioneer is configured with a `[versioneer]` +section in your `setup.cfg` file. Earlier versions required the `setup.py` to +set attributes on the `versioneer` module immediately after import. The new +version will refuse to run (raising an exception during import) until you +have provided the necessary `setup.cfg` section. + +In addition, the Versioneer package provides an executable named +`versioneer`, and the installation process is driven by running `versioneer +install`. In 0.14 and earlier, the executable was named +`versioneer-installer` and was run without an argument. + +### Upgrading to 0.14 + +0.14 changes the format of the version string. 0.13 and earlier used +hyphen-separated strings like "0.11-2-g1076c97-dirty". 0.14 and beyond use a +plus-separated "local version" section strings, with dot-separated +components, like "0.11+2.g1076c97". PEP440-strict tools did not like the old +format, but should be ok with the new one. + +### Upgrading from 0.11 to 0.12 + +Nothing special. + +### Upgrading from 0.10 to 0.11 + +You must add a `versioneer.VCS = "git"` to your `setup.py` before re-running +`setup.py setup_versioneer`. This will enable the use of additional +version-control systems (SVN, etc) in the future. + +## Future Directions + +This tool is designed to make it easily extended to other version-control +systems: all VCS-specific components are in separate directories like +src/git/ . The top-level `versioneer.py` script is assembled from these +components by running make-versioneer.py . In the future, make-versioneer.py +will take a VCS name as an argument, and will construct a version of +`versioneer.py` that is specific to the given VCS. It might also take the +configuration arguments that are currently provided manually during +installation by editing setup.py . Alternatively, it might go the other +direction and include code from all supported VCS systems, reducing the +number of intermediate scripts. + + +## License + +To make Versioneer easier to embed, all its code is dedicated to the public +domain. The `_version.py` that it creates is also in the public domain. +Specifically, both are released under the Creative Commons "Public Domain +Dedication" license (CC0-1.0), as described in +https://creativecommons.org/publicdomain/zero/1.0/ . + +""" + +from __future__ import print_function +try: + import configparser +except ImportError: + import ConfigParser as configparser +import errno +import json +import os +import re +import subprocess +import sys + + +class VersioneerConfig: + """Container for Versioneer configuration parameters.""" + + +def get_root(): + """Get the project root directory. + + We require that all commands are run from the project root, i.e. the + directory that contains setup.py, setup.cfg, and versioneer.py . + """ + root = os.path.realpath(os.path.abspath(os.getcwd())) + setup_py = os.path.join(root, "setup.py") + versioneer_py = os.path.join(root, "versioneer.py") + if not (os.path.exists(setup_py) or os.path.exists(versioneer_py)): + # allow 'python path/to/setup.py COMMAND' + root = os.path.dirname(os.path.realpath(os.path.abspath(sys.argv[0]))) + setup_py = os.path.join(root, "setup.py") + versioneer_py = os.path.join(root, "versioneer.py") + if not (os.path.exists(setup_py) or os.path.exists(versioneer_py)): + err = ("Versioneer was unable to run the project root directory. " + "Versioneer requires setup.py to be executed from " + "its immediate directory (like 'python setup.py COMMAND'), " + "or in a way that lets it use sys.argv[0] to find the root " + "(like 'python path/to/setup.py COMMAND').") + raise VersioneerBadRootError(err) + try: + # Certain runtime workflows (setup.py install/develop in a setuptools + # tree) execute all dependencies in a single python process, so + # "versioneer" may be imported multiple times, and python's shared + # module-import table will cache the first one. So we can't use + # os.path.dirname(__file__), as that will find whichever + # versioneer.py was first imported, even in later projects. + me = os.path.realpath(os.path.abspath(__file__)) + if os.path.splitext(me)[0] != os.path.splitext(versioneer_py)[0]: + print("Warning: build in %s is using versioneer.py from %s" + % (os.path.dirname(me), versioneer_py)) + except NameError: + pass + return root + + +def get_config_from_root(root): + """Read the project setup.cfg file to determine Versioneer config.""" + # This might raise EnvironmentError (if setup.cfg is missing), or + # configparser.NoSectionError (if it lacks a [versioneer] section), or + # configparser.NoOptionError (if it lacks "VCS="). See the docstring at + # the top of versioneer.py for instructions on writing your setup.cfg . + setup_cfg = os.path.join(root, "setup.cfg") + parser = configparser.SafeConfigParser() + with open(setup_cfg, "r") as f: + parser.readfp(f) + VCS = parser.get("versioneer", "VCS") # mandatory + + def get(parser, name): + if parser.has_option("versioneer", name): + return parser.get("versioneer", name) + return None + cfg = VersioneerConfig() + cfg.VCS = VCS + cfg.style = get(parser, "style") or "" + cfg.versionfile_source = get(parser, "versionfile_source") + cfg.versionfile_build = get(parser, "versionfile_build") + cfg.tag_prefix = get(parser, "tag_prefix") + if cfg.tag_prefix in ("''", '""'): + cfg.tag_prefix = "" + cfg.parentdir_prefix = get(parser, "parentdir_prefix") + cfg.verbose = get(parser, "verbose") + return cfg + + +class NotThisMethod(Exception): + """Exception raised if a method is not valid for the current scenario.""" + +# these dictionaries contain VCS-specific tools +LONG_VERSION_PY = {} +HANDLERS = {} + + +def register_vcs_handler(vcs, method): # decorator + """Decorator to mark a method as the handler for a particular VCS.""" + def decorate(f): + """Store f in HANDLERS[vcs][method].""" + if vcs not in HANDLERS: + HANDLERS[vcs] = {} + HANDLERS[vcs][method] = f + return f + return decorate + + +def run_command(commands, args, cwd=None, verbose=False, hide_stderr=False): + """Call the given command(s).""" + assert isinstance(commands, list) + p = None + for c in commands: + try: + dispcmd = str([c] + args) + # remember shell=False, so use git.cmd on windows, not just git + p = subprocess.Popen([c] + args, cwd=cwd, stdout=subprocess.PIPE, + stderr=(subprocess.PIPE if hide_stderr + else None)) + break + except EnvironmentError: + e = sys.exc_info()[1] + if e.errno == errno.ENOENT: + continue + if verbose: + print("unable to run %s" % dispcmd) + print(e) + return None + else: + if verbose: + print("unable to find command, tried %s" % (commands,)) + return None + stdout = p.communicate()[0].strip() + if sys.version_info[0] >= 3: + stdout = stdout.decode() + if p.returncode != 0: + if verbose: + print("unable to run %s (error)" % dispcmd) + return None + return stdout +LONG_VERSION_PY['git'] = ''' +# This file helps to compute a version number in source trees obtained from +# git-archive tarball (such as those provided by githubs download-from-tag +# feature). Distribution tarballs (built by setup.py sdist) and build +# directories (produced by setup.py build) will contain a much shorter file +# that just contains the computed version number. + +# This file is released into the public domain. Generated by +# versioneer-0.16 (https://github.com/warner/python-versioneer) + +"""Git implementation of _version.py.""" + +import errno +import os +import re +import subprocess +import sys + + +def get_keywords(): + """Get the keywords needed to look up the version information.""" + # these strings will be replaced by git during git-archive. + # setup.py/versioneer.py will grep for the variable names, so they must + # each be defined on a line of their own. _version.py will just call + # get_keywords(). + git_refnames = "%(DOLLAR)sFormat:%%d%(DOLLAR)s" + git_full = "%(DOLLAR)sFormat:%%H%(DOLLAR)s" + keywords = {"refnames": git_refnames, "full": git_full} + return keywords + + +class VersioneerConfig: + """Container for Versioneer configuration parameters.""" + + +def get_config(): + """Create, populate and return the VersioneerConfig() object.""" + # these strings are filled in when 'setup.py versioneer' creates + # _version.py + cfg = VersioneerConfig() + cfg.VCS = "git" + cfg.style = "%(STYLE)s" + cfg.tag_prefix = "%(TAG_PREFIX)s" + cfg.parentdir_prefix = "%(PARENTDIR_PREFIX)s" + cfg.versionfile_source = "%(VERSIONFILE_SOURCE)s" + cfg.verbose = False + return cfg + + +class NotThisMethod(Exception): + """Exception raised if a method is not valid for the current scenario.""" + + +LONG_VERSION_PY = {} +HANDLERS = {} + + +def register_vcs_handler(vcs, method): # decorator + """Decorator to mark a method as the handler for a particular VCS.""" + def decorate(f): + """Store f in HANDLERS[vcs][method].""" + if vcs not in HANDLERS: + HANDLERS[vcs] = {} + HANDLERS[vcs][method] = f + return f + return decorate + + +def run_command(commands, args, cwd=None, verbose=False, hide_stderr=False): + """Call the given command(s).""" + assert isinstance(commands, list) + p = None + for c in commands: + try: + dispcmd = str([c] + args) + # remember shell=False, so use git.cmd on windows, not just git + p = subprocess.Popen([c] + args, cwd=cwd, stdout=subprocess.PIPE, + stderr=(subprocess.PIPE if hide_stderr + else None)) + break + except EnvironmentError: + e = sys.exc_info()[1] + if e.errno == errno.ENOENT: + continue + if verbose: + print("unable to run %%s" %% dispcmd) + print(e) + return None + else: + if verbose: + print("unable to find command, tried %%s" %% (commands,)) + return None + stdout = p.communicate()[0].strip() + if sys.version_info[0] >= 3: + stdout = stdout.decode() + if p.returncode != 0: + if verbose: + print("unable to run %%s (error)" %% dispcmd) + return None + return stdout + + +def versions_from_parentdir(parentdir_prefix, root, verbose): + """Try to determine the version from the parent directory name. + + Source tarballs conventionally unpack into a directory that includes + both the project name and a version string. + """ + dirname = os.path.basename(root) + if not dirname.startswith(parentdir_prefix): + if verbose: + print("guessing rootdir is '%%s', but '%%s' doesn't start with " + "prefix '%%s'" %% (root, dirname, parentdir_prefix)) + raise NotThisMethod("rootdir doesn't start with parentdir_prefix") + return {"version": dirname[len(parentdir_prefix):], + "full-revisionid": None, + "dirty": False, "error": None} + + +@register_vcs_handler("git", "get_keywords") +def git_get_keywords(versionfile_abs): + """Extract version information from the given file.""" + # the code embedded in _version.py can just fetch the value of these + # keywords. When used from setup.py, we don't want to import _version.py, + # so we do it with a regexp instead. This function is not used from + # _version.py. + keywords = {} + try: + f = open(versionfile_abs, "r") + for line in f.readlines(): + if line.strip().startswith("git_refnames ="): + mo = re.search(r'=\s*"(.*)"', line) + if mo: + keywords["refnames"] = mo.group(1) + if line.strip().startswith("git_full ="): + mo = re.search(r'=\s*"(.*)"', line) + if mo: + keywords["full"] = mo.group(1) + f.close() + except EnvironmentError: + pass + return keywords + + +@register_vcs_handler("git", "keywords") +def git_versions_from_keywords(keywords, tag_prefix, verbose): + """Get version information from git keywords.""" + if not keywords: + raise NotThisMethod("no keywords at all, weird") + refnames = keywords["refnames"].strip() + if refnames.startswith("$Format"): + if verbose: + print("keywords are unexpanded, not using") + raise NotThisMethod("unexpanded keywords, not a git-archive tarball") + refs = set([r.strip() for r in refnames.strip("()").split(",")]) + # starting in git-1.8.3, tags are listed as "tag: foo-1.0" instead of + # just "foo-1.0". If we see a "tag: " prefix, prefer those. + TAG = "tag: " + tags = set([r[len(TAG):] for r in refs if r.startswith(TAG)]) + if not tags: + # Either we're using git < 1.8.3, or there really are no tags. We use + # a heuristic: assume all version tags have a digit. The old git %%d + # expansion behaves like git log --decorate=short and strips out the + # refs/heads/ and refs/tags/ prefixes that would let us distinguish + # between branches and tags. By ignoring refnames without digits, we + # filter out many common branch names like "release" and + # "stabilization", as well as "HEAD" and "master". + tags = set([r for r in refs if re.search(r'\d', r)]) + if verbose: + print("discarding '%%s', no digits" %% ",".join(refs-tags)) + if verbose: + print("likely tags: %%s" %% ",".join(sorted(tags))) + for ref in sorted(tags): + # sorting will prefer e.g. "2.0" over "2.0rc1" + if ref.startswith(tag_prefix): + r = ref[len(tag_prefix):] + if verbose: + print("picking %%s" %% r) + return {"version": r, + "full-revisionid": keywords["full"].strip(), + "dirty": False, "error": None + } + # no suitable tags, so version is "0+unknown", but full hex is still there + if verbose: + print("no suitable tags, using unknown + full revision id") + return {"version": "0+unknown", + "full-revisionid": keywords["full"].strip(), + "dirty": False, "error": "no suitable tags"} + + +@register_vcs_handler("git", "pieces_from_vcs") +def git_pieces_from_vcs(tag_prefix, root, verbose, run_command=run_command): + """Get version from 'git describe' in the root of the source tree. + + This only gets called if the git-archive 'subst' keywords were *not* + expanded, and _version.py hasn't already been rewritten with a short + version string, meaning we're inside a checked out source tree. + """ + if not os.path.exists(os.path.join(root, ".git")): + if verbose: + print("no .git in %%s" %% root) + raise NotThisMethod("no .git directory") + + GITS = ["git"] + if sys.platform == "win32": + GITS = ["git.cmd", "git.exe"] + # if there is a tag matching tag_prefix, this yields TAG-NUM-gHEX[-dirty] + # if there isn't one, this yields HEX[-dirty] (no NUM) + describe_out = run_command(GITS, ["describe", "--tags", "--dirty", + "--always", "--long", + "--match", "%%s*" %% tag_prefix], + cwd=root) + # --long was added in git-1.5.5 + if describe_out is None: + raise NotThisMethod("'git describe' failed") + describe_out = describe_out.strip() + full_out = run_command(GITS, ["rev-parse", "HEAD"], cwd=root) + if full_out is None: + raise NotThisMethod("'git rev-parse' failed") + full_out = full_out.strip() + + pieces = {} + pieces["long"] = full_out + pieces["short"] = full_out[:7] # maybe improved later + pieces["error"] = None + + # parse describe_out. It will be like TAG-NUM-gHEX[-dirty] or HEX[-dirty] + # TAG might have hyphens. + git_describe = describe_out + + # look for -dirty suffix + dirty = git_describe.endswith("-dirty") + pieces["dirty"] = dirty + if dirty: + git_describe = git_describe[:git_describe.rindex("-dirty")] + + # now we have TAG-NUM-gHEX or HEX + + if "-" in git_describe: + # TAG-NUM-gHEX + mo = re.search(r'^(.+)-(\d+)-g([0-9a-f]+)$', git_describe) + if not mo: + # unparseable. Maybe git-describe is misbehaving? + pieces["error"] = ("unable to parse git-describe output: '%%s'" + %% describe_out) + return pieces + + # tag + full_tag = mo.group(1) + if not full_tag.startswith(tag_prefix): + if verbose: + fmt = "tag '%%s' doesn't start with prefix '%%s'" + print(fmt %% (full_tag, tag_prefix)) + pieces["error"] = ("tag '%%s' doesn't start with prefix '%%s'" + %% (full_tag, tag_prefix)) + return pieces + pieces["closest-tag"] = full_tag[len(tag_prefix):] + + # distance: number of commits since tag + pieces["distance"] = int(mo.group(2)) + + # commit: short hex revision ID + pieces["short"] = mo.group(3) + + else: + # HEX: no tags + pieces["closest-tag"] = None + count_out = run_command(GITS, ["rev-list", "HEAD", "--count"], + cwd=root) + pieces["distance"] = int(count_out) # total number of commits + + return pieces + + +def plus_or_dot(pieces): + """Return a + if we don't already have one, else return a .""" + if "+" in pieces.get("closest-tag", ""): + return "." + return "+" + + +def render_pep440(pieces): + """Build up version string, with post-release "local version identifier". + + Our goal: TAG[+DISTANCE.gHEX[.dirty]] . Note that if you + get a tagged build and then dirty it, you'll get TAG+0.gHEX.dirty + + Exceptions: + 1: no tags. git_describe was just HEX. 0+untagged.DISTANCE.gHEX[.dirty] + """ + if pieces["closest-tag"]: + rendered = pieces["closest-tag"] + if pieces["distance"] or pieces["dirty"]: + rendered += plus_or_dot(pieces) + rendered += "%%d.g%%s" %% (pieces["distance"], pieces["short"]) + if pieces["dirty"]: + rendered += ".dirty" + else: + # exception #1 + rendered = "0+untagged.%%d.g%%s" %% (pieces["distance"], + pieces["short"]) + if pieces["dirty"]: + rendered += ".dirty" + return rendered + + +def render_pep440_pre(pieces): + """TAG[.post.devDISTANCE] -- No -dirty. + + Exceptions: + 1: no tags. 0.post.devDISTANCE + """ + if pieces["closest-tag"]: + rendered = pieces["closest-tag"] + if pieces["distance"]: + rendered += ".post.dev%%d" %% pieces["distance"] + else: + # exception #1 + rendered = "0.post.dev%%d" %% pieces["distance"] + return rendered + + +def render_pep440_post(pieces): + """TAG[.postDISTANCE[.dev0]+gHEX] . + + The ".dev0" means dirty. Note that .dev0 sorts backwards + (a dirty tree will appear "older" than the corresponding clean one), + but you shouldn't be releasing software with -dirty anyways. + + Exceptions: + 1: no tags. 0.postDISTANCE[.dev0] + """ + if pieces["closest-tag"]: + rendered = pieces["closest-tag"] + if pieces["distance"] or pieces["dirty"]: + rendered += ".post%%d" %% pieces["distance"] + if pieces["dirty"]: + rendered += ".dev0" + rendered += plus_or_dot(pieces) + rendered += "g%%s" %% pieces["short"] + else: + # exception #1 + rendered = "0.post%%d" %% pieces["distance"] + if pieces["dirty"]: + rendered += ".dev0" + rendered += "+g%%s" %% pieces["short"] + return rendered + + +def render_pep440_old(pieces): + """TAG[.postDISTANCE[.dev0]] . + + The ".dev0" means dirty. + + Eexceptions: + 1: no tags. 0.postDISTANCE[.dev0] + """ + if pieces["closest-tag"]: + rendered = pieces["closest-tag"] + if pieces["distance"] or pieces["dirty"]: + rendered += ".post%%d" %% pieces["distance"] + if pieces["dirty"]: + rendered += ".dev0" + else: + # exception #1 + rendered = "0.post%%d" %% pieces["distance"] + if pieces["dirty"]: + rendered += ".dev0" + return rendered + + +def render_git_describe(pieces): + """TAG[-DISTANCE-gHEX][-dirty]. + + Like 'git describe --tags --dirty --always'. + + Exceptions: + 1: no tags. HEX[-dirty] (note: no 'g' prefix) + """ + if pieces["closest-tag"]: + rendered = pieces["closest-tag"] + if pieces["distance"]: + rendered += "-%%d-g%%s" %% (pieces["distance"], pieces["short"]) + else: + # exception #1 + rendered = pieces["short"] + if pieces["dirty"]: + rendered += "-dirty" + return rendered + + +def render_git_describe_long(pieces): + """TAG-DISTANCE-gHEX[-dirty]. + + Like 'git describe --tags --dirty --always -long'. + The distance/hash is unconditional. + + Exceptions: + 1: no tags. HEX[-dirty] (note: no 'g' prefix) + """ + if pieces["closest-tag"]: + rendered = pieces["closest-tag"] + rendered += "-%%d-g%%s" %% (pieces["distance"], pieces["short"]) + else: + # exception #1 + rendered = pieces["short"] + if pieces["dirty"]: + rendered += "-dirty" + return rendered + + +def render(pieces, style): + """Render the given version pieces into the requested style.""" + if pieces["error"]: + return {"version": "unknown", + "full-revisionid": pieces.get("long"), + "dirty": None, + "error": pieces["error"]} + + if not style or style == "default": + style = "pep440" # the default + + if style == "pep440": + rendered = render_pep440(pieces) + elif style == "pep440-pre": + rendered = render_pep440_pre(pieces) + elif style == "pep440-post": + rendered = render_pep440_post(pieces) + elif style == "pep440-old": + rendered = render_pep440_old(pieces) + elif style == "git-describe": + rendered = render_git_describe(pieces) + elif style == "git-describe-long": + rendered = render_git_describe_long(pieces) + else: + raise ValueError("unknown style '%%s'" %% style) + + return {"version": rendered, "full-revisionid": pieces["long"], + "dirty": pieces["dirty"], "error": None} + + +def get_versions(): + """Get version information or return default if unable to do so.""" + # I am in _version.py, which lives at ROOT/VERSIONFILE_SOURCE. If we have + # __file__, we can work backwards from there to the root. Some + # py2exe/bbfreeze/non-CPython implementations don't do __file__, in which + # case we can only use expanded keywords. + + cfg = get_config() + verbose = cfg.verbose + + try: + return git_versions_from_keywords(get_keywords(), cfg.tag_prefix, + verbose) + except NotThisMethod: + pass + + try: + root = os.path.realpath(__file__) + # versionfile_source is the relative path from the top of the source + # tree (where the .git directory might live) to this file. Invert + # this to find the root from __file__. + for i in cfg.versionfile_source.split('/'): + root = os.path.dirname(root) + except NameError: + return {"version": "0+unknown", "full-revisionid": None, + "dirty": None, + "error": "unable to find root of source tree"} + + try: + pieces = git_pieces_from_vcs(cfg.tag_prefix, root, verbose) + return render(pieces, cfg.style) + except NotThisMethod: + pass + + try: + if cfg.parentdir_prefix: + return versions_from_parentdir(cfg.parentdir_prefix, root, verbose) + except NotThisMethod: + pass + + return {"version": "0+unknown", "full-revisionid": None, + "dirty": None, + "error": "unable to compute version"} +''' + + +@register_vcs_handler("git", "get_keywords") +def git_get_keywords(versionfile_abs): + """Extract version information from the given file.""" + # the code embedded in _version.py can just fetch the value of these + # keywords. When used from setup.py, we don't want to import _version.py, + # so we do it with a regexp instead. This function is not used from + # _version.py. + keywords = {} + try: + f = open(versionfile_abs, "r") + for line in f.readlines(): + if line.strip().startswith("git_refnames ="): + mo = re.search(r'=\s*"(.*)"', line) + if mo: + keywords["refnames"] = mo.group(1) + if line.strip().startswith("git_full ="): + mo = re.search(r'=\s*"(.*)"', line) + if mo: + keywords["full"] = mo.group(1) + f.close() + except EnvironmentError: + pass + return keywords + + +@register_vcs_handler("git", "keywords") +def git_versions_from_keywords(keywords, tag_prefix, verbose): + """Get version information from git keywords.""" + if not keywords: + raise NotThisMethod("no keywords at all, weird") + refnames = keywords["refnames"].strip() + if refnames.startswith("$Format"): + if verbose: + print("keywords are unexpanded, not using") + raise NotThisMethod("unexpanded keywords, not a git-archive tarball") + refs = set([r.strip() for r in refnames.strip("()").split(",")]) + # starting in git-1.8.3, tags are listed as "tag: foo-1.0" instead of + # just "foo-1.0". If we see a "tag: " prefix, prefer those. + TAG = "tag: " + tags = set([r[len(TAG):] for r in refs if r.startswith(TAG)]) + if not tags: + # Either we're using git < 1.8.3, or there really are no tags. We use + # a heuristic: assume all version tags have a digit. The old git %d + # expansion behaves like git log --decorate=short and strips out the + # refs/heads/ and refs/tags/ prefixes that would let us distinguish + # between branches and tags. By ignoring refnames without digits, we + # filter out many common branch names like "release" and + # "stabilization", as well as "HEAD" and "master". + tags = set([r for r in refs if re.search(r'\d', r)]) + if verbose: + print("discarding '%s', no digits" % ",".join(refs-tags)) + if verbose: + print("likely tags: %s" % ",".join(sorted(tags))) + for ref in sorted(tags): + # sorting will prefer e.g. "2.0" over "2.0rc1" + if ref.startswith(tag_prefix): + r = ref[len(tag_prefix):] + if verbose: + print("picking %s" % r) + return {"version": r, + "full-revisionid": keywords["full"].strip(), + "dirty": False, "error": None + } + # no suitable tags, so version is "0+unknown", but full hex is still there + if verbose: + print("no suitable tags, using unknown + full revision id") + return {"version": "0+unknown", + "full-revisionid": keywords["full"].strip(), + "dirty": False, "error": "no suitable tags"} + + +@register_vcs_handler("git", "pieces_from_vcs") +def git_pieces_from_vcs(tag_prefix, root, verbose, run_command=run_command): + """Get version from 'git describe' in the root of the source tree. + + This only gets called if the git-archive 'subst' keywords were *not* + expanded, and _version.py hasn't already been rewritten with a short + version string, meaning we're inside a checked out source tree. + """ + if not os.path.exists(os.path.join(root, ".git")): + if verbose: + print("no .git in %s" % root) + raise NotThisMethod("no .git directory") + + GITS = ["git"] + if sys.platform == "win32": + GITS = ["git.cmd", "git.exe"] + # if there is a tag matching tag_prefix, this yields TAG-NUM-gHEX[-dirty] + # if there isn't one, this yields HEX[-dirty] (no NUM) + describe_out = run_command(GITS, ["describe", "--tags", "--dirty", + "--always", "--long", + "--match", "%s*" % tag_prefix], + cwd=root) + # --long was added in git-1.5.5 + if describe_out is None: + raise NotThisMethod("'git describe' failed") + describe_out = describe_out.strip() + full_out = run_command(GITS, ["rev-parse", "HEAD"], cwd=root) + if full_out is None: + raise NotThisMethod("'git rev-parse' failed") + full_out = full_out.strip() + + pieces = {} + pieces["long"] = full_out + pieces["short"] = full_out[:7] # maybe improved later + pieces["error"] = None + + # parse describe_out. It will be like TAG-NUM-gHEX[-dirty] or HEX[-dirty] + # TAG might have hyphens. + git_describe = describe_out + + # look for -dirty suffix + dirty = git_describe.endswith("-dirty") + pieces["dirty"] = dirty + if dirty: + git_describe = git_describe[:git_describe.rindex("-dirty")] + + # now we have TAG-NUM-gHEX or HEX + + if "-" in git_describe: + # TAG-NUM-gHEX + mo = re.search(r'^(.+)-(\d+)-g([0-9a-f]+)$', git_describe) + if not mo: + # unparseable. Maybe git-describe is misbehaving? + pieces["error"] = ("unable to parse git-describe output: '%s'" + % describe_out) + return pieces + + # tag + full_tag = mo.group(1) + if not full_tag.startswith(tag_prefix): + if verbose: + fmt = "tag '%s' doesn't start with prefix '%s'" + print(fmt % (full_tag, tag_prefix)) + pieces["error"] = ("tag '%s' doesn't start with prefix '%s'" + % (full_tag, tag_prefix)) + return pieces + pieces["closest-tag"] = full_tag[len(tag_prefix):] + + # distance: number of commits since tag + pieces["distance"] = int(mo.group(2)) + + # commit: short hex revision ID + pieces["short"] = mo.group(3) + + else: + # HEX: no tags + pieces["closest-tag"] = None + count_out = run_command(GITS, ["rev-list", "HEAD", "--count"], + cwd=root) + pieces["distance"] = int(count_out) # total number of commits + + return pieces + + +def do_vcs_install(manifest_in, versionfile_source, ipy): + """Git-specific installation logic for Versioneer. + + For Git, this means creating/changing .gitattributes to mark _version.py + for export-time keyword substitution. + """ + GITS = ["git"] + if sys.platform == "win32": + GITS = ["git.cmd", "git.exe"] + files = [manifest_in, versionfile_source] + if ipy: + files.append(ipy) + try: + me = __file__ + if me.endswith(".pyc") or me.endswith(".pyo"): + me = os.path.splitext(me)[0] + ".py" + versioneer_file = os.path.relpath(me) + except NameError: + versioneer_file = "versioneer.py" + files.append(versioneer_file) + present = False + try: + f = open(".gitattributes", "r") + for line in f.readlines(): + if line.strip().startswith(versionfile_source): + if "export-subst" in line.strip().split()[1:]: + present = True + f.close() + except EnvironmentError: + pass + if not present: + f = open(".gitattributes", "a+") + f.write("%s export-subst\n" % versionfile_source) + f.close() + files.append(".gitattributes") + run_command(GITS, ["add", "--"] + files) + + +def versions_from_parentdir(parentdir_prefix, root, verbose): + """Try to determine the version from the parent directory name. + + Source tarballs conventionally unpack into a directory that includes + both the project name and a version string. + """ + dirname = os.path.basename(root) + if not dirname.startswith(parentdir_prefix): + if verbose: + print("guessing rootdir is '%s', but '%s' doesn't start with " + "prefix '%s'" % (root, dirname, parentdir_prefix)) + raise NotThisMethod("rootdir doesn't start with parentdir_prefix") + return {"version": dirname[len(parentdir_prefix):], + "full-revisionid": None, + "dirty": False, "error": None} + +SHORT_VERSION_PY = """ +# This file was generated by 'versioneer.py' (0.16) from +# revision-control system data, or from the parent directory name of an +# unpacked source archive. Distribution tarballs contain a pre-generated copy +# of this file. + +import json +import sys + +version_json = ''' +%s +''' # END VERSION_JSON + + +def get_versions(): + return json.loads(version_json) +""" + + +def versions_from_file(filename): + """Try to determine the version from _version.py if present.""" + try: + with open(filename) as f: + contents = f.read() + except EnvironmentError: + raise NotThisMethod("unable to read _version.py") + mo = re.search(r"version_json = '''\n(.*)''' # END VERSION_JSON", + contents, re.M | re.S) + if not mo: + raise NotThisMethod("no version_json in _version.py") + return json.loads(mo.group(1)) + + +def write_to_version_file(filename, versions): + """Write the given version number to the given _version.py file.""" + os.unlink(filename) + contents = json.dumps(versions, sort_keys=True, + indent=1, separators=(",", ": ")) + with open(filename, "w") as f: + f.write(SHORT_VERSION_PY % contents) + + print("set %s to '%s'" % (filename, versions["version"])) + + +def plus_or_dot(pieces): + """Return a + if we don't already have one, else return a .""" + if "+" in pieces.get("closest-tag", ""): + return "." + return "+" + + +def render_pep440(pieces): + """Build up version string, with post-release "local version identifier". + + Our goal: TAG[+DISTANCE.gHEX[.dirty]] . Note that if you + get a tagged build and then dirty it, you'll get TAG+0.gHEX.dirty + + Exceptions: + 1: no tags. git_describe was just HEX. 0+untagged.DISTANCE.gHEX[.dirty] + """ + if pieces["closest-tag"]: + rendered = pieces["closest-tag"] + if pieces["distance"] or pieces["dirty"]: + rendered += plus_or_dot(pieces) + rendered += "%d.g%s" % (pieces["distance"], pieces["short"]) + if pieces["dirty"]: + rendered += ".dirty" + else: + # exception #1 + rendered = "0+untagged.%d.g%s" % (pieces["distance"], + pieces["short"]) + if pieces["dirty"]: + rendered += ".dirty" + return rendered + + +def render_pep440_pre(pieces): + """TAG[.post.devDISTANCE] -- No -dirty. + + Exceptions: + 1: no tags. 0.post.devDISTANCE + """ + if pieces["closest-tag"]: + rendered = pieces["closest-tag"] + if pieces["distance"]: + rendered += ".post.dev%d" % pieces["distance"] + else: + # exception #1 + rendered = "0.post.dev%d" % pieces["distance"] + return rendered + + +def render_pep440_post(pieces): + """TAG[.postDISTANCE[.dev0]+gHEX] . + + The ".dev0" means dirty. Note that .dev0 sorts backwards + (a dirty tree will appear "older" than the corresponding clean one), + but you shouldn't be releasing software with -dirty anyways. + + Exceptions: + 1: no tags. 0.postDISTANCE[.dev0] + """ + if pieces["closest-tag"]: + rendered = pieces["closest-tag"] + if pieces["distance"] or pieces["dirty"]: + rendered += ".post%d" % pieces["distance"] + if pieces["dirty"]: + rendered += ".dev0" + rendered += plus_or_dot(pieces) + rendered += "g%s" % pieces["short"] + else: + # exception #1 + rendered = "0.post%d" % pieces["distance"] + if pieces["dirty"]: + rendered += ".dev0" + rendered += "+g%s" % pieces["short"] + return rendered + + +def render_pep440_old(pieces): + """TAG[.postDISTANCE[.dev0]] . + + The ".dev0" means dirty. + + Eexceptions: + 1: no tags. 0.postDISTANCE[.dev0] + """ + if pieces["closest-tag"]: + rendered = pieces["closest-tag"] + if pieces["distance"] or pieces["dirty"]: + rendered += ".post%d" % pieces["distance"] + if pieces["dirty"]: + rendered += ".dev0" + else: + # exception #1 + rendered = "0.post%d" % pieces["distance"] + if pieces["dirty"]: + rendered += ".dev0" + return rendered + + +def render_git_describe(pieces): + """TAG[-DISTANCE-gHEX][-dirty]. + + Like 'git describe --tags --dirty --always'. + + Exceptions: + 1: no tags. HEX[-dirty] (note: no 'g' prefix) + """ + if pieces["closest-tag"]: + rendered = pieces["closest-tag"] + if pieces["distance"]: + rendered += "-%d-g%s" % (pieces["distance"], pieces["short"]) + else: + # exception #1 + rendered = pieces["short"] + if pieces["dirty"]: + rendered += "-dirty" + return rendered + + +def render_git_describe_long(pieces): + """TAG-DISTANCE-gHEX[-dirty]. + + Like 'git describe --tags --dirty --always -long'. + The distance/hash is unconditional. + + Exceptions: + 1: no tags. HEX[-dirty] (note: no 'g' prefix) + """ + if pieces["closest-tag"]: + rendered = pieces["closest-tag"] + rendered += "-%d-g%s" % (pieces["distance"], pieces["short"]) + else: + # exception #1 + rendered = pieces["short"] + if pieces["dirty"]: + rendered += "-dirty" + return rendered + + +def render(pieces, style): + """Render the given version pieces into the requested style.""" + if pieces["error"]: + return {"version": "unknown", + "full-revisionid": pieces.get("long"), + "dirty": None, + "error": pieces["error"]} + + if not style or style == "default": + style = "pep440" # the default + + if style == "pep440": + rendered = render_pep440(pieces) + elif style == "pep440-pre": + rendered = render_pep440_pre(pieces) + elif style == "pep440-post": + rendered = render_pep440_post(pieces) + elif style == "pep440-old": + rendered = render_pep440_old(pieces) + elif style == "git-describe": + rendered = render_git_describe(pieces) + elif style == "git-describe-long": + rendered = render_git_describe_long(pieces) + else: + raise ValueError("unknown style '%s'" % style) + + return {"version": rendered, "full-revisionid": pieces["long"], + "dirty": pieces["dirty"], "error": None} + + +class VersioneerBadRootError(Exception): + """The project root directory is unknown or missing key files.""" + + +def get_versions(verbose=False): + """Get the project version from whatever source is available. + + Returns dict with two keys: 'version' and 'full'. + """ + if "versioneer" in sys.modules: + # see the discussion in cmdclass.py:get_cmdclass() + del sys.modules["versioneer"] + + root = get_root() + cfg = get_config_from_root(root) + + assert cfg.VCS is not None, "please set [versioneer]VCS= in setup.cfg" + handlers = HANDLERS.get(cfg.VCS) + assert handlers, "unrecognized VCS '%s'" % cfg.VCS + verbose = verbose or cfg.verbose + assert cfg.versionfile_source is not None, \ + "please set versioneer.versionfile_source" + assert cfg.tag_prefix is not None, "please set versioneer.tag_prefix" + + versionfile_abs = os.path.join(root, cfg.versionfile_source) + + # extract version from first of: _version.py, VCS command (e.g. 'git + # describe'), parentdir. This is meant to work for developers using a + # source checkout, for users of a tarball created by 'setup.py sdist', + # and for users of a tarball/zipball created by 'git archive' or github's + # download-from-tag feature or the equivalent in other VCSes. + + get_keywords_f = handlers.get("get_keywords") + from_keywords_f = handlers.get("keywords") + if get_keywords_f and from_keywords_f: + try: + keywords = get_keywords_f(versionfile_abs) + ver = from_keywords_f(keywords, cfg.tag_prefix, verbose) + if verbose: + print("got version from expanded keyword %s" % ver) + return ver + except NotThisMethod: + pass + + try: + ver = versions_from_file(versionfile_abs) + if verbose: + print("got version from file %s %s" % (versionfile_abs, ver)) + return ver + except NotThisMethod: + pass + + from_vcs_f = handlers.get("pieces_from_vcs") + if from_vcs_f: + try: + pieces = from_vcs_f(cfg.tag_prefix, root, verbose) + ver = render(pieces, cfg.style) + if verbose: + print("got version from VCS %s" % ver) + return ver + except NotThisMethod: + pass + + try: + if cfg.parentdir_prefix: + ver = versions_from_parentdir(cfg.parentdir_prefix, root, verbose) + if verbose: + print("got version from parentdir %s" % ver) + return ver + except NotThisMethod: + pass + + if verbose: + print("unable to compute version") + + return {"version": "0+unknown", "full-revisionid": None, + "dirty": None, "error": "unable to compute version"} + + +def get_version(): + """Get the short version string for this project.""" + return get_versions()["version"] + + +def get_cmdclass(): + """Get the custom setuptools/distutils subclasses used by Versioneer.""" + if "versioneer" in sys.modules: + del sys.modules["versioneer"] + # this fixes the "python setup.py develop" case (also 'install' and + # 'easy_install .'), in which subdependencies of the main project are + # built (using setup.py bdist_egg) in the same python process. Assume + # a main project A and a dependency B, which use different versions + # of Versioneer. A's setup.py imports A's Versioneer, leaving it in + # sys.modules by the time B's setup.py is executed, causing B to run + # with the wrong versioneer. Setuptools wraps the sub-dep builds in a + # sandbox that restores sys.modules to it's pre-build state, so the + # parent is protected against the child's "import versioneer". By + # removing ourselves from sys.modules here, before the child build + # happens, we protect the child from the parent's versioneer too. + # Also see https://github.com/warner/python-versioneer/issues/52 + + cmds = {} + + # we add "version" to both distutils and setuptools + from distutils.core import Command + + class cmd_version(Command): + description = "report generated version string" + user_options = [] + boolean_options = [] + + def initialize_options(self): + pass + + def finalize_options(self): + pass + + def run(self): + vers = get_versions(verbose=True) + print("Version: %s" % vers["version"]) + print(" full-revisionid: %s" % vers.get("full-revisionid")) + print(" dirty: %s" % vers.get("dirty")) + if vers["error"]: + print(" error: %s" % vers["error"]) + cmds["version"] = cmd_version + + # we override "build_py" in both distutils and setuptools + # + # most invocation pathways end up running build_py: + # distutils/build -> build_py + # distutils/install -> distutils/build ->.. + # setuptools/bdist_wheel -> distutils/install ->.. + # setuptools/bdist_egg -> distutils/install_lib -> build_py + # setuptools/install -> bdist_egg ->.. + # setuptools/develop -> ? + + # we override different "build_py" commands for both environments + if "setuptools" in sys.modules: + from setuptools.command.build_py import build_py as _build_py + else: + from distutils.command.build_py import build_py as _build_py + + class cmd_build_py(_build_py): + def run(self): + root = get_root() + cfg = get_config_from_root(root) + versions = get_versions() + _build_py.run(self) + # now locate _version.py in the new build/ directory and replace + # it with an updated value + if cfg.versionfile_build: + target_versionfile = os.path.join(self.build_lib, + cfg.versionfile_build) + print("UPDATING %s" % target_versionfile) + write_to_version_file(target_versionfile, versions) + cmds["build_py"] = cmd_build_py + + if "cx_Freeze" in sys.modules: # cx_freeze enabled? + from cx_Freeze.dist import build_exe as _build_exe + + class cmd_build_exe(_build_exe): + def run(self): + root = get_root() + cfg = get_config_from_root(root) + versions = get_versions() + target_versionfile = cfg.versionfile_source + print("UPDATING %s" % target_versionfile) + write_to_version_file(target_versionfile, versions) + + _build_exe.run(self) + os.unlink(target_versionfile) + with open(cfg.versionfile_source, "w") as f: + LONG = LONG_VERSION_PY[cfg.VCS] + f.write(LONG % + {"DOLLAR": "$", + "STYLE": cfg.style, + "TAG_PREFIX": cfg.tag_prefix, + "PARENTDIR_PREFIX": cfg.parentdir_prefix, + "VERSIONFILE_SOURCE": cfg.versionfile_source, + }) + cmds["build_exe"] = cmd_build_exe + del cmds["build_py"] + + # we override different "sdist" commands for both environments + if "setuptools" in sys.modules: + from setuptools.command.sdist import sdist as _sdist + else: + from distutils.command.sdist import sdist as _sdist + + class cmd_sdist(_sdist): + def run(self): + versions = get_versions() + self._versioneer_generated_versions = versions + # unless we update this, the command will keep using the old + # version + self.distribution.metadata.version = versions["version"] + return _sdist.run(self) + + def make_release_tree(self, base_dir, files): + root = get_root() + cfg = get_config_from_root(root) + _sdist.make_release_tree(self, base_dir, files) + # now locate _version.py in the new base_dir directory + # (remembering that it may be a hardlink) and replace it with an + # updated value + target_versionfile = os.path.join(base_dir, cfg.versionfile_source) + print("UPDATING %s" % target_versionfile) + write_to_version_file(target_versionfile, + self._versioneer_generated_versions) + cmds["sdist"] = cmd_sdist + + return cmds + + +CONFIG_ERROR = """ +setup.cfg is missing the necessary Versioneer configuration. You need +a section like: + + [versioneer] + VCS = git + style = pep440 + versionfile_source = src/myproject/_version.py + versionfile_build = myproject/_version.py + tag_prefix = + parentdir_prefix = myproject- + +You will also need to edit your setup.py to use the results: + + import versioneer + setup(version=versioneer.get_version(), + cmdclass=versioneer.get_cmdclass(), ...) + +Please read the docstring in ./versioneer.py for configuration instructions, +edit setup.cfg, and re-run the installer or 'python versioneer.py setup'. +""" + +SAMPLE_CONFIG = """ +# See the docstring in versioneer.py for instructions. Note that you must +# re-run 'versioneer.py setup' after changing this section, and commit the +# resulting files. + +[versioneer] +#VCS = git +#style = pep440 +#versionfile_source = +#versionfile_build = +#tag_prefix = +#parentdir_prefix = + +""" + +INIT_PY_SNIPPET = """ +from ._version import get_versions +__version__ = get_versions()['version'] +del get_versions +""" + + +def do_setup(): + """Main VCS-independent setup function for installing Versioneer.""" + root = get_root() + try: + cfg = get_config_from_root(root) + except (EnvironmentError, configparser.NoSectionError, + configparser.NoOptionError) as e: + if isinstance(e, (EnvironmentError, configparser.NoSectionError)): + print("Adding sample versioneer config to setup.cfg", + file=sys.stderr) + with open(os.path.join(root, "setup.cfg"), "a") as f: + f.write(SAMPLE_CONFIG) + print(CONFIG_ERROR, file=sys.stderr) + return 1 + + print(" creating %s" % cfg.versionfile_source) + with open(cfg.versionfile_source, "w") as f: + LONG = LONG_VERSION_PY[cfg.VCS] + f.write(LONG % {"DOLLAR": "$", + "STYLE": cfg.style, + "TAG_PREFIX": cfg.tag_prefix, + "PARENTDIR_PREFIX": cfg.parentdir_prefix, + "VERSIONFILE_SOURCE": cfg.versionfile_source, + }) + + ipy = os.path.join(os.path.dirname(cfg.versionfile_source), + "__init__.py") + if os.path.exists(ipy): + try: + with open(ipy, "r") as f: + old = f.read() + except EnvironmentError: + old = "" + if INIT_PY_SNIPPET not in old: + print(" appending to %s" % ipy) + with open(ipy, "a") as f: + f.write(INIT_PY_SNIPPET) + else: + print(" %s unmodified" % ipy) + else: + print(" %s doesn't exist, ok" % ipy) + ipy = None + + # Make sure both the top-level "versioneer.py" and versionfile_source + # (PKG/_version.py, used by runtime code) are in MANIFEST.in, so + # they'll be copied into source distributions. Pip won't be able to + # install the package without this. + manifest_in = os.path.join(root, "MANIFEST.in") + simple_includes = set() + try: + with open(manifest_in, "r") as f: + for line in f: + if line.startswith("include "): + for include in line.split()[1:]: + simple_includes.add(include) + except EnvironmentError: + pass + # That doesn't cover everything MANIFEST.in can do + # (http://docs.python.org/2/distutils/sourcedist.html#commands), so + # it might give some false negatives. Appending redundant 'include' + # lines is safe, though. + if "versioneer.py" not in simple_includes: + print(" appending 'versioneer.py' to MANIFEST.in") + with open(manifest_in, "a") as f: + f.write("include versioneer.py\n") + else: + print(" 'versioneer.py' already in MANIFEST.in") + if cfg.versionfile_source not in simple_includes: + print(" appending versionfile_source ('%s') to MANIFEST.in" % + cfg.versionfile_source) + with open(manifest_in, "a") as f: + f.write("include %s\n" % cfg.versionfile_source) + else: + print(" versionfile_source already in MANIFEST.in") + + # Make VCS-specific changes. For git, this means creating/changing + # .gitattributes to mark _version.py for export-time keyword + # substitution. + do_vcs_install(manifest_in, cfg.versionfile_source, ipy) + return 0 + + +def scan_setup_py(): + """Validate the contents of setup.py against Versioneer's expectations.""" + found = set() + setters = False + errors = 0 + with open("setup.py", "r") as f: + for line in f.readlines(): + if "import versioneer" in line: + found.add("import") + if "versioneer.get_cmdclass()" in line: + found.add("cmdclass") + if "versioneer.get_version()" in line: + found.add("get_version") + if "versioneer.VCS" in line: + setters = True + if "versioneer.versionfile_source" in line: + setters = True + if len(found) != 3: + print("") + print("Your setup.py appears to be missing some important items") + print("(but I might be wrong). Please make sure it has something") + print("roughly like the following:") + print("") + print(" import versioneer") + print(" setup( version=versioneer.get_version(),") + print(" cmdclass=versioneer.get_cmdclass(), ...)") + print("") + errors += 1 + if setters: + print("You should remove lines like 'versioneer.VCS = ' and") + print("'versioneer.versionfile_source = ' . This configuration") + print("now lives in setup.cfg, and should be removed from setup.py") + print("") + errors += 1 + return errors + +if __name__ == "__main__": + cmd = sys.argv[1] + if cmd == "setup": + errors = do_setup() + errors += scan_setup_py() + if errors: + sys.exit(1) |