From 0b48d0e739db82c28dc8a125c1ca85762dac0ba6 Mon Sep 17 00:00:00 2001 From: Kali Kaneko Date: Thu, 29 Oct 2015 11:02:09 -0400 Subject: [docs] add version badge for pypi --- README.rst | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/README.rst b/README.rst index 81b4cec..f201baa 100644 --- a/README.rst +++ b/README.rst @@ -2,8 +2,8 @@ leap.mail ========= Mail services for the LEAP Client. -.. image:: https://pypip.in/v/leap.mail/badge.png - :target: https://crate.io/packages/leap.mail +.. image:: https://badge.fury.io/py/leap.mail.svg + :target: http://badge.fury.io/py/leap.mail .. image:: https://readthedocs.org/projects/leapmail/badge/?version=latest :target: http://leapmail.readthedocs.org/en/latest/ -- cgit v1.2.3 From 569449e633abbe22b63b9c6b8c680119a3bdceda Mon Sep 17 00:00:00 2001 From: Kali Kaneko Date: Mon, 30 Nov 2015 15:37:41 -0400 Subject: [feat] make events multi-user aware - Resolves: #7656 - Releases: 0.4.1 --- changes/next-changelog.rst | 29 +++++++++++++++++++++++++++++ src/leap/mail/imap/server.py | 2 +- src/leap/mail/incoming/service.py | 16 +++++++++------- src/leap/mail/outgoing/service.py | 13 +++++++++---- src/leap/mail/smtp/gateway.py | 10 ++++++---- 5 files changed, 54 insertions(+), 16 deletions(-) create mode 100644 changes/next-changelog.rst diff --git a/changes/next-changelog.rst b/changes/next-changelog.rst new file mode 100644 index 0000000..e218bb9 --- /dev/null +++ b/changes/next-changelog.rst @@ -0,0 +1,29 @@ +0.4.1 - xxx ++++++++++++++++++++++++++++++++ + +Please add lines to this file, they will be moved to the CHANGELOG.rst during +the next release. + +There are two template lines for each category, use them as reference. + +I've added a new category `Misc` so we can track doc/style/packaging stuff. + +Features +~~~~~~~~ +- `#7656 `_: Emit multi-user aware events. +- `#1234 `_: Description of the new feature corresponding with issue #1234. +- New feature without related issue number. + +Bugfixes +~~~~~~~~ +- `#1235 `_: Description for the fixed stuff corresponding with issue #1235. +- Bugfix without related issue number. + +Misc +~~~~ +- `#1236 `_: Description of the new feature corresponding with issue #1236. +- Some change without issue number. + +Known Issues +~~~~~~~~~~~~ +- `#1236 `_: Description of the known issue corresponding with issue #1236. diff --git a/src/leap/mail/imap/server.py b/src/leap/mail/imap/server.py index 99e7174..0e5d011 100644 --- a/src/leap/mail/imap/server.py +++ b/src/leap/mail/imap/server.py @@ -224,7 +224,7 @@ class LEAPIMAPServer(imap4.IMAP4Server): # bad username, reject. raise cred.error.UnauthorizedLogin() # any dummy password is allowed so far. use realm instead! - emit_async(catalog.IMAP_CLIENT_LOGIN, "1") + emit_async(catalog.IMAP_CLIENT_LOGIN, username, "1") return imap4.IAccount, self.theAccount, lambda: None def do_FETCH(self, tag, messages, query, uid=0): diff --git a/src/leap/mail/incoming/service.py b/src/leap/mail/incoming/service.py index d8b91ba..3896c17 100644 --- a/src/leap/mail/incoming/service.py +++ b/src/leap/mail/incoming/service.py @@ -233,7 +233,7 @@ class IncomingMail(Service): failure.trap(InvalidAuthTokenError) # if the token is invalid, send an event so the GUI can # disable mail and show an error message. - emit_async(catalog.SOLEDAD_INVALID_AUTH_TOKEN) + emit_async(catalog.SOLEDAD_INVALID_AUTH_TOKEN, self._userid) log.msg('FETCH: syncing soledad...') d = self._soledad.sync() @@ -254,7 +254,7 @@ class IncomingMail(Service): num_mails = len(doclist) if doclist is not None else 0 if num_mails != 0: log.msg("there are %s mails" % (num_mails,)) - emit_async(catalog.MAIL_FETCHED_INCOMING, + emit_async(catalog.MAIL_FETCHED_INCOMING, self._userid, str(num_mails), str(fetched_ts)) return doclist @@ -262,7 +262,7 @@ class IncomingMail(Service): """ Sends unread event to ui. """ - emit_async(catalog.MAIL_UNREAD_MESSAGES, + emit_async(catalog.MAIL_UNREAD_MESSAGES, self._userid, str(self._inbox_collection.count_unseen())) # process incoming mail. @@ -286,7 +286,7 @@ class IncomingMail(Service): deferreds = [] for index, doc in enumerate(doclist): logger.debug("processing doc %d of %d" % (index + 1, num_mails)) - emit_async(catalog.MAIL_MSG_PROCESSING, + emit_async(catalog.MAIL_MSG_PROCESSING, self._userid, str(index), str(num_mails)) keys = doc.content.keys() @@ -336,7 +336,8 @@ class IncomingMail(Service): decrdata = "" success = False - emit_async(catalog.MAIL_MSG_DECRYPTED, "1" if success else "0") + emit_async(catalog.MAIL_MSG_DECRYPTED, self._userid, + "1" if success else "0") return self._process_decrypted_doc(doc, decrdata) d = self._keymanager.decrypt( @@ -743,10 +744,11 @@ class IncomingMail(Service): listener(result) def signal_deleted(doc_id): - emit_async(catalog.MAIL_MSG_DELETED_INCOMING) + emit_async(catalog.MAIL_MSG_DELETED_INCOMING, + self._userid) return doc_id - emit_async(catalog.MAIL_MSG_SAVED_LOCALLY) + emit_async(catalog.MAIL_MSG_SAVED_LOCALLY, self._userid) d = self._delete_incoming_message(doc) d.addCallback(signal_deleted) return d diff --git a/src/leap/mail/outgoing/service.py b/src/leap/mail/outgoing/service.py index 7cc5a24..3bd0ea2 100644 --- a/src/leap/mail/outgoing/service.py +++ b/src/leap/mail/outgoing/service.py @@ -135,7 +135,8 @@ class OutgoingMail: """ dest_addrstr = smtp_sender_result[1][0][0] log.msg('Message sent to %s' % dest_addrstr) - emit_async(catalog.SMTP_SEND_MESSAGE_SUCCESS, dest_addrstr) + emit_async(catalog.SMTP_SEND_MESSAGE_SUCCESS, + self._from_address, dest_addrstr) def sendError(self, failure): """ @@ -145,7 +146,8 @@ class OutgoingMail: :type e: anything """ # XXX: need to get the address from the exception to send signal - # emit_async(catalog.SMTP_SEND_MESSAGE_ERROR, self._user.dest.addrstr) + # emit_async(catalog.SMTP_SEND_MESSAGE_ERROR, self._from_address, + # self._user.dest.addrstr) err = failure.value log.err(err) raise err @@ -178,7 +180,8 @@ class OutgoingMail: requireAuthentication=False, requireTransportSecurity=True) factory.domain = __version__ - emit_async(catalog.SMTP_SEND_MESSAGE_START, recipient.dest.addrstr) + emit_async(catalog.SMTP_SEND_MESSAGE_START, + self._from_address, recipient.dest.addrstr) reactor.connectSSL( self._host, self._port, factory, contextFactory=SSLContextFactory(self._cert, self._key)) @@ -241,6 +244,7 @@ class OutgoingMail: def signal_encrypt_sign(newmsg): emit_async(catalog.SMTP_END_ENCRYPT_AND_SIGN, + self._from_address, "%s,%s" % (self._from_address, to_address)) return newmsg, recipient @@ -248,7 +252,7 @@ class OutgoingMail: failure.trap(KeyNotFound, KeyAddressMismatch) log.msg('Will send unencrypted message to %s.' % to_address) - emit_async(catalog.SMTP_START_SIGN, self._from_address) + emit_async(catalog.SMTP_START_SIGN, self._from_address, to_address) d = self._sign(message, from_address) d.addCallback(signal_sign) return d @@ -260,6 +264,7 @@ class OutgoingMail: log.msg("Will encrypt the message with %s and sign with %s." % (to_address, from_address)) emit_async(catalog.SMTP_START_ENCRYPT_AND_SIGN, + self._from_address, "%s,%s" % (self._from_address, to_address)) d = self._maybe_attach_key(origmsg, from_address, to_address) d.addCallback(maybe_encrypt_and_sign) diff --git a/src/leap/mail/smtp/gateway.py b/src/leap/mail/smtp/gateway.py index 45560bf..3657250 100644 --- a/src/leap/mail/smtp/gateway.py +++ b/src/leap/mail/smtp/gateway.py @@ -202,20 +202,21 @@ class SMTPDelivery(object): def found(_): log.msg("Accepting mail for %s..." % user.dest.addrstr) emit_async(catalog.SMTP_RECIPIENT_ACCEPTED_ENCRYPTED, - user.dest.addrstr) + self._userid, user.dest.addrstr) def not_found(failure): failure.trap(KeyNotFound) # if key was not found, check config to see if will send anyway if self._encrypted_only: - emit_async(catalog.SMTP_RECIPIENT_REJECTED, user.dest.addrstr) + emit_async(catalog.SMTP_RECIPIENT_REJECTED, self._userid, + user.dest.addrstr) raise smtp.SMTPBadRcpt(user.dest.addrstr) log.msg("Warning: will send an unencrypted message (because " "encrypted_only' is set to False).") emit_async( catalog.SMTP_RECIPIENT_ACCEPTED_UNENCRYPTED, - user.dest.addrstr) + self._userid, user.dest.addrstr) def encrypt_func(_): return lambda: EncryptedMessage(user, self._outgoing_mail) @@ -307,7 +308,8 @@ class EncryptedMessage(object): """ log.msg("Connection lost unexpectedly!") log.err() - emit_async(catalog.SMTP_CONNECTION_LOST, self._user.dest.addrstr) + emit_async(catalog.SMTP_CONNECTION_LOST, self._userid, + self._user.dest.addrstr) # unexpected loss of connection; don't save self._lines = [] -- cgit v1.2.3 From 71cf1c4a243181330c0e973b8f69f9143b3d18de Mon Sep 17 00:00:00 2001 From: Bruno Wagner Date: Wed, 4 Nov 2015 12:10:56 -0200 Subject: Fixed the get_body logic It won't break anymore if the body is None, but will return an empty body in that case --- src/leap/mail/adaptors/soledad.py | 15 ++++++++------- 1 file changed, 8 insertions(+), 7 deletions(-) diff --git a/src/leap/mail/adaptors/soledad.py b/src/leap/mail/adaptors/soledad.py index 8de83f7..7f2b1cf 100644 --- a/src/leap/mail/adaptors/soledad.py +++ b/src/leap/mail/adaptors/soledad.py @@ -687,13 +687,14 @@ class MessageWrapper(object): :rtype: deferred """ body_phash = self.hdoc.body - if not body_phash: - if self.cdocs: - return self.cdocs[1] - d = store.get_doc('C-' + body_phash) - d.addCallback(lambda doc: ContentDocWrapper(**doc.content)) - return d - + if body_phash: + d = store.get_doc('C-' + body_phash) + d.addCallback(lambda doc: ContentDocWrapper(**doc.content)) + return d + elif self.cdocs: + return self.cdocs[1] + else: + return '' # # Mailboxes -- cgit v1.2.3 From cbfc7cd2acb1ab1d58030ab9a5205a295be1ca5d Mon Sep 17 00:00:00 2001 From: Kali Kaneko Date: Mon, 14 Dec 2015 18:40:31 -0400 Subject: [docs] document bugfix on pr 215 by bwagner --- changes/next-changelog.rst | 1 + 1 file changed, 1 insertion(+) diff --git a/changes/next-changelog.rst b/changes/next-changelog.rst index e218bb9..7de9d98 100644 --- a/changes/next-changelog.rst +++ b/changes/next-changelog.rst @@ -17,6 +17,7 @@ Features Bugfixes ~~~~~~~~ - `#1235 `_: Description for the fixed stuff corresponding with issue #1235. +- Fix the get_body logic for corner-cases in which body is None (yet-to-be synced docs, mainly). - Bugfix without related issue number. Misc -- cgit v1.2.3 From 8ddddd77478984777be79e817458183a41f0a978 Mon Sep 17 00:00:00 2001 From: Kali Kaneko Date: Thu, 26 Nov 2015 21:27:23 -0400 Subject: [feat] credentials handling: use twisted.cred --- src/leap/mail/imap/account.py | 5 +- src/leap/mail/imap/server.py | 47 ------- src/leap/mail/imap/service/imap-server.tac | 9 +- src/leap/mail/imap/service/imap.py | 200 +++++++++++++++++++---------- src/leap/mail/imap/tests/utils.py | 5 +- src/leap/mail/outgoing/service.py | 2 +- src/leap/mail/smtp/gateway.py | 3 + 7 files changed, 150 insertions(+), 121 deletions(-) diff --git a/src/leap/mail/imap/account.py b/src/leap/mail/imap/account.py index cc56fff..2f9ed1d 100644 --- a/src/leap/mail/imap/account.py +++ b/src/leap/mail/imap/account.py @@ -49,6 +49,7 @@ if PROFILE_CMD: # Soledad IMAP Account ####################################### + class IMAPAccount(object): """ An implementation of an imap4 Account @@ -68,8 +69,8 @@ class IMAPAccount(object): You can either pass a deferred to this constructor, or use `callWhenReady` method. - :param user_id: The name of the account (user id, in the form - user@provider). + :param user_id: The identifier of the user this account belongs to + (user id, in the form user@provider). :type user_id: str :param store: a Soledad instance. diff --git a/src/leap/mail/imap/server.py b/src/leap/mail/imap/server.py index 0e5d011..2682db7 100644 --- a/src/leap/mail/imap/server.py +++ b/src/leap/mail/imap/server.py @@ -20,16 +20,10 @@ LEAP IMAP4 Server Implementation. import StringIO from copy import copy -from twisted import cred -from twisted.internet import reactor from twisted.internet.defer import maybeDeferred from twisted.mail import imap4 from twisted.python import log -from leap.common.check import leap_assert, leap_assert_type -from leap.common.events import emit_async, catalog -from leap.soledad.client import Soledad - # imports for LITERAL+ patch from twisted.internet import defer, interfaces from twisted.mail.imap4 import IllegalClientResponse @@ -72,25 +66,6 @@ class LEAPIMAPServer(imap4.IMAP4Server): """ An IMAP4 Server with a LEAP Storage Backend. """ - def __init__(self, *args, **kwargs): - # pop extraneous arguments - soledad = kwargs.pop('soledad', None) - uuid = kwargs.pop('uuid', None) - userid = kwargs.pop('userid', None) - - leap_assert(soledad, "need a soledad instance") - leap_assert_type(soledad, Soledad) - leap_assert(uuid, "need a user in the initialization") - - self._userid = userid - - # initialize imap server! - imap4.IMAP4Server.__init__(self, *args, **kwargs) - - # we should initialize the account here, - # but we move it to the factory so we can - # populate the test account properly (and only once - # per session) ############################################################# # @@ -181,10 +156,6 @@ class LEAPIMAPServer(imap4.IMAP4Server): :param line: the line from the server, without the line delimiter. :type line: str """ - if self.theAccount.session_ended is True and self.state != "unauth": - log.msg("Closing the session. State: unauth") - self.state = "unauth" - if "login" in line.lower(): # avoid to log the pass, even though we are using a dummy auth # by now. @@ -208,24 +179,6 @@ class LEAPIMAPServer(imap4.IMAP4Server): self.mbox = None self.state = 'unauth' - def authenticateLogin(self, username, password): - """ - Lookup the account with the given parameters, and deny - the improper combinations. - - :param username: the username that is attempting authentication. - :type username: str - :param password: the password to authenticate with. - :type password: str - """ - # XXX this should use portal: - # return portal.login(cred.credentials.UsernamePassword(user, pass) - if username != self._userid: - # bad username, reject. - raise cred.error.UnauthorizedLogin() - # any dummy password is allowed so far. use realm instead! - emit_async(catalog.IMAP_CLIENT_LOGIN, username, "1") - return imap4.IAccount, self.theAccount, lambda: None def do_FETCH(self, tag, messages, query, uid=0): """ diff --git a/src/leap/mail/imap/service/imap-server.tac b/src/leap/mail/imap/service/imap-server.tac index 2045757..c4d602d 100644 --- a/src/leap/mail/imap/service/imap-server.tac +++ b/src/leap/mail/imap/service/imap-server.tac @@ -1,3 +1,4 @@ +#!/usr/bin/env python # -*- coding: utf-8 -*- # imap-server.tac # Copyright (C) 2013,2014 LEAP @@ -27,6 +28,9 @@ userid = 'user@provider' uuid = 'deadbeefdeadabad' passwd = 'supersecret' # optional, will get prompted if not found. """ + +# TODO -- this .tac file should be deprecated in favor of bitmask.core.bitmaskd + import ConfigParser import getpass import os @@ -112,7 +116,7 @@ tempdir = "/tmp/" print "[~] user:", userid soledad = initialize_soledad(uuid, userid, passwd, secrets, - localdb, gnupg_home, tempdir) + localdb, gnupg_home, tempdir, userid=userid) km_args = (userid, "https://localhost", soledad) km_kwargs = { "token": "", @@ -131,7 +135,8 @@ keymanager = KeyManager(*km_args, **km_kwargs) def getIMAPService(): - factory = imap.LeapIMAPFactory(uuid, userid, soledad) + soledad_sessions = {userid: soledad} + factory = imap.LeapIMAPFactory(soledad_sessions) return internet.TCPServer(port, factory, interface="localhost") diff --git a/src/leap/mail/imap/service/imap.py b/src/leap/mail/imap/service/imap.py index a50611b..24fa865 100644 --- a/src/leap/mail/imap/service/imap.py +++ b/src/leap/mail/imap/service/imap.py @@ -15,21 +15,26 @@ # You should have received a copy of the GNU General Public License # along with this program. If not, see . """ -IMAP service initialization +IMAP Service Initialization. """ import logging import os from collections import defaultdict +from twisted.cred.portal import Portal, IRealm +from twisted.cred.credentials import IUsernamePassword +from twisted.cred.checkers import ICredentialsChecker +from twisted.cred.error import UnauthorizedLogin +from twisted.mail.imap4 import IAccount +from twisted.internet import defer from twisted.internet import reactor from twisted.internet.error import CannotListenError from twisted.internet.protocol import ServerFactory -from twisted.mail import imap4 from twisted.python import log +from zope.interface import implementer from leap.common.events import emit_async, catalog -from leap.common.check import leap_check from leap.mail.imap.account import IMAPAccount from leap.mail.imap.server import LEAPIMAPServer @@ -41,57 +46,145 @@ DO_MANHOLE = os.environ.get("LEAP_MAIL_MANHOLE", None) if DO_MANHOLE: from leap.mail.imap.service import manhole -DO_PROFILE = os.environ.get("LEAP_PROFILE", None) -if DO_PROFILE: - import cProfile - log.msg("Starting PROFILING...") - - PROFILE_DAT = "/tmp/leap_mail_profile.pstats" - pr = cProfile.Profile() - pr.enable() - # The default port in which imap service will run + IMAP_PORT = 1984 +# +# Credentials Handling +# + + +@implementer(IRealm) +class LocalSoledadIMAPRealm(object): + + _encoding = 'utf-8' + + def __init__(self, soledad_sessions): + """ + :param soledad_sessions: a dict-like object, containing instances + of a Store (soledad instances), indexed by + userid. + """ + self._soledad_sessions = soledad_sessions + + def requestAvatar(self, avatarId, mind, *interfaces): + if isinstance(avatarId, str): + avatarId = avatarId.decode(self._encoding) + + def gotSoledad(soledad): + for iface in interfaces: + if iface is IAccount: + avatar = IMAPAccount(avatarId, soledad) + return (IAccount, avatar, + getattr(avatar, 'logout', lambda: None)) + raise NotImplementedError(self, interfaces) + + return self.lookupSoledadInstance(avatarId).addCallback(gotSoledad) + + def lookupSoledadInstance(self, userid): + soledad = self._soledad_sessions[userid] + # XXX this should return the instance after whenReady callback + return defer.succeed(soledad) + + +@implementer(ICredentialsChecker) +class LocalSoledadTokenChecker(object): -class IMAPAuthRealm(object): """ - Dummy authentication realm. Do not use in production! + A Credentials Checker for a LocalSoledad store. + + It checks that: + + 1) The Local SoledadStorage has been correctly unlocked for the given + user. This currently means that the right passphrase has been passed + to the Local SoledadStorage. + + 2) The password passed in the credentials matches whatever token has + been stored in the local encrypted SoledadStorage, associated to the + Protocol that is requesting the authentication. """ - theAccount = None - def requestAvatar(self, avatarId, mind, *interfaces): - return imap4.IAccount, self.theAccount, lambda: None + credentialInterfaces = (IUsernamePassword,) + service = None + + def __init__(self, soledad_sessions): + """ + :param soledad_sessions: a dict-like object, containing instances + of a Store (soledad instances), indexed by + userid. + """ + self._soledad_sessions = soledad_sessions + + def requestAvatarId(self, credentials): + if self.service is None: + raise NotImplementedError( + "this checker has not defined its service name") + username, password = credentials.username, credentials.password + d = self.checkSoledadToken(username, password, self.service) + d.addErrback(lambda f: defer.fail(UnauthorizedLogin())) + return d + + def checkSoledadToken(self, username, password, service): + soledad = self._soledad_sessions.get(username) + if not soledad: + return defer.fail(Exception("No soledad")) + + def match_token(token): + if token is None: + raise RuntimeError('no token') + if token == password: + return username + else: + raise RuntimeError('bad token') + + d = soledad.get_or_create_service_token(service) + d.addCallback(match_token) + return d + + +class IMAPTokenChecker(LocalSoledadTokenChecker): + """A credentials checker that will lookup a token for the IMAP service.""" + service = 'imap' + + +class LocalSoledadIMAPServer(LEAPIMAPServer): + + """ + An IMAP Server that authenticates against a LocalSoledad store. + """ + + def __init__(self, soledad_sessions, *args, **kw): + + LEAPIMAPServer.__init__(self, *args, **kw) + + realm = LocalSoledadIMAPRealm(soledad_sessions) + portal = Portal(realm) + checker = IMAPTokenChecker(soledad_sessions) + self.checker = checker + self.portal = portal + portal.registerChecker(checker) class LeapIMAPFactory(ServerFactory): + """ Factory for a IMAP4 server with soledad remote sync and gpg-decryption capabilities. """ - protocol = LEAPIMAPServer - def __init__(self, uuid, userid, soledad): + protocol = LocalSoledadIMAPServer + + def __init__(self, soledad_sessions): """ Initializes the server factory. - :param uuid: user uuid - :type uuid: str - - :param userid: user id (user@provider.org) - :type userid: str - - :param soledad: soledad instance - :type soledad: Soledad + :param soledad_sessions: a dict-like object, containing instances + of a Store (soledad instances), indexed by + userid. """ - self._uuid = uuid - self._userid = userid - self._soledad = soledad - - theAccount = IMAPAccount(uuid, soledad) - self.theAccount = theAccount + self._soledad_sessions = soledad_sessions self._connections = defaultdict() - # XXX how to pass the store along? def buildProtocol(self, addr): """ @@ -103,13 +196,7 @@ class LeapIMAPFactory(ServerFactory): # TODO should reject anything from addr != localhost, # just in case. log.msg("Building protocol for connection %s" % addr) - imapProtocol = self.protocol( - uuid=self._uuid, - userid=self._userid, - soledad=self._soledad) - imapProtocol.theAccount = self.theAccount - imapProtocol.factory = self - + imapProtocol = self.protocol(self._soledad_sessions) self._connections[addr] = imapProtocol return imapProtocol @@ -123,39 +210,21 @@ class LeapIMAPFactory(ServerFactory): """ Stops imap service (fetcher, factory and port). """ - # mark account as unusable, so any imap command will fail - # with unauth state. - self.theAccount.end_session() - - # TODO should wait for all the pending deferreds, - # the twisted way! - if DO_PROFILE: - log.msg("Stopping PROFILING") - pr.disable() - pr.dump_stats(PROFILE_DAT) - return ServerFactory.doStop(self) -def run_service(store, **kwargs): +def run_service(soledad_sessions, port=IMAP_PORT): """ Main entry point to run the service from the client. - :param store: a soledad instance + :param soledad_sessions: a dict-like object, containing instances + of a Store (soledad instances), indexed by userid. :returns: the port as returned by the reactor when starts listening, and the factory for the protocol. + :rtype: tuple """ - leap_check(store, "store cannot be None") - # XXX this can also be a ProxiedObject, FIXME - # leap_assert_type(store, Soledad) - - port = kwargs.get('port', IMAP_PORT) - userid = kwargs.get('userid', None) - leap_check(userid is not None, "need an user id") - - uuid = store.uuid - factory = LeapIMAPFactory(uuid, userid, store) + factory = LeapIMAPFactory(soledad_sessions) try: interface = "localhost" @@ -164,6 +233,7 @@ def run_service(store, **kwargs): if os.environ.get("LEAP_DOCKERIZED"): interface = '' + # TODO use Endpoints !!! tport = reactor.listenTCP(port, factory, interface=interface) except CannotListenError: @@ -178,9 +248,9 @@ def run_service(store, **kwargs): # TODO get pass from env var.too. manhole_factory = manhole.getManholeFactory( {'f': factory, - 'a': factory.theAccount, 'gm': factory.theAccount.getMailbox}, "boss", "leap") + # TODO use Endpoints !!! reactor.listenTCP(manhole.MANHOLE_PORT, manhole_factory, interface="127.0.0.1") logger.debug("IMAP4 Server is RUNNING in port %s" % (port,)) diff --git a/src/leap/mail/imap/tests/utils.py b/src/leap/mail/imap/tests/utils.py index a34538b..b1f8563 100644 --- a/src/leap/mail/imap/tests/utils.py +++ b/src/leap/mail/imap/tests/utils.py @@ -77,14 +77,11 @@ class IMAP4HelperMixin(SoledadTestMixin): soledad_adaptor.cleanup_deferred_locks() - UUID = 'deadbeef', USERID = TEST_USER def setup_server(account): self.server = LEAPIMAPServer( - uuid=UUID, userid=USERID, - contextFactory=self.serverCTX, - soledad=self._soledad) + contextFactory=self.serverCTX) self.server.theAccount = account d_server_ready = defer.Deferred() diff --git a/src/leap/mail/outgoing/service.py b/src/leap/mail/outgoing/service.py index 3bd0ea2..7943c12 100644 --- a/src/leap/mail/outgoing/service.py +++ b/src/leap/mail/outgoing/service.py @@ -71,7 +71,7 @@ class OutgoingMail: def __init__(self, from_address, keymanager, cert, key, host, port): """ - Initialize the mail service. + Initialize the outgoing mail service. :param from_address: The sender address. :type from_address: str diff --git a/src/leap/mail/smtp/gateway.py b/src/leap/mail/smtp/gateway.py index 3657250..3c86d7e 100644 --- a/src/leap/mail/smtp/gateway.py +++ b/src/leap/mail/smtp/gateway.py @@ -49,6 +49,9 @@ from email import generator generator.Generator = RFC3156CompliantGenerator +# TODO -- implement Queue using twisted.mail.mail.MailService + + # # Helper utilities # -- cgit v1.2.3 From 14cc0595eb58db6710782f7082dfc4e175eb86fc Mon Sep 17 00:00:00 2001 From: Kali Kaneko Date: Tue, 15 Dec 2015 02:04:07 -0400 Subject: [fix] dummy credentials for tests imap tests must be adapted, using a dummy credential checker. --- src/leap/mail/imap/tests/test_imap.py | 4 +-- src/leap/mail/imap/tests/utils.py | 59 ++++++++++++++++++++++++++++++++++- 2 files changed, 60 insertions(+), 3 deletions(-) diff --git a/src/leap/mail/imap/tests/test_imap.py b/src/leap/mail/imap/tests/test_imap.py index 62c3c41..ccce285 100644 --- a/src/leap/mail/imap/tests/test_imap.py +++ b/src/leap/mail/imap/tests/test_imap.py @@ -575,8 +575,8 @@ class LEAPIMAP4ServerTestCase(IMAP4HelperMixin): """ Test login requiring quoting """ - self.server._userid = '{test}user@leap.se' - self.server._password = '{test}password' + self.server.checker.userid = '{test}user@leap.se' + self.server.checker.password = '{test}password' def login(): d = self.client.login('{test}user@leap.se', '{test}password') diff --git a/src/leap/mail/imap/tests/utils.py b/src/leap/mail/imap/tests/utils.py index b1f8563..ad89e92 100644 --- a/src/leap/mail/imap/tests/utils.py +++ b/src/leap/mail/imap/tests/utils.py @@ -20,10 +20,15 @@ Common utilities for testing Soledad IMAP Server. from email import parser from mock import Mock +from twisted.cred.checkers import ICredentialsChecker +from twisted.cred.credentials import IUsernamePassword +from twisted.cred.error import UnauthorizedLogin +from twisted.cred.portal import Portal, IRealm from twisted.mail import imap4 from twisted.internet import defer from twisted.protocols import loopback from twisted.python import log +from zope.interface import implementer from leap.mail.adaptors import soledad as soledad_adaptor from leap.mail.imap.account import IMAPAccount @@ -64,6 +69,57 @@ class SimpleClient(imap4.IMAP4Client): self.events.append(['newMessages', exists, recent]) self.transport.loseConnection() +# +# Dummy credentials for tests +# + + +@implementer(IRealm) +class TestRealm(object): + + def __init__(self, account): + self._account = account + + def requestAvatar(self, avatarId, mind, *interfaces): + avatar = self._account + return (imap4.IAccount, avatar, + getattr(avatar, 'logout', lambda: None)) + + +@implementer(ICredentialsChecker) +class TestCredentialsChecker(object): + + credentialInterfaces = (IUsernamePassword,) + + userid = TEST_USER + password = TEST_PASSWD + + def requestAvatarId(self, credentials): + username, password = credentials.username, credentials.password + d = self.checkTestCredentials(username, password) + d.addErrback(lambda f: defer.fail(UnauthorizedLogin())) + return d + + def checkTestCredentials(self, username, password): + if username == self.userid and password == self.password: + return defer.succeed(username) + else: + return defer.fail(Exception("Wrong credentials")) + + +class TestSoledadIMAPServer(LEAPIMAPServer): + + def __init__(self, account, *args, **kw): + + LEAPIMAPServer.__init__(self, *args, **kw) + + realm = TestRealm(account) + portal = Portal(realm) + checker = TestCredentialsChecker() + self.checker = checker + self.portal = portal + portal.registerChecker(checker) + class IMAP4HelperMixin(SoledadTestMixin): """ @@ -80,7 +136,8 @@ class IMAP4HelperMixin(SoledadTestMixin): USERID = TEST_USER def setup_server(account): - self.server = LEAPIMAPServer( + self.server = TestSoledadIMAPServer( + account=account, contextFactory=self.serverCTX) self.server.theAccount = account -- cgit v1.2.3 From 5c0bf0d6a84de6c69d85ab4c08a0cb64e96a5cf2 Mon Sep 17 00:00:00 2001 From: Kali Kaneko Date: Tue, 15 Dec 2015 02:42:14 -0400 Subject: [docs] add entry about cred-based token authentication to next-changelog --- changes/next-changelog.rst | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/changes/next-changelog.rst b/changes/next-changelog.rst index 7de9d98..625dcac 100644 --- a/changes/next-changelog.rst +++ b/changes/next-changelog.rst @@ -11,6 +11,9 @@ I've added a new category `Misc` so we can track doc/style/packaging stuff. Features ~~~~~~~~ - `#7656 `_: Emit multi-user aware events. +- `#4008 `_: Add token-based authentication to local IMAP/SMTP services. +- Use twisted.cred to authenticate IMAP users. + - `#1234 `_: Description of the new feature corresponding with issue #1234. - New feature without related issue number. @@ -18,6 +21,7 @@ Bugfixes ~~~~~~~~ - `#1235 `_: Description for the fixed stuff corresponding with issue #1235. - Fix the get_body logic for corner-cases in which body is None (yet-to-be synced docs, mainly). + - Bugfix without related issue number. Misc -- cgit v1.2.3 From cb676ab65d41c8821683a10bb675e94ac59e06ff Mon Sep 17 00:00:00 2001 From: Kali Kaneko Date: Tue, 15 Dec 2015 16:40:46 -0400 Subject: [style] pep8 --- src/leap/mail/imap/server.py | 1 - 1 file changed, 1 deletion(-) diff --git a/src/leap/mail/imap/server.py b/src/leap/mail/imap/server.py index 2682db7..b6f1e47 100644 --- a/src/leap/mail/imap/server.py +++ b/src/leap/mail/imap/server.py @@ -179,7 +179,6 @@ class LEAPIMAPServer(imap4.IMAP4Server): self.mbox = None self.state = 'unauth' - def do_FETCH(self, tag, messages, query, uid=0): """ Overwritten fetch dispatcher to use the fast fetch_flags -- cgit v1.2.3 From ea41bb44afee34e8ad6baf917ba461a7e95bf70d Mon Sep 17 00:00:00 2001 From: Kali Kaneko Date: Wed, 16 Dec 2015 01:12:30 -0400 Subject: [feat] cred authentication for SMTP service --- src/leap/mail/cred.py | 80 +++++++++++++++++++ src/leap/mail/errors.py | 27 +++++++ src/leap/mail/imap/service/imap.py | 59 +------------- src/leap/mail/outgoing/service.py | 32 +++++++- src/leap/mail/smtp/__init__.py | 50 +++++------- src/leap/mail/smtp/gateway.py | 159 +++++++++++++++++++++++-------------- 6 files changed, 257 insertions(+), 150 deletions(-) create mode 100644 src/leap/mail/cred.py create mode 100644 src/leap/mail/errors.py diff --git a/src/leap/mail/cred.py b/src/leap/mail/cred.py new file mode 100644 index 0000000..7eab1f0 --- /dev/null +++ b/src/leap/mail/cred.py @@ -0,0 +1,80 @@ +# -*- coding: utf-8 -*- +# cred.py +# Copyright (C) 2015 LEAP +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . +""" +Credentials handling. +""" + +from zope.interface import implementer +from twisted.cred.checkers import ICredentialsChecker +from twisted.cred.credentials import IUsernamePassword +from twisted.cred.error import UnauthorizedLogin +from twisted.internet import defer + + +@implementer(ICredentialsChecker) +class LocalSoledadTokenChecker(object): + + """ + A Credentials Checker for a LocalSoledad store. + + It checks that: + + 1) The Local SoledadStorage has been correctly unlocked for the given + user. This currently means that the right passphrase has been passed + to the Local SoledadStorage. + + 2) The password passed in the credentials matches whatever token has + been stored in the local encrypted SoledadStorage, associated to the + Protocol that is requesting the authentication. + """ + + credentialInterfaces = (IUsernamePassword,) + service = None + + def __init__(self, soledad_sessions): + """ + :param soledad_sessions: a dict-like object, containing instances + of a Store (soledad instances), indexed by + userid. + """ + self._soledad_sessions = soledad_sessions + + def requestAvatarId(self, credentials): + if self.service is None: + raise NotImplementedError( + "this checker has not defined its service name") + username, password = credentials.username, credentials.password + d = self.checkSoledadToken(username, password, self.service) + d.addErrback(lambda f: defer.fail(UnauthorizedLogin())) + return d + + def checkSoledadToken(self, username, password, service): + soledad = self._soledad_sessions.get(username) + if not soledad: + return defer.fail(Exception("No soledad")) + + def match_token(token): + if token is None: + raise RuntimeError('no token') + if token == password: + return username + else: + raise RuntimeError('bad token') + + d = soledad.get_or_create_service_token(service) + d.addCallback(match_token) + return d diff --git a/src/leap/mail/errors.py b/src/leap/mail/errors.py new file mode 100644 index 0000000..2f18e87 --- /dev/null +++ b/src/leap/mail/errors.py @@ -0,0 +1,27 @@ +# -*- coding: utf-8 -*- +# errors.py +# Copyright (C) 2015 LEAP +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . +""" +Exceptions for leap.mail +""" + + +class AuthenticationError(Exception): + pass + + +class ConfigurationError(Exception): + pass diff --git a/src/leap/mail/imap/service/imap.py b/src/leap/mail/imap/service/imap.py index 24fa865..9e34454 100644 --- a/src/leap/mail/imap/service/imap.py +++ b/src/leap/mail/imap/service/imap.py @@ -23,9 +23,6 @@ import os from collections import defaultdict from twisted.cred.portal import Portal, IRealm -from twisted.cred.credentials import IUsernamePassword -from twisted.cred.checkers import ICredentialsChecker -from twisted.cred.error import UnauthorizedLogin from twisted.mail.imap4 import IAccount from twisted.internet import defer from twisted.internet import reactor @@ -35,6 +32,7 @@ from twisted.python import log from zope.interface import implementer from leap.common.events import emit_async, catalog +from leap.mail.cred import LocalSoledadTokenChecker from leap.mail.imap.account import IMAPAccount from leap.mail.imap.server import LEAPIMAPServer @@ -88,61 +86,6 @@ class LocalSoledadIMAPRealm(object): return defer.succeed(soledad) -@implementer(ICredentialsChecker) -class LocalSoledadTokenChecker(object): - - """ - A Credentials Checker for a LocalSoledad store. - - It checks that: - - 1) The Local SoledadStorage has been correctly unlocked for the given - user. This currently means that the right passphrase has been passed - to the Local SoledadStorage. - - 2) The password passed in the credentials matches whatever token has - been stored in the local encrypted SoledadStorage, associated to the - Protocol that is requesting the authentication. - """ - - credentialInterfaces = (IUsernamePassword,) - service = None - - def __init__(self, soledad_sessions): - """ - :param soledad_sessions: a dict-like object, containing instances - of a Store (soledad instances), indexed by - userid. - """ - self._soledad_sessions = soledad_sessions - - def requestAvatarId(self, credentials): - if self.service is None: - raise NotImplementedError( - "this checker has not defined its service name") - username, password = credentials.username, credentials.password - d = self.checkSoledadToken(username, password, self.service) - d.addErrback(lambda f: defer.fail(UnauthorizedLogin())) - return d - - def checkSoledadToken(self, username, password, service): - soledad = self._soledad_sessions.get(username) - if not soledad: - return defer.fail(Exception("No soledad")) - - def match_token(token): - if token is None: - raise RuntimeError('no token') - if token == password: - return username - else: - raise RuntimeError('bad token') - - d = soledad.get_or_create_service_token(service) - d.addCallback(match_token) - return d - - class IMAPTokenChecker(LocalSoledadTokenChecker): """A credentials checker that will lookup a token for the IMAP service.""" service = 'imap' diff --git a/src/leap/mail/outgoing/service.py b/src/leap/mail/outgoing/service.py index 7943c12..3e14fbd 100644 --- a/src/leap/mail/outgoing/service.py +++ b/src/leap/mail/outgoing/service.py @@ -14,6 +14,14 @@ # # You should have received a copy of the GNU General Public License # along with this program. If not, see . + +""" +OutgoingMail module. + +The OutgoingMail class allows to send mail, and encrypts/signs it if needed. +""" + +import os.path import re from StringIO import StringIO from copy import deepcopy @@ -35,6 +43,7 @@ from leap.common.events import emit_async, catalog from leap.keymanager.openpgp import OpenPGPKey from leap.keymanager.errors import KeyNotFound, KeyAddressMismatch from leap.mail import __version__ +from leap.mail import errors from leap.mail.utils import validate_address from leap.mail.rfc3156 import MultipartEncrypted from leap.mail.rfc3156 import MultipartSigned @@ -64,9 +73,23 @@ class SSLContextFactory(ssl.ClientContextFactory): return ctx -class OutgoingMail: +def outgoingFactory(userid, keymanager, opts): + + cert = unicode(opts.cert) + key = unicode(opts.key) + hostname = str(opts.hostname) + port = opts.port + + if not os.path.isfile(cert): + raise errors.ConfigurationError( + 'No valid SMTP certificate could be found for %s!' % userid) + + return OutgoingMail(str(userid), keymanager, cert, key, hostname, port) + + +class OutgoingMail(object): """ - A service for handling encrypted outgoing mail. + Sends Outgoing Mail, encrypting and signing if needed. """ def __init__(self, from_address, keymanager, cert, key, host, port): @@ -134,9 +157,10 @@ class OutgoingMail: :type smtp_sender_result: tuple(int, list(tuple)) """ dest_addrstr = smtp_sender_result[1][0][0] - log.msg('Message sent to %s' % dest_addrstr) + fromaddr = self._from_address + log.msg('Message sent from %s to %s' % (fromaddr, dest_addrstr)) emit_async(catalog.SMTP_SEND_MESSAGE_SUCCESS, - self._from_address, dest_addrstr) + fromaddr, dest_addrstr) def sendError(self, failure): """ diff --git a/src/leap/mail/smtp/__init__.py b/src/leap/mail/smtp/__init__.py index 7b62808..9fab70a 100644 --- a/src/leap/mail/smtp/__init__.py +++ b/src/leap/mail/smtp/__init__.py @@ -23,47 +23,35 @@ import os from twisted.internet import reactor from twisted.internet.error import CannotListenError -from leap.mail.outgoing.service import OutgoingMail from leap.common.events import emit_async, catalog + from leap.mail.smtp.gateway import SMTPFactory logger = logging.getLogger(__name__) -def setup_smtp_gateway(port, userid, keymanager, smtp_host, smtp_port, - smtp_cert, smtp_key, encrypted_only): - """ - Setup SMTP gateway to run with Twisted. +SMTP_PORT = 2013 + - This function sets up the SMTP gateway configuration and the Twisted - reactor. +def run_service(soledad_sessions, keymanager_sessions, sendmail_opts, + port=SMTP_PORT): + """ + Main entry point to run the service from the client. - :param port: The port in which to run the server. - :type port: int - :param userid: The user currently logged in - :type userid: str - :param keymanager: A Key Manager from where to get recipients' public - keys. - :type keymanager: leap.common.keymanager.KeyManager - :param smtp_host: The hostname of the remote SMTP server. - :type smtp_host: str - :param smtp_port: The port of the remote SMTP server. - :type smtp_port: int - :param smtp_cert: The client certificate for authentication. - :type smtp_cert: str - :param smtp_key: The client key for authentication. - :type smtp_key: str - :param encrypted_only: Whether the SMTP gateway should send unencrypted - mail or not. - :type encrypted_only: bool + :param soledad_sessions: a dict-like object, containing instances + of a Store (soledad instances), indexed by userid. + :param keymanager_sessions: a dict-like object, containing instances + of Keymanager, indexed by userid. + :param sendmail_opts: a dict-like object of sendmailOptions. - :returns: tuple of SMTPFactory, twisted.internet.tcp.Port + :returns: the port as returned by the reactor when starts listening, and + the factory for the protocol. + :rtype: tuple """ - # configure the use of this service with twistd - outgoing_mail = OutgoingMail( - userid, keymanager, smtp_cert, smtp_key, smtp_host, smtp_port) - factory = SMTPFactory(userid, keymanager, encrypted_only, outgoing_mail) + factory = SMTPFactory(soledad_sessions, keymanager_sessions, + sendmail_opts) + try: interface = "localhost" # don't bind just to localhost if we are running on docker since we @@ -71,8 +59,10 @@ def setup_smtp_gateway(port, userid, keymanager, smtp_host, smtp_port, if os.environ.get("LEAP_DOCKERIZED"): interface = '' + # TODO Use Endpoints instead -------------------------------- tport = reactor.listenTCP(port, factory, interface=interface) emit_async(catalog.SMTP_SERVICE_STARTED, str(port)) + return factory, tport except CannotListenError: logger.error("STMP Service failed to start: " diff --git a/src/leap/mail/smtp/gateway.py b/src/leap/mail/smtp/gateway.py index 3c86d7e..85b1560 100644 --- a/src/leap/mail/smtp/gateway.py +++ b/src/leap/mail/smtp/gateway.py @@ -30,18 +30,26 @@ The following classes comprise the SMTP gateway service: knows how to encrypt/sign itself before sending. """ +from email.Header import Header + from zope.interface import implements +from zope.interface import implementer + +from twisted.cred.portal import Portal, IRealm from twisted.mail import smtp -from twisted.internet.protocol import ServerFactory +from twisted.mail.imap4 import LOGINCredentials, PLAINCredentials +from twisted.internet import defer, protocol from twisted.python import log -from email.Header import Header from leap.common.check import leap_assert_type from leap.common.events import emit_async, catalog -from leap.keymanager.openpgp import OpenPGPKey -from leap.keymanager.errors import KeyNotFound +from leap.mail import errors +from leap.mail.cred import LocalSoledadTokenChecker from leap.mail.utils import validate_address from leap.mail.rfc3156 import RFC3156CompliantGenerator +from leap.mail.outgoing.service import outgoingFactory +from leap.keymanager.openpgp import OpenPGPKey +from leap.keymanager.errors import KeyNotFound # replace email generator with a RFC 3156 compliant one. from email import generator @@ -49,87 +57,122 @@ from email import generator generator.Generator = RFC3156CompliantGenerator -# TODO -- implement Queue using twisted.mail.mail.MailService +LOCAL_FQDN = "bitmask.local" -# -# Helper utilities -# +@implementer(IRealm) +class LocalSMTPRealm(object): -LOCAL_FQDN = "bitmask.local" + _encoding = 'utf-8' + + def __init__(self, keymanager_sessions, sendmail_opts): + """ + :param keymanager_sessions: a dict-like object, containing instances + of a Keymanager objects, indexed by + userid. + """ + self._keymanager_sessions = keymanager_sessions + self._sendmail_opts = sendmail_opts + def requestAvatar(self, avatarId, mind, *interfaces): + if isinstance(avatarId, str): + avatarId = avatarId.decode(self._encoding) -class SMTPHeloLocalhost(smtp.SMTP): - """ - An SMTP class that ensures a proper FQDN - for localhost. + def gotKeymanager(keymanager): - This avoids a problem in which unproperly configured providers - would complain about the helo not being a fqdn. - """ + # TODO use IMessageDeliveryFactory instead ? + # it could reuse the connections. + if smtp.IMessageDelivery in interfaces: + userid = avatarId + opts = self.getSendingOpts(userid) + outgoing = outgoingFactory(userid, keymanager, opts) + avatar = SMTPDelivery(userid, keymanager, False, outgoing) + + return (smtp.IMessageDelivery, avatar, + getattr(avatar, 'logout', lambda: None)) + + raise NotImplementedError(self, interfaces) + + return self.lookupKeymanagerInstance(avatarId).addCallback( + gotKeymanager) - def __init__(self, *args): - smtp.SMTP.__init__(self, *args) - self.host = LOCAL_FQDN + def lookupKeymanagerInstance(self, userid): + try: + keymanager = self._keymanager_sessions[userid] + except: + raise errors.AuthenticationError( + 'No keymanager session found for user %s. Is it authenticated?' + % userid) + # XXX this should return the instance after whenReady callback + return defer.succeed(keymanager) + + def getSendingOpts(self, userid): + try: + opts = self._sendmail_opts[userid] + except KeyError: + raise errors.ConfigurationError( + 'No sendingMail options found for user %s' % userid) + return opts + + +class SMTPTokenChecker(LocalSoledadTokenChecker): + """A credentials checker that will lookup a token for the SMTP service.""" + service = 'smtp' + + # TODO besides checking for token credential, + # we could also verify the certificate here. + + +# TODO -- implement Queue using twisted.mail.mail.MailService +class LocalSMTPServer(smtp.ESMTP): + def __init__(self, soledad_sessions, keymanager_sessions, sendmail_opts, + *args, **kw): -class SMTPFactory(ServerFactory): + smtp.ESMTP.__init__(self, *args, **kw) + + realm = LocalSMTPRealm(keymanager_sessions, sendmail_opts) + portal = Portal(realm) + checker = SMTPTokenChecker(soledad_sessions) + self.checker = checker + self.portal = portal + portal.registerChecker(checker) + + +class SMTPFactory(protocol.ServerFactory): """ Factory for an SMTP server with encrypted gatewaying capabilities. """ - domain = LOCAL_FQDN - - def __init__(self, userid, keymanager, encrypted_only, outgoing_mail): - """ - Initialize the SMTP factory. - :param userid: The user currently logged in - :type userid: unicode - :param keymanager: A Key Manager from where to get recipients' public - keys. - :param encrypted_only: Whether the SMTP gateway should send unencrypted - mail or not. - :type encrypted_only: bool - :param outgoing_mail: The outgoing mail to send the message - :type outgoing_mail: leap.mail.outgoing.service.OutgoingMail - """ + protocol = LocalSMTPServer + domain = LOCAL_FQDN + timeout = 600 - leap_assert_type(encrypted_only, bool) - # and store them - self._userid = userid - self._km = keymanager - self._outgoing_mail = outgoing_mail - self._encrypted_only = encrypted_only + def __init__(self, soledad_sessions, keymanager_sessions, sendmail_opts): + self._soledad_sessions = soledad_sessions + self._keymanager_sessions = keymanager_sessions + self._sendmail_opts = sendmail_opts def buildProtocol(self, addr): - """ - Return a protocol suitable for the job. - - :param addr: An address, e.g. a TCP (host, port). - :type addr: twisted.internet.interfaces.IAddress - - @return: The protocol. - @rtype: SMTPDelivery - """ - smtpProtocol = SMTPHeloLocalhost( - SMTPDelivery( - self._userid, self._km, self._encrypted_only, - self._outgoing_mail)) - smtpProtocol.factory = self - return smtpProtocol + p = self.protocol( + self._soledad_sessions, self._keymanager_sessions, + self._sendmail_opts) + p.factory = self + p.host = LOCAL_FQDN + p.challengers = {"LOGIN": LOGINCredentials, "PLAIN": PLAINCredentials} + return p # # SMTPDelivery # +@implementer(smtp.IMessageDelivery) class SMTPDelivery(object): """ Validate email addresses and handle message delivery. """ - implements(smtp.IMessageDelivery) - def __init__(self, userid, keymanager, encrypted_only, outgoing_mail): """ Initialize the SMTP delivery object. -- cgit v1.2.3 From f87b25c7b942f509372a14ed0ee7073f8f17e053 Mon Sep 17 00:00:00 2001 From: Kali Kaneko Date: Thu, 17 Dec 2015 01:34:26 -0400 Subject: [tests] make tests use dummy authentication --- src/leap/mail/outgoing/service.py | 9 +-- src/leap/mail/outgoing/tests/test_outgoing.py | 24 +++---- src/leap/mail/smtp/gateway.py | 43 ++++++++++--- src/leap/mail/smtp/tests/test_gateway.py | 91 ++++++++++++++++++--------- 4 files changed, 112 insertions(+), 55 deletions(-) diff --git a/src/leap/mail/outgoing/service.py b/src/leap/mail/outgoing/service.py index 3e14fbd..8d7c0f8 100644 --- a/src/leap/mail/outgoing/service.py +++ b/src/leap/mail/outgoing/service.py @@ -73,16 +73,17 @@ class SSLContextFactory(ssl.ClientContextFactory): return ctx -def outgoingFactory(userid, keymanager, opts): +def outgoingFactory(userid, keymanager, opts, check_cert=True): cert = unicode(opts.cert) key = unicode(opts.key) hostname = str(opts.hostname) port = opts.port - if not os.path.isfile(cert): - raise errors.ConfigurationError( - 'No valid SMTP certificate could be found for %s!' % userid) + if check_cert: + if not os.path.isfile(cert): + raise errors.ConfigurationError( + 'No valid SMTP certificate could be found for %s!' % userid) return OutgoingMail(str(userid), keymanager, cert, key, hostname, port) diff --git a/src/leap/mail/outgoing/tests/test_outgoing.py b/src/leap/mail/outgoing/tests/test_outgoing.py index 5518b33..79eafd9 100644 --- a/src/leap/mail/outgoing/tests/test_outgoing.py +++ b/src/leap/mail/outgoing/tests/test_outgoing.py @@ -29,20 +29,19 @@ from twisted.mail.smtp import User from mock import Mock -from leap.mail.smtp.gateway import SMTPFactory from leap.mail.rfc3156 import RFC3156CompliantGenerator from leap.mail.outgoing.service import OutgoingMail -from leap.mail.tests import ( - TestCaseWithKeyManager, - ADDRESS, - ADDRESS_2, - PUBLIC_KEY_2, -) +from leap.mail.tests import TestCaseWithKeyManager +from leap.mail.tests import ADDRESS, ADDRESS_2, PUBLIC_KEY_2 +from leap.mail.smtp.tests.test_gateway import getSMTPFactory + from leap.keymanager import openpgp, errors BEGIN_PUBLIC_KEY = "-----BEGIN PGP PUBLIC KEY BLOCK-----" +TEST_USER = u'anotheruser@leap.se' + class TestOutgoingMail(TestCaseWithKeyManager): EMAIL_DATA = ['HELO gateway.leap.se', @@ -73,11 +72,12 @@ class TestOutgoingMail(TestCaseWithKeyManager): self.fromAddr, self._km, self._config['cert'], self._config['key'], self._config['host'], self._config['port']) - self.proto = SMTPFactory( - u'anotheruser@leap.se', - self._km, - self._config['encrypted_only'], - self.outgoing_mail).buildProtocol(('127.0.0.1', 0)) + + user = TEST_USER + + # TODO -- this shouldn't need SMTP to be tested!? or does it? + self.proto = getSMTPFactory( + {user: None}, {user: self._km}, {user: None}) self.dest = User(ADDRESS, 'gateway.leap.se', self.proto, ADDRESS_2) d = TestCaseWithKeyManager.setUp(self) diff --git a/src/leap/mail/smtp/gateway.py b/src/leap/mail/smtp/gateway.py index 85b1560..7ff6b14 100644 --- a/src/leap/mail/smtp/gateway.py +++ b/src/leap/mail/smtp/gateway.py @@ -65,7 +65,8 @@ class LocalSMTPRealm(object): _encoding = 'utf-8' - def __init__(self, keymanager_sessions, sendmail_opts): + def __init__(self, keymanager_sessions, sendmail_opts, + encrypted_only=False): """ :param keymanager_sessions: a dict-like object, containing instances of a Keymanager objects, indexed by @@ -73,6 +74,7 @@ class LocalSMTPRealm(object): """ self._keymanager_sessions = keymanager_sessions self._sendmail_opts = sendmail_opts + self.encrypted_only = encrypted_only def requestAvatar(self, avatarId, mind, *interfaces): if isinstance(avatarId, str): @@ -86,7 +88,8 @@ class LocalSMTPRealm(object): userid = avatarId opts = self.getSendingOpts(userid) outgoing = outgoingFactory(userid, keymanager, opts) - avatar = SMTPDelivery(userid, keymanager, False, outgoing) + avatar = SMTPDelivery(userid, keymanager, self.encrypted_only, + outgoing) return (smtp.IMessageDelivery, avatar, getattr(avatar, 'logout', lambda: None)) @@ -123,22 +126,41 @@ class SMTPTokenChecker(LocalSoledadTokenChecker): # we could also verify the certificate here. -# TODO -- implement Queue using twisted.mail.mail.MailService -class LocalSMTPServer(smtp.ESMTP): +class LEAPInitMixin(object): + """ + A Mixin that takes care of initialization of all the data needed to access + LEAP sessions. + """ def __init__(self, soledad_sessions, keymanager_sessions, sendmail_opts, - *args, **kw): - - smtp.ESMTP.__init__(self, *args, **kw) - - realm = LocalSMTPRealm(keymanager_sessions, sendmail_opts) + encrypted_only=False): + realm = LocalSMTPRealm(keymanager_sessions, sendmail_opts, + encrypted_only) portal = Portal(realm) + checker = SMTPTokenChecker(soledad_sessions) self.checker = checker self.portal = portal portal.registerChecker(checker) +class LocalSMTPServer(smtp.ESMTP, LEAPInitMixin): + """ + The Production ESMTP Server: Authentication Needed. + Authenticates against SMTP Token stored in Local Soledad instance. + The Realm will produce a Delivery Object that handles encryption/signing. + """ + + # TODO: implement Queue using twisted.mail.mail.MailService + + def __init__(self, soledads, keyms, sendmailopts, *args, **kw): + encrypted_only = kw.pop('encrypted_only', False) + + LEAPInitMixin.__init__(self, soledads, keyms, sendmailopts, + encrypted_only) + smtp.ESMTP.__init__(self, *args, **kw) + + class SMTPFactory(protocol.ServerFactory): """ Factory for an SMTP server with encrypted gatewaying capabilities. @@ -147,6 +169,7 @@ class SMTPFactory(protocol.ServerFactory): protocol = LocalSMTPServer domain = LOCAL_FQDN timeout = 600 + encrypted_only = False def __init__(self, soledad_sessions, keymanager_sessions, sendmail_opts): self._soledad_sessions = soledad_sessions @@ -156,7 +179,7 @@ class SMTPFactory(protocol.ServerFactory): def buildProtocol(self, addr): p = self.protocol( self._soledad_sessions, self._keymanager_sessions, - self._sendmail_opts) + self._sendmail_opts, encrypted_only=self.encrypted_only) p.factory = self p.host = LOCAL_FQDN p.challengers = {"LOGIN": LOGINCredentials, "PLAIN": PLAINCredentials} diff --git a/src/leap/mail/smtp/tests/test_gateway.py b/src/leap/mail/smtp/tests/test_gateway.py index 0b9a364..df83cf0 100644 --- a/src/leap/mail/smtp/tests/test_gateway.py +++ b/src/leap/mail/smtp/tests/test_gateway.py @@ -15,7 +15,6 @@ # You should have received a copy of the GNU General Public License # along with this program. If not, see . - """ SMTP gateway tests. """ @@ -23,19 +22,18 @@ SMTP gateway tests. import re from datetime import datetime +from twisted.mail import smtp from twisted.internet import reactor from twisted.internet.defer import inlineCallbacks, fail, succeed, Deferred from twisted.test import proto_helpers from mock import Mock -from leap.mail.smtp.gateway import ( - SMTPFactory -) -from leap.mail.tests import ( - TestCaseWithKeyManager, - ADDRESS, - ADDRESS_2, -) +from leap.mail.smtp.gateway import SMTPFactory, LOCAL_FQDN +from leap.mail.smtp.gateway import SMTPDelivery + +from leap.mail.outgoing.service import outgoingFactory +from leap.mail.tests import TestCaseWithKeyManager +from leap.mail.tests import ADDRESS, ADDRESS_2 from leap.keymanager import openpgp, errors @@ -46,6 +44,52 @@ HOSTNAME_REGEX = "(([a-zA-Z0-9]|[a-zA-Z0-9][a-zA-Z0-9\-]*[a-zA-Z0-9])\.)*" + \ "([A-Za-z0-9]|[A-Za-z0-9][A-Za-z0-9\-]*[A-Za-z0-9])" IP_OR_HOST_REGEX = '(' + IP_REGEX + '|' + HOSTNAME_REGEX + ')' +TEST_USER = u'anotheruser@leap.se' + + +def getSMTPFactory(soledad_s, keymanager_s, sendmail_opts, + encrypted_only=False): + factory = UnauthenticatedSMTPFactory + factory.encrypted_only = encrypted_only + proto = factory( + soledad_s, keymanager_s, sendmail_opts).buildProtocol(('127.0.0.1', 0)) + return proto + + +class UnauthenticatedSMTPServer(smtp.SMTP): + + encrypted_only = False + + def __init__(self, soledads, keyms, opts, encrypted_only=False): + smtp.SMTP.__init__(self) + + userid = TEST_USER + keym = keyms[userid] + + class Opts: + cert = '/tmp/cert' + key = '/tmp/cert' + hostname = 'remote' + port = 666 + + outgoing = outgoingFactory( + userid, keym, Opts, check_cert=False) + avatar = SMTPDelivery(userid, keym, encrypted_only, outgoing) + self.delivery = avatar + + def validateFrom(self, helo, origin): + return origin + + +class UnauthenticatedSMTPFactory(SMTPFactory): + """ + A Factory that produces a SMTP server that does not authenticate user. + Only for tests! + """ + protocol = UnauthenticatedSMTPServer + domain = LOCAL_FQDN + encrypted_only = False + class TestSmtpGateway(TestCaseWithKeyManager): @@ -85,14 +129,8 @@ class TestSmtpGateway(TestCaseWithKeyManager): '250 Recipient address accepted', '354 Continue'] - # XXX this bit can be refactored away in a helper - # method... - proto = SMTPFactory( - u'anotheruser@leap.se', - self._km, - self._config['encrypted_only'], - outgoing_mail=Mock()).buildProtocol(('127.0.0.1', 0)) - # snip... + user = TEST_USER + proto = getSMTPFactory({user: None}, {user: self._km}, {user: None}) transport = proto_helpers.StringTransport() proto.makeConnection(transport) reply = "" @@ -116,12 +154,10 @@ class TestSmtpGateway(TestCaseWithKeyManager): # mock the key fetching self._km._fetch_keys_from_server = Mock( return_value=fail(errors.KeyNotFound())) - # prepare the SMTP factory - proto = SMTPFactory( - u'anotheruser@leap.se', - self._km, - self._config['encrypted_only'], - outgoing_mail=Mock()).buildProtocol(('127.0.0.1', 0)) + user = TEST_USER + proto = getSMTPFactory( + {user: None}, {user: self._km}, {user: None}, + encrypted_only=True) transport = proto_helpers.StringTransport() proto.makeConnection(transport) yield self.getReply(self.EMAIL_DATA[0] + '\r\n', proto, transport) @@ -132,7 +168,7 @@ class TestSmtpGateway(TestCaseWithKeyManager): self.assertEqual( '550 Cannot receive for specified address\r\n', reply, - 'Address should have been rejecetd with appropriate message.') + 'Address should have been rejected with appropriate message.') proto.setTimeout(None) @inlineCallbacks @@ -149,11 +185,8 @@ class TestSmtpGateway(TestCaseWithKeyManager): # mock the key fetching self._km._fetch_keys_from_server = Mock( return_value=fail(errors.KeyNotFound())) - # prepare the SMTP factory with encrypted only equal to false - proto = SMTPFactory( - u'anotheruser@leap.se', - self._km, - False, outgoing_mail=Mock()).buildProtocol(('127.0.0.1', 0)) + user = TEST_USER + proto = getSMTPFactory({user: None}, {user: self._km}, {user: None}) transport = proto_helpers.StringTransport() proto.makeConnection(transport) yield self.getReply(self.EMAIL_DATA[0] + '\r\n', proto, transport) -- cgit v1.2.3 From 50f0659459169d297fa28ec8f42b7541da970175 Mon Sep 17 00:00:00 2001 From: Giovane Date: Tue, 19 Jan 2016 17:45:52 -0200 Subject: [feat] Verify plain text signed email - Extract message serialization to a method - Add new condition to verify signature on plain text mail - Return InvalidSignature if cannot verify --- src/leap/mail/incoming/service.py | 43 ++++++++++++++++++++++++++++++++------- 1 file changed, 36 insertions(+), 7 deletions(-) diff --git a/src/leap/mail/incoming/service.py b/src/leap/mail/incoming/service.py index 3896c17..1716816 100644 --- a/src/leap/mail/incoming/service.py +++ b/src/leap/mail/incoming/service.py @@ -440,6 +440,7 @@ class IncomingMail(Service): fromHeader = msg.get('from', None) senderAddress = None + if (fromHeader is not None and (msg.get_content_type() == MULTIPART_ENCRYPTED or msg.get_content_type() == MULTIPART_SIGNED)): @@ -466,6 +467,8 @@ class IncomingMail(Service): if msg.get_content_type() == MULTIPART_ENCRYPTED: d = self._decrypt_multipart_encrypted_msg( msg, encoding, senderAddress) + elif msg.get_content_type() == MULTIPART_SIGNED: + d = self._verify_signature_not_encrypted_msg(msg, senderAddress) else: d = self._maybe_decrypt_inline_encrypted_msg( msg, encoding, senderAddress) @@ -522,8 +525,8 @@ class IncomingMail(Service): return (msg, signkey) d = self._keymanager.decrypt( - encdata, self._userid, OpenPGPKey, - verify=senderAddress) + encdata, self._userid, OpenPGPKey, + verify=senderAddress) d.addCallbacks(build_msg, self._decryption_error, errbackArgs=(msg,)) return d @@ -545,11 +548,8 @@ class IncomingMail(Service): :rtype: Deferred """ log.msg('maybe decrypting inline encrypted msg') - # serialize the original message - buf = StringIO() - g = Generator(buf) - g.flatten(origmsg) - data = buf.getvalue() + + data = self._serialize_msg(origmsg) def decrypted_data(res): decrdata, signkey = res @@ -578,6 +578,35 @@ class IncomingMail(Service): d.addCallback(encode_and_return) return d + def _verify_signature_not_encrypted_msg(self, origmsg, sender_address): + """ + Possibly decrypt an inline OpenPGP encrypted message. + + :param origmsg: The original, possibly encrypted message. + :type origmsg: Message + :param sender_address: The email address of the sender of the message. + :type sender_address: str + + :return: A Deferred that will be fired with a tuple containing a + signed Message and the signing OpenPGPKey if the signature + is valid or InvalidSignature. + :rtype: Deferred + """ + msg = copy.deepcopy(origmsg) + data = msg.get_payload()[0].as_string() + detached_sig = msg.get_payload()[1].get_payload() + d = self._keymanager.verify(data, sender_address, OpenPGPKey, detached_sig) + + d.addCallback(lambda sign_key: (msg, sign_key)) + d.addErrback(lambda _: (msg, keymanager_errors.InvalidSignature())) + return d + + def _serialize_msg(self, origmsg): + buf = StringIO() + g = Generator(buf) + g.flatten(origmsg) + return buf.getvalue() + def _decryption_error(self, failure, msg): """ Check for known decryption errors -- cgit v1.2.3 From 1c3b8d861db49ee723a7470301a63a8ca2994f5c Mon Sep 17 00:00:00 2001 From: Giovane Date: Thu, 21 Jan 2016 16:31:14 -0200 Subject: [feat] Validate signature with attachments - Create a new Generator that doesn't trim the headers - Extract detached signature from message - Convert message to the body an attachments level - Add coment to the generator workaround and shows which python version has the patch --- src/leap/mail/generator.py | 21 +++++++++++++++++++++ src/leap/mail/incoming/service.py | 25 ++++++++++++++++++------- 2 files changed, 39 insertions(+), 7 deletions(-) create mode 100644 src/leap/mail/generator.py diff --git a/src/leap/mail/generator.py b/src/leap/mail/generator.py new file mode 100644 index 0000000..28db8da --- /dev/null +++ b/src/leap/mail/generator.py @@ -0,0 +1,21 @@ +from email.generator import Generator as EmailGenerator + +class Generator(EmailGenerator): + """ + Generates output from a Message object tree, keeping signatures. + + This code was extracted from Mailman.Generator.Generator, version 2.1.4: + + Most other Generator will be created not setting the foldheader flag, + as we do not overwrite clone(). The original clone() does not + set foldheaders. + + So you need to set foldheaders if you want the toplevel to fold headers + + TODO: Python 3.3 is patched against this problems. See issue 1590744 on python bug tracker. + """ + def _write_headers(self, msg): + for h, v in msg.items(): + print >> self._fp, '%s:' % h, + print >> self._fp, v + print >> self._fp diff --git a/src/leap/mail/incoming/service.py b/src/leap/mail/incoming/service.py index 1716816..49bca50 100644 --- a/src/leap/mail/incoming/service.py +++ b/src/leap/mail/incoming/service.py @@ -24,7 +24,6 @@ import time import warnings from email.parser import Parser -from email.generator import Generator from email.utils import parseaddr from email.utils import formatdate from StringIO import StringIO @@ -43,6 +42,7 @@ from leap.common.mail import get_email_charset from leap.keymanager import errors as keymanager_errors from leap.keymanager.openpgp import OpenPGPKey from leap.mail.adaptors import soledad_indexes as fields +from leap.mail.generator import Generator from leap.mail.utils import json_loads, empty from leap.soledad.client import Soledad from leap.soledad.common.crypto import ENC_SCHEME_KEY, ENC_JSON_KEY @@ -394,7 +394,7 @@ class IncomingMail(Service): # ok, this is an incoming message rawmsg = msg.get(self.CONTENT_KEY, None) - if not rawmsg: + if rawmsg is None: return "" return self._maybe_decrypt_msg(rawmsg) @@ -525,8 +525,8 @@ class IncomingMail(Service): return (msg, signkey) d = self._keymanager.decrypt( - encdata, self._userid, OpenPGPKey, - verify=senderAddress) + encdata, self._userid, OpenPGPKey, + verify=senderAddress) d.addCallbacks(build_msg, self._decryption_error, errbackArgs=(msg,)) return d @@ -593,9 +593,10 @@ class IncomingMail(Service): :rtype: Deferred """ msg = copy.deepcopy(origmsg) - data = msg.get_payload()[0].as_string() - detached_sig = msg.get_payload()[1].get_payload() - d = self._keymanager.verify(data, sender_address, OpenPGPKey, detached_sig) + data = self._serialize_msg(msg.get_payload(0)) + detached_sig = self._extract_signature(msg) + d = self._keymanager.verify(data, sender_address, OpenPGPKey, + detached_sig) d.addCallback(lambda sign_key: (msg, sign_key)) d.addErrback(lambda _: (msg, keymanager_errors.InvalidSignature())) @@ -607,6 +608,16 @@ class IncomingMail(Service): g.flatten(origmsg) return buf.getvalue() + def _extract_signature(self, msg): + body = msg.get_payload(0).get_payload() + + if isinstance(body, str): + body = msg.get_payload(0) + + detached_sig = msg.get_payload(1).get_payload() + msg.set_payload(body) + return detached_sig + def _decryption_error(self, failure, msg): """ Check for known decryption errors -- cgit v1.2.3 From 27f1a84897bf60135820a99f92bfdd36e97450e5 Mon Sep 17 00:00:00 2001 From: Folker Bernitt Date: Tue, 9 Feb 2016 10:29:52 +0100 Subject: [tests] fix missing pycryptopp dependency and mock async calls - leap_mail still uses pycryptopp and therefore still needs the dependency - Keymanager calls to async HTTPClient had not been mocked, causing a test to fail - fixed a pep8 warning --- pkg/requirements-latest.pip | 2 ++ src/leap/mail/generator.py | 1 + src/leap/mail/tests/__init__.py | 2 ++ 3 files changed, 5 insertions(+) diff --git a/pkg/requirements-latest.pip b/pkg/requirements-latest.pip index 846a319..f561d4e 100644 --- a/pkg/requirements-latest.pip +++ b/pkg/requirements-latest.pip @@ -1,5 +1,7 @@ --index-url https://pypi.python.org/simple/ +pycryptopp + --allow-external u1db --allow-unverified u1db --allow-external dirspec --allow-unverified dirspec -e 'git+https://github.com/pixelated-project/leap_pycommon.git@develop#egg=leap.common' diff --git a/src/leap/mail/generator.py b/src/leap/mail/generator.py index 28db8da..7028817 100644 --- a/src/leap/mail/generator.py +++ b/src/leap/mail/generator.py @@ -1,5 +1,6 @@ from email.generator import Generator as EmailGenerator + class Generator(EmailGenerator): """ Generates output from a Message object tree, keeping signatures. diff --git a/src/leap/mail/tests/__init__.py b/src/leap/mail/tests/__init__.py index 71452d2..8094c11 100644 --- a/src/leap/mail/tests/__init__.py +++ b/src/leap/mail/tests/__init__.py @@ -94,6 +94,8 @@ class TestCaseWithKeyManager(unittest.TestCase, BaseLeapTest): gpgbinary=self.GPG_BINARY_PATH) self._km._fetcher.put = Mock() self._km._fetcher.get = Mock(return_value=Response()) + self._km._async_client.request = Mock(return_value="") + self._km._async_client_pinned.request = Mock(return_value="") d1 = self._km.put_raw_key(PRIVATE_KEY, OpenPGPKey, ADDRESS) d2 = self._km.put_raw_key(PRIVATE_KEY_2, OpenPGPKey, ADDRESS_2) -- cgit v1.2.3 From aa87f3bb4205b6a756668aac15fe92acb41bf067 Mon Sep 17 00:00:00 2001 From: Ruben Pollan Date: Tue, 9 Feb 2016 17:09:14 +0100 Subject: [style] fix pep8 --- src/leap/mail/generator.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/leap/mail/generator.py b/src/leap/mail/generator.py index 7028817..bb3f26e 100644 --- a/src/leap/mail/generator.py +++ b/src/leap/mail/generator.py @@ -13,7 +13,8 @@ class Generator(EmailGenerator): So you need to set foldheaders if you want the toplevel to fold headers - TODO: Python 3.3 is patched against this problems. See issue 1590744 on python bug tracker. + TODO: Python 3.3 is patched against this problems. See issue 1590744 on + python bug tracker. """ def _write_headers(self, msg): for h, v in msg.items(): -- cgit v1.2.3 From 51e6f7a5eb8cb3e5bd69e4c55c5360b37a6f2111 Mon Sep 17 00:00:00 2001 From: Ruben Pollan Date: Thu, 11 Feb 2016 01:18:11 +0100 Subject: [feat] Remove debug from walk --- src/leap/mail/walk.py | 17 +++-------------- 1 file changed, 3 insertions(+), 14 deletions(-) diff --git a/src/leap/mail/walk.py b/src/leap/mail/walk.py index 1c74366..7be1bb8 100644 --- a/src/leap/mail/walk.py +++ b/src/leap/mail/walk.py @@ -17,20 +17,13 @@ """ Utilities for walking along a message tree. """ -import os - from pycryptopp.hash import sha256 from leap.mail.utils import first -DEBUG = os.environ.get("BITMASK_MAIL_DEBUG") -if DEBUG: - def get_hash(s): - return sha256.SHA256(s).hexdigest()[:10] -else: - def get_hash(s): - return sha256.SHA256(s).hexdigest() +def get_hash(s): + return sha256.SHA256(s).hexdigest() """ @@ -92,7 +85,7 @@ def get_raw_docs(msg, parts): return ( { "type": "cnt", # type content they'll be - "raw": payload if not DEBUG else payload[:100], + "raw": payload, "phash": get_hash(payload), "content-disposition": first(headers.get( 'content-disposition', '').split(';')), @@ -168,10 +161,6 @@ def walk_msg_tree(parts, body_phash=None): inner_headers = parts[1].get(HEADERS, None) if ( len(parts) == 2) else None - if DEBUG: - print "parts vector: ", pv - print - # wrappers vector def getwv(pv): return [ -- cgit v1.2.3 From f2fa5804879f6d7614d05537b042be3586958e7f Mon Sep 17 00:00:00 2001 From: Ruben Pollan Date: Thu, 11 Feb 2016 01:20:33 +0100 Subject: [feat] Use cryptography instead of pycryptopp to reduce dependencies. * Resolves: #7889 --- changes/next-changelog.rst | 1 + pkg/requirements-latest.pip | 2 -- src/leap/mail/adaptors/soledad.py | 3 +-- src/leap/mail/walk.py | 7 +++++-- 4 files changed, 7 insertions(+), 6 deletions(-) diff --git a/changes/next-changelog.rst b/changes/next-changelog.rst index 625dcac..7979246 100644 --- a/changes/next-changelog.rst +++ b/changes/next-changelog.rst @@ -12,6 +12,7 @@ Features ~~~~~~~~ - `#7656 `_: Emit multi-user aware events. - `#4008 `_: Add token-based authentication to local IMAP/SMTP services. +- `#7889 `_: Use cryptography instead of pycryptopp to reduce dependencies. - Use twisted.cred to authenticate IMAP users. - `#1234 `_: Description of the new feature corresponding with issue #1234. diff --git a/pkg/requirements-latest.pip b/pkg/requirements-latest.pip index f561d4e..846a319 100644 --- a/pkg/requirements-latest.pip +++ b/pkg/requirements-latest.pip @@ -1,7 +1,5 @@ --index-url https://pypi.python.org/simple/ -pycryptopp - --allow-external u1db --allow-unverified u1db --allow-external dirspec --allow-unverified dirspec -e 'git+https://github.com/pixelated-project/leap_pycommon.git@develop#egg=leap.common' diff --git a/src/leap/mail/adaptors/soledad.py b/src/leap/mail/adaptors/soledad.py index 7f2b1cf..f4af020 100644 --- a/src/leap/mail/adaptors/soledad.py +++ b/src/leap/mail/adaptors/soledad.py @@ -22,7 +22,6 @@ import re from collections import defaultdict from email import message_from_string -from pycryptopp.hash import sha256 from twisted.internet import defer from twisted.python import log from zope.interface import implements @@ -1208,7 +1207,7 @@ def _split_into_parts(raw): def _parse_msg(raw): msg = message_from_string(raw) parts = walk.get_parts(msg) - chash = sha256.SHA256(raw).hexdigest() + chash = walk.get_hash(raw) multi = msg.is_multipart() return msg, parts, chash, multi diff --git a/src/leap/mail/walk.py b/src/leap/mail/walk.py index 7be1bb8..8693bdd 100644 --- a/src/leap/mail/walk.py +++ b/src/leap/mail/walk.py @@ -17,13 +17,16 @@ """ Utilities for walking along a message tree. """ -from pycryptopp.hash import sha256 +from cryptography.hazmat.backends import default_backend +from cryptography.hazmat.primitives import hashes from leap.mail.utils import first def get_hash(s): - return sha256.SHA256(s).hexdigest() + digest = hashes.Hash(hashes.SHA256(), default_backend()) + digest.update(s) + return digest.finalize().encode("hex").upper() """ -- cgit v1.2.3 From b962e9bc7973f3a826dccfa9aa417b0139b94104 Mon Sep 17 00:00:00 2001 From: Ruben Pollan Date: Tue, 2 Feb 2016 18:10:26 +0100 Subject: [bug] Use the right succeed function for passthrough encrypted email - Resolves #7861 --- changes/next-changelog.rst | 3 ++- src/leap/mail/outgoing/service.py | 2 +- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/changes/next-changelog.rst b/changes/next-changelog.rst index 7979246..67bf940 100644 --- a/changes/next-changelog.rst +++ b/changes/next-changelog.rst @@ -20,9 +20,10 @@ Features Bugfixes ~~~~~~~~ -- `#1235 `_: Description for the fixed stuff corresponding with issue #1235. +- `#7861 `_: Use the right succeed function for passthrough encrypted email. - Fix the get_body logic for corner-cases in which body is None (yet-to-be synced docs, mainly). +- `#1235 `_: Description for the fixed stuff corresponding with issue #1235. - Bugfix without related issue number. Misc diff --git a/src/leap/mail/outgoing/service.py b/src/leap/mail/outgoing/service.py index 8d7c0f8..8e06bd4 100644 --- a/src/leap/mail/outgoing/service.py +++ b/src/leap/mail/outgoing/service.py @@ -254,7 +254,7 @@ class OutgoingMail(object): origmsg = Parser().parsestr(raw) if origmsg.get_content_type() == 'multipart/encrypted': - return defer.success((origmsg, recipient)) + return defer.succeed((origmsg, recipient)) from_address = validate_address(self._from_address) username, domain = from_address.split('@') -- cgit v1.2.3 From 8a828b7a2158c01215c3cbab006caa694a41af03 Mon Sep 17 00:00:00 2001 From: Ruben Pollan Date: Mon, 21 Dec 2015 15:35:56 +0100 Subject: [feat] use fingerprint instead of key_id to address keys --- src/leap/mail/incoming/service.py | 2 +- src/leap/mail/outgoing/service.py | 2 +- src/leap/mail/outgoing/tests/test_outgoing.py | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/src/leap/mail/incoming/service.py b/src/leap/mail/incoming/service.py index 49bca50..98ed416 100644 --- a/src/leap/mail/incoming/service.py +++ b/src/leap/mail/incoming/service.py @@ -461,7 +461,7 @@ class IncomingMail(Service): decrmsg.add_header( self.LEAP_SIGNATURE_HEADER, self.LEAP_SIGNATURE_VALID, - pubkey=signkey.key_id) + pubkey=signkey.fingerprint) return decrmsg.as_string() if msg.get_content_type() == MULTIPART_ENCRYPTED: diff --git a/src/leap/mail/outgoing/service.py b/src/leap/mail/outgoing/service.py index 8e06bd4..eeb5d32 100644 --- a/src/leap/mail/outgoing/service.py +++ b/src/leap/mail/outgoing/service.py @@ -487,7 +487,7 @@ class OutgoingMail(object): def add_openpgp_header(signkey): username, domain = sign_address.split('@') newmsg.add_header( - 'OpenPGP', 'id=%s' % signkey.key_id, + 'OpenPGP', 'id=%s' % signkey.fingerprint, url='https://%s/key/%s' % (domain, username), preference='signencrypt') return newmsg, origmsg diff --git a/src/leap/mail/outgoing/tests/test_outgoing.py b/src/leap/mail/outgoing/tests/test_outgoing.py index 79eafd9..ad7803d 100644 --- a/src/leap/mail/outgoing/tests/test_outgoing.py +++ b/src/leap/mail/outgoing/tests/test_outgoing.py @@ -236,7 +236,7 @@ class TestOutgoingMail(TestCaseWithKeyManager): def _set_sign_used(self, address): def set_sign(key): key.sign_used = True - return self._km.put_key(key, address) + return self._km.put_key(key) d = self._km.get_key(address, openpgp.OpenPGPKey, fetch_remote=False) d.addCallback(set_sign) -- cgit v1.2.3 From df901569fc77d7c990b6da8546a21e7a8957a5e7 Mon Sep 17 00:00:00 2001 From: Kali Kaneko Date: Wed, 9 Mar 2016 12:54:22 -0400 Subject: [bug] specify openssl backend explicitely for some reason, available_backends does not work inside a frozen PyInstaller binary. - Resolves: #7952 --- src/leap/mail/walk.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/src/leap/mail/walk.py b/src/leap/mail/walk.py index 8693bdd..b2aa304 100644 --- a/src/leap/mail/walk.py +++ b/src/leap/mail/walk.py @@ -17,14 +17,16 @@ """ Utilities for walking along a message tree. """ -from cryptography.hazmat.backends import default_backend +from cryptography.hazmat.backends.multibackend import MultiBackend +from cryptography.hazmat.backends.openssl.backend import Backend as OpenSSLBackend from cryptography.hazmat.primitives import hashes from leap.mail.utils import first def get_hash(s): - digest = hashes.Hash(hashes.SHA256(), default_backend()) + backend = MultiBackend([OpenSSLBackend()]) + digest = hashes.Hash(hashes.SHA256(), backend) digest.update(s) return digest.finalize().encode("hex").upper() -- cgit v1.2.3 From 352995b60f3181023e09f378a4652688b24a37f8 Mon Sep 17 00:00:00 2001 From: Kali Kaneko Date: Fri, 11 Mar 2016 15:06:25 -0400 Subject: [style] pep8! --- src/leap/mail/walk.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/leap/mail/walk.py b/src/leap/mail/walk.py index b2aa304..b6fea8d 100644 --- a/src/leap/mail/walk.py +++ b/src/leap/mail/walk.py @@ -18,7 +18,8 @@ Utilities for walking along a message tree. """ from cryptography.hazmat.backends.multibackend import MultiBackend -from cryptography.hazmat.backends.openssl.backend import Backend as OpenSSLBackend +from cryptography.hazmat.backends.openssl.backend import ( + Backend as OpenSSLBackend) from cryptography.hazmat.primitives import hashes from leap.mail.utils import first -- cgit v1.2.3 From 0f1b68a131b57bcde93ee3d6d4d2f20f6b2bcf17 Mon Sep 17 00:00:00 2001 From: Giovane Date: Tue, 5 Jan 2016 17:54:17 -0200 Subject: Fix pixelated repos reference on requirements --- pkg/requirements-latest.pip | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/pkg/requirements-latest.pip b/pkg/requirements-latest.pip index 846a319..3526bbd 100644 --- a/pkg/requirements-latest.pip +++ b/pkg/requirements-latest.pip @@ -1,9 +1,9 @@ --index-url https://pypi.python.org/simple/ ---allow-external u1db --allow-unverified u1db ---allow-external dirspec --allow-unverified dirspec --e 'git+https://github.com/pixelated-project/leap_pycommon.git@develop#egg=leap.common' --e 'git+https://github.com/pixelated-project/soledad.git@develop#egg=leap.soledad.common&subdirectory=common/' --e 'git+https://github.com/pixelated-project/soledad.git@develop#egg=leap.soledad.client&subdirectory=client/' --e 'git+https://github.com/pixelated-project/keymanager.git@develop#egg=leap.keymanager' +https://launchpad.net/dirspec/stable-13-10/13.10/+download/dirspec-13.10.tar.gz +https://launchpad.net/ubuntu/+archive/primary/+files/u1db_13.09.orig.tar.bz2 +-e 'git+https://github.com/pixelated/leap_pycommon.git@develop#egg=leap.common' +-e 'git+https://github.com/pixelated/soledad.git@develop#egg=leap.soledad.common&subdirectory=common/' +-e 'git+https://github.com/pixelated/soledad.git@develop#egg=leap.soledad.client&subdirectory=client/' +-e 'git+https://github.com/pixelated/keymanager.git@develop#egg=leap.keymanager' -e . -- cgit v1.2.3 From e57f215f2af9b8b3d155c6a8992973e327a8b7b7 Mon Sep 17 00:00:00 2001 From: Ruben Pollan Date: Wed, 16 Mar 2016 18:02:04 +0100 Subject: [bug] Fix IMAP fetch headers - Resolves: #7898 --- changes/next-changelog.rst | 1 + src/leap/mail/imap/mailbox.py | 22 ++++++++++++---------- 2 files changed, 13 insertions(+), 10 deletions(-) diff --git a/changes/next-changelog.rst b/changes/next-changelog.rst index 67bf940..40efb19 100644 --- a/changes/next-changelog.rst +++ b/changes/next-changelog.rst @@ -21,6 +21,7 @@ Features Bugfixes ~~~~~~~~ - `#7861 `_: Use the right succeed function for passthrough encrypted email. +- `#7898 `_: Fix IMAP fetch headers - Fix the get_body logic for corner-cases in which body is None (yet-to-be synced docs, mainly). - `#1235 `_: Description for the fixed stuff corresponding with issue #1235. diff --git a/src/leap/mail/imap/mailbox.py b/src/leap/mail/imap/mailbox.py index bfc0bfc..d545c00 100644 --- a/src/leap/mail/imap/mailbox.py +++ b/src/leap/mail/imap/mailbox.py @@ -508,7 +508,7 @@ class IMAPMailbox(object): def get_range(messages_asked): return self._filter_msg_seq(messages_asked) - d = defer.maybeDeferred(self._bound_seq, messages_asked, uid) + d = self._bound_seq(messages_asked, uid) if uid: d.addCallback(get_range) d.addErrback(lambda f: log.err(f)) @@ -520,7 +520,7 @@ class IMAPMailbox(object): :param messages_asked: IDs of the messages. :type messages_asked: MessageSet - :rtype: MessageSet + :return: a Deferred that will fire with a MessageSet """ def set_last_uid(last_uid): @@ -543,7 +543,7 @@ class IMAPMailbox(object): d = self.collection.all_uid_iter() d.addCallback(set_last_seq) return d - return messages_asked + return defer.succeed(messages_asked) def _filter_msg_seq(self, messages_asked): """ @@ -713,6 +713,7 @@ class IMAPMailbox(object): d_seq.addCallback(get_flags_for_seq) return d_seq + @defer.inlineCallbacks def fetch_headers(self, messages_asked, uid): """ A fast method to fetch all headers, tricking just the @@ -757,14 +758,15 @@ class IMAPMailbox(object): for key, value in self.headers.items()) - messages_asked = self._bound_seq(messages_asked) - seq_messg = self._filter_msg_seq(messages_asked) + messages_asked = yield self._bound_seq(messages_asked, uid) + seq_messg = yield self._filter_msg_seq(messages_asked) - all_headers = self.messages.all_headers() - result = ((msgid, headersPart( - msgid, all_headers.get(msgid, {}))) - for msgid in seq_messg) - return result + result = [] + for msgid in seq_messg: + msg = yield self.collection.get_message_by_uid(msgid) + headers = headersPart(msgid, msg.get_headers()) + result.append((msgid, headers)) + defer.returnValue(iter(result)) def store(self, messages_asked, flags, mode, uid): """ -- cgit v1.2.3 From 13927ac178c00b729d8d660107f72d878879a5c3 Mon Sep 17 00:00:00 2001 From: Ruben Pollan Date: Sun, 20 Mar 2016 19:52:46 +0100 Subject: [bug] Decode attached keys so they are recognized by keymanager - Resolves: #7977 --- changes/next-changelog.rst | 1 + src/leap/mail/incoming/service.py | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/changes/next-changelog.rst b/changes/next-changelog.rst index 40efb19..f89af89 100644 --- a/changes/next-changelog.rst +++ b/changes/next-changelog.rst @@ -22,6 +22,7 @@ Bugfixes ~~~~~~~~ - `#7861 `_: Use the right succeed function for passthrough encrypted email. - `#7898 `_: Fix IMAP fetch headers +- `#7977 `_: Decode attached keys so they are recognized by keymanager. - Fix the get_body logic for corner-cases in which body is None (yet-to-be synced docs, mainly). - `#1235 `_: Description for the fixed stuff corresponding with issue #1235. diff --git a/src/leap/mail/incoming/service.py b/src/leap/mail/incoming/service.py index 98ed416..c7d194d 100644 --- a/src/leap/mail/incoming/service.py +++ b/src/leap/mail/incoming/service.py @@ -749,7 +749,7 @@ class IncomingMail(Service): for attachment in attachments: if MIME_KEY == attachment.get_content_type(): d = self._keymanager.put_raw_key( - attachment.get_payload(), + attachment.get_payload(decode=True), OpenPGPKey, address=address) d.addCallbacks(log_key_added, failed_put_key) -- cgit v1.2.3 From 7c9082152155100b171450f54c56c614104175df Mon Sep 17 00:00:00 2001 From: Kali Kaneko Date: Wed, 23 Mar 2016 17:48:36 -0400 Subject: [bug] emit imap-login event again this was gone with the imap/cred refactor, but the client relies on it to hide the 'congratulations!' welcome display on the mail widget. --- src/leap/mail/imap/server.py | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/src/leap/mail/imap/server.py b/src/leap/mail/imap/server.py index b6f1e47..0397337 100644 --- a/src/leap/mail/imap/server.py +++ b/src/leap/mail/imap/server.py @@ -29,6 +29,8 @@ from twisted.internet import defer, interfaces from twisted.mail.imap4 import IllegalClientResponse from twisted.mail.imap4 import LiteralString, LiteralFile +from leap.common.events import emit_async, catalog + def _getContentType(msg): """ @@ -609,7 +611,6 @@ class LEAPIMAPServer(imap4.IMAP4Server): d.addCallback(send_response) return d # XXX patched --------------------------------- - # ----------------------------------------------------------------------- auth_APPEND = (do_APPEND, arg_astring, imap4.IMAP4Server.opt_plist, @@ -685,3 +686,9 @@ class LEAPIMAPServer(imap4.IMAP4Server): ############################################################# # END of Twisted imap4 patch to support LITERAL+ extension ############################################################# + + def authenticateLogin(self, user, passwd): + result = imap4.IMAP4Server.authenticateLogin(self, user, passwd) + emit_async(catalog.IMAP_CLIENT_LOGIN, str(user)) + return result + -- cgit v1.2.3 From 49ba2965434c6f771e7946899901c594beed8908 Mon Sep 17 00:00:00 2001 From: Kali Kaneko Date: Wed, 23 Mar 2016 17:50:21 -0400 Subject: [bug] let the inbox used in IncomingMail notify any subscribed Mailbox the mail service uses an Account object created from scratch, so it wasn't sharing the collections mapping with the other Account object that is created in the IMAP Service. I make it a class attribute to allow mailbox notifications. However, with the transition to a single service tree, this class attribute can again become a class instance. This is somehow related to a PR proposed recently by cz8s in pixelated team: https://github.com/leapcode/leap_mail/pull/228 However, I'm reluctant to re-use IMAPMailbox instances, since they represent concurrent views over the same collection. I believe that sharing the same underlying collection might be enough. --- src/leap/mail/mail.py | 31 ++++++++++++++++++++----------- 1 file changed, 20 insertions(+), 11 deletions(-) diff --git a/src/leap/mail/mail.py b/src/leap/mail/mail.py index c0e16a6..b9c97f6 100644 --- a/src/leap/mail/mail.py +++ b/src/leap/mail/mail.py @@ -29,6 +29,8 @@ import StringIO import time import weakref +from collections import defaultdict + from twisted.internet import defer from twisted.python import log @@ -924,19 +926,25 @@ class Account(object): adaptor_class = SoledadMailAdaptor + # this is a defaultdict, indexed by userid, that returns a + # WeakValueDictionary mapping to collection instances so that we always + # return a reference to them instead of creating new ones. however, + # being a dictionary of weakrefs values, they automagically vanish + # from the dict when no hard refs is left to them (so they can be + # garbage collected) this is important because the different wrappers + # rely on several kinds of deferredlocks that are kept as class or + # instance variables. + + # We need it to be a class property because we create more than one Account + # object in the current usage pattern (ie, one in the mail service, and + # another one in the IncomingMailService). When we move to a proper service + # tree we can let it be an instance attribute. + _collection_mapping = defaultdict(weakref.WeakValueDictionary) + def __init__(self, store, ready_cb=None): self.store = store self.adaptor = self.adaptor_class() - # this is a mapping to collection instances so that we always - # return a reference to them instead of creating new ones. however, - # being a dictionary of weakrefs values, they automagically vanish - # from the dict when no hard refs is left to them (so they can be - # garbage collected) this is important because the different wrappers - # rely on several kinds of deferredlocks that are kept as class or - # instance variables - self._collection_mapping = weakref.WeakValueDictionary() - self.mbox_indexer = MailboxIndexer(self.store) # This flag is only used from the imap service for the moment. @@ -1069,7 +1077,8 @@ class Account(object): :rtype: deferred :return: a deferred that will fire with a MessageCollection """ - collection = self._collection_mapping.get(name, None) + collection = self._collection_mapping[self.store.userid].get( + name, None) if collection: return defer.succeed(collection) @@ -1077,7 +1086,7 @@ class Account(object): def get_collection_for_mailbox(mbox_wrapper): collection = MessageCollection( self.adaptor, self.store, self.mbox_indexer, mbox_wrapper) - self._collection_mapping[name] = collection + self._collection_mapping[self.store.userid][name] = collection return collection d = self.adaptor.get_or_create_mbox(self.store, name) -- cgit v1.2.3 From bbc29c8f3d018d5f13e358fa7509ff1d71b64015 Mon Sep 17 00:00:00 2001 From: Kali Kaneko Date: Wed, 23 Mar 2016 18:14:53 -0400 Subject: [bug] Fix unread mails notification this one was missing after the events refactor. the bug is that client was discarding the first parameter, assuming it was the userid. --- src/leap/mail/mail.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/leap/mail/mail.py b/src/leap/mail/mail.py index b9c97f6..c6e053c 100644 --- a/src/leap/mail/mail.py +++ b/src/leap/mail/mail.py @@ -746,7 +746,7 @@ class MessageCollection(object): :param unseen: number of unseen messages. :type unseen: int """ - emit_async(catalog.MAIL_UNREAD_MESSAGES, str(unseen)) + emit_async(catalog.MAIL_UNREAD_MESSAGES, self.store.uuid, str(unseen)) def copy_msg(self, msg, new_mbox_uuid): """ -- cgit v1.2.3 From 303df4aceba4d9dfff74ec4024373cbadae36d75 Mon Sep 17 00:00:00 2001 From: Kali Kaneko Date: Wed, 30 Mar 2016 11:23:12 -0400 Subject: [style] pep8 --- src/leap/mail/imap/server.py | 1 - 1 file changed, 1 deletion(-) diff --git a/src/leap/mail/imap/server.py b/src/leap/mail/imap/server.py index 0397337..5a63af0 100644 --- a/src/leap/mail/imap/server.py +++ b/src/leap/mail/imap/server.py @@ -691,4 +691,3 @@ class LEAPIMAPServer(imap4.IMAP4Server): result = imap4.IMAP4Server.authenticateLogin(self, user, passwd) emit_async(catalog.IMAP_CLIENT_LOGIN, str(user)) return result - -- cgit v1.2.3 From f05b0ec45ac667e85a610219d77906f049eb58cc Mon Sep 17 00:00:00 2001 From: Kali Kaneko Date: Tue, 29 Mar 2016 17:21:43 -0400 Subject: [feature] SMTP delivery bounces We catch any error on SMTP delivery and format it as a bounce message delivered to the user Inbox. this doesn't comply with the bounce format, but it's a nice first start. leaving proper structuring of the delivery failure report for future iterations. - Resolves: #7263 --- changes/next-changelog.rst | 1 + src/leap/mail/outgoing/service.py | 44 +++++++++++++------ src/leap/mail/smtp/bounces.py | 89 +++++++++++++++++++++++++++++++++++++++ src/leap/mail/smtp/gateway.py | 47 +++++++++++++++++---- 4 files changed, 160 insertions(+), 21 deletions(-) create mode 100644 src/leap/mail/smtp/bounces.py diff --git a/changes/next-changelog.rst b/changes/next-changelog.rst index f89af89..9b2a9d6 100644 --- a/changes/next-changelog.rst +++ b/changes/next-changelog.rst @@ -13,6 +13,7 @@ Features - `#7656 `_: Emit multi-user aware events. - `#4008 `_: Add token-based authentication to local IMAP/SMTP services. - `#7889 `_: Use cryptography instead of pycryptopp to reduce dependencies. +- `#7263 `_: Implement local bounces to notify user of SMTP delivery errors. - Use twisted.cred to authenticate IMAP users. - `#1234 `_: Description of the new feature corresponding with issue #1234. diff --git a/src/leap/mail/outgoing/service.py b/src/leap/mail/outgoing/service.py index eeb5d32..335cae4 100644 --- a/src/leap/mail/outgoing/service.py +++ b/src/leap/mail/outgoing/service.py @@ -73,7 +73,7 @@ class SSLContextFactory(ssl.ClientContextFactory): return ctx -def outgoingFactory(userid, keymanager, opts, check_cert=True): +def outgoingFactory(userid, keymanager, opts, check_cert=True, bouncer=None): cert = unicode(opts.cert) key = unicode(opts.key) @@ -85,7 +85,9 @@ def outgoingFactory(userid, keymanager, opts, check_cert=True): raise errors.ConfigurationError( 'No valid SMTP certificate could be found for %s!' % userid) - return OutgoingMail(str(userid), keymanager, cert, key, hostname, port) + return OutgoingMail( + str(userid), keymanager, cert, key, hostname, port, + bouncer) class OutgoingMail(object): @@ -93,7 +95,8 @@ class OutgoingMail(object): Sends Outgoing Mail, encrypting and signing if needed. """ - def __init__(self, from_address, keymanager, cert, key, host, port): + def __init__(self, from_address, keymanager, cert, key, host, port, + bouncer=None): """ Initialize the outgoing mail service. @@ -133,6 +136,7 @@ class OutgoingMail(object): self._cert = cert self._from_address = from_address self._keymanager = keymanager + self._bouncer = bouncer def send_message(self, raw, recipient): """ @@ -145,8 +149,8 @@ class OutgoingMail(object): :return: a deferred which delivers the message when fired """ d = self._maybe_encrypt_and_sign(raw, recipient) - d.addCallback(self._route_msg) - d.addErrback(self.sendError) + d.addCallback(self._route_msg, raw) + d.addErrback(self.sendError, raw) return d def sendSuccess(self, smtp_sender_result): @@ -163,21 +167,36 @@ class OutgoingMail(object): emit_async(catalog.SMTP_SEND_MESSAGE_SUCCESS, fromaddr, dest_addrstr) - def sendError(self, failure): + def sendError(self, failure, origmsg): """ Callback for an unsuccessfull send. - :param e: The result from the last errback. - :type e: anything + :param failure: The result from the last errback. + :type failure: anything + :param origmsg: the original, unencrypted, raw message, to be passed to + the bouncer. + :type origmsg: str """ - # XXX: need to get the address from the exception to send signal + # XXX: need to get the address from the original message to send signal # emit_async(catalog.SMTP_SEND_MESSAGE_ERROR, self._from_address, # self._user.dest.addrstr) + + # TODO when we implement outgoing queues/long-term-retries, we could + # examine the error *here* and delay the notification if it's just a + # temporal error. We might want to notify the permanent errors + # differently. + err = failure.value log.err(err) - raise err - def _route_msg(self, encrypt_and_sign_result): + if self._bouncer: + self._bouncer.bounce_message( + err.message, to=self._from_address, + orig=origmsg) + else: + raise err + + def _route_msg(self, encrypt_and_sign_result, raw): """ Sends the msg using the ESMTPSenderFactory. @@ -191,7 +210,8 @@ class OutgoingMail(object): # we construct a defer to pass to the ESMTPSenderFactory d = defer.Deferred() - d.addCallbacks(self.sendSuccess, self.sendError) + d.addCallback(self.sendSuccess) + d.addErrback(self.sendError, raw) # we don't pass an ssl context factory to the ESMTPSenderFactory # because ssl will be handled by reactor.connectSSL() below. factory = smtp.ESMTPSenderFactory( diff --git a/src/leap/mail/smtp/bounces.py b/src/leap/mail/smtp/bounces.py new file mode 100644 index 0000000..64f2dd7 --- /dev/null +++ b/src/leap/mail/smtp/bounces.py @@ -0,0 +1,89 @@ +# -*- coding: utf-8 -*- +# bounces.py +# Copyright (C) 2016 LEAP +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . +""" +Deliver bounces to the user Inbox. +""" +import time +from email.message import Message +from email.utils import formatdate + +from leap.mail.constants import INBOX_NAME +from leap.mail.mail import Account + + +# TODO implement localization for this template. + +BOUNCE_TEMPLATE = """This is your local Bitmask Mail Agent running at localhost. + +I'm sorry to have to inform you that your message could not be delivered to one +or more recipients. + +The reasons I got for the error are: + +{raw_error} + +If the problem persists and it's not a network connectivity issue, you might +want to contact your provider ({provider}) with this information (remove any +sensitive data before). + +--- Original message (*before* it was encrypted by bitmask) below ----: + +{orig}""" + + +class Bouncer(object): + """ + Implements a mechanism to deliver bounces to user inbox. + """ + # TODO this should follow RFC 6522, and compose a correct multipart + # attaching the report and the original message. Leaving this for a future + # iteration. + + def __init__(self, inbox_collection): + self._inbox_collection = inbox_collection + + def bounce_message(self, error_data, to, date=None, orig=''): + if not date: + date = formatdate(time.time()) + + raw_data = self._format_msg(error_data, to, date, orig) + d = self._inbox_collection.add_msg( + raw_data, ('\\Recent',), date=date) + return d + + def _format_msg(self, error_data, to, date, orig): + provider = to.split('@')[1] + + msg = Message() + msg.add_header( + 'From', 'bitmask-bouncer@localhost (Bitmask Local Agent)') + msg.add_header('To', to) + msg.add_header('Subject', 'Undelivered Message') + msg.add_header('Date', date) + msg.set_payload(BOUNCE_TEMPLATE.format( + raw_error=error_data, + provider=provider, + orig=orig)) + + return msg.as_string() + + +def bouncerFactory(soledad): + acc = Account(soledad) + d = acc.callWhenReady(lambda _: acc.get_collection_by_mailbox(INBOX_NAME)) + d.addCallback(lambda inbox: Bouncer(inbox)) + return d diff --git a/src/leap/mail/smtp/gateway.py b/src/leap/mail/smtp/gateway.py index 7ff6b14..cb1b060 100644 --- a/src/leap/mail/smtp/gateway.py +++ b/src/leap/mail/smtp/gateway.py @@ -29,7 +29,6 @@ The following classes comprise the SMTP gateway service: * EncryptedMessage - An implementation of twisted.mail.smtp.IMessage that knows how to encrypt/sign itself before sending. """ - from email.Header import Header from zope.interface import implements @@ -48,6 +47,7 @@ from leap.mail.cred import LocalSoledadTokenChecker from leap.mail.utils import validate_address from leap.mail.rfc3156 import RFC3156CompliantGenerator from leap.mail.outgoing.service import outgoingFactory +from leap.mail.smtp.bounces import bouncerFactory from leap.keymanager.openpgp import OpenPGPKey from leap.keymanager.errors import KeyNotFound @@ -65,7 +65,7 @@ class LocalSMTPRealm(object): _encoding = 'utf-8' - def __init__(self, keymanager_sessions, sendmail_opts, + def __init__(self, keymanager_sessions, soledad_sessions, sendmail_opts, encrypted_only=False): """ :param keymanager_sessions: a dict-like object, containing instances @@ -73,21 +73,31 @@ class LocalSMTPRealm(object): userid. """ self._keymanager_sessions = keymanager_sessions + self._soledad_sessions = soledad_sessions self._sendmail_opts = sendmail_opts self.encrypted_only = encrypted_only def requestAvatar(self, avatarId, mind, *interfaces): + if isinstance(avatarId, str): avatarId = avatarId.decode(self._encoding) - def gotKeymanager(keymanager): + def gotKeymanagerAndSoledad(result): + keymanager, soledad = result + d = bouncerFactory(soledad) + d.addCallback(lambda bouncer: (keymanager, soledad, bouncer)) + return d + def getMessageDelivery(result): + keymanager, soledad, bouncer = result # TODO use IMessageDeliveryFactory instead ? # it could reuse the connections. if smtp.IMessageDelivery in interfaces: userid = avatarId opts = self.getSendingOpts(userid) - outgoing = outgoingFactory(userid, keymanager, opts) + + outgoing = outgoingFactory( + userid, keymanager, opts, bouncer=bouncer) avatar = SMTPDelivery(userid, keymanager, self.encrypted_only, outgoing) @@ -96,10 +106,15 @@ class LocalSMTPRealm(object): raise NotImplementedError(self, interfaces) - return self.lookupKeymanagerInstance(avatarId).addCallback( - gotKeymanager) + d1 = self.lookupKeymanagerInstance(avatarId) + d2 = self.lookupSoledadInstance(avatarId) + d = defer.gatherResults([d1, d2]) + d.addCallback(gotKeymanagerAndSoledad) + d.addCallback(getMessageDelivery) + return d def lookupKeymanagerInstance(self, userid): + print 'getting KM INSTNACE>>>' try: keymanager = self._keymanager_sessions[userid] except: @@ -109,6 +124,16 @@ class LocalSMTPRealm(object): # XXX this should return the instance after whenReady callback return defer.succeed(keymanager) + def lookupSoledadInstance(self, userid): + try: + soledad = self._soledad_sessions[userid] + except: + raise errors.AuthenticationError( + 'No soledad session found for user %s. Is it authenticated?' + % userid) + # XXX this should return the instance after whenReady callback + return defer.succeed(soledad) + def getSendingOpts(self, userid): try: opts = self._sendmail_opts[userid] @@ -134,8 +159,9 @@ class LEAPInitMixin(object): """ def __init__(self, soledad_sessions, keymanager_sessions, sendmail_opts, encrypted_only=False): - realm = LocalSMTPRealm(keymanager_sessions, sendmail_opts, - encrypted_only) + realm = LocalSMTPRealm( + keymanager_sessions, soledad_sessions, sendmail_opts, + encrypted_only) portal = Portal(realm) checker = SMTPTokenChecker(soledad_sessions) @@ -161,6 +187,7 @@ class LocalSMTPServer(smtp.ESMTP, LEAPInitMixin): smtp.ESMTP.__init__(self, *args, **kw) +# TODO implement retries -- see smtp.SenderMixin class SMTPFactory(protocol.ServerFactory): """ Factory for an SMTP server with encrypted gatewaying capabilities. @@ -171,7 +198,9 @@ class SMTPFactory(protocol.ServerFactory): timeout = 600 encrypted_only = False - def __init__(self, soledad_sessions, keymanager_sessions, sendmail_opts): + def __init__(self, soledad_sessions, keymanager_sessions, sendmail_opts, + deferred=None, retries=3): + self._soledad_sessions = soledad_sessions self._keymanager_sessions = keymanager_sessions self._sendmail_opts = sendmail_opts -- cgit v1.2.3 From 727a14957928a4d6f99d70a5a8521a6fe183c70d Mon Sep 17 00:00:00 2001 From: Kali Kaneko Date: Fri, 1 Apr 2016 13:07:18 -0400 Subject: [pkg] update to versioneer 0.16 --- MANIFEST.in | 1 + setup.cfg | 7 + setup.py | 42 +- src/leap/mail/_version.py | 549 +++++++++--- versioneer.py | 2053 ++++++++++++++++++++++++++++++++++----------- 5 files changed, 2020 insertions(+), 632 deletions(-) diff --git a/MANIFEST.in b/MANIFEST.in index 83264d4..1821bf4 100644 --- a/MANIFEST.in +++ b/MANIFEST.in @@ -3,3 +3,4 @@ include versioneer.py include LICENSE include CHANGELOG include README.rst +include src/leap/mail/_version.py diff --git a/setup.cfg b/setup.cfg index 51070c6..501ecf1 100644 --- a/setup.cfg +++ b/setup.cfg @@ -8,3 +8,10 @@ ignore = E731 [flake8] exclude = versioneer.py,_version.py,*.egg,build,docs ignore = E731 + +[versioneer] +VCS = git +style = pep440 +versionfile_source = src/leap/mail/_version.py +versionfile_build = leap/mail/_version.py +tag_prefix = diff --git a/setup.py b/setup.py index 575a6ec..e9c3e41 100644 --- a/setup.py +++ b/setup.py @@ -26,11 +26,6 @@ from pkg import utils import versioneer -versioneer.versionfile_source = 'src/leap/mail/_version.py' -versioneer.versionfile_build = 'leap/mail/_version.py' -versioneer.tag_prefix = '' # tags are like 1.2.0 -versioneer.parentdir_prefix = 'leap.mail-' - trove_classifiers = [ 'Development Status :: 4 - Beta', @@ -54,7 +49,7 @@ DOWNLOAD_BASE = ('https://github.com/leapcode/leap_mail/' 'archive/%s.tar.gz') _versions = versioneer.get_versions() VERSION = _versions['version'] -VERSION_FULL = _versions['full'] +VERSION_REVISION = _versions['full-revisionid'] DOWNLOAD_URL = "" # get the short version for the download url @@ -72,6 +67,22 @@ class freeze_debianver(Command): To be used after merging the development branch onto the debian one. """ user_options = [] + template = r""" +# This file was generated by the `freeze_debianver` command in setup.py +# Using 'versioneer.py' (0.16) from +# revision-control system data, or from the parent directory name of an +# unpacked source archive. Distribution tarballs contain a pre-generated copy +# of this file. + +version_version = '{version}' +full_revisionid = '{full_revisionid}' +""" + templatefun = r""" + +def get_versions(default={}, verbose=False): + return {'version': version_version, + 'full-revisionid': full_revisionid} +""" def initialize_options(self): pass @@ -85,24 +96,9 @@ class freeze_debianver(Command): if proceed != "y": print("He. You scared. Aborting.") return - template = r""" -# This file was generated by the `freeze_debianver` command in setup.py -# Using 'versioneer.py' (0.7+) from -# revision-control system data, or from the parent directory name of an -# unpacked source archive. Distribution tarballs contain a pre-generated copy -# of this file. - -version_version = '{version}' -version_full = '{version_full}' -""" - templatefun = r""" - -def get_versions(default={}, verbose=False): - return {'version': version_version, 'full': version_full} -""" - subst_template = template.format( + subst_template = self.template.format( version=VERSION_SHORT, - version_full=VERSION_FULL) + templatefun + version_full=VERSION_REVISION) + self.templatefun with open(versioneer.versionfile_source, 'w') as f: f.write(subst_template) diff --git a/src/leap/mail/_version.py b/src/leap/mail/_version.py index b77d552..954f488 100644 --- a/src/leap/mail/_version.py +++ b/src/leap/mail/_version.py @@ -1,72 +1,157 @@ -import subprocess -import sys -import re -import os.path -IN_LONG_VERSION_PY = True # This file helps to compute a version number in source trees obtained from # git-archive tarball (such as those provided by githubs download-from-tag -# feature). Distribution tarballs (build by setup.py sdist) and build +# feature). Distribution tarballs (built by setup.py sdist) and build # directories (produced by setup.py build) will contain a much shorter file # that just contains the computed version number. # This file is released into the public domain. Generated by -# versioneer-0.7+ (https://github.com/warner/python-versioneer) +# versioneer-0.16 (https://github.com/warner/python-versioneer) -# these strings will be replaced by git during git-archive -git_refnames = "$Format:%d$" -git_full = "$Format:%H$" +"""Git implementation of _version.py.""" +import errno +import os +import re +import subprocess +import sys -def run_command(args, cwd=None, verbose=False): - try: - # remember shell=False, so use git.cmd on windows, not just git - p = subprocess.Popen(args, stdout=subprocess.PIPE, cwd=cwd) - except EnvironmentError: - e = sys.exc_info()[1] + +def get_keywords(): + """Get the keywords needed to look up the version information.""" + # these strings will be replaced by git during git-archive. + # setup.py/versioneer.py will grep for the variable names, so they must + # each be defined on a line of their own. _version.py will just call + # get_keywords(). + git_refnames = "$Format:%d$" + git_full = "$Format:%H$" + keywords = {"refnames": git_refnames, "full": git_full} + return keywords + + +class VersioneerConfig: + """Container for Versioneer configuration parameters.""" + + +def get_config(): + """Create, populate and return the VersioneerConfig() object.""" + # these strings are filled in when 'setup.py versioneer' creates + # _version.py + cfg = VersioneerConfig() + cfg.VCS = "git" + cfg.style = "pep440" + cfg.tag_prefix = "" + cfg.parentdir_prefix = "None" + cfg.versionfile_source = "src/leap/mail/_version.py" + cfg.verbose = False + return cfg + + +class NotThisMethod(Exception): + """Exception raised if a method is not valid for the current scenario.""" + + +LONG_VERSION_PY = {} +HANDLERS = {} + + +def register_vcs_handler(vcs, method): # decorator + """Decorator to mark a method as the handler for a particular VCS.""" + def decorate(f): + """Store f in HANDLERS[vcs][method].""" + if vcs not in HANDLERS: + HANDLERS[vcs] = {} + HANDLERS[vcs][method] = f + return f + return decorate + + +def run_command(commands, args, cwd=None, verbose=False, hide_stderr=False): + """Call the given command(s).""" + assert isinstance(commands, list) + p = None + for c in commands: + try: + dispcmd = str([c] + args) + # remember shell=False, so use git.cmd on windows, not just git + p = subprocess.Popen([c] + args, cwd=cwd, stdout=subprocess.PIPE, + stderr=(subprocess.PIPE if hide_stderr + else None)) + break + except EnvironmentError: + e = sys.exc_info()[1] + if e.errno == errno.ENOENT: + continue + if verbose: + print("unable to run %s" % dispcmd) + print(e) + return None + else: if verbose: - print("unable to run %s" % args[0]) - print(e) + print("unable to find command, tried %s" % (commands,)) return None stdout = p.communicate()[0].strip() - if sys.version >= '3': + if sys.version_info[0] >= 3: stdout = stdout.decode() if p.returncode != 0: if verbose: - print("unable to run %s (error)" % args[0]) + print("unable to run %s (error)" % dispcmd) return None return stdout -def get_expanded_variables(versionfile_source): +def versions_from_parentdir(parentdir_prefix, root, verbose): + """Try to determine the version from the parent directory name. + + Source tarballs conventionally unpack into a directory that includes + both the project name and a version string. + """ + dirname = os.path.basename(root) + if not dirname.startswith(parentdir_prefix): + if verbose: + print("guessing rootdir is '%s', but '%s' doesn't start with " + "prefix '%s'" % (root, dirname, parentdir_prefix)) + raise NotThisMethod("rootdir doesn't start with parentdir_prefix") + return {"version": dirname[len(parentdir_prefix):], + "full-revisionid": None, + "dirty": False, "error": None} + + +@register_vcs_handler("git", "get_keywords") +def git_get_keywords(versionfile_abs): + """Extract version information from the given file.""" # the code embedded in _version.py can just fetch the value of these - # variables. When used from setup.py, we don't want to import - # _version.py, so we do it with a regexp instead. This function is not - # used from _version.py. - variables = {} + # keywords. When used from setup.py, we don't want to import _version.py, + # so we do it with a regexp instead. This function is not used from + # _version.py. + keywords = {} try: - f = open(versionfile_source, "r") + f = open(versionfile_abs, "r") for line in f.readlines(): if line.strip().startswith("git_refnames ="): mo = re.search(r'=\s*"(.*)"', line) if mo: - variables["refnames"] = mo.group(1) + keywords["refnames"] = mo.group(1) if line.strip().startswith("git_full ="): mo = re.search(r'=\s*"(.*)"', line) if mo: - variables["full"] = mo.group(1) + keywords["full"] = mo.group(1) f.close() except EnvironmentError: pass - return variables + return keywords -def versions_from_expanded_variables(variables, tag_prefix, verbose=False): - refnames = variables["refnames"].strip() +@register_vcs_handler("git", "keywords") +def git_versions_from_keywords(keywords, tag_prefix, verbose): + """Get version information from git keywords.""" + if not keywords: + raise NotThisMethod("no keywords at all, weird") + refnames = keywords["refnames"].strip() if refnames.startswith("$Format"): if verbose: - print("variables are unexpanded, not using") - return {} # unexpanded, so not in an unpacked git-archive tarball + print("keywords are unexpanded, not using") + raise NotThisMethod("unexpanded keywords, not a git-archive tarball") refs = set([r.strip() for r in refnames.strip("()").split(",")]) # starting in git-1.8.3, tags are listed as "tag: foo-1.0" instead of # just "foo-1.0". If we see a "tag: " prefix, prefer those. @@ -82,7 +167,7 @@ def versions_from_expanded_variables(variables, tag_prefix, verbose=False): # "stabilization", as well as "HEAD" and "master". tags = set([r for r in refs if re.search(r'\d', r)]) if verbose: - print("discarding '%s', no digits" % ",".join(refs - tags)) + print("discarding '%s', no digits" % ",".join(refs-tags)) if verbose: print("likely tags: %s" % ",".join(sorted(tags))) for ref in sorted(tags): @@ -92,114 +177,308 @@ def versions_from_expanded_variables(variables, tag_prefix, verbose=False): if verbose: print("picking %s" % r) return {"version": r, - "full": variables["full"].strip()} - # no suitable tags, so we use the full revision id + "full-revisionid": keywords["full"].strip(), + "dirty": False, "error": None + } + # no suitable tags, so version is "0+unknown", but full hex is still there if verbose: - print("no suitable tags, using full revision id") - return {"version": variables["full"].strip(), - "full": variables["full"].strip()} - - -def versions_from_vcs(tag_prefix, versionfile_source, verbose=False): - # this runs 'git' from the root of the source tree. That either means - # someone ran a setup.py command (and this code is in versioneer.py, so - # IN_LONG_VERSION_PY=False, thus the containing directory is the root of - # the source tree), or someone ran a project-specific entry point (and - # this code is in _version.py, so IN_LONG_VERSION_PY=True, thus the - # containing directory is somewhere deeper in the source tree). This only - # gets called if the git-archive 'subst' variables were *not* expanded, - # and _version.py hasn't already been rewritten with a short version - # string, meaning we're inside a checked out source tree. + print("no suitable tags, using unknown + full revision id") + return {"version": "0+unknown", + "full-revisionid": keywords["full"].strip(), + "dirty": False, "error": "no suitable tags"} - try: - here = os.path.abspath(__file__) - except NameError: - # some py2exe/bbfreeze/non-CPython implementations don't do __file__ - return {} # not always correct - - # versionfile_source is the relative path from the top of the source tree - # (where the .git directory might live) to this file. Invert this to find - # the root from __file__. - root = here - if IN_LONG_VERSION_PY: - for i in range(len(versionfile_source.split("/"))): - root = os.path.dirname(root) - else: - root = os.path.dirname(here) + +@register_vcs_handler("git", "pieces_from_vcs") +def git_pieces_from_vcs(tag_prefix, root, verbose, run_command=run_command): + """Get version from 'git describe' in the root of the source tree. + + This only gets called if the git-archive 'subst' keywords were *not* + expanded, and _version.py hasn't already been rewritten with a short + version string, meaning we're inside a checked out source tree. + """ if not os.path.exists(os.path.join(root, ".git")): if verbose: print("no .git in %s" % root) - return {} + raise NotThisMethod("no .git directory") - GIT = "git" + GITS = ["git"] if sys.platform == "win32": - GIT = "git.cmd" - stdout = run_command([GIT, "describe", "--tags", "--dirty", "--always"], - cwd=root) - if stdout is None: - return {} - if not stdout.startswith(tag_prefix): - if verbose: - print("tag '%s' doesn't start with prefix '%s'" % - (stdout, tag_prefix)) - return {} - tag = stdout[len(tag_prefix):] - stdout = run_command([GIT, "rev-parse", "HEAD"], cwd=root) - if stdout is None: - return {} - full = stdout.strip() - if tag.endswith("-dirty"): - full += "-dirty" - return {"version": tag, "full": full} - - -def versions_from_parentdir(parentdir_prefix, versionfile_source, - verbose=False): - if IN_LONG_VERSION_PY: - # We're running from _version.py. If it's from a source tree - # (execute-in-place), we can work upwards to find the root of the - # tree, and then check the parent directory for a version string. If - # it's in an installed application, there's no hope. - try: - here = os.path.abspath(__file__) - except NameError: - # py2exe/bbfreeze/non-CPython don't have __file__ - return {} # without __file__, we have no hope + GITS = ["git.cmd", "git.exe"] + # if there is a tag matching tag_prefix, this yields TAG-NUM-gHEX[-dirty] + # if there isn't one, this yields HEX[-dirty] (no NUM) + describe_out = run_command(GITS, ["describe", "--tags", "--dirty", + "--always", "--long", + "--match", "%s*" % tag_prefix], + cwd=root) + # --long was added in git-1.5.5 + if describe_out is None: + raise NotThisMethod("'git describe' failed") + describe_out = describe_out.strip() + full_out = run_command(GITS, ["rev-parse", "HEAD"], cwd=root) + if full_out is None: + raise NotThisMethod("'git rev-parse' failed") + full_out = full_out.strip() + + pieces = {} + pieces["long"] = full_out + pieces["short"] = full_out[:7] # maybe improved later + pieces["error"] = None + + # parse describe_out. It will be like TAG-NUM-gHEX[-dirty] or HEX[-dirty] + # TAG might have hyphens. + git_describe = describe_out + + # look for -dirty suffix + dirty = git_describe.endswith("-dirty") + pieces["dirty"] = dirty + if dirty: + git_describe = git_describe[:git_describe.rindex("-dirty")] + + # now we have TAG-NUM-gHEX or HEX + + if "-" in git_describe: + # TAG-NUM-gHEX + mo = re.search(r'^(.+)-(\d+)-g([0-9a-f]+)$', git_describe) + if not mo: + # unparseable. Maybe git-describe is misbehaving? + pieces["error"] = ("unable to parse git-describe output: '%s'" + % describe_out) + return pieces + + # tag + full_tag = mo.group(1) + if not full_tag.startswith(tag_prefix): + if verbose: + fmt = "tag '%s' doesn't start with prefix '%s'" + print(fmt % (full_tag, tag_prefix)) + pieces["error"] = ("tag '%s' doesn't start with prefix '%s'" + % (full_tag, tag_prefix)) + return pieces + pieces["closest-tag"] = full_tag[len(tag_prefix):] + + # distance: number of commits since tag + pieces["distance"] = int(mo.group(2)) + + # commit: short hex revision ID + pieces["short"] = mo.group(3) + + else: + # HEX: no tags + pieces["closest-tag"] = None + count_out = run_command(GITS, ["rev-list", "HEAD", "--count"], + cwd=root) + pieces["distance"] = int(count_out) # total number of commits + + return pieces + + +def plus_or_dot(pieces): + """Return a + if we don't already have one, else return a .""" + if "+" in pieces.get("closest-tag", ""): + return "." + return "+" + + +def render_pep440(pieces): + """Build up version string, with post-release "local version identifier". + + Our goal: TAG[+DISTANCE.gHEX[.dirty]] . Note that if you + get a tagged build and then dirty it, you'll get TAG+0.gHEX.dirty + + Exceptions: + 1: no tags. git_describe was just HEX. 0+untagged.DISTANCE.gHEX[.dirty] + """ + if pieces["closest-tag"]: + rendered = pieces["closest-tag"] + if pieces["distance"] or pieces["dirty"]: + rendered += plus_or_dot(pieces) + rendered += "%d.g%s" % (pieces["distance"], pieces["short"]) + if pieces["dirty"]: + rendered += ".dirty" + else: + # exception #1 + rendered = "0+untagged.%d.g%s" % (pieces["distance"], + pieces["short"]) + if pieces["dirty"]: + rendered += ".dirty" + return rendered + + +def render_pep440_pre(pieces): + """TAG[.post.devDISTANCE] -- No -dirty. + + Exceptions: + 1: no tags. 0.post.devDISTANCE + """ + if pieces["closest-tag"]: + rendered = pieces["closest-tag"] + if pieces["distance"]: + rendered += ".post.dev%d" % pieces["distance"] + else: + # exception #1 + rendered = "0.post.dev%d" % pieces["distance"] + return rendered + + +def render_pep440_post(pieces): + """TAG[.postDISTANCE[.dev0]+gHEX] . + + The ".dev0" means dirty. Note that .dev0 sorts backwards + (a dirty tree will appear "older" than the corresponding clean one), + but you shouldn't be releasing software with -dirty anyways. + + Exceptions: + 1: no tags. 0.postDISTANCE[.dev0] + """ + if pieces["closest-tag"]: + rendered = pieces["closest-tag"] + if pieces["distance"] or pieces["dirty"]: + rendered += ".post%d" % pieces["distance"] + if pieces["dirty"]: + rendered += ".dev0" + rendered += plus_or_dot(pieces) + rendered += "g%s" % pieces["short"] + else: + # exception #1 + rendered = "0.post%d" % pieces["distance"] + if pieces["dirty"]: + rendered += ".dev0" + rendered += "+g%s" % pieces["short"] + return rendered + + +def render_pep440_old(pieces): + """TAG[.postDISTANCE[.dev0]] . + + The ".dev0" means dirty. + + Eexceptions: + 1: no tags. 0.postDISTANCE[.dev0] + """ + if pieces["closest-tag"]: + rendered = pieces["closest-tag"] + if pieces["distance"] or pieces["dirty"]: + rendered += ".post%d" % pieces["distance"] + if pieces["dirty"]: + rendered += ".dev0" + else: + # exception #1 + rendered = "0.post%d" % pieces["distance"] + if pieces["dirty"]: + rendered += ".dev0" + return rendered + + +def render_git_describe(pieces): + """TAG[-DISTANCE-gHEX][-dirty]. + + Like 'git describe --tags --dirty --always'. + + Exceptions: + 1: no tags. HEX[-dirty] (note: no 'g' prefix) + """ + if pieces["closest-tag"]: + rendered = pieces["closest-tag"] + if pieces["distance"]: + rendered += "-%d-g%s" % (pieces["distance"], pieces["short"]) + else: + # exception #1 + rendered = pieces["short"] + if pieces["dirty"]: + rendered += "-dirty" + return rendered + + +def render_git_describe_long(pieces): + """TAG-DISTANCE-gHEX[-dirty]. + + Like 'git describe --tags --dirty --always -long'. + The distance/hash is unconditional. + + Exceptions: + 1: no tags. HEX[-dirty] (note: no 'g' prefix) + """ + if pieces["closest-tag"]: + rendered = pieces["closest-tag"] + rendered += "-%d-g%s" % (pieces["distance"], pieces["short"]) + else: + # exception #1 + rendered = pieces["short"] + if pieces["dirty"]: + rendered += "-dirty" + return rendered + + +def render(pieces, style): + """Render the given version pieces into the requested style.""" + if pieces["error"]: + return {"version": "unknown", + "full-revisionid": pieces.get("long"), + "dirty": None, + "error": pieces["error"]} + + if not style or style == "default": + style = "pep440" # the default + + if style == "pep440": + rendered = render_pep440(pieces) + elif style == "pep440-pre": + rendered = render_pep440_pre(pieces) + elif style == "pep440-post": + rendered = render_pep440_post(pieces) + elif style == "pep440-old": + rendered = render_pep440_old(pieces) + elif style == "git-describe": + rendered = render_git_describe(pieces) + elif style == "git-describe-long": + rendered = render_git_describe_long(pieces) + else: + raise ValueError("unknown style '%s'" % style) + + return {"version": rendered, "full-revisionid": pieces["long"], + "dirty": pieces["dirty"], "error": None} + + +def get_versions(): + """Get version information or return default if unable to do so.""" + # I am in _version.py, which lives at ROOT/VERSIONFILE_SOURCE. If we have + # __file__, we can work backwards from there to the root. Some + # py2exe/bbfreeze/non-CPython implementations don't do __file__, in which + # case we can only use expanded keywords. + + cfg = get_config() + verbose = cfg.verbose + + try: + return git_versions_from_keywords(get_keywords(), cfg.tag_prefix, + verbose) + except NotThisMethod: + pass + + try: + root = os.path.realpath(__file__) # versionfile_source is the relative path from the top of the source - # tree to _version.py. Invert this to find the root from __file__. - root = here - for i in range(len(versionfile_source.split("/"))): + # tree (where the .git directory might live) to this file. Invert + # this to find the root from __file__. + for i in cfg.versionfile_source.split('/'): root = os.path.dirname(root) - else: - # we're running from versioneer.py, which means we're running from - # the setup.py in a source tree. sys.argv[0] is setup.py in the root. - here = os.path.abspath(sys.argv[0]) - root = os.path.dirname(here) + except NameError: + return {"version": "0+unknown", "full-revisionid": None, + "dirty": None, + "error": "unable to find root of source tree"} - # Source tarballs conventionally unpack into a directory that includes - # both the project name and a version string. - dirname = os.path.basename(root) - if not dirname.startswith(parentdir_prefix): - if verbose: - print("guessing rootdir is '%s', but '%s' doesn't " - "start with prefix '%s'" % - (root, dirname, parentdir_prefix)) - return None - return {"version": dirname[len(parentdir_prefix):], "full": ""} - -tag_prefix = "" -parentdir_prefix = "leap-mail" -versionfile_source = "src/leap/mail/_version.py" - - -def get_versions(default={"version": "unknown", "full": ""}, verbose=False): - variables = {"refnames": git_refnames, "full": git_full} - ver = versions_from_expanded_variables(variables, tag_prefix, verbose) - if not ver: - ver = versions_from_vcs(tag_prefix, versionfile_source, verbose) - if not ver: - ver = versions_from_parentdir(parentdir_prefix, versionfile_source, - verbose) - if not ver: - ver = default - return ver + try: + pieces = git_pieces_from_vcs(cfg.tag_prefix, root, verbose) + return render(pieces, cfg.style) + except NotThisMethod: + pass + + try: + if cfg.parentdir_prefix: + return versions_from_parentdir(cfg.parentdir_prefix, root, verbose) + except NotThisMethod: + pass + + return {"version": "0+unknown", "full-revisionid": None, + "dirty": None, + "error": "unable to compute version"} diff --git a/versioneer.py b/versioneer.py index 4e2c0a5..7ed2a21 100644 --- a/versioneer.py +++ b/versioneer.py @@ -1,170 +1,641 @@ -#! /usr/bin/python -"""versioneer.py +# Version: 0.16 -(like a rocketeer, but for versions) +"""The Versioneer - like a rocketeer, but for versions. +The Versioneer +============== + +* like a rocketeer, but for versions! * https://github.com/warner/python-versioneer * Brian Warner * License: Public Domain -* Version: 0.7+ - -This file helps distutils-based projects manage their version number by just -creating version-control tags. - -For developers who work from a VCS-generated tree (e.g. 'git clone' etc), -each 'setup.py version', 'setup.py build', 'setup.py sdist' will compute a -version number by asking your version-control tool about the current -checkout. The version number will be written into a generated _version.py -file of your choosing, where it can be included by your __init__.py - -For users who work from a VCS-generated tarball (e.g. 'git archive'), it will -compute a version number by looking at the name of the directory created when -te tarball is unpacked. This conventionally includes both the name of the -project and a version number. - -For users who work from a tarball built by 'setup.py sdist', it will get a -version number from a previously-generated _version.py file. - -As a result, loading code directly from the source tree will not result in a -real version. If you want real versions from VCS trees (where you frequently -update from the upstream repository, or do new development), you will need to -do a 'setup.py version' after each update, and load code from the build/ -directory. - -You need to provide this code with a few configuration values: - - versionfile_source: - A project-relative pathname into which the generated version strings - should be written. This is usually a _version.py next to your project's - main __init__.py file. If your project uses src/myproject/__init__.py, - this should be 'src/myproject/_version.py'. This file should be checked - in to your VCS as usual: the copy created below by 'setup.py - update_files' will include code that parses expanded VCS keywords in - generated tarballs. The 'build' and 'sdist' commands will replace it with - a copy that has just the calculated version string. - - versionfile_build: - Like versionfile_source, but relative to the build directory instead of - the source directory. These will differ when your setup.py uses - 'package_dir='. If you have package_dir={'myproject': 'src/myproject'}, - then you will probably have versionfile_build='myproject/_version.py' and - versionfile_source='src/myproject/_version.py'. - - tag_prefix: a string, like 'PROJECTNAME-', which appears at the start of all - VCS tags. If your tags look like 'myproject-1.2.0', then you - should use tag_prefix='myproject-'. If you use unprefixed tags - like '1.2.0', this should be an empty string. - - parentdir_prefix: a string, frequently the same as tag_prefix, which - appears at the start of all unpacked tarball filenames. If - your tarball unpacks into 'myproject-1.2.0', this should - be 'myproject-'. - -To use it: - - 1: include this file in the top level of your project - 2: make the following changes to the top of your setup.py: - import versioneer - versioneer.versionfile_source = 'src/myproject/_version.py' - versioneer.versionfile_build = 'myproject/_version.py' - versioneer.tag_prefix = '' # tags are like 1.2.0 - versioneer.parentdir_prefix = 'myproject-' # dirname like 'myproject-1.2.0' - 3: add the following arguments to the setup() call in your setup.py: - version=versioneer.get_version(), - cmdclass=versioneer.get_cmdclass(), - 4: run 'setup.py update_files', which will create _version.py, and will - modify your __init__.py to define __version__ (by calling a function - from _version.py) - 5: modify your MANIFEST.in to include versioneer.py - 6: add both versioneer.py and the generated _version.py to your VCS -""" +* Compatible With: python2.6, 2.7, 3.3, 3.4, 3.5, and pypy +* [![Latest Version] +(https://pypip.in/version/versioneer/badge.svg?style=flat) +](https://pypi.python.org/pypi/versioneer/) +* [![Build Status] +(https://travis-ci.org/warner/python-versioneer.png?branch=master) +](https://travis-ci.org/warner/python-versioneer) + +This is a tool for managing a recorded version number in distutils-based +python projects. The goal is to remove the tedious and error-prone "update +the embedded version string" step from your release process. Making a new +release should be as easy as recording a new tag in your version-control +system, and maybe making new tarballs. + + +## Quick Install + +* `pip install versioneer` to somewhere to your $PATH +* add a `[versioneer]` section to your setup.cfg (see below) +* run `versioneer install` in your source tree, commit the results + +## Version Identifiers + +Source trees come from a variety of places: + +* a version-control system checkout (mostly used by developers) +* a nightly tarball, produced by build automation +* a snapshot tarball, produced by a web-based VCS browser, like github's + "tarball from tag" feature +* a release tarball, produced by "setup.py sdist", distributed through PyPI + +Within each source tree, the version identifier (either a string or a number, +this tool is format-agnostic) can come from a variety of places: + +* ask the VCS tool itself, e.g. "git describe" (for checkouts), which knows + about recent "tags" and an absolute revision-id +* the name of the directory into which the tarball was unpacked +* an expanded VCS keyword ($Id$, etc) +* a `_version.py` created by some earlier build step + +For released software, the version identifier is closely related to a VCS +tag. Some projects use tag names that include more than just the version +string (e.g. "myproject-1.2" instead of just "1.2"), in which case the tool +needs to strip the tag prefix to extract the version identifier. For +unreleased software (between tags), the version identifier should provide +enough information to help developers recreate the same tree, while also +giving them an idea of roughly how old the tree is (after version 1.2, before +version 1.3). Many VCS systems can report a description that captures this, +for example `git describe --tags --dirty --always` reports things like +"0.7-1-g574ab98-dirty" to indicate that the checkout is one revision past the +0.7 tag, has a unique revision id of "574ab98", and is "dirty" (it has +uncommitted changes. + +The version identifier is used for multiple purposes: + +* to allow the module to self-identify its version: `myproject.__version__` +* to choose a name and prefix for a 'setup.py sdist' tarball + +## Theory of Operation + +Versioneer works by adding a special `_version.py` file into your source +tree, where your `__init__.py` can import it. This `_version.py` knows how to +dynamically ask the VCS tool for version information at import time. + +`_version.py` also contains `$Revision$` markers, and the installation +process marks `_version.py` to have this marker rewritten with a tag name +during the `git archive` command. As a result, generated tarballs will +contain enough information to get the proper version. + +To allow `setup.py` to compute a version too, a `versioneer.py` is added to +the top level of your source tree, next to `setup.py` and the `setup.cfg` +that configures it. This overrides several distutils/setuptools commands to +compute the version when invoked, and changes `setup.py build` and `setup.py +sdist` to replace `_version.py` with a small static file that contains just +the generated version data. + +## Installation + +First, decide on values for the following configuration variables: + +* `VCS`: the version control system you use. Currently accepts "git". + +* `style`: the style of version string to be produced. See "Styles" below for + details. Defaults to "pep440", which looks like + `TAG[+DISTANCE.gSHORTHASH[.dirty]]`. + +* `versionfile_source`: + + A project-relative pathname into which the generated version strings should + be written. This is usually a `_version.py` next to your project's main + `__init__.py` file, so it can be imported at runtime. If your project uses + `src/myproject/__init__.py`, this should be `src/myproject/_version.py`. + This file should be checked in to your VCS as usual: the copy created below + by `setup.py setup_versioneer` will include code that parses expanded VCS + keywords in generated tarballs. The 'build' and 'sdist' commands will + replace it with a copy that has just the calculated version string. + + This must be set even if your project does not have any modules (and will + therefore never import `_version.py`), since "setup.py sdist" -based trees + still need somewhere to record the pre-calculated version strings. Anywhere + in the source tree should do. If there is a `__init__.py` next to your + `_version.py`, the `setup.py setup_versioneer` command (described below) + will append some `__version__`-setting assignments, if they aren't already + present. + +* `versionfile_build`: + + Like `versionfile_source`, but relative to the build directory instead of + the source directory. These will differ when your setup.py uses + 'package_dir='. If you have `package_dir={'myproject': 'src/myproject'}`, + then you will probably have `versionfile_build='myproject/_version.py'` and + `versionfile_source='src/myproject/_version.py'`. + + If this is set to None, then `setup.py build` will not attempt to rewrite + any `_version.py` in the built tree. If your project does not have any + libraries (e.g. if it only builds a script), then you should use + `versionfile_build = None`. To actually use the computed version string, + your `setup.py` will need to override `distutils.command.build_scripts` + with a subclass that explicitly inserts a copy of + `versioneer.get_version()` into your script file. See + `test/demoapp-script-only/setup.py` for an example. + +* `tag_prefix`: + + a string, like 'PROJECTNAME-', which appears at the start of all VCS tags. + If your tags look like 'myproject-1.2.0', then you should use + tag_prefix='myproject-'. If you use unprefixed tags like '1.2.0', this + should be an empty string, using either `tag_prefix=` or `tag_prefix=''`. + +* `parentdir_prefix`: + + a optional string, frequently the same as tag_prefix, which appears at the + start of all unpacked tarball filenames. If your tarball unpacks into + 'myproject-1.2.0', this should be 'myproject-'. To disable this feature, + just omit the field from your `setup.cfg`. + +This tool provides one script, named `versioneer`. That script has one mode, +"install", which writes a copy of `versioneer.py` into the current directory +and runs `versioneer.py setup` to finish the installation. + +To versioneer-enable your project: + +* 1: Modify your `setup.cfg`, adding a section named `[versioneer]` and + populating it with the configuration values you decided earlier (note that + the option names are not case-sensitive): + + ```` + [versioneer] + VCS = git + style = pep440 + versionfile_source = src/myproject/_version.py + versionfile_build = myproject/_version.py + tag_prefix = + parentdir_prefix = myproject- + ```` + +* 2: Run `versioneer install`. This will do the following: + + * copy `versioneer.py` into the top of your source tree + * create `_version.py` in the right place (`versionfile_source`) + * modify your `__init__.py` (if one exists next to `_version.py`) to define + `__version__` (by calling a function from `_version.py`) + * modify your `MANIFEST.in` to include both `versioneer.py` and the + generated `_version.py` in sdist tarballs + + `versioneer install` will complain about any problems it finds with your + `setup.py` or `setup.cfg`. Run it multiple times until you have fixed all + the problems. + +* 3: add a `import versioneer` to your setup.py, and add the following + arguments to the setup() call: + + version=versioneer.get_version(), + cmdclass=versioneer.get_cmdclass(), + +* 4: commit these changes to your VCS. To make sure you won't forget, + `versioneer install` will mark everything it touched for addition using + `git add`. Don't forget to add `setup.py` and `setup.cfg` too. + +## Post-Installation Usage + +Once established, all uses of your tree from a VCS checkout should get the +current version string. All generated tarballs should include an embedded +version string (so users who unpack them will not need a VCS tool installed). + +If you distribute your project through PyPI, then the release process should +boil down to two steps: + +* 1: git tag 1.0 +* 2: python setup.py register sdist upload + +If you distribute it through github (i.e. users use github to generate +tarballs with `git archive`), the process is: + +* 1: git tag 1.0 +* 2: git push; git push --tags + +Versioneer will report "0+untagged.NUMCOMMITS.gHASH" until your tree has at +least one tag in its history. + +## Version-String Flavors + +Code which uses Versioneer can learn about its version string at runtime by +importing `_version` from your main `__init__.py` file and running the +`get_versions()` function. From the "outside" (e.g. in `setup.py`), you can +import the top-level `versioneer.py` and run `get_versions()`. + +Both functions return a dictionary with different flavors of version +information: + +* `['version']`: A condensed version string, rendered using the selected + style. This is the most commonly used value for the project's version + string. The default "pep440" style yields strings like `0.11`, + `0.11+2.g1076c97`, or `0.11+2.g1076c97.dirty`. See the "Styles" section + below for alternative styles. + +* `['full-revisionid']`: detailed revision identifier. For Git, this is the + full SHA1 commit id, e.g. "1076c978a8d3cfc70f408fe5974aa6c092c949ac". + +* `['dirty']`: a boolean, True if the tree has uncommitted changes. Note that + this is only accurate if run in a VCS checkout, otherwise it is likely to + be False or None + +* `['error']`: if the version string could not be computed, this will be set + to a string describing the problem, otherwise it will be None. It may be + useful to throw an exception in setup.py if this is set, to avoid e.g. + creating tarballs with a version string of "unknown". + +Some variants are more useful than others. Including `full-revisionid` in a +bug report should allow developers to reconstruct the exact code being tested +(or indicate the presence of local changes that should be shared with the +developers). `version` is suitable for display in an "about" box or a CLI +`--version` output: it can be easily compared against release notes and lists +of bugs fixed in various releases. + +The installer adds the following text to your `__init__.py` to place a basic +version in `YOURPROJECT.__version__`: + + from ._version import get_versions + __version__ = get_versions()['version'] + del get_versions + +## Styles + +The setup.cfg `style=` configuration controls how the VCS information is +rendered into a version string. + +The default style, "pep440", produces a PEP440-compliant string, equal to the +un-prefixed tag name for actual releases, and containing an additional "local +version" section with more detail for in-between builds. For Git, this is +TAG[+DISTANCE.gHEX[.dirty]] , using information from `git describe --tags +--dirty --always`. For example "0.11+2.g1076c97.dirty" indicates that the +tree is like the "1076c97" commit but has uncommitted changes (".dirty"), and +that this commit is two revisions ("+2") beyond the "0.11" tag. For released +software (exactly equal to a known tag), the identifier will only contain the +stripped tag, e.g. "0.11". + +Other styles are available. See details.md in the Versioneer source tree for +descriptions. + +## Debugging + +Versioneer tries to avoid fatal errors: if something goes wrong, it will tend +to return a version of "0+unknown". To investigate the problem, run `setup.py +version`, which will run the version-lookup code in a verbose mode, and will +display the full contents of `get_versions()` (including the `error` string, +which may help identify what went wrong). + +## Updating Versioneer + +To upgrade your project to a new release of Versioneer, do the following: + +* install the new Versioneer (`pip install -U versioneer` or equivalent) +* edit `setup.cfg`, if necessary, to include any new configuration settings + indicated by the release notes +* re-run `versioneer install` in your source tree, to replace + `SRC/_version.py` +* commit any changed files + +### Upgrading to 0.16 + +Nothing special. + +### Upgrading to 0.15 + +Starting with this version, Versioneer is configured with a `[versioneer]` +section in your `setup.cfg` file. Earlier versions required the `setup.py` to +set attributes on the `versioneer` module immediately after import. The new +version will refuse to run (raising an exception during import) until you +have provided the necessary `setup.cfg` section. + +In addition, the Versioneer package provides an executable named +`versioneer`, and the installation process is driven by running `versioneer +install`. In 0.14 and earlier, the executable was named +`versioneer-installer` and was run without an argument. + +### Upgrading to 0.14 -import os, sys, re -from distutils.core import Command -from distutils.command.sdist import sdist as _sdist -from distutils.command.build import build as _build +0.14 changes the format of the version string. 0.13 and earlier used +hyphen-separated strings like "0.11-2-g1076c97-dirty". 0.14 and beyond use a +plus-separated "local version" section strings, with dot-separated +components, like "0.11+2.g1076c97". PEP440-strict tools did not like the old +format, but should be ok with the new one. -versionfile_source = None -versionfile_build = None -tag_prefix = None -parentdir_prefix = None +### Upgrading from 0.11 to 0.12 -VCS = "git" -IN_LONG_VERSION_PY = False +Nothing special. +### Upgrading from 0.10 to 0.11 -LONG_VERSION_PY = ''' -IN_LONG_VERSION_PY = True +You must add a `versioneer.VCS = "git"` to your `setup.py` before re-running +`setup.py setup_versioneer`. This will enable the use of additional +version-control systems (SVN, etc) in the future. + +## Future Directions + +This tool is designed to make it easily extended to other version-control +systems: all VCS-specific components are in separate directories like +src/git/ . The top-level `versioneer.py` script is assembled from these +components by running make-versioneer.py . In the future, make-versioneer.py +will take a VCS name as an argument, and will construct a version of +`versioneer.py` that is specific to the given VCS. It might also take the +configuration arguments that are currently provided manually during +installation by editing setup.py . Alternatively, it might go the other +direction and include code from all supported VCS systems, reducing the +number of intermediate scripts. + + +## License + +To make Versioneer easier to embed, all its code is dedicated to the public +domain. The `_version.py` that it creates is also in the public domain. +Specifically, both are released under the Creative Commons "Public Domain +Dedication" license (CC0-1.0), as described in +https://creativecommons.org/publicdomain/zero/1.0/ . + +""" + +from __future__ import print_function +try: + import configparser +except ImportError: + import ConfigParser as configparser +import errno +import json +import os +import re +import subprocess +import sys + + +class VersioneerConfig: + """Container for Versioneer configuration parameters.""" + + +def get_root(): + """Get the project root directory. + + We require that all commands are run from the project root, i.e. the + directory that contains setup.py, setup.cfg, and versioneer.py . + """ + root = os.path.realpath(os.path.abspath(os.getcwd())) + setup_py = os.path.join(root, "setup.py") + versioneer_py = os.path.join(root, "versioneer.py") + if not (os.path.exists(setup_py) or os.path.exists(versioneer_py)): + # allow 'python path/to/setup.py COMMAND' + root = os.path.dirname(os.path.realpath(os.path.abspath(sys.argv[0]))) + setup_py = os.path.join(root, "setup.py") + versioneer_py = os.path.join(root, "versioneer.py") + if not (os.path.exists(setup_py) or os.path.exists(versioneer_py)): + err = ("Versioneer was unable to run the project root directory. " + "Versioneer requires setup.py to be executed from " + "its immediate directory (like 'python setup.py COMMAND'), " + "or in a way that lets it use sys.argv[0] to find the root " + "(like 'python path/to/setup.py COMMAND').") + raise VersioneerBadRootError(err) + try: + # Certain runtime workflows (setup.py install/develop in a setuptools + # tree) execute all dependencies in a single python process, so + # "versioneer" may be imported multiple times, and python's shared + # module-import table will cache the first one. So we can't use + # os.path.dirname(__file__), as that will find whichever + # versioneer.py was first imported, even in later projects. + me = os.path.realpath(os.path.abspath(__file__)) + if os.path.splitext(me)[0] != os.path.splitext(versioneer_py)[0]: + print("Warning: build in %s is using versioneer.py from %s" + % (os.path.dirname(me), versioneer_py)) + except NameError: + pass + return root + + +def get_config_from_root(root): + """Read the project setup.cfg file to determine Versioneer config.""" + # This might raise EnvironmentError (if setup.cfg is missing), or + # configparser.NoSectionError (if it lacks a [versioneer] section), or + # configparser.NoOptionError (if it lacks "VCS="). See the docstring at + # the top of versioneer.py for instructions on writing your setup.cfg . + setup_cfg = os.path.join(root, "setup.cfg") + parser = configparser.SafeConfigParser() + with open(setup_cfg, "r") as f: + parser.readfp(f) + VCS = parser.get("versioneer", "VCS") # mandatory + + def get(parser, name): + if parser.has_option("versioneer", name): + return parser.get("versioneer", name) + return None + cfg = VersioneerConfig() + cfg.VCS = VCS + cfg.style = get(parser, "style") or "" + cfg.versionfile_source = get(parser, "versionfile_source") + cfg.versionfile_build = get(parser, "versionfile_build") + cfg.tag_prefix = get(parser, "tag_prefix") + if cfg.tag_prefix in ("''", '""'): + cfg.tag_prefix = "" + cfg.parentdir_prefix = get(parser, "parentdir_prefix") + cfg.verbose = get(parser, "verbose") + return cfg + + +class NotThisMethod(Exception): + """Exception raised if a method is not valid for the current scenario.""" + +# these dictionaries contain VCS-specific tools +LONG_VERSION_PY = {} +HANDLERS = {} + + +def register_vcs_handler(vcs, method): # decorator + """Decorator to mark a method as the handler for a particular VCS.""" + def decorate(f): + """Store f in HANDLERS[vcs][method].""" + if vcs not in HANDLERS: + HANDLERS[vcs] = {} + HANDLERS[vcs][method] = f + return f + return decorate + + +def run_command(commands, args, cwd=None, verbose=False, hide_stderr=False): + """Call the given command(s).""" + assert isinstance(commands, list) + p = None + for c in commands: + try: + dispcmd = str([c] + args) + # remember shell=False, so use git.cmd on windows, not just git + p = subprocess.Popen([c] + args, cwd=cwd, stdout=subprocess.PIPE, + stderr=(subprocess.PIPE if hide_stderr + else None)) + break + except EnvironmentError: + e = sys.exc_info()[1] + if e.errno == errno.ENOENT: + continue + if verbose: + print("unable to run %s" % dispcmd) + print(e) + return None + else: + if verbose: + print("unable to find command, tried %s" % (commands,)) + return None + stdout = p.communicate()[0].strip() + if sys.version_info[0] >= 3: + stdout = stdout.decode() + if p.returncode != 0: + if verbose: + print("unable to run %s (error)" % dispcmd) + return None + return stdout +LONG_VERSION_PY['git'] = ''' # This file helps to compute a version number in source trees obtained from # git-archive tarball (such as those provided by githubs download-from-tag -# feature). Distribution tarballs (build by setup.py sdist) and build +# feature). Distribution tarballs (built by setup.py sdist) and build # directories (produced by setup.py build) will contain a much shorter file # that just contains the computed version number. # This file is released into the public domain. Generated by -# versioneer-0.7+ (https://github.com/warner/python-versioneer) - -# these strings will be replaced by git during git-archive -git_refnames = "%(DOLLAR)sFormat:%%d%(DOLLAR)s" -git_full = "%(DOLLAR)sFormat:%%H%(DOLLAR)s" +# versioneer-0.16 (https://github.com/warner/python-versioneer) +"""Git implementation of _version.py.""" +import errno +import os +import re import subprocess import sys -def run_command(args, cwd=None, verbose=False): - try: - # remember shell=False, so use git.exe on windows, not just git - p = subprocess.Popen(args, stdout=subprocess.PIPE, cwd=cwd) - except EnvironmentError: - e = sys.exc_info()[1] + +def get_keywords(): + """Get the keywords needed to look up the version information.""" + # these strings will be replaced by git during git-archive. + # setup.py/versioneer.py will grep for the variable names, so they must + # each be defined on a line of their own. _version.py will just call + # get_keywords(). + git_refnames = "%(DOLLAR)sFormat:%%d%(DOLLAR)s" + git_full = "%(DOLLAR)sFormat:%%H%(DOLLAR)s" + keywords = {"refnames": git_refnames, "full": git_full} + return keywords + + +class VersioneerConfig: + """Container for Versioneer configuration parameters.""" + + +def get_config(): + """Create, populate and return the VersioneerConfig() object.""" + # these strings are filled in when 'setup.py versioneer' creates + # _version.py + cfg = VersioneerConfig() + cfg.VCS = "git" + cfg.style = "%(STYLE)s" + cfg.tag_prefix = "%(TAG_PREFIX)s" + cfg.parentdir_prefix = "%(PARENTDIR_PREFIX)s" + cfg.versionfile_source = "%(VERSIONFILE_SOURCE)s" + cfg.verbose = False + return cfg + + +class NotThisMethod(Exception): + """Exception raised if a method is not valid for the current scenario.""" + + +LONG_VERSION_PY = {} +HANDLERS = {} + + +def register_vcs_handler(vcs, method): # decorator + """Decorator to mark a method as the handler for a particular VCS.""" + def decorate(f): + """Store f in HANDLERS[vcs][method].""" + if vcs not in HANDLERS: + HANDLERS[vcs] = {} + HANDLERS[vcs][method] = f + return f + return decorate + + +def run_command(commands, args, cwd=None, verbose=False, hide_stderr=False): + """Call the given command(s).""" + assert isinstance(commands, list) + p = None + for c in commands: + try: + dispcmd = str([c] + args) + # remember shell=False, so use git.cmd on windows, not just git + p = subprocess.Popen([c] + args, cwd=cwd, stdout=subprocess.PIPE, + stderr=(subprocess.PIPE if hide_stderr + else None)) + break + except EnvironmentError: + e = sys.exc_info()[1] + if e.errno == errno.ENOENT: + continue + if verbose: + print("unable to run %%s" %% dispcmd) + print(e) + return None + else: if verbose: - print("unable to run %%s" %% args[0]) - print(e) + print("unable to find command, tried %%s" %% (commands,)) return None stdout = p.communicate()[0].strip() - if sys.version >= '3': + if sys.version_info[0] >= 3: stdout = stdout.decode() if p.returncode != 0: if verbose: - print("unable to run %%s (error)" %% args[0]) + print("unable to run %%s (error)" %% dispcmd) return None return stdout -import sys -import re -import os.path +def versions_from_parentdir(parentdir_prefix, root, verbose): + """Try to determine the version from the parent directory name. + + Source tarballs conventionally unpack into a directory that includes + both the project name and a version string. + """ + dirname = os.path.basename(root) + if not dirname.startswith(parentdir_prefix): + if verbose: + print("guessing rootdir is '%%s', but '%%s' doesn't start with " + "prefix '%%s'" %% (root, dirname, parentdir_prefix)) + raise NotThisMethod("rootdir doesn't start with parentdir_prefix") + return {"version": dirname[len(parentdir_prefix):], + "full-revisionid": None, + "dirty": False, "error": None} -def get_expanded_variables(versionfile_source): + +@register_vcs_handler("git", "get_keywords") +def git_get_keywords(versionfile_abs): + """Extract version information from the given file.""" # the code embedded in _version.py can just fetch the value of these - # variables. When used from setup.py, we don't want to import - # _version.py, so we do it with a regexp instead. This function is not - # used from _version.py. - variables = {} + # keywords. When used from setup.py, we don't want to import _version.py, + # so we do it with a regexp instead. This function is not used from + # _version.py. + keywords = {} try: - f = open(versionfile_source,"r") + f = open(versionfile_abs, "r") for line in f.readlines(): if line.strip().startswith("git_refnames ="): mo = re.search(r'=\s*"(.*)"', line) if mo: - variables["refnames"] = mo.group(1) + keywords["refnames"] = mo.group(1) if line.strip().startswith("git_full ="): mo = re.search(r'=\s*"(.*)"', line) if mo: - variables["full"] = mo.group(1) + keywords["full"] = mo.group(1) f.close() except EnvironmentError: pass - return variables + return keywords + -def versions_from_expanded_variables(variables, tag_prefix, verbose=False): - refnames = variables["refnames"].strip() +@register_vcs_handler("git", "keywords") +def git_versions_from_keywords(keywords, tag_prefix, verbose): + """Get version information from git keywords.""" + if not keywords: + raise NotThisMethod("no keywords at all, weird") + refnames = keywords["refnames"].strip() if refnames.startswith("$Format"): if verbose: - print("variables are unexpanded, not using") - return {} # unexpanded, so not in an unpacked git-archive tarball + print("keywords are unexpanded, not using") + raise NotThisMethod("unexpanded keywords, not a git-archive tarball") refs = set([r.strip() for r in refnames.strip("()").split(",")]) # starting in git-1.8.3, tags are listed as "tag: foo-1.0" instead of # just "foo-1.0". If we see a "tag: " prefix, prefer those. @@ -189,172 +660,350 @@ def versions_from_expanded_variables(variables, tag_prefix, verbose=False): r = ref[len(tag_prefix):] if verbose: print("picking %%s" %% r) - return { "version": r, - "full": variables["full"].strip() } - # no suitable tags, so we use the full revision id + return {"version": r, + "full-revisionid": keywords["full"].strip(), + "dirty": False, "error": None + } + # no suitable tags, so version is "0+unknown", but full hex is still there if verbose: - print("no suitable tags, using full revision id") - return { "version": variables["full"].strip(), - "full": variables["full"].strip() } - -def versions_from_vcs(tag_prefix, versionfile_source, verbose=False): - # this runs 'git' from the root of the source tree. That either means - # someone ran a setup.py command (and this code is in versioneer.py, so - # IN_LONG_VERSION_PY=False, thus the containing directory is the root of - # the source tree), or someone ran a project-specific entry point (and - # this code is in _version.py, so IN_LONG_VERSION_PY=True, thus the - # containing directory is somewhere deeper in the source tree). This only - # gets called if the git-archive 'subst' variables were *not* expanded, - # and _version.py hasn't already been rewritten with a short version - # string, meaning we're inside a checked out source tree. + print("no suitable tags, using unknown + full revision id") + return {"version": "0+unknown", + "full-revisionid": keywords["full"].strip(), + "dirty": False, "error": "no suitable tags"} - try: - here = os.path.abspath(__file__) - except NameError: - # some py2exe/bbfreeze/non-CPython implementations don't do __file__ - return {} # not always correct - - # versionfile_source is the relative path from the top of the source tree - # (where the .git directory might live) to this file. Invert this to find - # the root from __file__. - root = here - if IN_LONG_VERSION_PY: - for i in range(len(versionfile_source.split("/"))): - root = os.path.dirname(root) - else: - root = os.path.dirname(here) + +@register_vcs_handler("git", "pieces_from_vcs") +def git_pieces_from_vcs(tag_prefix, root, verbose, run_command=run_command): + """Get version from 'git describe' in the root of the source tree. + + This only gets called if the git-archive 'subst' keywords were *not* + expanded, and _version.py hasn't already been rewritten with a short + version string, meaning we're inside a checked out source tree. + """ if not os.path.exists(os.path.join(root, ".git")): if verbose: print("no .git in %%s" %% root) - return {} + raise NotThisMethod("no .git directory") - GIT = "git" + GITS = ["git"] if sys.platform == "win32": - GIT = "git.exe" - stdout = run_command([GIT, "describe", "--tags", "--dirty", "--always"], - cwd=root) - if stdout is None: - return {} - if not stdout.startswith(tag_prefix): - if verbose: - print("tag '%%s' doesn't start with prefix '%%s'" %% (stdout, tag_prefix)) - return {} - tag = stdout[len(tag_prefix):] - stdout = run_command([GIT, "rev-parse", "HEAD"], cwd=root) - if stdout is None: - return {} - full = stdout.strip() - if tag.endswith("-dirty"): - full += "-dirty" - return {"version": tag, "full": full} - - -def versions_from_parentdir(parentdir_prefix, versionfile_source, verbose=False): - if IN_LONG_VERSION_PY: - # We're running from _version.py. If it's from a source tree - # (execute-in-place), we can work upwards to find the root of the - # tree, and then check the parent directory for a version string. If - # it's in an installed application, there's no hope. - try: - here = os.path.abspath(__file__) - except NameError: - # py2exe/bbfreeze/non-CPython don't have __file__ - return {} # without __file__, we have no hope - # versionfile_source is the relative path from the top of the source - # tree to _version.py. Invert this to find the root from __file__. - root = here - for i in range(len(versionfile_source.split("/"))): - root = os.path.dirname(root) + GITS = ["git.cmd", "git.exe"] + # if there is a tag matching tag_prefix, this yields TAG-NUM-gHEX[-dirty] + # if there isn't one, this yields HEX[-dirty] (no NUM) + describe_out = run_command(GITS, ["describe", "--tags", "--dirty", + "--always", "--long", + "--match", "%%s*" %% tag_prefix], + cwd=root) + # --long was added in git-1.5.5 + if describe_out is None: + raise NotThisMethod("'git describe' failed") + describe_out = describe_out.strip() + full_out = run_command(GITS, ["rev-parse", "HEAD"], cwd=root) + if full_out is None: + raise NotThisMethod("'git rev-parse' failed") + full_out = full_out.strip() + + pieces = {} + pieces["long"] = full_out + pieces["short"] = full_out[:7] # maybe improved later + pieces["error"] = None + + # parse describe_out. It will be like TAG-NUM-gHEX[-dirty] or HEX[-dirty] + # TAG might have hyphens. + git_describe = describe_out + + # look for -dirty suffix + dirty = git_describe.endswith("-dirty") + pieces["dirty"] = dirty + if dirty: + git_describe = git_describe[:git_describe.rindex("-dirty")] + + # now we have TAG-NUM-gHEX or HEX + + if "-" in git_describe: + # TAG-NUM-gHEX + mo = re.search(r'^(.+)-(\d+)-g([0-9a-f]+)$', git_describe) + if not mo: + # unparseable. Maybe git-describe is misbehaving? + pieces["error"] = ("unable to parse git-describe output: '%%s'" + %% describe_out) + return pieces + + # tag + full_tag = mo.group(1) + if not full_tag.startswith(tag_prefix): + if verbose: + fmt = "tag '%%s' doesn't start with prefix '%%s'" + print(fmt %% (full_tag, tag_prefix)) + pieces["error"] = ("tag '%%s' doesn't start with prefix '%%s'" + %% (full_tag, tag_prefix)) + return pieces + pieces["closest-tag"] = full_tag[len(tag_prefix):] + + # distance: number of commits since tag + pieces["distance"] = int(mo.group(2)) + + # commit: short hex revision ID + pieces["short"] = mo.group(3) + + else: + # HEX: no tags + pieces["closest-tag"] = None + count_out = run_command(GITS, ["rev-list", "HEAD", "--count"], + cwd=root) + pieces["distance"] = int(count_out) # total number of commits + + return pieces + + +def plus_or_dot(pieces): + """Return a + if we don't already have one, else return a .""" + if "+" in pieces.get("closest-tag", ""): + return "." + return "+" + + +def render_pep440(pieces): + """Build up version string, with post-release "local version identifier". + + Our goal: TAG[+DISTANCE.gHEX[.dirty]] . Note that if you + get a tagged build and then dirty it, you'll get TAG+0.gHEX.dirty + + Exceptions: + 1: no tags. git_describe was just HEX. 0+untagged.DISTANCE.gHEX[.dirty] + """ + if pieces["closest-tag"]: + rendered = pieces["closest-tag"] + if pieces["distance"] or pieces["dirty"]: + rendered += plus_or_dot(pieces) + rendered += "%%d.g%%s" %% (pieces["distance"], pieces["short"]) + if pieces["dirty"]: + rendered += ".dirty" + else: + # exception #1 + rendered = "0+untagged.%%d.g%%s" %% (pieces["distance"], + pieces["short"]) + if pieces["dirty"]: + rendered += ".dirty" + return rendered + + +def render_pep440_pre(pieces): + """TAG[.post.devDISTANCE] -- No -dirty. + + Exceptions: + 1: no tags. 0.post.devDISTANCE + """ + if pieces["closest-tag"]: + rendered = pieces["closest-tag"] + if pieces["distance"]: + rendered += ".post.dev%%d" %% pieces["distance"] else: - # we're running from versioneer.py, which means we're running from - # the setup.py in a source tree. sys.argv[0] is setup.py in the root. - here = os.path.abspath(sys.argv[0]) - root = os.path.dirname(here) + # exception #1 + rendered = "0.post.dev%%d" %% pieces["distance"] + return rendered + + +def render_pep440_post(pieces): + """TAG[.postDISTANCE[.dev0]+gHEX] . + + The ".dev0" means dirty. Note that .dev0 sorts backwards + (a dirty tree will appear "older" than the corresponding clean one), + but you shouldn't be releasing software with -dirty anyways. + + Exceptions: + 1: no tags. 0.postDISTANCE[.dev0] + """ + if pieces["closest-tag"]: + rendered = pieces["closest-tag"] + if pieces["distance"] or pieces["dirty"]: + rendered += ".post%%d" %% pieces["distance"] + if pieces["dirty"]: + rendered += ".dev0" + rendered += plus_or_dot(pieces) + rendered += "g%%s" %% pieces["short"] + else: + # exception #1 + rendered = "0.post%%d" %% pieces["distance"] + if pieces["dirty"]: + rendered += ".dev0" + rendered += "+g%%s" %% pieces["short"] + return rendered + + +def render_pep440_old(pieces): + """TAG[.postDISTANCE[.dev0]] . + + The ".dev0" means dirty. + + Eexceptions: + 1: no tags. 0.postDISTANCE[.dev0] + """ + if pieces["closest-tag"]: + rendered = pieces["closest-tag"] + if pieces["distance"] or pieces["dirty"]: + rendered += ".post%%d" %% pieces["distance"] + if pieces["dirty"]: + rendered += ".dev0" + else: + # exception #1 + rendered = "0.post%%d" %% pieces["distance"] + if pieces["dirty"]: + rendered += ".dev0" + return rendered - # Source tarballs conventionally unpack into a directory that includes - # both the project name and a version string. - dirname = os.path.basename(root) - if not dirname.startswith(parentdir_prefix): - if verbose: - print("guessing rootdir is '%%s', but '%%s' doesn't start with prefix '%%s'" %% - (root, dirname, parentdir_prefix)) - return None - return {"version": dirname[len(parentdir_prefix):], "full": ""} - -tag_prefix = "%(TAG_PREFIX)s" -parentdir_prefix = "%(PARENTDIR_PREFIX)s" -versionfile_source = "%(VERSIONFILE_SOURCE)s" - -def get_versions(default={"version": "unknown", "full": ""}, verbose=False): - variables = { "refnames": git_refnames, "full": git_full } - ver = versions_from_expanded_variables(variables, tag_prefix, verbose) - if not ver: - ver = versions_from_vcs(tag_prefix, versionfile_source, verbose) - if not ver: - ver = versions_from_parentdir(parentdir_prefix, versionfile_source, - verbose) - if not ver: - ver = default - return ver -''' +def render_git_describe(pieces): + """TAG[-DISTANCE-gHEX][-dirty]. + Like 'git describe --tags --dirty --always'. -import subprocess -import sys + Exceptions: + 1: no tags. HEX[-dirty] (note: no 'g' prefix) + """ + if pieces["closest-tag"]: + rendered = pieces["closest-tag"] + if pieces["distance"]: + rendered += "-%%d-g%%s" %% (pieces["distance"], pieces["short"]) + else: + # exception #1 + rendered = pieces["short"] + if pieces["dirty"]: + rendered += "-dirty" + return rendered + + +def render_git_describe_long(pieces): + """TAG-DISTANCE-gHEX[-dirty]. + + Like 'git describe --tags --dirty --always -long'. + The distance/hash is unconditional. + + Exceptions: + 1: no tags. HEX[-dirty] (note: no 'g' prefix) + """ + if pieces["closest-tag"]: + rendered = pieces["closest-tag"] + rendered += "-%%d-g%%s" %% (pieces["distance"], pieces["short"]) + else: + # exception #1 + rendered = pieces["short"] + if pieces["dirty"]: + rendered += "-dirty" + return rendered + + +def render(pieces, style): + """Render the given version pieces into the requested style.""" + if pieces["error"]: + return {"version": "unknown", + "full-revisionid": pieces.get("long"), + "dirty": None, + "error": pieces["error"]} + + if not style or style == "default": + style = "pep440" # the default + + if style == "pep440": + rendered = render_pep440(pieces) + elif style == "pep440-pre": + rendered = render_pep440_pre(pieces) + elif style == "pep440-post": + rendered = render_pep440_post(pieces) + elif style == "pep440-old": + rendered = render_pep440_old(pieces) + elif style == "git-describe": + rendered = render_git_describe(pieces) + elif style == "git-describe-long": + rendered = render_git_describe_long(pieces) + else: + raise ValueError("unknown style '%%s'" %% style) + + return {"version": rendered, "full-revisionid": pieces["long"], + "dirty": pieces["dirty"], "error": None} + + +def get_versions(): + """Get version information or return default if unable to do so.""" + # I am in _version.py, which lives at ROOT/VERSIONFILE_SOURCE. If we have + # __file__, we can work backwards from there to the root. Some + # py2exe/bbfreeze/non-CPython implementations don't do __file__, in which + # case we can only use expanded keywords. + + cfg = get_config() + verbose = cfg.verbose -def run_command(args, cwd=None, verbose=False): try: - # remember shell=False, so use git.exe on windows, not just git - p = subprocess.Popen(args, stdout=subprocess.PIPE, cwd=cwd) - except EnvironmentError: - e = sys.exc_info()[1] - if verbose: - print("unable to run %s" % args[0]) - print(e) - return None - stdout = p.communicate()[0].strip() - if sys.version >= '3': - stdout = stdout.decode() - if p.returncode != 0: - if verbose: - print("unable to run %s (error)" % args[0]) - return None - return stdout + return git_versions_from_keywords(get_keywords(), cfg.tag_prefix, + verbose) + except NotThisMethod: + pass + try: + root = os.path.realpath(__file__) + # versionfile_source is the relative path from the top of the source + # tree (where the .git directory might live) to this file. Invert + # this to find the root from __file__. + for i in cfg.versionfile_source.split('/'): + root = os.path.dirname(root) + except NameError: + return {"version": "0+unknown", "full-revisionid": None, + "dirty": None, + "error": "unable to find root of source tree"} -import sys -import re -import os.path + try: + pieces = git_pieces_from_vcs(cfg.tag_prefix, root, verbose) + return render(pieces, cfg.style) + except NotThisMethod: + pass + + try: + if cfg.parentdir_prefix: + return versions_from_parentdir(cfg.parentdir_prefix, root, verbose) + except NotThisMethod: + pass -def get_expanded_variables(versionfile_source): + return {"version": "0+unknown", "full-revisionid": None, + "dirty": None, + "error": "unable to compute version"} +''' + + +@register_vcs_handler("git", "get_keywords") +def git_get_keywords(versionfile_abs): + """Extract version information from the given file.""" # the code embedded in _version.py can just fetch the value of these - # variables. When used from setup.py, we don't want to import - # _version.py, so we do it with a regexp instead. This function is not - # used from _version.py. - variables = {} + # keywords. When used from setup.py, we don't want to import _version.py, + # so we do it with a regexp instead. This function is not used from + # _version.py. + keywords = {} try: - f = open(versionfile_source,"r") + f = open(versionfile_abs, "r") for line in f.readlines(): if line.strip().startswith("git_refnames ="): mo = re.search(r'=\s*"(.*)"', line) if mo: - variables["refnames"] = mo.group(1) + keywords["refnames"] = mo.group(1) if line.strip().startswith("git_full ="): mo = re.search(r'=\s*"(.*)"', line) if mo: - variables["full"] = mo.group(1) + keywords["full"] = mo.group(1) f.close() except EnvironmentError: pass - return variables + return keywords + -def versions_from_expanded_variables(variables, tag_prefix, verbose=False): - refnames = variables["refnames"].strip() +@register_vcs_handler("git", "keywords") +def git_versions_from_keywords(keywords, tag_prefix, verbose): + """Get version information from git keywords.""" + if not keywords: + raise NotThisMethod("no keywords at all, weird") + refnames = keywords["refnames"].strip() if refnames.startswith("$Format"): if verbose: - print("variables are unexpanded, not using") - return {} # unexpanded, so not in an unpacked git-archive tarball + print("keywords are unexpanded, not using") + raise NotThisMethod("unexpanded keywords, not a git-archive tarball") refs = set([r.strip() for r in refnames.strip("()").split(",")]) # starting in git-1.8.3, tags are listed as "tag: foo-1.0" instead of # just "foo-1.0". If we see a "tag: " prefix, prefer those. @@ -379,107 +1028,122 @@ def versions_from_expanded_variables(variables, tag_prefix, verbose=False): r = ref[len(tag_prefix):] if verbose: print("picking %s" % r) - return { "version": r, - "full": variables["full"].strip() } - # no suitable tags, so we use the full revision id + return {"version": r, + "full-revisionid": keywords["full"].strip(), + "dirty": False, "error": None + } + # no suitable tags, so version is "0+unknown", but full hex is still there if verbose: - print("no suitable tags, using full revision id") - return { "version": variables["full"].strip(), - "full": variables["full"].strip() } - -def versions_from_vcs(tag_prefix, versionfile_source, verbose=False): - # this runs 'git' from the root of the source tree. That either means - # someone ran a setup.py command (and this code is in versioneer.py, so - # IN_LONG_VERSION_PY=False, thus the containing directory is the root of - # the source tree), or someone ran a project-specific entry point (and - # this code is in _version.py, so IN_LONG_VERSION_PY=True, thus the - # containing directory is somewhere deeper in the source tree). This only - # gets called if the git-archive 'subst' variables were *not* expanded, - # and _version.py hasn't already been rewritten with a short version - # string, meaning we're inside a checked out source tree. + print("no suitable tags, using unknown + full revision id") + return {"version": "0+unknown", + "full-revisionid": keywords["full"].strip(), + "dirty": False, "error": "no suitable tags"} - try: - here = os.path.abspath(__file__) - except NameError: - # some py2exe/bbfreeze/non-CPython implementations don't do __file__ - return {} # not always correct - - # versionfile_source is the relative path from the top of the source tree - # (where the .git directory might live) to this file. Invert this to find - # the root from __file__. - root = here - if IN_LONG_VERSION_PY: - for i in range(len(versionfile_source.split("/"))): - root = os.path.dirname(root) - else: - root = os.path.dirname(here) + +@register_vcs_handler("git", "pieces_from_vcs") +def git_pieces_from_vcs(tag_prefix, root, verbose, run_command=run_command): + """Get version from 'git describe' in the root of the source tree. + + This only gets called if the git-archive 'subst' keywords were *not* + expanded, and _version.py hasn't already been rewritten with a short + version string, meaning we're inside a checked out source tree. + """ if not os.path.exists(os.path.join(root, ".git")): if verbose: print("no .git in %s" % root) - return {} + raise NotThisMethod("no .git directory") - GIT = "git" + GITS = ["git"] if sys.platform == "win32": - GIT = "git.exe" - stdout = run_command([GIT, "describe", "--tags", "--dirty", "--always"], - cwd=root) - if stdout is None: - return {} - if not stdout.startswith(tag_prefix): - if verbose: - print("tag '%s' doesn't start with prefix '%s'" % (stdout, tag_prefix)) - return {} - tag = stdout[len(tag_prefix):] - stdout = run_command([GIT, "rev-parse", "HEAD"], cwd=root) - if stdout is None: - return {} - full = stdout.strip() - if tag.endswith("-dirty"): - full += "-dirty" - return {"version": tag, "full": full} - - -def versions_from_parentdir(parentdir_prefix, versionfile_source, verbose=False): - if IN_LONG_VERSION_PY: - # We're running from _version.py. If it's from a source tree - # (execute-in-place), we can work upwards to find the root of the - # tree, and then check the parent directory for a version string. If - # it's in an installed application, there's no hope. - try: - here = os.path.abspath(__file__) - except NameError: - # py2exe/bbfreeze/non-CPython don't have __file__ - return {} # without __file__, we have no hope - # versionfile_source is the relative path from the top of the source - # tree to _version.py. Invert this to find the root from __file__. - root = here - for i in range(len(versionfile_source.split("/"))): - root = os.path.dirname(root) + GITS = ["git.cmd", "git.exe"] + # if there is a tag matching tag_prefix, this yields TAG-NUM-gHEX[-dirty] + # if there isn't one, this yields HEX[-dirty] (no NUM) + describe_out = run_command(GITS, ["describe", "--tags", "--dirty", + "--always", "--long", + "--match", "%s*" % tag_prefix], + cwd=root) + # --long was added in git-1.5.5 + if describe_out is None: + raise NotThisMethod("'git describe' failed") + describe_out = describe_out.strip() + full_out = run_command(GITS, ["rev-parse", "HEAD"], cwd=root) + if full_out is None: + raise NotThisMethod("'git rev-parse' failed") + full_out = full_out.strip() + + pieces = {} + pieces["long"] = full_out + pieces["short"] = full_out[:7] # maybe improved later + pieces["error"] = None + + # parse describe_out. It will be like TAG-NUM-gHEX[-dirty] or HEX[-dirty] + # TAG might have hyphens. + git_describe = describe_out + + # look for -dirty suffix + dirty = git_describe.endswith("-dirty") + pieces["dirty"] = dirty + if dirty: + git_describe = git_describe[:git_describe.rindex("-dirty")] + + # now we have TAG-NUM-gHEX or HEX + + if "-" in git_describe: + # TAG-NUM-gHEX + mo = re.search(r'^(.+)-(\d+)-g([0-9a-f]+)$', git_describe) + if not mo: + # unparseable. Maybe git-describe is misbehaving? + pieces["error"] = ("unable to parse git-describe output: '%s'" + % describe_out) + return pieces + + # tag + full_tag = mo.group(1) + if not full_tag.startswith(tag_prefix): + if verbose: + fmt = "tag '%s' doesn't start with prefix '%s'" + print(fmt % (full_tag, tag_prefix)) + pieces["error"] = ("tag '%s' doesn't start with prefix '%s'" + % (full_tag, tag_prefix)) + return pieces + pieces["closest-tag"] = full_tag[len(tag_prefix):] + + # distance: number of commits since tag + pieces["distance"] = int(mo.group(2)) + + # commit: short hex revision ID + pieces["short"] = mo.group(3) + else: - # we're running from versioneer.py, which means we're running from - # the setup.py in a source tree. sys.argv[0] is setup.py in the root. - here = os.path.abspath(sys.argv[0]) - root = os.path.dirname(here) + # HEX: no tags + pieces["closest-tag"] = None + count_out = run_command(GITS, ["rev-list", "HEAD", "--count"], + cwd=root) + pieces["distance"] = int(count_out) # total number of commits - # Source tarballs conventionally unpack into a directory that includes - # both the project name and a version string. - dirname = os.path.basename(root) - if not dirname.startswith(parentdir_prefix): - if verbose: - print("guessing rootdir is '%s', but '%s' doesn't start with prefix '%s'" % - (root, dirname, parentdir_prefix)) - return None - return {"version": dirname[len(parentdir_prefix):], "full": ""} + return pieces -import sys -def do_vcs_install(versionfile_source, ipy): - GIT = "git" +def do_vcs_install(manifest_in, versionfile_source, ipy): + """Git-specific installation logic for Versioneer. + + For Git, this means creating/changing .gitattributes to mark _version.py + for export-time keyword substitution. + """ + GITS = ["git"] if sys.platform == "win32": - GIT = "git.exe" - run_command([GIT, "add", "versioneer.py"]) - run_command([GIT, "add", versionfile_source]) - run_command([GIT, "add", ipy]) + GITS = ["git.cmd", "git.exe"] + files = [manifest_in, versionfile_source] + if ipy: + files.append(ipy) + try: + me = __file__ + if me.endswith(".pyc") or me.endswith(".pyo"): + me = os.path.splitext(me)[0] + ".py" + versioneer_file = os.path.relpath(me) + except NameError: + versioneer_file = "versioneer.py" + files.append(versioneer_file) present = False try: f = open(".gitattributes", "r") @@ -494,135 +1158,487 @@ def do_vcs_install(versionfile_source, ipy): f = open(".gitattributes", "a+") f.write("%s export-subst\n" % versionfile_source) f.close() - run_command([GIT, "add", ".gitattributes"]) + files.append(".gitattributes") + run_command(GITS, ["add", "--"] + files) + +def versions_from_parentdir(parentdir_prefix, root, verbose): + """Try to determine the version from the parent directory name. + + Source tarballs conventionally unpack into a directory that includes + both the project name and a version string. + """ + dirname = os.path.basename(root) + if not dirname.startswith(parentdir_prefix): + if verbose: + print("guessing rootdir is '%s', but '%s' doesn't start with " + "prefix '%s'" % (root, dirname, parentdir_prefix)) + raise NotThisMethod("rootdir doesn't start with parentdir_prefix") + return {"version": dirname[len(parentdir_prefix):], + "full-revisionid": None, + "dirty": False, "error": None} SHORT_VERSION_PY = """ -# This file was generated by 'versioneer.py' (0.7+) from +# This file was generated by 'versioneer.py' (0.16) from # revision-control system data, or from the parent directory name of an # unpacked source archive. Distribution tarballs contain a pre-generated copy # of this file. -version_version = '%(version)s' -version_full = '%(full)s' -def get_versions(default={}, verbose=False): - return {'version': version_version, 'full': version_full} +import json +import sys + +version_json = ''' +%s +''' # END VERSION_JSON + +def get_versions(): + return json.loads(version_json) """ -DEFAULT = {"version": "unknown", "full": "unknown"} def versions_from_file(filename): - versions = {} + """Try to determine the version from _version.py if present.""" try: - f = open(filename) + with open(filename) as f: + contents = f.read() except EnvironmentError: - return versions - for line in f.readlines(): - mo = re.match("version_version = '([^']+)'", line) - if mo: - versions["version"] = mo.group(1) - mo = re.match("version_full = '([^']+)'", line) - if mo: - versions["full"] = mo.group(1) - f.close() - return versions + raise NotThisMethod("unable to read _version.py") + mo = re.search(r"version_json = '''\n(.*)''' # END VERSION_JSON", + contents, re.M | re.S) + if not mo: + raise NotThisMethod("no version_json in _version.py") + return json.loads(mo.group(1)) + def write_to_version_file(filename, versions): - f = open(filename, "w") - f.write(SHORT_VERSION_PY % versions) - f.close() + """Write the given version number to the given _version.py file.""" + os.unlink(filename) + contents = json.dumps(versions, sort_keys=True, + indent=1, separators=(",", ": ")) + with open(filename, "w") as f: + f.write(SHORT_VERSION_PY % contents) + print("set %s to '%s'" % (filename, versions["version"])) -def get_best_versions(versionfile, tag_prefix, parentdir_prefix, - default=DEFAULT, verbose=False): - # returns dict with two keys: 'version' and 'full' - # - # extract version from first of _version.py, 'git describe', parentdir. - # This is meant to work for developers using a source checkout, for users - # of a tarball created by 'setup.py sdist', and for users of a - # tarball/zipball created by 'git archive' or github's download-from-tag - # feature. - - variables = get_expanded_variables(versionfile_source) - if variables: - ver = versions_from_expanded_variables(variables, tag_prefix) - if ver: - if verbose: print("got version from expanded variable %s" % ver) - return ver +def plus_or_dot(pieces): + """Return a + if we don't already have one, else return a .""" + if "+" in pieces.get("closest-tag", ""): + return "." + return "+" - ver = versions_from_file(versionfile) - if ver: - if verbose: print("got version from file %s %s" % (versionfile, ver)) - return ver - ver = versions_from_vcs(tag_prefix, versionfile_source, verbose) - if ver: - if verbose: print("got version from git %s" % ver) - return ver +def render_pep440(pieces): + """Build up version string, with post-release "local version identifier". - ver = versions_from_parentdir(parentdir_prefix, versionfile_source, verbose) - if ver: - if verbose: print("got version from parentdir %s" % ver) - return ver + Our goal: TAG[+DISTANCE.gHEX[.dirty]] . Note that if you + get a tagged build and then dirty it, you'll get TAG+0.gHEX.dirty + + Exceptions: + 1: no tags. git_describe was just HEX. 0+untagged.DISTANCE.gHEX[.dirty] + """ + if pieces["closest-tag"]: + rendered = pieces["closest-tag"] + if pieces["distance"] or pieces["dirty"]: + rendered += plus_or_dot(pieces) + rendered += "%d.g%s" % (pieces["distance"], pieces["short"]) + if pieces["dirty"]: + rendered += ".dirty" + else: + # exception #1 + rendered = "0+untagged.%d.g%s" % (pieces["distance"], + pieces["short"]) + if pieces["dirty"]: + rendered += ".dirty" + return rendered + + +def render_pep440_pre(pieces): + """TAG[.post.devDISTANCE] -- No -dirty. + + Exceptions: + 1: no tags. 0.post.devDISTANCE + """ + if pieces["closest-tag"]: + rendered = pieces["closest-tag"] + if pieces["distance"]: + rendered += ".post.dev%d" % pieces["distance"] + else: + # exception #1 + rendered = "0.post.dev%d" % pieces["distance"] + return rendered + + +def render_pep440_post(pieces): + """TAG[.postDISTANCE[.dev0]+gHEX] . + + The ".dev0" means dirty. Note that .dev0 sorts backwards + (a dirty tree will appear "older" than the corresponding clean one), + but you shouldn't be releasing software with -dirty anyways. + + Exceptions: + 1: no tags. 0.postDISTANCE[.dev0] + """ + if pieces["closest-tag"]: + rendered = pieces["closest-tag"] + if pieces["distance"] or pieces["dirty"]: + rendered += ".post%d" % pieces["distance"] + if pieces["dirty"]: + rendered += ".dev0" + rendered += plus_or_dot(pieces) + rendered += "g%s" % pieces["short"] + else: + # exception #1 + rendered = "0.post%d" % pieces["distance"] + if pieces["dirty"]: + rendered += ".dev0" + rendered += "+g%s" % pieces["short"] + return rendered + + +def render_pep440_old(pieces): + """TAG[.postDISTANCE[.dev0]] . + + The ".dev0" means dirty. + + Eexceptions: + 1: no tags. 0.postDISTANCE[.dev0] + """ + if pieces["closest-tag"]: + rendered = pieces["closest-tag"] + if pieces["distance"] or pieces["dirty"]: + rendered += ".post%d" % pieces["distance"] + if pieces["dirty"]: + rendered += ".dev0" + else: + # exception #1 + rendered = "0.post%d" % pieces["distance"] + if pieces["dirty"]: + rendered += ".dev0" + return rendered + + +def render_git_describe(pieces): + """TAG[-DISTANCE-gHEX][-dirty]. + + Like 'git describe --tags --dirty --always'. - if verbose: print("got version from default %s" % ver) - return default - -def get_versions(default=DEFAULT, verbose=False): - assert versionfile_source is not None, "please set versioneer.versionfile_source" - assert tag_prefix is not None, "please set versioneer.tag_prefix" - assert parentdir_prefix is not None, "please set versioneer.parentdir_prefix" - return get_best_versions(versionfile_source, tag_prefix, parentdir_prefix, - default=default, verbose=verbose) -def get_version(verbose=False): - return get_versions(verbose=verbose)["version"] - -class cmd_version(Command): - description = "report generated version string" - user_options = [] - boolean_options = [] - def initialize_options(self): + Exceptions: + 1: no tags. HEX[-dirty] (note: no 'g' prefix) + """ + if pieces["closest-tag"]: + rendered = pieces["closest-tag"] + if pieces["distance"]: + rendered += "-%d-g%s" % (pieces["distance"], pieces["short"]) + else: + # exception #1 + rendered = pieces["short"] + if pieces["dirty"]: + rendered += "-dirty" + return rendered + + +def render_git_describe_long(pieces): + """TAG-DISTANCE-gHEX[-dirty]. + + Like 'git describe --tags --dirty --always -long'. + The distance/hash is unconditional. + + Exceptions: + 1: no tags. HEX[-dirty] (note: no 'g' prefix) + """ + if pieces["closest-tag"]: + rendered = pieces["closest-tag"] + rendered += "-%d-g%s" % (pieces["distance"], pieces["short"]) + else: + # exception #1 + rendered = pieces["short"] + if pieces["dirty"]: + rendered += "-dirty" + return rendered + + +def render(pieces, style): + """Render the given version pieces into the requested style.""" + if pieces["error"]: + return {"version": "unknown", + "full-revisionid": pieces.get("long"), + "dirty": None, + "error": pieces["error"]} + + if not style or style == "default": + style = "pep440" # the default + + if style == "pep440": + rendered = render_pep440(pieces) + elif style == "pep440-pre": + rendered = render_pep440_pre(pieces) + elif style == "pep440-post": + rendered = render_pep440_post(pieces) + elif style == "pep440-old": + rendered = render_pep440_old(pieces) + elif style == "git-describe": + rendered = render_git_describe(pieces) + elif style == "git-describe-long": + rendered = render_git_describe_long(pieces) + else: + raise ValueError("unknown style '%s'" % style) + + return {"version": rendered, "full-revisionid": pieces["long"], + "dirty": pieces["dirty"], "error": None} + + +class VersioneerBadRootError(Exception): + """The project root directory is unknown or missing key files.""" + + +def get_versions(verbose=False): + """Get the project version from whatever source is available. + + Returns dict with two keys: 'version' and 'full'. + """ + if "versioneer" in sys.modules: + # see the discussion in cmdclass.py:get_cmdclass() + del sys.modules["versioneer"] + + root = get_root() + cfg = get_config_from_root(root) + + assert cfg.VCS is not None, "please set [versioneer]VCS= in setup.cfg" + handlers = HANDLERS.get(cfg.VCS) + assert handlers, "unrecognized VCS '%s'" % cfg.VCS + verbose = verbose or cfg.verbose + assert cfg.versionfile_source is not None, \ + "please set versioneer.versionfile_source" + assert cfg.tag_prefix is not None, "please set versioneer.tag_prefix" + + versionfile_abs = os.path.join(root, cfg.versionfile_source) + + # extract version from first of: _version.py, VCS command (e.g. 'git + # describe'), parentdir. This is meant to work for developers using a + # source checkout, for users of a tarball created by 'setup.py sdist', + # and for users of a tarball/zipball created by 'git archive' or github's + # download-from-tag feature or the equivalent in other VCSes. + + get_keywords_f = handlers.get("get_keywords") + from_keywords_f = handlers.get("keywords") + if get_keywords_f and from_keywords_f: + try: + keywords = get_keywords_f(versionfile_abs) + ver = from_keywords_f(keywords, cfg.tag_prefix, verbose) + if verbose: + print("got version from expanded keyword %s" % ver) + return ver + except NotThisMethod: + pass + + try: + ver = versions_from_file(versionfile_abs) + if verbose: + print("got version from file %s %s" % (versionfile_abs, ver)) + return ver + except NotThisMethod: pass - def finalize_options(self): + + from_vcs_f = handlers.get("pieces_from_vcs") + if from_vcs_f: + try: + pieces = from_vcs_f(cfg.tag_prefix, root, verbose) + ver = render(pieces, cfg.style) + if verbose: + print("got version from VCS %s" % ver) + return ver + except NotThisMethod: + pass + + try: + if cfg.parentdir_prefix: + ver = versions_from_parentdir(cfg.parentdir_prefix, root, verbose) + if verbose: + print("got version from parentdir %s" % ver) + return ver + except NotThisMethod: pass - def run(self): - ver = get_version(verbose=True) - print("Version is currently: %s" % ver) - - -class cmd_build(_build): - def run(self): - versions = get_versions(verbose=True) - _build.run(self) - # now locate _version.py in the new build/ directory and replace it - # with an updated value - target_versionfile = os.path.join(self.build_lib, versionfile_build) - print("UPDATING %s" % target_versionfile) - os.unlink(target_versionfile) - f = open(target_versionfile, "w") - f.write(SHORT_VERSION_PY % versions) - f.close() -class cmd_sdist(_sdist): - def run(self): - versions = get_versions(verbose=True) - self._versioneer_generated_versions = versions - # unless we update this, the command will keep using the old version - self.distribution.metadata.version = versions["version"] - return _sdist.run(self) - - def make_release_tree(self, base_dir, files): - _sdist.make_release_tree(self, base_dir, files) - # now locate _version.py in the new base_dir directory (remembering - # that it may be a hardlink) and replace it with an updated value - target_versionfile = os.path.join(base_dir, versionfile_source) - print("UPDATING %s" % target_versionfile) - os.unlink(target_versionfile) - f = open(target_versionfile, "w") - f.write(SHORT_VERSION_PY % self._versioneer_generated_versions) - f.close() + if verbose: + print("unable to compute version") + + return {"version": "0+unknown", "full-revisionid": None, + "dirty": None, "error": "unable to compute version"} + + +def get_version(): + """Get the short version string for this project.""" + return get_versions()["version"] + + +def get_cmdclass(): + """Get the custom setuptools/distutils subclasses used by Versioneer.""" + if "versioneer" in sys.modules: + del sys.modules["versioneer"] + # this fixes the "python setup.py develop" case (also 'install' and + # 'easy_install .'), in which subdependencies of the main project are + # built (using setup.py bdist_egg) in the same python process. Assume + # a main project A and a dependency B, which use different versions + # of Versioneer. A's setup.py imports A's Versioneer, leaving it in + # sys.modules by the time B's setup.py is executed, causing B to run + # with the wrong versioneer. Setuptools wraps the sub-dep builds in a + # sandbox that restores sys.modules to it's pre-build state, so the + # parent is protected against the child's "import versioneer". By + # removing ourselves from sys.modules here, before the child build + # happens, we protect the child from the parent's versioneer too. + # Also see https://github.com/warner/python-versioneer/issues/52 + + cmds = {} + + # we add "version" to both distutils and setuptools + from distutils.core import Command + + class cmd_version(Command): + description = "report generated version string" + user_options = [] + boolean_options = [] + + def initialize_options(self): + pass + + def finalize_options(self): + pass + + def run(self): + vers = get_versions(verbose=True) + print("Version: %s" % vers["version"]) + print(" full-revisionid: %s" % vers.get("full-revisionid")) + print(" dirty: %s" % vers.get("dirty")) + if vers["error"]: + print(" error: %s" % vers["error"]) + cmds["version"] = cmd_version + + # we override "build_py" in both distutils and setuptools + # + # most invocation pathways end up running build_py: + # distutils/build -> build_py + # distutils/install -> distutils/build ->.. + # setuptools/bdist_wheel -> distutils/install ->.. + # setuptools/bdist_egg -> distutils/install_lib -> build_py + # setuptools/install -> bdist_egg ->.. + # setuptools/develop -> ? + + # we override different "build_py" commands for both environments + if "setuptools" in sys.modules: + from setuptools.command.build_py import build_py as _build_py + else: + from distutils.command.build_py import build_py as _build_py + + class cmd_build_py(_build_py): + def run(self): + root = get_root() + cfg = get_config_from_root(root) + versions = get_versions() + _build_py.run(self) + # now locate _version.py in the new build/ directory and replace + # it with an updated value + if cfg.versionfile_build: + target_versionfile = os.path.join(self.build_lib, + cfg.versionfile_build) + print("UPDATING %s" % target_versionfile) + write_to_version_file(target_versionfile, versions) + cmds["build_py"] = cmd_build_py + + if "cx_Freeze" in sys.modules: # cx_freeze enabled? + from cx_Freeze.dist import build_exe as _build_exe + + class cmd_build_exe(_build_exe): + def run(self): + root = get_root() + cfg = get_config_from_root(root) + versions = get_versions() + target_versionfile = cfg.versionfile_source + print("UPDATING %s" % target_versionfile) + write_to_version_file(target_versionfile, versions) + + _build_exe.run(self) + os.unlink(target_versionfile) + with open(cfg.versionfile_source, "w") as f: + LONG = LONG_VERSION_PY[cfg.VCS] + f.write(LONG % + {"DOLLAR": "$", + "STYLE": cfg.style, + "TAG_PREFIX": cfg.tag_prefix, + "PARENTDIR_PREFIX": cfg.parentdir_prefix, + "VERSIONFILE_SOURCE": cfg.versionfile_source, + }) + cmds["build_exe"] = cmd_build_exe + del cmds["build_py"] + + # we override different "sdist" commands for both environments + if "setuptools" in sys.modules: + from setuptools.command.sdist import sdist as _sdist + else: + from distutils.command.sdist import sdist as _sdist + + class cmd_sdist(_sdist): + def run(self): + versions = get_versions() + self._versioneer_generated_versions = versions + # unless we update this, the command will keep using the old + # version + self.distribution.metadata.version = versions["version"] + return _sdist.run(self) + + def make_release_tree(self, base_dir, files): + root = get_root() + cfg = get_config_from_root(root) + _sdist.make_release_tree(self, base_dir, files) + # now locate _version.py in the new base_dir directory + # (remembering that it may be a hardlink) and replace it with an + # updated value + target_versionfile = os.path.join(base_dir, cfg.versionfile_source) + print("UPDATING %s" % target_versionfile) + write_to_version_file(target_versionfile, + self._versioneer_generated_versions) + cmds["sdist"] = cmd_sdist + + return cmds + + +CONFIG_ERROR = """ +setup.cfg is missing the necessary Versioneer configuration. You need +a section like: + + [versioneer] + VCS = git + style = pep440 + versionfile_source = src/myproject/_version.py + versionfile_build = myproject/_version.py + tag_prefix = + parentdir_prefix = myproject- + +You will also need to edit your setup.py to use the results: + + import versioneer + setup(version=versioneer.get_version(), + cmdclass=versioneer.get_cmdclass(), ...) + +Please read the docstring in ./versioneer.py for configuration instructions, +edit setup.cfg, and re-run the installer or 'python versioneer.py setup'. +""" + +SAMPLE_CONFIG = """ +# See the docstring in versioneer.py for instructions. Note that you must +# re-run 'versioneer.py setup' after changing this section, and commit the +# resulting files. + +[versioneer] +#VCS = git +#style = pep440 +#versionfile_source = +#versionfile_build = +#tag_prefix = +#parentdir_prefix = + +""" INIT_PY_SNIPPET = """ from ._version import get_versions @@ -630,40 +1646,129 @@ __version__ = get_versions()['version'] del get_versions """ -class cmd_update_files(Command): - description = "modify __init__.py and create _version.py" - user_options = [] - boolean_options = [] - def initialize_options(self): - pass - def finalize_options(self): - pass - def run(self): - ipy = os.path.join(os.path.dirname(versionfile_source), "__init__.py") - print(" creating %s" % versionfile_source) - f = open(versionfile_source, "w") - f.write(LONG_VERSION_PY % {"DOLLAR": "$", - "TAG_PREFIX": tag_prefix, - "PARENTDIR_PREFIX": parentdir_prefix, - "VERSIONFILE_SOURCE": versionfile_source, - }) - f.close() + +def do_setup(): + """Main VCS-independent setup function for installing Versioneer.""" + root = get_root() + try: + cfg = get_config_from_root(root) + except (EnvironmentError, configparser.NoSectionError, + configparser.NoOptionError) as e: + if isinstance(e, (EnvironmentError, configparser.NoSectionError)): + print("Adding sample versioneer config to setup.cfg", + file=sys.stderr) + with open(os.path.join(root, "setup.cfg"), "a") as f: + f.write(SAMPLE_CONFIG) + print(CONFIG_ERROR, file=sys.stderr) + return 1 + + print(" creating %s" % cfg.versionfile_source) + with open(cfg.versionfile_source, "w") as f: + LONG = LONG_VERSION_PY[cfg.VCS] + f.write(LONG % {"DOLLAR": "$", + "STYLE": cfg.style, + "TAG_PREFIX": cfg.tag_prefix, + "PARENTDIR_PREFIX": cfg.parentdir_prefix, + "VERSIONFILE_SOURCE": cfg.versionfile_source, + }) + + ipy = os.path.join(os.path.dirname(cfg.versionfile_source), + "__init__.py") + if os.path.exists(ipy): try: - old = open(ipy, "r").read() + with open(ipy, "r") as f: + old = f.read() except EnvironmentError: old = "" if INIT_PY_SNIPPET not in old: print(" appending to %s" % ipy) - f = open(ipy, "a") - f.write(INIT_PY_SNIPPET) - f.close() + with open(ipy, "a") as f: + f.write(INIT_PY_SNIPPET) else: print(" %s unmodified" % ipy) - do_vcs_install(versionfile_source, ipy) + else: + print(" %s doesn't exist, ok" % ipy) + ipy = None + + # Make sure both the top-level "versioneer.py" and versionfile_source + # (PKG/_version.py, used by runtime code) are in MANIFEST.in, so + # they'll be copied into source distributions. Pip won't be able to + # install the package without this. + manifest_in = os.path.join(root, "MANIFEST.in") + simple_includes = set() + try: + with open(manifest_in, "r") as f: + for line in f: + if line.startswith("include "): + for include in line.split()[1:]: + simple_includes.add(include) + except EnvironmentError: + pass + # That doesn't cover everything MANIFEST.in can do + # (http://docs.python.org/2/distutils/sourcedist.html#commands), so + # it might give some false negatives. Appending redundant 'include' + # lines is safe, though. + if "versioneer.py" not in simple_includes: + print(" appending 'versioneer.py' to MANIFEST.in") + with open(manifest_in, "a") as f: + f.write("include versioneer.py\n") + else: + print(" 'versioneer.py' already in MANIFEST.in") + if cfg.versionfile_source not in simple_includes: + print(" appending versionfile_source ('%s') to MANIFEST.in" % + cfg.versionfile_source) + with open(manifest_in, "a") as f: + f.write("include %s\n" % cfg.versionfile_source) + else: + print(" versionfile_source already in MANIFEST.in") -def get_cmdclass(): - return {'version': cmd_version, - 'update_files': cmd_update_files, - 'build': cmd_build, - 'sdist': cmd_sdist, - } + # Make VCS-specific changes. For git, this means creating/changing + # .gitattributes to mark _version.py for export-time keyword + # substitution. + do_vcs_install(manifest_in, cfg.versionfile_source, ipy) + return 0 + + +def scan_setup_py(): + """Validate the contents of setup.py against Versioneer's expectations.""" + found = set() + setters = False + errors = 0 + with open("setup.py", "r") as f: + for line in f.readlines(): + if "import versioneer" in line: + found.add("import") + if "versioneer.get_cmdclass()" in line: + found.add("cmdclass") + if "versioneer.get_version()" in line: + found.add("get_version") + if "versioneer.VCS" in line: + setters = True + if "versioneer.versionfile_source" in line: + setters = True + if len(found) != 3: + print("") + print("Your setup.py appears to be missing some important items") + print("(but I might be wrong). Please make sure it has something") + print("roughly like the following:") + print("") + print(" import versioneer") + print(" setup( version=versioneer.get_version(),") + print(" cmdclass=versioneer.get_cmdclass(), ...)") + print("") + errors += 1 + if setters: + print("You should remove lines like 'versioneer.VCS = ' and") + print("'versioneer.versionfile_source = ' . This configuration") + print("now lives in setup.cfg, and should be removed from setup.py") + print("") + errors += 1 + return errors + +if __name__ == "__main__": + cmd = sys.argv[1] + if cmd == "setup": + errors = do_setup() + errors += scan_setup_py() + if errors: + sys.exit(1) -- cgit v1.2.3 From 861b16d49e81a30be9386d529d4a3339d91ff30c Mon Sep 17 00:00:00 2001 From: Kali Kaneko Date: Thu, 7 Apr 2016 09:54:55 -0400 Subject: [feature] use same token for imap/stmp authentication This greatly simplifies the handling of the password in the thunderbird extension. Related: #6041 --- src/leap/mail/imap/service/imap.py | 6 ++++-- src/leap/mail/smtp/gateway.py | 6 ++++-- 2 files changed, 8 insertions(+), 4 deletions(-) diff --git a/src/leap/mail/imap/service/imap.py b/src/leap/mail/imap/service/imap.py index 9e34454..6a2fca8 100644 --- a/src/leap/mail/imap/service/imap.py +++ b/src/leap/mail/imap/service/imap.py @@ -87,8 +87,10 @@ class LocalSoledadIMAPRealm(object): class IMAPTokenChecker(LocalSoledadTokenChecker): - """A credentials checker that will lookup a token for the IMAP service.""" - service = 'imap' + """A credentials checker that will lookup a token for the IMAP service. + For now it will be using the same identifier than SMTPTokenChecker""" + + service = 'mail_auth' class LocalSoledadIMAPServer(LEAPIMAPServer): diff --git a/src/leap/mail/smtp/gateway.py b/src/leap/mail/smtp/gateway.py index cb1b060..bd0be6f 100644 --- a/src/leap/mail/smtp/gateway.py +++ b/src/leap/mail/smtp/gateway.py @@ -144,8 +144,10 @@ class LocalSMTPRealm(object): class SMTPTokenChecker(LocalSoledadTokenChecker): - """A credentials checker that will lookup a token for the SMTP service.""" - service = 'smtp' + """A credentials checker that will lookup a token for the SMTP service. + For now it will be using the same identifier than IMAPTokenChecker""" + + service = 'mail_auth' # TODO besides checking for token credential, # we could also verify the certificate here. -- cgit v1.2.3 From ec7ccd73dd80b51c349fc6240d5bc2a7a556fbd4 Mon Sep 17 00:00:00 2001 From: Caio Carrara Date: Mon, 11 Apr 2016 09:42:40 -0300 Subject: Remove leftover print statement The print statement only printed a number. Seeing the print you cannot know what was printed. Seems that this line was left during a debug process. --- src/leap/mail/walk.py | 1 - 1 file changed, 1 deletion(-) diff --git a/src/leap/mail/walk.py b/src/leap/mail/walk.py index b6fea8d..17349e6 100644 --- a/src/leap/mail/walk.py +++ b/src/leap/mail/walk.py @@ -204,7 +204,6 @@ def walk_msg_tree(parts, body_phash=None): last_part = max(main_pmap.keys()) main_pmap[last_part][PART_MAP] = {} for partind in range(len(pv) - 1): - print partind + 1, len(parts) main_pmap[last_part][PART_MAP][partind] = parts[partind + 1] outer = parts[0] -- cgit v1.2.3 From d8ee6357c518a17a243840e42aeeae948bc03b2f Mon Sep 17 00:00:00 2001 From: Caio Carrara Date: Wed, 13 Apr 2016 16:12:09 -0300 Subject: [bug] Adds user_id to Account Previously Account used user id from the store, but this attribute is optional and None by default. This caused the collection_mapping to be unable to distinct between multiple users message collections. This chance adds a non optional user_id attribute to Account and use it to index the collection_mapping. - Resolves: https://github.com/pixelated/pixelated-user-agent/issues/674 - Releases: 0.4.0 --- src/leap/mail/imap/account.py | 2 +- src/leap/mail/mail.py | 7 ++++--- src/leap/mail/smtp/bounces.py | 3 ++- src/leap/mail/tests/test_mail.py | 14 +++++++------- 4 files changed, 14 insertions(+), 12 deletions(-) diff --git a/src/leap/mail/imap/account.py b/src/leap/mail/imap/account.py index 2f9ed1d..0b8e019 100644 --- a/src/leap/mail/imap/account.py +++ b/src/leap/mail/imap/account.py @@ -88,7 +88,7 @@ class IMAPAccount(object): # about user_id, only the client backend. self.user_id = user_id - self.account = Account(store, ready_cb=lambda: d.callback(self)) + self.account = Account(store, user_id, ready_cb=lambda: d.callback(self)) def end_session(self): """ diff --git a/src/leap/mail/mail.py b/src/leap/mail/mail.py index c6e053c..d3659de 100644 --- a/src/leap/mail/mail.py +++ b/src/leap/mail/mail.py @@ -941,8 +941,9 @@ class Account(object): # tree we can let it be an instance attribute. _collection_mapping = defaultdict(weakref.WeakValueDictionary) - def __init__(self, store, ready_cb=None): + def __init__(self, store, user_id, ready_cb=None): self.store = store + self.user_id = user_id self.adaptor = self.adaptor_class() self.mbox_indexer = MailboxIndexer(self.store) @@ -1077,7 +1078,7 @@ class Account(object): :rtype: deferred :return: a deferred that will fire with a MessageCollection """ - collection = self._collection_mapping[self.store.userid].get( + collection = self._collection_mapping[self.user_id].get( name, None) if collection: return defer.succeed(collection) @@ -1086,7 +1087,7 @@ class Account(object): def get_collection_for_mailbox(mbox_wrapper): collection = MessageCollection( self.adaptor, self.store, self.mbox_indexer, mbox_wrapper) - self._collection_mapping[self.store.userid][name] = collection + self._collection_mapping[self.user_id][name] = collection return collection d = self.adaptor.get_or_create_mbox(self.store, name) diff --git a/src/leap/mail/smtp/bounces.py b/src/leap/mail/smtp/bounces.py index 64f2dd7..7a4674b 100644 --- a/src/leap/mail/smtp/bounces.py +++ b/src/leap/mail/smtp/bounces.py @@ -83,7 +83,8 @@ class Bouncer(object): def bouncerFactory(soledad): - acc = Account(soledad) + user_id = soledad.uuid + acc = Account(soledad, user_id) d = acc.callWhenReady(lambda _: acc.get_collection_by_mailbox(INBOX_NAME)) d.addCallback(lambda inbox: Bouncer(inbox)) return d diff --git a/src/leap/mail/tests/test_mail.py b/src/leap/mail/tests/test_mail.py index 9f40ffb..aca406f 100644 --- a/src/leap/mail/tests/test_mail.py +++ b/src/leap/mail/tests/test_mail.py @@ -317,12 +317,12 @@ class AccountTestCase(SoledadTestMixin): """ Tests for the Account class. """ - def get_account(self): + def get_account(self, user_id): store = self._soledad - return Account(store) + return Account(store, user_id) def test_add_mailbox(self): - acc = self.get_account() + acc = self.get_account('some_user_id') d = acc.callWhenReady(lambda _: acc.add_mailbox("TestMailbox")) d.addCallback(lambda _: acc.list_all_mailbox_names()) d.addCallback(self._test_add_mailbox_cb) @@ -333,7 +333,7 @@ class AccountTestCase(SoledadTestMixin): self.assertItemsEqual(mboxes, expected) def test_delete_mailbox(self): - acc = self.get_account() + acc = self.get_account('some_user_id') d = acc.callWhenReady(lambda _: acc.delete_mailbox("Inbox")) d.addCallback(lambda _: acc.list_all_mailbox_names()) d.addCallback(self._test_delete_mailbox_cb) @@ -344,7 +344,7 @@ class AccountTestCase(SoledadTestMixin): self.assertItemsEqual(mboxes, expected) def test_rename_mailbox(self): - acc = self.get_account() + acc = self.get_account('some_user_id') d = acc.callWhenReady(lambda _: acc.add_mailbox("OriginalMailbox")) d.addCallback(lambda _: acc.rename_mailbox( "OriginalMailbox", "RenamedMailbox")) @@ -357,7 +357,7 @@ class AccountTestCase(SoledadTestMixin): self.assertItemsEqual(mboxes, expected) def test_get_all_mailboxes(self): - acc = self.get_account() + acc = self.get_account('some_user_id') d = acc.callWhenReady(lambda _: acc.add_mailbox("OneMailbox")) d.addCallback(lambda _: acc.add_mailbox("TwoMailbox")) d.addCallback(lambda _: acc.add_mailbox("ThreeMailbox")) @@ -374,7 +374,7 @@ class AccountTestCase(SoledadTestMixin): self.assertItemsEqual(names, expected) def test_get_collection_by_mailbox(self): - acc = self.get_account() + acc = self.get_account('some_user_id') d = acc.callWhenReady(lambda _: acc.get_collection_by_mailbox("INBOX")) d.addCallback(self._test_get_collection_by_mailbox_cb) return d -- cgit v1.2.3 From ec0183a792223e6acd8343beb0e5ba1bf3df43b1 Mon Sep 17 00:00:00 2001 From: Kali Kaneko Date: Fri, 15 Apr 2016 16:06:56 -0400 Subject: [refactor] change IMAPAccount signature for consistency with the previous Account change. --- src/leap/mail/imap/account.py | 7 ++++--- src/leap/mail/imap/service/imap.py | 2 +- src/leap/mail/imap/tests/utils.py | 2 +- src/leap/mail/incoming/tests/test_incoming_mail.py | 2 +- 4 files changed, 7 insertions(+), 6 deletions(-) diff --git a/src/leap/mail/imap/account.py b/src/leap/mail/imap/account.py index 0b8e019..459b0ba 100644 --- a/src/leap/mail/imap/account.py +++ b/src/leap/mail/imap/account.py @@ -60,7 +60,7 @@ class IMAPAccount(object): selected = None - def __init__(self, user_id, store, d=defer.Deferred()): + def __init__(self, store, user_id, d=defer.Deferred()): """ Keeps track of the mailboxes and subscriptions handled by this account. @@ -69,12 +69,13 @@ class IMAPAccount(object): You can either pass a deferred to this constructor, or use `callWhenReady` method. + :param store: a Soledad instance. + :type store: Soledad + :param user_id: The identifier of the user this account belongs to (user id, in the form user@provider). :type user_id: str - :param store: a Soledad instance. - :type store: Soledad :param d: a deferred that will be fired with this IMAPAccount instance when the account is ready to be used. diff --git a/src/leap/mail/imap/service/imap.py b/src/leap/mail/imap/service/imap.py index 6a2fca8..4663854 100644 --- a/src/leap/mail/imap/service/imap.py +++ b/src/leap/mail/imap/service/imap.py @@ -73,7 +73,7 @@ class LocalSoledadIMAPRealm(object): def gotSoledad(soledad): for iface in interfaces: if iface is IAccount: - avatar = IMAPAccount(avatarId, soledad) + avatar = IMAPAccount(soledad, avatarId) return (IAccount, avatar, getattr(avatar, 'logout', lambda: None)) raise NotImplementedError(self, interfaces) diff --git a/src/leap/mail/imap/tests/utils.py b/src/leap/mail/imap/tests/utils.py index ad89e92..64a0326 100644 --- a/src/leap/mail/imap/tests/utils.py +++ b/src/leap/mail/imap/tests/utils.py @@ -158,7 +158,7 @@ class IMAP4HelperMixin(SoledadTestMixin): self._soledad.sync = Mock() d = defer.Deferred() - self.acc = IMAPAccount(USERID, self._soledad, d=d) + self.acc = IMAPAccount(self._soledad, USERID, d=d) return d d = super(IMAP4HelperMixin, self).setUp() diff --git a/src/leap/mail/incoming/tests/test_incoming_mail.py b/src/leap/mail/incoming/tests/test_incoming_mail.py index 6880496..754df9f 100644 --- a/src/leap/mail/incoming/tests/test_incoming_mail.py +++ b/src/leap/mail/incoming/tests/test_incoming_mail.py @@ -77,7 +77,7 @@ subject: independence of cyberspace def setUp(self): def getInbox(_): d = defer.Deferred() - theAccount = IMAPAccount(ADDRESS, self._soledad, d=d) + theAccount = IMAPAccount(self._soledad, ADDRESS, d=d) d.addCallback( lambda _: theAccount.getMailbox(INBOX_NAME)) return d -- cgit v1.2.3 From 0c06b3dd5b3b3277303742498300a6c90c692ee5 Mon Sep 17 00:00:00 2001 From: Kali Kaneko Date: Mon, 18 Apr 2016 11:41:53 -0400 Subject: [pkg] bump leap deps --- pkg/requirements-leap.pip | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/pkg/requirements-leap.pip b/pkg/requirements-leap.pip index feb9f37..134b783 100644 --- a/pkg/requirements-leap.pip +++ b/pkg/requirements-leap.pip @@ -1,3 +1,3 @@ -leap.common>=0.4.3 -leap.soledad.client>=0.7.0 -leap.keymanager>=0.4.0 +leap.common>=0.5.1 +leap.soledad.client>=0.8.0 +leap.keymanager>=0.5.0 -- cgit v1.2.3 From d6f260f85f8464c6db6b9e158ecc85cfc02761ac Mon Sep 17 00:00:00 2001 From: Kali Kaneko Date: Mon, 18 Apr 2016 11:51:46 -0400 Subject: [pkg] Update changelog --- CHANGELOG | 179 ---------------------------------------------------------- CHANGELOG.rst | 28 +++++++++ HISTORY | 179 ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ 3 files changed, 207 insertions(+), 179 deletions(-) delete mode 100644 CHANGELOG create mode 100644 CHANGELOG.rst create mode 100644 HISTORY diff --git a/CHANGELOG b/CHANGELOG deleted file mode 100644 index 6ca54e7..0000000 --- a/CHANGELOG +++ /dev/null @@ -1,179 +0,0 @@ -0.4.0 Oct 28, 2015: - o Expose generic and protocol-agnostic public mail API. - o Make use of the twisted-based, async soledad API. - o Create a OutgoingMail class that has the logic for encrypting, signing and - sending messages. Factors that logic out of EncryptedMessage so it can be - used by other clients. Closes: #6357. - o Refactor email fetching outside IMAP to its own independient IncomingMail - class. Closes: #6361. - o Adapt to new events api on leap.common. Related to #5359. - o Discover public keys via attachment. Closes: #5937. - o Add public key as attachment. Closes: #6617. - o Parse OpenPGP header and import keys from it. Closes: #3879. - o Don't add any footer to the emails. Closes: #4692. - o Add listener for each email added to inbox in IncomingMail. Closes: #6742. - o Ability to reindex local UIDs after a soledad sync. Closes: #6996. - o Feature: add very basic support for message sequence numbers. - o Send a BYE command to all open connections, so that the MUA is notified - when the server is shutted down. - o Fix nested multipart rendering. Closes: #7244 - o Update SMTP gateway docs. Closes #7169. - o Bugfix: fix keyerror when inserting msg on pending_inserts dict. - o Bugfix: Return the first cdoc if no body found - o Lots of style fixes and tests updates. - o If the auth token has expired signal the GUI to request her to log in again - (Closes: #7430) - o don't extract openpgp header if valid attached key (Closes: #7480) - o disable local only tcp bind on docker containers to allow access to IMAP - and SMTP. Related to #7471. - -0.3.10 Sept 26, 2014: - o MessageCollection iterator now creates the LeapMessage with the - collection reference, so setFlags will work properly. - o account#addMailbox can't allow empty mailbox names since it makes - it impossible to create it later (mailbox#__init__ will throw an - error), which makes it impossible to getMailbox or even delete it. - -0.3.9 Apr 4, 2014: - o Footer url shouldn't end in period. Closes #4791. - o Handle non-ascii headers. Closes #5021. - o Soledad writer consumes messages eagerly. Fixes failing - tests. Closes #4715. - o Convert unicode to str when raising exceptions in IMAP server. - Fixes #4830. - o Remove conversion of IMAP folder names to string. This makes the - IMAP server use twisted's transparent 7bit conversion. Fixes - #4830. - o Add a flag to be able to reset the session. Closes #4925. - o Check for none in payload detection. Closes #4933. - o Check for flags doc uniqueness before adding a message. Avoids - duplicates of a single message in the same mailbox while copying - or moving. Closes #4949. - o Correctly process attachments when signing. Fixes #5014. - o Fix bug in which destination folder sometimes was not showing - messages after copy/append. Closes #5167. - o Fix unread notifications to client UI. Only INBOX is - notified. Closes #5177. - o Fix bug in which deleted folder wouldn't show its messages - inside. Closes #5179. - o Keep processing after a decryption error. Closes #5307. - o Enqueue unsetting of recent flag. this was holding the new mails - from being displayed soonish. - o Properly parse emails crafted by Mail.app. Fixes #5013. - o Restrict adding outgoing footer to text/plain messages. - o Sanity check on last_uid setter. Avoids incomplete fetches. - o Stop providing hostname for helo in smtp gateway. Fixes #4335. - o Only try to fetch keys for multipart signed or encrypted emails. - Fixes #4671. - o Add a flag for offline mode in imap. Related to #4943. - o Flush IMAP data to disk when stopping. Closes #5095. - o Signal the client when auth token is invalid for syncing Soledad. - Fixes #5191. - o Ability to support SEARCH Commands, limited to HEADER Message-ID. - This is a quick workaround for avoiding duplicate saves in Drafts - Folder. Closes #4209. - o Use a memory store as write-buffer and read-cache. - o Implement IMAP4 non-synchronizing literals (rfc2088), so APPENDs - can be made in a single round-trip. Closes #5190. - o Defer costly operations to a pool of threads. - o Split the internal representation of messages into three distinct - documents: 1) Flags 2) Headers 3) Content. - o Make use of the Twisted MIME interface. - o Add deduplication ability to the save operation, for body and - attachments. - o Add IMessageCopier interface to mailbox implementation, so bulk - moves are costless. Closes #4654. - o Makes efficient use of indexes and count method. Closes #4616. - o Handle correctly unicode characters in emails. Closes #4838. - -0.3.8 Dec 6, 2013: - o Fail gracefully when failing to decrypt incoming messages. Closes - #4589. - o Fix a bug when adding a message with empty flags. Closes #4496 - o Allow to iterate in an empty mailbox during fetch. Closes #4603 - o Add 'signencrypt' preference to OpenPGP header on outgoing - email. Closes #3878. - o Add a header to incoming emails that reflects if a valid signature - was found when decrypting. Closes #4354. - o Add a footer to outgoing email pointing to the address where - sender keys can be fetched. Closes #4526. - o Serialize Soledad Writes for new messages. Fixes segmentation - fault when sqlcipher was been concurrently accessed from many - threads. Closes #4606 - o Set remote mail polling time to 60 seconds. Closes #4499 - -0.3.7 Nov 15, 2013: - o Uses deferToThread for sendMail. Closes #3937 - o Update pkey to allow multiple accounts. Solves: #4394 - o Change SMTP service name from "relay" to "gateway". Closes #4416. - o Identify ourselves with a fqdn, always. Closes: #4441 - o Remove 'multipart/encrypted' header after decrypting incoming - mail. Closes #4454. - o Fix several bugs with imap mailbox getUIDNext and notifiers that - were breaking the mail indexing after message deletion. This - solves also the perceived mismatch between the number of unread - mails reported by bitmask_client and the number reported by - MUAs. Closes: #4461 - o Check username in authentications. Closes: #4299 - o Reject senders that aren't the user that is currently logged - in. Fixes #3952. - o Prevent already encrypted outgoing messages from being encrypted - again. Closes #4324. - o Correctly handle email headers when gatewaying messages. Also add - OpenPGP header. Closes #4322 and #4447. - -0.3.6 Nov 1, 2013: - o Add support for non-ascii characters in emails. Closes #4000. - o Default to UTF-8 when there is no charset parsed from the mail - contents. - o Refactor get_email_charset to leap.common. - o Return the necessary references (factory, port) from IMAP4 launch - in order to be able to properly stop it. Related to #4199. - o Notify MUA of new mail, using IDLE as advertised. Closes: #3671 - o Use TLS wrapper mode instead of STARTTLS. Closes #3637. - -0.3.5 Oct 18, 2013: - o Do not log mail doc contents. - o Comply with RFC 3156. Closes #4029. - -0.3.4 Oct 4, 2013: - o Improve charset handling when exposing mails to the mail - client. Related to #3660. - o Return Twisted's smtp Port object to be able to stop listening to - it whenever we want. Related to #3873. - -0.3.3 Sep 20, 2013: - o Remove cleartext mail from logs. Closes: #3877. - -0.3.2 Sep 6, 2013: - o Make mail services bind to 127.0.0.1. Closes: #3627. - o Signal unread message to UI when message is saved locally. Closes: - #3654. - o Signal unread to UI when flag in message change. Closes: #3662. - o Use dirspec instead of plain xdg. Closes #3574. - o SMTP service invocation returns factory instance. - -0.3.1 Aug 23, 2013: - o Avoid logging dummy password on imap server. Closes: #3416 - o Do not fail while processing an empty mail, just skip it. Fixes - #3457. - o Notify of unread email explicitly every time the mailbox is - sync'ed. - o Fix signals to emit only string in the contents instead of bool or - int values. - o Improve unseen filter of email. - o Make default imap fetch period 5 minutes. Client can config it via - environment variable for debug. Closes: #3409 - o Refactor imap fetch code for better defer handling. Closes: #3423 - o Emit signals to notify UI for SMTP relay events. Closes #3464. - o Add events for notifications about imap activity. Closes: #3480 - o Update to new soledad package scheme (common, client and - server). Closes #3487. - o Improve packaging: add versioneer, parse_requirements, - classifiers. - -0.3.0 Aug 9, 2013: - o Add dependency for leap.keymanager. - o User 1984 default port for imap. - o Add client certificate authentication. Closes #3376. - o SMTP relay signs outgoing messages. diff --git a/CHANGELOG.rst b/CHANGELOG.rst new file mode 100644 index 0000000..af315ed --- /dev/null +++ b/CHANGELOG.rst @@ -0,0 +1,28 @@ +0.4.1 - 18 Apr, 2016 ++++++++++++++++++++++ + +Features +~~~~~~~~ +- `#7656 `_: Emit multi-user aware events. +- `#4008 `_: Add token-based authentication to local IMAP/SMTP services. +- `#7889 `_: Use cryptography instead of pycryptopp to reduce dependencies. +- `#7263 `_: Implement local bounces to notify user of SMTP delivery errors. +- Use twisted.cred to authenticate IMAP/SMTP users. +- Verify plain text signed email. +- Validate signature with attachments. +- Use fingerprint instead of key_id to address keys. + + +Bugfixes +~~~~~~~~ +- `#7861 `_: Use the right succeed function for passthrough encrypted email. +- `#7898 `_: Fix IMAP fetch headers +- `#7977 `_: Decode attached keys so they are recognized by keymanager. +- `#7952 `_: Specify openssl backend explicitely. +- Fix the get_body logic for corner-cases in which body is None (yet-to-be synced docs, mainly). +- Let the inbox used in IncomingMail notify any subscribed Mailbox. +- Adds user_id to Account (fixes Pixelated mail leakage). + +Misc +~~~~ +- Change IMAPAccount signature, for consistency with a previous Account change. diff --git a/HISTORY b/HISTORY new file mode 100644 index 0000000..6ca54e7 --- /dev/null +++ b/HISTORY @@ -0,0 +1,179 @@ +0.4.0 Oct 28, 2015: + o Expose generic and protocol-agnostic public mail API. + o Make use of the twisted-based, async soledad API. + o Create a OutgoingMail class that has the logic for encrypting, signing and + sending messages. Factors that logic out of EncryptedMessage so it can be + used by other clients. Closes: #6357. + o Refactor email fetching outside IMAP to its own independient IncomingMail + class. Closes: #6361. + o Adapt to new events api on leap.common. Related to #5359. + o Discover public keys via attachment. Closes: #5937. + o Add public key as attachment. Closes: #6617. + o Parse OpenPGP header and import keys from it. Closes: #3879. + o Don't add any footer to the emails. Closes: #4692. + o Add listener for each email added to inbox in IncomingMail. Closes: #6742. + o Ability to reindex local UIDs after a soledad sync. Closes: #6996. + o Feature: add very basic support for message sequence numbers. + o Send a BYE command to all open connections, so that the MUA is notified + when the server is shutted down. + o Fix nested multipart rendering. Closes: #7244 + o Update SMTP gateway docs. Closes #7169. + o Bugfix: fix keyerror when inserting msg on pending_inserts dict. + o Bugfix: Return the first cdoc if no body found + o Lots of style fixes and tests updates. + o If the auth token has expired signal the GUI to request her to log in again + (Closes: #7430) + o don't extract openpgp header if valid attached key (Closes: #7480) + o disable local only tcp bind on docker containers to allow access to IMAP + and SMTP. Related to #7471. + +0.3.10 Sept 26, 2014: + o MessageCollection iterator now creates the LeapMessage with the + collection reference, so setFlags will work properly. + o account#addMailbox can't allow empty mailbox names since it makes + it impossible to create it later (mailbox#__init__ will throw an + error), which makes it impossible to getMailbox or even delete it. + +0.3.9 Apr 4, 2014: + o Footer url shouldn't end in period. Closes #4791. + o Handle non-ascii headers. Closes #5021. + o Soledad writer consumes messages eagerly. Fixes failing + tests. Closes #4715. + o Convert unicode to str when raising exceptions in IMAP server. + Fixes #4830. + o Remove conversion of IMAP folder names to string. This makes the + IMAP server use twisted's transparent 7bit conversion. Fixes + #4830. + o Add a flag to be able to reset the session. Closes #4925. + o Check for none in payload detection. Closes #4933. + o Check for flags doc uniqueness before adding a message. Avoids + duplicates of a single message in the same mailbox while copying + or moving. Closes #4949. + o Correctly process attachments when signing. Fixes #5014. + o Fix bug in which destination folder sometimes was not showing + messages after copy/append. Closes #5167. + o Fix unread notifications to client UI. Only INBOX is + notified. Closes #5177. + o Fix bug in which deleted folder wouldn't show its messages + inside. Closes #5179. + o Keep processing after a decryption error. Closes #5307. + o Enqueue unsetting of recent flag. this was holding the new mails + from being displayed soonish. + o Properly parse emails crafted by Mail.app. Fixes #5013. + o Restrict adding outgoing footer to text/plain messages. + o Sanity check on last_uid setter. Avoids incomplete fetches. + o Stop providing hostname for helo in smtp gateway. Fixes #4335. + o Only try to fetch keys for multipart signed or encrypted emails. + Fixes #4671. + o Add a flag for offline mode in imap. Related to #4943. + o Flush IMAP data to disk when stopping. Closes #5095. + o Signal the client when auth token is invalid for syncing Soledad. + Fixes #5191. + o Ability to support SEARCH Commands, limited to HEADER Message-ID. + This is a quick workaround for avoiding duplicate saves in Drafts + Folder. Closes #4209. + o Use a memory store as write-buffer and read-cache. + o Implement IMAP4 non-synchronizing literals (rfc2088), so APPENDs + can be made in a single round-trip. Closes #5190. + o Defer costly operations to a pool of threads. + o Split the internal representation of messages into three distinct + documents: 1) Flags 2) Headers 3) Content. + o Make use of the Twisted MIME interface. + o Add deduplication ability to the save operation, for body and + attachments. + o Add IMessageCopier interface to mailbox implementation, so bulk + moves are costless. Closes #4654. + o Makes efficient use of indexes and count method. Closes #4616. + o Handle correctly unicode characters in emails. Closes #4838. + +0.3.8 Dec 6, 2013: + o Fail gracefully when failing to decrypt incoming messages. Closes + #4589. + o Fix a bug when adding a message with empty flags. Closes #4496 + o Allow to iterate in an empty mailbox during fetch. Closes #4603 + o Add 'signencrypt' preference to OpenPGP header on outgoing + email. Closes #3878. + o Add a header to incoming emails that reflects if a valid signature + was found when decrypting. Closes #4354. + o Add a footer to outgoing email pointing to the address where + sender keys can be fetched. Closes #4526. + o Serialize Soledad Writes for new messages. Fixes segmentation + fault when sqlcipher was been concurrently accessed from many + threads. Closes #4606 + o Set remote mail polling time to 60 seconds. Closes #4499 + +0.3.7 Nov 15, 2013: + o Uses deferToThread for sendMail. Closes #3937 + o Update pkey to allow multiple accounts. Solves: #4394 + o Change SMTP service name from "relay" to "gateway". Closes #4416. + o Identify ourselves with a fqdn, always. Closes: #4441 + o Remove 'multipart/encrypted' header after decrypting incoming + mail. Closes #4454. + o Fix several bugs with imap mailbox getUIDNext and notifiers that + were breaking the mail indexing after message deletion. This + solves also the perceived mismatch between the number of unread + mails reported by bitmask_client and the number reported by + MUAs. Closes: #4461 + o Check username in authentications. Closes: #4299 + o Reject senders that aren't the user that is currently logged + in. Fixes #3952. + o Prevent already encrypted outgoing messages from being encrypted + again. Closes #4324. + o Correctly handle email headers when gatewaying messages. Also add + OpenPGP header. Closes #4322 and #4447. + +0.3.6 Nov 1, 2013: + o Add support for non-ascii characters in emails. Closes #4000. + o Default to UTF-8 when there is no charset parsed from the mail + contents. + o Refactor get_email_charset to leap.common. + o Return the necessary references (factory, port) from IMAP4 launch + in order to be able to properly stop it. Related to #4199. + o Notify MUA of new mail, using IDLE as advertised. Closes: #3671 + o Use TLS wrapper mode instead of STARTTLS. Closes #3637. + +0.3.5 Oct 18, 2013: + o Do not log mail doc contents. + o Comply with RFC 3156. Closes #4029. + +0.3.4 Oct 4, 2013: + o Improve charset handling when exposing mails to the mail + client. Related to #3660. + o Return Twisted's smtp Port object to be able to stop listening to + it whenever we want. Related to #3873. + +0.3.3 Sep 20, 2013: + o Remove cleartext mail from logs. Closes: #3877. + +0.3.2 Sep 6, 2013: + o Make mail services bind to 127.0.0.1. Closes: #3627. + o Signal unread message to UI when message is saved locally. Closes: + #3654. + o Signal unread to UI when flag in message change. Closes: #3662. + o Use dirspec instead of plain xdg. Closes #3574. + o SMTP service invocation returns factory instance. + +0.3.1 Aug 23, 2013: + o Avoid logging dummy password on imap server. Closes: #3416 + o Do not fail while processing an empty mail, just skip it. Fixes + #3457. + o Notify of unread email explicitly every time the mailbox is + sync'ed. + o Fix signals to emit only string in the contents instead of bool or + int values. + o Improve unseen filter of email. + o Make default imap fetch period 5 minutes. Client can config it via + environment variable for debug. Closes: #3409 + o Refactor imap fetch code for better defer handling. Closes: #3423 + o Emit signals to notify UI for SMTP relay events. Closes #3464. + o Add events for notifications about imap activity. Closes: #3480 + o Update to new soledad package scheme (common, client and + server). Closes #3487. + o Improve packaging: add versioneer, parse_requirements, + classifiers. + +0.3.0 Aug 9, 2013: + o Add dependency for leap.keymanager. + o User 1984 default port for imap. + o Add client certificate authentication. Closes #3376. + o SMTP relay signs outgoing messages. -- cgit v1.2.3