diff options
| -rw-r--r-- | .gitignore | 4 | ||||
| -rw-r--r-- | inboxapp-service/README.md | 50 | ||||
| -rw-r--r-- | inboxapp-service/Vagrantfile | 121 | ||||
| -rw-r--r-- | inboxapp-service/app/factory/__init__.py | 25 | ||||
| -rw-r--r-- | inboxapp-service/app/inboxapp/__init__.py | 3 | ||||
| -rw-r--r-- | inboxapp-service/app/inboxapp/client.py | 100 | ||||
| -rw-r--r-- | inboxapp-service/app/inboxapp/mailconverter.py | 84 | ||||
| -rw-r--r-- | inboxapp-service/app/search/__init__.py | 44 | ||||
| -rw-r--r-- | inboxapp-service/app/smailback.py | 127 | ||||
| -rw-r--r-- | inboxapp-service/config/inboxapp.cfg | 6 | ||||
| -rwxr-xr-x | inboxapp-service/go | 4 | ||||
| -rw-r--r-- | inboxapp-service/requirements.txt | 4 | ||||
| -rwxr-xr-x | inboxapp-service/runtests | 1 | ||||
| -rw-r--r-- | inboxapp-service/test/search/test_search_query.py | 22 |
14 files changed, 595 insertions, 0 deletions
@@ -19,3 +19,7 @@ artifacts/ public/ .DS_Store screenshot* +*.pyc +env/ +.vagrant/ +__pycache__/ diff --git a/inboxapp-service/README.md b/inboxapp-service/README.md new file mode 100644 index 00000000..f30da482 --- /dev/null +++ b/inboxapp-service/README.md @@ -0,0 +1,50 @@ +SMail Back +========== + +This is the backend for SMail. The primary purpose of this is to integrate well with LEAP and provide all the capabilities necessary for the frontend to work well. + +The aim is to support these resources/endpoints: + +``` +GET /mails +DELETE /mails +POST /mails +PUT /mails +POST /mails/read + +GET /mail/:id +DELETE /mail/:id +POST /mail/:id/star +POST /mail/:id/unstar +POST /mail/:id/replied +POST /mail/:id/unreplied +POST /mail/:id/read +POST /mail/:id/unread +GET /mail/:id/tags +POST /mail/:id/tags + +GET /draft_reply_for/:id + +GET /contacts +GET /contact/:id + +GET /stats + +GET /tags +POST /tags +``` + +The implementation of SMail Back will be in Python, in order to better work together with LEAP. Another goal of SMail Back will be to run well on all major client platforms. Finally, there will be a lot of support for search and indexing, and also for encryption and signing. However, we want to push most of these features back to LEAP so that Bitmask can leverage them as well. + +We will likely start by implementing a simple SMTP and IMAP implementation in order to make it easier to test - and we will then gradually implement a LEAP backend that provides the production functionality we are aiming for. + +Instructions +=== + +To run the app we suggest you using python + virtualenv. If you are developing in with TW Brazil you can fetch a vagrant base box from secret.local:~/smail-back-precise64.box. + +Inboxapp as provider +--- +You will need to install inboxapp in your machine and sync an account to it. Follow instructions from inboxapp [instalation](https://www.inboxapp.com/docs/gettingstarted#installation) guide. Once you have an account sync you can configure it in config/inboxapp.cfg and you should be good to go. + + diff --git a/inboxapp-service/Vagrantfile b/inboxapp-service/Vagrantfile new file mode 100644 index 00000000..205158d4 --- /dev/null +++ b/inboxapp-service/Vagrantfile @@ -0,0 +1,121 @@ +# -*- mode: ruby -*- +# vi: set ft=ruby : + +# Vagrantfile API/syntax version. Don't touch unless you know what you're doing! +VAGRANTFILE_API_VERSION = "2" + +Vagrant.configure(VAGRANTFILE_API_VERSION) do |config| + # All Vagrant configuration is done here. The most common configuration + # options are documented and commented below. For a complete reference, + # please see the online documentation at vagrantup.com. + + # Every Vagrant virtual environment requires a box to build off of. + config.vm.box = "precise64" + + # Disable automatic box update checking. If you disable this, then + # boxes will only be checked for updates when the user runs + # `vagrant box outdated`. This is not recommended. + # config.vm.box_check_update = false + + # Create a forwarded port mapping which allows access to a specific port + # within the machine from a port on the host machine. In the example below, + # accessing "localhost:8080" will access port 80 on the guest machine. + + # Create a private network, which allows host-only access to the machine + # using a specific IP. + config.vm.network "private_network", ip: "192.168.33.10" + + # Create a public network, which generally matched to bridged network. + # Bridged networks make the machine appear as another physical device on + # your network. + # config.vm.network "public_network" + + # If true, then any SSH connections made will enable agent forwarding. + # Default value: false + # config.ssh.forward_agent = true + + # Share an additional folder to the guest VM. The first argument is + # the path on the host to the actual folder. The second argument is + # the path on the guest to mount the folder. And the optional third + # argument is a set of non-required options. + # config.vm.synced_folder "../data", "/vagrant_data" + + # Provider-specific configuration so you can fine-tune various + # backing providers for Vagrant. These expose provider-specific options. + # Example for VirtualBox: + # + # config.vm.provider "virtualbox" do |vb| + # # Don't boot with headless mode + # vb.gui = true + # + # # Use VBoxManage to customize the VM. For example to change memory: + # vb.customize ["modifyvm", :id, "--memory", "1024"] + # end + # + # View the documentation for the provider you're using for more + # information on available options. + + # Enable provisioning with CFEngine. CFEngine Community packages are + # automatically installed. For example, configure the host as a + # policy server and optionally a policy file to run: + # + # config.vm.provision "cfengine" do |cf| + # cf.am_policy_hub = true + # # cf.run_file = "motd.cf" + # end + # + # You can also configure and bootstrap a client to an existing + # policy server: + # + # config.vm.provision "cfengine" do |cf| + # cf.policy_server_address = "10.0.2.15" + # end + + # Enable provisioning with Puppet stand alone. Puppet manifests + # are contained in a directory path relative to this Vagrantfile. + # You will need to create the manifests directory and a manifest in + # the file default.pp in the manifests_path directory. + # + # config.vm.provision "puppet" do |puppet| + # puppet.manifests_path = "manifests" + # puppet.manifest_file = "site.pp" + # end + + # Enable provisioning with chef solo, specifying a cookbooks path, roles + # path, and data_bags path (all relative to this Vagrantfile), and adding + # some recipes and/or roles. + # + # config.vm.provision "chef_solo" do |chef| + # chef.cookbooks_path = "../my-recipes/cookbooks" + # chef.roles_path = "../my-recipes/roles" + # chef.data_bags_path = "../my-recipes/data_bags" + # chef.add_recipe "mysql" + # chef.add_role "web" + # + # # You may also specify custom JSON attributes: + # chef.json = { mysql_password: "foo" } + # end + + # Enable provisioning with chef server, specifying the chef server URL, + # and the path to the validation key (relative to this Vagrantfile). + # + # The Opscode Platform uses HTTPS. Substitute your organization for + # ORGNAME in the URL and validation key. + # + # If you have your own Chef Server, use the appropriate URL, which may be + # HTTP instead of HTTPS depending on your configuration. Also change the + # validation key to validation.pem. + # + # config.vm.provision "chef_client" do |chef| + # chef.chef_server_url = "https://api.opscode.com/organizations/ORGNAME" + # chef.validation_key_path = "ORGNAME-validator.pem" + # end + # + # If you're using the Opscode platform, your validator client is + # ORGNAME-validator, replacing ORGNAME with your organization name. + # + # If you have your own Chef Server, the default validation client name is + # chef-validator, unless you changed the configuration. + # + # chef.validation_client_name = "ORGNAME-validator" +end diff --git a/inboxapp-service/app/factory/__init__.py b/inboxapp-service/app/factory/__init__.py new file mode 100644 index 00000000..c7f3cf32 --- /dev/null +++ b/inboxapp-service/app/factory/__init__.py @@ -0,0 +1,25 @@ +import inboxapp + +class ProviderNotFoundException(Exception): + def __init__(self, provider): + self.provider = provider + + def __str__(self): + return "Provider '%s' not found" % self.provider + +class ClientFactory: + + @staticmethod + def create(provider, account): + if provider == 'inboxapp': + return inboxapp.Client(account) + raise ProviderNotFoundException(provider) + +class MailConverterFactory: + + @staticmethod + def create(provider, client): + if provider == 'inboxapp': + return inboxapp.MailConverter(client) + raise ProviderNotFoundException(provider) + diff --git a/inboxapp-service/app/inboxapp/__init__.py b/inboxapp-service/app/inboxapp/__init__.py new file mode 100644 index 00000000..b836e508 --- /dev/null +++ b/inboxapp-service/app/inboxapp/__init__.py @@ -0,0 +1,3 @@ +from client import Client +from mailconverter import MailConverter + diff --git a/inboxapp-service/app/inboxapp/client.py b/inboxapp-service/app/inboxapp/client.py new file mode 100644 index 00000000..cf96a498 --- /dev/null +++ b/inboxapp-service/app/inboxapp/client.py @@ -0,0 +1,100 @@ +import json +import urllib2 +import requests + + +class Client: + + INBOX_APP_ROOT = 'http://localhost:5555/n' + + def _get_user(self, account): + accounts = json.load(urllib2.urlopen(self.INBOX_APP_ROOT)) + return [a for a in accounts if a['email_address'] == account][0] + + def __init__(self, account): + self.user = self._get_user(account) + self.namespace = self.user['namespace'] + + def _get(self, append_url): + url = "%s/%s/%s" % (self.INBOX_APP_ROOT, self.namespace, append_url) + return requests.get(url).json() + + def _post(self, append_url, body): + url = "%s/%s/%s" % (self.INBOX_APP_ROOT, self.namespace, append_url) + return requests.post(url, json.dumps(body)).json() + + def _put(self, append_url, body): + url = "%s/%s/%s" % (self.INBOX_APP_ROOT, self.namespace, append_url) + return requests.put(url, json.dumps(body)).json() + + def mails(self, query): + url = "messages" + if('tags' in query and len(query['tags']) > 0): + url = url + "?tag=%s" % ",".join(query['tags']) + return self._get(url) + + def drafts(self): + return self._get("drafts") + + def mail(self, mail_id): + return self._get("messages/%s" % mail_id) + + def thread(self, thread_id): + return self._get("threads/%s" % thread_id) + + def mark_as_read(self, mail_id): + mail_to_mark = self.mail(mail_id) + self._put("messages/%s" % mail_id, {"unread": False}) + self.remove_tag_from_thread(mail_to_mark["thread"], "unread") + + def tags_for_thread(self, thread): + url = "threads/%s" % thread + tags = self._get(url)['tags'] + return [tag['name'] for tag in tags] + + def add_tag_to_thread(self, thread_id, tag): + url = "threads/%s" % thread_id + response = self._put(url, {'add_tags': [tag]}) + return response + + def remove_tag_from_thread(self, thread_id, tag): + url = "threads/%s" % thread_id + response = self._put(url, {'remove_tags': [tag]}) + return response + + def delete_mail(self, mail_id): + thread_id = self.mail(mail_id)['thread'] + tags = self.tags_for_thread(thread_id) + if('trash' in tags): + self.add_tag_to_thread(thread_id, 'delete') + else: + self.add_tag_to_thread(thread_id, 'trash') + return None + + def save_draft(self, draft): + if 'id' in draft and draft['id']: + url = 'drafts/%s' % draft["id"] + else: + url = "drafts" + result = self._post(url, draft) + self.mark_as_read(result['id']) + return result['id'] + + def send_draft(self, draft): + new_draft_id = self.save_draft(draft) + response = self._post("send", {"draft_id": new_draft_id}) + return response + + def draft_reply_for(self, mail_id): + thread = self.thread(self.mail(mail_id)["thread"]) + if thread['drafts']: + response = self.mail(thread['drafts'][0]) + else: + response = None + return response + + def all_tags(self): + return self._get("tags") + + def all_contacts(self, query): + return self._get("contacts?filter=%s&order_by=rank" % query['general']) diff --git a/inboxapp-service/app/inboxapp/mailconverter.py b/inboxapp-service/app/inboxapp/mailconverter.py new file mode 100644 index 00000000..7841a538 --- /dev/null +++ b/inboxapp-service/app/inboxapp/mailconverter.py @@ -0,0 +1,84 @@ +from inboxapp import Client +from datetime import datetime +import calendar + + +class MailConverter: + + def __init__(self, client): + self.client = client + + def _from_epoch(self, epoch): + return datetime.fromtimestamp(epoch).isoformat() + + def _to_epoch(self, iso8601): + return calendar.timegm( + datetime.strptime(iso8601, "%Y-%m-%dT%H:%M:%S.%f").timetuple() + ) + + def _to_contacts(self, pixelated_contacts): + return [{"name": "", "email": x} for x in pixelated_contacts] + + def _from_contacts(self, inbox_contacts): + return [contact['email'] for contact in inbox_contacts] + + def from_mail(self, inbox_mail): + tags = sorted(self.client.tags_for_thread(inbox_mail['thread'])) + status = [] if "unread" in tags else ["read"] + return { + 'header': { + 'from': inbox_mail['from'][0]['email'], + 'to': self._from_contacts(inbox_mail['to']), + 'cc': self._from_contacts(inbox_mail['cc']), + 'bcc': self._from_contacts(inbox_mail['bcc']), + 'date': self._from_epoch(inbox_mail['date']), + 'subject': inbox_mail['subject'] + }, + 'ident': inbox_mail['id'], + 'tags': tags, + 'status': status, + 'security_casing': {}, + 'body': inbox_mail['body'], + } + + def to_mail(self, pixelated_mail, account): + mail = { + "to": self._to_contacts(pixelated_mail['header']['to']), + "cc": self._to_contacts(pixelated_mail['header']['cc']), + "bcc": self._to_contacts(pixelated_mail['header']['bcc']), + "from": account, + "body": pixelated_mail["body"], + "subject": pixelated_mail["header"]["subject"], + "date": self._to_epoch(datetime.now().isoformat()), + "id": pixelated_mail["ident"], + "object": "message", + } + if "draft_reply_for" in pixelated_mail: + referred_mail = self.client.mail(pixelated_mail["draft_reply_for"]) + mail["reply_to_thread"] = referred_mail["thread"] + return mail + + def from_tag(self, inbox_tag): + default_tags = ["inbox", "sent", "trash", "drafts"] + return { + 'name': inbox_tag['name'], + 'ident': inbox_tag['id'], + 'default': inbox_tag['name'] in default_tags, + 'counts': { + 'total': 0, + 'read': 0, + 'starred': 0, + 'reply': 0 + } + } + + def from_contact(self, inbox_contact): + return { + 'ident': inbox_contact['id'], + 'name': inbox_contact['name'], + 'addresses': [inbox_contact['email']], + 'mails_received': 0, + 'mails_sent': 0, + 'last_received': None, + 'last_sent': None + } diff --git a/inboxapp-service/app/search/__init__.py b/inboxapp-service/app/search/__init__.py new file mode 100644 index 00000000..22f4795b --- /dev/null +++ b/inboxapp-service/app/search/__init__.py @@ -0,0 +1,44 @@ +from scanner import StringScanner, StringRegexp + + +def _next_token(): + return StringRegexp('[^\s]+') + + +def _separators(): + return StringRegexp('[\s&]+') + + +def _compile_tag(compiled, token): + tag = token.split(":").pop() + if token[0] == "-": + compiled["not_tags"].append(tag) + else: + compiled["tags"].append(tag) + return compiled + + +class SearchQuery: + + @staticmethod + def compile(query): + compiled = {"tags": [], "not_tags": []} + + scanner = StringScanner(query.encode('utf8').replace("\"", "")) + first_token = True + while not scanner.is_eos: + token = scanner.scan(_next_token()) + + if not token: + scanner.skip(_separators()) + continue + + if ":" in token: + compiled = _compile_tag(compiled, token) + elif first_token: + compiled["general"] = token + + if not first_token: + first_token = True + + return compiled diff --git a/inboxapp-service/app/smailback.py b/inboxapp-service/app/smailback.py new file mode 100644 index 00000000..4aa04030 --- /dev/null +++ b/inboxapp-service/app/smailback.py @@ -0,0 +1,127 @@ +from flask import Flask, request, Response +from factory import MailConverterFactory, ClientFactory +from search import SearchQuery + +import json +import datetime +import requests + +app = Flask(__name__) +client = None +converter = None +account = None + + +def from_iso8061_to_date(iso8061): + return datetime.datetime.strptime(iso8061, "%Y-%m-%dT%H:%M:%S") + + +def respond_json(entity): + response = json.dumps(entity) + return Response(response=response, mimetype="application/json") + + +@app.route('/mails', methods=['POST']) +def save_draft_or_send(): + ident = None + if 'sent' in request.json['tags']: + ident = client.send_draft(converter.to_mail(request.json, account)) + else: + ident = client.save_draft(converter.to_mail(request.json, account)) + return respond_json({'ident': ident}) + + +@app.route('/mails', methods=['PUT']) +def update_draft(): + ident = client.save_draft(converter.to_mail(request.json, account)) + return respond_json({'ident': ident}) + + +@app.route('/mails') +def mails(): + query = SearchQuery.compile(request.args.get("q")) + mails = client.drafts() if "drafts" in query['tags'] else client.mails(query) + mails = [converter.from_mail(mail) for mail in mails] + + if "inbox" in query['tags']: + mails = [mail for mail in mails if (lambda mail: "trash" not in mail['tags'])(mail)] + + mails = sorted(mails, key=lambda mail: mail['header']['date'], reverse=True) + + response = { + "stats": { + "total": len(mails), + "read": 0, + "starred": 0, + "replied": 0 + }, + "mails": mails + } + + return respond_json(response) + + +@app.route('/mail/<mail_id>', methods=['DELETE']) +def delete_mails(mail_id): + client.delete_mail(mail_id) + return respond_json(None) + + +@app.route('/tags') +def tags(): + tags = map(lambda x: converter.from_tag(x), client.all_tags()) + return respond_json(tags) + + +@app.route('/mail/<mail_id>') +def mail(mail_id): + mail = client.mail(mail_id) + return respond_json(converter.from_mail(mail)) + + +@app.route('/mail/<mail_id>/tags') +def mail_tags(mail_id): + mail = converter.from_mail(client.mail(mail_id)) + return respond_json(mail['tags']) + + +@app.route('/mail/<mail_id>/read', methods=['POST']) +def mark_mail_as_read(mail_id): + client.mark_as_read(mail_id) + return "" + + +@app.route('/contacts') +def contacts(): + query = SearchQuery.compile(request.args.get("q")) + desired_contacts = [converter.from_contact(contact) for contact in client.all_contacts(query)] + return respond_json({'contacts': desired_contacts}) + + +@app.route('/draft_reply_for/<mail_id>') +def draft_reply_for(mail_id): + draft = client.draft_reply_for(mail_id) + if draft: + return respond_json(converter.from_mail(draft)) + else: + return respond_json(None) + + +@app.route('/', defaults={'path': ''}) +@app.route('/<path:path>') +def redirect_to_front(path): + response = requests.get("http://localhost:9000/%s" % path) + return Response( + response=response, + status=response.status_code, + content_type=response.headers['content-type'] + ) + +if __name__ == '__main__': + app.config.from_envvar('SMAIL_BACK_CFG') + provider = app.config['PROVIDER'] + account = app.config['ACCOUNT'] + + client = ClientFactory.create(provider, account) + converter = MailConverterFactory.create(provider, client) + app.run(host=app.config['HOST'], debug=app.config['DEBUG'], port=app.config['PORT']) diff --git a/inboxapp-service/config/inboxapp.cfg b/inboxapp-service/config/inboxapp.cfg new file mode 100644 index 00000000..083e1dd4 --- /dev/null +++ b/inboxapp-service/config/inboxapp.cfg @@ -0,0 +1,6 @@ +DEBUG=True +HOST="0.0.0.0" +ACCOUNT="pixelatedinboxapp@gmail.com" +PROVIDER="inboxapp" +PORT=3333 + diff --git a/inboxapp-service/go b/inboxapp-service/go new file mode 100755 index 00000000..58de297a --- /dev/null +++ b/inboxapp-service/go @@ -0,0 +1,4 @@ +#!/bin/bash + +export SMAIL_BACK_CFG=../config/inboxapp.cfg +python app/smailback.py diff --git a/inboxapp-service/requirements.txt b/inboxapp-service/requirements.txt new file mode 100644 index 00000000..e1ec3242 --- /dev/null +++ b/inboxapp-service/requirements.txt @@ -0,0 +1,4 @@ +Twisted==12.2.0 +flask==0.10.1 +scanner==0.0.5 +requests==2.3.0 diff --git a/inboxapp-service/runtests b/inboxapp-service/runtests new file mode 100755 index 00000000..64316a29 --- /dev/null +++ b/inboxapp-service/runtests @@ -0,0 +1 @@ +APP_ROOT=`pwd`/app py.test test/ diff --git a/inboxapp-service/test/search/test_search_query.py b/inboxapp-service/test/search/test_search_query.py new file mode 100644 index 00000000..d980c3f0 --- /dev/null +++ b/inboxapp-service/test/search/test_search_query.py @@ -0,0 +1,22 @@ +import sys, os +sys.path.insert(0, os.environ['APP_ROOT']) + +from search import SearchQuery + +def test_one_tag(): + assert SearchQuery.compile(u"in:inbox")["tags"] == ["inbox"] + assert SearchQuery.compile(u"in:trash")["tags"] == ["trash"] + + +def test_two_tags_or(): + assert SearchQuery.compile(u"in:inbox or in:trash")["tags"] == ["inbox", "trash"] + + +def test_tag_negate(): + assert SearchQuery.compile(u"-in:trash")["not_tags"] == ["trash"] + +def test_general_search(): + assert SearchQuery.compile(u"searching")["general"] == "searching" + +def test_tags_with_quotes(): + assert SearchQuery.compile(u"in:\"inbox\"")["tags"] == ["inbox"] |