#157: mail api to return htmlBody and textPlainBody so the UI doesnt have to parse the multipart mails anymore

7 files changed, 74 insertions, 214 deletions

diff --git a/service/pixelated/adapter/model/mail.py b/service/pixelated/adapter/model/mail.py
index 4b128d8c..98533a74 100644
--- a/service/pixelated/adapter/model/mail.py
+++ b/service/pixelated/adapter/model/mail.py
@@ -25,7 +25,6 @@ import pixelated.support.date
 from email.MIMEMultipart import MIMEMultipart
 from pycryptopp.hash import sha256
 import re
-import base64
 from pixelated.support.functional import compact
 
 
@@ -65,7 +64,13 @@ class Mail(object):
         mime = MIMEMultipart()
         for key, value in self.headers.items():
             mime[str(key)] = str(value)
-        mime.attach(MIMEText(self.body, 'plain', self._charset()))
+
+        try:
+            body_to_use = self.body
+        except AttributeError:
+            body_to_use = self.text_plain_body
+
+        mime.attach(MIMEText(body_to_use, 'plain', self._charset()))
         self._mime = mime
         return mime
 
@@ -193,6 +198,7 @@ class InputMail(Mail):
 
 
 class PixelatedMail(Mail):
+
     @staticmethod
     def from_soledad(fdoc, hdoc, bdoc, parts=None, soledad_querier=None):
         mail = PixelatedMail()
@@ -205,24 +211,33 @@ class PixelatedMail(Mail):
         mail._mime = None
         return mail
 
+    def _decode_part(self, part):
+        encoding = part['headers'].get('Content-Transfer-Encoding', '')
+
+        decoding_map = {
+            'quoted-printable': lambda content: unicode(content.decode('quopri')),
+            'base64': lambda content: content.decode('base64').decode('utf-8')
+        }
+        if encoding:
+            return decoding_map[encoding](part['content'])
+        return part['content']
+
     @property
-    def body(self):
-        if self.parts and len(self.parts['alternatives']) > 1:
-            body = ''
-            for alternative in self.parts['alternatives']:
-                body += '--' + self.boundary + '\n'
-                for header, value in alternative['headers'].items():
-                    body += '%s: %s\n' % (header, value)
-                body += '\n'
-                body += alternative['content']
-                body += '\n'
-            body += '--' + self.boundary + '--'
-            return body
+    def alternatives(self):
+        return self.parts.get('alternatives')
+
+    @property
+    def text_plain_body(self):
+        if self.parts and len(self.alternatives) == 1:
+            return self._decode_part(self.alternatives[0])
         else:
-            if self.parts and self.parts['alternatives'][0]['headers'].get('Content-Transfer-Encoding', '') == 'base64':
-                return unicode(base64.b64decode(self.parts['alternatives'][0]['content']), 'utf-8')
-            else:
-                return self.bdoc.content['raw']
+            return self.bdoc.content['raw']  # plain
+
+    @property
+    def html_body(self):
+        if self.parts and len(self.alternatives) > 1:
+            html_part = [e for e in self.alternatives if re.match('text/html', e['headers']['Content-Type'])][0]
+            return self._decode_part(html_part)
 
     @property
     def headers(self):
@@ -352,7 +367,8 @@ class PixelatedMail(Mail):
                      'tags': list(self.tags),
                      'status': list(self.status),
                      'security_casing': self.security_casing,
-                     'body': self.body,
+                     'textPlainBody': self.text_plain_body,
+                     'htmlBody': self.html_body,
                      'mailbox': self.mailbox_name.lower(),
                      'attachments': self.parts['attachments'] if self.parts else []}
         dict_mail['replying'] = {'single': None, 'all': {'to-field': [], 'cc-field': []}}
diff --git a/service/pixelated/adapter/search/__init__.py b/service/pixelated/adapter/search/__init__.py
index bbefc487..a5eb4ef2 100644
--- a/service/pixelated/adapter/search/__init__.py
+++ b/service/pixelated/adapter/search/__init__.py
@@ -128,7 +128,7 @@ class SearchEngine(object):
             'cc': u','.join(header.get('cc', [''])),
             'bcc': u','.join(header.get('bcc', [''])),
             'tag': u','.join(unique(tags)),
-            'body': unicode(mdict['body']),
+            'body': unicode(mdict['textPlainBody']),
             'ident': unicode(mdict['ident']),
             'flags': unicode(','.join(unique(mail.flags))),
             'raw': unicode(mail.raw)
diff --git a/service/test/unit/adapter/mail_test.py b/service/test/unit/adapter/mail_test.py
index e0b7a498..c35bdb62 100644
--- a/service/test/unit/adapter/mail_test.py
+++ b/service/test/unit/adapter/mail_test.py
@@ -95,7 +95,10 @@ class TestPixelatedMail(unittest.TestCase):
 
         _dict = mail.as_dict()
 
-        self.assertEquals(_dict, {'body': 'body',
+        self.maxDiff = None
+
+        self.assertEquals(_dict, {'htmlBody': None,
+                                  'textPlainBody': 'body',
                                   'header': {
                                       'date': dateparser.parse(hdoc.content['date']).isoformat(),
                                       'from': 'someone@pixelated.org',
@@ -143,21 +146,20 @@ class TestPixelatedMail(unittest.TestCase):
         parts['alternatives'].append({'content': 'blablabla', 'headers': {'Content-Type': 'text/plain'}})
         parts['alternatives'].append({'content': '<p>blablabla</p>', 'headers': {'Content-Type': 'text/html'}})
 
-        mail = PixelatedMail.from_soledad(None, None, None, parts=parts, soledad_querier=None)
+        mail = PixelatedMail.from_soledad(None, None, self._create_bdoc(raw='blablabla'), parts=parts, soledad_querier=None)
 
-        self.assertRegexpMatches(mail.body, '^--' + mail.boundary + '\n.*')
-        self.assertRegexpMatches(mail.body, '\nContent-Type: text/html\n\n<p>blablabla</p>\n')
-        self.assertRegexpMatches(mail.body, '\nContent-Type: text/plain\n\nblablabla\n')
-        self.assertRegexpMatches(mail.body, '.*--' + mail.boundary + '--$')
+        self.assertRegexpMatches(mail.html_body, '^<p>blablabla</p>$')
+        self.assertRegexpMatches(mail.text_plain_body, '^blablabla$')
 
     def test_percent_character_is_allowed_on_body(self):
         parts = {'alternatives': [], 'attachments': []}
         parts['alternatives'].append({'content': '100% happy with percentage symbol', 'headers': {'Content-Type': 'text/plain'}})
         parts['alternatives'].append({'content': '<p>100% happy with percentage symbol</p>', 'headers': {'Content-Type': 'text/html'}})
 
-        mail = PixelatedMail.from_soledad(None, None, None, parts=parts, soledad_querier=None)
+        mail = PixelatedMail.from_soledad(None, None, self._create_bdoc(raw="100% happy with percentage symbol"), parts=parts, soledad_querier=None)
 
-        self.assertRegexpMatches(mail.body, '([\s\S]*100%){2}')
+        self.assertRegexpMatches(mail.text_plain_body, '([\s\S]*100%)')
+        self.assertRegexpMatches(mail.html_body, '([\s\S]*100%)')
 
     def test_clean_line_breaks_on_address_headers(self):
         many_recipients = 'One <one@mail.com>,\nTwo <two@mail.com>, Normal <normal@mail.com>,\nalone@mail.com'
@@ -175,25 +177,22 @@ class TestPixelatedMail(unittest.TestCase):
                 self.assertNotIn(',', address)
             self.assertEquals(4, len(mail.headers[header_label]))
 
-    def test_content_type_is_read_from_headers_for_plain_mail_when_converted_to_raw(self):
-        fdoc, hdoc, bdoc = test_helper.leap_mail(flags=['\\Recent'], body=u'some umlaut \xc3', extra_headers={'Content-Type': 'text/plain; charset=ISO-8859-1'})
-        hdoc.content['headers']['Subject'] = 'The subject'
-        hdoc.content['headers']['From'] = 'me@pixelated.org'
-        mail = PixelatedMail.from_soledad(fdoc, hdoc, bdoc, soledad_querier=self.querier)
-
-        mail.raw
-
     def test_that_body_understands_base64(self):
         body = u'bl\xe1'
-        encoded_body = unicode(base64.b64encode(body.encode('utf-8')))
+        encoded_body = unicode(body.encode('utf-8').encode('base64'))
 
         fdoc, hdoc, bdoc = test_helper.leap_mail()
         parts = {'alternatives': []}
         parts['alternatives'].append({'content': encoded_body, 'headers': {'Content-Transfer-Encoding': 'base64'}})
         mail = PixelatedMail.from_soledad(fdoc, hdoc, bdoc, soledad_querier=self.querier, parts=parts)
 
-        self.assertEquals(body, mail.body)
+        self.assertEquals(body, mail.text_plain_body)
 
+    def _create_bdoc(self, raw):
+        class FakeBDoc:
+            def __init__(self, raw):
+                self.content = {'raw': raw}
+        return FakeBDoc(raw)
 
 class InputMailTest(unittest.TestCase):
     mail_dict = lambda x: {
diff --git a/web-ui/app/js/helpers/view_helper.js b/web-ui/app/js/helpers/view_helper.js
index 72ced4af..48d9a9c7 100644
--- a/web-ui/app/js/helpers/view_helper.js
+++ b/web-ui/app/js/helpers/view_helper.js
@@ -31,52 +31,17 @@ define(
     }).join(' ');
   }
 
-  function addParagraphsToPlainText(plainTextBodyPart) {
-    return _.map(plainTextBodyPart.split('\n'), function (paragraph) {
+  function addParagraphsToPlainText(textPlainBody) {
+    return _.map(textPlainBody.split('\n'), function (paragraph) {
       return '<p>' + paragraph + '</p>';
     }).join('');
   }
 
-  function isQuotedPrintableBodyPart (bodyPart) {
-    return bodyPart.headers &&
-      bodyPart.headers['Content-Transfer-Encoding'] &&
-      bodyPart.headers['Content-Transfer-Encoding'] === 'quoted-printable';
-  }
-
-  function getHtmlContentType (mail) {
-    return _.find(mail.availableBodyPartsContentType(), function (contentType) {
-      return contentType.indexOf('text/html') >= 0;
-    });
-  }
-
-  function getSanitizedAndDecodedMailBody (bodyPart) {
-    var body;
-
-    if (isQuotedPrintableBodyPart(bodyPart)) {
-      body = utf8.decode(quotedPrintable.decode(bodyPart.body));
-    } else if (bodyPart.body) {
-      body = bodyPart.body;
-    } else {
-      body = bodyPart;
-    }
-
-    return htmlWhitelister.sanitize(body, htmlWhitelister.tagPolicy);
-  }
-
   function formatMailBody (mail) {
-    if (mail.isMailMultipartAlternative()) {
-      var htmlContentType;
-
-      htmlContentType = getHtmlContentType(mail);
-
-      if (htmlContentType) {
-        return $(getSanitizedAndDecodedMailBody(mail.getMailPartByContentType(htmlContentType)));
-      }
-
-      return $(getSanitizedAndDecodedMailBody(addParagraphsToPlainText(mail.getMailMultiParts[0])));
-    }
-
-    return $(getSanitizedAndDecodedMailBody(addParagraphsToPlainText(mail.body)));
+    var body = mail.htmlBodyPart ?
+                htmlWhitelister.sanitize(mail.htmlBody, htmlWhitelister.tagPolicy) :
+                addParagraphsToPlainText(mail.textPlainBody)
+    return $(body)
   }
 
   function moveCaretToEnd(el) {
@@ -125,7 +90,7 @@ define(
   }
 
   function quoteMail(mail) {
-    var quotedLines = _.map(mail.body.split('\n'), function (line) {
+    var quotedLines = _.map(mail.textPlainBody.split('\n'), function (line) {
       return '> ' + line;
     });
 
diff --git a/web-ui/test/spec/helpers/view_helper.spec.js b/web-ui/test/spec/helpers/view_helper.spec.js
index 217ac890..3399baa8 100644
--- a/web-ui/test/spec/helpers/view_helper.spec.js
+++ b/web-ui/test/spec/helpers/view_helper.spec.js
@@ -11,7 +11,7 @@ define(['helpers/view_helper'], function (viewHelper) {
 
     describe('quote email', function() {
       it('should add > to body text', function() {
-        testData.rawMail.mail.body = 'First Line\nSecond Line';
+        testData.rawMail.mail.textPlainBody = 'First Line\nSecond Line';
 
         var quotedMail = viewHelper.quoteMail(testData.rawMail.mail);
 
@@ -55,20 +55,10 @@ define(['helpers/view_helper'], function (viewHelper) {
       });
     });
 
-    it('formats the body of a multipart email', function () {
-      expect(viewHelper.formatMailBody(testData.parsedMail.html)).toContainHtml('<p>Hello everyone!</p>');
-    });
-
     it('formats the body of a plain text email', function () {
       var formatedMail = $('<div></div>');
       formatedMail.html(viewHelper.formatMailBody(testData.parsedMail.simpleTextPlain));
-      expect(formatedMail).toContainHtml('<p>Hello Everyone</p>');
-    });
-
-    it('decodes a quoted-printable email body', function () {
-      var result = viewHelper.formatMailBody(testData.parsedMail.htmlQuotedPrintable);
-
-      expect(result).toContainHtml('<p style="border: 5px;">Hello everyone!</p>');
+      expect(formatedMail).toContainHtml('<p>HNello Everyone</p>');
     });
 
     it('move caret to the end of text after 1ms', function () {
diff --git a/web-ui/test/spec/services/model/mail.spec.js b/web-ui/test/spec/services/model/mail.spec.js
index 35c4bc9b..f9d076f0 100644
--- a/web-ui/test/spec/services/model/mail.spec.js
+++ b/web-ui/test/spec/services/model/mail.spec.js
@@ -34,57 +34,6 @@ require(['services/model/mail'], function (Mail) {
         });
       });
 
-      describe('multipart email', function () {
-        var parsedMultipartMail;
-
-        beforeEach(function () {
-          parsedMultipartMail = Mail.create(Pixelated.testData().rawMail.multipart);
-        });
-
-        it('parses the mail as multipart/alternative', function () {
-          expect(parsedMultipartMail.isMailMultipartAlternative()).toBe(true);
-        });
-
-        it('lists the correct available content-type of the parts', function () {
-          expect(parsedMultipartMail.availableBodyPartsContentType()).toEqual(['text/plain;', 'text/html;']);
-        });
-
-        it('gets the list of parts', function () {
-          var expectedParts = [
-            {
-              headers: { 'Content-Type': 'text/plain;' },
-              body: 'Hello everyone!\n'
-            },
-            {
-              headers: {
-                'Content-Type': 'text/html;',
-                'Content-Transfer-Encoding': 'quoted-printable'
-              },
-              body: '<p><b>Hello everyone!</b></p>\n'
-            }
-          ];
-
-          expect(parsedMultipartMail.getMailMultiParts()).toEqual(expectedParts);
-        });
-
-        it('gets the text/plain body by the content-type', function () {
-          expect(parsedMultipartMail.getMailPartByContentType('text/plain;')).toEqual(
-            {
-              headers: { 'Content-Type': 'text/plain;' },
-              body: 'Hello everyone!\n'
-            });
-        });
-
-        it('parses the content type of a text/html body', function () {
-          expect(parsedMultipartMail.getMailPartByContentType('text/html;')).toEqual({
-              headers: {
-                'Content-Type': 'text/html;',
-                'Content-Transfer-Encoding': 'quoted-printable'
-              },
-              body: '<p><b>Hello everyone!</b></p>\n'
-          });
-        });
-      });
     });
   });
 });
diff --git a/web-ui/test/test_data.js b/web-ui/test/test_data.js
index 20b6ee0f..64ec60a7 100644
--- a/web-ui/test/test_data.js
+++ b/web-ui/test/test_data.js
@@ -12,7 +12,7 @@ define(function() {
     ident:2048,
     tags:['gang_family','garden','nailartaddicts','inbox'],
     status:[],
-    body: 'Porro quam minus. Doloribus odio vel. Placeat alias sed est assumenda qui esse. Tenetur tempora deserunt est consequatur ducimus laborum. Velit dolor voluptatibus.\n\nRerum repellendus tempore. Aliquam dolores laudantium amet et dolor voluptas. Quod eos magni mollitia et ex. Corrupti quis reprehenderit quasi. Quam cum nobis voluptas accusamus quisquam ut asperiores.\n\nFacilis dicta mollitia non molestiae. Eligendi perspiciatis aut qui eos qui. Laborum cumque odit velit nobis. Cumque quo impedit dignissimos quia.',
+    textPlainBody: 'Porro quam minus. Doloribus odio vel. Placeat alias sed est assumenda qui esse. Tenetur tempora deserunt est consequatur ducimus laborum. Velit dolor voluptatibus.\n\nRerum repellendus tempore. Aliquam dolores laudantium amet et dolor voluptas. Quod eos magni mollitia et ex. Corrupti quis reprehenderit quasi. Quam cum nobis voluptas accusamus quisquam ut asperiores.\n\nFacilis dicta mollitia non molestiae. Eligendi perspiciatis aut qui eos qui. Laborum cumque odit velit nobis. Cumque quo impedit dignissimos quia.',
     security_casing: {
       locks: [],
       imprints: []
@@ -31,7 +31,7 @@ define(function() {
     'ident':9359,
     'tags':['photography','sky'],
     'status':['read'],
-    'body':'Illum eos nihil commodi voluptas. Velit consequatur odio quibusdam. Beatae aliquam hic quos.',
+    textPlainBody:'Illum eos nihil commodi voluptas. Velit consequatur odio quibusdam. Beatae aliquam hic quos.',
     'mailbox': 'SENT',
     replying: {
       single: 'laurel@hamil.info',
@@ -47,7 +47,7 @@ define(function() {
     'ident':9360,
     'tags':['photography','sky'],
     'status':['read'],
-    'body':'Illum eos nihil commodi voluptas. Velit consequatur odio quibusdam. Beatae aliquam hic quos.',
+    textPlainBody:'Illum eos nihil commodi voluptas. Velit consequatur odio quibusdam. Beatae aliquam hic quos.',
     'mailbox': 'DRAFTS',
     replying: {
       single: 'afton_braun@botsford.biz',
@@ -63,7 +63,7 @@ define(function() {
     'ident':9360,
     'tags':['photography','sky'],
     'status':['read'],
-    'body':'Illum eos nihil commodi voluptas. Velit consequatur odio quibusdam. Beatae aliquam hic quos.',
+    textPlainBody:'Illum eos nihil commodi voluptas. Velit consequatur odio quibusdam. Beatae aliquam hic quos.',
     'mailbox': 'TRASH',
     replying: {
       single: 'afton_braun@botsford.biz',
@@ -79,7 +79,7 @@ define(function() {
     'ident':242,
     'tags':['garden','instalovers','popularpic'],
     'status':['read'],
-    'body':'Sed est neque tempore. Alias officiis pariatur ullam porro corporis. Tempore eum quia placeat. Sapiente fuga cum.',
+    textPlainBody: 'Sed est neque tempore. Alias officiis pariatur ullam porro corporis. Tempore eum quia placeat. Sapiente fuga cum.',
     replying: {
       single: 'afton_braun@botsford.biz',
       all: {
@@ -96,6 +96,7 @@ define(function() {
     'tags':['garden','instalovers','popularpic'],
     'status':['read'],
     'body':'Sed est neque tempore. Alias officiis pariatur ullam porro corporis. Tempore eum quia placeat. Sapiente fuga cum.',
+    textPlainBody: 'body',
     replying: {
       single: 'cleve_jaskolski@schimmelhirthe.net',
       all: {
@@ -111,7 +112,7 @@ define(function() {
     'ident':242,
     'tags':['garden','instalovers','popularpic'],
     'status':['read'],
-    'body':'Sed est neque tempore. Alias officiis pariatur ullam porro corporis. Tempore eum quia placeat. Sapiente fuga cum.',
+    textPlainBody:'Sed est neque tempore. Alias officiis pariatur ullam porro corporis. Tempore eum quia placeat. Sapiente fuga cum.',
     replying: {
       single: 'cleve_jaskolski@schimmelhirthe.net',
       all: {
@@ -122,37 +123,6 @@ define(function() {
 
   };
 
-  var rawMultipartMail = {
-    header: {
-      to:'multipart@multipart.info',
-      from:'laurel@hamill.info',
-      subject:'multipart email with text/plain and text/html',
-      content_type: 'multipart/alternative; boundary=asdfghjkl',
-      date:'2014-06-04T14:41:13-03:00'
-    },
-    ident: 11,
-    tags:['multipart','inbox'],
-    status:[],
-    body: '--asdfghjkl\n' +
-          'Content-Type: text/plain;\n' +
-          '\n' +
-          'Hello everyone!\n' +
-          '--asdfghjkl\n' +
-          'Content-Type: text/html;\n' +
-          'Content-Transfer-Encoding: quoted-printable\n' +
-          '\n' +
-          '<p><b>Hello everyone!</b></p>\n' +
-          '--asdfghjkl--\n',
-    replying: {
-      single: 'laurel@hamil.info',
-      all: {
-        'to-field': ['laurel@hamil.info'],
-        'cc-field': []
-      }
-    }
-
-  };
-
   var simpleTextPlainMail = {
     header: {
       to:'jed_waelchi@cummerata.info',
@@ -164,7 +134,7 @@ define(function() {
     tags:['textplain'],
     mailbox: ['inbox'],
     status:[],
-    body: 'Hello Everyone',
+    textPlainBody: 'Hello Everyone',
     isSentMail: function() { return false; },
     isDraftMail: function() { return false; },
     replyToAddress: function() { return { to: ['laurel@hamill.info'], cc: [] }; },
@@ -185,7 +155,8 @@ define(function() {
     ident:2,
     tags:['html','noencoding','inbox'],
     status:[],
-    body: '--asdfghjkl\nContent-Type: text/html; charset=utf8\n\n<DOCTYPE html>\n<body> <div> <p>Hello everyone!</p> </div> </body>\n--asdfghjkl--\n',
+    textPlainBody: 'Hello everyone!',
+    htmlBody: '<DOCTYPE html>\n<body> <div> <p>Hello everyone!</p> </div> </body>',
     isSentMail: function() { return false; },
     isDraftMail: function() { return false; },
     replyToAddress: function() { return { to: ['laurel@hamill.info'], cc: [] }; },
@@ -200,34 +171,6 @@ define(function() {
     }
   };
 
-  var htmlQuotedPrintableMail = {
-    header: {
-      to:'jed_waelchi@cummerata.info',
-      from:'laurel@hamill.info',
-      subject:'Velit aut tempora animi ut nulla esse.',
-      content_type: 'multipart/alternative; boundary=asdfghjkl',
-      date:'2014-06-04T14:41:13-03:00'
-    },
-    ident:3,
-    tags:['html','quotedprintable','inbox'],
-    status:[],
-    body: '--asdfghjkl\nContent-Type: text/html; charset=utf8\nContent-Transfer-Encoding: quoted-printable\n\n<DOCTYPE html>\n<body> <div style=3D"border: 5px;"> <p>Hello everyone!</p> </div> </body>\n--asdfghjkl--\n',
-    isSentMail: function() { return false; },
-    isDraftMail: function() { return false; },
-    replyToAddress: function() { return { to: ['laurel@hamill.info'], cc: [] }; },
-    replyToAllAddress: function() { return { to: ['laurel@hamill.info'], cc: [] }; },
-    isMailMultipartAlternative: function () { return true; },
-    availableBodyPartsContentType: function () { return ['text/html']; },
-    getMailPartByContentType: function () {
-      return {
-        headers: {
-          'Content-Type': 'text/html; charset=utf-8',
-          'Content-Transfer-Encoding': 'quoted-printable'},
-        body: '<!DOCTYPE html> <body> <div> <p style=3D"border: 5px;">Hello everyone!</p> </div> </body>'
-      };
-    }
-  };
-
   var testData = {
     rawMail: {
       mail: rawMail,
@@ -236,13 +179,11 @@ define(function() {
       trash: rawMailInTrash,
       received: rawReceivedMail,
       receivedWithCC: rawReceivedWithCCMail,
-      rawMailWithMultipleTo: rawMailWithMultipleTo,
-      multipart: rawMultipartMail
+      rawMailWithMultipleTo: rawMailWithMultipleTo
     },
     parsedMail: {
       simpleTextPlain: simpleTextPlainMail,
-      html: htmlNoEncodingMail,
-      htmlQuotedPrintable: htmlQuotedPrintableMail
+      html: htmlNoEncodingMail
     }
   };