Merge branch 'master' of https://github.com/pixelated/pixelated-user-agent

# By Felix Hammerl (5) and others
# Via NavaL
* 'master' of https://github.com/pixelated/pixelated-user-agent:
  serving the client directly, as the current dependency on proxy strips out xsrf cookies -fixing functional test
  only adding feature resource in root_resource test -- fixing build
  changed logout to post Issue #612
  Backend and frontend protection against csrf attacks: - root resources changes the csrf token cookie everytime it is loaded, in particular during the intestitial load during login - it will also add that cookie on single user mode - initialize will still load all resources - but they you cant access them if the csrf token do not match - all ajax calls needs to add the token to the header - non ajax get requests do not need xsrf token validation - non ajax post will have to send the token in as a form input or in the content
  Consolidate stylesheets
  Remove unused font and stylesheetgit s
  Create a new deferred for all IMAPAccount calls
  Clean up jshintrc
  Recreate session on soledad problems
  issue #617: Remove old html whitelister
  Issue #617: Sanitize received content

10 files changed, 196 insertions, 19 deletions

diff --git a/service/test/functional/features/environment.py b/service/test/functional/features/environment.py
index 37b5d612..2c07faf3 100644
--- a/service/test/functional/features/environment.py
+++ b/service/test/functional/features/environment.py
@@ -44,11 +44,9 @@ def before_all(context):
     PixelatedSite.disable_csp_requests()
     client = AppTestClient()
     start_app_test_client(client, UserAgentMode(is_single_user=True))
-    client.listenTCP()
-    proxy = Proxy(proxy_port='8889', app_port='4567')
+    client.listenTCP(port=8889)
     FeaturesResource.DISABLED_FEATURES.append('autoRefresh')
     context.client = client
-    context.call_to_terminate_proxy = proxy.run_on_a_thread()
 
     multi_user_client = AppTestClient()
     start_app_test_client(multi_user_client, UserAgentMode(is_single_user=False))
@@ -57,7 +55,7 @@ def before_all(context):
 
 
 def after_all(context):
-    context.call_to_terminate_proxy()
+    context.client.stop()
 
 
 def before_feature(context, feature):
diff --git a/service/test/integration/test_logout.py b/service/test/integration/test_logout.py
index 52f7e34f..da414126 100644
--- a/service/test/integration/test_logout.py
+++ b/service/test/integration/test_logout.py
@@ -13,10 +13,11 @@
 #
 # You should have received a copy of the GNU Affero General Public License
 # along with Pixelated. If not, see <http://www.gnu.org/licenses/>.
+import json
+
 from mockito import verify
 from twisted.internet import defer
 
-from test.support.integration import load_mail_from_file
 from test.support.integration.multi_user_client import MultiUserClient
 from test.support.integration.soledad_test_base import SoledadTestBase
 
@@ -34,7 +35,8 @@ class MultiUserLogoutTest(MultiUserClient, SoledadTestBase):
 
         yield self.wait_for_session_user_id_to_finish()
 
-        response, request = self.get("/logout", as_json=False, from_request=login_request)
+        response, request = self.post("/logout", json.dumps({'csrftoken': [login_request.getCookie('XSRF-TOKEN')]}),
+                                      from_request=login_request, as_json=False)
         yield response
 
         self.assertEqual(302, request.responseCode)     # redirected
diff --git a/service/test/integration/test_retrieve_attachment.py b/service/test/integration/test_retrieve_attachment.py
index 4aaeadc2..31c8c5df 100644
--- a/service/test/integration/test_retrieve_attachment.py
+++ b/service/test/integration/test_retrieve_attachment.py
@@ -43,6 +43,23 @@ class RetrieveAttachmentTest(SoledadTestBase):
         self.assertEquals(expected_content_disposition, req.outgoingHeaders['content-disposition'])
         self.assertEquals(expected_content_type, req.outgoingHeaders['content-type'])
 
+    @defer.inlineCallbacks
+    def test_should_retrieve_attachment_even_if_xsrf_token_not_passed(self):
+        attachment_id, input_mail = self._create_mail_with_attachment()
+        yield self.mail_store.add_mail('INBOX', input_mail.as_string())
+
+        requested_filename = "file name with space"
+        expected_content_type = 'text/plain'
+        expected_content_disposition = 'attachment; filename="file name with space"'
+
+        attachment, req = yield self.get_attachment(attachment_id, 'base64', filename=requested_filename,
+                                                    content_type=expected_content_type, ajax=False, csrf='mismatched token')
+
+        self.assertEqual(200, req.code)
+        self.assertEquals('pretend to be binary attachment data', attachment)
+        self.assertEquals(expected_content_disposition, req.outgoingHeaders['content-disposition'])
+        self.assertEquals(expected_content_type, req.outgoingHeaders['content-type'])
+
     def _create_mail_with_attachment(self):
         input_mail = MIMEMultipart()
         input_mail.attach(MIMEText(u'a utf8 message', _charset='utf-8'))
diff --git a/service/test/support/integration/app_test_client.py b/service/test/support/integration/app_test_client.py
index 8ab58397..f3ec5d25 100644
--- a/service/test/support/integration/app_test_client.py
+++ b/service/test/support/integration/app_test_client.py
@@ -244,22 +244,25 @@ class AppTestClient(object):
         time.sleep(1)
         return lambda: process.terminate()
 
-    def get(self, path, get_args='', as_json=True):
-        request = request_mock(path)
+    def stop(self):
+        reactor.stop()
+
+    def get(self, path, get_args='', as_json=True, ajax=True, csrf='token'):
+        request = request_mock(path, ajax=ajax, csrf=csrf)
         request.args = get_args
         return self._render(request, as_json)
 
-    def post(self, path, body='', headers=None):
+    def post(self, path, body='', headers=None, ajax=True, csrf='token'):
         headers = headers or {'Content-Type': 'application/json'}
-        request = request_mock(path=path, method="POST", body=body, headers=headers)
+        request = request_mock(path=path, method="POST", body=body, headers=headers, ajax=ajax, csrf=csrf)
         return self._render(request)
 
-    def put(self, path, body):
-        request = request_mock(path=path, method="PUT", body=body, headers={'Content-Type': ['application/json']})
+    def put(self, path, body, ajax=True, csrf='token'):
+        request = request_mock(path=path, method="PUT", body=body, headers={'Content-Type': ['application/json']}, ajax=ajax, csrf=csrf)
         return self._render(request)
 
-    def delete(self, path, body=""):
-        request = request_mock(path=path, body=body, headers={'Content-Type': ['application/json']}, method="DELETE")
+    def delete(self, path, body="", ajax=True, csrf='token'):
+        request = request_mock(path=path, body=body, headers={'Content-Type': ['application/json']}, method="DELETE", ajax=ajax, csrf=csrf)
         return self._render(request)
 
     @defer.inlineCallbacks
@@ -322,13 +325,13 @@ class AppTestClient(object):
         defer.returnValue(mails)
 
     @defer.inlineCallbacks
-    def get_attachment(self, ident, encoding, filename=None, content_type=None):
+    def get_attachment(self, ident, encoding, filename=None, content_type=None, ajax=True, csrf='token'):
         params = {'encoding': [encoding]}
         if filename:
             params['filename'] = [filename]
         if content_type:
             params['content_type'] = [content_type]
-        deferred_result, req = self.get("/attachment/%s" % ident, params, as_json=False)
+        deferred_result, req = self.get("/attachment/%s" % ident, params, as_json=False, ajax=ajax, csrf=csrf)
         res = yield deferred_result
         defer.returnValue((res, req))
 
diff --git a/service/test/support/integration/multi_user_client.py b/service/test/support/integration/multi_user_client.py
index fa65fb06..5f24456b 100644
--- a/service/test/support/integration/multi_user_client.py
+++ b/service/test/support/integration/multi_user_client.py
@@ -82,3 +82,12 @@ class MultiUserClient(AppTestClient):
             session = from_request.getSession()
             request.session = session
         return self._render(request, as_json)
+
+    def post(self, path, body='', headers=None, ajax=True, csrf='token', as_json=True, from_request=None):
+        headers = headers or {'Content-Type': 'application/json'}
+        request = request_mock(path=path, method="POST", body=body, headers=headers, ajax=ajax, csrf=csrf)
+
+        if from_request:
+            session = from_request.getSession()
+            request.session = session
+        return self._render(request, as_json)
diff --git a/service/test/support/test_helper.py b/service/test/support/test_helper.py
index 77c74407..640baf6f 100644
--- a/service/test/support/test_helper.py
+++ b/service/test/support/test_helper.py
@@ -88,6 +88,7 @@ class PixRequestMock(DummyRequest):
         DummyRequest.__init__(self, path)
         self.content = None
         self.code = None
+        self.cookies = {}
 
     def getWrittenData(self):
         if len(self.written):
@@ -97,8 +98,14 @@ class PixRequestMock(DummyRequest):
         self.setResponseCode(302)
         self.setHeader(b"location", url)
 
+    def addCookie(self, key, value):
+        self.cookies[key] = value
 
-def request_mock(path='', method='GET', body='', headers={}):
+    def getCookie(self, key):
+        return self.cookies.get(key)
+
+
+def request_mock(path='', method='GET', body='', headers={}, ajax=True, csrf='token'):
     dummy = PixRequestMock(path.split('/'))
     for name, val in headers.iteritems():
         dummy.headers[name.lower()] = val
@@ -108,5 +115,9 @@ def request_mock(path='', method='GET', body='', headers={}):
     else:
         for key, val in body.items():
             dummy.addArg(key, val)
+    if ajax:
+        dummy.headers['x-requested-with'] = 'XMLHttpRequest'
+        dummy.headers['x-xsrf-token'] = csrf
+        dummy.addCookie('XSRF-TOKEN', csrf)
 
     return dummy
diff --git a/service/test/unit/bitmask_libraries/test_session.py b/service/test/unit/bitmask_libraries/test_session.py
index a41cb805..aad2cac2 100644
--- a/service/test/unit/bitmask_libraries/test_session.py
+++ b/service/test/unit/bitmask_libraries/test_session.py
@@ -133,6 +133,16 @@ class SessionTest(AbstractLeapTest):
         self.assertTrue(session.fresh_account)
 
     @patch('pixelated.bitmask_libraries.session.register')
+    def test_closed_session_not_reused(self, _):
+        session = self._create_session()
+        SessionCache.remember_session('somekey', session)
+        session._is_closed = True
+
+        result = SessionCache.lookup_session('somekey')
+
+        self.assertIsNone(result)
+
+    @patch('pixelated.bitmask_libraries.session.register')
     def test_session_does_not_set_status_fresh_for_unkown_emails(self, _):
         session = self._create_session()
         self.provider.address_for.return_value = 'someone@somedomain.tld'
@@ -150,7 +160,7 @@ class SessionTest(AbstractLeapTest):
         with patch('pixelated.bitmask_libraries.session.reactor.callFromThread', new=_execute_func) as _:
             with patch.object(LeapSession, '_create_incoming_mail_fetcher', return_value=mailFetcherMock) as _:
                 session = self._create_session()
-                session._has_been_synced = True
+                session._has_been_initially_synced = True
                 yield session.initial_sync()
                 self.assertFalse(mailFetcherMock.startService.called)
 
diff --git a/service/test/unit/config/test_leap.py b/service/test/unit/config/test_leap.py
new file mode 100644
index 00000000..6b34d717
--- /dev/null
+++ b/service/test/unit/config/test_leap.py
@@ -0,0 +1,38 @@
+from leap.soledad.common.errors import InvalidAuthTokenError
+from mock import MagicMock, patch
+from twisted.trial import unittest
+from twisted.internet import defer
+from pixelated.config.leap import authenticate_user
+
+
+class TestAuth(unittest.TestCase):
+
+    @patch('pixelated.config.leap.LeapSessionFactory')
+    @defer.inlineCallbacks
+    def test_authenticate_user_calls_initinal_sync(self, session_factory__ctor_mock):
+        session_factory_mock = session_factory__ctor_mock.return_value
+        provider_mock = MagicMock()
+        auth_mock = MagicMock()
+        session = MagicMock()
+
+        session_factory_mock.create.return_value = session
+
+        yield authenticate_user(provider_mock, 'username', 'password', auth=auth_mock)
+
+        session.initial_sync.assert_called_with()
+
+    @patch('pixelated.config.leap.LeapSessionFactory')
+    @defer.inlineCallbacks
+    def test_authenticate_user_calls_initial_sync_a_second_time_if_invalid_auth_exception_is_raised(self, session_factory__ctor_mock):
+        session_factory_mock = session_factory__ctor_mock.return_value
+        provider_mock = MagicMock()
+        auth_mock = MagicMock()
+        session = MagicMock()
+
+        session.initial_sync.side_effect = [InvalidAuthTokenError, defer.succeed(None)]
+        session_factory_mock.create.return_value = session
+
+        yield authenticate_user(provider_mock, 'username', 'password', auth=auth_mock)
+
+        session.close.assert_called_with()
+        self.assertEqual(2, session.initial_sync.call_count)
diff --git a/service/test/unit/resources/test_logout_resources.py b/service/test/unit/resources/test_logout_resources.py
index 48cf9db9..6246eeb9 100644
--- a/service/test/unit/resources/test_logout_resources.py
+++ b/service/test/unit/resources/test_logout_resources.py
@@ -1,6 +1,7 @@
 from mock import patch
 from mockito import mock, verify
 from twisted.trial import unittest
+from twisted.web.error import UnsupportedMethod
 from twisted.web.test.requesthelper import DummyRequest
 
 from pixelated.resources.logout_resource import LogoutResource
@@ -16,6 +17,7 @@ class TestLogoutResource(unittest.TestCase):
     @patch('twisted.web.util.redirectTo')
     def test_logout(self, mock_redirect):
         request = DummyRequest(['/logout'])
+        request.method = 'POST'
 
         mock_redirect.return_value = 'haha'
 
@@ -29,3 +31,9 @@ class TestLogoutResource(unittest.TestCase):
 
         d.addCallback(expire_session_and_redirect)
         return d
+
+    def test_get_is_not_supported_for_logout(self):
+        request = DummyRequest(['/logout'])
+        request.method = 'GET'
+
+        self.assertRaises(UnsupportedMethod, self.web.get, request)
diff --git a/service/test/unit/resources/test_root_resource.py b/service/test/unit/resources/test_root_resource.py
index 3b0846ee..cc052d8b 100644
--- a/service/test/unit/resources/test_root_resource.py
+++ b/service/test/unit/resources/test_root_resource.py
@@ -1,11 +1,14 @@
 import unittest
 import re
+
+from mock import MagicMock, patch
 from mockito import mock, when, any as ANY
 
 from pixelated.application import UserAgentMode
+from pixelated.resources.features_resource import FeaturesResource
 from test.unit.resources import DummySite
 from twisted.web.test.requesthelper import DummyRequest
-from pixelated.resources.root_resource import RootResource
+from pixelated.resources.root_resource import RootResource, MODE_STARTUP, MODE_RUNNING
 
 
 class TestRootResource(unittest.TestCase):
@@ -25,9 +28,11 @@ class TestRootResource(unittest.TestCase):
         root_resource._html_template = "<html><head><title>$account_email</title></head></html>"
         root_resource._mode = root_resource
         self.web = DummySite(root_resource)
+        self.root_resource = root_resource
 
     def test_render_GET_should_template_account_email(self):
         request = DummyRequest([''])
+        request.addCookie = lambda key, value: 'stubbed'
 
         d = self.web.get(request)
 
@@ -38,3 +43,79 @@ class TestRootResource(unittest.TestCase):
 
         d.addCallback(assert_response)
         return d
+
+    def _test_should_renew_xsrf_cookie(self):
+        request = DummyRequest([''])
+        request.addCookie = MagicMock()
+        generated_csrf_token = 'csrf_token'
+        mock_sha = MagicMock()
+        mock_sha.hexdigest = MagicMock(return_value=generated_csrf_token)
+
+        with patch('hashlib.sha256', return_value=mock_sha):
+            d = self.web.get(request)
+
+        def assert_csrf_cookie(_):
+            request.addCookie.assert_called_once_with('XSRF-TOKEN', generated_csrf_token)
+
+        d.addCallback(assert_csrf_cookie)
+        return d
+
+    def test_should_renew_xsrf_cookie_on_startup_mode(self):
+        self.root_resource._mode = MODE_STARTUP
+        self._test_should_renew_xsrf_cookie()
+
+    def test_should_renew_xsrf_cookie_on_running_mode(self):
+        self.root_resource._mode = MODE_RUNNING
+        self._test_should_renew_xsrf_cookie()
+
+    def _mock_ajax_csrf(self, request, csrf_token):
+        request.headers['x-requested-with'] = 'XMLHttpRequest'
+        request.headers['x-xsrf-token'] = csrf_token
+
+    def test_should_unauthorize_child_resource_ajax_requests_when_csrf_mismatch(self):
+        request = DummyRequest(['/child'])
+        self._mock_ajax_csrf(request, 'stubbed csrf token')
+
+        request.getCookie = MagicMock(return_value='mismatched csrf token')
+
+        d = self.web.get(request)
+
+        def assert_unauthorized(_):
+            self.assertEqual(401, request.responseCode)
+            self.assertEqual("Unauthorized!", request.written[0])
+
+        d.addCallback(assert_unauthorized)
+        return d
+
+    def test_should_authorize_child_resource_non_ajax_GET_requests(self):
+        request = DummyRequest(['features'])
+
+        request.getCookie = MagicMock(return_value='irrelevant -- stubbed')
+        self.root_resource._child_resources.add('features', FeaturesResource())
+
+        d = self.web.get(request)
+
+        def assert_unauthorized(_):
+            self.assertEqual(200, request.code)
+
+        d.addCallback(assert_unauthorized)
+        return d
+
+    def test_should_unauthorize_child_resource_non_ajax_POST_requests_when_csrf_input_mismatch(self):
+        request = DummyRequest(['mails'])
+        request.method = 'POST'
+        request.addArg('csrftoken', 'some csrf token')
+        mock_content = MagicMock()
+        mock_content.read = MagicMock(return_value={})
+        request.content = mock_content
+
+        request.getCookie = MagicMock(return_value='mismatched csrf token')
+
+        d = self.web.get(request)
+
+        def assert_unauthorized(_):
+            self.assertEqual(401, request.responseCode)
+            self.assertEqual("Unauthorized!", request.written[0])
+
+        d.addCallback(assert_unauthorized)
+        return d