Merge branch 'functional_tests'

10 files changed, 366 insertions, 6 deletions

diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
index c05494fb..cc7520f0 100644
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@@ -1,8 +1,8 @@
 stages:
   - test
+  - e2e
   - bundle
   - build_ui
-  - e2e
 
 variables:
   DOCKER_DRIVER: overlay
@@ -28,7 +28,7 @@ bitmask_latest_bundle:
   stage: bundle
   script:
     - pkg/build_bundle_with_venv.sh
-    - mv "dist/bitmask-`cat pkg/next-version`" . 
+    - mv "dist/bitmask-`cat pkg/next-version`" .
   artifacts:
     paths:
       - "bitmask-`cat pkg/next-version`"
@@ -59,6 +59,25 @@ e2e_tests:
   tags:
     - linux
 
+functional_tests:
+  image: 0xacab.org:4567/leap/bitmask-dev:latest
+  stage: e2e
+  before_script:
+    - virtualenv venv
+    - source venv/bin/activate
+    - make dev-latest-all
+    - make test_functional_setup
+  script:
+    - make test_functional
+  artifacts:
+    when: on_failure
+    paths:
+      - "/tmp/*.png"
+    name: "Bitmask_linux64_${CI_BUILD_REF}_e2e_screenshots"
+    expire_in: 1 month
+  tags:
+    - linux
+
 build_docker_image:
   image: 0xacab.org:4567/leap/bitmask-dev:latest
   stage: test
diff --git a/Makefile b/Makefile
index 4605c792..c9ce3a7e 100644
--- a/Makefile
+++ b/Makefile
@@ -9,7 +9,6 @@ dev-mail:
 
 dev-gui: install_pixelated
 	pip install -e '.[gui]'
-
 dev-backend:
 	pip install -e '.[backend]'
 
@@ -18,6 +17,7 @@ dev-latest-backend: dev-backend
 	pip install -e 'git+https://0xacab.org/leap/soledad@master#egg=leap.soledad'
 
 dev-all: install_pixelated
+	pip install -I --install-option="--bundled" pysqlcipher
 	pip install -e '.[all]'
 
 dev-latest-all: dev-all
@@ -34,6 +34,15 @@ test_e2e:
 	tests/e2e/e2e-test-mail.sh
 	tests/e2e/e2e-test-vpn.sh
 
+test_functional_setup:
+	pip install behave selenium
+
+test_functional:
+	xvfb-run --server-args="-screen 0 1280x1024x24" behave --tags ~@wip --tags @smoke tests/functional/features -k -D host=localhost
+
+test_functional_graphical:
+	behave --tags ~@wip --tags @smoke tests/functional/features -k -D host=localhost
+
 install_helpers:
 	cp src/leap/bitmask/vpn/helpers/linux/bitmask-root /usr/local/sbin/
 	cp src/leap/bitmask/vpn/helpers/linux/se.leap.bitmask.policy /usr/share/polkit-1/actions/
@@ -41,7 +50,7 @@ install_helpers:
 install_pixelated:
 	# install pixelated from our repo until assets get packaged.
 	pip install requests==2.11.1 whoosh chardet
-	pip install pixelated-www pixelated-user-agent --find-links https://downloads.leap.se/libs/pixelated/  
+	pip install pixelated-www pixelated-user-agent --find-links https://downloads.leap.se/libs/pixelated/
 
 qt-resources:
 	pyrcc5 pkg/branding/icons.qrc -o src/leap/bitmask/gui/app_rc.py
diff --git a/tests/docker/Dockerfile b/tests/docker/Dockerfile
index 4dfeab33..876ba1d4 100644
--- a/tests/docker/Dockerfile
+++ b/tests/docker/Dockerfile
@@ -15,3 +15,5 @@ RUN apt-get -y install build-essential python-virtualenv libpython-dev \
   openvpn policykit-1 lxpolkit \
   wget patchelf libusb-0.1-4 \
   docker.io \
+  xvfb chromium-chromedriver
+RUN ln -s /usr/lib/chromium-browser/chromedriver /usr/local/bin/chromedriver
diff --git a/tests/functional/README.md b/tests/functional/README.md
new file mode 100644
index 00000000..9405d8e0
--- /dev/null
+++ b/tests/functional/README.md
@@ -0,0 +1,41 @@
+# Bitmask functional UI tests
+
+## Todo
+
+Moved to https://0xacab.org/leap/bitmask-dev/issues/8929#note_111673
+
+## Setup
+
+Ubuntu:
+
+    sudo apt install xvfb chromium-chromedriver
+    ln -s /usr/lib/chromium-browser/chromedriver venv-all/bin/chromedriver
+
+Debian:
+
+    sudo apt install xvfb chromedriver
+
+
+Setup your virtualenv and python packages:
+
+    virtualenv venv-all
+    source ./venv-all/bin/activate
+    make dev-all
+    make test_functional_setup
+
+## Run tests
+
+    source ./venv-all/bin/activate
+    export TEST_USERNAME='user@provider.tld' TEST_PASSWORD='...'
+    make test_functional
+
+# Develop tests
+
+When tests are run using `make test_functional` no window shows you what the browser sees.
+In order to see tests running in the browser run:
+
+    make test_functional_graphical
+
+You can also run behave by itself and have a browser window to watch, i.e. to run all tests tagged as `@wip`:
+
+    behave --wip -k -D host=localhost tests/functional/features
diff --git a/tests/functional/features/environment.py b/tests/functional/features/environment.py
new file mode 100644
index 00000000..4ed0caa3
--- /dev/null
+++ b/tests/functional/features/environment.py
@@ -0,0 +1,67 @@
+import os
+import re
+import time
+from urlparse import urlparse
+import commands
+
+from selenium import webdriver
+from selenium.webdriver.chrome.options import Options
+
+DEFAULT_IMPLICIT_WAIT_TIMEOUT_IN_S = 10
+
+
+def before_all(context):
+    _setup_webdriver(context)
+    userdata = context.config.userdata
+    context.host = userdata.get('host', 'http://localhost')
+    if not context.host.startswith('http'):
+        context.host = 'https://{}'.format(context.host)
+    context.hostname = urlparse(context.host).hostname
+
+    context.username = os.environ['TEST_USERNAME']
+    context.password = os.environ['TEST_PASSWORD']
+    context.user_email = '{}@{}'.format(context.username, context.hostname)
+
+
+def _setup_webdriver(context):
+    chrome_options = Options()
+    # argument to switch off suid sandBox and no sandBox in Chrome
+    chrome_options.add_argument("--no-sandbox")
+    chrome_options.add_argument("--disable-setuid-sandbox")
+
+    context.browser = webdriver.Chrome(chrome_options=chrome_options)
+    context.browser.set_window_size(1280, 1024)
+    context.browser.implicitly_wait(DEFAULT_IMPLICIT_WAIT_TIMEOUT_IN_S)
+    context.browser.set_page_load_timeout(60)
+
+
+def after_all(context):
+    context.browser.quit()
+    commands.getoutput('bitmaskctl stop')
+
+
+def after_step(context, step):
+    if step.status == 'failed':
+        _save_screenshot(context, step)
+        _debug_on_error(context, step)
+
+
+def _debug_on_error(context, step):
+    if context.config.userdata.getbool("debug"):
+        try:
+            import ipdb
+            ipdb.post_mortem(step.exc_traceback)
+        except ImportError:
+            import pdb
+            pdb.post_mortem(step.exc_traceback)
+
+
+def _save_screenshot(context, step):
+    timestamp = time.strftime("%Y-%m-%d-%H-%M-%S")
+    filename = _slugify('{} failed {}'.format(timestamp, str(step.name)))
+    filepath = os.path.join('/tmp/', filename + '.png')
+    context.browser.save_screenshot(filepath)
+
+
+def _slugify(string_):
+    return re.sub('\W', '-', string_)
diff --git a/tests/functional/features/smoke.feature b/tests/functional/features/smoke.feature
new file mode 100644
index 00000000..6209ad6a
--- /dev/null
+++ b/tests/functional/features/smoke.feature
@@ -0,0 +1,14 @@
+@smoke
+Feature: login and logout
+
+  Scenario: See user panel after login
+    Given I start bitmask for the first time
+    When I login
+    Then I should see the user panel
+
+  Scenario: Log in and log out
+    Given I start bitmask for the first time
+    When I login
+    And I logout
+    Then I should see the second login page
+
diff --git a/tests/functional/features/steps/bitmask.py b/tests/functional/features/steps/bitmask.py
new file mode 100644
index 00000000..b5b4cf1e
--- /dev/null
+++ b/tests/functional/features/steps/bitmask.py
@@ -0,0 +1,26 @@
+import commands
+import shutil
+import os
+import time
+from leap.common.config import get_path_prefix
+
+from behave import given
+
+
+@given('I start bitmask for the first time')
+def initial_run(context):
+    commands.getoutput('bitmaskctl stop')
+    # TODO: fix bitmaskctl to only exit once bitmaskd has stopped
+    time.sleep(2)
+    _initialize_home_path()
+    commands.getoutput('bitmaskctl start')
+    tokenpath = os.path.join(get_path_prefix(), 'leap', 'authtoken')
+    token = open(tokenpath).read().strip()
+    context.login_url = "http://localhost:7070/#%s" % token
+
+
+def _initialize_home_path():
+    home_path = '/tmp/bitmask-test'
+    shutil.rmtree(home_path, ignore_errors=True)
+    os.environ['HOME'] = home_path
+    os.makedirs(get_path_prefix())
diff --git a/tests/functional/features/steps/common.py b/tests/functional/features/steps/common.py
new file mode 100644
index 00000000..91858f85
--- /dev/null
+++ b/tests/functional/features/steps/common.py
@@ -0,0 +1,138 @@
+import time
+
+from selenium.common.exceptions import (
+    StaleElementReferenceException,
+    TimeoutException)
+from selenium.webdriver.common.by import By
+from selenium.webdriver.support import expected_conditions as EC
+from selenium.webdriver.support.wait import WebDriverWait
+
+TIMEOUT_IN_S = 10
+
+DEFAULT_IMPLICIT_WAIT_TIMEOUT_IN_S = 10
+
+
+def wait_until_element_is_invisible_by_locator(context, locator_tuple,
+                                               timeout=TIMEOUT_IN_S):
+    wait = WebDriverWait(context.browser, timeout)
+    wait.until(EC.invisibility_of_element_located(locator_tuple))
+
+
+def wait_for_loading_to_finish(context, timeout=TIMEOUT_IN_S):
+    wait_until_element_is_invisible_by_locator(
+        context, (By.ID, 'loading'), timeout)
+
+
+def wait_for_user_alert_to_disapear(context, timeout=TIMEOUT_IN_S):
+    wait_until_element_is_invisible_by_locator(
+        context, (By.ID, 'user-alerts'), timeout)
+
+
+def _wait_until_elements_are_visible_by_locator(context, locator_tuple,
+                                                timeout=TIMEOUT_IN_S):
+    wait = WebDriverWait(context.browser, timeout)
+    wait.until(EC.presence_of_all_elements_located(locator_tuple))
+    return context.browser.find_elements(locator_tuple[0], locator_tuple[1])
+
+
+def _wait_until_element_is_visible_by_locator(context, locator_tuple,
+                                              timeout=TIMEOUT_IN_S):
+    wait = WebDriverWait(context.browser, timeout)
+    wait.until(EC.visibility_of_element_located(locator_tuple))
+    return context.browser.find_element(locator_tuple[0], locator_tuple[1])
+
+
+def wait_for_condition(context, predicate_func, timeout=TIMEOUT_IN_S,
+                       poll_frequency=0.1):
+    wait = WebDriverWait(context.browser, timeout,
+                         poll_frequency=poll_frequency)
+    wait.until(predicate_func)
+
+
+def fill_by_xpath(context, xpath, text):
+    field = context.browser.find_element_by_xpath(xpath)
+    field.send_keys(text)
+
+
+def fill_by_css_selector(context, css_selector, text, timeout=TIMEOUT_IN_S):
+    field = find_element_by_css_selector(context, css_selector,
+                                         timeout=timeout)
+    field.send_keys(text)
+
+
+def take_screenshot(context, filename):
+    context.browser.save_screenshot(filename)
+
+
+def page_has_css(context, css):
+    try:
+        find_element_by_css_selector(context, css)
+        return True
+    except TimeoutException:
+        return False
+
+
+def find_element_by_xpath(context, xpath):
+    return _wait_until_element_is_visible_by_locator(
+        context, (By.XPATH, xpath))
+
+
+def find_element_by_id(context, id):
+    return _wait_until_element_is_visible_by_locator(context, (By.ID, id))
+
+
+def find_element_by_css_selector(context, css_selector, timeout=TIMEOUT_IN_S):
+    return _wait_until_element_is_visible_by_locator(
+        context, (By.CSS_SELECTOR, css_selector), timeout=timeout)
+
+
+def find_element_by_class_name(context, class_name):
+    return _wait_until_element_is_visible_by_locator(
+        context, (By.CLASS_NAME, class_name))
+
+
+def find_elements_by_css_selector(context, css_selector, timeout=TIMEOUT_IN_S):
+    return _wait_until_elements_are_visible_by_locator(
+        context, (By.CSS_SELECTOR, css_selector), timeout=timeout)
+
+
+def find_elements_by_xpath(context, xpath, timeout=TIMEOUT_IN_S):
+    return _wait_until_elements_are_visible_by_locator(
+        context, (By.XPATH, xpath), timeout=timeout)
+
+
+def find_element_containing_text(context, text, element_type='*'):
+    return find_element_by_xpath(
+        context, "//%s[contains(.,'%s')]" % (element_type, text))
+
+
+def element_should_have_content(context, css_selector, content):
+    e = find_element_by_css_selector(context, css_selector)
+    assert e.text == content
+
+
+def wait_until_button_is_visible(context, title, timeout=TIMEOUT_IN_S):
+    wait = WebDriverWait(context.browser, timeout)
+    locator_tuple = (By.XPATH, ("//%s[contains(.,'%s')]" % ('button', title)))
+    wait.until(EC.visibility_of_element_located(locator_tuple))
+
+
+def execute_ignoring_staleness(func, timeout=TIMEOUT_IN_S):
+    end_time = time.time() + timeout
+    while time.time() <= end_time:
+        try:
+            return func()
+        except StaleElementReferenceException:
+            pass
+    raise TimeoutException(
+        'did not solve stale state until timeout %f' % timeout)
+
+
+def click_button(context, title, element='button'):
+    button = find_element_containing_text(context, title, element_type=element)
+    button.click()
+
+
+def reply_subject(context):
+    e = find_element_by_css_selector(context, '#reply-subject')
+    return e.text
diff --git a/tests/functional/features/steps/login.py b/tests/functional/features/steps/login.py
new file mode 100644
index 00000000..4a5981a1
--- /dev/null
+++ b/tests/functional/features/steps/login.py
@@ -0,0 +1,45 @@
+from behave import given, when, then
+
+from common import (
+    click_button,
+    fill_by_css_selector,
+    find_element_by_css_selector
+)
+
+
+@when(u'I login')
+def login_user(context):
+    login_page(context)
+    enter_credentials(context)
+    click_button(context, 'Log In')
+
+
+def login_page(context):
+    context.browser.get(context.login_url)
+    context.browser.refresh()
+
+
+def enter_credentials(context):
+    fill_by_css_selector(context, 'textarea[id="loginUsername"]',
+                         context.username)
+    fill_by_css_selector(context, 'input[id="loginPassword"]',
+                         context.password)
+
+
+@then(u'I should see the user panel')
+def see_home_screen(context):
+    find_element_by_css_selector(context, '.main-panel')
+
+
+@then(u'I logout')
+@when(u'I logout')
+def click_logout(context):
+    # TODO: Have identifiers for the "second" login screen
+    click_button(context, 'Log Out')
+
+
+@then(u'I should see the second login page')
+def see_second_login_page(context):
+    # TODO: Have unique identifiers for the second login page
+    #       (that differentiates from user panel)
+    find_element_by_css_selector(context, '#loginUsername')
diff --git a/tox.ini b/tox.ini
index f7262fca..fd8f18e4 100644
--- a/tox.ini
+++ b/tox.ini
@@ -23,8 +23,7 @@ setenv =
 commands =
 # XXX workaround: use a bundled version of pysqlcipher to ensure HAVE_USLEEP is
 # set and we don't have problems with concurrent db access.
-    pip uninstall -y pysqlcipher
-    pip install --install-option="--bundled" pysqlcipher
+    pip install -I --install-option="--bundled" pysqlcipher
 #   Adding pixelated as a dependency brings a *lot* of trouble to the test
 #   infrastructure. Leaving them out for now, we'll be considering them as an
 #   optional extra until the dependencies/tests are fixed more sanely.