summaryrefslogtreecommitdiff
diff options
context:
space:
mode:
authorAzul <azul@riseup.net>2017-07-20 13:46:01 +0200
committerAzul <azul@riseup.net>2017-07-20 13:46:01 +0200
commiteecf6ad13801b63c6b7664d8a2e98e905262aaa0 (patch)
treefadf7d8270106a628c31b1f7017f343a94f311f0
parentc81b98df15e3d2006c991c9ca9d86500e3599e9b (diff)
parentd6abd906cb64ae68eed3348eba521bc44ebed7b2 (diff)
Merge branch 'functional_tests'
-rw-r--r--.gitlab-ci.yml23
-rw-r--r--Makefile13
-rw-r--r--tests/docker/Dockerfile2
-rw-r--r--tests/functional/README.md41
-rw-r--r--tests/functional/features/environment.py67
-rw-r--r--tests/functional/features/smoke.feature14
-rw-r--r--tests/functional/features/steps/bitmask.py26
-rw-r--r--tests/functional/features/steps/common.py138
-rw-r--r--tests/functional/features/steps/login.py45
-rw-r--r--tox.ini3
10 files changed, 366 insertions, 6 deletions
diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
index c05494fb..cc7520f0 100644
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@@ -1,8 +1,8 @@
stages:
- test
+ - e2e
- bundle
- build_ui
- - e2e
variables:
DOCKER_DRIVER: overlay
@@ -28,7 +28,7 @@ bitmask_latest_bundle:
stage: bundle
script:
- pkg/build_bundle_with_venv.sh
- - mv "dist/bitmask-`cat pkg/next-version`" .
+ - mv "dist/bitmask-`cat pkg/next-version`" .
artifacts:
paths:
- "bitmask-`cat pkg/next-version`"
@@ -59,6 +59,25 @@ e2e_tests:
tags:
- linux
+functional_tests:
+ image: 0xacab.org:4567/leap/bitmask-dev:latest
+ stage: e2e
+ before_script:
+ - virtualenv venv
+ - source venv/bin/activate
+ - make dev-latest-all
+ - make test_functional_setup
+ script:
+ - make test_functional
+ artifacts:
+ when: on_failure
+ paths:
+ - "/tmp/*.png"
+ name: "Bitmask_linux64_${CI_BUILD_REF}_e2e_screenshots"
+ expire_in: 1 month
+ tags:
+ - linux
+
build_docker_image:
image: 0xacab.org:4567/leap/bitmask-dev:latest
stage: test
diff --git a/Makefile b/Makefile
index 4605c792..c9ce3a7e 100644
--- a/Makefile
+++ b/Makefile
@@ -9,7 +9,6 @@ dev-mail:
dev-gui: install_pixelated
pip install -e '.[gui]'
-
dev-backend:
pip install -e '.[backend]'
@@ -18,6 +17,7 @@ dev-latest-backend: dev-backend
pip install -e 'git+https://0xacab.org/leap/soledad@master#egg=leap.soledad'
dev-all: install_pixelated
+ pip install -I --install-option="--bundled" pysqlcipher
pip install -e '.[all]'
dev-latest-all: dev-all
@@ -34,6 +34,15 @@ test_e2e:
tests/e2e/e2e-test-mail.sh
tests/e2e/e2e-test-vpn.sh
+test_functional_setup:
+ pip install behave selenium
+
+test_functional:
+ xvfb-run --server-args="-screen 0 1280x1024x24" behave --tags ~@wip --tags @smoke tests/functional/features -k -D host=localhost
+
+test_functional_graphical:
+ behave --tags ~@wip --tags @smoke tests/functional/features -k -D host=localhost
+
install_helpers:
cp src/leap/bitmask/vpn/helpers/linux/bitmask-root /usr/local/sbin/
cp src/leap/bitmask/vpn/helpers/linux/se.leap.bitmask.policy /usr/share/polkit-1/actions/
@@ -41,7 +50,7 @@ install_helpers:
install_pixelated:
# install pixelated from our repo until assets get packaged.
pip install requests==2.11.1 whoosh chardet
- pip install pixelated-www pixelated-user-agent --find-links https://downloads.leap.se/libs/pixelated/
+ pip install pixelated-www pixelated-user-agent --find-links https://downloads.leap.se/libs/pixelated/
qt-resources:
pyrcc5 pkg/branding/icons.qrc -o src/leap/bitmask/gui/app_rc.py
diff --git a/tests/docker/Dockerfile b/tests/docker/Dockerfile
index 4dfeab33..876ba1d4 100644
--- a/tests/docker/Dockerfile
+++ b/tests/docker/Dockerfile
@@ -15,3 +15,5 @@ RUN apt-get -y install build-essential python-virtualenv libpython-dev \
openvpn policykit-1 lxpolkit \
wget patchelf libusb-0.1-4 \
docker.io \
+ xvfb chromium-chromedriver
+RUN ln -s /usr/lib/chromium-browser/chromedriver /usr/local/bin/chromedriver
diff --git a/tests/functional/README.md b/tests/functional/README.md
new file mode 100644
index 00000000..9405d8e0
--- /dev/null
+++ b/tests/functional/README.md
@@ -0,0 +1,41 @@
+# Bitmask functional UI tests
+
+## Todo
+
+Moved to https://0xacab.org/leap/bitmask-dev/issues/8929#note_111673
+
+## Setup
+
+Ubuntu:
+
+ sudo apt install xvfb chromium-chromedriver
+ ln -s /usr/lib/chromium-browser/chromedriver venv-all/bin/chromedriver
+
+Debian:
+
+ sudo apt install xvfb chromedriver
+
+
+Setup your virtualenv and python packages:
+
+ virtualenv venv-all
+ source ./venv-all/bin/activate
+ make dev-all
+ make test_functional_setup
+
+## Run tests
+
+ source ./venv-all/bin/activate
+ export TEST_USERNAME='user@provider.tld' TEST_PASSWORD='...'
+ make test_functional
+
+# Develop tests
+
+When tests are run using `make test_functional` no window shows you what the browser sees.
+In order to see tests running in the browser run:
+
+ make test_functional_graphical
+
+You can also run behave by itself and have a browser window to watch, i.e. to run all tests tagged as `@wip`:
+
+ behave --wip -k -D host=localhost tests/functional/features
diff --git a/tests/functional/features/environment.py b/tests/functional/features/environment.py
new file mode 100644
index 00000000..4ed0caa3
--- /dev/null
+++ b/tests/functional/features/environment.py
@@ -0,0 +1,67 @@
+import os
+import re
+import time
+from urlparse import urlparse
+import commands
+
+from selenium import webdriver
+from selenium.webdriver.chrome.options import Options
+
+DEFAULT_IMPLICIT_WAIT_TIMEOUT_IN_S = 10
+
+
+def before_all(context):
+ _setup_webdriver(context)
+ userdata = context.config.userdata
+ context.host = userdata.get('host', 'http://localhost')
+ if not context.host.startswith('http'):
+ context.host = 'https://{}'.format(context.host)
+ context.hostname = urlparse(context.host).hostname
+
+ context.username = os.environ['TEST_USERNAME']
+ context.password = os.environ['TEST_PASSWORD']
+ context.user_email = '{}@{}'.format(context.username, context.hostname)
+
+
+def _setup_webdriver(context):
+ chrome_options = Options()
+ # argument to switch off suid sandBox and no sandBox in Chrome
+ chrome_options.add_argument("--no-sandbox")
+ chrome_options.add_argument("--disable-setuid-sandbox")
+
+ context.browser = webdriver.Chrome(chrome_options=chrome_options)
+ context.browser.set_window_size(1280, 1024)
+ context.browser.implicitly_wait(DEFAULT_IMPLICIT_WAIT_TIMEOUT_IN_S)
+ context.browser.set_page_load_timeout(60)
+
+
+def after_all(context):
+ context.browser.quit()
+ commands.getoutput('bitmaskctl stop')
+
+
+def after_step(context, step):
+ if step.status == 'failed':
+ _save_screenshot(context, step)
+ _debug_on_error(context, step)
+
+
+def _debug_on_error(context, step):
+ if context.config.userdata.getbool("debug"):
+ try:
+ import ipdb
+ ipdb.post_mortem(step.exc_traceback)
+ except ImportError:
+ import pdb
+ pdb.post_mortem(step.exc_traceback)
+
+
+def _save_screenshot(context, step):
+ timestamp = time.strftime("%Y-%m-%d-%H-%M-%S")
+ filename = _slugify('{} failed {}'.format(timestamp, str(step.name)))
+ filepath = os.path.join('/tmp/', filename + '.png')
+ context.browser.save_screenshot(filepath)
+
+
+def _slugify(string_):
+ return re.sub('\W', '-', string_)
diff --git a/tests/functional/features/smoke.feature b/tests/functional/features/smoke.feature
new file mode 100644
index 00000000..6209ad6a
--- /dev/null
+++ b/tests/functional/features/smoke.feature
@@ -0,0 +1,14 @@
+@smoke
+Feature: login and logout
+
+ Scenario: See user panel after login
+ Given I start bitmask for the first time
+ When I login
+ Then I should see the user panel
+
+ Scenario: Log in and log out
+ Given I start bitmask for the first time
+ When I login
+ And I logout
+ Then I should see the second login page
+
diff --git a/tests/functional/features/steps/bitmask.py b/tests/functional/features/steps/bitmask.py
new file mode 100644
index 00000000..b5b4cf1e
--- /dev/null
+++ b/tests/functional/features/steps/bitmask.py
@@ -0,0 +1,26 @@
+import commands
+import shutil
+import os
+import time
+from leap.common.config import get_path_prefix
+
+from behave import given
+
+
+@given('I start bitmask for the first time')
+def initial_run(context):
+ commands.getoutput('bitmaskctl stop')
+ # TODO: fix bitmaskctl to only exit once bitmaskd has stopped
+ time.sleep(2)
+ _initialize_home_path()
+ commands.getoutput('bitmaskctl start')
+ tokenpath = os.path.join(get_path_prefix(), 'leap', 'authtoken')
+ token = open(tokenpath).read().strip()
+ context.login_url = "http://localhost:7070/#%s" % token
+
+
+def _initialize_home_path():
+ home_path = '/tmp/bitmask-test'
+ shutil.rmtree(home_path, ignore_errors=True)
+ os.environ['HOME'] = home_path
+ os.makedirs(get_path_prefix())
diff --git a/tests/functional/features/steps/common.py b/tests/functional/features/steps/common.py
new file mode 100644
index 00000000..91858f85
--- /dev/null
+++ b/tests/functional/features/steps/common.py
@@ -0,0 +1,138 @@
+import time
+
+from selenium.common.exceptions import (
+ StaleElementReferenceException,
+ TimeoutException)
+from selenium.webdriver.common.by import By
+from selenium.webdriver.support import expected_conditions as EC
+from selenium.webdriver.support.wait import WebDriverWait
+
+TIMEOUT_IN_S = 10
+
+DEFAULT_IMPLICIT_WAIT_TIMEOUT_IN_S = 10
+
+
+def wait_until_element_is_invisible_by_locator(context, locator_tuple,
+ timeout=TIMEOUT_IN_S):
+ wait = WebDriverWait(context.browser, timeout)
+ wait.until(EC.invisibility_of_element_located(locator_tuple))
+
+
+def wait_for_loading_to_finish(context, timeout=TIMEOUT_IN_S):
+ wait_until_element_is_invisible_by_locator(
+ context, (By.ID, 'loading'), timeout)
+
+
+def wait_for_user_alert_to_disapear(context, timeout=TIMEOUT_IN_S):
+ wait_until_element_is_invisible_by_locator(
+ context, (By.ID, 'user-alerts'), timeout)
+
+
+def _wait_until_elements_are_visible_by_locator(context, locator_tuple,
+ timeout=TIMEOUT_IN_S):
+ wait = WebDriverWait(context.browser, timeout)
+ wait.until(EC.presence_of_all_elements_located(locator_tuple))
+ return context.browser.find_elements(locator_tuple[0], locator_tuple[1])
+
+
+def _wait_until_element_is_visible_by_locator(context, locator_tuple,
+ timeout=TIMEOUT_IN_S):
+ wait = WebDriverWait(context.browser, timeout)
+ wait.until(EC.visibility_of_element_located(locator_tuple))
+ return context.browser.find_element(locator_tuple[0], locator_tuple[1])
+
+
+def wait_for_condition(context, predicate_func, timeout=TIMEOUT_IN_S,
+ poll_frequency=0.1):
+ wait = WebDriverWait(context.browser, timeout,
+ poll_frequency=poll_frequency)
+ wait.until(predicate_func)
+
+
+def fill_by_xpath(context, xpath, text):
+ field = context.browser.find_element_by_xpath(xpath)
+ field.send_keys(text)
+
+
+def fill_by_css_selector(context, css_selector, text, timeout=TIMEOUT_IN_S):
+ field = find_element_by_css_selector(context, css_selector,
+ timeout=timeout)
+ field.send_keys(text)
+
+
+def take_screenshot(context, filename):
+ context.browser.save_screenshot(filename)
+
+
+def page_has_css(context, css):
+ try:
+ find_element_by_css_selector(context, css)
+ return True
+ except TimeoutException:
+ return False
+
+
+def find_element_by_xpath(context, xpath):
+ return _wait_until_element_is_visible_by_locator(
+ context, (By.XPATH, xpath))
+
+
+def find_element_by_id(context, id):
+ return _wait_until_element_is_visible_by_locator(context, (By.ID, id))
+
+
+def find_element_by_css_selector(context, css_selector, timeout=TIMEOUT_IN_S):
+ return _wait_until_element_is_visible_by_locator(
+ context, (By.CSS_SELECTOR, css_selector), timeout=timeout)
+
+
+def find_element_by_class_name(context, class_name):
+ return _wait_until_element_is_visible_by_locator(
+ context, (By.CLASS_NAME, class_name))
+
+
+def find_elements_by_css_selector(context, css_selector, timeout=TIMEOUT_IN_S):
+ return _wait_until_elements_are_visible_by_locator(
+ context, (By.CSS_SELECTOR, css_selector), timeout=timeout)
+
+
+def find_elements_by_xpath(context, xpath, timeout=TIMEOUT_IN_S):
+ return _wait_until_elements_are_visible_by_locator(
+ context, (By.XPATH, xpath), timeout=timeout)
+
+
+def find_element_containing_text(context, text, element_type='*'):
+ return find_element_by_xpath(
+ context, "//%s[contains(.,'%s')]" % (element_type, text))
+
+
+def element_should_have_content(context, css_selector, content):
+ e = find_element_by_css_selector(context, css_selector)
+ assert e.text == content
+
+
+def wait_until_button_is_visible(context, title, timeout=TIMEOUT_IN_S):
+ wait = WebDriverWait(context.browser, timeout)
+ locator_tuple = (By.XPATH, ("//%s[contains(.,'%s')]" % ('button', title)))
+ wait.until(EC.visibility_of_element_located(locator_tuple))
+
+
+def execute_ignoring_staleness(func, timeout=TIMEOUT_IN_S):
+ end_time = time.time() + timeout
+ while time.time() <= end_time:
+ try:
+ return func()
+ except StaleElementReferenceException:
+ pass
+ raise TimeoutException(
+ 'did not solve stale state until timeout %f' % timeout)
+
+
+def click_button(context, title, element='button'):
+ button = find_element_containing_text(context, title, element_type=element)
+ button.click()
+
+
+def reply_subject(context):
+ e = find_element_by_css_selector(context, '#reply-subject')
+ return e.text
diff --git a/tests/functional/features/steps/login.py b/tests/functional/features/steps/login.py
new file mode 100644
index 00000000..4a5981a1
--- /dev/null
+++ b/tests/functional/features/steps/login.py
@@ -0,0 +1,45 @@
+from behave import given, when, then
+
+from common import (
+ click_button,
+ fill_by_css_selector,
+ find_element_by_css_selector
+)
+
+
+@when(u'I login')
+def login_user(context):
+ login_page(context)
+ enter_credentials(context)
+ click_button(context, 'Log In')
+
+
+def login_page(context):
+ context.browser.get(context.login_url)
+ context.browser.refresh()
+
+
+def enter_credentials(context):
+ fill_by_css_selector(context, 'textarea[id="loginUsername"]',
+ context.username)
+ fill_by_css_selector(context, 'input[id="loginPassword"]',
+ context.password)
+
+
+@then(u'I should see the user panel')
+def see_home_screen(context):
+ find_element_by_css_selector(context, '.main-panel')
+
+
+@then(u'I logout')
+@when(u'I logout')
+def click_logout(context):
+ # TODO: Have identifiers for the "second" login screen
+ click_button(context, 'Log Out')
+
+
+@then(u'I should see the second login page')
+def see_second_login_page(context):
+ # TODO: Have unique identifiers for the second login page
+ # (that differentiates from user panel)
+ find_element_by_css_selector(context, '#loginUsername')
diff --git a/tox.ini b/tox.ini
index f7262fca..fd8f18e4 100644
--- a/tox.ini
+++ b/tox.ini
@@ -23,8 +23,7 @@ setenv =
commands =
# XXX workaround: use a bundled version of pysqlcipher to ensure HAVE_USLEEP is
# set and we don't have problems with concurrent db access.
- pip uninstall -y pysqlcipher
- pip install --install-option="--bundled" pysqlcipher
+ pip install -I --install-option="--bundled" pysqlcipher
# Adding pixelated as a dependency brings a *lot* of trouble to the test
# infrastructure. Leaving them out for now, we'll be considering them as an
# optional extra until the dependencies/tests are fixed more sanely.