diff options
| -rw-r--r-- | .gitlab-ci.yml | 23 | ||||
| -rw-r--r-- | Makefile | 13 | ||||
| -rw-r--r-- | tests/docker/Dockerfile | 2 | ||||
| -rw-r--r-- | tests/functional/README.md | 41 | ||||
| -rw-r--r-- | tests/functional/features/environment.py | 67 | ||||
| -rw-r--r-- | tests/functional/features/smoke.feature | 14 | ||||
| -rw-r--r-- | tests/functional/features/steps/bitmask.py | 26 | ||||
| -rw-r--r-- | tests/functional/features/steps/common.py | 138 | ||||
| -rw-r--r-- | tests/functional/features/steps/login.py | 45 | ||||
| -rw-r--r-- | tox.ini | 3 |
10 files changed, 366 insertions, 6 deletions
diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index c05494fb..cc7520f0 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -1,8 +1,8 @@ stages: - test + - e2e - bundle - build_ui - - e2e variables: DOCKER_DRIVER: overlay @@ -28,7 +28,7 @@ bitmask_latest_bundle: stage: bundle script: - pkg/build_bundle_with_venv.sh - - mv "dist/bitmask-`cat pkg/next-version`" . + - mv "dist/bitmask-`cat pkg/next-version`" . artifacts: paths: - "bitmask-`cat pkg/next-version`" @@ -59,6 +59,25 @@ e2e_tests: tags: - linux +functional_tests: + image: 0xacab.org:4567/leap/bitmask-dev:latest + stage: e2e + before_script: + - virtualenv venv + - source venv/bin/activate + - make dev-latest-all + - make test_functional_setup + script: + - make test_functional + artifacts: + when: on_failure + paths: + - "/tmp/*.png" + name: "Bitmask_linux64_${CI_BUILD_REF}_e2e_screenshots" + expire_in: 1 month + tags: + - linux + build_docker_image: image: 0xacab.org:4567/leap/bitmask-dev:latest stage: test @@ -9,7 +9,6 @@ dev-mail: dev-gui: install_pixelated pip install -e '.[gui]' - dev-backend: pip install -e '.[backend]' @@ -18,6 +17,7 @@ dev-latest-backend: dev-backend pip install -e 'git+https://0xacab.org/leap/soledad@master#egg=leap.soledad' dev-all: install_pixelated + pip install -I --install-option="--bundled" pysqlcipher pip install -e '.[all]' dev-latest-all: dev-all @@ -34,6 +34,15 @@ test_e2e: tests/e2e/e2e-test-mail.sh tests/e2e/e2e-test-vpn.sh +test_functional_setup: + pip install behave selenium + +test_functional: + xvfb-run --server-args="-screen 0 1280x1024x24" behave --tags ~@wip --tags @smoke tests/functional/features -k -D host=localhost + +test_functional_graphical: + behave --tags ~@wip --tags @smoke tests/functional/features -k -D host=localhost + install_helpers: cp src/leap/bitmask/vpn/helpers/linux/bitmask-root /usr/local/sbin/ cp src/leap/bitmask/vpn/helpers/linux/se.leap.bitmask.policy /usr/share/polkit-1/actions/ @@ -41,7 +50,7 @@ install_helpers: install_pixelated: # install pixelated from our repo until assets get packaged. pip install requests==2.11.1 whoosh chardet - pip install pixelated-www pixelated-user-agent --find-links https://downloads.leap.se/libs/pixelated/ + pip install pixelated-www pixelated-user-agent --find-links https://downloads.leap.se/libs/pixelated/ qt-resources: pyrcc5 pkg/branding/icons.qrc -o src/leap/bitmask/gui/app_rc.py diff --git a/tests/docker/Dockerfile b/tests/docker/Dockerfile index 4dfeab33..876ba1d4 100644 --- a/tests/docker/Dockerfile +++ b/tests/docker/Dockerfile @@ -15,3 +15,5 @@ RUN apt-get -y install build-essential python-virtualenv libpython-dev \ openvpn policykit-1 lxpolkit \ wget patchelf libusb-0.1-4 \ docker.io \ + xvfb chromium-chromedriver +RUN ln -s /usr/lib/chromium-browser/chromedriver /usr/local/bin/chromedriver diff --git a/tests/functional/README.md b/tests/functional/README.md new file mode 100644 index 00000000..9405d8e0 --- /dev/null +++ b/tests/functional/README.md @@ -0,0 +1,41 @@ +# Bitmask functional UI tests + +## Todo + +Moved to https://0xacab.org/leap/bitmask-dev/issues/8929#note_111673 + +## Setup + +Ubuntu: + + sudo apt install xvfb chromium-chromedriver + ln -s /usr/lib/chromium-browser/chromedriver venv-all/bin/chromedriver + +Debian: + + sudo apt install xvfb chromedriver + + +Setup your virtualenv and python packages: + + virtualenv venv-all + source ./venv-all/bin/activate + make dev-all + make test_functional_setup + +## Run tests + + source ./venv-all/bin/activate + export TEST_USERNAME='user@provider.tld' TEST_PASSWORD='...' + make test_functional + +# Develop tests + +When tests are run using `make test_functional` no window shows you what the browser sees. +In order to see tests running in the browser run: + + make test_functional_graphical + +You can also run behave by itself and have a browser window to watch, i.e. to run all tests tagged as `@wip`: + + behave --wip -k -D host=localhost tests/functional/features diff --git a/tests/functional/features/environment.py b/tests/functional/features/environment.py new file mode 100644 index 00000000..4ed0caa3 --- /dev/null +++ b/tests/functional/features/environment.py @@ -0,0 +1,67 @@ +import os +import re +import time +from urlparse import urlparse +import commands + +from selenium import webdriver +from selenium.webdriver.chrome.options import Options + +DEFAULT_IMPLICIT_WAIT_TIMEOUT_IN_S = 10 + + +def before_all(context): + _setup_webdriver(context) + userdata = context.config.userdata + context.host = userdata.get('host', 'http://localhost') + if not context.host.startswith('http'): + context.host = 'https://{}'.format(context.host) + context.hostname = urlparse(context.host).hostname + + context.username = os.environ['TEST_USERNAME'] + context.password = os.environ['TEST_PASSWORD'] + context.user_email = '{}@{}'.format(context.username, context.hostname) + + +def _setup_webdriver(context): + chrome_options = Options() + # argument to switch off suid sandBox and no sandBox in Chrome + chrome_options.add_argument("--no-sandbox") + chrome_options.add_argument("--disable-setuid-sandbox") + + context.browser = webdriver.Chrome(chrome_options=chrome_options) + context.browser.set_window_size(1280, 1024) + context.browser.implicitly_wait(DEFAULT_IMPLICIT_WAIT_TIMEOUT_IN_S) + context.browser.set_page_load_timeout(60) + + +def after_all(context): + context.browser.quit() + commands.getoutput('bitmaskctl stop') + + +def after_step(context, step): + if step.status == 'failed': + _save_screenshot(context, step) + _debug_on_error(context, step) + + +def _debug_on_error(context, step): + if context.config.userdata.getbool("debug"): + try: + import ipdb + ipdb.post_mortem(step.exc_traceback) + except ImportError: + import pdb + pdb.post_mortem(step.exc_traceback) + + +def _save_screenshot(context, step): + timestamp = time.strftime("%Y-%m-%d-%H-%M-%S") + filename = _slugify('{} failed {}'.format(timestamp, str(step.name))) + filepath = os.path.join('/tmp/', filename + '.png') + context.browser.save_screenshot(filepath) + + +def _slugify(string_): + return re.sub('\W', '-', string_) diff --git a/tests/functional/features/smoke.feature b/tests/functional/features/smoke.feature new file mode 100644 index 00000000..6209ad6a --- /dev/null +++ b/tests/functional/features/smoke.feature @@ -0,0 +1,14 @@ +@smoke +Feature: login and logout + + Scenario: See user panel after login + Given I start bitmask for the first time + When I login + Then I should see the user panel + + Scenario: Log in and log out + Given I start bitmask for the first time + When I login + And I logout + Then I should see the second login page + diff --git a/tests/functional/features/steps/bitmask.py b/tests/functional/features/steps/bitmask.py new file mode 100644 index 00000000..b5b4cf1e --- /dev/null +++ b/tests/functional/features/steps/bitmask.py @@ -0,0 +1,26 @@ +import commands +import shutil +import os +import time +from leap.common.config import get_path_prefix + +from behave import given + + +@given('I start bitmask for the first time') +def initial_run(context): + commands.getoutput('bitmaskctl stop') + # TODO: fix bitmaskctl to only exit once bitmaskd has stopped + time.sleep(2) + _initialize_home_path() + commands.getoutput('bitmaskctl start') + tokenpath = os.path.join(get_path_prefix(), 'leap', 'authtoken') + token = open(tokenpath).read().strip() + context.login_url = "http://localhost:7070/#%s" % token + + +def _initialize_home_path(): + home_path = '/tmp/bitmask-test' + shutil.rmtree(home_path, ignore_errors=True) + os.environ['HOME'] = home_path + os.makedirs(get_path_prefix()) diff --git a/tests/functional/features/steps/common.py b/tests/functional/features/steps/common.py new file mode 100644 index 00000000..91858f85 --- /dev/null +++ b/tests/functional/features/steps/common.py @@ -0,0 +1,138 @@ +import time + +from selenium.common.exceptions import ( + StaleElementReferenceException, + TimeoutException) +from selenium.webdriver.common.by import By +from selenium.webdriver.support import expected_conditions as EC +from selenium.webdriver.support.wait import WebDriverWait + +TIMEOUT_IN_S = 10 + +DEFAULT_IMPLICIT_WAIT_TIMEOUT_IN_S = 10 + + +def wait_until_element_is_invisible_by_locator(context, locator_tuple, + timeout=TIMEOUT_IN_S): + wait = WebDriverWait(context.browser, timeout) + wait.until(EC.invisibility_of_element_located(locator_tuple)) + + +def wait_for_loading_to_finish(context, timeout=TIMEOUT_IN_S): + wait_until_element_is_invisible_by_locator( + context, (By.ID, 'loading'), timeout) + + +def wait_for_user_alert_to_disapear(context, timeout=TIMEOUT_IN_S): + wait_until_element_is_invisible_by_locator( + context, (By.ID, 'user-alerts'), timeout) + + +def _wait_until_elements_are_visible_by_locator(context, locator_tuple, + timeout=TIMEOUT_IN_S): + wait = WebDriverWait(context.browser, timeout) + wait.until(EC.presence_of_all_elements_located(locator_tuple)) + return context.browser.find_elements(locator_tuple[0], locator_tuple[1]) + + +def _wait_until_element_is_visible_by_locator(context, locator_tuple, + timeout=TIMEOUT_IN_S): + wait = WebDriverWait(context.browser, timeout) + wait.until(EC.visibility_of_element_located(locator_tuple)) + return context.browser.find_element(locator_tuple[0], locator_tuple[1]) + + +def wait_for_condition(context, predicate_func, timeout=TIMEOUT_IN_S, + poll_frequency=0.1): + wait = WebDriverWait(context.browser, timeout, + poll_frequency=poll_frequency) + wait.until(predicate_func) + + +def fill_by_xpath(context, xpath, text): + field = context.browser.find_element_by_xpath(xpath) + field.send_keys(text) + + +def fill_by_css_selector(context, css_selector, text, timeout=TIMEOUT_IN_S): + field = find_element_by_css_selector(context, css_selector, + timeout=timeout) + field.send_keys(text) + + +def take_screenshot(context, filename): + context.browser.save_screenshot(filename) + + +def page_has_css(context, css): + try: + find_element_by_css_selector(context, css) + return True + except TimeoutException: + return False + + +def find_element_by_xpath(context, xpath): + return _wait_until_element_is_visible_by_locator( + context, (By.XPATH, xpath)) + + +def find_element_by_id(context, id): + return _wait_until_element_is_visible_by_locator(context, (By.ID, id)) + + +def find_element_by_css_selector(context, css_selector, timeout=TIMEOUT_IN_S): + return _wait_until_element_is_visible_by_locator( + context, (By.CSS_SELECTOR, css_selector), timeout=timeout) + + +def find_element_by_class_name(context, class_name): + return _wait_until_element_is_visible_by_locator( + context, (By.CLASS_NAME, class_name)) + + +def find_elements_by_css_selector(context, css_selector, timeout=TIMEOUT_IN_S): + return _wait_until_elements_are_visible_by_locator( + context, (By.CSS_SELECTOR, css_selector), timeout=timeout) + + +def find_elements_by_xpath(context, xpath, timeout=TIMEOUT_IN_S): + return _wait_until_elements_are_visible_by_locator( + context, (By.XPATH, xpath), timeout=timeout) + + +def find_element_containing_text(context, text, element_type='*'): + return find_element_by_xpath( + context, "//%s[contains(.,'%s')]" % (element_type, text)) + + +def element_should_have_content(context, css_selector, content): + e = find_element_by_css_selector(context, css_selector) + assert e.text == content + + +def wait_until_button_is_visible(context, title, timeout=TIMEOUT_IN_S): + wait = WebDriverWait(context.browser, timeout) + locator_tuple = (By.XPATH, ("//%s[contains(.,'%s')]" % ('button', title))) + wait.until(EC.visibility_of_element_located(locator_tuple)) + + +def execute_ignoring_staleness(func, timeout=TIMEOUT_IN_S): + end_time = time.time() + timeout + while time.time() <= end_time: + try: + return func() + except StaleElementReferenceException: + pass + raise TimeoutException( + 'did not solve stale state until timeout %f' % timeout) + + +def click_button(context, title, element='button'): + button = find_element_containing_text(context, title, element_type=element) + button.click() + + +def reply_subject(context): + e = find_element_by_css_selector(context, '#reply-subject') + return e.text diff --git a/tests/functional/features/steps/login.py b/tests/functional/features/steps/login.py new file mode 100644 index 00000000..4a5981a1 --- /dev/null +++ b/tests/functional/features/steps/login.py @@ -0,0 +1,45 @@ +from behave import given, when, then + +from common import ( + click_button, + fill_by_css_selector, + find_element_by_css_selector +) + + +@when(u'I login') +def login_user(context): + login_page(context) + enter_credentials(context) + click_button(context, 'Log In') + + +def login_page(context): + context.browser.get(context.login_url) + context.browser.refresh() + + +def enter_credentials(context): + fill_by_css_selector(context, 'textarea[id="loginUsername"]', + context.username) + fill_by_css_selector(context, 'input[id="loginPassword"]', + context.password) + + +@then(u'I should see the user panel') +def see_home_screen(context): + find_element_by_css_selector(context, '.main-panel') + + +@then(u'I logout') +@when(u'I logout') +def click_logout(context): + # TODO: Have identifiers for the "second" login screen + click_button(context, 'Log Out') + + +@then(u'I should see the second login page') +def see_second_login_page(context): + # TODO: Have unique identifiers for the second login page + # (that differentiates from user panel) + find_element_by_css_selector(context, '#loginUsername') @@ -23,8 +23,7 @@ setenv = commands = # XXX workaround: use a bundled version of pysqlcipher to ensure HAVE_USLEEP is # set and we don't have problems with concurrent db access. - pip uninstall -y pysqlcipher - pip install --install-option="--bundled" pysqlcipher + pip install -I --install-option="--bundled" pysqlcipher # Adding pixelated as a dependency brings a *lot* of trouble to the test # infrastructure. Leaving them out for now, we'll be considering them as an # optional extra until the dependencies/tests are fixed more sanely. |