Remove spend_time_in_reactor from functional tests

- Issue #456
- Protect mail list click click against staleness exceptions
  Mail list is manipulated while accessing first mail so page
  often is stale. Repeat function if that happens

2 files changed, 36 insertions, 32 deletions

diff --git a/service/test/functional/features/steps/common.py b/service/test/functional/features/steps/common.py
index b53359dc..9a547375 100644
--- a/service/test/functional/features/steps/common.py
+++ b/service/test/functional/features/steps/common.py
@@ -13,15 +13,11 @@
 #
 # You should have received a copy of the GNU Affero General Public License
 # along with Pixelated. If not, see <http://www.gnu.org/licenses/>.
-from crochet import wait_for
-
 from selenium.webdriver.common.by import By
 from selenium.webdriver.support import expected_conditions as EC
 from selenium.webdriver.support.wait import WebDriverWait
-from selenium.common.exceptions import TimeoutException
-from twisted.internet import reactor
-from twisted.internet import defer
-
+from selenium.common.exceptions import TimeoutException, StaleElementReferenceException
+import time
 from test.support.integration import MailBuilder
 
 LOADING = 'loading'
@@ -49,7 +45,6 @@ def wait_until_element_is_invisible_by_locator(context, locator_tuple, timeout=T
 
 
 def wait_until_element_is_deleted(context, locator_tuple, timeout=TIMEOUT_IN_S):
-    spend_time_in_reactor()
     wait = WebDriverWait(context.browser, timeout)
     wait.until(lambda s: len(s.find_elements(locator_tuple[0], locator_tuple[1])) == 0)
 
@@ -63,33 +58,35 @@ def wait_for_user_alert_to_disapear(context, timeout=TIMEOUT_IN_S):
 
 
 def wait_until_elements_are_visible_by_locator(context, locator_tuple, timeout=TIMEOUT_IN_S):
-    spend_time_in_reactor()
     wait = WebDriverWait(context.browser, timeout)
     wait.until(EC.presence_of_all_elements_located(locator_tuple))
     return context.browser.find_elements(locator_tuple[0], locator_tuple[1])
 
 
 def wait_until_elements_are_visible_by_xpath(context, locator_tuple, timeout=TIMEOUT_IN_S):
-    spend_time_in_reactor()
     wait = WebDriverWait(context.browser, timeout)
     wait.until(EC.presence_of_all_elements_located(locator_tuple))
     return context.browser.find_elements(locator_tuple[0], locator_tuple[1])
 
 
 def wait_until_element_is_visible_by_locator(context, locator_tuple, timeout=TIMEOUT_IN_S):
-    spend_time_in_reactor()
     wait = WebDriverWait(context.browser, timeout)
     wait.until(EC.visibility_of_element_located(locator_tuple))
     return context.browser.find_element(locator_tuple[0], locator_tuple[1])
 
 
+def wait_for_condition(context, predicate_func, timeout=TIMEOUT_IN_S, poll_frequency=0.1):
+    wait = WebDriverWait(context.browser, timeout, poll_frequency=poll_frequency)
+    wait.until(predicate_func)
+
+
 def fill_by_xpath(context, xpath, text):
     field = context.browser.find_element_by_xpath(xpath)
     field.send_keys(text)
 
 
 def fill_by_css_selector(context, css_selector, text):
-    field = context.browser.find_element_by_css_selector(css_selector)
+    field = find_element_by_css_selector(context, css_selector)
     field.send_keys(text)
 
 
@@ -126,8 +123,8 @@ def find_element_by_class_name(context, class_name):
     return wait_until_element_is_visible_by_locator(context, (By.CLASS_NAME, class_name))
 
 
-def find_elements_by_css_selector(context, css_selector):
-    return wait_until_elements_are_visible_by_locator(context, (By.CSS_SELECTOR, css_selector))
+def find_elements_by_css_selector(context, css_selector, timeout=TIMEOUT_IN_S):
+    return wait_until_elements_are_visible_by_locator(context, (By.CSS_SELECTOR, css_selector), timeout=timeout)
 
 
 def find_elements_by_xpath(context, xpath):
@@ -144,12 +141,21 @@ def element_should_have_content(context, css_selector, content):
 
 
 def wait_until_button_is_visible(context, title, timeout=TIMEOUT_IN_S):
-    spend_time_in_reactor()
     wait = WebDriverWait(context.browser, timeout)
     locator_tuple = (By.XPATH, ("//%s[contains(.,'%s')]" % ('button', title)))
     wait.until(EC.visibility_of_element_located(locator_tuple))
 
 
+def execute_ignoring_staleness(func, timeout=TIMEOUT_IN_S):
+    end_time = time.time() + timeout
+    while time.time() <= end_time:
+        try:
+            return func()
+        except StaleElementReferenceException:
+            pass
+    raise TimeoutException('did not solve stale state until timeout %f' % timeout)
+
+
 def click_button(context, title, element='button'):
     button = find_element_containing_text(context, title, element_type=element)
     button.click()
@@ -180,15 +186,3 @@ def get_console_log(context):
 def create_email(context):
     input_mail = MailBuilder().build_input_mail()
     context.client.add_mail_to_inbox(input_mail)
-
-
-@wait_for(timeout=5.0)
-def spend_time_in_reactor(reactor_time=1.0):
-    d = defer.Deferred()
-
-    def done_waiting():
-        d.callback(None)
-
-    reactor.callLater(reactor_time, done_waiting)
-
-    return d
diff --git a/service/test/functional/features/steps/mail_list.py b/service/test/functional/features/steps/mail_list.py
index 7962ee28..5eae2214 100644
--- a/service/test/functional/features/steps/mail_list.py
+++ b/service/test/functional/features/steps/mail_list.py
@@ -31,6 +31,10 @@ def open_current_mail(context):
     e.click()
 
 
+def get_first_email(context):
+    return wait_until_elements_are_visible_by_locator(context, (By.CSS_SELECTOR, '#mail-list li span a'))[0]
+
+
 @then('I see that mail under the \'{tag}\' tag')
 def impl(context, tag):
     context.execute_steps("when I select the tag '%s'" % tag)
@@ -44,9 +48,9 @@ def impl(context):
 
 @when('I open the first mail in the mail list')
 def impl(context):
-    first_email = wait_until_elements_are_visible_by_locator(context, (By.CSS_SELECTOR, '#mail-list li span a'))[0]
-    context.current_mail_id = 'mail-' + first_email.get_attribute('href').split('/')[-1]
-    first_email.click()
+    # it seems page is often still loading so staleness exceptions hapen often
+    context.current_mail_id = 'mail-' + execute_ignoring_staleness(lambda: get_first_email(context).get_attribute('href').split('/')[-1])
+    execute_ignoring_staleness(lambda: get_first_email(context).click())
 
 
 @when('I open the first mail in the \'{tag}\'')
@@ -83,9 +87,9 @@ def impl(context):
 
     for email in emails:
         if 'status-read' not in email.get_attribute('class'):
+            context.current_mail_id = email.get_attribute('id')  # we need to get the mail id before manipulating the page
             email.find_element_by_tag_name('input').click()
             find_element_by_id(context, 'mark-selected-as-read').click()
-            context.current_mail_id = email.get_attribute('id')
             break
     wait_until_elements_are_visible_by_locator(context, (By.CSS_SELECTOR, '#%s.status-read' % context.current_mail_id))
 
@@ -104,8 +108,14 @@ def impl(context):
 def _wait_for_mail_list_to_be_empty(context):
     wait_for_loading_to_finish(context)
 
-    with ImplicitWait(context, timeout=0.1):
-        assert 0 == len(context.browser.find_elements_by_css_selector('#mail-list li'))
+    def mail_list_is_empty(_):
+        with ImplicitWait(context, timeout=0.1):
+            try:
+                return 0 == len(context.browser.find_elements_by_css_selector('#mail-list li'))
+            except TimeoutException:
+                return False
+
+    wait_for_condition(context, mail_list_is_empty)
 
 
 @when('I check all emails')