[test] add --subdir option and blacklist

Use subdir path to select/deselect benchmark tests.

Because of the way pytest and pytest benchamrk select/deselect tests, it
makes more sense for this test suite to use a --subdir option to select
only a subset of tests and a blacklist of subdirectories that should not
be run unless explicitelly asked to.

This commit adds a --subdir option, that will deselect all tests no in
./tests/<subdir> when the option is passed to pytest. Also, a blacklist
is added so, unless explicitelly added as a --subdir, tests in
blacklisted subdirectories will not be run.

The reasons for this modification are:

- pytest-benchmarks selects/deselects tests based on the type of their
  fixtures. This means that fixtures have to be instantiated previous to
  test selection/deselection, something that may cause side effects that
  we want to avoid.

- the usual test suite is run for all benchmarks, while some tests
  should only be run in specific situations (as is the case of
  benchmarks and responsiveness tests). It is saner to implement subdir
  selection than marking all tests with pytest marks and using them to
  select/deselect.

2 files changed, 74 insertions, 22 deletions

diff --git a/testing/tests/conftest.py b/testing/tests/conftest.py
index 857934ff..d3a39289 100644
--- a/testing/tests/conftest.py
+++ b/testing/tests/conftest.py
@@ -21,13 +21,52 @@ from leap.soledad.common.couch import CouchDatabase
 from leap.soledad.client import Soledad
 
 
-# mark tests that depend on couchdb
-def pytest_collection_modifyitems(items):
+def _select_subdir(subdir, blacklist, items):
+
+    # allow blacklisted subdir if explicited in command line
+    if subdir and subdir in blacklist:
+        blacklist.remove(subdir)
+
+    # determine blacklisted subdirs
+    dirname = os.path.dirname(__file__)
+    blacklisted_subdirs = map(lambda s: os.path.join(dirname, s), blacklist)
+
+    # determine base path for selected tests
+    path = dirname
+    if subdir:
+        path = os.path.join(dirname, subdir)
+
+    # remove tests from blacklisted subdirs
+    selected = []
+    deselected = []
+    for item in items:
+        filename = item.module.__file__
+        blacklisted = any(
+            map(lambda s: filename.startswith(s), blacklisted_subdirs))
+        if blacklisted or not filename.startswith(path):
+            deselected.append(item)
+        else:
+            selected.append(item)
+
+    return selected, deselected
+
+
+def pytest_collection_modifyitems(items, config):
+
+    # mark tests that depend on couchdb
     marker = getattr(pytest.mark, 'needs_couch')
     for item in items:
         if 'soledad/testing/tests/couch/' in item.module.__file__:
             item.add_marker(marker)
 
+    # select/deselect tests based on a blacklist and the subdir option given in
+    # command line
+    blacklist = ['benchmarks', 'responsiveness']
+    subdir = config.getoption('subdir')
+    selected, deselected = _select_subdir(subdir, blacklist, items)
+    config.hook.pytest_deselected(items=deselected)
+    items[:] = selected
+
 
 #
 # default options for all tests
@@ -65,6 +104,10 @@ def pytest_addoption(parser):
         "--elasticsearch-url", type="string", default=None,
         help="the url for posting responsiveness results to elasticsearch")
 
+    parser.addoption(
+        "--subdir", type="string", default=None,
+        help="select only tests from a certain subdirectory of ./tests/")
+
 
 def _request(method, url, data=None, do=True):
     if do:
diff --git a/testing/tox.ini b/testing/tox.ini
index 6bc82b8e..a8186f70 100644
--- a/testing/tox.ini
+++ b/testing/tox.ini
@@ -6,11 +6,11 @@ skipsdist=True
 basepython = python2.7
 commands = 
     ./ensure-pysqlcipher-has-usleep.sh
-    py.test -x --ignore=tests/benchmarks --ignore=tests/responsiveness \
-                   --cov-report=html \
-                   --cov-report=term \
-                   --cov=leap.soledad \
-                   {posargs}
+    py.test -x \
+      --cov-report=html \
+      --cov-report=term \
+      --cov=leap.soledad \
+      {posargs}
 usedevelop = True
 deps =
     coverage
@@ -25,6 +25,12 @@ deps =
     requests
     service_identity
     leap.common
+# used by benchmarks
+    psutil
+    numpy
+    pytest-benchmark
+    elasticsearch
+    certifi
 # install soledad from current tree
     -e../
     -e../[client]
@@ -37,11 +43,12 @@ install_command = pip install {opts} {packages}
 
 [testenv:py34]
 basepython = python3.4
-commands = py.test --ignore=tests/benchmarks --ignore=tests/responsiveness \
-                   --cov-report=html \
-                   --cov-report=term \
-                   --cov=leap.soledad \
-                   {posargs}
+commands =
+    py.test \
+      --cov-report=html \
+      --cov-report=term \
+      --cov=leap.soledad \
+      {posargs}
 usedevelop = True
 deps =
     coverage
@@ -54,6 +61,12 @@ deps =
     couchdb
     requests
     service_identity
+# used by benchmarks
+    psutil
+    numpy
+    pytest-benchmark
+    elasticsearch
+    certifi
 # install soledad local packages
     -e../
     -e../[client]
@@ -66,18 +79,13 @@ install_command = pip3 install {opts} {packages}
 [testenv:benchmark]
 deps =
     {[testenv]deps}
-    psutil
-    numpy
-    pytest-benchmark
-    elasticsearch
-    certifi
 commands =
 # we must make sure that installed pysqlcipher was built with the HAVE_USLEEP
 # flag, or we might have problems with concurrent db access.
     ./ensure-pysqlcipher-has-usleep.sh
 # run benchmarks twice: once for time and cpu and a second time for memory
-    py.test --benchmark-only {posargs}
-    py.test --benchmark-only --watch-memory {posargs}
+    py.test --subdir=benchmarks {posargs}
+    py.test --subdir=benchmarks --watch-memory {posargs}
 passenv = HOST_HOSTNAME
 
 [testenv:responsiveness]
@@ -85,7 +93,7 @@ deps =
     {[testenv:benchmark]deps}
 commands =
     ./ensure-pysqlcipher-has-usleep.sh
-    pytest -m responsiveness {posargs}
+    py.test --subdir=responsiveness {posargs}
 
 [testenv:code-check]
 changedir = ..
@@ -101,5 +109,6 @@ deps =
     {[testenv]deps}
     pytest-xdist
 install_command = pip install {opts} {packages}
-commands = ./ensure-pysqlcipher-has-usleep.sh
-           py.test --ignore=tests/benchmarks {posargs} -n 4
+commands =
+    ./ensure-pysqlcipher-has-usleep.sh
+    py.test {posargs} -n 4