Add support for parallel testr workers in Keystone

Full support for parallel testr workers now works in keystone. The
major item to be aware of is that if non-SQLite backends or a LIVE
LDAP backend is being used for testing, the concurrency must be set
to 1 to avoid conflicts. To artificially set the concurrency set
the env variable ``TEST_RUN_CONCURRENCY`` to the desired number of
testr workers. The default is to run one worker. To use 1 per
available core, set ``TEST_RUN_CONCURRENCY`` to 0.

Each worker uses a subdirectory of the test temp dir  based upon
the PID. When the worker exists (via the use of atexit), the
worker-specific PID directory is automatically removed.

The first time concurrent tests are run in a given environment,
ensure that old .testrepository directories are cleaned up as they
may introduce bad learning data on how to bin the tests based upon
test runs that occured without concurrency.

Closes-Bug: #1240052
Change-Id: I5c3385a431774ce574266a82065ed6237e8b3665

.testr.conf

@@ -5,8 +5,13 @@ test_command=${PYTHON:-python} -m subunit.run discover \
 
 test_id_option=--load-list $IDFILE
 test_list_option=--list
+group_regex=.*(test_cert_setup|test_keystoneclient).+
+
+
+# NOTE(morganfainberg): If single-worker mode is wanted (e.g. for live tests)
+# the environment variable ``TEST_RUN_CONCURRENCY`` should be set to ``1``. If
+# a non-default (1 worker per available core) concurrency is desired, set
+# environment variable ``TEST_RUN_CONCURRENCY`` to the desired number of
+# workers.
+test_run_concurrency=echo ${TEST_RUN_CONCURRENCY:-1}
 
-# NOTE(dstanek): Ensures that Keystone test never run in parallel.
-#                Please remove once the issues have been worked out.
-#                Bug: #1240052
-test_run_concurrency=echo 1

keystone/tests/core.py

@@ -69,6 +69,7 @@ config.configure()
 
 
 LOG = log.getLogger(__name__)
+PID = six.text_type(os.getpid())
 TESTSDIR = os.path.dirname(os.path.abspath(__file__))
 TESTCONF = os.path.join(TESTSDIR, 'config_files')
 ROOTDIR = os.path.normpath(os.path.join(TESTSDIR, '..', '..'))
@@ -78,9 +79,9 @@ ETCDIR = os.path.join(ROOTDIR, 'etc')
 
 def _calc_tmpdir():
     env_val = os.environ.get('KEYSTONE_TEST_TEMP_DIR')
-    if env_val:
-        return env_val
-    return os.path.join(TESTSDIR, 'tmp')
+    if not env_val:
+        return os.path.join(TESTSDIR, 'tmp', PID)
+    return os.path.join(env_val, PID)
 
 
 TMPDIR = _calc_tmpdir()
@@ -88,6 +89,8 @@ TMPDIR = _calc_tmpdir()
 CONF = config.CONF
 
 exception._FATAL_EXCEPTION_FORMAT_ERRORS = True
+os.mkdir(TMPDIR)
+atexit.register(shutil.rmtree, TMPDIR)
 
 
 class dirs:
@@ -113,11 +116,14 @@ class dirs:
 
 
 # keystone.common.sql.initialize() for testing.
+DEFAULT_TEST_DB_FILE = dirs.tmp('test.db')
+
+
 def _initialize_sql_session():
     # Make sure the DB is located in the correct location, in this case set
     # the default value, as this should be able to be overridden in some
     # test cases.
-    db_file = dirs.tmp('test.db')
+    db_file = DEFAULT_TEST_DB_FILE
     db_options.set_defaults(
         sql_connection='sqlite:///%s' % db_file,
         sqlite_db=db_file)

keystone/tests/test_cert_setup.py

@@ -24,7 +24,7 @@ from keystone.tests import rest
 from keystone import token
 
 
-SSLDIR = tests.dirs.tests('ssl')
+SSLDIR = tests.dirs.tmp('ssl')
 CONF = tests.CONF
 DEFAULT_DOMAIN_ID = CONF.identity.default_domain_id
 

keystone/tests/test_sql_upgrade.py

@@ -74,6 +74,18 @@ class SqlMigrateBase(tests.SQLDriverOverrides, tests.TestCase):
 
         self.config(self.config_files())
 
+        conn_str = CONF.database.connection
+        if (conn_str.startswith('sqlite') and
+                conn_str[10:] == tests.DEFAULT_TEST_DB_FILE):
+            # Override the default with a DB that is specific to the migration
+            # tests only if the DB Connection string is the same as the global
+            # default. This is required so that no conflicts occur due to the
+            # global default DB already being under migrate control.
+            db_file = tests.dirs.tmp('keystone_migrate_test.db')
+            self.config_fixture.config(
+                group='database',
+                connection='sqlite:///%s' % db_file)
+
         # create and share a single sqlalchemy engine for testing
         self.engine = sql.get_engine()
         self.Session = db_session.get_maker(self.engine, autocommit=False)