Enable in-process func tests to optionally use encryption

Running functional tests in the in-process mode uses
the default value for the pipeline.  This patch adds support
to specify the SWIFT_TEST_IN_PROCESS_CONF_LOADER variable
to point to a labeled function that changes the proxy
configuration for the functional test.

The patch also adds a new tox environment
func-in-process-encryption
which runs with the environment variable
SWIFT_TEST_IN_PROCESS_CONF_LOADER=encryption

The motivation for this change is to put support in place for an
upstream CI job that will functionally test using encryption
middleware in the pipeline.  The gate job is proposed at:
https://review.openstack.org/#/c/348292/

Change-Id: I15c4b20f1d2be57ae21c69c614f6a9579145bee9

doc/source/development_guidelines.rst

@@ -106,6 +106,11 @@ set using environment variables:
 - the optional in-memory object server may be selected by setting the
   environment variable ``SWIFT_TEST_IN_MEMORY_OBJ`` to a true value.
 
+- encryption may be added to the proxy pipeline by setting the
+  environment variable ``SWIFT_TEST_IN_PROCESS_CONF_LOADER`` to
+  ``encryption``.  Or when using tox, specify the tox environment
+  ``func-in-process-encryption``
+
 - the proxy-server ``object_post_as_copy`` option may be set using the
   environment variable ``SWIFT_TEST_IN_PROCESS_OBJECT_POST_AS_COPY``.
 

test/functional/__init__.py

@@ -287,6 +287,56 @@ def _in_process_setup_ring(swift_conf, conf_src_dir, testdir):
     return obj_sockets
 
 
+def _load_encryption(proxy_conf_file, **kwargs):
+    """
+    Load encryption configuration and override proxy-server.conf contents.
+
+    :param proxy_conf_file: Source proxy conf filename
+    :returns: Path to the test proxy conf file to use
+    :raises InProcessException: raised if proxy conf contents are invalid
+    """
+    _debug('Setting configuration for encryption')
+
+    # The global conf dict cannot be used to modify the pipeline.
+    # The pipeline loader requires the pipeline to be set in the local_conf.
+    # If pipeline is set in the global conf dict (which in turn populates the
+    # DEFAULTS options) then it prevents pipeline being loaded into the local
+    # conf during wsgi load_app.
+    # Therefore we must modify the [pipeline:main] section.
+
+    conf = ConfigParser()
+    conf.read(proxy_conf_file)
+    try:
+        section = 'pipeline:main'
+        pipeline = conf.get(section, 'pipeline')
+        pipeline = pipeline.replace(
+            "proxy-logging proxy-server",
+            "keymaster encryption proxy-logging proxy-server")
+        conf.set(section, 'pipeline', pipeline)
+        root_secret = os.urandom(32).encode("base64")
+        conf.set('filter:keymaster', 'encryption_root_secret', root_secret)
+    except NoSectionError as err:
+        msg = 'Error problem with proxy conf file %s: %s' % \
+              (proxy_conf_file, err)
+        raise InProcessException(msg)
+
+    test_conf_file = os.path.join(_testdir, 'proxy-server.conf')
+    with open(test_conf_file, 'w') as fp:
+        conf.write(fp)
+
+    return test_conf_file
+
+
+# Mapping from possible values of the variable
+# SWIFT_TEST_IN_PROCESS_CONF_LOADER
+# to the method to call for loading the associated configuration
+# The expected signature for these methods is:
+# conf_filename_to_use loader(input_conf_filename, **kwargs)
+conf_loaders = {
+    'encryption': _load_encryption
+}
+
+
 def in_process_setup(the_object_server=object_server):
     _info('IN-PROCESS SERVERS IN USE FOR FUNCTIONAL TESTS')
     _info('Using object_server class: %s' % the_object_server.__name__)
@@ -318,6 +368,26 @@ def in_process_setup(the_object_server=object_server):
     utils.mkdirs(os.path.join(_testdir, 'sdb1'))
     utils.mkdirs(os.path.join(_testdir, 'sdb1', 'tmp'))
 
+    # Call the associated method for the value of
+    # 'SWIFT_TEST_IN_PROCESS_CONF_LOADER', if one exists
+    conf_loader_label = os.environ.get(
+        'SWIFT_TEST_IN_PROCESS_CONF_LOADER')
+    if conf_loader_label is not None:
+        try:
+            conf_loader = conf_loaders[conf_loader_label]
+            _debug('Calling method %s mapped to conf loader %s' %
+                   (conf_loader.__name__, conf_loader_label))
+        except KeyError as missing_key:
+            raise InProcessException('No function mapped for conf loader %s' %
+                                     missing_key)
+
+        try:
+            # Pass-in proxy_conf
+            proxy_conf = conf_loader(proxy_conf)
+            _debug('Now using proxy conf %s' % proxy_conf)
+        except Exception as err:  # noqa
+            raise InProcessException(err)
+
     swift_conf = _in_process_setup_swift_conf(swift_conf_src, _testdir)
     obj_sockets = _in_process_setup_ring(swift_conf, conf_src_dir, _testdir)
 

tox.ini

@@ -53,6 +53,11 @@ commands = ./.functests {posargs}
 setenv = SWIFT_TEST_IN_PROCESS=1
          SWIFT_TEST_IN_PROCESS_OBJECT_POST_AS_COPY=False
 
+[testenv:func-in-process-encryption]
+commands = ./.functests {posargs}
+setenv = SWIFT_TEST_IN_PROCESS=1
+         SWIFT_TEST_IN_PROCESS_CONF_LOADER=encryption
+
 [testenv:venv]
 commands = {posargs}