Update pipeline for refstack compatibility

Re-align pipeline with default example pipeline; this includes
adding the following new middleware:

    - copy
    - ratelimit
    - symlink

Re-work SLO middleware configuration to avoid duplicated section.

Re-order middleware inline with documentation avoiding authentication
issues when using DLO/SLO features.

Drop proxy-server.conf validation; its brittle and function should be
validated by other tests anyway.

Change-Id: Ic9df7c12ee0bb402245ad2e64da0d905fe68890c
Closes-Bug: 1794255
(cherry picked from commit e92e8a75e0)

templates/queens/proxy-server.conf

@@ -23,9 +23,9 @@ key_file = {{ ssl_key }}
 {% if auth_type == 'keystone' %}
 [pipeline:main]
 {% if transport_url %}
-pipeline = ceilometer catch_errors gatekeeper healthcheck proxy-logging cache swift3 s3token container_sync bulk tempurl slo dlo formpost authtoken keystoneauth staticweb versioned_writes container-quotas account-quotas proxy-logging proxy-server
+pipeline = ceilometer catch_errors gatekeeper healthcheck proxy-logging cache container_sync bulk tempurl ratelimit formpost authtoken keystoneauth swift3 s3token staticweb copy container-quotas account-quotas slo dlo versioned_writes symlink proxy-logging proxy-server
 {% else %}
-pipeline = catch_errors gatekeeper healthcheck proxy-logging cache authtoken swift3 s3token container_sync bulk tempurl slo dlo formpost keystoneauth staticweb versioned_writes container-quotas account-quotas proxy-logging proxy-server
+pipeline = catch_errors gatekeeper healthcheck proxy-logging cache container_sync bulk tempurl ratelimit formpost authtoken keystoneauth swift3 s3token staticweb copy container-quotas account-quotas slo dlo versioned_writes symlink proxy-logging proxy-server
 {% endif %}
 {% else %}
 [pipeline:main]
@@ -67,6 +67,10 @@ use = egg:swift#bulk
 
 [filter:slo]
 use = egg:swift#slo
+{% if static_large_object_segments and static_large_object_segments > 0 %}
+max_manifest_size = 536870912
+max_manifest_segments = {{ static_large_object_segments }}
+{% endif %}
 
 [filter:dlo]
 use = egg:swift#dlo
@@ -89,6 +93,15 @@ use = egg:swift#container_sync
 [filter:gatekeeper]
 use = egg:swift#gatekeeper
 
+[filter:ratelimit]
+use = egg:swift#ratelimit
+
+[filter:copy]
+use = egg:swift#copy
+
+[filter:symlink]
+use = egg:swift#symlink
+
 {% if auth_type == 'keystone' %}
 [filter:keystoneauth]
 use = egg:swift#keystoneauth
@@ -135,10 +148,3 @@ driver = messagingv2
 topic = notifications
 log_level = WARN
 {% endif -%}
-
-{% if static_large_object_segments and static_large_object_segments > 0 %}
-[filter:slo]
-use = egg:swift#slo
-max_manifest_size = 536870912
-max_manifest_segments = {{ static_large_object_segments }}
-{% endif %}

tests/basic_deployment.py

@@ -422,204 +422,6 @@ class SwiftProxyBasicDeployment(OpenStackAmuletDeployment):
             message = "swift config error: {}".format(ret)
             amulet.raise_status(amulet.FAIL, msg=message)
 
-    def test_302_proxy_server_config(self, auth_api_version=None):
-        """Verify the data in the proxy-server config file."""
-        if self._get_openstack_release() >= self.xenial_queens:
-            auth_api_version = auth_api_version or '3'
-        else:
-            auth_api_version = auth_api_version or '2.0'
-        u.log.debug("Checking swift proxy-server config auth_api_version={}..."
-                    "".format(auth_api_version))
-        unit = self.swift_proxy_sentry
-        conf = '/etc/swift/proxy-server.conf'
-        keystone_relation = self.keystone_sentry.relation(
-            'identity-service', 'swift-proxy:identity-service')
-        swift_proxy_relation = unit.relation(
-            'identity-service', 'keystone:identity-service')
-        swift_proxy_ip = swift_proxy_relation['private-address']
-        auth_host = keystone_relation['auth_host']
-        auth_protocol = keystone_relation['auth_protocol']
-
-        expected = {
-            'DEFAULT': {
-                'bind_port': '8070',
-                'user': 'swift',
-                'log_name': 'swift',
-                'log_facility': 'LOG_LOCAL0',
-                'log_level': 'INFO',
-                'log_headers': 'False',
-                'log_address': '/dev/log'
-            },
-            'pipeline:main': {
-                'pipeline': 'gatekeeper healthcheck proxy-logging cache '
-                            'swift3 s3token container_sync bulk tempurl '
-                            'slo dlo formpost authtoken keystoneauth '
-                            'staticweb container-quotas account-quotas '
-                            'proxy-logging proxy-server'
-            },
-            'app:proxy-server': {
-                'use': 'egg:swift#proxy',
-                'allow_account_management': 'true',
-                'account_autocreate': 'true',
-                'node_timeout': '60',
-                'recoverable_node_timeout': '30'
-            },
-            'filter:tempauth': {
-                'use': 'egg:swift#tempauth',
-                'user_system_root': 'testpass .admin https://{}:8080/v1/'
-                                    'AUTH_system'.format(swift_proxy_ip)
-            },
-            'filter:healthcheck': {'use': 'egg:swift#healthcheck'},
-            'filter:cache': {
-                'use': 'egg:swift#memcache',
-                'memcache_servers': '{}:11211'.format(swift_proxy_ip)
-            },
-            'filter:account-quotas': {'use': 'egg:swift#account_quotas'},
-            'filter:container-quotas': {'use': 'egg:swift#container_quotas'},
-            'filter:proxy-logging': {'use': 'egg:swift#proxy_logging'},
-            'filter:staticweb': {'use': 'egg:swift#staticweb'},
-            'filter:bulk': {'use': 'egg:swift#bulk'},
-            'filter:slo': {'use': 'egg:swift#slo'},
-            'filter:dlo': {'use': 'egg:swift#dlo'},
-            'filter:formpost': {'use': 'egg:swift#formpost'},
-            'filter:tempurl': {'use': 'egg:swift#tempurl'},
-            'filter:container_sync': {'use': 'egg:swift#container_sync'},
-            'filter:gatekeeper': {'use': 'egg:swift#gatekeeper'},
-            'filter:keystoneauth': {
-                'use': 'egg:swift#keystoneauth',
-                'operator_roles': 'Member,Admin'
-            },
-            'filter:authtoken': {
-                'auth_uri': '{}://{}:{}'.format(
-                    auth_protocol,
-                    auth_host,
-                    keystone_relation['service_port']),
-                'delay_auth_decision': 'true',
-                'signing_dir': '/var/cache/swift',
-                'cache': 'swift.cache'
-            },
-            'filter:swift3': {'use': 'egg:swift3#swift3'}
-        }
-        if auth_api_version == '2.0':
-            expected['filter:authtoken'].update({
-                'admin_tenant_name': keystone_relation['service_tenant'],
-                'admin_user': keystone_relation['service_username'],
-                'admin_password': keystone_relation['service_password'],
-            })
-
-        if self._get_openstack_release() >= self.xenial_queens:
-            expected['pipeline:main'] = {
-                'pipeline': 'catch_errors gatekeeper healthcheck proxy-logging'
-                ' cache authtoken swift3 s3token container_sync bulk tempurl'
-                ' slo dlo formpost keystoneauth staticweb'
-                ' versioned_writes container-quotas account-quotas'
-                ' proxy-logging proxy-server'
-            }
-        elif self._get_openstack_release() >= self.trusty_mitaka:
-            expected['pipeline:main'] = {
-                'pipeline': 'catch_errors gatekeeper healthcheck proxy-logging'
-                ' cache swift3 s3token container_sync bulk tempurl slo dlo'
-                ' formpost authtoken keystoneauth staticweb'
-                ' versioned_writes container-quotas account-quotas'
-                ' proxy-logging proxy-server'
-            }
-
-        s3_token_auth_settings_legacy = {
-            'auth_port': keystone_relation['auth_port'],
-            'auth_host': keystone_relation['auth_host'],
-            'service_host': keystone_relation['service_host'],
-            'service_port': keystone_relation['service_port'],
-            'auth_protocol': keystone_relation['auth_protocol'],
-            'auth_token': keystone_relation['admin_token'],
-            'admin_token': keystone_relation['admin_token']
-        }
-
-        if self._get_openstack_release() >= self.xenial_queens:
-            expected['filter:authtoken'].update({
-                'paste.filter_factory': 'keystonemiddleware.auth_token:'
-                                        'filter_factory',
-            })
-            expected['filter:authtoken'].update({
-                'auth_url': '{}://{}:{}'.format(
-                    auth_protocol,
-                    auth_host,
-                    keystone_relation['auth_port']),
-                'auth_plugin': 'password',
-                'username': keystone_relation['service_username'],
-                'password': keystone_relation['service_password'],
-                'project_domain_name': keystone_relation['service_domain'],
-                'user_domain_name': keystone_relation['service_domain'],
-                'project_name': keystone_relation['service_tenant'],
-            })
-            expected['filter:s3token'] = {
-                'use': 'egg:swift3#s3token',
-                'auth_uri': '{}://{}:{}'.format(
-                    auth_protocol,
-                    auth_host,
-                    keystone_relation['auth_port']),
-                'auth_version': '3'
-            }
-        elif self._get_openstack_release() >= self.trusty_kilo:
-            # Kilo and later
-            expected['filter:authtoken'].update({
-                'paste.filter_factory': 'keystonemiddleware.auth_token:'
-                                        'filter_factory',
-            })
-            if auth_api_version == '3':
-                expected['filter:authtoken'].update({
-                    'auth_url': '{}://{}:{}'.format(
-                        auth_protocol,
-                        auth_host,
-                        keystone_relation['auth_port']),
-                    'auth_plugin': 'password',
-                    'username': keystone_relation['service_username'],
-                    'password': keystone_relation['service_password'],
-                    'project_domain_name': keystone_relation['service_domain'],
-                    'user_domain_name': keystone_relation['service_domain'],
-                    'project_name': keystone_relation['service_tenant'],
-                })
-            else:
-                expected['filter:authtoken'].update({
-                    'identity_uri': '{}://{}:{}'.format(
-                        auth_protocol,
-                        auth_host,
-                        keystone_relation['auth_port']),
-                })
-            expected['filter:s3token'] = {
-                # No section commonality with J and earlier
-                'paste.filter_factory': 'keystoneclient.middleware.s3_token'
-                                        ':filter_factory',
-            }
-            expected['filter:s3token'].update(s3_token_auth_settings_legacy)
-
-            if self._get_openstack_release() >= self.trusty_mitaka:
-                expected['filter:s3token']['paste.filter_factory'] = \
-                    'keystonemiddleware.s3_token:filter_factory'
-
-            # NOTE(hopem): this will need extending for newer releases once
-            #              swift-plugin-s3 is updated in UCA. See LP: #1738063
-        else:
-            # Juno and earlier
-            expected['filter:authtoken'].update({
-                'paste.filter_factory': 'keystoneclient.middleware.'
-                                        'auth_token:filter_factory',
-                'auth_host': auth_host,
-                'auth_port': keystone_relation['auth_port'],
-                'auth_protocol': auth_protocol,
-            })
-            expected['filter:s3token'] = {
-                # No section commonality with K and later
-                'paste.filter_factory': 'keystoneclient.middleware.'
-                's3_token:filter_factory',
-            }
-            expected['filter:s3token'].update(s3_token_auth_settings_legacy)
-
-        for section, pairs in expected.items():
-            ret = u.validate_config_data(unit, conf, section, pairs)
-            if ret:
-                message = "proxy-server config error: {}".format(ret)
-                amulet.raise_status(amulet.FAIL, msg=message)
-
     def test_400_swift_backed_image_create(self):
         """Create an instance in glance, which is backed by swift, and validate
         that some of the metadata for the image match in glance and swift."""
@@ -728,7 +530,6 @@ class SwiftProxyBasicDeployment(OpenStackAmuletDeployment):
             # v3 to access backend swift services.  Early v3 deployments
             # still use v2 credentials in glance for swift access.
             self._initialize_tests(api_version=3)
-        self.test_302_proxy_server_config(auth_api_version='3')
         self.test_400_swift_backed_image_create()
 
     def test_900_restart_on_config_change(self):