Merge "Improve support for telemetry collection"

README.md

@@ -189,4 +189,15 @@ alternatively these can also be provided as part of a juju native bundle configu
 
 **NOTE:** Spaces must be configured in the underlying provider prior to attempting to use them.
 
-**NOTE:** Existing deployments using os-*-network configuration options will continue to function; these options are preferred over any network space binding provided if set.
+**NOTE:** Existing deployments using os-\*-network configuration options will continue to function; these options are preferred over any network space binding provided if set.
+
+Telemetry support
+------------------
+
+For OpenStack releases >= Mitaka, improved telemetry collection support is possible by
+adding a relation between swift-proxy and rabbitmq-server:
+
+    juju add-relation swift-proxy rabbitmq-server
+
+**NOTE:** In a busy Swift deployment this can place additional load on the underlying
+message bus.

config.yaml

@@ -327,3 +327,11 @@ options:
     description: |
       A comma-separated list of nagios servicegroups. If left empty, the
       nagios_context will be used as the servicegroup.
+  rabbit-user:
+    type: string
+    default: swift
+    description: Username used to access rabbitmq queue.
+  rabbit-vhost:
+    type: string
+    default: openstack
+    description: Rabbitmq vhost name.

hooks/amqp-relation-broken

@@ -0,0 +1 @@
+swift_hooks.py

hooks/amqp-relation-changed

@@ -0,0 +1 @@
+swift_hooks.py

hooks/amqp-relation-departed

@@ -0,0 +1 @@
+swift_hooks.py

hooks/amqp-relation-joined

@@ -0,0 +1 @@
+swift_hooks.py

hooks/swift_hooks.py

@@ -91,6 +91,7 @@ from charmhelpers.core.host import (
 from charmhelpers.fetch import (
     apt_install,
     apt_update,
+    filter_installed_packages,
 )
 from charmhelpers.payload.execd import execd_preinstall
 from charmhelpers.contrib.openstack.ip import (
@@ -184,6 +185,10 @@ def config_changed():
 
     for r_id in relation_ids('object-store'):
         object_store_joined(relation_id=r_id)
+
+    for r_id in relation_ids('amqp'):
+        amqp_joined(relation_id=r_id)
+
     try_initialize_swauth()
 
 
@@ -717,6 +722,11 @@ def update_nrpe_config():
 @hooks.hook('upgrade-charm')
 @harden()
 def upgrade_charm():
+    rel = openstack.get_os_codename_install_source(config('openstack-origin'))
+    pkgs = determine_packages(rel)
+    new_packages = filter_installed_packages(pkgs)
+    if new_packages:
+        apt_install(new_packages)
     update_rsync_acls()
 
 
@@ -726,6 +736,20 @@ def update_status():
     log('Updating status.')
 
 
+@hooks.hook('amqp-relation-joined')
+def amqp_joined(relation_id=None):
+    relation_set(relation_id=relation_id,
+                 username=config('rabbit-user'),
+                 vhost=config('rabbit-vhost'))
+
+
+@hooks.hook('amqp-relation-changed',
+            'amqp-relation-departed')
+@restart_on_change(restart_map())
+def amqp_changed():
+    CONFIGS.write_all()
+
+
 def main():
     try:
         hooks.execute(sys.argv)

lib/swift_utils.py

@@ -123,6 +123,8 @@ BASE_PACKAGES = [
 ]
 # > Folsom specific packages
 FOLSOM_PACKAGES = BASE_PACKAGES + ['swift-plugin-s3', 'swauth']
+# > Mitaka specific packages
+MITAKA_PACKAGES = FOLSOM_PACKAGES + ['python-ceilometermiddleware']
 
 SWIFT_HA_RES = 'grp_swift_vips'
 TEMPLATES = 'templates/'
@@ -136,7 +138,8 @@ CONFIG_FILES = OrderedDict([
     }),
     (SWIFT_PROXY_CONF, {
         'hook_contexts': [SwiftIdentityContext(),
-                          context.BindHostContext()],
+                          context.BindHostContext(),
+                          context.AMQPContext(ssl_dir=SWIFT_CONF_DIR)],
         'services': ['swift-proxy'],
     }),
     (HAPROXY_CONF, {
@@ -443,14 +446,13 @@ def ensure_swift_dir(conf_dir=os.path.dirname(SWIFT_CONF)):
 
 def determine_packages(release):
     """Determine what packages are needed for a given OpenStack release."""
-    if release == 'essex':
+    cmp_openstack = CompareOpenStackReleases(release)
-        return BASE_PACKAGES
+    if cmp_openstack >= 'mitaka':
-    elif release == 'folsom':
+        return MITAKA_PACKAGES
-        return FOLSOM_PACKAGES
+    elif cmp_openstack >= 'folsom':
-    elif release == 'grizzly':
         return FOLSOM_PACKAGES
     else:
-        return FOLSOM_PACKAGES
+        return BASE_PACKAGES
 
 
 def _load_builder(path):

metadata.yaml

@@ -40,6 +40,8 @@ requires:
   ha:
     interface: hacluster
     scope: container
+  amqp:
+    interface: rabbitmq
 peers:
   cluster:
     interface: swift-ha

templates/mitaka/proxy-server.conf

@@ -0,0 +1,135 @@
+[DEFAULT]
+bind_port = {{ bind_port }}
+workers = {{ workers }}
+user = swift
+bind_ip = {{ bind_host }}
+log_name = swift
+log_facility = LOG_LOCAL0
+log_level = {{ log_level }}
+log_address = /dev/log
+log_headers = {{ log_headers }}
+
+{% if ssl %}
+cert_file = {{ ssl_cert }}
+key_file = {{ ssl_key }}
+{% endif %}
+
+{% if auth_type == 'keystone' %}
+[pipeline:main]
+{% if transport_url %}
+pipeline = ceilometer catch_errors gatekeeper healthcheck proxy-logging cache swift3 s3token container_sync bulk tempurl slo dlo formpost authtoken keystoneauth staticweb versioned_writes container-quotas account-quotas proxy-logging proxy-server
+{% else %}
+pipeline = catch_errors gatekeeper healthcheck proxy-logging cache swift3 s3token container_sync bulk tempurl slo dlo formpost authtoken keystoneauth staticweb versioned_writes container-quotas account-quotas proxy-logging proxy-server
+{% endif %}
+{% else %}
+[pipeline:main]
+pipeline = catch_errors gatekeeper healthcheck proxy-logging cache container_sync bulk tempurl slo dlo formpost {{ auth_type }} staticweb versioned_writes container-quotas account-quotas proxy-logging proxy-server
+{% endif %}
+
+[app:proxy-server]
+use = egg:swift#proxy
+allow_account_management = true
+{% if auth_type == 'keystone' %}account_autocreate = true{% endif %}
+node_timeout = {{ node_timeout }}
+recoverable_node_timeout = {{ recoverable_node_timeout }}
+
+[filter:tempauth]
+use = egg:swift#tempauth
+user_system_root = testpass .admin https://{{ proxy_ip }}:8080/v1/AUTH_system
+
+[filter:healthcheck]
+use = egg:swift#healthcheck
+
+[filter:cache]
+use = egg:swift#memcache
+memcache_servers = {{ memcached_ip }}:11211
+
+[filter:account-quotas]
+use = egg:swift#account_quotas
+
+[filter:container-quotas]
+use = egg:swift#container_quotas
+
+[filter:proxy-logging]
+use = egg:swift#proxy_logging
+
+[filter:staticweb]
+use = egg:swift#staticweb
+
+[filter:bulk]
+use = egg:swift#bulk
+
+[filter:slo]
+use = egg:swift#slo
+
+[filter:dlo]
+use = egg:swift#dlo
+
+[filter:formpost]
+use = egg:swift#formpost
+
+[filter:tempurl]
+use = egg:swift#tempurl
+
+[filter:catch_errors]
+use = egg:swift#catch_errors
+
+[filter:versioned_writes]
+use = egg:swift#versioned_writes
+
+[filter:container_sync]
+use = egg:swift#container_sync
+
+[filter:gatekeeper]
+use = egg:swift#gatekeeper
+
+{% if auth_type == 'keystone' %}
+[filter:keystoneauth]
+use = egg:swift#keystoneauth
+operator_roles = {{ operator_roles }}
+
+[filter:authtoken]
+paste.filter_factory = keystonemiddleware.auth_token:filter_factory
+identity_uri = {{ auth_protocol }}://{{ keystone_host }}:{{ auth_port }}
+auth_uri = {{ service_protocol }}://{{ service_host }}:{{ service_port }}
+{% if api_version == '3' -%}
+auth_plugin = password
+auth_url = {{ auth_protocol }}://{{ keystone_host }}:{{ auth_port }}
+username = {{ service_user }}
+password = {{ service_password }}
+project_domain_name = {{ admin_domain_name }}
+user_domain_name = {{ admin_domain_name }}
+project_name = {{ admin_tenant_name }}
+{% else -%}
+admin_tenant_name = {{ service_tenant }}
+admin_user = {{ service_user }}
+admin_password = {{ service_password }}
+{% endif -%}
+delay_auth_decision = {{ delay_auth_decision|lower }}
+signing_dir = {{ signing_dir }}
+cache = swift.cache
+
+[filter:s3token]
+paste.filter_factory = keystonemiddleware.s3_token:filter_factory
+auth_uri = {{ auth_protocol }}://{{ keystone_host }}:{{ auth_port }}
+
+[filter:swift3]
+use = egg:swift3#swift3
+{% endif %}
+
+{% if auth_type == 'swauth' %}
+[filter:swauth]
+use = egg:swauth#swauth
+set log_name = swauth
+super_admin_key = {{ swauth_admin_key }}
+default_swift_cluster = local#https://{{ proxy_ip }}:8080/v1
+{% endif %}
+
+{% if transport_url -%}
+[filter:ceilometer]
+paste.filter_factory = ceilometermiddleware.swift:filter_factory
+url = {{ transport_url }}
+driver = messagingv2
+topic = notifications
+log_level = WARN
+{% endif -%}

tests/basic_deployment.py

@@ -468,6 +468,15 @@ class SwiftProxyBasicDeployment(OpenStackAmuletDeployment):
                 'admin_password': keystone_relation['service_password'],
             })
 
+        if self._get_openstack_release() >= self.trusty_mitaka:
+            expected['pipeline:main'] = {
+                'pipeline': 'catch_errors gatekeeper healthcheck proxy-logging'
+                ' cache swift3 s3token container_sync bulk tempurl slo dlo'
+                ' formpost authtoken keystoneauth staticweb'
+                ' versioned_writes container-quotas account-quotas'
+                ' proxy-logging proxy-server'
+            }
+
         if self._get_openstack_release() >= self.trusty_kilo:
             # Kilo and later
             expected['filter:authtoken'].update({