From 946790d780a610590235a7d1cb68a60e9043dbbb Mon Sep 17 00:00:00 2001 From: Edward Hope-Morley Date: Tue, 7 Apr 2015 22:47:03 +0100 Subject: [PATCH 01/20] [hopem,r=] Ensure that endpoint info is the same across units. --- hooks/swift_hooks.py | 43 ++++++++++++++++++++++++++++--------------- 1 file changed, 28 insertions(+), 15 deletions(-) diff --git a/hooks/swift_hooks.py b/hooks/swift_hooks.py index 2749c49..8bafd51 100755 --- a/hooks/swift_hooks.py +++ b/hooks/swift_hooks.py @@ -83,6 +83,11 @@ from charmhelpers.contrib.network.ip import ( format_ipv6_addr, ) from charmhelpers.contrib.openstack.context import ADDRESS_TYPES +from charmhelpers.contrib.peerstorage import ( + peer_echo, + peer_store, + peer_retrieve, +) from charmhelpers.contrib.charmsupport import nrpe @@ -148,22 +153,28 @@ def config_changed(): @hooks.hook('identity-service-relation-joined') def keystone_joined(relid=None): - if not is_elected_leader(SWIFT_HA_RES): - return - port = config('bind-port') - admin_url = '%s:%s' % (canonical_url(CONFIGS, ADMIN), port) - internal_url = '%s:%s/v1/AUTH_$(tenant_id)s' % \ - (canonical_url(CONFIGS, INTERNAL), port) - public_url = '%s:%s/v1/AUTH_$(tenant_id)s' % \ - (canonical_url(CONFIGS, PUBLIC), port) - relation_set(service='swift', - region=config('region'), - public_url=public_url, - internal_url=internal_url, - admin_url=admin_url, - requested_roles=config('operator-roles'), - relation_id=relid) + if not is_elected_leader(SWIFT_HA_RES): + settings = peer_retrieve() + admin_url = settings.get('admin_url') + internal_url = settings.get('internal_url') + public_url = settings.get('public_url') + else: + admin_url = '%s:%s' % (canonical_url(CONFIGS, ADMIN), port) + internal_url = ('%s:%s/v1/AUTH_$(tenant_id)s' % + (canonical_url(CONFIGS, INTERNAL), port)) + public_url = ('%s:%s/v1/AUTH_$(tenant_id)s' % + (canonical_url(CONFIGS, PUBLIC), port)) + peer_store('admin_url', admin_url) + peer_store('internal_url', internal_url) + peer_store('public_url', public_url) + + region = config('region') + roles = config('operator-roles') + + relation_set(service='swift', region=region, public_url=public_url, + internal_url=internal_url, admin_url=admin_url, + requested_roles=roles, relation_id=relid) @hooks.hook('identity-service-relation-changed') @@ -391,6 +402,8 @@ def cluster_non_leader_actions(): 'cluster-relation-departed') @restart_on_change(restart_map()) def cluster_changed(): + peer_echo(includes=['admin_url', 'internal_url', 'public_url']) + key = SwiftProxyClusterRPC.KEY_NOTIFY_LEADER_CHANGED leader_changed = relation_get(attribute=key) if leader_changed: From 8d5d5192c10ed3ac51cc7e5687f3b0e84e4c7619 Mon Sep 17 00:00:00 2001 From: Edward Hope-Morley Date: Tue, 7 Apr 2015 23:31:17 +0100 Subject: [PATCH 02/20] more --- hooks/swift_hooks.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/hooks/swift_hooks.py b/hooks/swift_hooks.py index 8bafd51..6982643 100755 --- a/hooks/swift_hooks.py +++ b/hooks/swift_hooks.py @@ -155,7 +155,7 @@ def config_changed(): def keystone_joined(relid=None): port = config('bind-port') if not is_elected_leader(SWIFT_HA_RES): - settings = peer_retrieve() + settings = peer_retrieve(key=None) admin_url = settings.get('admin_url') internal_url = settings.get('internal_url') public_url = settings.get('public_url') From 5cf9827d52325d0ef73d2783d56eabd321970fd9 Mon Sep 17 00:00:00 2001 From: Edward Hope-Morley Date: Wed, 8 Apr 2015 15:25:58 +0100 Subject: [PATCH 03/20] [trivial] adds missing ssl_ca config option --- config.yaml | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/config.yaml b/config.yaml index 587ee34..3181bb0 100644 --- a/config.yaml +++ b/config.yaml @@ -79,6 +79,12 @@ options: default: description: | Base64 encoded SSL key to use with certificate specified as ssl_cert. + ssl_ca: + type: string + default: + description: | + Base64 encoded SSL CA to use with the certificate and key provided - only + required if you are providing a privately signed ssl_cert and ssl_key. # General Swift Proxy configuration bind-port: default: 8080 From 9005b658468cd0408b11c797143dc058d699f90d Mon Sep 17 00:00:00 2001 From: Edward Hope-Morley Date: Wed, 8 Apr 2015 17:24:50 +0100 Subject: [PATCH 04/20] no need for peer store endpoints --- hooks/swift_hooks.py | 28 +++++----------------------- 1 file changed, 5 insertions(+), 23 deletions(-) diff --git a/hooks/swift_hooks.py b/hooks/swift_hooks.py index 6982643..87d1043 100755 --- a/hooks/swift_hooks.py +++ b/hooks/swift_hooks.py @@ -83,12 +83,6 @@ from charmhelpers.contrib.network.ip import ( format_ipv6_addr, ) from charmhelpers.contrib.openstack.context import ADDRESS_TYPES -from charmhelpers.contrib.peerstorage import ( - peer_echo, - peer_store, - peer_retrieve, -) - from charmhelpers.contrib.charmsupport import nrpe extra_pkgs = [ @@ -154,21 +148,11 @@ def config_changed(): @hooks.hook('identity-service-relation-joined') def keystone_joined(relid=None): port = config('bind-port') - if not is_elected_leader(SWIFT_HA_RES): - settings = peer_retrieve(key=None) - admin_url = settings.get('admin_url') - internal_url = settings.get('internal_url') - public_url = settings.get('public_url') - else: - admin_url = '%s:%s' % (canonical_url(CONFIGS, ADMIN), port) - internal_url = ('%s:%s/v1/AUTH_$(tenant_id)s' % - (canonical_url(CONFIGS, INTERNAL), port)) - public_url = ('%s:%s/v1/AUTH_$(tenant_id)s' % - (canonical_url(CONFIGS, PUBLIC), port)) - peer_store('admin_url', admin_url) - peer_store('internal_url', internal_url) - peer_store('public_url', public_url) - + admin_url = '%s:%s' % (canonical_url(CONFIGS, ADMIN), port) + internal_url = ('%s:%s/v1/AUTH_$(tenant_id)s' % + (canonical_url(CONFIGS, INTERNAL), port)) + public_url = ('%s:%s/v1/AUTH_$(tenant_id)s' % + (canonical_url(CONFIGS, PUBLIC), port)) region = config('region') roles = config('operator-roles') @@ -402,8 +386,6 @@ def cluster_non_leader_actions(): 'cluster-relation-departed') @restart_on_change(restart_map()) def cluster_changed(): - peer_echo(includes=['admin_url', 'internal_url', 'public_url']) - key = SwiftProxyClusterRPC.KEY_NOTIFY_LEADER_CHANGED leader_changed = relation_get(attribute=key) if leader_changed: From f666e53be24ccf629b9c7968638e1a610a2b2c3a Mon Sep 17 00:00:00 2001 From: Seyeong Kim Date: Thu, 9 Apr 2015 05:36:21 +0000 Subject: [PATCH 05/20] add log configuration --- config.yaml | 12 ++++++++++++ hooks/swift_context.py | 11 +++++++++++ templates/icehouse/proxy-server.conf | 13 +++++++++++-- tests/basic_deployment.py | 7 ++++++- 4 files changed, 40 insertions(+), 3 deletions(-) diff --git a/config.yaml b/config.yaml index 3181bb0..1945170 100644 --- a/config.yaml +++ b/config.yaml @@ -122,6 +122,18 @@ options: Timeouts from these requests can be recovered from so setting this to something lower than node-timeout would provide quicker error recovery while allowing for a longer timeout for non-recoverable requests (PUTs). + # Logging configuration + debug: + default: False + type: boolean + description: | + Set log-name, log-facility, log-level, log-address + log-headers: + default: False + type: boolean + description: | + Enables the ability to log request headers. + The default is False. # Manual Keystone configuration. keystone-auth-host: type: string diff --git a/hooks/swift_context.py b/hooks/swift_context.py index 0c996c2..50c8826 100644 --- a/hooks/swift_context.py +++ b/hooks/swift_context.py @@ -81,6 +81,10 @@ class SwiftIdentityContext(OSContextGenerator): def __call__(self): bind_port = config('bind-port') workers = config('workers') + log_name = 'swift' + log_facility = 'LOG_LOCAL0' + log_level = 'INFO' + log_address = '/dev/log' if workers == 0: import multiprocessing workers = multiprocessing.cpu_count() @@ -100,8 +104,15 @@ class SwiftIdentityContext(OSContextGenerator): 'delay_auth_decision': config('delay-auth-decision'), 'node_timeout': config('node-timeout'), 'recoverable_node_timeout': config('recoverable-node-timeout'), + 'log_headers': config('log-headers') } + if config('debug'): + ctxt['log_name'] = log_name + ctxt['log_facility'] = log_facility + ctxt['log_level'] = log_level + ctxt['log_address'] = log_address + # Instead of duplicating code lets use charm-helpers to set signing_dir # TODO(hopem): refactor this context handler to use charm-helpers # code. diff --git a/templates/icehouse/proxy-server.conf b/templates/icehouse/proxy-server.conf index ed7d947..1b4c0cb 100644 --- a/templates/icehouse/proxy-server.conf +++ b/templates/icehouse/proxy-server.conf @@ -3,6 +3,12 @@ bind_port = {{ bind_port }} workers = {{ workers }} user = swift bind_ip = {{ bind_host }} +log_name = {{ log_name }} +log_facility = {{ log_facility }} +log_level = {{ log_level }} +log_address = {{ log_address }} +log_headers = {{ log_headers }} + {% if ssl %} cert_file = {{ ssl_cert }} key_file = {{ ssl_key }} @@ -10,10 +16,10 @@ key_file = {{ ssl_key }} {% if auth_type == 'keystone' %} [pipeline:main] -pipeline = gatekeeper healthcheck cache swift3 s3token container_sync bulk tempurl slo dlo formpost authtoken keystoneauth staticweb container-quotas account-quotas proxy-server +pipeline = gatekeeper healthcheck proxy-logging cache swift3 s3token container_sync bulk tempurl slo dlo formpost authtoken keystoneauth staticweb container-quotas account-quotas proxy-logging proxy-server {% else %} [pipeline:main] -pipeline = gatekeeper healthcheck cache container_sync bulk tempurl slo dlo formpost tempauth staticweb container-quotas account-quotas proxy-server +pipeline = gatekeeper healthcheck proxy-logging cache container_sync bulk tempurl slo dlo formpost tempauth staticweb container-quotas account-quotas proxy-logging proxy-server {% endif %} [app:proxy-server] @@ -40,6 +46,9 @@ use = egg:swift#account_quotas [filter:container-quotas] use = egg:swift#container_quotas +[filter:proxy-logging] +use = egg:swift#proxy_logging + [filter:staticweb] use = egg:swift#staticweb diff --git a/tests/basic_deployment.py b/tests/basic_deployment.py index c4c9e1d..dbab0b4 100644 --- a/tests/basic_deployment.py +++ b/tests/basic_deployment.py @@ -368,7 +368,12 @@ class SwiftProxyBasicDeployment(OpenStackAmuletDeployment): expected = { 'DEFAULT': { 'bind_port': '8070', - 'user': 'swift' + 'user': 'swift', +# 'log_name': 'swift', +# 'log_facility': 'LOG_LOCAL0', +# 'log_level': 'INFO', +# 'log_address': '/dev/log', +# 'log_headers': 'False' }, 'pipeline:main': { 'pipeline': 'gatekeeper healthcheck cache swift3 s3token ' From 6f5a0f34daa33b93607f450588788a88c7281e0b Mon Sep 17 00:00:00 2001 From: Seyeong Kim Date: Thu, 9 Apr 2015 07:43:53 +0000 Subject: [PATCH 06/20] remove comment --- tests/basic_deployment.py | 5 ----- 1 file changed, 5 deletions(-) diff --git a/tests/basic_deployment.py b/tests/basic_deployment.py index dbab0b4..14f10f7 100644 --- a/tests/basic_deployment.py +++ b/tests/basic_deployment.py @@ -369,11 +369,6 @@ class SwiftProxyBasicDeployment(OpenStackAmuletDeployment): 'DEFAULT': { 'bind_port': '8070', 'user': 'swift', -# 'log_name': 'swift', -# 'log_facility': 'LOG_LOCAL0', -# 'log_level': 'INFO', -# 'log_address': '/dev/log', -# 'log_headers': 'False' }, 'pipeline:main': { 'pipeline': 'gatekeeper healthcheck cache swift3 s3token ' From c67ea4703f63544d792c99e9282b17db477cf5ba Mon Sep 17 00:00:00 2001 From: Seyeong Kim Date: Fri, 10 Apr 2015 00:55:17 +0000 Subject: [PATCH 07/20] add tests --- tests/basic_deployment.py | 15 +++++++++++---- 1 file changed, 11 insertions(+), 4 deletions(-) diff --git a/tests/basic_deployment.py b/tests/basic_deployment.py index 14f10f7..cd9cf51 100644 --- a/tests/basic_deployment.py +++ b/tests/basic_deployment.py @@ -369,12 +369,18 @@ class SwiftProxyBasicDeployment(OpenStackAmuletDeployment): 'DEFAULT': { 'bind_port': '8070', 'user': 'swift', + 'log_name': 'swift', + 'log_facility': 'LOG_LOCAL0', + 'log_level': 'INFO', + 'log_headers': 'False', + 'log_address': '/dev/log' }, 'pipeline:main': { - 'pipeline': 'gatekeeper healthcheck cache swift3 s3token ' - 'container_sync bulk tempurl slo dlo formpost ' - 'authtoken keystoneauth staticweb ' - 'container-quotas account-quotas proxy-server' + 'pipeline': 'gatekeeper healthcheck proxy-logging cache swift3 ' + 's3token container_sync bulk tempurl slo dlo ' + 'formpost authtoken keystoneauth staticweb ' + 'container-quotas account-quotas proxy-logging ' + 'proxy-server' }, 'app:proxy-server': { 'use': 'egg:swift#proxy', @@ -395,6 +401,7 @@ class SwiftProxyBasicDeployment(OpenStackAmuletDeployment): }, 'filter:account-quotas': {'use': 'egg:swift#account_quotas'}, 'filter:container-quotas': {'use': 'egg:swift#container_quotas'}, + 'filter:proxy-logging': {'use': 'egg:swift#proxy_logging'}, 'filter:staticweb': {'use': 'egg:swift#staticweb'}, 'filter:bulk': {'use': 'egg:swift#bulk'}, 'filter:slo': {'use': 'egg:swift#slo'}, From 945d03dc674475e75d0a07f68a07e46ba6040858 Mon Sep 17 00:00:00 2001 From: Seyeong Kim Date: Mon, 13 Apr 2015 22:57:02 +0000 Subject: [PATCH 08/20] default things to template, more option on if branch --- hooks/swift_context.py | 11 +++-------- templates/icehouse/proxy-server.conf | 6 +++--- 2 files changed, 6 insertions(+), 11 deletions(-) diff --git a/hooks/swift_context.py b/hooks/swift_context.py index 50c8826..f9b2519 100644 --- a/hooks/swift_context.py +++ b/hooks/swift_context.py @@ -81,10 +81,6 @@ class SwiftIdentityContext(OSContextGenerator): def __call__(self): bind_port = config('bind-port') workers = config('workers') - log_name = 'swift' - log_facility = 'LOG_LOCAL0' - log_level = 'INFO' - log_address = '/dev/log' if workers == 0: import multiprocessing workers = multiprocessing.cpu_count() @@ -108,10 +104,9 @@ class SwiftIdentityContext(OSContextGenerator): } if config('debug'): - ctxt['log_name'] = log_name - ctxt['log_facility'] = log_facility - ctxt['log_level'] = log_level - ctxt['log_address'] = log_address + ctxt['log_level'] = 'DEBUG' + else: + ctxt['log_level'] = 'INFO' # Instead of duplicating code lets use charm-helpers to set signing_dir # TODO(hopem): refactor this context handler to use charm-helpers diff --git a/templates/icehouse/proxy-server.conf b/templates/icehouse/proxy-server.conf index 1b4c0cb..561d457 100644 --- a/templates/icehouse/proxy-server.conf +++ b/templates/icehouse/proxy-server.conf @@ -3,10 +3,10 @@ bind_port = {{ bind_port }} workers = {{ workers }} user = swift bind_ip = {{ bind_host }} -log_name = {{ log_name }} -log_facility = {{ log_facility }} +log_name = swift +log_facility = LOG_LOCAL0 log_level = {{ log_level }} -log_address = {{ log_address }} +log_address = /dev/log log_headers = {{ log_headers }} {% if ssl %} From 7fc0830b3b1afbd365c30cc1a85c17d7137b287f Mon Sep 17 00:00:00 2001 From: Liam Young Date: Thu, 16 Apr 2015 11:26:51 +0100 Subject: [PATCH 09/20] [gnuoy,trivial] Pre-release charmhelper sync --- .../contrib/openstack/amulet/deployment.py | 29 ++- .../charmhelpers/contrib/openstack/context.py | 169 +++++++++++++++- .../charmhelpers/contrib/openstack/neutron.py | 13 ++ .../contrib/openstack/templates/git.upstart | 17 ++ .../templates/{zeromq => section-zeromq} | 4 +- hooks/charmhelpers/contrib/openstack/utils.py | 185 ++++++++++++------ hooks/charmhelpers/core/hookenv.py | 15 +- hooks/charmhelpers/core/strutils.py | 4 +- hooks/charmhelpers/core/unitdata.py | 2 +- tests/charmhelpers/contrib/amulet/utils.py | 4 +- .../contrib/openstack/amulet/deployment.py | 29 ++- 11 files changed, 392 insertions(+), 79 deletions(-) create mode 100644 hooks/charmhelpers/contrib/openstack/templates/git.upstart rename hooks/charmhelpers/contrib/openstack/templates/{zeromq => section-zeromq} (66%) diff --git a/hooks/charmhelpers/contrib/openstack/amulet/deployment.py b/hooks/charmhelpers/contrib/openstack/amulet/deployment.py index 0cfeaa4..fef9638 100644 --- a/hooks/charmhelpers/contrib/openstack/amulet/deployment.py +++ b/hooks/charmhelpers/contrib/openstack/amulet/deployment.py @@ -15,6 +15,7 @@ # along with charm-helpers. If not, see . import six +from collections import OrderedDict from charmhelpers.contrib.amulet.deployment import ( AmuletDeployment ) @@ -43,7 +44,7 @@ class OpenStackAmuletDeployment(AmuletDeployment): Determine if the local branch being tested is derived from its stable or next (dev) branch, and based on this, use the corresonding stable or next branches for the other_services.""" - base_charms = ['mysql', 'mongodb', 'rabbitmq-server'] + base_charms = ['mysql', 'mongodb'] if self.stable: for svc in other_services: @@ -100,12 +101,34 @@ class OpenStackAmuletDeployment(AmuletDeployment): """ (self.precise_essex, self.precise_folsom, self.precise_grizzly, self.precise_havana, self.precise_icehouse, - self.trusty_icehouse) = range(6) + self.trusty_icehouse, self.trusty_juno, self.trusty_kilo) = range(8) releases = { ('precise', None): self.precise_essex, ('precise', 'cloud:precise-folsom'): self.precise_folsom, ('precise', 'cloud:precise-grizzly'): self.precise_grizzly, ('precise', 'cloud:precise-havana'): self.precise_havana, ('precise', 'cloud:precise-icehouse'): self.precise_icehouse, - ('trusty', None): self.trusty_icehouse} + ('trusty', None): self.trusty_icehouse, + ('trusty', 'cloud:trusty-juno'): self.trusty_juno, + ('trusty', 'cloud:trusty-kilo'): self.trusty_kilo} return releases[(self.series, self.openstack)] + + def _get_openstack_release_string(self): + """Get openstack release string. + + Return a string representing the openstack release. + """ + releases = OrderedDict([ + ('precise', 'essex'), + ('quantal', 'folsom'), + ('raring', 'grizzly'), + ('saucy', 'havana'), + ('trusty', 'icehouse'), + ('utopic', 'juno'), + ('vivid', 'kilo'), + ]) + if self.openstack: + os_origin = self.openstack.split(':')[1] + return os_origin.split('%s-' % self.series)[1].split('/')[0] + else: + return releases[self.series] diff --git a/hooks/charmhelpers/contrib/openstack/context.py b/hooks/charmhelpers/contrib/openstack/context.py index 90ac6d6..c9914d0 100644 --- a/hooks/charmhelpers/contrib/openstack/context.py +++ b/hooks/charmhelpers/contrib/openstack/context.py @@ -47,6 +47,7 @@ from charmhelpers.core.hookenv import ( ) from charmhelpers.core.sysctl import create as sysctl_create +from charmhelpers.core.strutils import bool_from_string from charmhelpers.core.host import ( list_nics, @@ -67,6 +68,7 @@ from charmhelpers.contrib.hahelpers.apache import ( ) from charmhelpers.contrib.openstack.neutron import ( neutron_plugin_attribute, + parse_data_port_mappings, ) from charmhelpers.contrib.openstack.ip import ( resolve_address, @@ -82,7 +84,6 @@ from charmhelpers.contrib.network.ip import ( is_bridge_member, ) from charmhelpers.contrib.openstack.utils import get_host_ip - CA_CERT_PATH = '/usr/local/share/ca-certificates/keystone_juju_ca_cert.crt' ADDRESS_TYPES = ['admin', 'internal', 'public'] @@ -319,14 +320,15 @@ def db_ssl(rdata, ctxt, ssl_dir): class IdentityServiceContext(OSContextGenerator): - interfaces = ['identity-service'] - def __init__(self, service=None, service_user=None): + def __init__(self, service=None, service_user=None, rel_name='identity-service'): self.service = service self.service_user = service_user + self.rel_name = rel_name + self.interfaces = [self.rel_name] def __call__(self): - log('Generating template context for identity-service', level=DEBUG) + log('Generating template context for ' + self.rel_name, level=DEBUG) ctxt = {} if self.service and self.service_user: @@ -340,7 +342,7 @@ class IdentityServiceContext(OSContextGenerator): ctxt['signing_dir'] = cachedir - for rid in relation_ids('identity-service'): + for rid in relation_ids(self.rel_name): for unit in related_units(rid): rdata = relation_get(rid=rid, unit=unit) serv_host = rdata.get('service_host') @@ -806,6 +808,19 @@ class NeutronContext(OSContextGenerator): return ovs_ctxt + def nuage_ctxt(self): + driver = neutron_plugin_attribute(self.plugin, 'driver', + self.network_manager) + config = neutron_plugin_attribute(self.plugin, 'config', + self.network_manager) + nuage_ctxt = {'core_plugin': driver, + 'neutron_plugin': 'vsp', + 'neutron_security_groups': self.neutron_security_groups, + 'local_ip': unit_private_ip(), + 'config': config} + + return nuage_ctxt + def nvp_ctxt(self): driver = neutron_plugin_attribute(self.plugin, 'driver', self.network_manager) @@ -889,6 +904,8 @@ class NeutronContext(OSContextGenerator): ctxt.update(self.n1kv_ctxt()) elif self.plugin == 'Calico': ctxt.update(self.calico_ctxt()) + elif self.plugin == 'vsp': + ctxt.update(self.nuage_ctxt()) alchemy_flags = config('neutron-alchemy-flags') if alchemy_flags: @@ -1162,3 +1179,145 @@ class SysctlContext(OSContextGenerator): sysctl_create(sysctl_dict, '/etc/sysctl.d/50-{0}.conf'.format(charm_name())) return {'sysctl': sysctl_dict} + + +class NeutronAPIContext(OSContextGenerator): + ''' + Inspects current neutron-plugin-api relation for neutron settings. Return + defaults if it is not present. + ''' + interfaces = ['neutron-plugin-api'] + + def __call__(self): + self.neutron_defaults = { + 'l2_population': { + 'rel_key': 'l2-population', + 'default': False, + }, + 'overlay_network_type': { + 'rel_key': 'overlay-network-type', + 'default': 'gre', + }, + 'neutron_security_groups': { + 'rel_key': 'neutron-security-groups', + 'default': False, + }, + 'network_device_mtu': { + 'rel_key': 'network-device-mtu', + 'default': None, + }, + 'enable_dvr': { + 'rel_key': 'enable-dvr', + 'default': False, + }, + 'enable_l3ha': { + 'rel_key': 'enable-l3ha', + 'default': False, + }, + } + ctxt = self.get_neutron_options({}) + for rid in relation_ids('neutron-plugin-api'): + for unit in related_units(rid): + rdata = relation_get(rid=rid, unit=unit) + if 'l2-population' in rdata: + ctxt.update(self.get_neutron_options(rdata)) + + return ctxt + + def get_neutron_options(self, rdata): + settings = {} + for nkey in self.neutron_defaults.keys(): + defv = self.neutron_defaults[nkey]['default'] + rkey = self.neutron_defaults[nkey]['rel_key'] + if rkey in rdata.keys(): + if type(defv) is bool: + settings[nkey] = bool_from_string(rdata[rkey]) + else: + settings[nkey] = rdata[rkey] + else: + settings[nkey] = defv + return settings + + +class ExternalPortContext(NeutronPortContext): + + def __call__(self): + ctxt = {} + ports = config('ext-port') + if ports: + ports = [p.strip() for p in ports.split()] + ports = self.resolve_ports(ports) + if ports: + ctxt = {"ext_port": ports[0]} + napi_settings = NeutronAPIContext()() + mtu = napi_settings.get('network_device_mtu') + if mtu: + ctxt['ext_port_mtu'] = mtu + + return ctxt + + +class DataPortContext(NeutronPortContext): + + def __call__(self): + ports = config('data-port') + if ports: + portmap = parse_data_port_mappings(ports) + ports = portmap.values() + resolved = self.resolve_ports(ports) + normalized = {get_nic_hwaddr(port): port for port in resolved + if port not in ports} + normalized.update({port: port for port in resolved + if port in ports}) + if resolved: + return {bridge: normalized[port] for bridge, port in + six.iteritems(portmap) if port in normalized.keys()} + + return None + + +class PhyNICMTUContext(DataPortContext): + + def __call__(self): + ctxt = {} + mappings = super(PhyNICMTUContext, self).__call__() + if mappings and mappings.values(): + ports = mappings.values() + napi_settings = NeutronAPIContext()() + mtu = napi_settings.get('network_device_mtu') + if mtu: + ctxt["devs"] = '\\n'.join(ports) + ctxt['mtu'] = mtu + + return ctxt + + +class NetworkServiceContext(OSContextGenerator): + + def __init__(self, rel_name='quantum-network-service'): + self.rel_name = rel_name + self.interfaces = [rel_name] + + def __call__(self): + for rid in relation_ids(self.rel_name): + for unit in related_units(rid): + rdata = relation_get(rid=rid, unit=unit) + ctxt = { + 'keystone_host': rdata.get('keystone_host'), + 'service_port': rdata.get('service_port'), + 'auth_port': rdata.get('auth_port'), + 'service_tenant': rdata.get('service_tenant'), + 'service_username': rdata.get('service_username'), + 'service_password': rdata.get('service_password'), + 'quantum_host': rdata.get('quantum_host'), + 'quantum_port': rdata.get('quantum_port'), + 'quantum_url': rdata.get('quantum_url'), + 'region': rdata.get('region'), + 'service_protocol': + rdata.get('service_protocol') or 'http', + 'auth_protocol': + rdata.get('auth_protocol') or 'http', + } + if context_complete(ctxt): + return ctxt + return {} diff --git a/hooks/charmhelpers/contrib/openstack/neutron.py b/hooks/charmhelpers/contrib/openstack/neutron.py index f885105..02c92e9 100644 --- a/hooks/charmhelpers/contrib/openstack/neutron.py +++ b/hooks/charmhelpers/contrib/openstack/neutron.py @@ -180,6 +180,19 @@ def neutron_plugins(): 'nova-api-metadata']], 'server_packages': ['neutron-server', 'calico-control'], 'server_services': ['neutron-server'] + }, + 'vsp': { + 'config': '/etc/neutron/plugins/nuage/nuage_plugin.ini', + 'driver': 'neutron.plugins.nuage.plugin.NuagePlugin', + 'contexts': [ + context.SharedDBContext(user=config('neutron-database-user'), + database=config('neutron-database'), + relation_prefix='neutron', + ssl_dir=NEUTRON_CONF_DIR)], + 'services': [], + 'packages': [], + 'server_packages': ['neutron-server', 'neutron-plugin-nuage'], + 'server_services': ['neutron-server'] } } if release >= 'icehouse': diff --git a/hooks/charmhelpers/contrib/openstack/templates/git.upstart b/hooks/charmhelpers/contrib/openstack/templates/git.upstart new file mode 100644 index 0000000..4bed404 --- /dev/null +++ b/hooks/charmhelpers/contrib/openstack/templates/git.upstart @@ -0,0 +1,17 @@ +description "{{ service_description }}" +author "Juju {{ service_name }} Charm " + +start on runlevel [2345] +stop on runlevel [!2345] + +respawn + +exec start-stop-daemon --start --chuid {{ user_name }} \ + --chdir {{ start_dir }} --name {{ process_name }} \ + --exec {{ executable_name }} -- \ + {% for config_file in config_files -%} + --config-file={{ config_file }} \ + {% endfor -%} + {% if log_file -%} + --log-file={{ log_file }} + {% endif -%} diff --git a/hooks/charmhelpers/contrib/openstack/templates/zeromq b/hooks/charmhelpers/contrib/openstack/templates/section-zeromq similarity index 66% rename from hooks/charmhelpers/contrib/openstack/templates/zeromq rename to hooks/charmhelpers/contrib/openstack/templates/section-zeromq index 0695eef..95f1a76 100644 --- a/hooks/charmhelpers/contrib/openstack/templates/zeromq +++ b/hooks/charmhelpers/contrib/openstack/templates/section-zeromq @@ -3,12 +3,12 @@ rpc_backend = zmq rpc_zmq_host = {{ zmq_host }} {% if zmq_redis_address -%} -rpc_zmq_matchmaker = oslo.messaging._drivers.matchmaker_redis.MatchMakerRedis +rpc_zmq_matchmaker = redis matchmaker_heartbeat_freq = 15 matchmaker_heartbeat_ttl = 30 [matchmaker_redis] host = {{ zmq_redis_address }} {% else -%} -rpc_zmq_matchmaker = oslo.messaging._drivers.matchmaker_ring.MatchMakerRing +rpc_zmq_matchmaker = ring {% endif -%} {% endif -%} diff --git a/hooks/charmhelpers/contrib/openstack/utils.py b/hooks/charmhelpers/contrib/openstack/utils.py index 4f110c6..5a12c9d 100644 --- a/hooks/charmhelpers/contrib/openstack/utils.py +++ b/hooks/charmhelpers/contrib/openstack/utils.py @@ -30,6 +30,10 @@ import yaml from charmhelpers.contrib.network import ip +from charmhelpers.core import ( + unitdata, +) + from charmhelpers.core.hookenv import ( config, log as juju_log, @@ -330,6 +334,21 @@ def configure_installation_source(rel): error_out("Invalid openstack-release specified: %s" % rel) +def config_value_changed(option): + """ + Determine if config value changed since last call to this function. + """ + hook_data = unitdata.HookData() + with hook_data(): + db = unitdata.kv() + current = config(option) + saved = db.get(option) + db.set(option, current) + if saved is None: + return False + return current != saved + + def save_script_rc(script_path="scripts/scriptrc", **env_vars): """ Write an rc file in the charm-delivered directory containing @@ -469,82 +488,103 @@ def os_requires_version(ostack_release, pkg): def git_install_requested(): - """Returns true if openstack-origin-git is specified.""" - return config('openstack-origin-git') != "None" + """ + Returns true if openstack-origin-git is specified. + """ + return config('openstack-origin-git') is not None requirements_dir = None -def git_clone_and_install(file_name, core_project): - """Clone/install all OpenStack repos specified in yaml config file.""" - global requirements_dir +def git_clone_and_install(projects_yaml, core_project): + """ + Clone/install all specified OpenStack repositories. - if file_name == "None": + The expected format of projects_yaml is: + repositories: + - {name: keystone, + repository: 'git://git.openstack.org/openstack/keystone.git', + branch: 'stable/icehouse'} + - {name: requirements, + repository: 'git://git.openstack.org/openstack/requirements.git', + branch: 'stable/icehouse'} + directory: /mnt/openstack-git + http_proxy: http://squid.internal:3128 + https_proxy: https://squid.internal:3128 + + The directory, http_proxy, and https_proxy keys are optional. + """ + global requirements_dir + parent_dir = '/mnt/openstack-git' + + if not projects_yaml: return - yaml_file = os.path.join(charm_dir(), file_name) + projects = yaml.load(projects_yaml) + _git_validate_projects_yaml(projects, core_project) - # clone/install the requirements project first - installed = _git_clone_and_install_subset(yaml_file, - whitelist=['requirements']) - if 'requirements' not in installed: - error_out('requirements git repository must be specified') + if 'http_proxy' in projects.keys(): + os.environ['http_proxy'] = projects['http_proxy'] - # clone/install all other projects except requirements and the core project - blacklist = ['requirements', core_project] - _git_clone_and_install_subset(yaml_file, blacklist=blacklist, - update_requirements=True) + if 'https_proxy' in projects.keys(): + os.environ['https_proxy'] = projects['https_proxy'] - # clone/install the core project - whitelist = [core_project] - installed = _git_clone_and_install_subset(yaml_file, whitelist=whitelist, - update_requirements=True) - if core_project not in installed: - error_out('{} git repository must be specified'.format(core_project)) + if 'directory' in projects.keys(): + parent_dir = projects['directory'] + + for p in projects['repositories']: + repo = p['repository'] + branch = p['branch'] + if p['name'] == 'requirements': + repo_dir = _git_clone_and_install_single(repo, branch, parent_dir, + update_requirements=False) + requirements_dir = repo_dir + else: + repo_dir = _git_clone_and_install_single(repo, branch, parent_dir, + update_requirements=True) -def _git_clone_and_install_subset(yaml_file, whitelist=[], blacklist=[], - update_requirements=False): - """Clone/install subset of OpenStack repos specified in yaml config file.""" - global requirements_dir - installed = [] +def _git_validate_projects_yaml(projects, core_project): + """ + Validate the projects yaml. + """ + _git_ensure_key_exists('repositories', projects) - with open(yaml_file, 'r') as fd: - projects = yaml.load(fd) - for proj, val in projects.items(): - # The project subset is chosen based on the following 3 rules: - # 1) If project is in blacklist, we don't clone/install it, period. - # 2) If whitelist is empty, we clone/install everything else. - # 3) If whitelist is not empty, we clone/install everything in the - # whitelist. - if proj in blacklist: - continue - if whitelist and proj not in whitelist: - continue - repo = val['repository'] - branch = val['branch'] - repo_dir = _git_clone_and_install_single(repo, branch, - update_requirements) - if proj == 'requirements': - requirements_dir = repo_dir - installed.append(proj) - return installed + for project in projects['repositories']: + _git_ensure_key_exists('name', project.keys()) + _git_ensure_key_exists('repository', project.keys()) + _git_ensure_key_exists('branch', project.keys()) + + if projects['repositories'][0]['name'] != 'requirements': + error_out('{} git repo must be specified first'.format('requirements')) + + if projects['repositories'][-1]['name'] != core_project: + error_out('{} git repo must be specified last'.format(core_project)) -def _git_clone_and_install_single(repo, branch, update_requirements=False): - """Clone and install a single git repository.""" - dest_parent_dir = "/mnt/openstack-git/" - dest_dir = os.path.join(dest_parent_dir, os.path.basename(repo)) +def _git_ensure_key_exists(key, keys): + """ + Ensure that key exists in keys. + """ + if key not in keys: + error_out('openstack-origin-git key \'{}\' is missing'.format(key)) - if not os.path.exists(dest_parent_dir): - juju_log('Host dir not mounted at {}. ' - 'Creating directory there instead.'.format(dest_parent_dir)) - os.mkdir(dest_parent_dir) + +def _git_clone_and_install_single(repo, branch, parent_dir, update_requirements): + """ + Clone and install a single git repository. + """ + dest_dir = os.path.join(parent_dir, os.path.basename(repo)) + + if not os.path.exists(parent_dir): + juju_log('Directory already exists at {}. ' + 'No need to create directory.'.format(parent_dir)) + os.mkdir(parent_dir) if not os.path.exists(dest_dir): juju_log('Cloning git repo: {}, branch: {}'.format(repo, branch)) - repo_dir = install_remote(repo, dest=dest_parent_dir, branch=branch) + repo_dir = install_remote(repo, dest=parent_dir, branch=branch) else: repo_dir = dest_dir @@ -561,16 +601,39 @@ def _git_clone_and_install_single(repo, branch, update_requirements=False): def _git_update_requirements(package_dir, reqs_dir): - """Update from global requirements. + """ + Update from global requirements. - Update an OpenStack git directory's requirements.txt and - test-requirements.txt from global-requirements.txt.""" + Update an OpenStack git directory's requirements.txt and + test-requirements.txt from global-requirements.txt. + """ orig_dir = os.getcwd() os.chdir(reqs_dir) - cmd = "python update.py {}".format(package_dir) + cmd = ['python', 'update.py', package_dir] try: - subprocess.check_call(cmd.split(' ')) + subprocess.check_call(cmd) except subprocess.CalledProcessError: package = os.path.basename(package_dir) error_out("Error updating {} from global-requirements.txt".format(package)) os.chdir(orig_dir) + + +def git_src_dir(projects_yaml, project): + """ + Return the directory where the specified project's source is located. + """ + parent_dir = '/mnt/openstack-git' + + if not projects_yaml: + return + + projects = yaml.load(projects_yaml) + + if 'directory' in projects.keys(): + parent_dir = projects['directory'] + + for p in projects['repositories']: + if p['name'] == project: + return os.path.join(parent_dir, os.path.basename(p['repository'])) + + return None diff --git a/hooks/charmhelpers/core/hookenv.py b/hooks/charmhelpers/core/hookenv.py index 715dd4c..86f805f 100644 --- a/hooks/charmhelpers/core/hookenv.py +++ b/hooks/charmhelpers/core/hookenv.py @@ -20,11 +20,13 @@ # Authors: # Charm Helpers Developers +from __future__ import print_function import os import json import yaml import subprocess import sys +import errno from subprocess import CalledProcessError import six @@ -87,7 +89,18 @@ def log(message, level=None): if not isinstance(message, six.string_types): message = repr(message) command += [message] - subprocess.call(command) + # Missing juju-log should not cause failures in unit tests + # Send log output to stderr + try: + subprocess.call(command) + except OSError as e: + if e.errno == errno.ENOENT: + if level: + message = "{}: {}".format(level, message) + message = "juju-log: {}".format(message) + print(message, file=sys.stderr) + else: + raise class Serializable(UserDict): diff --git a/hooks/charmhelpers/core/strutils.py b/hooks/charmhelpers/core/strutils.py index efc4402..a2a784a 100644 --- a/hooks/charmhelpers/core/strutils.py +++ b/hooks/charmhelpers/core/strutils.py @@ -33,9 +33,9 @@ def bool_from_string(value): value = value.strip().lower() - if value in ['y', 'yes', 'true', 't']: + if value in ['y', 'yes', 'true', 't', 'on']: return True - elif value in ['n', 'no', 'false', 'f']: + elif value in ['n', 'no', 'false', 'f', 'off']: return False msg = "Unable to interpret string value '%s' as boolean" % (value) diff --git a/hooks/charmhelpers/core/unitdata.py b/hooks/charmhelpers/core/unitdata.py index 3000134..406a35c 100644 --- a/hooks/charmhelpers/core/unitdata.py +++ b/hooks/charmhelpers/core/unitdata.py @@ -443,7 +443,7 @@ class HookData(object): data = hookenv.execution_environment() self.conf = conf_delta = self.kv.delta(data['conf'], 'config') self.rels = rels_delta = self.kv.delta(data['rels'], 'rels') - self.kv.set('env', data['env']) + self.kv.set('env', dict(data['env'])) self.kv.set('unit', data['unit']) self.kv.set('relid', data.get('relid')) return conf_delta, rels_delta diff --git a/tests/charmhelpers/contrib/amulet/utils.py b/tests/charmhelpers/contrib/amulet/utils.py index 65219d3..5088b1d 100644 --- a/tests/charmhelpers/contrib/amulet/utils.py +++ b/tests/charmhelpers/contrib/amulet/utils.py @@ -118,6 +118,9 @@ class AmuletUtils(object): longs, or can be a function that evaluate a variable and returns a bool. """ + self.log.debug('actual: {}'.format(repr(actual))) + self.log.debug('expected: {}'.format(repr(expected))) + for k, v in six.iteritems(expected): if k in actual: if (isinstance(v, six.string_types) or @@ -134,7 +137,6 @@ class AmuletUtils(object): def validate_relation_data(self, sentry_unit, relation, expected): """Validate actual relation data based on expected relation data.""" actual = sentry_unit.relation(relation[0], relation[1]) - self.log.debug('actual: {}'.format(repr(actual))) return self._validate_dict_data(expected, actual) def _validate_list_data(self, expected, actual): diff --git a/tests/charmhelpers/contrib/openstack/amulet/deployment.py b/tests/charmhelpers/contrib/openstack/amulet/deployment.py index 0cfeaa4..fef9638 100644 --- a/tests/charmhelpers/contrib/openstack/amulet/deployment.py +++ b/tests/charmhelpers/contrib/openstack/amulet/deployment.py @@ -15,6 +15,7 @@ # along with charm-helpers. If not, see . import six +from collections import OrderedDict from charmhelpers.contrib.amulet.deployment import ( AmuletDeployment ) @@ -43,7 +44,7 @@ class OpenStackAmuletDeployment(AmuletDeployment): Determine if the local branch being tested is derived from its stable or next (dev) branch, and based on this, use the corresonding stable or next branches for the other_services.""" - base_charms = ['mysql', 'mongodb', 'rabbitmq-server'] + base_charms = ['mysql', 'mongodb'] if self.stable: for svc in other_services: @@ -100,12 +101,34 @@ class OpenStackAmuletDeployment(AmuletDeployment): """ (self.precise_essex, self.precise_folsom, self.precise_grizzly, self.precise_havana, self.precise_icehouse, - self.trusty_icehouse) = range(6) + self.trusty_icehouse, self.trusty_juno, self.trusty_kilo) = range(8) releases = { ('precise', None): self.precise_essex, ('precise', 'cloud:precise-folsom'): self.precise_folsom, ('precise', 'cloud:precise-grizzly'): self.precise_grizzly, ('precise', 'cloud:precise-havana'): self.precise_havana, ('precise', 'cloud:precise-icehouse'): self.precise_icehouse, - ('trusty', None): self.trusty_icehouse} + ('trusty', None): self.trusty_icehouse, + ('trusty', 'cloud:trusty-juno'): self.trusty_juno, + ('trusty', 'cloud:trusty-kilo'): self.trusty_kilo} return releases[(self.series, self.openstack)] + + def _get_openstack_release_string(self): + """Get openstack release string. + + Return a string representing the openstack release. + """ + releases = OrderedDict([ + ('precise', 'essex'), + ('quantal', 'folsom'), + ('raring', 'grizzly'), + ('saucy', 'havana'), + ('trusty', 'icehouse'), + ('utopic', 'juno'), + ('vivid', 'kilo'), + ]) + if self.openstack: + os_origin = self.openstack.split(':')[1] + return os_origin.split('%s-' % self.series)[1].split('/')[0] + else: + return releases[self.series] From 3e0971b408b5bc387d322f276c6a9ef56fa492c7 Mon Sep 17 00:00:00 2001 From: Ryan Beisner Date: Thu, 16 Apr 2015 21:31:35 +0000 Subject: [PATCH 10/20] auto flip amulet debug on --- tests/basic_deployment.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/basic_deployment.py b/tests/basic_deployment.py index c4c9e1d..a781d65 100644 --- a/tests/basic_deployment.py +++ b/tests/basic_deployment.py @@ -14,7 +14,7 @@ from charmhelpers.contrib.openstack.amulet.utils import ( ) # Use DEBUG to turn on debug logging -u = OpenStackAmuletUtils(ERROR) +u = OpenStackAmuletUtils(DEBUG) class SwiftProxyBasicDeployment(OpenStackAmuletDeployment): From 768994cb0d09b0f3ec0b4bedf9cde967090c211b Mon Sep 17 00:00:00 2001 From: Ryan Beisner Date: Thu, 16 Apr 2015 21:31:46 +0000 Subject: [PATCH 11/20] auto remove amulet tests for unsupported releases --- tests/11-basic-precise-folsom | 11 ----------- tests/12-basic-precise-grizzly | 11 ----------- tests/13-basic-precise-havana | 11 ----------- 3 files changed, 33 deletions(-) delete mode 100755 tests/11-basic-precise-folsom delete mode 100755 tests/12-basic-precise-grizzly delete mode 100755 tests/13-basic-precise-havana diff --git a/tests/11-basic-precise-folsom b/tests/11-basic-precise-folsom deleted file mode 100755 index 82fe6a6..0000000 --- a/tests/11-basic-precise-folsom +++ /dev/null @@ -1,11 +0,0 @@ -#!/usr/bin/python - -"""Amulet tests on a basic swift-proxy deployment on precise-folsom.""" - -from basic_deployment import SwiftProxyBasicDeployment - -if __name__ == '__main__': - deployment = SwiftProxyBasicDeployment(series='precise', - openstack='cloud:precise-folsom', - source='cloud:precise-updates/folsom') - deployment.run_tests() diff --git a/tests/12-basic-precise-grizzly b/tests/12-basic-precise-grizzly deleted file mode 100755 index c1a434b..0000000 --- a/tests/12-basic-precise-grizzly +++ /dev/null @@ -1,11 +0,0 @@ -#!/usr/bin/python - -"""Amulet tests on a basic swift-proxy deployment on precise-grizzly.""" - -from basic_deployment import SwiftProxyBasicDeployment - -if __name__ == '__main__': - deployment = SwiftProxyBasicDeployment(series='precise', - openstack='cloud:precise-grizzly', - source='cloud:precise-updates/grizzly') - deployment.run_tests() diff --git a/tests/13-basic-precise-havana b/tests/13-basic-precise-havana deleted file mode 100755 index 0a3780a..0000000 --- a/tests/13-basic-precise-havana +++ /dev/null @@ -1,11 +0,0 @@ -#!/usr/bin/python - -"""Amulet tests on a basic swift-proxy deployment on precise-havana.""" - -from basic_deployment import SwiftProxyBasicDeployment - -if __name__ == '__main__': - deployment = SwiftProxyBasicDeployment(series='precise', - openstack='cloud:precise-havana', - source='cloud:precise-updates/havana') - deployment.run_tests() From b590f069845dbb95d02372655e4c277746198a24 Mon Sep 17 00:00:00 2001 From: Ryan Beisner Date: Thu, 16 Apr 2015 21:31:58 +0000 Subject: [PATCH 12/20] auto add AMULET_OS_VIP to preserved env vars in makefile --- Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Makefile b/Makefile index 00e1717..a38e3db 100644 --- a/Makefile +++ b/Makefile @@ -15,7 +15,7 @@ test: # coreycb note: The -v should only be temporary until Amulet sends # raise_status() messages to stderr: # https://bugs.launchpad.net/amulet/+bug/1320357 - @juju test -v -p AMULET_HTTP_PROXY --timeout 1200 \ + @juju test -v -p AMULET_HTTP_PROXY,AMULET_OS_VIP --timeout 1200 \ 00-setup 14-basic-precise-icehouse 15-basic-trusty-icehouse bin/charm_helpers_sync.py: From 75fa4a9a3a423d81ba4e21be41b899f54c279fc2 Mon Sep 17 00:00:00 2001 From: Ryan Beisner Date: Thu, 16 Apr 2015 21:32:07 +0000 Subject: [PATCH 13/20] auto Makefile test target (amulet): bump juju test timeout to 2700s (same value as the juju-deployer default). Also remove explicit test names, which will cause all +x files in ./tests to be executed (as bundletester does by default). --- Makefile | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/Makefile b/Makefile index a38e3db..c353b10 100644 --- a/Makefile +++ b/Makefile @@ -15,8 +15,7 @@ test: # coreycb note: The -v should only be temporary until Amulet sends # raise_status() messages to stderr: # https://bugs.launchpad.net/amulet/+bug/1320357 - @juju test -v -p AMULET_HTTP_PROXY,AMULET_OS_VIP --timeout 1200 \ - 00-setup 14-basic-precise-icehouse 15-basic-trusty-icehouse + @juju test -v -p AMULET_HTTP_PROXY,AMULET_OS_VIP --timeout 2700 bin/charm_helpers_sync.py: @mkdir -p bin From c2b5441843f5a0c9f2d755c52c399f3325ae2f4a Mon Sep 17 00:00:00 2001 From: Ryan Beisner Date: Thu, 16 Apr 2015 21:35:35 +0000 Subject: [PATCH 14/20] auto sync charmhelpers --- hooks/charmhelpers/contrib/openstack/amulet/deployment.py | 7 +++++-- hooks/charmhelpers/contrib/openstack/context.py | 5 +++++ hooks/charmhelpers/contrib/openstack/utils.py | 5 ++++- tests/charmhelpers/contrib/openstack/amulet/deployment.py | 7 +++++-- 4 files changed, 19 insertions(+), 5 deletions(-) diff --git a/hooks/charmhelpers/contrib/openstack/amulet/deployment.py b/hooks/charmhelpers/contrib/openstack/amulet/deployment.py index fef9638..11d49a7 100644 --- a/hooks/charmhelpers/contrib/openstack/amulet/deployment.py +++ b/hooks/charmhelpers/contrib/openstack/amulet/deployment.py @@ -101,7 +101,8 @@ class OpenStackAmuletDeployment(AmuletDeployment): """ (self.precise_essex, self.precise_folsom, self.precise_grizzly, self.precise_havana, self.precise_icehouse, - self.trusty_icehouse, self.trusty_juno, self.trusty_kilo) = range(8) + self.trusty_icehouse, self.trusty_juno, self.trusty_kilo, + self.utopic_juno, self.vivid_kilo) = range(10) releases = { ('precise', None): self.precise_essex, ('precise', 'cloud:precise-folsom'): self.precise_folsom, @@ -110,7 +111,9 @@ class OpenStackAmuletDeployment(AmuletDeployment): ('precise', 'cloud:precise-icehouse'): self.precise_icehouse, ('trusty', None): self.trusty_icehouse, ('trusty', 'cloud:trusty-juno'): self.trusty_juno, - ('trusty', 'cloud:trusty-kilo'): self.trusty_kilo} + ('trusty', 'cloud:trusty-kilo'): self.trusty_kilo, + ('utopic', None): self.utopic_juno, + ('vivid', None): self.vivid_kilo} return releases[(self.series, self.openstack)] def _get_openstack_release_string(self): diff --git a/hooks/charmhelpers/contrib/openstack/context.py b/hooks/charmhelpers/contrib/openstack/context.py index c9914d0..400eaf8 100644 --- a/hooks/charmhelpers/contrib/openstack/context.py +++ b/hooks/charmhelpers/contrib/openstack/context.py @@ -459,6 +459,11 @@ class AMQPContext(OSContextGenerator): ctxt['rabbitmq_hosts'] = ','.join(sorted(rabbitmq_hosts)) + oslo_messaging_flags = conf.get('oslo-messaging-flags', None) + if oslo_messaging_flags: + ctxt['oslo_messaging_flags'] = config_flags_parser( + oslo_messaging_flags) + if not context_complete(ctxt): return {} diff --git a/hooks/charmhelpers/contrib/openstack/utils.py b/hooks/charmhelpers/contrib/openstack/utils.py index 5a12c9d..f90a028 100644 --- a/hooks/charmhelpers/contrib/openstack/utils.py +++ b/hooks/charmhelpers/contrib/openstack/utils.py @@ -524,9 +524,10 @@ def git_clone_and_install(projects_yaml, core_project): projects = yaml.load(projects_yaml) _git_validate_projects_yaml(projects, core_project) + old_environ = dict(os.environ) + if 'http_proxy' in projects.keys(): os.environ['http_proxy'] = projects['http_proxy'] - if 'https_proxy' in projects.keys(): os.environ['https_proxy'] = projects['https_proxy'] @@ -544,6 +545,8 @@ def git_clone_and_install(projects_yaml, core_project): repo_dir = _git_clone_and_install_single(repo, branch, parent_dir, update_requirements=True) + os.environ = old_environ + def _git_validate_projects_yaml(projects, core_project): """ diff --git a/tests/charmhelpers/contrib/openstack/amulet/deployment.py b/tests/charmhelpers/contrib/openstack/amulet/deployment.py index fef9638..11d49a7 100644 --- a/tests/charmhelpers/contrib/openstack/amulet/deployment.py +++ b/tests/charmhelpers/contrib/openstack/amulet/deployment.py @@ -101,7 +101,8 @@ class OpenStackAmuletDeployment(AmuletDeployment): """ (self.precise_essex, self.precise_folsom, self.precise_grizzly, self.precise_havana, self.precise_icehouse, - self.trusty_icehouse, self.trusty_juno, self.trusty_kilo) = range(8) + self.trusty_icehouse, self.trusty_juno, self.trusty_kilo, + self.utopic_juno, self.vivid_kilo) = range(10) releases = { ('precise', None): self.precise_essex, ('precise', 'cloud:precise-folsom'): self.precise_folsom, @@ -110,7 +111,9 @@ class OpenStackAmuletDeployment(AmuletDeployment): ('precise', 'cloud:precise-icehouse'): self.precise_icehouse, ('trusty', None): self.trusty_icehouse, ('trusty', 'cloud:trusty-juno'): self.trusty_juno, - ('trusty', 'cloud:trusty-kilo'): self.trusty_kilo} + ('trusty', 'cloud:trusty-kilo'): self.trusty_kilo, + ('utopic', None): self.utopic_juno, + ('vivid', None): self.vivid_kilo} return releases[(self.series, self.openstack)] def _get_openstack_release_string(self): From adefbd596339c83e6d9105acbf0610d0de58d4df Mon Sep 17 00:00:00 2001 From: Ryan Beisner Date: Thu, 16 Apr 2015 21:36:06 +0000 Subject: [PATCH 15/20] auto rename amulet tests --- tests/{10-basic-precise-essex => 010-basic-precise-essex} | 0 tests/{14-basic-precise-icehouse => 014-basic-precise-icehouse} | 0 tests/{15-basic-trusty-icehouse => 015-basic-trusty-icehouse} | 0 3 files changed, 0 insertions(+), 0 deletions(-) rename tests/{10-basic-precise-essex => 010-basic-precise-essex} (100%) rename tests/{14-basic-precise-icehouse => 014-basic-precise-icehouse} (100%) rename tests/{15-basic-trusty-icehouse => 015-basic-trusty-icehouse} (100%) diff --git a/tests/10-basic-precise-essex b/tests/010-basic-precise-essex similarity index 100% rename from tests/10-basic-precise-essex rename to tests/010-basic-precise-essex diff --git a/tests/14-basic-precise-icehouse b/tests/014-basic-precise-icehouse similarity index 100% rename from tests/14-basic-precise-icehouse rename to tests/014-basic-precise-icehouse diff --git a/tests/15-basic-trusty-icehouse b/tests/015-basic-trusty-icehouse similarity index 100% rename from tests/15-basic-trusty-icehouse rename to tests/015-basic-trusty-icehouse From 4648d341f97c1ab032e54e7ac76af567df57043e Mon Sep 17 00:00:00 2001 From: Ryan Beisner Date: Thu, 16 Apr 2015 21:36:19 +0000 Subject: [PATCH 16/20] auto add amulet tests for supported releases --- tests/016-basic-trusty-juno | 11 +++++++++++ tests/017-basic-trusty-kilo | 11 +++++++++++ tests/018-basic-utopic-juno | 9 +++++++++ tests/019-basic-vivid-kilo | 9 +++++++++ 4 files changed, 40 insertions(+) create mode 100755 tests/016-basic-trusty-juno create mode 100755 tests/017-basic-trusty-kilo create mode 100755 tests/018-basic-utopic-juno create mode 100755 tests/019-basic-vivid-kilo diff --git a/tests/016-basic-trusty-juno b/tests/016-basic-trusty-juno new file mode 100755 index 0000000..4dd2a18 --- /dev/null +++ b/tests/016-basic-trusty-juno @@ -0,0 +1,11 @@ +#!/usr/bin/python + +"""Amulet tests on a basic swift-proxy deployment on trusty-juno.""" + +from basic_deployment import SwiftProxyBasicDeployment + +if __name__ == '__main__': + deployment = SwiftProxyBasicDeployment(series='trusty', + openstack='cloud:trusty-juno', + source='cloud:trusty-updates/juno') + deployment.run_tests() diff --git a/tests/017-basic-trusty-kilo b/tests/017-basic-trusty-kilo new file mode 100755 index 0000000..79c7de4 --- /dev/null +++ b/tests/017-basic-trusty-kilo @@ -0,0 +1,11 @@ +#!/usr/bin/python + +"""Amulet tests on a basic swift-proxy deployment on trusty-kilo.""" + +from basic_deployment import SwiftProxyBasicDeployment + +if __name__ == '__main__': + deployment = SwiftProxyBasicDeployment(series='trusty', + openstack='cloud:trusty-kilo', + source='cloud:trusty-updates/kilo') + deployment.run_tests() diff --git a/tests/018-basic-utopic-juno b/tests/018-basic-utopic-juno new file mode 100755 index 0000000..d8c09ab --- /dev/null +++ b/tests/018-basic-utopic-juno @@ -0,0 +1,9 @@ +#!/usr/bin/python + +"""Amulet tests on a basic swift-proxy deployment on utopic-juno.""" + +from basic_deployment import SwiftProxyBasicDeployment + +if __name__ == '__main__': + deployment = SwiftProxyBasicDeployment(series='utopic') + deployment.run_tests() diff --git a/tests/019-basic-vivid-kilo b/tests/019-basic-vivid-kilo new file mode 100755 index 0000000..30c172b --- /dev/null +++ b/tests/019-basic-vivid-kilo @@ -0,0 +1,9 @@ +#!/usr/bin/python + +"""Amulet tests on a basic swift-proxy deployment on vivid-kilo.""" + +from basic_deployment import SwiftProxyBasicDeployment + +if __name__ == '__main__': + deployment = SwiftProxyBasicDeployment(series='vivid') + deployment.run_tests() From 7c81c3b8917a79ef08ecf580a7a4a3d6deae7b6a Mon Sep 17 00:00:00 2001 From: Ryan Beisner Date: Thu, 16 Apr 2015 21:36:28 +0000 Subject: [PATCH 17/20] auto disable kilo amulet tests (until later confirmed as functional) --- tests/017-basic-trusty-kilo | 0 tests/019-basic-vivid-kilo | 0 2 files changed, 0 insertions(+), 0 deletions(-) mode change 100755 => 100644 tests/017-basic-trusty-kilo mode change 100755 => 100644 tests/019-basic-vivid-kilo diff --git a/tests/017-basic-trusty-kilo b/tests/017-basic-trusty-kilo old mode 100755 new mode 100644 diff --git a/tests/019-basic-vivid-kilo b/tests/019-basic-vivid-kilo old mode 100755 new mode 100644 From ebc2c22c6457ad8abfdf7886d9ba4588149b0597 Mon Sep 17 00:00:00 2001 From: Liam Young Date: Sun, 19 Apr 2015 10:00:53 +0100 Subject: [PATCH 18/20] [gnuoy,trivial] Pre-release charmhelper sync --- hooks/charmhelpers/contrib/charmsupport/nrpe.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/hooks/charmhelpers/contrib/charmsupport/nrpe.py b/hooks/charmhelpers/contrib/charmsupport/nrpe.py index 9d961cf..95a79c2 100644 --- a/hooks/charmhelpers/contrib/charmsupport/nrpe.py +++ b/hooks/charmhelpers/contrib/charmsupport/nrpe.py @@ -247,7 +247,9 @@ class NRPE(object): service('restart', 'nagios-nrpe-server') - for rid in relation_ids("local-monitors"): + monitor_ids = relation_ids("local-monitors") + \ + relation_ids("nrpe-external-master") + for rid in monitor_ids: relation_set(relation_id=rid, monitors=yaml.dump(monitors)) From 1fbb862563b8eb61331dfac19a084ce95a689132 Mon Sep 17 00:00:00 2001 From: James Page Date: Mon, 20 Apr 2015 12:19:04 +0100 Subject: [PATCH 19/20] Tidy config descriptions --- config.yaml | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/config.yaml b/config.yaml index 1945170..dcb3aec 100644 --- a/config.yaml +++ b/config.yaml @@ -126,14 +126,11 @@ options: debug: default: False type: boolean - description: | - Set log-name, log-facility, log-level, log-address + description: Enable debug level logging. log-headers: default: False type: boolean - description: | - Enables the ability to log request headers. - The default is False. + description: Enable logging of all request headers. # Manual Keystone configuration. keystone-auth-host: type: string From bc2bb388031ce30050649a862d06046eac371a79 Mon Sep 17 00:00:00 2001 From: Liam Young Date: Thu, 23 Apr 2015 15:52:27 +0100 Subject: [PATCH 20/20] [gnuoy,trivial] Pre-release charmhelper sync --- .../contrib/openstack/amulet/deployment.py | 21 +++++++++++++------ tests/charmhelpers/contrib/amulet/utils.py | 9 +++++++- .../contrib/openstack/amulet/deployment.py | 21 +++++++++++++------ 3 files changed, 38 insertions(+), 13 deletions(-) diff --git a/hooks/charmhelpers/contrib/openstack/amulet/deployment.py b/hooks/charmhelpers/contrib/openstack/amulet/deployment.py index 11d49a7..461a702 100644 --- a/hooks/charmhelpers/contrib/openstack/amulet/deployment.py +++ b/hooks/charmhelpers/contrib/openstack/amulet/deployment.py @@ -46,15 +46,22 @@ class OpenStackAmuletDeployment(AmuletDeployment): stable or next branches for the other_services.""" base_charms = ['mysql', 'mongodb'] + if self.series in ['precise', 'trusty']: + base_series = self.series + else: + base_series = self.current_next + if self.stable: for svc in other_services: - temp = 'lp:charms/{}' - svc['location'] = temp.format(svc['name']) + temp = 'lp:charms/{}/{}' + svc['location'] = temp.format(base_series, + svc['name']) else: for svc in other_services: if svc['name'] in base_charms: - temp = 'lp:charms/{}' - svc['location'] = temp.format(svc['name']) + temp = 'lp:charms/{}/{}' + svc['location'] = temp.format(base_series, + svc['name']) else: temp = 'lp:~openstack-charmers/charms/{}/{}/next' svc['location'] = temp.format(self.current_next, @@ -99,10 +106,12 @@ class OpenStackAmuletDeployment(AmuletDeployment): Return an integer representing the enum value of the openstack release. """ + # Must be ordered by OpenStack release (not by Ubuntu release): (self.precise_essex, self.precise_folsom, self.precise_grizzly, self.precise_havana, self.precise_icehouse, - self.trusty_icehouse, self.trusty_juno, self.trusty_kilo, - self.utopic_juno, self.vivid_kilo) = range(10) + self.trusty_icehouse, self.trusty_juno, self.utopic_juno, + self.trusty_kilo, self.vivid_kilo) = range(10) + releases = { ('precise', None): self.precise_essex, ('precise', 'cloud:precise-folsom'): self.precise_folsom, diff --git a/tests/charmhelpers/contrib/amulet/utils.py b/tests/charmhelpers/contrib/amulet/utils.py index 5088b1d..f61c2e8 100644 --- a/tests/charmhelpers/contrib/amulet/utils.py +++ b/tests/charmhelpers/contrib/amulet/utils.py @@ -79,6 +79,9 @@ class AmuletUtils(object): for k, v in six.iteritems(commands): for cmd in v: output, code = k.run(cmd) + self.log.debug('{} `{}` returned ' + '{}'.format(k.info['unit_name'], + cmd, code)) if code != 0: return "command `{}` returned {}".format(cmd, str(code)) return None @@ -86,7 +89,11 @@ class AmuletUtils(object): def _get_config(self, unit, filename): """Get a ConfigParser object for parsing a unit's config file.""" file_contents = unit.file_contents(filename) - config = ConfigParser.ConfigParser() + + # NOTE(beisner): by default, ConfigParser does not handle options + # with no value, such as the flags used in the mysql my.cnf file. + # https://bugs.python.org/issue7005 + config = ConfigParser.ConfigParser(allow_no_value=True) config.readfp(io.StringIO(file_contents)) return config diff --git a/tests/charmhelpers/contrib/openstack/amulet/deployment.py b/tests/charmhelpers/contrib/openstack/amulet/deployment.py index 11d49a7..461a702 100644 --- a/tests/charmhelpers/contrib/openstack/amulet/deployment.py +++ b/tests/charmhelpers/contrib/openstack/amulet/deployment.py @@ -46,15 +46,22 @@ class OpenStackAmuletDeployment(AmuletDeployment): stable or next branches for the other_services.""" base_charms = ['mysql', 'mongodb'] + if self.series in ['precise', 'trusty']: + base_series = self.series + else: + base_series = self.current_next + if self.stable: for svc in other_services: - temp = 'lp:charms/{}' - svc['location'] = temp.format(svc['name']) + temp = 'lp:charms/{}/{}' + svc['location'] = temp.format(base_series, + svc['name']) else: for svc in other_services: if svc['name'] in base_charms: - temp = 'lp:charms/{}' - svc['location'] = temp.format(svc['name']) + temp = 'lp:charms/{}/{}' + svc['location'] = temp.format(base_series, + svc['name']) else: temp = 'lp:~openstack-charmers/charms/{}/{}/next' svc['location'] = temp.format(self.current_next, @@ -99,10 +106,12 @@ class OpenStackAmuletDeployment(AmuletDeployment): Return an integer representing the enum value of the openstack release. """ + # Must be ordered by OpenStack release (not by Ubuntu release): (self.precise_essex, self.precise_folsom, self.precise_grizzly, self.precise_havana, self.precise_icehouse, - self.trusty_icehouse, self.trusty_juno, self.trusty_kilo, - self.utopic_juno, self.vivid_kilo) = range(10) + self.trusty_icehouse, self.trusty_juno, self.utopic_juno, + self.trusty_kilo, self.vivid_kilo) = range(10) + releases = { ('precise', None): self.precise_essex, ('precise', 'cloud:precise-folsom'): self.precise_folsom,