From c654d7ca73c8df3b8095b1a5eb3fd84423703cb1 Mon Sep 17 00:00:00 2001 From: James Page Date: Wed, 4 Dec 2013 09:43:53 +0000 Subject: [PATCH 01/98] Icehouse support --- charm-helpers.yaml | 2 +- hooks/charmhelpers/contrib/openstack/utils.py | 4 ++ hooks/charmhelpers/core/host.py | 44 +++++++++++++++++++ hooks/charmhelpers/fetch/__init__.py | 42 +++++++++++++----- 4 files changed, 81 insertions(+), 11 deletions(-) diff --git a/charm-helpers.yaml b/charm-helpers.yaml index 35e5834f..e3a717da 100644 --- a/charm-helpers.yaml +++ b/charm-helpers.yaml @@ -1,4 +1,4 @@ -branch: lp:charm-helpers +branch: lp:~james-page/charm-helpers/icehouse destination: hooks/charmhelpers include: - core diff --git a/hooks/charmhelpers/contrib/openstack/utils.py b/hooks/charmhelpers/contrib/openstack/utils.py index d66afd74..43c7df3e 100644 --- a/hooks/charmhelpers/contrib/openstack/utils.py +++ b/hooks/charmhelpers/contrib/openstack/utils.py @@ -41,6 +41,7 @@ UBUNTU_OPENSTACK_RELEASE = OrderedDict([ ('quantal', 'folsom'), ('raring', 'grizzly'), ('saucy', 'havana'), + ('trusty', 'icehouse') ]) @@ -260,6 +261,9 @@ def configure_installation_source(rel): 'havana': 'precise-updates/havana', 'havana/updates': 'precise-updates/havana', 'havana/proposed': 'precise-proposed/havana', + 'icehouse': 'precise-updates/icehouse', + 'icehouse/updates': 'precise-updates/icehouse', + 'icehouse/proposed': 'precise-proposed/icehouse', } try: diff --git a/hooks/charmhelpers/core/host.py b/hooks/charmhelpers/core/host.py index 4a6a4a8c..c8c81b28 100644 --- a/hooks/charmhelpers/core/host.py +++ b/hooks/charmhelpers/core/host.py @@ -245,3 +245,47 @@ def pwgen(length=None): random_chars = [ random.choice(alphanumeric_chars) for _ in range(length)] return(''.join(random_chars)) + + +def list_nics(nic_type): + '''Return a list of nics of given type(s)''' + if isinstance(nic_type, basestring): + int_types = [nic_type] + else: + int_types = nic_type + interfaces = [] + for int_type in int_types: + cmd = ['ip', 'addr', 'show', 'label', int_type + '*'] + ip_output = subprocess.check_output(cmd).split('\n') + ip_output = (line for line in ip_output if line) + for line in ip_output: + if line.split()[1].startswith(int_type): + interfaces.append(line.split()[1].replace(":", "")) + return interfaces + + +def set_nic_mtu(nic, mtu): + '''Set MTU on a network interface''' + cmd = ['ip', 'link', 'set', nic, 'mtu', mtu] + subprocess.check_call(cmd) + + +def get_nic_mtu(nic): + cmd = ['ip', 'addr', 'show', nic] + ip_output = subprocess.check_output(cmd).split('\n') + mtu = "" + for line in ip_output: + words = line.split() + if 'mtu' in words: + mtu = words[words.index("mtu") + 1] + return mtu + + +def get_nic_hwaddr(nic): + cmd = ['ip', '-o', '-0', 'addr', 'show', nic] + ip_output = subprocess.check_output(cmd) + hwaddr = "" + words = ip_output.split() + if 'link/ether' in words: + hwaddr = words[words.index('link/ether') + 1] + return hwaddr diff --git a/hooks/charmhelpers/fetch/__init__.py b/hooks/charmhelpers/fetch/__init__.py index fa0172a9..1f4f6315 100644 --- a/hooks/charmhelpers/fetch/__init__.py +++ b/hooks/charmhelpers/fetch/__init__.py @@ -13,6 +13,7 @@ from charmhelpers.core.hookenv import ( log, ) import apt_pkg +import os CLOUD_ARCHIVE = """# Ubuntu Cloud Archive deb http://ubuntu-cloud.archive.canonical.com/ubuntu {} main @@ -43,8 +44,16 @@ CLOUD_ARCHIVE_POCKETS = { 'precise-havana/updates': 'precise-updates/havana', 'precise-updates/havana': 'precise-updates/havana', 'havana/proposed': 'precise-proposed/havana', - 'precies-havana/proposed': 'precise-proposed/havana', + 'precise-havana/proposed': 'precise-proposed/havana', 'precise-proposed/havana': 'precise-proposed/havana', + # Icehouse + 'icehouse': 'precise-updates/icehouse', + 'precise-icehouse': 'precise-updates/icehouse', + 'precise-icehouse/updates': 'precise-updates/icehouse', + 'precise-updates/icehouse': 'precise-updates/icehouse', + 'icehouse/proposed': 'precise-proposed/icehouse', + 'precise-icehouse/proposed': 'precise-proposed/icehouse', + 'precise-proposed/icehouse': 'precise-proposed/icehouse', } @@ -66,8 +75,10 @@ def filter_installed_packages(packages): def apt_install(packages, options=None, fatal=False): """Install one or more packages""" - options = options or [] - cmd = ['apt-get', '-y'] + if options is None: + options = ['--option=Dpkg::Options::=--force-confold'] + + cmd = ['apt-get', '--assume-yes'] cmd.extend(options) cmd.append('install') if isinstance(packages, basestring): @@ -76,10 +87,14 @@ def apt_install(packages, options=None, fatal=False): cmd.extend(packages) log("Installing {} with options: {}".format(packages, options)) + env = os.environ.copy() + if 'DEBIAN_FRONTEND' not in env: + env['DEBIAN_FRONTEND'] = 'noninteractive' + if fatal: - subprocess.check_call(cmd) + subprocess.check_call(cmd, env=env) else: - subprocess.call(cmd) + subprocess.call(cmd, env=env) def apt_update(fatal=False): @@ -93,7 +108,7 @@ def apt_update(fatal=False): def apt_purge(packages, fatal=False): """Purge one or more packages""" - cmd = ['apt-get', '-y', 'purge'] + cmd = ['apt-get', '--assume-yes', 'purge'] if isinstance(packages, basestring): cmd.append(packages) else: @@ -123,14 +138,16 @@ def add_source(source, key=None): if (source.startswith('ppa:') or source.startswith('http:') or source.startswith('deb ') or - source.startswith('cloud-archive:')): + source.startswith('cloud-archive:')): subprocess.check_call(['add-apt-repository', '--yes', source]) elif source.startswith('cloud:'): apt_install(filter_installed_packages(['ubuntu-cloud-keyring']), fatal=True) pocket = source.split(':')[-1] if pocket not in CLOUD_ARCHIVE_POCKETS: - raise SourceConfigError('Unsupported cloud: source option %s' % pocket) + raise SourceConfigError( + 'Unsupported cloud: source option %s' % + pocket) actual_pocket = CLOUD_ARCHIVE_POCKETS[pocket] with open('/etc/apt/sources.list.d/cloud-archive.list', 'w') as apt: apt.write(CLOUD_ARCHIVE.format(actual_pocket)) @@ -220,7 +237,9 @@ def install_from_config(config_var_name): class BaseFetchHandler(object): + """Base class for FetchHandler implementations in fetch plugins""" + def can_handle(self, source): """Returns True if the source can be handled. Otherwise returns a string explaining why it cannot""" @@ -248,10 +267,13 @@ def plugins(fetch_handlers=None): for handler_name in fetch_handlers: package, classname = handler_name.rsplit('.', 1) try: - handler_class = getattr(importlib.import_module(package), classname) + handler_class = getattr( + importlib.import_module(package), + classname) plugin_list.append(handler_class()) except (ImportError, AttributeError): # Skip missing plugins so that they can be ommitted from # installation if desired - log("FetchHandler {} not found, skipping plugin".format(handler_name)) + log("FetchHandler {} not found, skipping plugin".format( + handler_name)) return plugin_list From 806af5816a8533e676c2906181d84200b226e45e Mon Sep 17 00:00:00 2001 From: James Page Date: Wed, 4 Dec 2013 10:11:39 +0000 Subject: [PATCH 02/98] Use charm-helpers trunk --- charm-helpers.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/charm-helpers.yaml b/charm-helpers.yaml index e3a717da..35e5834f 100644 --- a/charm-helpers.yaml +++ b/charm-helpers.yaml @@ -1,4 +1,4 @@ -branch: lp:~james-page/charm-helpers/icehouse +branch: lp:charm-helpers destination: hooks/charmhelpers include: - core From c54a7ba6edbed4b246a732153ec1d5094c72be98 Mon Sep 17 00:00:00 2001 From: James Page Date: Tue, 17 Dec 2013 16:05:51 +0000 Subject: [PATCH 03/98] Sync neutron/icehouse updates --- charm-helpers.yaml | 2 +- hooks/charmhelpers/contrib/openstack/neutron.py | 8 +++++++- hooks/charmhelpers/contrib/openstack/utils.py | 2 +- hooks/charmhelpers/contrib/storage/linux/utils.py | 2 +- 4 files changed, 10 insertions(+), 4 deletions(-) diff --git a/charm-helpers.yaml b/charm-helpers.yaml index 35e5834f..64dd7f89 100644 --- a/charm-helpers.yaml +++ b/charm-helpers.yaml @@ -1,4 +1,4 @@ -branch: lp:charm-helpers +branch: lp:~openstack-charmers/charm-helpers/icehouse destination: hooks/charmhelpers include: - core diff --git a/hooks/charmhelpers/contrib/openstack/neutron.py b/hooks/charmhelpers/contrib/openstack/neutron.py index 8d32bd00..a2a0c7c7 100644 --- a/hooks/charmhelpers/contrib/openstack/neutron.py +++ b/hooks/charmhelpers/contrib/openstack/neutron.py @@ -57,7 +57,8 @@ def quantum_plugins(): def neutron_plugins(): from charmhelpers.contrib.openstack import context - return { + release = os_release('nova-common') + plugins = { 'ovs': { 'config': '/etc/neutron/plugins/openvswitch/' 'ovs_neutron_plugin.ini', @@ -89,6 +90,11 @@ def neutron_plugins(): 'server_services': ['neutron-server'] } } + # NOTE: patch in ml2 plugin for icehouse onwards + if release >= 'icehouse': + plugins['ovs']['config'] = '/etc/neutron/plugins/ml2/ml2_conf.ini' + plugins['ovs']['driver'] = 'neutron.plugins.ml2.plugin.Ml2Plugin' + return plugins def neutron_plugin_attribute(plugin, attr, net_manager=None): diff --git a/hooks/charmhelpers/contrib/openstack/utils.py b/hooks/charmhelpers/contrib/openstack/utils.py index 43c7df3e..67a23378 100644 --- a/hooks/charmhelpers/contrib/openstack/utils.py +++ b/hooks/charmhelpers/contrib/openstack/utils.py @@ -202,7 +202,7 @@ def os_release(package, base='essex'): def import_key(keyid): - cmd = "apt-key adv --keyserver keyserver.ubuntu.com " \ + cmd = "apt-key adv --keyserver hkp://keyserver.ubuntu.com:80 " \ "--recv-keys %s" % keyid try: subprocess.check_call(cmd.split(' ')) diff --git a/hooks/charmhelpers/contrib/storage/linux/utils.py b/hooks/charmhelpers/contrib/storage/linux/utils.py index 5b9b6d47..c40218f0 100644 --- a/hooks/charmhelpers/contrib/storage/linux/utils.py +++ b/hooks/charmhelpers/contrib/storage/linux/utils.py @@ -22,4 +22,4 @@ def zap_disk(block_device): :param block_device: str: Full path of block device to clean. ''' - check_call(['sgdisk', '--zap-all', block_device]) + check_call(['sgdisk', '--zap-all', '--mbrtogpt', block_device]) From 38e33db03ffaeb03a6cfe6951854b0ffb37d7ddc Mon Sep 17 00:00:00 2001 From: James Page Date: Tue, 17 Dec 2013 16:18:50 +0000 Subject: [PATCH 04/98] Resync helpers --- hooks/charmhelpers/contrib/openstack/neutron.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/hooks/charmhelpers/contrib/openstack/neutron.py b/hooks/charmhelpers/contrib/openstack/neutron.py index a2a0c7c7..2845bfda 100644 --- a/hooks/charmhelpers/contrib/openstack/neutron.py +++ b/hooks/charmhelpers/contrib/openstack/neutron.py @@ -70,7 +70,7 @@ def neutron_plugins(): relation_prefix='neutron')], 'services': ['neutron-plugin-openvswitch-agent'], 'packages': [[headers_package(), 'openvswitch-datapath-dkms'], - ['quantum-plugin-openvswitch-agent']], + ['neutron-plugin-openvswitch-agent']], 'server_packages': ['neutron-server', 'neutron-plugin-openvswitch'], 'server_services': ['neutron-server'] From e695d603a3141b6ccfeab845bd7b3d9b54e7aeba Mon Sep 17 00:00:00 2001 From: James Page Date: Wed, 18 Dec 2013 10:06:23 +0000 Subject: [PATCH 05/98] Add template for ml2 plugin --- templates/icehouse/ml2_conf.ini | 29 +++++++++++++++++++++++++++++ 1 file changed, 29 insertions(+) create mode 100644 templates/icehouse/ml2_conf.ini diff --git a/templates/icehouse/ml2_conf.ini b/templates/icehouse/ml2_conf.ini new file mode 100644 index 00000000..729f8f86 --- /dev/null +++ b/templates/icehouse/ml2_conf.ini @@ -0,0 +1,29 @@ +# icehouse +############################################################################### +# [ WARNING ] +# Configuration file maintained by Juju. Local changes may be overwritten. +############################################################################### +[ml2] +type_drivers = gre,vxlan +tenant_network_types = gre,vxlan +mechanism_drivers = openvswitch + +[ml2_type_gre] +tunnel_id_ranges = 1:1000 + +[ml2_type_vxlan] +vni_ranges = 1001:2000 + +[ovs] +enable_tunneling = True +local_ip = {{ local_ip }} + +[agent] +tunnel_types = gre + +[securitygroup] +{% if neutron_security_groups -%} +firewall_driver = neutron.agent.linux.iptables_firewall.OVSHybridIptablesFirewallDriver +{% else -%} +firewall_driver = neutron.agent.firewall.NoopFirewallDriver +{% endif -%} From 56644a732d53dc34119efa02aa39ef41aec983e4 Mon Sep 17 00:00:00 2001 From: James Page Date: Fri, 10 Jan 2014 12:36:18 +0000 Subject: [PATCH 06/98] Resync helpers --- hooks/charmhelpers/contrib/openstack/neutron.py | 2 ++ hooks/charmhelpers/contrib/openstack/utils.py | 1 + 2 files changed, 3 insertions(+) diff --git a/hooks/charmhelpers/contrib/openstack/neutron.py b/hooks/charmhelpers/contrib/openstack/neutron.py index 2845bfda..87094d0a 100644 --- a/hooks/charmhelpers/contrib/openstack/neutron.py +++ b/hooks/charmhelpers/contrib/openstack/neutron.py @@ -94,6 +94,8 @@ def neutron_plugins(): if release >= 'icehouse': plugins['ovs']['config'] = '/etc/neutron/plugins/ml2/ml2_conf.ini' plugins['ovs']['driver'] = 'neutron.plugins.ml2.plugin.Ml2Plugin' + plugins['ovs']['server_packages'] = ['neutron-server', + 'neutron-plugin-ml2'] return plugins diff --git a/hooks/charmhelpers/contrib/openstack/utils.py b/hooks/charmhelpers/contrib/openstack/utils.py index 67a23378..b6e491cf 100644 --- a/hooks/charmhelpers/contrib/openstack/utils.py +++ b/hooks/charmhelpers/contrib/openstack/utils.py @@ -65,6 +65,7 @@ SWIFT_CODENAMES = OrderedDict([ ('1.10.0', 'havana'), ('1.9.1', 'havana'), ('1.9.0', 'havana'), + ('1.11.0', 'icehouse'), ]) DEFAULT_LOOPBACK_SIZE = '5G' From c362981e0c19bfeac0afc8610db1a74b152c50e1 Mon Sep 17 00:00:00 2001 From: James Page Date: Mon, 20 Jan 2014 10:30:38 +0000 Subject: [PATCH 07/98] Specialize neutron.conf for icehouse --- templates/icehouse/neutron.conf | 59 +++++++++++++++++++++++++++++++++ 1 file changed, 59 insertions(+) create mode 100644 templates/icehouse/neutron.conf diff --git a/templates/icehouse/neutron.conf b/templates/icehouse/neutron.conf new file mode 100644 index 00000000..191eb6db --- /dev/null +++ b/templates/icehouse/neutron.conf @@ -0,0 +1,59 @@ +############################################################################### +# [ WARNING ] +# Configuration file maintained by Juju. Local changes may be overwritten. +############################################################################### +[DEFAULT] +state_path = /var/lib/neutron +lock_path = $state_path/lock +bind_host = 0.0.0.0 +auth_strategy = keystone +notification_driver = neutron.openstack.common.notifier.rpc_notifier +{% if core_plugin -%} +core_plugin = {{ core_plugin }} +{% if neutron_plugin in ['ovs', 'ml2'] -%} +service_plugins = neutron.services.l3_router.l3_router_plugin.L3RouterPlugin +{% endif -%} +{% endif -%} +{% if neutron_security_groups -%} +allow_overlapping_ips = True +neutron_firewall_driver = neutron.agent.linux.iptables_firewall.OVSHybridIptablesFirewallDriver +{% endif -%} +{% if rabbitmq_host -%} +rabbit_host = {{ rabbitmq_host }} +rabbit_userid = {{ rabbitmq_user }} +rabbit_password = {{ rabbitmq_password }} +rabbit_virtual_host = {{ rabbitmq_virtual_host }} +{% endif -%} + +[quotas] +quota_driver = neutron.db.quota_db.DbQuotaDriver +{% if neutron_security_groups -%} +quota_items = network,subnet,port,security_group,security_group_rule +{% endif -%} + +[agent] +root_helper = sudo /usr/bin/neutron-rootwrap /etc/neutron/rootwrap.conf + +[keystone_authtoken] +signing_dir = $state_path/keystone-signing +{% if service_host -%} +service_protocol = {{ service_protocol }} +service_host = {{ service_host }} +service_port = {{ service_port }} +auth_host = {{ auth_host }} +auth_port = {{ auth_port }} +auth_protocol = {{ auth_protocol }} +admin_tenant_name = {{ admin_tenant_name }} +admin_user = {{ admin_user }} +admin_password = {{ admin_password }} +{% endif -%} + +[database] +{% if database_host -%} +connection = mysql://{{ database_user }}:{{ database_password }}@{{ database_host }}/{{ database }} +{% else -%} +connection = sqlite:////var/lib/neutron/neutron.sqlite +{% endif -%} + +[lbaas] +[service_providers] From d07048b69371eb5db7d186b17a077e7ddb8f0851 Mon Sep 17 00:00:00 2001 From: "yolanda.robla@canonical.com" <> Date: Tue, 28 Jan 2014 12:46:09 +0100 Subject: [PATCH 08/98] adding support for rabbit active/active --- .../charmhelpers/contrib/openstack/context.py | 76 +++++++++--- hooks/charmhelpers/contrib/openstack/utils.py | 20 ++- .../contrib/storage/linux/utils.py | 2 +- hooks/charmhelpers/core/hookenv.py | 6 + hooks/charmhelpers/core/host.py | 44 +++++++ hooks/charmhelpers/fetch/__init__.py | 42 +++++-- revision | 2 +- templates/grizzly/nova.conf | 114 ++++++++++++++++++ templates/grizzly/quantum.conf | 55 +++++++++ templates/havana/neutron.conf | 12 +- 10 files changed, 333 insertions(+), 40 deletions(-) create mode 100644 templates/grizzly/nova.conf create mode 100644 templates/grizzly/quantum.conf diff --git a/hooks/charmhelpers/contrib/openstack/context.py b/hooks/charmhelpers/contrib/openstack/context.py index 8a982ffa..6331a92e 100644 --- a/hooks/charmhelpers/contrib/openstack/context.py +++ b/hooks/charmhelpers/contrib/openstack/context.py @@ -23,7 +23,6 @@ from charmhelpers.core.hookenv import ( unit_get, unit_private_ip, ERROR, - WARNING, ) from charmhelpers.contrib.hahelpers.cluster import ( @@ -182,10 +181,12 @@ class AMQPContext(OSContextGenerator): # Sufficient information found = break out! break # Used for active/active rabbitmq >= grizzly - ctxt['rabbitmq_hosts'] = [] - for unit in related_units(rid): - ctxt['rabbitmq_hosts'].append(relation_get('private-address', - rid=rid, unit=unit)) + if 'clustered' not in ctxt and len(related_units(rid)) > 1: + rabbitmq_hosts = [] + for unit in related_units(rid): + rabbitmq_hosts.append(relation_get('private-address', + rid=rid, unit=unit)) + ctxt['rabbitmq_hosts'] = ','.join(rabbitmq_hosts) if not context_complete(ctxt): return {} else: @@ -286,6 +287,7 @@ class ImageServiceContext(OSContextGenerator): class ApacheSSLContext(OSContextGenerator): + """ Generates a context for an apache vhost configuration that configures HTTPS reverse proxying for one or many endpoints. Generated context @@ -433,28 +435,61 @@ class NeutronContext(object): class OSConfigFlagContext(OSContextGenerator): - ''' - Responsible adding user-defined config-flags in charm config to a - to a template context. - ''' + + """ + Responsible for adding user-defined config-flags in charm config to a + template context. + + NOTE: the value of config-flags may be a comma-separated list of + key=value pairs and some Openstack config files support + comma-separated lists as values. + """ + def __call__(self): config_flags = config('config-flags') - if not config_flags or config_flags in ['None', '']: + if not config_flags: return {} - config_flags = config_flags.split(',') + + if config_flags.find('==') >= 0: + log("config_flags is not in expected format (key=value)", + level=ERROR) + raise OSContextError + + # strip the following from each value. + post_strippers = ' ,' + # we strip any leading/trailing '=' or ' ' from the string then + # split on '='. + split = config_flags.strip(' =').split('=') + limit = len(split) flags = {} - for flag in config_flags: - if '=' not in flag: - log('Improperly formatted config-flag, expected k=v ' - 'got %s' % flag, level=WARNING) - continue - k, v = flag.split('=') - flags[k.strip()] = v - ctxt = {'user_config_flags': flags} - return ctxt + for i in xrange(0, limit - 1): + current = split[i] + next = split[i + 1] + vindex = next.rfind(',') + if (i == limit - 2) or (vindex < 0): + value = next + else: + value = next[:vindex] + + if i == 0: + key = current + else: + # if this not the first entry, expect an embedded key. + index = current.rfind(',') + if index < 0: + log("invalid config value(s) at index %s" % (i), + level=ERROR) + raise OSContextError + key = current[index + 1:] + + # Add to collection. + flags[key.strip(post_strippers)] = value.rstrip(post_strippers) + + return {'user_config_flags': flags} class SubordinateConfigContext(OSContextGenerator): + """ Responsible for inspecting relations to subordinates that may be exporting required config via a json blob. @@ -495,6 +530,7 @@ class SubordinateConfigContext(OSContextGenerator): } """ + def __init__(self, service, config_file, interface): """ :param service : Service name key to query in any subordinate diff --git a/hooks/charmhelpers/contrib/openstack/utils.py b/hooks/charmhelpers/contrib/openstack/utils.py index d66afd74..56d04245 100644 --- a/hooks/charmhelpers/contrib/openstack/utils.py +++ b/hooks/charmhelpers/contrib/openstack/utils.py @@ -41,6 +41,7 @@ UBUNTU_OPENSTACK_RELEASE = OrderedDict([ ('quantal', 'folsom'), ('raring', 'grizzly'), ('saucy', 'havana'), + ('trusty', 'icehouse') ]) @@ -201,7 +202,7 @@ def os_release(package, base='essex'): def import_key(keyid): - cmd = "apt-key adv --keyserver keyserver.ubuntu.com " \ + cmd = "apt-key adv --keyserver hkp://keyserver.ubuntu.com:80 " \ "--recv-keys %s" % keyid try: subprocess.check_call(cmd.split(' ')) @@ -260,6 +261,9 @@ def configure_installation_source(rel): 'havana': 'precise-updates/havana', 'havana/updates': 'precise-updates/havana', 'havana/proposed': 'precise-proposed/havana', + 'icehouse': 'precise-updates/icehouse', + 'icehouse/updates': 'precise-updates/icehouse', + 'icehouse/proposed': 'precise-proposed/icehouse', } try: @@ -411,7 +415,7 @@ def get_host_ip(hostname): return ns_query(hostname) -def get_hostname(address): +def get_hostname(address, fqdn=True): """ Resolves hostname for given IP, or returns the input if it is already a hostname. @@ -430,7 +434,11 @@ def get_hostname(address): if not result: return None - # strip trailing . - if result.endswith('.'): - return result[:-1] - return result + if fqdn: + # strip trailing . + if result.endswith('.'): + return result[:-1] + else: + return result + else: + return result.split('.')[0] diff --git a/hooks/charmhelpers/contrib/storage/linux/utils.py b/hooks/charmhelpers/contrib/storage/linux/utils.py index 5b9b6d47..c40218f0 100644 --- a/hooks/charmhelpers/contrib/storage/linux/utils.py +++ b/hooks/charmhelpers/contrib/storage/linux/utils.py @@ -22,4 +22,4 @@ def zap_disk(block_device): :param block_device: str: Full path of block device to clean. ''' - check_call(['sgdisk', '--zap-all', block_device]) + check_call(['sgdisk', '--zap-all', '--mbrtogpt', block_device]) diff --git a/hooks/charmhelpers/core/hookenv.py b/hooks/charmhelpers/core/hookenv.py index bb196dfa..505c202d 100644 --- a/hooks/charmhelpers/core/hookenv.py +++ b/hooks/charmhelpers/core/hookenv.py @@ -8,6 +8,7 @@ import os import json import yaml import subprocess +import sys import UserDict from subprocess import CalledProcessError @@ -149,6 +150,11 @@ def service_name(): return local_unit().split('/')[0] +def hook_name(): + """The name of the currently executing hook""" + return os.path.basename(sys.argv[0]) + + @cached def config(scope=None): """Juju charm configuration""" diff --git a/hooks/charmhelpers/core/host.py b/hooks/charmhelpers/core/host.py index 4a6a4a8c..c8c81b28 100644 --- a/hooks/charmhelpers/core/host.py +++ b/hooks/charmhelpers/core/host.py @@ -245,3 +245,47 @@ def pwgen(length=None): random_chars = [ random.choice(alphanumeric_chars) for _ in range(length)] return(''.join(random_chars)) + + +def list_nics(nic_type): + '''Return a list of nics of given type(s)''' + if isinstance(nic_type, basestring): + int_types = [nic_type] + else: + int_types = nic_type + interfaces = [] + for int_type in int_types: + cmd = ['ip', 'addr', 'show', 'label', int_type + '*'] + ip_output = subprocess.check_output(cmd).split('\n') + ip_output = (line for line in ip_output if line) + for line in ip_output: + if line.split()[1].startswith(int_type): + interfaces.append(line.split()[1].replace(":", "")) + return interfaces + + +def set_nic_mtu(nic, mtu): + '''Set MTU on a network interface''' + cmd = ['ip', 'link', 'set', nic, 'mtu', mtu] + subprocess.check_call(cmd) + + +def get_nic_mtu(nic): + cmd = ['ip', 'addr', 'show', nic] + ip_output = subprocess.check_output(cmd).split('\n') + mtu = "" + for line in ip_output: + words = line.split() + if 'mtu' in words: + mtu = words[words.index("mtu") + 1] + return mtu + + +def get_nic_hwaddr(nic): + cmd = ['ip', '-o', '-0', 'addr', 'show', nic] + ip_output = subprocess.check_output(cmd) + hwaddr = "" + words = ip_output.split() + if 'link/ether' in words: + hwaddr = words[words.index('link/ether') + 1] + return hwaddr diff --git a/hooks/charmhelpers/fetch/__init__.py b/hooks/charmhelpers/fetch/__init__.py index fa0172a9..1f4f6315 100644 --- a/hooks/charmhelpers/fetch/__init__.py +++ b/hooks/charmhelpers/fetch/__init__.py @@ -13,6 +13,7 @@ from charmhelpers.core.hookenv import ( log, ) import apt_pkg +import os CLOUD_ARCHIVE = """# Ubuntu Cloud Archive deb http://ubuntu-cloud.archive.canonical.com/ubuntu {} main @@ -43,8 +44,16 @@ CLOUD_ARCHIVE_POCKETS = { 'precise-havana/updates': 'precise-updates/havana', 'precise-updates/havana': 'precise-updates/havana', 'havana/proposed': 'precise-proposed/havana', - 'precies-havana/proposed': 'precise-proposed/havana', + 'precise-havana/proposed': 'precise-proposed/havana', 'precise-proposed/havana': 'precise-proposed/havana', + # Icehouse + 'icehouse': 'precise-updates/icehouse', + 'precise-icehouse': 'precise-updates/icehouse', + 'precise-icehouse/updates': 'precise-updates/icehouse', + 'precise-updates/icehouse': 'precise-updates/icehouse', + 'icehouse/proposed': 'precise-proposed/icehouse', + 'precise-icehouse/proposed': 'precise-proposed/icehouse', + 'precise-proposed/icehouse': 'precise-proposed/icehouse', } @@ -66,8 +75,10 @@ def filter_installed_packages(packages): def apt_install(packages, options=None, fatal=False): """Install one or more packages""" - options = options or [] - cmd = ['apt-get', '-y'] + if options is None: + options = ['--option=Dpkg::Options::=--force-confold'] + + cmd = ['apt-get', '--assume-yes'] cmd.extend(options) cmd.append('install') if isinstance(packages, basestring): @@ -76,10 +87,14 @@ def apt_install(packages, options=None, fatal=False): cmd.extend(packages) log("Installing {} with options: {}".format(packages, options)) + env = os.environ.copy() + if 'DEBIAN_FRONTEND' not in env: + env['DEBIAN_FRONTEND'] = 'noninteractive' + if fatal: - subprocess.check_call(cmd) + subprocess.check_call(cmd, env=env) else: - subprocess.call(cmd) + subprocess.call(cmd, env=env) def apt_update(fatal=False): @@ -93,7 +108,7 @@ def apt_update(fatal=False): def apt_purge(packages, fatal=False): """Purge one or more packages""" - cmd = ['apt-get', '-y', 'purge'] + cmd = ['apt-get', '--assume-yes', 'purge'] if isinstance(packages, basestring): cmd.append(packages) else: @@ -123,14 +138,16 @@ def add_source(source, key=None): if (source.startswith('ppa:') or source.startswith('http:') or source.startswith('deb ') or - source.startswith('cloud-archive:')): + source.startswith('cloud-archive:')): subprocess.check_call(['add-apt-repository', '--yes', source]) elif source.startswith('cloud:'): apt_install(filter_installed_packages(['ubuntu-cloud-keyring']), fatal=True) pocket = source.split(':')[-1] if pocket not in CLOUD_ARCHIVE_POCKETS: - raise SourceConfigError('Unsupported cloud: source option %s' % pocket) + raise SourceConfigError( + 'Unsupported cloud: source option %s' % + pocket) actual_pocket = CLOUD_ARCHIVE_POCKETS[pocket] with open('/etc/apt/sources.list.d/cloud-archive.list', 'w') as apt: apt.write(CLOUD_ARCHIVE.format(actual_pocket)) @@ -220,7 +237,9 @@ def install_from_config(config_var_name): class BaseFetchHandler(object): + """Base class for FetchHandler implementations in fetch plugins""" + def can_handle(self, source): """Returns True if the source can be handled. Otherwise returns a string explaining why it cannot""" @@ -248,10 +267,13 @@ def plugins(fetch_handlers=None): for handler_name in fetch_handlers: package, classname = handler_name.rsplit('.', 1) try: - handler_class = getattr(importlib.import_module(package), classname) + handler_class = getattr( + importlib.import_module(package), + classname) plugin_list.append(handler_class()) except (ImportError, AttributeError): # Skip missing plugins so that they can be ommitted from # installation if desired - log("FetchHandler {} not found, skipping plugin".format(handler_name)) + log("FetchHandler {} not found, skipping plugin".format( + handler_name)) return plugin_list diff --git a/revision b/revision index b661fff6..a1e0432c 100644 --- a/revision +++ b/revision @@ -1 +1 @@ -311 +312 diff --git a/templates/grizzly/nova.conf b/templates/grizzly/nova.conf new file mode 100644 index 00000000..67c63a6a --- /dev/null +++ b/templates/grizzly/nova.conf @@ -0,0 +1,114 @@ +############################################################################### +# [ WARNING ] +# Configuration file maintained by Juju. Local changes may be overwritten. +############################################################################### +[DEFAULT] +dhcpbridge_flagfile=/etc/nova/nova.conf +dhcpbridge=/usr/bin/nova-dhcpbridge +logdir=/var/log/nova +state_path=/var/lib/nova +lock_path=/var/lock/nova +force_dhcp_release=True +iscsi_helper=tgtadm +libvirt_use_virtio_for_bridges=True +connection_type=libvirt +root_helper=sudo nova-rootwrap /etc/nova/rootwrap.conf +verbose=True +ec2_private_dns_show_ip=True +api_paste_config=/etc/nova/api-paste.ini +volumes_path=/var/lib/nova/volumes +enabled_apis=ec2,osapi_compute,metadata +auth_strategy=keystone +compute_driver=libvirt.LibvirtDriver +{% if keystone_ec2_url -%} +keystone_ec2_url = {{ keystone_ec2_url }} +{% endif -%} + +{% if database_host -%} +sql_connection = mysql://{{ database_user }}:{{ database_password }}@{{ database_host }}/{{ database }} +{% endif -%} + +{% if rabbitmq_host or rabbitmq_hosts -%} +rabbit_userid = {{ rabbitmq_user }} +rabbit_password = {{ rabbitmq_password }} +rabbit_virtual_host = {{ rabbitmq_virtual_host }} +{% if rabbitmq_hosts -%} +rabbit_hosts = {{ rabbitmq_hosts }} +{% if rabbitmq_ha_queues -%} +rabbit_ha_queues = true +rabbit_durable_queues = false +{% endif %} +{% else %} +rabbit_host = {{ rabbitmq_host }} +{% endif -%} +{% endif -%} + +{% if glance_api_servers -%} +glance_api_servers = {{ glance_api_servers }} +{% endif -%} + +{% if rbd_pool -%} +rbd_pool = {{ rbd_pool }} +rbd_user = {{ rbd_user }} +rbd_secret_uuid = {{ rbd_secret_uuid }} +{% endif -%} + +{% if neutron_plugin and neutron_plugin == 'ovs' -%} +libvirt_vif_driver = nova.virt.libvirt.vif.LibvirtGenericVIFDriver +libvirt_user_virtio_for_bridges = True +{% if neutron_security_groups -%} +security_group_api = {{ network_manager }} +nova_firewall_driver = nova.virt.firewall.NoopFirewallDriver +{% endif -%} +{% if external_network -%} +default_floating_pool = {{ external_network }} +{% endif -%} +{% endif -%} + +{% if neutron_plugin and neutron_plugin == 'nvp' -%} +security_group_api = neutron +nova_firewall_driver = nova.virt.firewall.NoopFirewallDriver +{% if external_network -%} +default_floating_pool = {{ external_network }} +{% endif -%} +{% endif -%} + +{% if network_manager_config -%} +{% for key, value in network_manager_config.iteritems() -%} +{{ key }} = {{ value }} +{% endfor -%} +{% endif -%} + +{% if network_manager and network_manager == 'quantum' -%} +network_api_class = nova.network.quantumv2.api.API +{% elif network_manager and network_manager == 'neutron' -%} +network_api_class = nova.network.neutronv2.api.API +{% else -%} +network_manager = nova.network.manager.FlatDHCPManager +{% endif -%} + +{% if default_floating_pool -%} +default_floating_pool = {{ default_floating_pool }} +{% endif -%} + +{% if volume_service -%} +volume_api_class=nova.volume.cinder.API +{% endif -%} + +{% if user_config_flags -%} +{% for key, value in user_config_flags.iteritems() -%} +{{ key }} = {{ value }} +{% endfor -%} +{% endif -%} + +{% if listen_ports -%} +{% for key, value in listen_ports.iteritems() -%} +{{ key }} = {{ value }} +{% endfor -%} +{% endif -%} + +{% if sections and 'DEFAULT' in sections -%} +{% for key, value in sections['DEFAULT'] -%} +{{ key }} = {{ value }} +{% endfor -%} +{% endif -%} diff --git a/templates/grizzly/quantum.conf b/templates/grizzly/quantum.conf new file mode 100644 index 00000000..f763d38a --- /dev/null +++ b/templates/grizzly/quantum.conf @@ -0,0 +1,55 @@ +# grizzly +############################################################################### +# [ WARNING ] +# Configuration file maintained by Juju. Local changes may be overwritten. +############################################################################### +[DEFAULT] +state_path = /var/lib/quantum +lock_path = $state_path/lock +bind_host = 0.0.0.0 +{% if neutron_bind_port -%} +bind_port = {{ neutron_bind_port }} +{% else -%} +bind_port = 9696 +{% endif -%} +{% if core_plugin -%} +core_plugin = {{ core_plugin }} +{% endif -%} +api_paste_config = /etc/quantum/api-paste.ini +auth_strategy = keystone +control_exchange = quantum +notification_driver = quantum.openstack.common.notifier.rpc_notifier +default_notification_level = INFO +notification_topics = notifications +{% if rabbitmq_host or rabbitmq_hosts -%} +rabbit_userid = {{ rabbitmq_user }} +rabbit_password = {{ rabbitmq_password }} +rabbit_virtual_host = {{ rabbitmq_virtual_host }} +{% if rabbitmq_hosts -%} +rabbit_hosts = {{ rabbitmq_hosts }} +{% if rabbitmq_ha_queues -%} +rabbit_ha_queues = true +rabbit_durable_queues = false +{% endif %} +{% else %} +rabbit_host = {{ rabbitmq_host }} +{% endif -%} +{% endif -%} +{% if neutron_security_groups -%} +allow_overlapping_ips = True +{% endif -%} + + +[QUOTAS] +quota_driver = quantum.db.quota_db.DbQuotaDriver +{% if neutron_security_groups -%} +quota_items = network,subnet,port,security_group,security_group_rule +{% endif -%} + +[DEFAULT_SERVICETYPE] + +[AGENT] +root_helper = sudo quantum-rootwrap /etc/quantum/rootwrap.conf + +[keystone_authtoken] +# auth_token middleware currently set in /etc/quantum/api-paste.ini diff --git a/templates/havana/neutron.conf b/templates/havana/neutron.conf index 0bfb440d..a3d3b735 100644 --- a/templates/havana/neutron.conf +++ b/templates/havana/neutron.conf @@ -20,11 +20,19 @@ core_plugin = {{ core_plugin }} allow_overlapping_ips = True neutron_firewall_driver = neutron.agent.linux.iptables_firewall.OVSHybridIptablesFirewallDriver {% endif -%} -{% if rabbitmq_host -%} -rabbit_host = {{ rabbitmq_host }} +{% if rabbitmq_host or rabbitmq_hosts -%} rabbit_userid = {{ rabbitmq_user }} rabbit_password = {{ rabbitmq_password }} rabbit_virtual_host = {{ rabbitmq_virtual_host }} +{% if rabbitmq_hosts -%} +rabbit_hosts = {{ rabbitmq_hosts }} +{% if rabbitmq_ha_queues -%} +rabbit_ha_queues = true +rabbit_durable_queues = false +{% endif %} +{% else %} +rabbit_host = {{ rabbitmq_host }} +{% endif -%} {% endif -%} [quotas] From d34dbddedf89b78bd909242af352d68bdc583610 Mon Sep 17 00:00:00 2001 From: "yolanda.robla@canonical.com" <> Date: Thu, 30 Jan 2014 12:26:36 +0100 Subject: [PATCH 09/98] added amqp relation departed --- hooks/amqp-relation-departed | 1 + hooks/nova_cc_hooks.py | 13 +++++++++++++ 2 files changed, 14 insertions(+) create mode 120000 hooks/amqp-relation-departed diff --git a/hooks/amqp-relation-departed b/hooks/amqp-relation-departed new file mode 120000 index 00000000..f6702415 --- /dev/null +++ b/hooks/amqp-relation-departed @@ -0,0 +1 @@ +nova_cc_hooks.py \ No newline at end of file diff --git a/hooks/nova_cc_hooks.py b/hooks/nova_cc_hooks.py index 72365a3c..153d49b4 100755 --- a/hooks/nova_cc_hooks.py +++ b/hooks/nova_cc_hooks.py @@ -122,6 +122,19 @@ def amqp_changed(): CONFIGS.write(NEUTRON_CONF) +@hooks.hook('amqp-relation-departed') +@restart_on_change(restart_map()) +def amqp_departed(): + if 'amqp' not in CONFIGS.complete_contexts(): + log('amqp relation incomplete. Peer not ready?') + return + CONFIGS.write(NOVA_CONF) + if network_manager() == 'quantum': + CONFIGS.write(QUANTUM_CONF) + if network_manager() == 'neutron': + CONFIGS.write(NEUTRON_CONF) + + @hooks.hook('shared-db-relation-joined') def db_joined(): relation_set(nova_database=config('database'), From 0794973e43c41db936d096e23a6d7d22154a3a57 Mon Sep 17 00:00:00 2001 From: "yolanda.robla@canonical.com" <> Date: Fri, 31 Jan 2014 14:08:35 +0100 Subject: [PATCH 10/98] refreshed amqpcontext --- hooks/charmhelpers/contrib/openstack/context.py | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/hooks/charmhelpers/contrib/openstack/context.py b/hooks/charmhelpers/contrib/openstack/context.py index 6331a92e..a4452c7b 100644 --- a/hooks/charmhelpers/contrib/openstack/context.py +++ b/hooks/charmhelpers/contrib/openstack/context.py @@ -181,7 +181,12 @@ class AMQPContext(OSContextGenerator): # Sufficient information found = break out! break # Used for active/active rabbitmq >= grizzly - if 'clustered' not in ctxt and len(related_units(rid)) > 1: + if ('clustered' not in ctxt or relation_get('ha-vip-only') == 'True') and \ + len(related_units(rid)) > 1: + if relation_get('ha_queues'): + ctxt['rabbitmq_ha_queues'] = relation_get('ha_queues') + else: + ctxt['rabbitmq_ha_queues'] = 'False' rabbitmq_hosts = [] for unit in related_units(rid): rabbitmq_hosts.append(relation_get('private-address', From 57ca54332f220c1160720b49b60bf4aeba2a1409 Mon Sep 17 00:00:00 2001 From: Ante Karamatic Date: Thu, 13 Feb 2014 09:39:47 +0100 Subject: [PATCH 11/98] Remove duplicated functions --- hooks/nova_cc_hooks.py | 14 +------------- 1 file changed, 1 insertion(+), 13 deletions(-) diff --git a/hooks/nova_cc_hooks.py b/hooks/nova_cc_hooks.py index 153d49b4..1caf794b 100755 --- a/hooks/nova_cc_hooks.py +++ b/hooks/nova_cc_hooks.py @@ -110,21 +110,9 @@ def amqp_joined(relation_id=None): @hooks.hook('amqp-relation-changed') -@restart_on_change(restart_map()) -def amqp_changed(): - if 'amqp' not in CONFIGS.complete_contexts(): - log('amqp relation incomplete. Peer not ready?') - return - CONFIGS.write(NOVA_CONF) - if network_manager() == 'quantum': - CONFIGS.write(QUANTUM_CONF) - if network_manager() == 'neutron': - CONFIGS.write(NEUTRON_CONF) - - @hooks.hook('amqp-relation-departed') @restart_on_change(restart_map()) -def amqp_departed(): +def amqp_changed(): if 'amqp' not in CONFIGS.complete_contexts(): log('amqp relation incomplete. Peer not ready?') return From 85d999ee5ba3eeb96eba3ead259bfa06145511d7 Mon Sep 17 00:00:00 2001 From: Kapil Thangavelu Date: Fri, 21 Feb 2014 09:30:50 -0500 Subject: [PATCH 12/98] sync helpers and update nova.conf template --- charm-helpers.yaml | 2 +- .../charmhelpers/contrib/openstack/context.py | 130 ++++++++++++++---- hooks/charmhelpers/contrib/openstack/utils.py | 20 ++- .../contrib/storage/linux/utils.py | 2 +- hooks/charmhelpers/core/hookenv.py | 6 + hooks/charmhelpers/core/host.py | 44 ++++++ hooks/charmhelpers/fetch/__init__.py | 48 +++++-- hooks/nova_cc_utils.py | 9 +- revision | 2 +- templates/folsom/nova.conf | 2 +- 10 files changed, 214 insertions(+), 51 deletions(-) diff --git a/charm-helpers.yaml b/charm-helpers.yaml index 35e5834f..b53ba5be 100644 --- a/charm-helpers.yaml +++ b/charm-helpers.yaml @@ -1,4 +1,4 @@ -branch: lp:charm-helpers +branch: lp:~hazmat/charm-helpers/ssl-everywhere destination: hooks/charmhelpers include: - core diff --git a/hooks/charmhelpers/contrib/openstack/context.py b/hooks/charmhelpers/contrib/openstack/context.py index 8a982ffa..4e1b8112 100644 --- a/hooks/charmhelpers/contrib/openstack/context.py +++ b/hooks/charmhelpers/contrib/openstack/context.py @@ -1,5 +1,6 @@ import json import os +import time from base64 import b64decode @@ -23,7 +24,6 @@ from charmhelpers.core.hookenv import ( unit_get, unit_private_ip, ERROR, - WARNING, ) from charmhelpers.contrib.hahelpers.cluster import ( @@ -68,6 +68,43 @@ def context_complete(ctxt): return True +def config_flags_parser(config_flags): + if config_flags.find('==') >= 0: + log("config_flags is not in expected format (key=value)", + level=ERROR) + raise OSContextError + # strip the following from each value. + post_strippers = ' ,' + # we strip any leading/trailing '=' or ' ' from the string then + # split on '='. + split = config_flags.strip(' =').split('=') + limit = len(split) + flags = {} + for i in xrange(0, limit - 1): + current = split[i] + next = split[i + 1] + vindex = next.rfind(',') + if (i == limit - 2) or (vindex < 0): + value = next + else: + value = next[:vindex] + + if i == 0: + key = current + else: + # if this not the first entry, expect an embedded key. + index = current.rfind(',') + if index < 0: + log("invalid config value(s) at index %s" % (i), + level=ERROR) + raise OSContextError + key = current[index + 1:] + + # Add to collection. + flags[key.strip(post_strippers)] = value.rstrip(post_strippers) + return flags + + class OSContextGenerator(object): interfaces = [] @@ -78,7 +115,8 @@ class OSContextGenerator(object): class SharedDBContext(OSContextGenerator): interfaces = ['shared-db'] - def __init__(self, database=None, user=None, relation_prefix=None): + def __init__(self, + database=None, user=None, relation_prefix=None, ssl_dir=None): ''' Allows inspecting relation for settings prefixed with relation_prefix. This is useful for parsing access for multiple databases returned via @@ -87,6 +125,7 @@ class SharedDBContext(OSContextGenerator): self.relation_prefix = relation_prefix self.database = database self.user = user + self.ssl_dir = ssl_dir def __call__(self): self.database = self.database or config('database') @@ -104,15 +143,38 @@ class SharedDBContext(OSContextGenerator): for rid in relation_ids('shared-db'): for unit in related_units(rid): - passwd = relation_get(password_setting, rid=rid, unit=unit) + rdata = relation_get(rid=rid, unit=unit) ctxt = { - 'database_host': relation_get('db_host', rid=rid, - unit=unit), + 'database_host': rdata.get('db_host'), 'database': self.database, 'database_user': self.user, - 'database_password': passwd, + 'database_password': rdata.get(password_setting) } if context_complete(ctxt): + ctxt.update({'database_ssl_ca': '', 'database_ssl_key': '', + 'database_ssl_cert': ''}) + if 'ssl_ca' in rdata and self.ssl_dir: + ca_path = os.path.join(self.ssl_dir, 'db-client.ca') + with open(ca_path, 'w') as fh: + fh.write(b64decode(rdata['ssl_ca'])) + ctxt['database_ssl_ca'] = ca_path + elif 'ssl_ca' in rdata: + log("Charm not setup for ssl support but ssl ca found") + return ctxt + if 'ssl_cert' in rdata: + cert_path = os.path.join( + self.ssl_dir, 'db-client.cert') + if not os.path.exists(cert_path): + log("Waiting 1m for ssl client cert validity") + time.sleep(60) + with open(cert_path, 'w') as fh: + fh.write(b64decode(rdata['ssl_cert'])) + ctxt['database_ssl_cert'] = cert_path + + key_path = os.path.join(self.ssl_dir, 'db-client.key') + with open(key_path, 'w') as fh: + fh.write(b64decode(rdata['ssl_key'])) + ctxt['database_ssl_key'] = key_path return ctxt return {} @@ -182,10 +244,12 @@ class AMQPContext(OSContextGenerator): # Sufficient information found = break out! break # Used for active/active rabbitmq >= grizzly - ctxt['rabbitmq_hosts'] = [] - for unit in related_units(rid): - ctxt['rabbitmq_hosts'].append(relation_get('private-address', - rid=rid, unit=unit)) + if 'clustered' not in ctxt and len(related_units(rid)) > 1: + rabbitmq_hosts = [] + for unit in related_units(rid): + rabbitmq_hosts.append(relation_get('private-address', + rid=rid, unit=unit)) + ctxt['rabbitmq_hosts'] = ','.join(rabbitmq_hosts) if not context_complete(ctxt): return {} else: @@ -286,6 +350,7 @@ class ImageServiceContext(OSContextGenerator): class ApacheSSLContext(OSContextGenerator): + """ Generates a context for an apache vhost configuration that configures HTTPS reverse proxying for one or many endpoints. Generated context @@ -428,33 +493,37 @@ class NeutronContext(object): elif self.plugin == 'nvp': ctxt.update(self.nvp_ctxt()) + alchemy_flags = config('neutron-alchemy-flags') + if alchemy_flags: + flags = config_flags_parser(alchemy_flags) + ctxt['neutron_alchemy_flags'] = flags + self._save_flag_file() return ctxt class OSConfigFlagContext(OSContextGenerator): - ''' - Responsible adding user-defined config-flags in charm config to a - to a template context. - ''' + + """ + Responsible for adding user-defined config-flags in charm config to a + template context. + + NOTE: the value of config-flags may be a comma-separated list of + key=value pairs and some Openstack config files support + comma-separated lists as values. + """ + def __call__(self): config_flags = config('config-flags') - if not config_flags or config_flags in ['None', '']: + if not config_flags: return {} - config_flags = config_flags.split(',') - flags = {} - for flag in config_flags: - if '=' not in flag: - log('Improperly formatted config-flag, expected k=v ' - 'got %s' % flag, level=WARNING) - continue - k, v = flag.split('=') - flags[k.strip()] = v - ctxt = {'user_config_flags': flags} - return ctxt + + flags = config_flags_parser(config_flags) + return {'user_config_flags': flags} class SubordinateConfigContext(OSContextGenerator): + """ Responsible for inspecting relations to subordinates that may be exporting required config via a json blob. @@ -495,6 +564,7 @@ class SubordinateConfigContext(OSContextGenerator): } """ + def __init__(self, service, config_file, interface): """ :param service : Service name key to query in any subordinate @@ -539,3 +609,11 @@ class SubordinateConfigContext(OSContextGenerator): ctxt['sections'] = {} return ctxt + + +class SyslogContext(OSContextGenerator): + def __call__(self): + ctxt = { + 'use_syslog': config('use-syslog') + } + return ctxt diff --git a/hooks/charmhelpers/contrib/openstack/utils.py b/hooks/charmhelpers/contrib/openstack/utils.py index d66afd74..56d04245 100644 --- a/hooks/charmhelpers/contrib/openstack/utils.py +++ b/hooks/charmhelpers/contrib/openstack/utils.py @@ -41,6 +41,7 @@ UBUNTU_OPENSTACK_RELEASE = OrderedDict([ ('quantal', 'folsom'), ('raring', 'grizzly'), ('saucy', 'havana'), + ('trusty', 'icehouse') ]) @@ -201,7 +202,7 @@ def os_release(package, base='essex'): def import_key(keyid): - cmd = "apt-key adv --keyserver keyserver.ubuntu.com " \ + cmd = "apt-key adv --keyserver hkp://keyserver.ubuntu.com:80 " \ "--recv-keys %s" % keyid try: subprocess.check_call(cmd.split(' ')) @@ -260,6 +261,9 @@ def configure_installation_source(rel): 'havana': 'precise-updates/havana', 'havana/updates': 'precise-updates/havana', 'havana/proposed': 'precise-proposed/havana', + 'icehouse': 'precise-updates/icehouse', + 'icehouse/updates': 'precise-updates/icehouse', + 'icehouse/proposed': 'precise-proposed/icehouse', } try: @@ -411,7 +415,7 @@ def get_host_ip(hostname): return ns_query(hostname) -def get_hostname(address): +def get_hostname(address, fqdn=True): """ Resolves hostname for given IP, or returns the input if it is already a hostname. @@ -430,7 +434,11 @@ def get_hostname(address): if not result: return None - # strip trailing . - if result.endswith('.'): - return result[:-1] - return result + if fqdn: + # strip trailing . + if result.endswith('.'): + return result[:-1] + else: + return result + else: + return result.split('.')[0] diff --git a/hooks/charmhelpers/contrib/storage/linux/utils.py b/hooks/charmhelpers/contrib/storage/linux/utils.py index 5b9b6d47..c40218f0 100644 --- a/hooks/charmhelpers/contrib/storage/linux/utils.py +++ b/hooks/charmhelpers/contrib/storage/linux/utils.py @@ -22,4 +22,4 @@ def zap_disk(block_device): :param block_device: str: Full path of block device to clean. ''' - check_call(['sgdisk', '--zap-all', block_device]) + check_call(['sgdisk', '--zap-all', '--mbrtogpt', block_device]) diff --git a/hooks/charmhelpers/core/hookenv.py b/hooks/charmhelpers/core/hookenv.py index bb196dfa..505c202d 100644 --- a/hooks/charmhelpers/core/hookenv.py +++ b/hooks/charmhelpers/core/hookenv.py @@ -8,6 +8,7 @@ import os import json import yaml import subprocess +import sys import UserDict from subprocess import CalledProcessError @@ -149,6 +150,11 @@ def service_name(): return local_unit().split('/')[0] +def hook_name(): + """The name of the currently executing hook""" + return os.path.basename(sys.argv[0]) + + @cached def config(scope=None): """Juju charm configuration""" diff --git a/hooks/charmhelpers/core/host.py b/hooks/charmhelpers/core/host.py index 4a6a4a8c..c8c81b28 100644 --- a/hooks/charmhelpers/core/host.py +++ b/hooks/charmhelpers/core/host.py @@ -245,3 +245,47 @@ def pwgen(length=None): random_chars = [ random.choice(alphanumeric_chars) for _ in range(length)] return(''.join(random_chars)) + + +def list_nics(nic_type): + '''Return a list of nics of given type(s)''' + if isinstance(nic_type, basestring): + int_types = [nic_type] + else: + int_types = nic_type + interfaces = [] + for int_type in int_types: + cmd = ['ip', 'addr', 'show', 'label', int_type + '*'] + ip_output = subprocess.check_output(cmd).split('\n') + ip_output = (line for line in ip_output if line) + for line in ip_output: + if line.split()[1].startswith(int_type): + interfaces.append(line.split()[1].replace(":", "")) + return interfaces + + +def set_nic_mtu(nic, mtu): + '''Set MTU on a network interface''' + cmd = ['ip', 'link', 'set', nic, 'mtu', mtu] + subprocess.check_call(cmd) + + +def get_nic_mtu(nic): + cmd = ['ip', 'addr', 'show', nic] + ip_output = subprocess.check_output(cmd).split('\n') + mtu = "" + for line in ip_output: + words = line.split() + if 'mtu' in words: + mtu = words[words.index("mtu") + 1] + return mtu + + +def get_nic_hwaddr(nic): + cmd = ['ip', '-o', '-0', 'addr', 'show', nic] + ip_output = subprocess.check_output(cmd) + hwaddr = "" + words = ip_output.split() + if 'link/ether' in words: + hwaddr = words[words.index('link/ether') + 1] + return hwaddr diff --git a/hooks/charmhelpers/fetch/__init__.py b/hooks/charmhelpers/fetch/__init__.py index fa0172a9..07bb707d 100644 --- a/hooks/charmhelpers/fetch/__init__.py +++ b/hooks/charmhelpers/fetch/__init__.py @@ -13,6 +13,7 @@ from charmhelpers.core.hookenv import ( log, ) import apt_pkg +import os CLOUD_ARCHIVE = """# Ubuntu Cloud Archive deb http://ubuntu-cloud.archive.canonical.com/ubuntu {} main @@ -43,8 +44,16 @@ CLOUD_ARCHIVE_POCKETS = { 'precise-havana/updates': 'precise-updates/havana', 'precise-updates/havana': 'precise-updates/havana', 'havana/proposed': 'precise-proposed/havana', - 'precies-havana/proposed': 'precise-proposed/havana', + 'precise-havana/proposed': 'precise-proposed/havana', 'precise-proposed/havana': 'precise-proposed/havana', + # Icehouse + 'icehouse': 'precise-updates/icehouse', + 'precise-icehouse': 'precise-updates/icehouse', + 'precise-icehouse/updates': 'precise-updates/icehouse', + 'precise-updates/icehouse': 'precise-updates/icehouse', + 'icehouse/proposed': 'precise-proposed/icehouse', + 'precise-icehouse/proposed': 'precise-proposed/icehouse', + 'precise-proposed/icehouse': 'precise-proposed/icehouse', } @@ -66,8 +75,10 @@ def filter_installed_packages(packages): def apt_install(packages, options=None, fatal=False): """Install one or more packages""" - options = options or [] - cmd = ['apt-get', '-y'] + if options is None: + options = ['--option=Dpkg::Options::=--force-confold'] + + cmd = ['apt-get', '--assume-yes'] cmd.extend(options) cmd.append('install') if isinstance(packages, basestring): @@ -76,10 +87,14 @@ def apt_install(packages, options=None, fatal=False): cmd.extend(packages) log("Installing {} with options: {}".format(packages, options)) + env = os.environ.copy() + if 'DEBIAN_FRONTEND' not in env: + env['DEBIAN_FRONTEND'] = 'noninteractive' + if fatal: - subprocess.check_call(cmd) + subprocess.check_call(cmd, env=env) else: - subprocess.call(cmd) + subprocess.call(cmd, env=env) def apt_update(fatal=False): @@ -93,7 +108,7 @@ def apt_update(fatal=False): def apt_purge(packages, fatal=False): """Purge one or more packages""" - cmd = ['apt-get', '-y', 'purge'] + cmd = ['apt-get', '--assume-yes', 'purge'] if isinstance(packages, basestring): cmd.append(packages) else: @@ -121,16 +136,18 @@ def apt_hold(packages, fatal=False): def add_source(source, key=None): if (source.startswith('ppa:') or - source.startswith('http:') or + source.startswith('http') or source.startswith('deb ') or - source.startswith('cloud-archive:')): + source.startswith('cloud-archive:')): subprocess.check_call(['add-apt-repository', '--yes', source]) elif source.startswith('cloud:'): apt_install(filter_installed_packages(['ubuntu-cloud-keyring']), fatal=True) pocket = source.split(':')[-1] if pocket not in CLOUD_ARCHIVE_POCKETS: - raise SourceConfigError('Unsupported cloud: source option %s' % pocket) + raise SourceConfigError( + 'Unsupported cloud: source option %s' % + pocket) actual_pocket = CLOUD_ARCHIVE_POCKETS[pocket] with open('/etc/apt/sources.list.d/cloud-archive.list', 'w') as apt: apt.write(CLOUD_ARCHIVE.format(actual_pocket)) @@ -139,7 +156,9 @@ def add_source(source, key=None): with open('/etc/apt/sources.list.d/proposed.list', 'w') as apt: apt.write(PROPOSED_POCKET.format(release)) if key: - subprocess.check_call(['apt-key', 'import', key]) + subprocess.check_call(['apt-key', 'adv', '--keyserver', + 'keyserver.ubuntu.com', '--recv', + key]) class SourceConfigError(Exception): @@ -220,7 +239,9 @@ def install_from_config(config_var_name): class BaseFetchHandler(object): + """Base class for FetchHandler implementations in fetch plugins""" + def can_handle(self, source): """Returns True if the source can be handled. Otherwise returns a string explaining why it cannot""" @@ -248,10 +269,13 @@ def plugins(fetch_handlers=None): for handler_name in fetch_handlers: package, classname = handler_name.rsplit('.', 1) try: - handler_class = getattr(importlib.import_module(package), classname) + handler_class = getattr( + importlib.import_module(package), + classname) plugin_list.append(handler_class()) except (ImportError, AttributeError): # Skip missing plugins so that they can be ommitted from # installation if desired - log("FetchHandler {} not found, skipping plugin".format(handler_name)) + log("FetchHandler {} not found, skipping plugin".format( + handler_name)) return plugin_list diff --git a/hooks/nova_cc_utils.py b/hooks/nova_cc_utils.py index a305cfec..25bc7020 100644 --- a/hooks/nova_cc_utils.py +++ b/hooks/nova_cc_utils.py @@ -69,8 +69,10 @@ API_PORTS = { 'quantum-server': 9696, } -NOVA_CONF = '/etc/nova/nova.conf' -NOVA_API_PASTE = '/etc/nova/api-paste.ini' +NOVA_CONF_DIR = "/etc/nova" + +NOVA_CONF = '%s/nova.conf' % NOVA_CONF_DIR +NOVA_API_PASTE = '%s/api-paste.ini' % NOVA_CONF_DIR QUANTUM_CONF = '/etc/quantum/quantum.conf' QUANTUM_API_PASTE = '/etc/quantum/api-paste.ini' NEUTRON_CONF = '/etc/neutron/neutron.conf' @@ -84,7 +86,8 @@ BASE_RESOURCE_MAP = OrderedDict([ (NOVA_CONF, { 'services': BASE_SERVICES, 'contexts': [context.AMQPContext(), - context.SharedDBContext(relation_prefix='nova'), + context.SharedDBContext( + relation_prefix='nova', ssl_dir=NOVA_CONF_DIR), context.ImageServiceContext(), context.OSConfigFlagContext(), context.SubordinateConfigContext( diff --git a/revision b/revision index b661fff6..a1e0432c 100644 --- a/revision +++ b/revision @@ -1 +1 @@ -311 +312 diff --git a/templates/folsom/nova.conf b/templates/folsom/nova.conf index 5a6576ed..b0fb911f 100644 --- a/templates/folsom/nova.conf +++ b/templates/folsom/nova.conf @@ -25,7 +25,7 @@ keystone_ec2_url = {{ keystone_ec2_url }} {% endif -%} {% if database_host -%} -sql_connection = mysql://{{ database_user }}:{{ database_password }}@{{ database_host }}/{{ database }} +sql_connection = mysql://{{ database_user }}:{{ database_password }}@{{ database_host }}/{{ database }}{% if database_ssl_ca %}?ssl_ca={{ database_ssl_ca }}{% if database_ssl_cert %}&ssl_cert={{ database_ssl_cert }}&ssl_key={{ database_ssl_key }}{% endif %}{% endif %} {% endif -%} {% if rabbitmq_host -%} From 4f7a0d4ba01fb9543f1d59f41466533b5e24add6 Mon Sep 17 00:00:00 2001 From: Kapil Thangavelu Date: Sun, 23 Feb 2014 17:42:40 -0500 Subject: [PATCH 13/98] amqp ssl support --- .../charmhelpers/contrib/openstack/context.py | 23 ++++++++++++++++++- hooks/nova_cc_utils.py | 14 ++++++----- templates/folsom/nova.conf | 7 ++++++ templates/havana/neutron.conf | 7 ++++++ 4 files changed, 44 insertions(+), 7 deletions(-) diff --git a/hooks/charmhelpers/contrib/openstack/context.py b/hooks/charmhelpers/contrib/openstack/context.py index 4e1b8112..f1df9d8d 100644 --- a/hooks/charmhelpers/contrib/openstack/context.py +++ b/hooks/charmhelpers/contrib/openstack/context.py @@ -213,6 +213,9 @@ class IdentityServiceContext(OSContextGenerator): class AMQPContext(OSContextGenerator): interfaces = ['amqp'] + def __init__(self, ssl_dir=None): + self.ssl_dir = ssl_dir + def __call__(self): log('Generating template context for amqp') conf = config() @@ -223,7 +226,6 @@ class AMQPContext(OSContextGenerator): log('Could not generate shared_db context. ' 'Missing required charm config options: %s.' % e) raise OSContextError - ctxt = {} for rid in relation_ids('amqp'): for unit in related_units(rid): @@ -240,7 +242,24 @@ class AMQPContext(OSContextGenerator): unit=unit), 'rabbitmq_virtual_host': vhost, }) + ssl_port = relation_get('ssl_port', rid=rid, unit=unit) + if ssl_port: + ctxt['rabbit_ssl_port'] = ssl_port + ssl_ca = relation_get('ssl_ca', rid=rid, unit=unit) + if ssl_ca: + ctxt['rabbit_ssl_ca'] = ssl_ca + if context_complete(ctxt): + if 'rabbit_ssl_ca' in ctxt: + if not self.ssl_dir: + log(("Charm not setup for ssl support " + "but ssl ca found")) + break + ca_path = os.path.join( + self.ssl_dir, 'rabbit-client-ca.pem') + with open(ca_path, 'w') as fh: + fh.write(b64decode(ctxt['rabbit_ssl_ca'])) + ctxt['rabbit_ssl_ca'] = ca_path # Sufficient information found = break out! break # Used for active/active rabbitmq >= grizzly @@ -253,6 +272,8 @@ class AMQPContext(OSContextGenerator): if not context_complete(ctxt): return {} else: + ctxt.setdefault('rabbit_ssl_port', '') + ctxt.setdefault('rabbit_ssl_ca', '') return ctxt diff --git a/hooks/nova_cc_utils.py b/hooks/nova_cc_utils.py index 25bc7020..d99b8879 100644 --- a/hooks/nova_cc_utils.py +++ b/hooks/nova_cc_utils.py @@ -70,12 +70,14 @@ API_PORTS = { } NOVA_CONF_DIR = "/etc/nova" +QUANTUM_CONF_DIR = "/etc/quantum" +NEUTRON_CONF_DIR = "/etc/neutron" NOVA_CONF = '%s/nova.conf' % NOVA_CONF_DIR NOVA_API_PASTE = '%s/api-paste.ini' % NOVA_CONF_DIR -QUANTUM_CONF = '/etc/quantum/quantum.conf' -QUANTUM_API_PASTE = '/etc/quantum/api-paste.ini' -NEUTRON_CONF = '/etc/neutron/neutron.conf' +QUANTUM_CONF = '%s/quantum.conf' % QUANTUM_CONF_DIR +QUANTUM_API_PASTE = '%s/api-paste.ini' % QUANTUM_CONF_DIR +NEUTRON_CONF = '%s/neutron.conf' % NEUTRON_CONF_DIR HAPROXY_CONF = '/etc/haproxy/haproxy.cfg' APACHE_CONF = '/etc/apache2/sites-available/openstack_https_frontend' APACHE_24_CONF = '/etc/apache2/sites-available/openstack_https_frontend.conf' @@ -85,7 +87,7 @@ QUANTUM_DEFAULT = '/etc/default/quantum-server' BASE_RESOURCE_MAP = OrderedDict([ (NOVA_CONF, { 'services': BASE_SERVICES, - 'contexts': [context.AMQPContext(), + 'contexts': [context.AMQPContext(ssl_dir=NOVA_CONF_DIR), context.SharedDBContext( relation_prefix='nova', ssl_dir=NOVA_CONF_DIR), context.ImageServiceContext(), @@ -106,7 +108,7 @@ BASE_RESOURCE_MAP = OrderedDict([ }), (QUANTUM_CONF, { 'services': ['quantum-server'], - 'contexts': [context.AMQPContext(), + 'contexts': [context.AMQPContext(ssl_dir=QUANTUM_CONF_DIR), nova_cc_context.HAProxyContext(), nova_cc_context.IdentityServiceContext(), nova_cc_context.NeutronCCContext()], @@ -121,7 +123,7 @@ BASE_RESOURCE_MAP = OrderedDict([ }), (NEUTRON_CONF, { 'services': ['neutron-server'], - 'contexts': [context.AMQPContext(), + 'contexts': [context.AMQPContext(ssl_dir=NEUTRON_CONF_DIR), nova_cc_context.IdentityServiceContext(), nova_cc_context.NeutronCCContext(), nova_cc_context.HAProxyContext()], diff --git a/templates/folsom/nova.conf b/templates/folsom/nova.conf index b0fb911f..79e607e5 100644 --- a/templates/folsom/nova.conf +++ b/templates/folsom/nova.conf @@ -29,6 +29,13 @@ sql_connection = mysql://{{ database_user }}:{{ database_password }}@{{ database {% endif -%} {% if rabbitmq_host -%} +{% if rabbit_ssl_port %} +rabbit_use_ssl=True +rabbit_port={{ rabbit_ssl_port }} +{% if rabbit_ssl_ca %} +kombu_ssl_ca_certs={{rabbit_ssl_ca}} +{% endif %} +{% endif %} rabbit_host = {{ rabbitmq_host }} rabbit_userid = {{ rabbitmq_user }} rabbit_password = {{ rabbitmq_password }} diff --git a/templates/havana/neutron.conf b/templates/havana/neutron.conf index 0bfb440d..4ea89417 100644 --- a/templates/havana/neutron.conf +++ b/templates/havana/neutron.conf @@ -21,6 +21,13 @@ allow_overlapping_ips = True neutron_firewall_driver = neutron.agent.linux.iptables_firewall.OVSHybridIptablesFirewallDriver {% endif -%} {% if rabbitmq_host -%} +{% if rabbit_ssl_port %} +rabbit_use_ssl=True +rabbit_port={{ rabbit_ssl_port }} +{% if rabbit_ssl_ca %} +kombu_ssl_ca_certs={{rabbit_ssl_ca}} +{% endif %} +{% endif %} rabbit_host = {{ rabbitmq_host }} rabbit_userid = {{ rabbitmq_user }} rabbit_password = {{ rabbitmq_password }} From e17e55d1381f6b41a0f41ec1600b9303803aca40 Mon Sep 17 00:00:00 2001 From: Kapil Thangavelu Date: Mon, 24 Feb 2014 06:49:04 -0500 Subject: [PATCH 14/98] sync charm-helpers --- .../charmhelpers/contrib/openstack/context.py | 52 ++++++++++--------- 1 file changed, 28 insertions(+), 24 deletions(-) diff --git a/hooks/charmhelpers/contrib/openstack/context.py b/hooks/charmhelpers/contrib/openstack/context.py index f1df9d8d..1954e933 100644 --- a/hooks/charmhelpers/contrib/openstack/context.py +++ b/hooks/charmhelpers/contrib/openstack/context.py @@ -151,34 +151,38 @@ class SharedDBContext(OSContextGenerator): 'database_password': rdata.get(password_setting) } if context_complete(ctxt): - ctxt.update({'database_ssl_ca': '', 'database_ssl_key': '', - 'database_ssl_cert': ''}) - if 'ssl_ca' in rdata and self.ssl_dir: - ca_path = os.path.join(self.ssl_dir, 'db-client.ca') - with open(ca_path, 'w') as fh: - fh.write(b64decode(rdata['ssl_ca'])) - ctxt['database_ssl_ca'] = ca_path - elif 'ssl_ca' in rdata: - log("Charm not setup for ssl support but ssl ca found") - return ctxt - if 'ssl_cert' in rdata: - cert_path = os.path.join( - self.ssl_dir, 'db-client.cert') - if not os.path.exists(cert_path): - log("Waiting 1m for ssl client cert validity") - time.sleep(60) - with open(cert_path, 'w') as fh: - fh.write(b64decode(rdata['ssl_cert'])) - ctxt['database_ssl_cert'] = cert_path - - key_path = os.path.join(self.ssl_dir, 'db-client.key') - with open(key_path, 'w') as fh: - fh.write(b64decode(rdata['ssl_key'])) - ctxt['database_ssl_key'] = key_path + db_ssl(rdata, ctxt, self.ssl_dir) return ctxt return {} +def db_ssl(rdata, ctxt, ssl_dir, log): + ctxt.update({'database_ssl_ca': '', 'database_ssl_key': '', + 'database_ssl_cert': ''}) + if 'ssl_ca' in rdata and ssl_dir: + ca_path = os.path.join(ssl_dir, 'db-client.ca') + with open(ca_path, 'w') as fh: + fh.write(b64decode(rdata['ssl_ca'])) + ctxt['database_ssl_ca'] = ca_path + elif 'ssl_ca' in rdata: + log("Charm not setup for ssl support but ssl ca found") + return ctxt + if 'ssl_cert' in rdata: + cert_path = os.path.join( + ssl_dir, 'db-client.cert') + if not os.path.exists(cert_path): + log("Waiting 1m for ssl client cert validity") + time.sleep(60) + with open(cert_path, 'w') as fh: + fh.write(b64decode(rdata['ssl_cert'])) + ctxt['database_ssl_cert'] = cert_path + key_path = os.path.join(ssl_dir, 'db-client.key') + with open(key_path, 'w') as fh: + fh.write(b64decode(rdata['ssl_key'])) + ctxt['database_ssl_key'] = key_path + return ctxt + + class IdentityServiceContext(OSContextGenerator): interfaces = ['identity-service'] From 4dec32838bf30284f4c908f5478337b0e1ae2470 Mon Sep 17 00:00:00 2001 From: Kapil Thangavelu Date: Mon, 24 Feb 2014 08:46:47 -0500 Subject: [PATCH 15/98] sync charm helpers to correct db_ssl err --- hooks/charmhelpers/contrib/openstack/context.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/hooks/charmhelpers/contrib/openstack/context.py b/hooks/charmhelpers/contrib/openstack/context.py index 1954e933..b4583b19 100644 --- a/hooks/charmhelpers/contrib/openstack/context.py +++ b/hooks/charmhelpers/contrib/openstack/context.py @@ -156,7 +156,7 @@ class SharedDBContext(OSContextGenerator): return {} -def db_ssl(rdata, ctxt, ssl_dir, log): +def db_ssl(rdata, ctxt, ssl_dir): ctxt.update({'database_ssl_ca': '', 'database_ssl_key': '', 'database_ssl_cert': ''}) if 'ssl_ca' in rdata and ssl_dir: From 8fc48bffb1de78929a986a3f971c0768dff17ebe Mon Sep 17 00:00:00 2001 From: Kapil Thangavelu Date: Mon, 24 Feb 2014 14:16:35 -0500 Subject: [PATCH 16/98] sync helpers --- .../charmhelpers/contrib/openstack/context.py | 65 ++++++++++++------- .../contrib/openstack/templates/ceph.conf | 3 + .../contrib/openstack/templates/haproxy.cfg | 5 +- .../contrib/openstack/templating.py | 4 ++ .../contrib/storage/linux/ceph.py | 8 ++- hooks/charmhelpers/core/host.py | 12 +++- hooks/nova_cc_utils.py | 4 +- 7 files changed, 68 insertions(+), 33 deletions(-) diff --git a/hooks/charmhelpers/contrib/openstack/context.py b/hooks/charmhelpers/contrib/openstack/context.py index b4583b19..f78887aa 100644 --- a/hooks/charmhelpers/contrib/openstack/context.py +++ b/hooks/charmhelpers/contrib/openstack/context.py @@ -27,11 +27,9 @@ from charmhelpers.core.hookenv import ( ) from charmhelpers.contrib.hahelpers.cluster import ( + determine_apache_port, determine_api_port, - determine_haproxy_port, https, - is_clustered, - peer_units, ) from charmhelpers.contrib.hahelpers.apache import ( @@ -111,6 +109,15 @@ class OSContextGenerator(object): def __call__(self): raise NotImplementedError + def post_execute(self, ctxt_data): + """Called after all contexts for a config template have been invoked. + + Only invoked if the context returned data when invoked and + receives the final context data. + + Used to work around dependency ordering issues and multiple contexts. + """ + class SharedDBContext(OSContextGenerator): interfaces = ['shared-db'] @@ -157,8 +164,6 @@ class SharedDBContext(OSContextGenerator): def db_ssl(rdata, ctxt, ssl_dir): - ctxt.update({'database_ssl_ca': '', 'database_ssl_key': '', - 'database_ssl_cert': ''}) if 'ssl_ca' in rdata and ssl_dir: ca_path = os.path.join(ssl_dir, 'db-client.ca') with open(ca_path, 'w') as fh: @@ -254,20 +259,16 @@ class AMQPContext(OSContextGenerator): ctxt['rabbit_ssl_ca'] = ssl_ca if context_complete(ctxt): - if 'rabbit_ssl_ca' in ctxt: - if not self.ssl_dir: - log(("Charm not setup for ssl support " - "but ssl ca found")) - break - ca_path = os.path.join( - self.ssl_dir, 'rabbit-client-ca.pem') - with open(ca_path, 'w') as fh: - fh.write(b64decode(ctxt['rabbit_ssl_ca'])) - ctxt['rabbit_ssl_ca'] = ca_path # Sufficient information found = break out! break # Used for active/active rabbitmq >= grizzly - if 'clustered' not in ctxt and len(related_units(rid)) > 1: + if (('clustered' not in ctxt or + relation_get('ha-vip-only') == 'True') and + len(related_units(rid)) > 1): + if relation_get('ha_queues'): + ctxt['rabbitmq_ha_queues'] = relation_get('ha_queues') + else: + ctxt['rabbitmq_ha_queues'] = False rabbitmq_hosts = [] for unit in related_units(rid): rabbitmq_hosts.append(relation_get('private-address', @@ -276,10 +277,27 @@ class AMQPContext(OSContextGenerator): if not context_complete(ctxt): return {} else: - ctxt.setdefault('rabbit_ssl_port', '') - ctxt.setdefault('rabbit_ssl_ca', '') return ctxt + def post_execute(self, ctxt): + """ + AMQP is sometimes called as part of a list of contexts where the later + contexts perform package installation that install parent directories + for ssl certs. We delay writing out certs till those directories + are present but before the config file is written. + """ + if not 'rabbit_ssl_ca' in ctxt: + return + + if not self.ssl_dir: + log("Charm not setup for ssl support but ssl ca found") + return + + ca_path = os.path.join(self.ssl_dir, 'rabbit-client-ca.pem') + with open(ca_path, 'w') as fh: + fh.write(b64decode(ctxt['rabbit_ssl_ca'])) + ctxt['rabbit_ssl_ca'] = ca_path + class CephContext(OSContextGenerator): interfaces = ['ceph'] @@ -298,11 +316,13 @@ class CephContext(OSContextGenerator): unit=unit)) auth = relation_get('auth', rid=rid, unit=unit) key = relation_get('key', rid=rid, unit=unit) + use_syslog = str(config('use-syslog')).lower() ctxt = { 'mon_hosts': ' '.join(mon_hosts), 'auth': auth, 'key': key, + 'use_syslog': use_syslog } if not os.path.isdir('/etc/ceph'): @@ -431,11 +451,9 @@ class ApacheSSLContext(OSContextGenerator): 'private_address': unit_get('private-address'), 'endpoints': [] } - for ext_port in self.external_ports: - if peer_units() or is_clustered(): - int_port = determine_haproxy_port(ext_port) - else: - int_port = determine_api_port(ext_port) + for api_port in self.external_ports: + ext_port = determine_apache_port(api_port) + int_port = determine_api_port(api_port) portmap = (int(ext_port), int(int_port)) ctxt['endpoints'].append(portmap) return ctxt @@ -637,6 +655,7 @@ class SubordinateConfigContext(OSContextGenerator): class SyslogContext(OSContextGenerator): + def __call__(self): ctxt = { 'use_syslog': config('use-syslog') diff --git a/hooks/charmhelpers/contrib/openstack/templates/ceph.conf b/hooks/charmhelpers/contrib/openstack/templates/ceph.conf index 49d07c80..874ad2bd 100644 --- a/hooks/charmhelpers/contrib/openstack/templates/ceph.conf +++ b/hooks/charmhelpers/contrib/openstack/templates/ceph.conf @@ -9,3 +9,6 @@ keyring = /etc/ceph/$cluster.$name.keyring mon host = {{ mon_hosts }} {% endif -%} +log to syslog = {{ use_syslog }} +err to syslog = {{ use_syslog }} +clog to syslog = {{ use_syslog }} diff --git a/hooks/charmhelpers/contrib/openstack/templates/haproxy.cfg b/hooks/charmhelpers/contrib/openstack/templates/haproxy.cfg index a1694e44..56ed913e 100644 --- a/hooks/charmhelpers/contrib/openstack/templates/haproxy.cfg +++ b/hooks/charmhelpers/contrib/openstack/templates/haproxy.cfg @@ -8,8 +8,8 @@ global defaults log global - mode http - option httplog + mode tcp + option tcplog option dontlognull retries 3 timeout queue 1000 @@ -29,7 +29,6 @@ listen stats :8888 {% for service, ports in service_ports.iteritems() -%} listen {{ service }} 0.0.0.0:{{ ports[0] }} balance roundrobin - option tcplog {% for unit, address in units.iteritems() -%} server {{ unit }} {{ address }}:{{ ports[1] }} check {% endfor %} diff --git a/hooks/charmhelpers/contrib/openstack/templating.py b/hooks/charmhelpers/contrib/openstack/templating.py index 4595778c..0718a6eb 100644 --- a/hooks/charmhelpers/contrib/openstack/templating.py +++ b/hooks/charmhelpers/contrib/openstack/templating.py @@ -84,14 +84,18 @@ class OSConfigTemplate(object): def context(self): ctxt = {} + data_contexts = [] for context in self.contexts: _ctxt = context() if _ctxt: + data_contexts.append(context) ctxt.update(_ctxt) # track interfaces for every complete context. [self._complete_contexts.append(interface) for interface in context.interfaces if interface not in self._complete_contexts] + for context in data_contexts: + context.post_execute(ctxt) return ctxt def complete_contexts(self): diff --git a/hooks/charmhelpers/contrib/storage/linux/ceph.py b/hooks/charmhelpers/contrib/storage/linux/ceph.py index 69b879ca..12417410 100644 --- a/hooks/charmhelpers/contrib/storage/linux/ceph.py +++ b/hooks/charmhelpers/contrib/storage/linux/ceph.py @@ -49,6 +49,9 @@ CEPH_CONF = """[global] auth supported = {auth} keyring = {keyring} mon host = {mon_hosts} + log to syslog = {use_syslog} + err to syslog = {use_syslog} + clog to syslog = {use_syslog} """ @@ -194,7 +197,7 @@ def get_ceph_nodes(): return hosts -def configure(service, key, auth): +def configure(service, key, auth, use_syslog): ''' Perform basic configuration of Ceph ''' create_keyring(service, key) create_key_file(service, key) @@ -202,7 +205,8 @@ def configure(service, key, auth): with open('/etc/ceph/ceph.conf', 'w') as ceph_conf: ceph_conf.write(CEPH_CONF.format(auth=auth, keyring=_keyring_path(service), - mon_hosts=",".join(map(str, hosts)))) + mon_hosts=",".join(map(str, hosts)), + use_syslog=use_syslog)) modprobe('rbd') diff --git a/hooks/charmhelpers/core/host.py b/hooks/charmhelpers/core/host.py index c8c81b28..cfd26847 100644 --- a/hooks/charmhelpers/core/host.py +++ b/hooks/charmhelpers/core/host.py @@ -194,7 +194,7 @@ def file_hash(path): return None -def restart_on_change(restart_map): +def restart_on_change(restart_map, stopstart=False): """Restart services based on configuration files changing This function is used a decorator, for example @@ -219,8 +219,14 @@ def restart_on_change(restart_map): for path in restart_map: if checksums[path] != file_hash(path): restarts += restart_map[path] - for service_name in list(OrderedDict.fromkeys(restarts)): - service('restart', service_name) + services_list = list(OrderedDict.fromkeys(restarts)) + if not stopstart: + for service_name in services_list: + service('restart', service_name) + else: + for action in ['stop', 'start']: + for service_name in services_list: + service(action, service_name) return wrapped_f return wrap diff --git a/hooks/nova_cc_utils.py b/hooks/nova_cc_utils.py index d99b8879..14a416bf 100644 --- a/hooks/nova_cc_utils.py +++ b/hooks/nova_cc_utils.py @@ -317,10 +317,10 @@ def migrate_database(): def auth_token_config(setting): - ''' + """ Returns currently configured value for setting in api-paste.ini's authtoken section, or None. - ''' + """ config = ConfigParser.RawConfigParser() config.read('/etc/nova/api-paste.ini') try: From 2751ae3bbde03ca5c4b757c488347be83c2ffcfd Mon Sep 17 00:00:00 2001 From: Kapil Thangavelu Date: Mon, 24 Feb 2014 14:31:57 -0500 Subject: [PATCH 17/98] sync helpers --- .../charmhelpers/contrib/openstack/context.py | 43 ++++++------------- .../contrib/openstack/templating.py | 4 -- 2 files changed, 12 insertions(+), 35 deletions(-) diff --git a/hooks/charmhelpers/contrib/openstack/context.py b/hooks/charmhelpers/contrib/openstack/context.py index f78887aa..38e4eac7 100644 --- a/hooks/charmhelpers/contrib/openstack/context.py +++ b/hooks/charmhelpers/contrib/openstack/context.py @@ -109,15 +109,6 @@ class OSContextGenerator(object): def __call__(self): raise NotImplementedError - def post_execute(self, ctxt_data): - """Called after all contexts for a config template have been invoked. - - Only invoked if the context returned data when invoked and - receives the final context data. - - Used to work around dependency ordering issues and multiple contexts. - """ - class SharedDBContext(OSContextGenerator): interfaces = ['shared-db'] @@ -259,12 +250,21 @@ class AMQPContext(OSContextGenerator): ctxt['rabbit_ssl_ca'] = ssl_ca if context_complete(ctxt): + if 'rabbit_ssl_ca' in ctxt: + if not self.ssl_dir: + log(("Charm not setup for ssl support " + "but ssl ca found")) + break + ca_path = os.path.join( + self.ssl_dir, 'rabbit-client-ca.pem') + with open(ca_path, 'w') as fh: + fh.write(b64decode(ctxt['rabbit_ssl_ca'])) + ctxt['rabbit_ssl_ca'] = ca_path # Sufficient information found = break out! break # Used for active/active rabbitmq >= grizzly - if (('clustered' not in ctxt or - relation_get('ha-vip-only') == 'True') and - len(related_units(rid)) > 1): + if ('clustered' not in ctxt or relation_get('ha-vip-only') == 'True') and \ + len(related_units(rid)) > 1: if relation_get('ha_queues'): ctxt['rabbitmq_ha_queues'] = relation_get('ha_queues') else: @@ -279,25 +279,6 @@ class AMQPContext(OSContextGenerator): else: return ctxt - def post_execute(self, ctxt): - """ - AMQP is sometimes called as part of a list of contexts where the later - contexts perform package installation that install parent directories - for ssl certs. We delay writing out certs till those directories - are present but before the config file is written. - """ - if not 'rabbit_ssl_ca' in ctxt: - return - - if not self.ssl_dir: - log("Charm not setup for ssl support but ssl ca found") - return - - ca_path = os.path.join(self.ssl_dir, 'rabbit-client-ca.pem') - with open(ca_path, 'w') as fh: - fh.write(b64decode(ctxt['rabbit_ssl_ca'])) - ctxt['rabbit_ssl_ca'] = ca_path - class CephContext(OSContextGenerator): interfaces = ['ceph'] diff --git a/hooks/charmhelpers/contrib/openstack/templating.py b/hooks/charmhelpers/contrib/openstack/templating.py index 0718a6eb..4595778c 100644 --- a/hooks/charmhelpers/contrib/openstack/templating.py +++ b/hooks/charmhelpers/contrib/openstack/templating.py @@ -84,18 +84,14 @@ class OSConfigTemplate(object): def context(self): ctxt = {} - data_contexts = [] for context in self.contexts: _ctxt = context() if _ctxt: - data_contexts.append(context) ctxt.update(_ctxt) # track interfaces for every complete context. [self._complete_contexts.append(interface) for interface in context.interfaces if interface not in self._complete_contexts] - for context in data_contexts: - context.post_execute(ctxt) return ctxt def complete_contexts(self): From cac5f4ca06be4f922a03f298aae6c139922b82cd Mon Sep 17 00:00:00 2001 From: Kapil Thangavelu Date: Tue, 25 Feb 2014 09:23:13 -0500 Subject: [PATCH 18/98] sync helpers --- .../charmhelpers/contrib/openstack/context.py | 22 ++++++++----------- .../contrib/openstack/templates/ceph.conf | 3 --- .../contrib/openstack/templates/haproxy.cfg | 5 +++-- .../contrib/storage/linux/ceph.py | 8 ++----- hooks/charmhelpers/core/host.py | 12 +++------- 5 files changed, 17 insertions(+), 33 deletions(-) diff --git a/hooks/charmhelpers/contrib/openstack/context.py b/hooks/charmhelpers/contrib/openstack/context.py index 38e4eac7..a21c0c84 100644 --- a/hooks/charmhelpers/contrib/openstack/context.py +++ b/hooks/charmhelpers/contrib/openstack/context.py @@ -27,9 +27,11 @@ from charmhelpers.core.hookenv import ( ) from charmhelpers.contrib.hahelpers.cluster import ( - determine_apache_port, determine_api_port, + determine_haproxy_port, https, + is_clustered, + peer_units, ) from charmhelpers.contrib.hahelpers.apache import ( @@ -263,12 +265,7 @@ class AMQPContext(OSContextGenerator): # Sufficient information found = break out! break # Used for active/active rabbitmq >= grizzly - if ('clustered' not in ctxt or relation_get('ha-vip-only') == 'True') and \ - len(related_units(rid)) > 1: - if relation_get('ha_queues'): - ctxt['rabbitmq_ha_queues'] = relation_get('ha_queues') - else: - ctxt['rabbitmq_ha_queues'] = False + if 'clustered' not in ctxt and len(related_units(rid)) > 1: rabbitmq_hosts = [] for unit in related_units(rid): rabbitmq_hosts.append(relation_get('private-address', @@ -297,13 +294,11 @@ class CephContext(OSContextGenerator): unit=unit)) auth = relation_get('auth', rid=rid, unit=unit) key = relation_get('key', rid=rid, unit=unit) - use_syslog = str(config('use-syslog')).lower() ctxt = { 'mon_hosts': ' '.join(mon_hosts), 'auth': auth, 'key': key, - 'use_syslog': use_syslog } if not os.path.isdir('/etc/ceph'): @@ -432,9 +427,11 @@ class ApacheSSLContext(OSContextGenerator): 'private_address': unit_get('private-address'), 'endpoints': [] } - for api_port in self.external_ports: - ext_port = determine_apache_port(api_port) - int_port = determine_api_port(api_port) + for ext_port in self.external_ports: + if peer_units() or is_clustered(): + int_port = determine_haproxy_port(ext_port) + else: + int_port = determine_api_port(ext_port) portmap = (int(ext_port), int(int_port)) ctxt['endpoints'].append(portmap) return ctxt @@ -636,7 +633,6 @@ class SubordinateConfigContext(OSContextGenerator): class SyslogContext(OSContextGenerator): - def __call__(self): ctxt = { 'use_syslog': config('use-syslog') diff --git a/hooks/charmhelpers/contrib/openstack/templates/ceph.conf b/hooks/charmhelpers/contrib/openstack/templates/ceph.conf index 874ad2bd..49d07c80 100644 --- a/hooks/charmhelpers/contrib/openstack/templates/ceph.conf +++ b/hooks/charmhelpers/contrib/openstack/templates/ceph.conf @@ -9,6 +9,3 @@ keyring = /etc/ceph/$cluster.$name.keyring mon host = {{ mon_hosts }} {% endif -%} -log to syslog = {{ use_syslog }} -err to syslog = {{ use_syslog }} -clog to syslog = {{ use_syslog }} diff --git a/hooks/charmhelpers/contrib/openstack/templates/haproxy.cfg b/hooks/charmhelpers/contrib/openstack/templates/haproxy.cfg index 56ed913e..a1694e44 100644 --- a/hooks/charmhelpers/contrib/openstack/templates/haproxy.cfg +++ b/hooks/charmhelpers/contrib/openstack/templates/haproxy.cfg @@ -8,8 +8,8 @@ global defaults log global - mode tcp - option tcplog + mode http + option httplog option dontlognull retries 3 timeout queue 1000 @@ -29,6 +29,7 @@ listen stats :8888 {% for service, ports in service_ports.iteritems() -%} listen {{ service }} 0.0.0.0:{{ ports[0] }} balance roundrobin + option tcplog {% for unit, address in units.iteritems() -%} server {{ unit }} {{ address }}:{{ ports[1] }} check {% endfor %} diff --git a/hooks/charmhelpers/contrib/storage/linux/ceph.py b/hooks/charmhelpers/contrib/storage/linux/ceph.py index 12417410..69b879ca 100644 --- a/hooks/charmhelpers/contrib/storage/linux/ceph.py +++ b/hooks/charmhelpers/contrib/storage/linux/ceph.py @@ -49,9 +49,6 @@ CEPH_CONF = """[global] auth supported = {auth} keyring = {keyring} mon host = {mon_hosts} - log to syslog = {use_syslog} - err to syslog = {use_syslog} - clog to syslog = {use_syslog} """ @@ -197,7 +194,7 @@ def get_ceph_nodes(): return hosts -def configure(service, key, auth, use_syslog): +def configure(service, key, auth): ''' Perform basic configuration of Ceph ''' create_keyring(service, key) create_key_file(service, key) @@ -205,8 +202,7 @@ def configure(service, key, auth, use_syslog): with open('/etc/ceph/ceph.conf', 'w') as ceph_conf: ceph_conf.write(CEPH_CONF.format(auth=auth, keyring=_keyring_path(service), - mon_hosts=",".join(map(str, hosts)), - use_syslog=use_syslog)) + mon_hosts=",".join(map(str, hosts)))) modprobe('rbd') diff --git a/hooks/charmhelpers/core/host.py b/hooks/charmhelpers/core/host.py index cfd26847..c8c81b28 100644 --- a/hooks/charmhelpers/core/host.py +++ b/hooks/charmhelpers/core/host.py @@ -194,7 +194,7 @@ def file_hash(path): return None -def restart_on_change(restart_map, stopstart=False): +def restart_on_change(restart_map): """Restart services based on configuration files changing This function is used a decorator, for example @@ -219,14 +219,8 @@ def restart_on_change(restart_map, stopstart=False): for path in restart_map: if checksums[path] != file_hash(path): restarts += restart_map[path] - services_list = list(OrderedDict.fromkeys(restarts)) - if not stopstart: - for service_name in services_list: - service('restart', service_name) - else: - for action in ['stop', 'start']: - for service_name in services_list: - service(action, service_name) + for service_name in list(OrderedDict.fromkeys(restarts)): + service('restart', service_name) return wrapped_f return wrap From c904729f604e4f7bb8a1a095373fcabf90027287 Mon Sep 17 00:00:00 2001 From: Kapil Thangavelu Date: Wed, 26 Feb 2014 19:54:21 -0500 Subject: [PATCH 19/98] sync helpers --- .../charmhelpers/contrib/openstack/context.py | 34 +++++-------------- .../contrib/openstack/templates/ceph.conf | 3 -- .../contrib/openstack/templates/haproxy.cfg | 5 +-- .../contrib/storage/linux/ceph.py | 8 ++--- hooks/charmhelpers/core/host.py | 12 ++----- 5 files changed, 17 insertions(+), 45 deletions(-) diff --git a/hooks/charmhelpers/contrib/openstack/context.py b/hooks/charmhelpers/contrib/openstack/context.py index c579dead..a21c0c84 100644 --- a/hooks/charmhelpers/contrib/openstack/context.py +++ b/hooks/charmhelpers/contrib/openstack/context.py @@ -27,10 +27,11 @@ from charmhelpers.core.hookenv import ( ) from charmhelpers.contrib.hahelpers.cluster import ( - determine_apache_port, determine_api_port, + determine_haproxy_port, https, - is_clustered + is_clustered, + peer_units, ) from charmhelpers.contrib.hahelpers.apache import ( @@ -287,7 +288,6 @@ class CephContext(OSContextGenerator): mon_hosts = [] auth = None key = None - use_syslog = str(config('use-syslog')).lower() for rid in relation_ids('ceph'): for unit in related_units(rid): mon_hosts.append(relation_get('private-address', rid=rid, @@ -299,7 +299,6 @@ class CephContext(OSContextGenerator): 'mon_hosts': ' '.join(mon_hosts), 'auth': auth, 'key': key, - 'use_syslog': use_syslog } if not os.path.isdir('/etc/ceph'): @@ -428,9 +427,11 @@ class ApacheSSLContext(OSContextGenerator): 'private_address': unit_get('private-address'), 'endpoints': [] } - for api_port in self.external_ports: - ext_port = determine_apache_port(api_port) - int_port = determine_api_port(api_port) + for ext_port in self.external_ports: + if peer_units() or is_clustered(): + int_port = determine_haproxy_port(ext_port) + else: + int_port = determine_api_port(ext_port) portmap = (int(ext_port), int(int_port)) ctxt['endpoints'].append(portmap) return ctxt @@ -497,22 +498,6 @@ class NeutronContext(object): return nvp_ctxt - def neutron_ctxt(self): - if https(): - proto = 'https' - else: - proto = 'http' - if is_clustered(): - host = config('vip') - else: - host = unit_get('private-address') - url = '%s://%s:%s' % (proto, host, '9292') - ctxt = { - 'network_manager': self.network_manager, - 'neutron_url': url, - } - return ctxt - def __call__(self): self._ensure_packages() @@ -522,7 +507,7 @@ class NeutronContext(object): if not self.plugin: return {} - ctxt = self.neutron_ctxt() + ctxt = {'network_manager': self.network_manager} if self.plugin == 'ovs': ctxt.update(self.ovs_ctxt()) @@ -648,7 +633,6 @@ class SubordinateConfigContext(OSContextGenerator): class SyslogContext(OSContextGenerator): - def __call__(self): ctxt = { 'use_syslog': config('use-syslog') diff --git a/hooks/charmhelpers/contrib/openstack/templates/ceph.conf b/hooks/charmhelpers/contrib/openstack/templates/ceph.conf index 874ad2bd..49d07c80 100644 --- a/hooks/charmhelpers/contrib/openstack/templates/ceph.conf +++ b/hooks/charmhelpers/contrib/openstack/templates/ceph.conf @@ -9,6 +9,3 @@ keyring = /etc/ceph/$cluster.$name.keyring mon host = {{ mon_hosts }} {% endif -%} -log to syslog = {{ use_syslog }} -err to syslog = {{ use_syslog }} -clog to syslog = {{ use_syslog }} diff --git a/hooks/charmhelpers/contrib/openstack/templates/haproxy.cfg b/hooks/charmhelpers/contrib/openstack/templates/haproxy.cfg index 56ed913e..a1694e44 100644 --- a/hooks/charmhelpers/contrib/openstack/templates/haproxy.cfg +++ b/hooks/charmhelpers/contrib/openstack/templates/haproxy.cfg @@ -8,8 +8,8 @@ global defaults log global - mode tcp - option tcplog + mode http + option httplog option dontlognull retries 3 timeout queue 1000 @@ -29,6 +29,7 @@ listen stats :8888 {% for service, ports in service_ports.iteritems() -%} listen {{ service }} 0.0.0.0:{{ ports[0] }} balance roundrobin + option tcplog {% for unit, address in units.iteritems() -%} server {{ unit }} {{ address }}:{{ ports[1] }} check {% endfor %} diff --git a/hooks/charmhelpers/contrib/storage/linux/ceph.py b/hooks/charmhelpers/contrib/storage/linux/ceph.py index 12417410..69b879ca 100644 --- a/hooks/charmhelpers/contrib/storage/linux/ceph.py +++ b/hooks/charmhelpers/contrib/storage/linux/ceph.py @@ -49,9 +49,6 @@ CEPH_CONF = """[global] auth supported = {auth} keyring = {keyring} mon host = {mon_hosts} - log to syslog = {use_syslog} - err to syslog = {use_syslog} - clog to syslog = {use_syslog} """ @@ -197,7 +194,7 @@ def get_ceph_nodes(): return hosts -def configure(service, key, auth, use_syslog): +def configure(service, key, auth): ''' Perform basic configuration of Ceph ''' create_keyring(service, key) create_key_file(service, key) @@ -205,8 +202,7 @@ def configure(service, key, auth, use_syslog): with open('/etc/ceph/ceph.conf', 'w') as ceph_conf: ceph_conf.write(CEPH_CONF.format(auth=auth, keyring=_keyring_path(service), - mon_hosts=",".join(map(str, hosts)), - use_syslog=use_syslog)) + mon_hosts=",".join(map(str, hosts)))) modprobe('rbd') diff --git a/hooks/charmhelpers/core/host.py b/hooks/charmhelpers/core/host.py index cfd26847..c8c81b28 100644 --- a/hooks/charmhelpers/core/host.py +++ b/hooks/charmhelpers/core/host.py @@ -194,7 +194,7 @@ def file_hash(path): return None -def restart_on_change(restart_map, stopstart=False): +def restart_on_change(restart_map): """Restart services based on configuration files changing This function is used a decorator, for example @@ -219,14 +219,8 @@ def restart_on_change(restart_map, stopstart=False): for path in restart_map: if checksums[path] != file_hash(path): restarts += restart_map[path] - services_list = list(OrderedDict.fromkeys(restarts)) - if not stopstart: - for service_name in services_list: - service('restart', service_name) - else: - for action in ['stop', 'start']: - for service_name in services_list: - service(action, service_name) + for service_name in list(OrderedDict.fromkeys(restarts)): + service('restart', service_name) return wrapped_f return wrap From 41a684fb7ee044e6a05077b95f065cbe657f4c86 Mon Sep 17 00:00:00 2001 From: James Page Date: Thu, 27 Feb 2014 09:26:38 +0000 Subject: [PATCH 20/98] Resync latest ssl-everywhere helpers, add ssl_ca option. --- charm-helpers.yaml | 2 +- config.yaml | 7 +- .../charmhelpers/contrib/hahelpers/apache.py | 17 ++--- .../charmhelpers/contrib/openstack/context.py | 72 ++++++++++++------- .../contrib/openstack/templates/ceph.conf | 3 + .../contrib/openstack/templates/haproxy.cfg | 5 +- .../contrib/storage/linux/ceph.py | 8 ++- hooks/charmhelpers/core/host.py | 12 +++- hooks/nova_cc_hooks.py | 5 +- 9 files changed, 86 insertions(+), 45 deletions(-) diff --git a/charm-helpers.yaml b/charm-helpers.yaml index b53ba5be..a5690c9b 100644 --- a/charm-helpers.yaml +++ b/charm-helpers.yaml @@ -1,4 +1,4 @@ -branch: lp:~hazmat/charm-helpers/ssl-everywhere +branch: lp:~openstack-charmers/charm-helpers/ssl-everywhere destination: hooks/charmhelpers include: - core diff --git a/config.yaml b/config.yaml index a91e0f4a..edc30762 100644 --- a/config.yaml +++ b/config.yaml @@ -120,12 +120,17 @@ options: type: string description: | SSL certificate to install and use for API ports. Setting this value - and ssl_key will enable reverse proxying, point Glance's entry in the + and ssl_key will enable reverse proxying, point Nova's entry in the Keystone catalog to use https, and override any certficiate and key issued by Keystone (if it is configured to do so). ssl_key: type: string description: SSL key to use with certificate specified as ssl_cert. + ssl_ca: + type: string + description: | + SSL CA to use with the certificate and key provided - this is only + required if you are providing a privately signed ssl_cert and ssl_key. # Neutron NVP Plugin configuration nvp-controllers: type: string diff --git a/hooks/charmhelpers/contrib/hahelpers/apache.py b/hooks/charmhelpers/contrib/hahelpers/apache.py index 3208a85c..8d5fb8ba 100644 --- a/hooks/charmhelpers/contrib/hahelpers/apache.py +++ b/hooks/charmhelpers/contrib/hahelpers/apache.py @@ -39,14 +39,15 @@ def get_cert(): def get_ca_cert(): - ca_cert = None - log("Inspecting identity-service relations for CA SSL certificate.", - level=INFO) - for r_id in relation_ids('identity-service'): - for unit in relation_list(r_id): - if not ca_cert: - ca_cert = relation_get('ca_cert', - rid=r_id, unit=unit) + ca_cert = config_get('ssl_ca') + if ca_cert is None: + log("Inspecting identity-service relations for CA SSL certificate.", + level=INFO) + for r_id in relation_ids('identity-service'): + for unit in relation_list(r_id): + if ca_cert is None: + ca_cert = relation_get('ca_cert', + rid=r_id, unit=unit) return ca_cert diff --git a/hooks/charmhelpers/contrib/openstack/context.py b/hooks/charmhelpers/contrib/openstack/context.py index a21c0c84..11ddc0b0 100644 --- a/hooks/charmhelpers/contrib/openstack/context.py +++ b/hooks/charmhelpers/contrib/openstack/context.py @@ -27,11 +27,10 @@ from charmhelpers.core.hookenv import ( ) from charmhelpers.contrib.hahelpers.cluster import ( + determine_apache_port, determine_api_port, - determine_haproxy_port, https, - is_clustered, - peer_units, + is_clustered ) from charmhelpers.contrib.hahelpers.apache import ( @@ -190,22 +189,19 @@ class IdentityServiceContext(OSContextGenerator): for rid in relation_ids('identity-service'): for unit in related_units(rid): + rdata = relation_get(rid=rid, unit=unit) ctxt = { - 'service_port': relation_get('service_port', rid=rid, - unit=unit), - 'service_host': relation_get('service_host', rid=rid, - unit=unit), - 'auth_host': relation_get('auth_host', rid=rid, unit=unit), - 'auth_port': relation_get('auth_port', rid=rid, unit=unit), - 'admin_tenant_name': relation_get('service_tenant', - rid=rid, unit=unit), - 'admin_user': relation_get('service_username', rid=rid, - unit=unit), - 'admin_password': relation_get('service_password', rid=rid, - unit=unit), - # XXX: Hard-coded http. - 'service_protocol': 'http', - 'auth_protocol': 'http', + 'service_port': rdata.get('service_port'), + 'service_host': rdata.get('service_host'), + 'auth_host': rdata.get('auth_host'), + 'auth_port': rdata.get('auth_port'), + 'admin_tenant_name': rdata.get('service_tenant'), + 'admin_user': rdata.get('service_username'), + 'admin_password': rdata.get('service_password'), + 'service_protocol': + rdata.get('service_protocol') or 'http', + 'auth_protocol': + rdata.get('auth_protocol') or 'http', } if context_complete(ctxt): return ctxt @@ -265,7 +261,12 @@ class AMQPContext(OSContextGenerator): # Sufficient information found = break out! break # Used for active/active rabbitmq >= grizzly - if 'clustered' not in ctxt and len(related_units(rid)) > 1: + if ('clustered' not in ctxt or relation_get('ha-vip-only') == 'True') and \ + len(related_units(rid)) > 1: + if relation_get('ha_queues'): + ctxt['rabbitmq_ha_queues'] = relation_get('ha_queues') + else: + ctxt['rabbitmq_ha_queues'] = False rabbitmq_hosts = [] for unit in related_units(rid): rabbitmq_hosts.append(relation_get('private-address', @@ -284,10 +285,13 @@ class CephContext(OSContextGenerator): '''This generates context for /etc/ceph/ceph.conf templates''' if not relation_ids('ceph'): return {} + log('Generating template context for ceph') + mon_hosts = [] auth = None key = None + use_syslog = str(config('use-syslog')).lower() for rid in relation_ids('ceph'): for unit in related_units(rid): mon_hosts.append(relation_get('private-address', rid=rid, @@ -299,6 +303,7 @@ class CephContext(OSContextGenerator): 'mon_hosts': ' '.join(mon_hosts), 'auth': auth, 'key': key, + 'use_syslog': use_syslog } if not os.path.isdir('/etc/ceph'): @@ -427,17 +432,15 @@ class ApacheSSLContext(OSContextGenerator): 'private_address': unit_get('private-address'), 'endpoints': [] } - for ext_port in self.external_ports: - if peer_units() or is_clustered(): - int_port = determine_haproxy_port(ext_port) - else: - int_port = determine_api_port(ext_port) + for api_port in self.external_ports: + ext_port = determine_apache_port(api_port) + int_port = determine_api_port(api_port) portmap = (int(ext_port), int(int_port)) ctxt['endpoints'].append(portmap) return ctxt -class NeutronContext(object): +class NeutronContext(OSContextGenerator): interfaces = [] @property @@ -498,6 +501,22 @@ class NeutronContext(object): return nvp_ctxt + def neutron_ctxt(self): + if https(): + proto = 'https' + else: + proto = 'http' + if is_clustered(): + host = config('vip') + else: + host = unit_get('private-address') + url = '%s://%s:%s' % (proto, host, '9292') + ctxt = { + 'network_manager': self.network_manager, + 'neutron_url': url, + } + return ctxt + def __call__(self): self._ensure_packages() @@ -507,7 +526,7 @@ class NeutronContext(object): if not self.plugin: return {} - ctxt = {'network_manager': self.network_manager} + ctxt = self.neutron_ctxt() if self.plugin == 'ovs': ctxt.update(self.ovs_ctxt()) @@ -633,6 +652,7 @@ class SubordinateConfigContext(OSContextGenerator): class SyslogContext(OSContextGenerator): + def __call__(self): ctxt = { 'use_syslog': config('use-syslog') diff --git a/hooks/charmhelpers/contrib/openstack/templates/ceph.conf b/hooks/charmhelpers/contrib/openstack/templates/ceph.conf index 49d07c80..874ad2bd 100644 --- a/hooks/charmhelpers/contrib/openstack/templates/ceph.conf +++ b/hooks/charmhelpers/contrib/openstack/templates/ceph.conf @@ -9,3 +9,6 @@ keyring = /etc/ceph/$cluster.$name.keyring mon host = {{ mon_hosts }} {% endif -%} +log to syslog = {{ use_syslog }} +err to syslog = {{ use_syslog }} +clog to syslog = {{ use_syslog }} diff --git a/hooks/charmhelpers/contrib/openstack/templates/haproxy.cfg b/hooks/charmhelpers/contrib/openstack/templates/haproxy.cfg index a1694e44..56ed913e 100644 --- a/hooks/charmhelpers/contrib/openstack/templates/haproxy.cfg +++ b/hooks/charmhelpers/contrib/openstack/templates/haproxy.cfg @@ -8,8 +8,8 @@ global defaults log global - mode http - option httplog + mode tcp + option tcplog option dontlognull retries 3 timeout queue 1000 @@ -29,7 +29,6 @@ listen stats :8888 {% for service, ports in service_ports.iteritems() -%} listen {{ service }} 0.0.0.0:{{ ports[0] }} balance roundrobin - option tcplog {% for unit, address in units.iteritems() -%} server {{ unit }} {{ address }}:{{ ports[1] }} check {% endfor %} diff --git a/hooks/charmhelpers/contrib/storage/linux/ceph.py b/hooks/charmhelpers/contrib/storage/linux/ceph.py index 69b879ca..12417410 100644 --- a/hooks/charmhelpers/contrib/storage/linux/ceph.py +++ b/hooks/charmhelpers/contrib/storage/linux/ceph.py @@ -49,6 +49,9 @@ CEPH_CONF = """[global] auth supported = {auth} keyring = {keyring} mon host = {mon_hosts} + log to syslog = {use_syslog} + err to syslog = {use_syslog} + clog to syslog = {use_syslog} """ @@ -194,7 +197,7 @@ def get_ceph_nodes(): return hosts -def configure(service, key, auth): +def configure(service, key, auth, use_syslog): ''' Perform basic configuration of Ceph ''' create_keyring(service, key) create_key_file(service, key) @@ -202,7 +205,8 @@ def configure(service, key, auth): with open('/etc/ceph/ceph.conf', 'w') as ceph_conf: ceph_conf.write(CEPH_CONF.format(auth=auth, keyring=_keyring_path(service), - mon_hosts=",".join(map(str, hosts)))) + mon_hosts=",".join(map(str, hosts)), + use_syslog=use_syslog)) modprobe('rbd') diff --git a/hooks/charmhelpers/core/host.py b/hooks/charmhelpers/core/host.py index c8c81b28..cfd26847 100644 --- a/hooks/charmhelpers/core/host.py +++ b/hooks/charmhelpers/core/host.py @@ -194,7 +194,7 @@ def file_hash(path): return None -def restart_on_change(restart_map): +def restart_on_change(restart_map, stopstart=False): """Restart services based on configuration files changing This function is used a decorator, for example @@ -219,8 +219,14 @@ def restart_on_change(restart_map): for path in restart_map: if checksums[path] != file_hash(path): restarts += restart_map[path] - for service_name in list(OrderedDict.fromkeys(restarts)): - service('restart', service_name) + services_list = list(OrderedDict.fromkeys(restarts)) + if not stopstart: + for service_name in services_list: + service('restart', service_name) + else: + for action in ['stop', 'start']: + for service_name in services_list: + service(action, service_name) return wrapped_f return wrap diff --git a/hooks/nova_cc_hooks.py b/hooks/nova_cc_hooks.py index 6dfab01b..a92125e5 100755 --- a/hooks/nova_cc_hooks.py +++ b/hooks/nova_cc_hooks.py @@ -211,6 +211,8 @@ def _auth_config(): cfg = { 'auth_host': ks_auth_host, 'auth_port': auth_token_config('auth_port'), + 'auth_protocol': auth_token_config('auth_protocol'), + 'service_protocol': auth_token_config('service_protocol'), 'service_port': auth_token_config('service_port'), 'service_username': auth_token_config('admin_user'), 'service_password': auth_token_config('admin_password'), @@ -226,7 +228,8 @@ def _auth_config(): def save_novarc(): auth = _auth_config() # XXX hard-coded http - ks_url = 'http://%s:%s/v2.0' % (auth['auth_host'], auth['auth_port']) + ks_url = '%s://%s:%s/v2.0' % (auth['auth_protocol'], + auth['auth_host'], auth['auth_port']) with open('/etc/quantum/novarc', 'wb') as out: out.write('export OS_USERNAME=%s\n' % auth['service_username']) out.write('export OS_PASSWORD=%s\n' % auth['service_password']) From 382d22c10fde260f758b7183efca70b0faf7c680 Mon Sep 17 00:00:00 2001 From: James Page Date: Thu, 27 Feb 2014 11:12:34 +0000 Subject: [PATCH 21/98] Fixup mysql ssl for neutron configuration files --- hooks/charmhelpers/contrib/openstack/neutron.py | 14 ++++++++++---- templates/folsom/ovs_quantum_plugin.ini | 2 +- templates/havana/nvp.ini | 2 +- templates/havana/ovs_neutron_plugin.ini | 2 +- 4 files changed, 13 insertions(+), 7 deletions(-) diff --git a/hooks/charmhelpers/contrib/openstack/neutron.py b/hooks/charmhelpers/contrib/openstack/neutron.py index 8d32bd00..ee662498 100644 --- a/hooks/charmhelpers/contrib/openstack/neutron.py +++ b/hooks/charmhelpers/contrib/openstack/neutron.py @@ -17,6 +17,7 @@ def headers_package(): kver = check_output(['uname', '-r']).strip() return 'linux-headers-%s' % kver +QUANTUM_CONF_DIR = '/etc/quantum' # legacy def quantum_plugins(): @@ -30,7 +31,8 @@ def quantum_plugins(): 'contexts': [ context.SharedDBContext(user=config('neutron-database-user'), database=config('neutron-database'), - relation_prefix='neutron')], + relation_prefix='neutron', + ssl_dir=QUANTUM_CONF_DIR)], 'services': ['quantum-plugin-openvswitch-agent'], 'packages': [[headers_package(), 'openvswitch-datapath-dkms'], ['quantum-plugin-openvswitch-agent']], @@ -45,7 +47,8 @@ def quantum_plugins(): 'contexts': [ context.SharedDBContext(user=config('neutron-database-user'), database=config('neutron-database'), - relation_prefix='neutron')], + relation_prefix='neutron', + ssl_dir=QUANTUM_CONF_DIR)], 'services': [], 'packages': [], 'server_packages': ['quantum-server', @@ -54,6 +57,7 @@ def quantum_plugins(): } } +NEUTRON_CONF_DIR = '/etc/neutron' def neutron_plugins(): from charmhelpers.contrib.openstack import context @@ -66,7 +70,8 @@ def neutron_plugins(): 'contexts': [ context.SharedDBContext(user=config('neutron-database-user'), database=config('neutron-database'), - relation_prefix='neutron')], + relation_prefix='neutron', + ssl_dir=NEUTRON_CONF_DIR)], 'services': ['neutron-plugin-openvswitch-agent'], 'packages': [[headers_package(), 'openvswitch-datapath-dkms'], ['quantum-plugin-openvswitch-agent']], @@ -81,7 +86,8 @@ def neutron_plugins(): 'contexts': [ context.SharedDBContext(user=config('neutron-database-user'), database=config('neutron-database'), - relation_prefix='neutron')], + relation_prefix='neutron', + ssl_dir=NEUTRON_CONF_DIR)], 'services': [], 'packages': [], 'server_packages': ['neutron-server', diff --git a/templates/folsom/ovs_quantum_plugin.ini b/templates/folsom/ovs_quantum_plugin.ini index a80cca3a..37050079 100644 --- a/templates/folsom/ovs_quantum_plugin.ini +++ b/templates/folsom/ovs_quantum_plugin.ini @@ -11,7 +11,7 @@ local_ip = {{ local_ip }} [DATABASE] {% if database_host -%} -sql_connection = mysql://{{ database_user }}:{{ database_password }}@{{ database_host }}/{{ database }}?quantum?charset=utf8 +sql_connection = mysql://{{ database_user }}:{{ database_password }}@{{ database_host }}/{{ database }}{% if database_ssl_ca %}?ssl_ca={{ database_ssl_ca }}{% if database_ssl_cert %}&ssl_cert={{ database_ssl_cert }}&ssl_key={{ database_ssl_key }}{% endif %}{% endif %} reconnect_interval = 2 {% else -%} connection = sqlite:////var/lib/quantum/quantum.sqlite diff --git a/templates/havana/nvp.ini b/templates/havana/nvp.ini index a6225421..65d544c6 100644 --- a/templates/havana/nvp.ini +++ b/templates/havana/nvp.ini @@ -12,7 +12,7 @@ default_l3_gw_service_uuid = {{ nvp_l3_uuid }} [database] {% if database_host -%} -connection = mysql://{{ database_user }}:{{ database_password }}@{{ database_host }}/{{ database }} +connection = mysql://{{ database_user }}:{{ database_password }}@{{ database_host }}/{{ database }}{% if database_ssl_ca %}?ssl_ca={{ database_ssl_ca }}{% if database_ssl_cert %}&ssl_cert={{ database_ssl_cert }}&ssl_key={{ database_ssl_key }}{% endif %}{% endif %} {% else -%} connection = sqlite:////var/lib/neutron/neutron.sqlite {% endif -%} diff --git a/templates/havana/ovs_neutron_plugin.ini b/templates/havana/ovs_neutron_plugin.ini index 73e5a205..155b6f0f 100644 --- a/templates/havana/ovs_neutron_plugin.ini +++ b/templates/havana/ovs_neutron_plugin.ini @@ -6,7 +6,7 @@ local_ip = {{ local_ip }} [database] {% if database_host -%} -connection = mysql://{{ database_user }}:{{ database_password }}@{{ database_host }}/{{ database }} +connection = mysql://{{ database_user }}:{{ database_password }}@{{ database_host }}/{{ database }}{% if database_ssl_ca %}?ssl_ca={{ database_ssl_ca }}{% if database_ssl_cert %}&ssl_cert={{ database_ssl_cert }}&ssl_key={{ database_ssl_key }}{% endif %}{% endif %} {% else -%} connection = sqlite:////var/lib/neutron/neutron.sqlite {% endif -%} From 66171396faa1e4c71e8e6119f107d05120b9110c Mon Sep 17 00:00:00 2001 From: James Page Date: Thu, 27 Feb 2014 14:36:16 +0000 Subject: [PATCH 22/98] Temp fix - use correct port for neutron --- hooks/charmhelpers/contrib/openstack/context.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/hooks/charmhelpers/contrib/openstack/context.py b/hooks/charmhelpers/contrib/openstack/context.py index 11ddc0b0..62f127e0 100644 --- a/hooks/charmhelpers/contrib/openstack/context.py +++ b/hooks/charmhelpers/contrib/openstack/context.py @@ -510,7 +510,7 @@ class NeutronContext(OSContextGenerator): host = config('vip') else: host = unit_get('private-address') - url = '%s://%s:%s' % (proto, host, '9292') + url = '%s://%s:%s' % (proto, host, '9696') ctxt = { 'network_manager': self.network_manager, 'neutron_url': url, From c8411158a69e31bfeedb47e17ef1700c7d55448a Mon Sep 17 00:00:00 2001 From: James Page Date: Mon, 3 Mar 2014 11:31:44 +0000 Subject: [PATCH 23/98] Enable metering >= havana --- templates/havana/neutron.conf | 3 +++ templates/icehouse/neutron.conf | 2 +- 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/templates/havana/neutron.conf b/templates/havana/neutron.conf index 0bfb440d..ecd92488 100644 --- a/templates/havana/neutron.conf +++ b/templates/havana/neutron.conf @@ -15,6 +15,9 @@ bind_port = 9696 {% endif -%} {% if core_plugin -%} core_plugin = {{ core_plugin }} +{% if neutron_plugin in ['ovs', 'ml2'] -%} +service_plugins = neutron.services.metering.metering_plugin.MeteringPlugin +{% endif -%} {% endif -%} {% if neutron_security_groups -%} allow_overlapping_ips = True diff --git a/templates/icehouse/neutron.conf b/templates/icehouse/neutron.conf index 191eb6db..5b293a31 100644 --- a/templates/icehouse/neutron.conf +++ b/templates/icehouse/neutron.conf @@ -11,7 +11,7 @@ notification_driver = neutron.openstack.common.notifier.rpc_notifier {% if core_plugin -%} core_plugin = {{ core_plugin }} {% if neutron_plugin in ['ovs', 'ml2'] -%} -service_plugins = neutron.services.l3_router.l3_router_plugin.L3RouterPlugin +service_plugins = neutron.services.l3_router.l3_router_plugin.L3RouterPlugin,neutron.services.metering.metering_plugin.MeteringPlugin {% endif -%} {% endif -%} {% if neutron_security_groups -%} From 78d8f95fcad6beff351bcb49a4846be5b7a36db6 Mon Sep 17 00:00:00 2001 From: James Page Date: Tue, 4 Mar 2014 16:57:51 +0000 Subject: [PATCH 24/98] Add computev3 type if appropriate --- hooks/nova_cc_utils.py | 21 ++++++++++++++++++--- 1 file changed, 18 insertions(+), 3 deletions(-) diff --git a/hooks/nova_cc_utils.py b/hooks/nova_cc_utils.py index 9eb15113..aa28bdb2 100644 --- a/hooks/nova_cc_utils.py +++ b/hooks/nova_cc_utils.py @@ -456,10 +456,16 @@ def determine_endpoints(url): '''Generates a dictionary containing all relevant endpoints to be passed to keystone as relation settings.''' region = config('region') + os_rel = os_release('nova-common') - # TODO: Configurable nova API version. - nova_url = ('%s:%s/v1.1/$(tenant_id)s' % - (url, api_port('nova-api-os-compute'))) + if os_rel >= 'grizzly': + nova_url = ('%s:%s/v2.0/$(tenant_id)s' % + (url, api_port('nova-api-os-compute'))) + else: + nova_url = ('%s:%s/v1.1/$(tenant_id)s' % + (url, api_port('nova-api-os-compute'))) + novav3_url = ('%s:%s/v3.0/$(tenant_id)s' % + (url, api_port('nova-api-os-compute'))) ec2_url = '%s:%s/services/Cloud' % (url, api_port('nova-api-ec2')) nova_volume_url = ('%s:%s/v1/$(tenant_id)s' % (url, api_port('nova-api-os-compute'))) @@ -504,6 +510,15 @@ def determine_endpoints(url): 'quantum_internal_url': neutron_url, }) + if os_rel >= 'havana': + endpoints.update({ + 'novav3_service': 'novav3', + 'novav3_region': region, + 'novav3_public_url': novav3_url, + 'novav3_admin_url': novav3_url, + 'novav3_internal_url': novav3_url + }) + return endpoints From bf188abc79a8523c1852248944ed93376c956fee Mon Sep 17 00:00:00 2001 From: James Page Date: Tue, 4 Mar 2014 17:06:08 +0000 Subject: [PATCH 25/98] Deal with upgrades --- hooks/nova_cc_hooks.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/hooks/nova_cc_hooks.py b/hooks/nova_cc_hooks.py index 6dfab01b..3b259c42 100755 --- a/hooks/nova_cc_hooks.py +++ b/hooks/nova_cc_hooks.py @@ -436,6 +436,8 @@ def nova_vmware_relation_changed(): def upgrade_charm(): for r_id in relation_ids('amqp'): amqp_joined(relation_id=r_id) + for r_id in relation_ids('identity-service'): + identity_joined(relation_id=r_id) def main(): From 7d032a082e08e3e5da1bc2e8b19db8183013ab2a Mon Sep 17 00:00:00 2001 From: James Page Date: Tue, 4 Mar 2014 17:07:06 +0000 Subject: [PATCH 26/98] Fixup joined --- hooks/nova_cc_hooks.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/hooks/nova_cc_hooks.py b/hooks/nova_cc_hooks.py index 3b259c42..198958d0 100755 --- a/hooks/nova_cc_hooks.py +++ b/hooks/nova_cc_hooks.py @@ -437,7 +437,7 @@ def upgrade_charm(): for r_id in relation_ids('amqp'): amqp_joined(relation_id=r_id) for r_id in relation_ids('identity-service'): - identity_joined(relation_id=r_id) + identity_joined(rid=r_id) def main(): From b6bdcd322660cd951aeddb2beb78e7e0c8d2a677 Mon Sep 17 00:00:00 2001 From: James Page Date: Tue, 4 Mar 2014 17:10:07 +0000 Subject: [PATCH 27/98] Fixup endpoints --- hooks/nova_cc_utils.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/hooks/nova_cc_utils.py b/hooks/nova_cc_utils.py index aa28bdb2..464f9671 100644 --- a/hooks/nova_cc_utils.py +++ b/hooks/nova_cc_utils.py @@ -459,12 +459,12 @@ def determine_endpoints(url): os_rel = os_release('nova-common') if os_rel >= 'grizzly': - nova_url = ('%s:%s/v2.0/$(tenant_id)s' % + nova_url = ('%s:%s/v2/$(tenant_id)s' % (url, api_port('nova-api-os-compute'))) else: nova_url = ('%s:%s/v1.1/$(tenant_id)s' % (url, api_port('nova-api-os-compute'))) - novav3_url = ('%s:%s/v3.0/$(tenant_id)s' % + novav3_url = ('%s:%s/v3/$(tenant_id)s' % (url, api_port('nova-api-os-compute'))) ec2_url = '%s:%s/services/Cloud' % (url, api_port('nova-api-ec2')) nova_volume_url = ('%s:%s/v1/$(tenant_id)s' % From bb0bb7504a95e4cd98d9cffbdd26362f79d43cc2 Mon Sep 17 00:00:00 2001 From: James Page Date: Tue, 4 Mar 2014 17:20:02 +0000 Subject: [PATCH 28/98] Specialize nova.conf for havana --- templates/havana/nova.conf | 124 +++++++++++++++++++++++++++++++++++++ 1 file changed, 124 insertions(+) create mode 100644 templates/havana/nova.conf diff --git a/templates/havana/nova.conf b/templates/havana/nova.conf new file mode 100644 index 00000000..e94f3df0 --- /dev/null +++ b/templates/havana/nova.conf @@ -0,0 +1,124 @@ +############################################################################### +# [ WARNING ] +# Configuration file maintained by Juju. Local changes may be overwritten. +############################################################################### +[DEFAULT] +dhcpbridge_flagfile=/etc/nova/nova.conf +dhcpbridge=/usr/bin/nova-dhcpbridge +logdir=/var/log/nova +state_path=/var/lib/nova +lock_path=/var/lock/nova +force_dhcp_release=True +iscsi_helper=tgtadm +libvirt_use_virtio_for_bridges=True +connection_type=libvirt +root_helper=sudo nova-rootwrap /etc/nova/rootwrap.conf +verbose=True +ec2_private_dns_show_ip=True +api_paste_config=/etc/nova/api-paste.ini +volumes_path=/var/lib/nova/volumes +enabled_apis=ec2,osapi_compute,metadata +auth_strategy=keystone +compute_driver=libvirt.LibvirtDriver +{% if keystone_ec2_url -%} +keystone_ec2_url = {{ keystone_ec2_url }} +{% endif -%} + +{% if database_host -%} +sql_connection = mysql://{{ database_user }}:{{ database_password }}@{{ database_host }}/{{ database }} +{% endif -%} + +{% if rabbitmq_host -%} +rabbit_host = {{ rabbitmq_host }} +rabbit_userid = {{ rabbitmq_user }} +rabbit_password = {{ rabbitmq_password }} +rabbit_virtual_host = {{ rabbitmq_virtual_host }} +{% endif -%} + +{% if glance_api_servers -%} +glance_api_servers = {{ glance_api_servers }} +{% endif -%} + +{% if rbd_pool -%} +rbd_pool = {{ rbd_pool }} +rbd_user = {{ rbd_user }} +rbd_secret_uuid = {{ rbd_secret_uuid }} +{% endif -%} + +{% if neutron_plugin and neutron_plugin == 'ovs' -%} +libvirt_vif_driver = nova.virt.libvirt.vif.LibvirtGenericVIFDriver +libvirt_user_virtio_for_bridges = True +{% if neutron_security_groups -%} +security_group_api = {{ network_manager }} +nova_firewall_driver = nova.virt.firewall.NoopFirewallDriver +{% endif -%} +{% if external_network -%} +default_floating_pool = {{ external_network }} +{% endif -%} +{% endif -%} + +{% if neutron_plugin and neutron_plugin == 'nvp' -%} +security_group_api = neutron +nova_firewall_driver = nova.virt.firewall.NoopFirewallDriver +{% if external_network -%} +default_floating_pool = {{ external_network }} +{% endif -%} +{% endif -%} + +{% if network_manager_config -%} +{% for key, value in network_manager_config.iteritems() -%} +{{ key }} = {{ value }} +{% endfor -%} +{% endif -%} + +{% if network_manager and network_manager == 'quantum' -%} +network_api_class = nova.network.quantumv2.api.API +{% if auth_host -%} +quantum_auth_strategy = keystone +quantum_admin_tenant_name = {{ admin_tenant_name }} +quantum_admin_username = {{ admin_user }} +quantum_admin_password = {{ admin_password }} +quantum_admin_auth_url = {{ auth_protocol }}://{{ auth_host }}:{{ auth_port }}/v2.0 +{% endif -%} +{% elif network_manager and network_manager == 'neutron' -%} +network_api_class = nova.network.neutronv2.api.API +{% if auth_host -%} +neutron_auth_strategy = keystone +neutron_admin_tenant_name = {{ admin_tenant_name }} +neutron_admin_username = {{ admin_user }} +neutron_admin_password = {{ admin_password }} +neutron_admin_auth_url = {{ auth_protocol }}://{{ auth_host }}:{{ auth_port }}/v2.0 +{% endif -%} +{% else -%} +network_manager = nova.network.manager.FlatDHCPManager +{% endif -%} + +{% if default_floating_pool -%} +default_floating_pool = {{ default_floating_pool }} +{% endif -%} + +{% if volume_service -%} +volume_api_class=nova.volume.cinder.API +{% endif -%} + +{% if user_config_flags -%} +{% for key, value in user_config_flags.iteritems() -%} +{{ key }} = {{ value }} +{% endfor -%} +{% endif -%} + +{% if listen_ports -%} +{% for key, value in listen_ports.iteritems() -%} +{{ key }} = {{ value }} +{% endfor -%} +{% endif -%} + +{% if sections and 'DEFAULT' in sections -%} +{% for key, value in sections['DEFAULT'] -%} +{{ key }} = {{ value }} +{% endfor -%} +{% endif -%} + +[osapi_v3] +enabled=True + From be8c5d5bd1ad90bd08ab860b4d84a45d01f124b1 Mon Sep 17 00:00:00 2001 From: James Page Date: Tue, 4 Mar 2014 17:26:48 +0000 Subject: [PATCH 29/98] Fixup v3 endpoints --- hooks/nova_cc_utils.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/hooks/nova_cc_utils.py b/hooks/nova_cc_utils.py index 464f9671..c88e564b 100644 --- a/hooks/nova_cc_utils.py +++ b/hooks/nova_cc_utils.py @@ -464,8 +464,8 @@ def determine_endpoints(url): else: nova_url = ('%s:%s/v1.1/$(tenant_id)s' % (url, api_port('nova-api-os-compute'))) - novav3_url = ('%s:%s/v3/$(tenant_id)s' % - (url, api_port('nova-api-os-compute'))) + novav3_url = ('%s:%s/v3' % + (api_port('nova-api-os-compute'))) ec2_url = '%s:%s/services/Cloud' % (url, api_port('nova-api-ec2')) nova_volume_url = ('%s:%s/v1/$(tenant_id)s' % (url, api_port('nova-api-os-compute'))) From 5382bca7fe1decebad0d4b416dd4062d29dc8d4b Mon Sep 17 00:00:00 2001 From: James Page Date: Tue, 4 Mar 2014 17:35:58 +0000 Subject: [PATCH 30/98] Fixup --- hooks/nova_cc_utils.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/hooks/nova_cc_utils.py b/hooks/nova_cc_utils.py index c88e564b..184924ce 100644 --- a/hooks/nova_cc_utils.py +++ b/hooks/nova_cc_utils.py @@ -465,7 +465,7 @@ def determine_endpoints(url): nova_url = ('%s:%s/v1.1/$(tenant_id)s' % (url, api_port('nova-api-os-compute'))) novav3_url = ('%s:%s/v3' % - (api_port('nova-api-os-compute'))) + (url, api_port('nova-api-os-compute'))) ec2_url = '%s:%s/services/Cloud' % (url, api_port('nova-api-ec2')) nova_volume_url = ('%s:%s/v1/$(tenant_id)s' % (url, api_port('nova-api-os-compute'))) From d0803a6049591619d9fa924fd432f990809fde5c Mon Sep 17 00:00:00 2001 From: "yolanda.robla@canonical.com" <> Date: Wed, 5 Mar 2014 15:31:31 +0100 Subject: [PATCH 31/98] adding keystone config settings to grizzly/nova.conf --- templates/grizzly/nova.conf | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/templates/grizzly/nova.conf b/templates/grizzly/nova.conf index 67c63a6a..007e4f0e 100644 --- a/templates/grizzly/nova.conf +++ b/templates/grizzly/nova.conf @@ -81,8 +81,24 @@ default_floating_pool = {{ external_network }} {% if network_manager and network_manager == 'quantum' -%} network_api_class = nova.network.quantumv2.api.API +quantum_url = {{ neutron_url }} +{% if auth_host -%} +quantum_auth_strategy = keystone +quantum_admin_tenant_name = {{ admin_tenant_name }} +quantum_admin_username = {{ admin_user }} +quantum_admin_password = {{ admin_password }} +quantum_admin_auth_url = {{ auth_protocol }}://{{ auth_host }}:{{ auth_port }}/v2.0 +{% endif -%} {% elif network_manager and network_manager == 'neutron' -%} network_api_class = nova.network.neutronv2.api.API +neutron_url = {{ neutron_url }} +{% if auth_host -%} +neutron_auth_strategy = keystone +neutron_admin_tenant_name = {{ admin_tenant_name }} +neutron_admin_username = {{ admin_user }} +neutron_admin_password = {{ admin_password }} +neutron_admin_auth_url = {{ auth_protocol }}://{{ auth_host }}:{{ auth_port }}/v2.0 +{% endif -%} {% else -%} network_manager = nova.network.manager.FlatDHCPManager {% endif -%} From 99e4905950098a0f317811bc966808bae3449d1c Mon Sep 17 00:00:00 2001 From: James Page Date: Thu, 6 Mar 2014 12:46:15 +0000 Subject: [PATCH 32/98] Use upgrade for upgrades --- .../charmhelpers/contrib/openstack/context.py | 3 +-- hooks/charmhelpers/contrib/openstack/utils.py | 2 ++ hooks/charmhelpers/fetch/__init__.py | 20 +++++++++++++++++++ hooks/nova_cc_utils.py | 4 ++-- 4 files changed, 25 insertions(+), 4 deletions(-) diff --git a/hooks/charmhelpers/contrib/openstack/context.py b/hooks/charmhelpers/contrib/openstack/context.py index 81ac12b6..97658dc9 100644 --- a/hooks/charmhelpers/contrib/openstack/context.py +++ b/hooks/charmhelpers/contrib/openstack/context.py @@ -250,13 +250,12 @@ class CephContext(OSContextGenerator): unit=unit)) auth = relation_get('auth', rid=rid, unit=unit) key = relation_get('key', rid=rid, unit=unit) - use_syslog = str(config('use-syslog')).lower() ctxt = { 'mon_hosts': ' '.join(mon_hosts), 'auth': auth, 'key': key, - 'use_syslog': use_syslog + 'use_syslog': str(config('use-syslog')).lower() } if not os.path.isdir('/etc/ceph'): diff --git a/hooks/charmhelpers/contrib/openstack/utils.py b/hooks/charmhelpers/contrib/openstack/utils.py index d0d0998a..f1e60b95 100644 --- a/hooks/charmhelpers/contrib/openstack/utils.py +++ b/hooks/charmhelpers/contrib/openstack/utils.py @@ -65,6 +65,8 @@ SWIFT_CODENAMES = OrderedDict([ ('1.10.0', 'havana'), ('1.9.1', 'havana'), ('1.9.0', 'havana'), + ('1.13.0', 'icehouse'), + ('1.12.0', 'icehouse'), ('1.11.0', 'icehouse'), ]) diff --git a/hooks/charmhelpers/fetch/__init__.py b/hooks/charmhelpers/fetch/__init__.py index 07bb707d..702b63e5 100644 --- a/hooks/charmhelpers/fetch/__init__.py +++ b/hooks/charmhelpers/fetch/__init__.py @@ -97,6 +97,26 @@ def apt_install(packages, options=None, fatal=False): subprocess.call(cmd, env=env) +def apt_upgrade(options=None, fatal=False): + """Install one or more packages""" + if options is None: + options = ['--option=Dpkg::Options::=--force-confold'] + + cmd = ['apt-get', '--assume-yes'] + cmd.extend(options) + cmd.append('upgrade') + log("Upgrading with options: {}".format(options)) + + env = os.environ.copy() + if 'DEBIAN_FRONTEND' not in env: + env['DEBIAN_FRONTEND'] = 'noninteractive' + + if fatal: + subprocess.check_call(cmd, env=env) + else: + subprocess.call(cmd, env=env) + + def apt_update(fatal=False): """Update local apt cache""" cmd = ['apt-get', 'update'] diff --git a/hooks/nova_cc_utils.py b/hooks/nova_cc_utils.py index 184924ce..2254f2af 100644 --- a/hooks/nova_cc_utils.py +++ b/hooks/nova_cc_utils.py @@ -22,7 +22,7 @@ from charmhelpers.contrib.openstack.utils import ( save_script_rc as _save_script_rc) from charmhelpers.fetch import ( - apt_install, + apt_upgrade, apt_update, ) @@ -283,7 +283,7 @@ def do_openstack_upgrade(configs): '--option', 'Dpkg::Options::=--force-confdef', ] - apt_install(packages=determine_packages(), options=dpkg_opts, fatal=True) + apt_upgrade(options=dpkg_opts, fatal=True) # set CONFIGS to load templates from new release and regenerate config configs.set_release(openstack_release=new_os_rel) From 96689a8bb3edf8296c0b3054bdd22c1e81ed1866 Mon Sep 17 00:00:00 2001 From: James Page Date: Thu, 6 Mar 2014 13:41:46 +0000 Subject: [PATCH 33/98] Use dist-upgrade --- hooks/charmhelpers/fetch/__init__.py | 9 ++++++--- hooks/nova_cc_utils.py | 2 +- 2 files changed, 7 insertions(+), 4 deletions(-) diff --git a/hooks/charmhelpers/fetch/__init__.py b/hooks/charmhelpers/fetch/__init__.py index 702b63e5..bcc5b397 100644 --- a/hooks/charmhelpers/fetch/__init__.py +++ b/hooks/charmhelpers/fetch/__init__.py @@ -97,14 +97,17 @@ def apt_install(packages, options=None, fatal=False): subprocess.call(cmd, env=env) -def apt_upgrade(options=None, fatal=False): - """Install one or more packages""" +def apt_upgrade(options=None, fatal=False, dist=False): + """Upgrade all packages""" if options is None: options = ['--option=Dpkg::Options::=--force-confold'] cmd = ['apt-get', '--assume-yes'] cmd.extend(options) - cmd.append('upgrade') + if dist: + cmd.append('dist-upgrade') + else: + cmd.append('upgrade') log("Upgrading with options: {}".format(options)) env = os.environ.copy() diff --git a/hooks/nova_cc_utils.py b/hooks/nova_cc_utils.py index 2254f2af..3e5231eb 100644 --- a/hooks/nova_cc_utils.py +++ b/hooks/nova_cc_utils.py @@ -283,7 +283,7 @@ def do_openstack_upgrade(configs): '--option', 'Dpkg::Options::=--force-confdef', ] - apt_upgrade(options=dpkg_opts, fatal=True) + apt_upgrade(options=dpkg_opts, fatal=True, dist=True) # set CONFIGS to load templates from new release and regenerate config configs.set_release(openstack_release=new_os_rel) From eaab552bca6a02ec1723f6568a3ba2a23f66c2dc Mon Sep 17 00:00:00 2001 From: James Page Date: Thu, 6 Mar 2014 16:09:47 +0000 Subject: [PATCH 34/98] Rework updates to deal with config file name changes --- hooks/nova_cc_utils.py | 19 +++++++++++++++++-- unit_tests/test_nova_cc_utils.py | 2 ++ 2 files changed, 19 insertions(+), 2 deletions(-) diff --git a/hooks/nova_cc_utils.py b/hooks/nova_cc_utils.py index 3e5231eb..5c906ce0 100644 --- a/hooks/nova_cc_utils.py +++ b/hooks/nova_cc_utils.py @@ -36,6 +36,11 @@ from charmhelpers.core.hookenv import ( ERROR, ) +from charmhelpers.core.host import ( + service_stop, + service_start +) + import nova_cc_context @@ -223,6 +228,14 @@ def restart_map(): if v['services']]) +def services(): + ''' Returns a list of services associate with this charm ''' + _services = [] + for v in restart_map().values(): + _services = _services + v + return list(set(_services)) + + def determine_ports(): '''Assemble a list of API ports for services we are managing''' ports = [] @@ -285,12 +298,14 @@ def do_openstack_upgrade(configs): apt_upgrade(options=dpkg_opts, fatal=True, dist=True) - # set CONFIGS to load templates from new release and regenerate config - configs.set_release(openstack_release=new_os_rel) + # Re-register all configs to accomodate changes in filesname etc + configs = register_configs() configs.write_all() + [service_stop(s) for s in services()] if eligible_leader(CLUSTER_RES): migrate_database() + [service_start(s) for s in services()] def volume_service(): diff --git a/unit_tests/test_nova_cc_utils.py b/unit_tests/test_nova_cc_utils.py index 034a13ae..8003ad67 100644 --- a/unit_tests/test_nova_cc_utils.py +++ b/unit_tests/test_nova_cc_utils.py @@ -22,6 +22,8 @@ TO_PATCH = [ 'relation_ids', 'remote_unit', '_save_script_rc', + 'service_stop', + 'service_start' ] SCRIPTRC_ENV_VARS = { From e1c6411ac396dfd8b1642f792c001e013d5e7bf7 Mon Sep 17 00:00:00 2001 From: James Page Date: Thu, 6 Mar 2014 16:26:40 +0000 Subject: [PATCH 35/98] Sort out passback of new release --- hooks/nova_cc_utils.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/hooks/nova_cc_utils.py b/hooks/nova_cc_utils.py index 5c906ce0..099af8cd 100644 --- a/hooks/nova_cc_utils.py +++ b/hooks/nova_cc_utils.py @@ -299,8 +299,9 @@ def do_openstack_upgrade(configs): apt_upgrade(options=dpkg_opts, fatal=True, dist=True) # Re-register all configs to accomodate changes in filesname etc - configs = register_configs() - configs.write_all() + configs.set_release(openstack_release=new_os_rel) + new_configs = register_configs() + new_configs.write_all() [service_stop(s) for s in services()] if eligible_leader(CLUSTER_RES): From 9ea07940085edfb104520470f83949f039278412 Mon Sep 17 00:00:00 2001 From: "yolanda.robla@canonical.com" <> Date: Fri, 7 Mar 2014 11:25:47 +0100 Subject: [PATCH 36/98] refreshed charmhelpers refreshed amqpcontext --- hooks/charmhelpers/contrib/openstack/context.py | 13 +++++++------ hooks/charmhelpers/fetch/__init__.py | 4 ++++ 2 files changed, 11 insertions(+), 6 deletions(-) diff --git a/hooks/charmhelpers/contrib/openstack/context.py b/hooks/charmhelpers/contrib/openstack/context.py index a2d4636b..865e4201 100644 --- a/hooks/charmhelpers/contrib/openstack/context.py +++ b/hooks/charmhelpers/contrib/openstack/context.py @@ -199,6 +199,7 @@ class AMQPContext(OSContextGenerator): ctxt = {} for rid in relation_ids('amqp'): + ha_vip_only = False for unit in related_units(rid): if relation_get('clustered', rid=rid, unit=unit): ctxt['clustered'] = True @@ -213,16 +214,16 @@ class AMQPContext(OSContextGenerator): unit=unit), 'rabbitmq_virtual_host': vhost, }) + if relation_get('ha_queues', rid=rid, unit=unit): + ctxt['rabbitmq_ha_queues'] = True + + ha_vip_only = (relation_get('ha-vip-only', rid=rid, unit=unit) == 'True') + if context_complete(ctxt): # Sufficient information found = break out! break # Used for active/active rabbitmq >= grizzly - if ('clustered' not in ctxt or relation_get('ha-vip-only') == 'True') and \ - len(related_units(rid)) > 1: - if relation_get('ha_queues'): - ctxt['rabbitmq_ha_queues'] = relation_get('ha_queues') - else: - ctxt['rabbitmq_ha_queues'] = False + if ('clustered' not in ctxt or ha_vip_only) and len(related_units(rid)) > 1: rabbitmq_hosts = [] for unit in related_units(rid): rabbitmq_hosts.append(relation_get('private-address', diff --git a/hooks/charmhelpers/fetch/__init__.py b/hooks/charmhelpers/fetch/__init__.py index 07bb707d..c05e0335 100644 --- a/hooks/charmhelpers/fetch/__init__.py +++ b/hooks/charmhelpers/fetch/__init__.py @@ -135,6 +135,10 @@ def apt_hold(packages, fatal=False): def add_source(source, key=None): + if source is None: + log('Source is not present. Skipping') + return + if (source.startswith('ppa:') or source.startswith('http') or source.startswith('deb ') or From 1b86c4fba2cec8bec34b013a1222a2efa7f731dd Mon Sep 17 00:00:00 2001 From: "yolanda.robla@canonical.com" <> Date: Wed, 12 Mar 2014 13:06:55 +0100 Subject: [PATCH 37/98] refreshed charmhelper --- hooks/charmhelpers/contrib/openstack/utils.py | 24 +++++++++---------- revision | 2 +- 2 files changed, 13 insertions(+), 13 deletions(-) diff --git a/hooks/charmhelpers/contrib/openstack/utils.py b/hooks/charmhelpers/contrib/openstack/utils.py index 56d04245..de95a4b6 100644 --- a/hooks/charmhelpers/contrib/openstack/utils.py +++ b/hooks/charmhelpers/contrib/openstack/utils.py @@ -420,19 +420,19 @@ def get_hostname(address, fqdn=True): Resolves hostname for given IP, or returns the input if it is already a hostname. """ - if not is_ip(address): - return address + if is_ip(address): + try: + import dns.reversename + except ImportError: + apt_install('python-dnspython') + import dns.reversename - try: - import dns.reversename - except ImportError: - apt_install('python-dnspython') - import dns.reversename - - rev = dns.reversename.from_address(address) - result = ns_query(rev) - if not result: - return None + rev = dns.reversename.from_address(address) + result = ns_query(rev) + if not result: + return None + else: + result = address if fqdn: # strip trailing . diff --git a/revision b/revision index 9346fabb..66953656 100644 --- a/revision +++ b/revision @@ -1 +1 @@ -314 +315 From 07e8140a563c86cf25d3be5466c1aafe2976c99c Mon Sep 17 00:00:00 2001 From: James Page Date: Fri, 14 Mar 2014 11:23:48 +0000 Subject: [PATCH 38/98] Use install and upgrade during upgrade --- hooks/nova_cc_utils.py | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/hooks/nova_cc_utils.py b/hooks/nova_cc_utils.py index 099af8cd..bb23cf3d 100644 --- a/hooks/nova_cc_utils.py +++ b/hooks/nova_cc_utils.py @@ -24,6 +24,7 @@ from charmhelpers.contrib.openstack.utils import ( from charmhelpers.fetch import ( apt_upgrade, apt_update, + apt_install, ) from charmhelpers.core.hookenv import ( @@ -289,19 +290,19 @@ def do_openstack_upgrade(configs): log('Performing OpenStack upgrade to %s.' % (new_os_rel)) configure_installation_source(new_src) - apt_update() - dpkg_opts = [ '--option', 'Dpkg::Options::=--force-confnew', '--option', 'Dpkg::Options::=--force-confdef', ] + apt_update(fatal=True) apt_upgrade(options=dpkg_opts, fatal=True, dist=True) + apt_install(determine_packages(), fatal=True) # Re-register all configs to accomodate changes in filesname etc configs.set_release(openstack_release=new_os_rel) - new_configs = register_configs() - new_configs.write_all() + configs.write_all() + # NOTE(jamespage) upgrades that change contexts needs to be resolved still [service_stop(s) for s in services()] if eligible_leader(CLUSTER_RES): From 1c04e49d4044b95d39f00bce8191421726c1877b Mon Sep 17 00:00:00 2001 From: James Page Date: Mon, 17 Mar 2014 11:44:21 +0000 Subject: [PATCH 39/98] Resync helpers --- charm-helpers.yaml | 2 +- hooks/charmhelpers/contrib/openstack/context.py | 8 +++++--- 2 files changed, 6 insertions(+), 4 deletions(-) diff --git a/charm-helpers.yaml b/charm-helpers.yaml index 35e5834f..efd994db 100644 --- a/charm-helpers.yaml +++ b/charm-helpers.yaml @@ -1,4 +1,4 @@ -branch: lp:charm-helpers +branch: lp:~openstack-charmers/charm-helpers/active-active destination: hooks/charmhelpers include: - core diff --git a/hooks/charmhelpers/contrib/openstack/context.py b/hooks/charmhelpers/contrib/openstack/context.py index 865e4201..6014ee1b 100644 --- a/hooks/charmhelpers/contrib/openstack/context.py +++ b/hooks/charmhelpers/contrib/openstack/context.py @@ -214,16 +214,18 @@ class AMQPContext(OSContextGenerator): unit=unit), 'rabbitmq_virtual_host': vhost, }) - if relation_get('ha_queues', rid=rid, unit=unit): + if relation_get('ha_queues', rid=rid, unit=unit) is not None: ctxt['rabbitmq_ha_queues'] = True - ha_vip_only = (relation_get('ha-vip-only', rid=rid, unit=unit) == 'True') + ha_vip_only = relation_get('ha-vip-only', + rid=rid, unit=unit) is not None if context_complete(ctxt): # Sufficient information found = break out! break # Used for active/active rabbitmq >= grizzly - if ('clustered' not in ctxt or ha_vip_only) and len(related_units(rid)) > 1: + if ('clustered' not in ctxt or ha_vip_only) \ + and len(related_units(rid)) > 1: rabbitmq_hosts = [] for unit in related_units(rid): rabbitmq_hosts.append(relation_get('private-address', From 70e61d50fd9d42e46f94e232d339c3843f205b10 Mon Sep 17 00:00:00 2001 From: "yolanda.robla@canonical.com" <> Date: Thu, 20 Mar 2014 10:31:25 +0100 Subject: [PATCH 40/98] added database type --- hooks/nova_cc_context.py | 11 +++++++++++ hooks/nova_cc_utils.py | 2 +- templates/essex/nova.conf | 2 +- templates/folsom/nova.conf | 2 +- templates/folsom/ovs_quantum_plugin.ini | 2 +- templates/havana/nvp.ini | 2 +- templates/havana/ovs_neutron_plugin.ini | 2 +- 7 files changed, 17 insertions(+), 6 deletions(-) diff --git a/hooks/nova_cc_context.py b/hooks/nova_cc_context.py index d6cafccc..59ddf4bc 100644 --- a/hooks/nova_cc_context.py +++ b/hooks/nova_cc_context.py @@ -163,3 +163,14 @@ class IdentityServiceContext(context.IdentityServiceContext): ctxt['service_port']) ctxt['keystone_ec2_url'] = ec2_tokens return ctxt + +class SharedDBContext(context.SharedDBContext): + + interfaces = ['shared-db'] + + def __call__(self): + # add database type + ctxt = super(SharedDBContext, self).__call__() + ctxt['database_type'] = 'mysql' + return ctxt + diff --git a/hooks/nova_cc_utils.py b/hooks/nova_cc_utils.py index 9eb15113..412e895e 100644 --- a/hooks/nova_cc_utils.py +++ b/hooks/nova_cc_utils.py @@ -84,7 +84,7 @@ BASE_RESOURCE_MAP = OrderedDict([ (NOVA_CONF, { 'services': BASE_SERVICES, 'contexts': [context.AMQPContext(), - context.SharedDBContext(relation_prefix='nova'), + nova_cc_context.SharedDBContext(relation_prefix='nova'), context.ImageServiceContext(), context.OSConfigFlagContext(), context.SubordinateConfigContext( diff --git a/templates/essex/nova.conf b/templates/essex/nova.conf index d0bb3725..bdb0dacd 100644 --- a/templates/essex/nova.conf +++ b/templates/essex/nova.conf @@ -16,7 +16,7 @@ --verbose --ec2_private_dns_show_ip {% if database_host -%} ---sql_connection=mysql://{{ database_user }}:{{ database_password }}@{{ database_host }}/{{ database }} +--sql_connection={{ database_type }}://{{ database_user }}:{{ database_password }}@{{ database_host }}/{{ database }} {% endif -%} {% if rabbitmq_host -%} --rabbit_host={{ rabbitmq_host }} diff --git a/templates/folsom/nova.conf b/templates/folsom/nova.conf index 7c620918..f41408eb 100644 --- a/templates/folsom/nova.conf +++ b/templates/folsom/nova.conf @@ -25,7 +25,7 @@ keystone_ec2_url = {{ keystone_ec2_url }} {% endif -%} {% if database_host -%} -sql_connection = mysql://{{ database_user }}:{{ database_password }}@{{ database_host }}/{{ database }} +sql_connection = {{ database_type }}://{{ database_user }}:{{ database_password }}@{{ database_host }}/{{ database }} {% endif -%} {% if rabbitmq_host -%} diff --git a/templates/folsom/ovs_quantum_plugin.ini b/templates/folsom/ovs_quantum_plugin.ini index a80cca3a..7c1c39c6 100644 --- a/templates/folsom/ovs_quantum_plugin.ini +++ b/templates/folsom/ovs_quantum_plugin.ini @@ -11,7 +11,7 @@ local_ip = {{ local_ip }} [DATABASE] {% if database_host -%} -sql_connection = mysql://{{ database_user }}:{{ database_password }}@{{ database_host }}/{{ database }}?quantum?charset=utf8 +sql_connection = {{ database_type }}://{{ database_user }}:{{ database_password }}@{{ database_host }}/{{ database }}?quantum?charset=utf8 reconnect_interval = 2 {% else -%} connection = sqlite:////var/lib/quantum/quantum.sqlite diff --git a/templates/havana/nvp.ini b/templates/havana/nvp.ini index a6225421..2876cc1d 100644 --- a/templates/havana/nvp.ini +++ b/templates/havana/nvp.ini @@ -12,7 +12,7 @@ default_l3_gw_service_uuid = {{ nvp_l3_uuid }} [database] {% if database_host -%} -connection = mysql://{{ database_user }}:{{ database_password }}@{{ database_host }}/{{ database }} +connection = {{ database_type }}://{{ database_user }}:{{ database_password }}@{{ database_host }}/{{ database }} {% else -%} connection = sqlite:////var/lib/neutron/neutron.sqlite {% endif -%} diff --git a/templates/havana/ovs_neutron_plugin.ini b/templates/havana/ovs_neutron_plugin.ini index 73e5a205..ca12b082 100644 --- a/templates/havana/ovs_neutron_plugin.ini +++ b/templates/havana/ovs_neutron_plugin.ini @@ -6,7 +6,7 @@ local_ip = {{ local_ip }} [database] {% if database_host -%} -connection = mysql://{{ database_user }}:{{ database_password }}@{{ database_host }}/{{ database }} +connection = {{ database_type }}://{{ database_user }}:{{ database_password }}@{{ database_host }}/{{ database }} {% else -%} connection = sqlite:////var/lib/neutron/neutron.sqlite {% endif -%} From 499930bcd8bf6e4b2af8952b479cf3fd71643357 Mon Sep 17 00:00:00 2001 From: "yolanda.robla@canonical.com" <> Date: Thu, 20 Mar 2014 11:18:23 +0100 Subject: [PATCH 41/98] added postgresql support --- .../charmhelpers/contrib/openstack/context.py | 29 ++++++++++ .../contrib/storage/linux/utils.py | 3 +- hooks/charmhelpers/fetch/__init__.py | 4 ++ hooks/nova_cc_context.py | 6 ++ hooks/nova_cc_hooks.py | 57 ++++++++++++++++++- hooks/nova_cc_utils.py | 2 + hooks/pgsql-neutron-db-relation-broken | 1 + hooks/pgsql-neutron-db-relation-changed | 1 + hooks/pgsql-neutron-db-relation-joined | 1 + hooks/pgsql-nova-db-relation-broken | 1 + hooks/pgsql-nova-db-relation-changed | 1 + hooks/pgsql-nova-db-relation-joined | 1 + metadata.yaml | 4 ++ revision | 2 +- 14 files changed, 110 insertions(+), 3 deletions(-) create mode 120000 hooks/pgsql-neutron-db-relation-broken create mode 120000 hooks/pgsql-neutron-db-relation-changed create mode 120000 hooks/pgsql-neutron-db-relation-joined create mode 120000 hooks/pgsql-nova-db-relation-broken create mode 120000 hooks/pgsql-nova-db-relation-changed create mode 120000 hooks/pgsql-nova-db-relation-joined diff --git a/hooks/charmhelpers/contrib/openstack/context.py b/hooks/charmhelpers/contrib/openstack/context.py index a2d4636b..6cc3c971 100644 --- a/hooks/charmhelpers/contrib/openstack/context.py +++ b/hooks/charmhelpers/contrib/openstack/context.py @@ -152,6 +152,35 @@ class SharedDBContext(OSContextGenerator): return {} +class PostgresqlDBContext(OSContextGenerator): + interfaces = ['pgsql-db'] + + def __init__(self, database=None): + self.database = database + + def __call__(self): + self.database = self.database or config('database') + if self.database is None: + log('Could not generate shared_db context. ' + 'Missing required charm config options. ' + '(database name)') + raise OSContextError + ctxt = {} + + for rid in relation_ids(self.interfaces[0]): + for unit in related_units(rid): + ctxt = { + 'database_host': relation_get('host', rid=rid, unit=unit), + 'database': self.database, + 'database_user': relation_get('user', rid=rid, unit=unit), + 'database_password': relation_get('password', rid=rid, unit=unit), + 'database_type': 'postgresql', + } + if context_complete(ctxt): + return ctxt + return {} + + class IdentityServiceContext(OSContextGenerator): interfaces = ['identity-service'] diff --git a/hooks/charmhelpers/contrib/storage/linux/utils.py b/hooks/charmhelpers/contrib/storage/linux/utils.py index c40218f0..5349c3ea 100644 --- a/hooks/charmhelpers/contrib/storage/linux/utils.py +++ b/hooks/charmhelpers/contrib/storage/linux/utils.py @@ -22,4 +22,5 @@ def zap_disk(block_device): :param block_device: str: Full path of block device to clean. ''' - check_call(['sgdisk', '--zap-all', '--mbrtogpt', block_device]) + check_call(['sgdisk', '--zap-all', '--clear', + '--mbrtogpt', block_device]) diff --git a/hooks/charmhelpers/fetch/__init__.py b/hooks/charmhelpers/fetch/__init__.py index 07bb707d..c05e0335 100644 --- a/hooks/charmhelpers/fetch/__init__.py +++ b/hooks/charmhelpers/fetch/__init__.py @@ -135,6 +135,10 @@ def apt_hold(packages, fatal=False): def add_source(source, key=None): + if source is None: + log('Source is not present. Skipping') + return + if (source.startswith('ppa:') or source.startswith('http') or source.startswith('deb ') or diff --git a/hooks/nova_cc_context.py b/hooks/nova_cc_context.py index 59ddf4bc..382a5f81 100644 --- a/hooks/nova_cc_context.py +++ b/hooks/nova_cc_context.py @@ -174,3 +174,9 @@ class SharedDBContext(context.SharedDBContext): ctxt['database_type'] = 'mysql' return ctxt +class NovaPostgresqlDBContext(context.PostgresqlDBContext): + interfaces = ['pgsql-nova-db'] + + +class NeutronPostgresqlDBContext(context.PostgresqlDBContext): + interfaces = ['pgsql-neutron-db'] diff --git a/hooks/nova_cc_hooks.py b/hooks/nova_cc_hooks.py index 6dfab01b..f4e2c40f 100755 --- a/hooks/nova_cc_hooks.py +++ b/hooks/nova_cc_hooks.py @@ -13,6 +13,7 @@ from charmhelpers.core.hookenv import ( UnregisteredHookError, config, charm_dir, + is_relation_made, log, relation_get, relation_ids, @@ -29,6 +30,8 @@ from charmhelpers.fetch import ( apt_install, apt_update ) +from charmhelpers.contrib.openstack import context + from charmhelpers.contrib.openstack.utils import ( configure_installation_source, openstack_upgrade_available, @@ -124,6 +127,12 @@ def amqp_changed(): @hooks.hook('shared-db-relation-joined') def db_joined(): + if is_relation_made('pgsql-nova-db') or is_relation_made('pgsql-neutron-db'): + # error, postgresql is used + e = ('Attempting to associate a mysql database when there is already ' + 'associated a postgresql one') + raise context.OSContextError(e) + relation_set(nova_database=config('database'), nova_username=config('database-user'), nova_hostname=unit_get('private-address')) @@ -134,6 +143,28 @@ def db_joined(): neutron_hostname=unit_get('private-address')) +@hooks.hook('pgsql-nova-db-relation-joined') +def pgsql_nova_db_joined(): + if is_relation_made('shared-db'): + # raise error + e = ('Attempting to associate a postgresql database when there is already ' + 'associated a mysql one') + raise context.OSContextError(e) + + relation_set(database=config('database')), + + +@hooks.hook('pgsql-neutron-db-relation-joined') +def pgsql_neutron_db_joined(): + if is_relation_made('shared-db'): + # raise error + e = ('Attempting to associate a postgresql database when there is already ' + 'associated a mysql one') + raise context.OSContextError(e) + + relation_set(database=config('neutron-database')), + + @hooks.hook('shared-db-relation-changed') @restart_on_change(restart_map()) def db_changed(): @@ -154,6 +185,28 @@ def db_changed(): for rid in relation_ids('cloud-compute')] +@hooks.hook('pgsql-nova-db-relation-changed') +@hooks.hook('pgsql-neutron-db-relation-changed') +@restart_on_change(restart_map()) +def db_changed(): + if 'pgsql-nova-db' not in CONFIGS.complete_contexts() or \ + 'pgsql-neutron-db' not in CONFIGS.complete_contexts(): + log('pgsql-*-db relation incomplete. Peer not ready?') + return + CONFIGS.write(NOVA_CONF) + + if network_manager() in ['neutron', 'quantum']: + plugin = neutron_plugin() + # DB config might have been moved to main neutron.conf in H? + CONFIGS.write(neutron_plugin_attribute(plugin, 'config')) + + if eligible_leader(CLUSTER_RES): + migrate_database() + log('Triggering remote cloud-compute restarts.') + [compute_joined(rid=rid, remote_restart=True) + for rid in relation_ids('cloud-compute')] + + @hooks.hook('image-service-relation-changed') @restart_on_change(restart_map()) def image_service_changed(): @@ -385,7 +438,9 @@ def ha_changed(): 'identity-service-relation-broken', 'image-service-relation-broken', 'nova-volume-service-relation-broken', - 'shared-db-relation-broken' + 'shared-db-relation-broken', + 'pgsql-nova-db-relation-broken', + 'pgsql-neutron-db-relation-broken', 'quantum-network-service-relation-broken') def relation_broken(): CONFIGS.write_all() diff --git a/hooks/nova_cc_utils.py b/hooks/nova_cc_utils.py index 412e895e..f5190ed1 100644 --- a/hooks/nova_cc_utils.py +++ b/hooks/nova_cc_utils.py @@ -85,6 +85,8 @@ BASE_RESOURCE_MAP = OrderedDict([ 'services': BASE_SERVICES, 'contexts': [context.AMQPContext(), nova_cc_context.SharedDBContext(relation_prefix='nova'), + nova_cc_context.NovaPostgresqlDBContext(), + nova_cc_context.NeutronPostgresqlDBContext(), context.ImageServiceContext(), context.OSConfigFlagContext(), context.SubordinateConfigContext( diff --git a/hooks/pgsql-neutron-db-relation-broken b/hooks/pgsql-neutron-db-relation-broken new file mode 120000 index 00000000..f6702415 --- /dev/null +++ b/hooks/pgsql-neutron-db-relation-broken @@ -0,0 +1 @@ +nova_cc_hooks.py \ No newline at end of file diff --git a/hooks/pgsql-neutron-db-relation-changed b/hooks/pgsql-neutron-db-relation-changed new file mode 120000 index 00000000..f6702415 --- /dev/null +++ b/hooks/pgsql-neutron-db-relation-changed @@ -0,0 +1 @@ +nova_cc_hooks.py \ No newline at end of file diff --git a/hooks/pgsql-neutron-db-relation-joined b/hooks/pgsql-neutron-db-relation-joined new file mode 120000 index 00000000..f6702415 --- /dev/null +++ b/hooks/pgsql-neutron-db-relation-joined @@ -0,0 +1 @@ +nova_cc_hooks.py \ No newline at end of file diff --git a/hooks/pgsql-nova-db-relation-broken b/hooks/pgsql-nova-db-relation-broken new file mode 120000 index 00000000..f6702415 --- /dev/null +++ b/hooks/pgsql-nova-db-relation-broken @@ -0,0 +1 @@ +nova_cc_hooks.py \ No newline at end of file diff --git a/hooks/pgsql-nova-db-relation-changed b/hooks/pgsql-nova-db-relation-changed new file mode 120000 index 00000000..f6702415 --- /dev/null +++ b/hooks/pgsql-nova-db-relation-changed @@ -0,0 +1 @@ +nova_cc_hooks.py \ No newline at end of file diff --git a/hooks/pgsql-nova-db-relation-joined b/hooks/pgsql-nova-db-relation-joined new file mode 120000 index 00000000..f6702415 --- /dev/null +++ b/hooks/pgsql-nova-db-relation-joined @@ -0,0 +1 @@ +nova_cc_hooks.py \ No newline at end of file diff --git a/metadata.yaml b/metadata.yaml index 366bdb75..5907e5d7 100644 --- a/metadata.yaml +++ b/metadata.yaml @@ -12,6 +12,10 @@ provides: requires: shared-db: interface: mysql-shared + pgsql-nova-db: + interface: pgsql + pgsql-neutron-db: + interface: pgsql amqp: interface: rabbitmq image-service: diff --git a/revision b/revision index 9346fabb..66953656 100644 --- a/revision +++ b/revision @@ -1 +1 @@ -314 +315 From f0214bc3c1dd83c234e954a50dda6a25710634f6 Mon Sep 17 00:00:00 2001 From: "yolanda.robla@canonical.com" <> Date: Thu, 20 Mar 2014 11:33:09 +0100 Subject: [PATCH 42/98] adding 2 interfaces to discriminate --- metadata.yaml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/metadata.yaml b/metadata.yaml index 5907e5d7..d8e523e0 100644 --- a/metadata.yaml +++ b/metadata.yaml @@ -9,6 +9,10 @@ categories: provides: cloud-controller: interface: nova + db: + interface: nova + neutrondb: + interface: nova requires: shared-db: interface: mysql-shared From a381d773f1a83dbfa0daa2b9e9d31bf33f24780d Mon Sep 17 00:00:00 2001 From: "yolanda.robla@canonical.com" <> Date: Thu, 20 Mar 2014 11:51:13 +0100 Subject: [PATCH 43/98] updating metadata --- metadata.yaml | 2 -- 1 file changed, 2 deletions(-) diff --git a/metadata.yaml b/metadata.yaml index d8e523e0..28003ae4 100644 --- a/metadata.yaml +++ b/metadata.yaml @@ -11,8 +11,6 @@ provides: interface: nova db: interface: nova - neutrondb: - interface: nova requires: shared-db: interface: mysql-shared From 890b904f99c8cb984b250abfc290ff57cb61b13b Mon Sep 17 00:00:00 2001 From: "yolanda.robla@canonical.com" <> Date: Thu, 20 Mar 2014 12:07:52 +0100 Subject: [PATCH 44/98] adding psycopg2 package --- hooks/nova_cc_utils.py | 1 + 1 file changed, 1 insertion(+) diff --git a/hooks/nova_cc_utils.py b/hooks/nova_cc_utils.py index f5190ed1..8b439201 100644 --- a/hooks/nova_cc_utils.py +++ b/hooks/nova_cc_utils.py @@ -49,6 +49,7 @@ BASE_PACKAGES = [ 'haproxy', 'python-keystoneclient', 'python-mysqldb', + 'python-psycopg2', 'uuid', ] From b279031051ad90756a97997bce872b0408bb3955 Mon Sep 17 00:00:00 2001 From: James Page Date: Thu, 20 Mar 2014 12:17:32 +0000 Subject: [PATCH 45/98] Fixup ssh_know_host_key to deal with non-zero exit codes --- hooks/nova_cc_utils.py | 5 ++++- templates/folsom/ovs_quantum_plugin.ini | 2 +- 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/hooks/nova_cc_utils.py b/hooks/nova_cc_utils.py index bb23cf3d..7d3b034f 100644 --- a/hooks/nova_cc_utils.py +++ b/hooks/nova_cc_utils.py @@ -377,7 +377,10 @@ def authorized_keys(user=None): def ssh_known_host_key(host, user=None): cmd = ['ssh-keygen', '-f', known_hosts(user), '-H', '-F', host] - return subprocess.check_output(cmd).strip() + try: + return subprocess.check_output(cmd).strip() + except subprocess.CalledProcessError: + return None def remove_known_host(host, user=None): diff --git a/templates/folsom/ovs_quantum_plugin.ini b/templates/folsom/ovs_quantum_plugin.ini index a80cca3a..1ff50088 100644 --- a/templates/folsom/ovs_quantum_plugin.ini +++ b/templates/folsom/ovs_quantum_plugin.ini @@ -11,7 +11,7 @@ local_ip = {{ local_ip }} [DATABASE] {% if database_host -%} -sql_connection = mysql://{{ database_user }}:{{ database_password }}@{{ database_host }}/{{ database }}?quantum?charset=utf8 +sql_connection = mysql://{{ database_user }}:{{ database_password }}@{{ database_host }}/{{ database }}?charset=utf8 reconnect_interval = 2 {% else -%} connection = sqlite:////var/lib/quantum/quantum.sqlite From 0a70e07aedbdffa8c04ea88eb33186f57232ac8d Mon Sep 17 00:00:00 2001 From: James Page Date: Thu, 20 Mar 2014 13:47:08 +0000 Subject: [PATCH 46/98] Resynced helpers --- .../charmhelpers/contrib/openstack/context.py | 26 ++++++++++++++++--- .../charmhelpers/contrib/openstack/neutron.py | 2 +- .../contrib/storage/linux/utils.py | 3 ++- hooks/charmhelpers/fetch/__init__.py | 4 +++ hooks/charmhelpers/fetch/archiveurl.py | 15 +++++++++++ 5 files changed, 45 insertions(+), 5 deletions(-) diff --git a/hooks/charmhelpers/contrib/openstack/context.py b/hooks/charmhelpers/contrib/openstack/context.py index 97658dc9..a2d4636b 100644 --- a/hooks/charmhelpers/contrib/openstack/context.py +++ b/hooks/charmhelpers/contrib/openstack/context.py @@ -29,6 +29,7 @@ from charmhelpers.contrib.hahelpers.cluster import ( determine_apache_port, determine_api_port, https, + is_clustered ) from charmhelpers.contrib.hahelpers.apache import ( @@ -240,10 +241,13 @@ class CephContext(OSContextGenerator): '''This generates context for /etc/ceph/ceph.conf templates''' if not relation_ids('ceph'): return {} + log('Generating template context for ceph') + mon_hosts = [] auth = None key = None + use_syslog = str(config('use-syslog')).lower() for rid in relation_ids('ceph'): for unit in related_units(rid): mon_hosts.append(relation_get('private-address', rid=rid, @@ -255,7 +259,7 @@ class CephContext(OSContextGenerator): 'mon_hosts': ' '.join(mon_hosts), 'auth': auth, 'key': key, - 'use_syslog': str(config('use-syslog')).lower() + 'use_syslog': use_syslog } if not os.path.isdir('/etc/ceph'): @@ -392,7 +396,7 @@ class ApacheSSLContext(OSContextGenerator): return ctxt -class NeutronContext(object): +class NeutronContext(OSContextGenerator): interfaces = [] @property @@ -453,6 +457,22 @@ class NeutronContext(object): return nvp_ctxt + def neutron_ctxt(self): + if https(): + proto = 'https' + else: + proto = 'http' + if is_clustered(): + host = config('vip') + else: + host = unit_get('private-address') + url = '%s://%s:%s' % (proto, host, '9696') + ctxt = { + 'network_manager': self.network_manager, + 'neutron_url': url, + } + return ctxt + def __call__(self): self._ensure_packages() @@ -462,7 +482,7 @@ class NeutronContext(object): if not self.plugin: return {} - ctxt = {'network_manager': self.network_manager} + ctxt = self.neutron_ctxt() if self.plugin == 'ovs': ctxt.update(self.ovs_ctxt()) diff --git a/hooks/charmhelpers/contrib/openstack/neutron.py b/hooks/charmhelpers/contrib/openstack/neutron.py index 91e55d36..5badc39c 100644 --- a/hooks/charmhelpers/contrib/openstack/neutron.py +++ b/hooks/charmhelpers/contrib/openstack/neutron.py @@ -20,7 +20,7 @@ def headers_package(): def kernel_version(): """ Retrieve the current major kernel version as a tuple e.g. (3, 13) """ - kver = check_output(['uname' , '-r']).strip() + kver = check_output(['uname', '-r']).strip() kver = kver.split('.') return (int(kver[0]), int(kver[1])) diff --git a/hooks/charmhelpers/contrib/storage/linux/utils.py b/hooks/charmhelpers/contrib/storage/linux/utils.py index c40218f0..5349c3ea 100644 --- a/hooks/charmhelpers/contrib/storage/linux/utils.py +++ b/hooks/charmhelpers/contrib/storage/linux/utils.py @@ -22,4 +22,5 @@ def zap_disk(block_device): :param block_device: str: Full path of block device to clean. ''' - check_call(['sgdisk', '--zap-all', '--mbrtogpt', block_device]) + check_call(['sgdisk', '--zap-all', '--clear', + '--mbrtogpt', block_device]) diff --git a/hooks/charmhelpers/fetch/__init__.py b/hooks/charmhelpers/fetch/__init__.py index bcc5b397..97a19912 100644 --- a/hooks/charmhelpers/fetch/__init__.py +++ b/hooks/charmhelpers/fetch/__init__.py @@ -158,6 +158,10 @@ def apt_hold(packages, fatal=False): def add_source(source, key=None): + if source is None: + log('Source is not present. Skipping') + return + if (source.startswith('ppa:') or source.startswith('http') or source.startswith('deb ') or diff --git a/hooks/charmhelpers/fetch/archiveurl.py b/hooks/charmhelpers/fetch/archiveurl.py index e35b8f15..87e7071a 100644 --- a/hooks/charmhelpers/fetch/archiveurl.py +++ b/hooks/charmhelpers/fetch/archiveurl.py @@ -1,5 +1,7 @@ import os import urllib2 +import urlparse + from charmhelpers.fetch import ( BaseFetchHandler, UnhandledSource @@ -24,6 +26,19 @@ class ArchiveUrlFetchHandler(BaseFetchHandler): def download(self, source, dest): # propogate all exceptions # URLError, OSError, etc + proto, netloc, path, params, query, fragment = urlparse.urlparse(source) + if proto in ('http', 'https'): + auth, barehost = urllib2.splituser(netloc) + if auth is not None: + source = urlparse.urlunparse((proto, barehost, path, params, query, fragment)) + username, password = urllib2.splitpasswd(auth) + passman = urllib2.HTTPPasswordMgrWithDefaultRealm() + # Realm is set to None in add_password to force the username and password + # to be used whatever the realm + passman.add_password(None, source, username, password) + authhandler = urllib2.HTTPBasicAuthHandler(passman) + opener = urllib2.build_opener(authhandler) + urllib2.install_opener(opener) response = urllib2.urlopen(source) try: with open(dest, 'w') as dest_file: From f8a29467b1eb12b83d2a92065120a49ba9379ba7 Mon Sep 17 00:00:00 2001 From: "yolanda.robla@canonical.com" <> Date: Thu, 20 Mar 2014 14:54:08 +0100 Subject: [PATCH 47/98] adding postgresql to neutron --- hooks/charmhelpers/contrib/openstack/context.py | 1 + hooks/nova_cc_context.py | 9 --------- hooks/nova_cc_utils.py | 8 +++++++- 3 files changed, 8 insertions(+), 10 deletions(-) diff --git a/hooks/charmhelpers/contrib/openstack/context.py b/hooks/charmhelpers/contrib/openstack/context.py index 6cc3c971..b6069259 100644 --- a/hooks/charmhelpers/contrib/openstack/context.py +++ b/hooks/charmhelpers/contrib/openstack/context.py @@ -146,6 +146,7 @@ class SharedDBContext(OSContextGenerator): 'database': self.database, 'database_user': self.user, 'database_password': passwd, + 'database_type': 'mysql' } if context_complete(ctxt): return ctxt diff --git a/hooks/nova_cc_context.py b/hooks/nova_cc_context.py index 382a5f81..3ce72f3f 100644 --- a/hooks/nova_cc_context.py +++ b/hooks/nova_cc_context.py @@ -164,15 +164,6 @@ class IdentityServiceContext(context.IdentityServiceContext): ctxt['keystone_ec2_url'] = ec2_tokens return ctxt -class SharedDBContext(context.SharedDBContext): - - interfaces = ['shared-db'] - - def __call__(self): - # add database type - ctxt = super(SharedDBContext, self).__call__() - ctxt['database_type'] = 'mysql' - return ctxt class NovaPostgresqlDBContext(context.PostgresqlDBContext): interfaces = ['pgsql-nova-db'] diff --git a/hooks/nova_cc_utils.py b/hooks/nova_cc_utils.py index 8b439201..0e06e028 100644 --- a/hooks/nova_cc_utils.py +++ b/hooks/nova_cc_utils.py @@ -29,6 +29,7 @@ from charmhelpers.fetch import ( from charmhelpers.core.hookenv import ( config, log, + is_relation_made, relation_get, relation_ids, remote_unit, @@ -85,7 +86,7 @@ BASE_RESOURCE_MAP = OrderedDict([ (NOVA_CONF, { 'services': BASE_SERVICES, 'contexts': [context.AMQPContext(), - nova_cc_context.SharedDBContext(relation_prefix='nova'), + context.SharedDBContext(relation_prefix='nova'), nova_cc_context.NovaPostgresqlDBContext(), nova_cc_context.NeutronPostgresqlDBContext(), context.ImageServiceContext(), @@ -195,6 +196,11 @@ def resource_map(): resource_map[conf]['contexts'].append( nova_cc_context.NeutronCCContext()) + # update for postgres + if is_relation_made('pgsql-neutron-db'): + resource_map[conf]['contexts'].append( + context.PostgresqlDBContext()) + # nova-conductor for releases >= G. if os_release('nova-common') not in ['essex', 'folsom']: resource_map['/etc/nova/nova.conf']['services'] += ['nova-conductor'] From d0811c15566e6605654a0b3c5f8d1301ced599d7 Mon Sep 17 00:00:00 2001 From: James Page Date: Thu, 20 Mar 2014 14:53:14 +0000 Subject: [PATCH 48/98] Update icehouse for active-active --- templates/icehouse/neutron.conf | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/templates/icehouse/neutron.conf b/templates/icehouse/neutron.conf index 5b293a31..726697a5 100644 --- a/templates/icehouse/neutron.conf +++ b/templates/icehouse/neutron.conf @@ -18,11 +18,19 @@ service_plugins = neutron.services.l3_router.l3_router_plugin.L3RouterPlugin,neu allow_overlapping_ips = True neutron_firewall_driver = neutron.agent.linux.iptables_firewall.OVSHybridIptablesFirewallDriver {% endif -%} -{% if rabbitmq_host -%} -rabbit_host = {{ rabbitmq_host }} +{% if rabbitmq_host or rabbitmq_hosts -%} rabbit_userid = {{ rabbitmq_user }} rabbit_password = {{ rabbitmq_password }} rabbit_virtual_host = {{ rabbitmq_virtual_host }} +{% if rabbitmq_hosts -%} +rabbit_hosts = {{ rabbitmq_hosts }} +{% if rabbitmq_ha_queues -%} +rabbit_ha_queues = true +rabbit_durable_queues = false +{% endif %} +{% else %} +rabbit_host = {{ rabbitmq_host }} +{% endif -%} {% endif -%} [quotas] From a7ca95766881e5b88234e9e3caba6cb5aa934877 Mon Sep 17 00:00:00 2001 From: James Page Date: Thu, 20 Mar 2014 15:23:07 +0000 Subject: [PATCH 49/98] Make rabbit config ha --- templates/havana/nova.conf | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/templates/havana/nova.conf b/templates/havana/nova.conf index e94f3df0..9725037c 100644 --- a/templates/havana/nova.conf +++ b/templates/havana/nova.conf @@ -28,11 +28,19 @@ keystone_ec2_url = {{ keystone_ec2_url }} sql_connection = mysql://{{ database_user }}:{{ database_password }}@{{ database_host }}/{{ database }} {% endif -%} -{% if rabbitmq_host -%} -rabbit_host = {{ rabbitmq_host }} +{% if rabbitmq_host or rabbitmq_hosts -%} rabbit_userid = {{ rabbitmq_user }} rabbit_password = {{ rabbitmq_password }} rabbit_virtual_host = {{ rabbitmq_virtual_host }} +{% if rabbitmq_hosts -%} +rabbit_hosts = {{ rabbitmq_hosts }} +{% if rabbitmq_ha_queues -%} +rabbit_ha_queues = true +rabbit_durable_queues = false +{% endif %} +{% else %} +rabbit_host = {{ rabbitmq_host }} +{% endif -%} {% endif -%} {% if glance_api_servers -%} From 36a0262ea3d7a17400c8d11e441d240485b028f2 Mon Sep 17 00:00:00 2001 From: James Page Date: Thu, 20 Mar 2014 15:26:44 +0000 Subject: [PATCH 50/98] Make sure neutron_url is set for >= havana --- templates/havana/nova.conf | 2 ++ 1 file changed, 2 insertions(+) diff --git a/templates/havana/nova.conf b/templates/havana/nova.conf index 9725037c..5b2f22cd 100644 --- a/templates/havana/nova.conf +++ b/templates/havana/nova.conf @@ -81,6 +81,7 @@ default_floating_pool = {{ external_network }} {% if network_manager and network_manager == 'quantum' -%} network_api_class = nova.network.quantumv2.api.API +quantum_url = {{ neutron_url }} {% if auth_host -%} quantum_auth_strategy = keystone quantum_admin_tenant_name = {{ admin_tenant_name }} @@ -90,6 +91,7 @@ quantum_admin_auth_url = {{ auth_protocol }}://{{ auth_host }}:{{ auth_port }}/v {% endif -%} {% elif network_manager and network_manager == 'neutron' -%} network_api_class = nova.network.neutronv2.api.API +neutron_url = {{ neutron_url }} {% if auth_host -%} neutron_auth_strategy = keystone neutron_admin_tenant_name = {{ admin_tenant_name }} From 3ba20e7e846b9b7d8b7cd929218b9432f2cbbac0 Mon Sep 17 00:00:00 2001 From: James Page Date: Thu, 20 Mar 2014 16:54:23 +0000 Subject: [PATCH 51/98] Tidy icehouse template --- templates/icehouse/neutron.conf | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/templates/icehouse/neutron.conf b/templates/icehouse/neutron.conf index 726697a5..5b3e1088 100644 --- a/templates/icehouse/neutron.conf +++ b/templates/icehouse/neutron.conf @@ -8,16 +8,25 @@ lock_path = $state_path/lock bind_host = 0.0.0.0 auth_strategy = keystone notification_driver = neutron.openstack.common.notifier.rpc_notifier + +{% if neutron_bind_port -%} +bind_port = {{ neutron_bind_port }} +{% else -%} +bind_port = 9696 +{% endif -%} + {% if core_plugin -%} core_plugin = {{ core_plugin }} {% if neutron_plugin in ['ovs', 'ml2'] -%} service_plugins = neutron.services.l3_router.l3_router_plugin.L3RouterPlugin,neutron.services.metering.metering_plugin.MeteringPlugin {% endif -%} {% endif -%} + {% if neutron_security_groups -%} allow_overlapping_ips = True neutron_firewall_driver = neutron.agent.linux.iptables_firewall.OVSHybridIptablesFirewallDriver {% endif -%} + {% if rabbitmq_host or rabbitmq_hosts -%} rabbit_userid = {{ rabbitmq_user }} rabbit_password = {{ rabbitmq_password }} From d9f165297cc203189691618010db5e24af711314 Mon Sep 17 00:00:00 2001 From: James Page Date: Thu, 20 Mar 2014 16:58:05 +0000 Subject: [PATCH 52/98] Add db context for neutron.conf --- hooks/nova_cc_utils.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/hooks/nova_cc_utils.py b/hooks/nova_cc_utils.py index 7d3b034f..2e3ef06a 100644 --- a/hooks/nova_cc_utils.py +++ b/hooks/nova_cc_utils.py @@ -125,6 +125,10 @@ BASE_RESOURCE_MAP = OrderedDict([ (NEUTRON_CONF, { 'services': ['neutron-server'], 'contexts': [context.AMQPContext(), + context.SharedDBContext( + user=config('neutron-database-user'), + database=config('neutron-database'), + relation_prefix='neutron'), nova_cc_context.IdentityServiceContext(), nova_cc_context.NeutronCCContext(), nova_cc_context.HAProxyContext()], From 72b2dbf352dd5c4fd8d8e57efb9fc5238410b220 Mon Sep 17 00:00:00 2001 From: "yolanda.robla@canonical.com" <> Date: Thu, 20 Mar 2014 18:58:12 +0100 Subject: [PATCH 53/98] only adding nova postgres db --- hooks/nova_cc_utils.py | 1 - 1 file changed, 1 deletion(-) diff --git a/hooks/nova_cc_utils.py b/hooks/nova_cc_utils.py index 0e06e028..2450ee2c 100644 --- a/hooks/nova_cc_utils.py +++ b/hooks/nova_cc_utils.py @@ -88,7 +88,6 @@ BASE_RESOURCE_MAP = OrderedDict([ 'contexts': [context.AMQPContext(), context.SharedDBContext(relation_prefix='nova'), nova_cc_context.NovaPostgresqlDBContext(), - nova_cc_context.NeutronPostgresqlDBContext(), context.ImageServiceContext(), context.OSConfigFlagContext(), context.SubordinateConfigContext( From 6cb64f567da782a3549b721ff2cda6cd95d1a8bb Mon Sep 17 00:00:00 2001 From: "yolanda.robla@canonical.com" <> Date: Thu, 20 Mar 2014 19:17:35 +0100 Subject: [PATCH 54/98] fixes in postgres for neutron --- hooks/nova_cc_utils.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/hooks/nova_cc_utils.py b/hooks/nova_cc_utils.py index 2450ee2c..31ffa2aa 100644 --- a/hooks/nova_cc_utils.py +++ b/hooks/nova_cc_utils.py @@ -196,9 +196,9 @@ def resource_map(): nova_cc_context.NeutronCCContext()) # update for postgres - if is_relation_made('pgsql-neutron-db'): - resource_map[conf]['contexts'].append( - context.PostgresqlDBContext()) + resource_map[conf]['contexts'].append( + nova_cc_context.NeutronPostgresqlDBContext()) + # nova-conductor for releases >= G. if os_release('nova-common') not in ['essex', 'folsom']: From b7b3a6a29dc09d7759e32e2d845f32a00cc3ae6a Mon Sep 17 00:00:00 2001 From: "yolanda.robla@canonical.com" <> Date: Mon, 24 Mar 2014 12:36:04 +0100 Subject: [PATCH 55/98] fixes in oscontext and in db relations --- hooks/nova_cc_hooks.py | 29 +++++++++++++++++------------ 1 file changed, 17 insertions(+), 12 deletions(-) diff --git a/hooks/nova_cc_hooks.py b/hooks/nova_cc_hooks.py index f4e2c40f..546300ab 100755 --- a/hooks/nova_cc_hooks.py +++ b/hooks/nova_cc_hooks.py @@ -131,7 +131,8 @@ def db_joined(): # error, postgresql is used e = ('Attempting to associate a mysql database when there is already ' 'associated a postgresql one') - raise context.OSContextError(e) + log(e, level=ERROR) + raise relation_set(nova_database=config('database'), nova_username=config('database-user'), @@ -149,7 +150,8 @@ def pgsql_nova_db_joined(): # raise error e = ('Attempting to associate a postgresql database when there is already ' 'associated a mysql one') - raise context.OSContextError(e) + log(e, level=ERROR) + raise relation_set(database=config('database')), @@ -160,7 +162,8 @@ def pgsql_neutron_db_joined(): # raise error e = ('Attempting to associate a postgresql database when there is already ' 'associated a mysql one') - raise context.OSContextError(e) + log(e, level=ERROR) + raise relation_set(database=config('neutron-database')), @@ -186,20 +189,13 @@ def db_changed(): @hooks.hook('pgsql-nova-db-relation-changed') -@hooks.hook('pgsql-neutron-db-relation-changed') @restart_on_change(restart_map()) -def db_changed(): - if 'pgsql-nova-db' not in CONFIGS.complete_contexts() or \ - 'pgsql-neutron-db' not in CONFIGS.complete_contexts(): +def postgresql_nova_db_changed(): + if 'pgsql-nova-db' not in CONFIGS.complete_contexts(): log('pgsql-*-db relation incomplete. Peer not ready?') return CONFIGS.write(NOVA_CONF) - if network_manager() in ['neutron', 'quantum']: - plugin = neutron_plugin() - # DB config might have been moved to main neutron.conf in H? - CONFIGS.write(neutron_plugin_attribute(plugin, 'config')) - if eligible_leader(CLUSTER_RES): migrate_database() log('Triggering remote cloud-compute restarts.') @@ -207,6 +203,15 @@ def db_changed(): for rid in relation_ids('cloud-compute')] +@hooks.hook('pgsql-neutron-db-relation-changed') +@restart_on_change(restart_map()) +def postgresql_neutron_db_changed(): + if network_manager() in ['neutron', 'quantum']: + plugin = neutron_plugin() + # DB config might have been moved to main neutron.conf in H? + CONFIGS.write(neutron_plugin_attribute(plugin, 'config')) + + @hooks.hook('image-service-relation-changed') @restart_on_change(restart_map()) def image_service_changed(): From ef3fabf1c863503f3c5521c9db0a02843ffa1162 Mon Sep 17 00:00:00 2001 From: "yolanda.robla@canonical.com" <> Date: Mon, 24 Mar 2014 13:01:12 +0100 Subject: [PATCH 56/98] raising generic exception --- hooks/nova_cc_hooks.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/hooks/nova_cc_hooks.py b/hooks/nova_cc_hooks.py index 546300ab..59719c80 100755 --- a/hooks/nova_cc_hooks.py +++ b/hooks/nova_cc_hooks.py @@ -15,6 +15,7 @@ from charmhelpers.core.hookenv import ( charm_dir, is_relation_made, log, + ERROR, relation_get, relation_ids, relation_set, @@ -132,7 +133,7 @@ def db_joined(): e = ('Attempting to associate a mysql database when there is already ' 'associated a postgresql one') log(e, level=ERROR) - raise + raise Exception(e) relation_set(nova_database=config('database'), nova_username=config('database-user'), @@ -151,7 +152,7 @@ def pgsql_nova_db_joined(): e = ('Attempting to associate a postgresql database when there is already ' 'associated a mysql one') log(e, level=ERROR) - raise + raise Exception(e) relation_set(database=config('database')), @@ -163,7 +164,7 @@ def pgsql_neutron_db_joined(): e = ('Attempting to associate a postgresql database when there is already ' 'associated a mysql one') log(e, level=ERROR) - raise + raise Exception(e) relation_set(database=config('neutron-database')), From 129bcd6e457ba2b7b8635d3943e455bb1d0e1446 Mon Sep 17 00:00:00 2001 From: "yolanda.robla@canonical.com" <> Date: Mon, 24 Mar 2014 15:31:59 +0100 Subject: [PATCH 57/98] fixed lint bugs resync charmhelpers --- hooks/charmhelpers/contrib/openstack/context.py | 4 ++-- .../contrib/openstack/templates/ceph.conf | 9 +++++---- hooks/charmhelpers/fetch/archiveurl.py | 15 +++++++++++++++ hooks/nova_cc_hooks.py | 5 +---- 4 files changed, 23 insertions(+), 10 deletions(-) diff --git a/hooks/charmhelpers/contrib/openstack/context.py b/hooks/charmhelpers/contrib/openstack/context.py index b6069259..d547a4fb 100644 --- a/hooks/charmhelpers/contrib/openstack/context.py +++ b/hooks/charmhelpers/contrib/openstack/context.py @@ -146,7 +146,7 @@ class SharedDBContext(OSContextGenerator): 'database': self.database, 'database_user': self.user, 'database_password': passwd, - 'database_type': 'mysql' + 'database_type': 'mysql', } if context_complete(ctxt): return ctxt @@ -162,7 +162,7 @@ class PostgresqlDBContext(OSContextGenerator): def __call__(self): self.database = self.database or config('database') if self.database is None: - log('Could not generate shared_db context. ' + log('Could not generate postgresql_db context. ' 'Missing required charm config options. ' '(database name)') raise OSContextError diff --git a/hooks/charmhelpers/contrib/openstack/templates/ceph.conf b/hooks/charmhelpers/contrib/openstack/templates/ceph.conf index 874ad2bd..81a9719f 100644 --- a/hooks/charmhelpers/contrib/openstack/templates/ceph.conf +++ b/hooks/charmhelpers/contrib/openstack/templates/ceph.conf @@ -3,12 +3,13 @@ # cinder configuration file maintained by Juju # local changes may be overwritten. ############################################################################### -{% if auth -%} [global] +{% if auth -%} auth_supported = {{ auth }} keyring = /etc/ceph/$cluster.$name.keyring mon host = {{ mon_hosts }} {% endif -%} -log to syslog = {{ use_syslog }} -err to syslog = {{ use_syslog }} -clog to syslog = {{ use_syslog }} + log to syslog = {{ use_syslog }} + err to syslog = {{ use_syslog }} + clog to syslog = {{ use_syslog }} + diff --git a/hooks/charmhelpers/fetch/archiveurl.py b/hooks/charmhelpers/fetch/archiveurl.py index e35b8f15..87e7071a 100644 --- a/hooks/charmhelpers/fetch/archiveurl.py +++ b/hooks/charmhelpers/fetch/archiveurl.py @@ -1,5 +1,7 @@ import os import urllib2 +import urlparse + from charmhelpers.fetch import ( BaseFetchHandler, UnhandledSource @@ -24,6 +26,19 @@ class ArchiveUrlFetchHandler(BaseFetchHandler): def download(self, source, dest): # propogate all exceptions # URLError, OSError, etc + proto, netloc, path, params, query, fragment = urlparse.urlparse(source) + if proto in ('http', 'https'): + auth, barehost = urllib2.splituser(netloc) + if auth is not None: + source = urlparse.urlunparse((proto, barehost, path, params, query, fragment)) + username, password = urllib2.splitpasswd(auth) + passman = urllib2.HTTPPasswordMgrWithDefaultRealm() + # Realm is set to None in add_password to force the username and password + # to be used whatever the realm + passman.add_password(None, source, username, password) + authhandler = urllib2.HTTPBasicAuthHandler(passman) + opener = urllib2.build_opener(authhandler) + urllib2.install_opener(opener) response = urllib2.urlopen(source) try: with open(dest, 'w') as dest_file: diff --git a/hooks/nova_cc_hooks.py b/hooks/nova_cc_hooks.py index 59719c80..cc2c8de1 100755 --- a/hooks/nova_cc_hooks.py +++ b/hooks/nova_cc_hooks.py @@ -31,8 +31,6 @@ from charmhelpers.fetch import ( apt_install, apt_update ) -from charmhelpers.contrib.openstack import context - from charmhelpers.contrib.openstack.utils import ( configure_installation_source, openstack_upgrade_available, @@ -134,7 +132,7 @@ def db_joined(): 'associated a postgresql one') log(e, level=ERROR) raise Exception(e) - + relation_set(nova_database=config('database'), nova_username=config('database-user'), nova_hostname=unit_get('private-address')) @@ -203,7 +201,6 @@ def postgresql_nova_db_changed(): [compute_joined(rid=rid, remote_restart=True) for rid in relation_ids('cloud-compute')] - @hooks.hook('pgsql-neutron-db-relation-changed') @restart_on_change(restart_map()) def postgresql_neutron_db_changed(): From ceda7d6a7048900429b9ecb719597d8ce9c83744 Mon Sep 17 00:00:00 2001 From: "yolanda.robla@canonical.com" <> Date: Tue, 25 Mar 2014 09:50:53 +0100 Subject: [PATCH 58/98] changes in templates to support database type --- hooks/nova_cc_utils.py | 8 +++----- templates/grizzly/nova.conf | 2 +- templates/havana/nova.conf | 2 +- templates/icehouse/neutron.conf | 2 +- 4 files changed, 6 insertions(+), 8 deletions(-) diff --git a/hooks/nova_cc_utils.py b/hooks/nova_cc_utils.py index 77b8ef73..dba3a586 100644 --- a/hooks/nova_cc_utils.py +++ b/hooks/nova_cc_utils.py @@ -30,7 +30,6 @@ from charmhelpers.fetch import ( from charmhelpers.core.hookenv import ( config, log, - is_relation_made, relation_get, relation_ids, remote_unit, @@ -129,9 +128,9 @@ BASE_RESOURCE_MAP = OrderedDict([ 'services': ['neutron-server'], 'contexts': [context.AMQPContext(), context.SharedDBContext( - user=config('neutron-database-user'), - database=config('neutron-database'), - relation_prefix='neutron'), + user=config('neutron-database-user'), + database=config('neutron-database'), + relation_prefix='neutron'), nova_cc_context.IdentityServiceContext(), nova_cc_context.NeutronCCContext(), nova_cc_context.HAProxyContext()], @@ -209,7 +208,6 @@ def resource_map(): resource_map[conf]['contexts'].append( nova_cc_context.NeutronPostgresqlDBContext()) - # nova-conductor for releases >= G. if os_release('nova-common') not in ['essex', 'folsom']: resource_map['/etc/nova/nova.conf']['services'] += ['nova-conductor'] diff --git a/templates/grizzly/nova.conf b/templates/grizzly/nova.conf index 007e4f0e..7bf5c9bb 100644 --- a/templates/grizzly/nova.conf +++ b/templates/grizzly/nova.conf @@ -25,7 +25,7 @@ keystone_ec2_url = {{ keystone_ec2_url }} {% endif -%} {% if database_host -%} -sql_connection = mysql://{{ database_user }}:{{ database_password }}@{{ database_host }}/{{ database }} +sql_connection = {{database_type}}://{{ database_user }}:{{ database_password }}@{{ database_host }}/{{ database }} {% endif -%} {% if rabbitmq_host or rabbitmq_hosts -%} diff --git a/templates/havana/nova.conf b/templates/havana/nova.conf index 5b2f22cd..d252516c 100644 --- a/templates/havana/nova.conf +++ b/templates/havana/nova.conf @@ -25,7 +25,7 @@ keystone_ec2_url = {{ keystone_ec2_url }} {% endif -%} {% if database_host -%} -sql_connection = mysql://{{ database_user }}:{{ database_password }}@{{ database_host }}/{{ database }} +sql_connection = {{database_type}}://{{ database_user }}:{{ database_password }}@{{ database_host }}/{{ database }} {% endif -%} {% if rabbitmq_host or rabbitmq_hosts -%} diff --git a/templates/icehouse/neutron.conf b/templates/icehouse/neutron.conf index 5b3e1088..ff60656f 100644 --- a/templates/icehouse/neutron.conf +++ b/templates/icehouse/neutron.conf @@ -67,7 +67,7 @@ admin_password = {{ admin_password }} [database] {% if database_host -%} -connection = mysql://{{ database_user }}:{{ database_password }}@{{ database_host }}/{{ database }} +connection = {{database_type}}://{{ database_user }}:{{ database_password }}@{{ database_host }}/{{ database }} {% else -%} connection = sqlite:////var/lib/neutron/neutron.sqlite {% endif -%} From abbe9589ceff3aa792fbe416ea116bba567d22ad Mon Sep 17 00:00:00 2001 From: "yolanda.robla@canonical.com" <> Date: Tue, 25 Mar 2014 10:00:50 +0100 Subject: [PATCH 59/98] typo --- hooks/nova_cc_hooks.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/hooks/nova_cc_hooks.py b/hooks/nova_cc_hooks.py index 745f3087..24c1a72e 100755 --- a/hooks/nova_cc_hooks.py +++ b/hooks/nova_cc_hooks.py @@ -153,7 +153,7 @@ def pgsql_nova_db_joined(): log(e, level=ERROR) raise Exception(e) - relation_set(database=config('database')), + relation_set(database=config('database')) @hooks.hook('pgsql-neutron-db-relation-joined') @@ -165,7 +165,7 @@ def pgsql_neutron_db_joined(): log(e, level=ERROR) raise Exception(e) - relation_set(database=config('neutron-database')), + relation_set(database=config('neutron-database')) @hooks.hook('shared-db-relation-changed') From 5f48dbbc1f53249c60c84d4c6cd4696a9bfbaf74 Mon Sep 17 00:00:00 2001 From: "yolanda.robla@canonical.com" <> Date: Tue, 25 Mar 2014 10:11:04 +0100 Subject: [PATCH 60/98] pushed initial readme --- README.txt | 15 +++++++++++++++ 1 file changed, 15 insertions(+) create mode 100644 README.txt diff --git a/README.txt b/README.txt new file mode 100644 index 00000000..984859ad --- /dev/null +++ b/README.txt @@ -0,0 +1,15 @@ +===================== +nova-cloud-controller +===================== + +Cloud controller node for Openstack nova. Contains nova-schedule, nova-api, nova-network and nova-objectstore. + +****************************************************** +Special considerations to be deployed using Postgresql +****************************************************** + +juju deploy nova-cloud-controller +juju deploy postgresql + +juju add-relation "nova-cloud-controller:pgsql-nova-db" "postgresql:db" +juju add-relation "nova-cloud-controller:pgsql-neutron-db" "postgresql:db" From 4e35f67581c944ead7b0c1dbd6931942228e3203 Mon Sep 17 00:00:00 2001 From: James Page Date: Tue, 25 Mar 2014 09:34:20 +0000 Subject: [PATCH 61/98] Drop compute v3 registration --- hooks/nova_cc_utils.py | 11 ----------- 1 file changed, 11 deletions(-) diff --git a/hooks/nova_cc_utils.py b/hooks/nova_cc_utils.py index 2e3ef06a..32b7a4e7 100644 --- a/hooks/nova_cc_utils.py +++ b/hooks/nova_cc_utils.py @@ -488,8 +488,6 @@ def determine_endpoints(url): else: nova_url = ('%s:%s/v1.1/$(tenant_id)s' % (url, api_port('nova-api-os-compute'))) - novav3_url = ('%s:%s/v3' % - (url, api_port('nova-api-os-compute'))) ec2_url = '%s:%s/services/Cloud' % (url, api_port('nova-api-ec2')) nova_volume_url = ('%s:%s/v1/$(tenant_id)s' % (url, api_port('nova-api-os-compute'))) @@ -534,15 +532,6 @@ def determine_endpoints(url): 'quantum_internal_url': neutron_url, }) - if os_rel >= 'havana': - endpoints.update({ - 'novav3_service': 'novav3', - 'novav3_region': region, - 'novav3_public_url': novav3_url, - 'novav3_admin_url': novav3_url, - 'novav3_internal_url': novav3_url - }) - return endpoints From 75d80b0f15dbffad9e3f892df4c69ea76debc658 Mon Sep 17 00:00:00 2001 From: James Page Date: Tue, 25 Mar 2014 17:04:30 +0000 Subject: [PATCH 62/98] Resynced helpers --- charm-helpers.yaml | 2 +- .../charmhelpers/contrib/openstack/context.py | 15 +++++++----- .../contrib/openstack/templates/ceph.conf | 9 +++---- hooks/charmhelpers/contrib/openstack/utils.py | 24 +++++++++---------- 4 files changed, 27 insertions(+), 23 deletions(-) diff --git a/charm-helpers.yaml b/charm-helpers.yaml index 64dd7f89..35e5834f 100644 --- a/charm-helpers.yaml +++ b/charm-helpers.yaml @@ -1,4 +1,4 @@ -branch: lp:~openstack-charmers/charm-helpers/icehouse +branch: lp:charm-helpers destination: hooks/charmhelpers include: - core diff --git a/hooks/charmhelpers/contrib/openstack/context.py b/hooks/charmhelpers/contrib/openstack/context.py index a2d4636b..6014ee1b 100644 --- a/hooks/charmhelpers/contrib/openstack/context.py +++ b/hooks/charmhelpers/contrib/openstack/context.py @@ -199,6 +199,7 @@ class AMQPContext(OSContextGenerator): ctxt = {} for rid in relation_ids('amqp'): + ha_vip_only = False for unit in related_units(rid): if relation_get('clustered', rid=rid, unit=unit): ctxt['clustered'] = True @@ -213,16 +214,18 @@ class AMQPContext(OSContextGenerator): unit=unit), 'rabbitmq_virtual_host': vhost, }) + if relation_get('ha_queues', rid=rid, unit=unit) is not None: + ctxt['rabbitmq_ha_queues'] = True + + ha_vip_only = relation_get('ha-vip-only', + rid=rid, unit=unit) is not None + if context_complete(ctxt): # Sufficient information found = break out! break # Used for active/active rabbitmq >= grizzly - if ('clustered' not in ctxt or relation_get('ha-vip-only') == 'True') and \ - len(related_units(rid)) > 1: - if relation_get('ha_queues'): - ctxt['rabbitmq_ha_queues'] = relation_get('ha_queues') - else: - ctxt['rabbitmq_ha_queues'] = False + if ('clustered' not in ctxt or ha_vip_only) \ + and len(related_units(rid)) > 1: rabbitmq_hosts = [] for unit in related_units(rid): rabbitmq_hosts.append(relation_get('private-address', diff --git a/hooks/charmhelpers/contrib/openstack/templates/ceph.conf b/hooks/charmhelpers/contrib/openstack/templates/ceph.conf index 874ad2bd..81a9719f 100644 --- a/hooks/charmhelpers/contrib/openstack/templates/ceph.conf +++ b/hooks/charmhelpers/contrib/openstack/templates/ceph.conf @@ -3,12 +3,13 @@ # cinder configuration file maintained by Juju # local changes may be overwritten. ############################################################################### -{% if auth -%} [global] +{% if auth -%} auth_supported = {{ auth }} keyring = /etc/ceph/$cluster.$name.keyring mon host = {{ mon_hosts }} {% endif -%} -log to syslog = {{ use_syslog }} -err to syslog = {{ use_syslog }} -clog to syslog = {{ use_syslog }} + log to syslog = {{ use_syslog }} + err to syslog = {{ use_syslog }} + clog to syslog = {{ use_syslog }} + diff --git a/hooks/charmhelpers/contrib/openstack/utils.py b/hooks/charmhelpers/contrib/openstack/utils.py index f1e60b95..3261adf4 100644 --- a/hooks/charmhelpers/contrib/openstack/utils.py +++ b/hooks/charmhelpers/contrib/openstack/utils.py @@ -423,19 +423,19 @@ def get_hostname(address, fqdn=True): Resolves hostname for given IP, or returns the input if it is already a hostname. """ - if not is_ip(address): - return address + if is_ip(address): + try: + import dns.reversename + except ImportError: + apt_install('python-dnspython') + import dns.reversename - try: - import dns.reversename - except ImportError: - apt_install('python-dnspython') - import dns.reversename - - rev = dns.reversename.from_address(address) - result = ns_query(rev) - if not result: - return None + rev = dns.reversename.from_address(address) + result = ns_query(rev) + if not result: + return None + else: + result = address if fqdn: # strip trailing . From 3189a1800e445df40c39795aa16f36c1bc07c757 Mon Sep 17 00:00:00 2001 From: James Page Date: Thu, 27 Mar 2014 10:20:18 +0000 Subject: [PATCH 63/98] Resynced helpers --- .../charmhelpers/contrib/hahelpers/apache.py | 17 ++-- .../charmhelpers/contrib/openstack/context.py | 89 ++++++++++++++----- .../charmhelpers/contrib/openstack/neutron.py | 18 +++- 3 files changed, 91 insertions(+), 33 deletions(-) diff --git a/hooks/charmhelpers/contrib/hahelpers/apache.py b/hooks/charmhelpers/contrib/hahelpers/apache.py index 3208a85c..8d5fb8ba 100644 --- a/hooks/charmhelpers/contrib/hahelpers/apache.py +++ b/hooks/charmhelpers/contrib/hahelpers/apache.py @@ -39,14 +39,15 @@ def get_cert(): def get_ca_cert(): - ca_cert = None - log("Inspecting identity-service relations for CA SSL certificate.", - level=INFO) - for r_id in relation_ids('identity-service'): - for unit in relation_list(r_id): - if not ca_cert: - ca_cert = relation_get('ca_cert', - rid=r_id, unit=unit) + ca_cert = config_get('ssl_ca') + if ca_cert is None: + log("Inspecting identity-service relations for CA SSL certificate.", + level=INFO) + for r_id in relation_ids('identity-service'): + for unit in relation_list(r_id): + if ca_cert is None: + ca_cert = relation_get('ca_cert', + rid=r_id, unit=unit) return ca_cert diff --git a/hooks/charmhelpers/contrib/openstack/context.py b/hooks/charmhelpers/contrib/openstack/context.py index 6014ee1b..d254de18 100644 --- a/hooks/charmhelpers/contrib/openstack/context.py +++ b/hooks/charmhelpers/contrib/openstack/context.py @@ -1,5 +1,6 @@ import json import os +import time from base64 import b64decode @@ -113,7 +114,8 @@ class OSContextGenerator(object): class SharedDBContext(OSContextGenerator): interfaces = ['shared-db'] - def __init__(self, database=None, user=None, relation_prefix=None): + def __init__(self, + database=None, user=None, relation_prefix=None, ssl_dir=None): ''' Allows inspecting relation for settings prefixed with relation_prefix. This is useful for parsing access for multiple databases returned via @@ -122,6 +124,7 @@ class SharedDBContext(OSContextGenerator): self.relation_prefix = relation_prefix self.database = database self.user = user + self.ssl_dir = ssl_dir def __call__(self): self.database = self.database or config('database') @@ -139,19 +142,44 @@ class SharedDBContext(OSContextGenerator): for rid in relation_ids('shared-db'): for unit in related_units(rid): - passwd = relation_get(password_setting, rid=rid, unit=unit) + rdata = relation_get(rid=rid, unit=unit) ctxt = { - 'database_host': relation_get('db_host', rid=rid, - unit=unit), + 'database_host': rdata.get('db_host'), 'database': self.database, 'database_user': self.user, - 'database_password': passwd, + 'database_password': rdata.get(password_setting) } if context_complete(ctxt): + db_ssl(rdata, ctxt, self.ssl_dir) return ctxt return {} +def db_ssl(rdata, ctxt, ssl_dir): + if 'ssl_ca' in rdata and ssl_dir: + ca_path = os.path.join(ssl_dir, 'db-client.ca') + with open(ca_path, 'w') as fh: + fh.write(b64decode(rdata['ssl_ca'])) + ctxt['database_ssl_ca'] = ca_path + elif 'ssl_ca' in rdata: + log("Charm not setup for ssl support but ssl ca found") + return ctxt + if 'ssl_cert' in rdata: + cert_path = os.path.join( + ssl_dir, 'db-client.cert') + if not os.path.exists(cert_path): + log("Waiting 1m for ssl client cert validity") + time.sleep(60) + with open(cert_path, 'w') as fh: + fh.write(b64decode(rdata['ssl_cert'])) + ctxt['database_ssl_cert'] = cert_path + key_path = os.path.join(ssl_dir, 'db-client.key') + with open(key_path, 'w') as fh: + fh.write(b64decode(rdata['ssl_key'])) + ctxt['database_ssl_key'] = key_path + return ctxt + + class IdentityServiceContext(OSContextGenerator): interfaces = ['identity-service'] @@ -161,22 +189,19 @@ class IdentityServiceContext(OSContextGenerator): for rid in relation_ids('identity-service'): for unit in related_units(rid): + rdata = relation_get(rid=rid, unit=unit) ctxt = { - 'service_port': relation_get('service_port', rid=rid, - unit=unit), - 'service_host': relation_get('service_host', rid=rid, - unit=unit), - 'auth_host': relation_get('auth_host', rid=rid, unit=unit), - 'auth_port': relation_get('auth_port', rid=rid, unit=unit), - 'admin_tenant_name': relation_get('service_tenant', - rid=rid, unit=unit), - 'admin_user': relation_get('service_username', rid=rid, - unit=unit), - 'admin_password': relation_get('service_password', rid=rid, - unit=unit), - # XXX: Hard-coded http. - 'service_protocol': 'http', - 'auth_protocol': 'http', + 'service_port': rdata.get('service_port'), + 'service_host': rdata.get('service_host'), + 'auth_host': rdata.get('auth_host'), + 'auth_port': rdata.get('auth_port'), + 'admin_tenant_name': rdata.get('service_tenant'), + 'admin_user': rdata.get('service_username'), + 'admin_password': rdata.get('service_password'), + 'service_protocol': + rdata.get('service_protocol') or 'http', + 'auth_protocol': + rdata.get('auth_protocol') or 'http', } if context_complete(ctxt): return ctxt @@ -186,6 +211,9 @@ class IdentityServiceContext(OSContextGenerator): class AMQPContext(OSContextGenerator): interfaces = ['amqp'] + def __init__(self, ssl_dir=None): + self.ssl_dir = ssl_dir + def __call__(self): log('Generating template context for amqp') conf = config() @@ -196,7 +224,6 @@ class AMQPContext(OSContextGenerator): log('Could not generate shared_db context. ' 'Missing required charm config options: %s.' % e) raise OSContextError - ctxt = {} for rid in relation_ids('amqp'): ha_vip_only = False @@ -214,6 +241,14 @@ class AMQPContext(OSContextGenerator): unit=unit), 'rabbitmq_virtual_host': vhost, }) + + ssl_port = relation_get('ssl_port', rid=rid, unit=unit) + if ssl_port: + ctxt['rabbit_ssl_port'] = ssl_port + ssl_ca = relation_get('ssl_ca', rid=rid, unit=unit) + if ssl_ca: + ctxt['rabbit_ssl_ca'] = ssl_ca + if relation_get('ha_queues', rid=rid, unit=unit) is not None: ctxt['rabbitmq_ha_queues'] = True @@ -221,6 +256,16 @@ class AMQPContext(OSContextGenerator): rid=rid, unit=unit) is not None if context_complete(ctxt): + if 'rabbit_ssl_ca' in ctxt: + if not self.ssl_dir: + log(("Charm not setup for ssl support " + "but ssl ca found")) + break + ca_path = os.path.join( + self.ssl_dir, 'rabbit-client-ca.pem') + with open(ca_path, 'w') as fh: + fh.write(b64decode(ctxt['rabbit_ssl_ca'])) + ctxt['rabbit_ssl_ca'] = ca_path # Sufficient information found = break out! break # Used for active/active rabbitmq >= grizzly @@ -391,6 +436,8 @@ class ApacheSSLContext(OSContextGenerator): 'private_address': unit_get('private-address'), 'endpoints': [] } + if is_clustered(): + ctxt['private_address'] = config('vip') for api_port in self.external_ports: ext_port = determine_apache_port(api_port) int_port = determine_api_port(api_port) diff --git a/hooks/charmhelpers/contrib/openstack/neutron.py b/hooks/charmhelpers/contrib/openstack/neutron.py index 5badc39c..563e804d 100644 --- a/hooks/charmhelpers/contrib/openstack/neutron.py +++ b/hooks/charmhelpers/contrib/openstack/neutron.py @@ -17,6 +17,8 @@ def headers_package(): kver = check_output(['uname', '-r']).strip() return 'linux-headers-%s' % kver +QUANTUM_CONF_DIR = '/etc/quantum' + def kernel_version(): """ Retrieve the current major kernel version as a tuple e.g. (3, 13) """ @@ -35,6 +37,8 @@ def determine_dkms_package(): # legacy + + def quantum_plugins(): from charmhelpers.contrib.openstack import context return { @@ -46,7 +50,8 @@ def quantum_plugins(): 'contexts': [ context.SharedDBContext(user=config('neutron-database-user'), database=config('neutron-database'), - relation_prefix='neutron')], + relation_prefix='neutron', + ssl_dir=QUANTUM_CONF_DIR)], 'services': ['quantum-plugin-openvswitch-agent'], 'packages': [[headers_package()] + determine_dkms_package(), ['quantum-plugin-openvswitch-agent']], @@ -61,7 +66,8 @@ def quantum_plugins(): 'contexts': [ context.SharedDBContext(user=config('neutron-database-user'), database=config('neutron-database'), - relation_prefix='neutron')], + relation_prefix='neutron', + ssl_dir=QUANTUM_CONF_DIR)], 'services': [], 'packages': [], 'server_packages': ['quantum-server', @@ -70,6 +76,8 @@ def quantum_plugins(): } } +NEUTRON_CONF_DIR = '/etc/neutron' + def neutron_plugins(): from charmhelpers.contrib.openstack import context @@ -83,7 +91,8 @@ def neutron_plugins(): 'contexts': [ context.SharedDBContext(user=config('neutron-database-user'), database=config('neutron-database'), - relation_prefix='neutron')], + relation_prefix='neutron', + ssl_dir=NEUTRON_CONF_DIR)], 'services': ['neutron-plugin-openvswitch-agent'], 'packages': [[headers_package()] + determine_dkms_package(), ['neutron-plugin-openvswitch-agent']], @@ -98,7 +107,8 @@ def neutron_plugins(): 'contexts': [ context.SharedDBContext(user=config('neutron-database-user'), database=config('neutron-database'), - relation_prefix='neutron')], + relation_prefix='neutron', + ssl_dir=NEUTRON_CONF_DIR)], 'services': [], 'packages': [], 'server_packages': ['neutron-server', From af49e5c8a0e86e02196d3d29224834eaa4a3f627 Mon Sep 17 00:00:00 2001 From: James Page Date: Thu, 27 Mar 2014 11:12:27 +0000 Subject: [PATCH 64/98] Restore SSL config for rabbit --- templates/grizzly/nova.conf | 7 +++++++ templates/grizzly/quantum.conf | 7 +++++++ templates/havana/neutron.conf | 7 +++++++ templates/havana/nova.conf | 7 +++++++ templates/icehouse/neutron.conf | 7 +++++++ 5 files changed, 35 insertions(+) diff --git a/templates/grizzly/nova.conf b/templates/grizzly/nova.conf index 007e4f0e..7744c552 100644 --- a/templates/grizzly/nova.conf +++ b/templates/grizzly/nova.conf @@ -41,6 +41,13 @@ rabbit_durable_queues = false {% else %} rabbit_host = {{ rabbitmq_host }} {% endif -%} +{% if rabbit_ssl_port %} +rabbit_use_ssl=True +rabbit_port={{ rabbit_ssl_port }} +{% if rabbit_ssl_ca %} +kombu_ssl_ca_certs={{rabbit_ssl_ca}} +{% endif %} +{% endif %} {% endif -%} {% if glance_api_servers -%} diff --git a/templates/grizzly/quantum.conf b/templates/grizzly/quantum.conf index f763d38a..1c5cacd6 100644 --- a/templates/grizzly/quantum.conf +++ b/templates/grizzly/quantum.conf @@ -34,6 +34,13 @@ rabbit_durable_queues = false {% else %} rabbit_host = {{ rabbitmq_host }} {% endif -%} +{% if rabbit_ssl_port %} +rabbit_use_ssl=True +rabbit_port={{ rabbit_ssl_port }} +{% if rabbit_ssl_ca %} +kombu_ssl_ca_certs={{rabbit_ssl_ca}} +{% endif %} +{% endif %} {% endif -%} {% if neutron_security_groups -%} allow_overlapping_ips = True diff --git a/templates/havana/neutron.conf b/templates/havana/neutron.conf index 01ad661a..022b52a3 100644 --- a/templates/havana/neutron.conf +++ b/templates/havana/neutron.conf @@ -36,6 +36,13 @@ rabbit_durable_queues = false {% else %} rabbit_host = {{ rabbitmq_host }} {% endif -%} +{% if rabbit_ssl_port %} +rabbit_use_ssl=True +rabbit_port={{ rabbit_ssl_port }} +{% if rabbit_ssl_ca %} +kombu_ssl_ca_certs={{rabbit_ssl_ca}} +{% endif %} +{% endif %} {% endif -%} [quotas] diff --git a/templates/havana/nova.conf b/templates/havana/nova.conf index 5b2f22cd..aaf411c0 100644 --- a/templates/havana/nova.conf +++ b/templates/havana/nova.conf @@ -41,6 +41,13 @@ rabbit_durable_queues = false {% else %} rabbit_host = {{ rabbitmq_host }} {% endif -%} +{% if rabbit_ssl_port %} +rabbit_use_ssl=True +rabbit_port={{ rabbit_ssl_port }} +{% if rabbit_ssl_ca %} +kombu_ssl_ca_certs={{rabbit_ssl_ca}} +{% endif %} +{% endif %} {% endif -%} {% if glance_api_servers -%} diff --git a/templates/icehouse/neutron.conf b/templates/icehouse/neutron.conf index e604c1a4..9e6c402f 100644 --- a/templates/icehouse/neutron.conf +++ b/templates/icehouse/neutron.conf @@ -40,6 +40,13 @@ rabbit_durable_queues = false {% else %} rabbit_host = {{ rabbitmq_host }} {% endif -%} +{% if rabbit_ssl_port %} +rabbit_use_ssl=True +rabbit_port={{ rabbit_ssl_port }} +{% if rabbit_ssl_ca %} +kombu_ssl_ca_certs={{rabbit_ssl_ca}} +{% endif %} +{% endif %} {% endif -%} [quotas] From 16364f3eb231d1fbd1d0fc3e900d984bf9420bc2 Mon Sep 17 00:00:00 2001 From: "yolanda.robla@canonical.com" <> Date: Thu, 27 Mar 2014 12:46:44 +0100 Subject: [PATCH 65/98] added db testing --- hooks/nova_cc_hooks.py | 2 +- unit_tests/test_nova_cc_hooks.py | 103 +++++++++++++++++++++++++++++-- 2 files changed, 100 insertions(+), 5 deletions(-) diff --git a/hooks/nova_cc_hooks.py b/hooks/nova_cc_hooks.py index 24c1a72e..6b9ad522 100755 --- a/hooks/nova_cc_hooks.py +++ b/hooks/nova_cc_hooks.py @@ -192,7 +192,7 @@ def db_changed(): @restart_on_change(restart_map()) def postgresql_nova_db_changed(): if 'pgsql-nova-db' not in CONFIGS.complete_contexts(): - log('pgsql-*-db relation incomplete. Peer not ready?') + log('pgsql-nova-db relation incomplete. Peer not ready?') return CONFIGS.write(NOVA_CONF) diff --git a/unit_tests/test_nova_cc_hooks.py b/unit_tests/test_nova_cc_hooks.py index e0d93e76..a0e5a941 100644 --- a/unit_tests/test_nova_cc_hooks.py +++ b/unit_tests/test_nova_cc_hooks.py @@ -1,8 +1,9 @@ -from mock import MagicMock, patch - +from mock import call, MagicMock, patch from test_utils import CharmTestCase -import nova_cc_utils as utils +with patch('charmhelpers.core.hookenv.config') as config: + config.return_value = 'neutron' + import nova_cc_utils as utils _reg = utils.register_configs _map = utils.restart_map @@ -15,7 +16,6 @@ import nova_cc_hooks as hooks utils.register_configs = _reg utils.restart_map = _map - TO_PATCH = [ 'api_port', 'apt_update', @@ -29,8 +29,11 @@ TO_PATCH = [ 'determine_packages', 'determine_ports', 'open_port', + 'is_relation_made', + 'log', 'relation_get', 'relation_set', + 'relation_ids', 'ssh_compute_add', 'ssh_known_hosts_b64', 'ssh_authorized_keys_b64', @@ -42,6 +45,7 @@ TO_PATCH = [ 'eligible_leader', 'keystone_ca_cert_b64', 'neutron_plugin', + 'migrate_database', ] @@ -63,6 +67,7 @@ class NovaCCHooksTests(CharmTestCase): def setUp(self): super(NovaCCHooksTests, self).setUp(hooks, TO_PATCH) + self.config.side_effect = self.test_config.get self.relation_get.side_effect = self.test_relation.get self.charm_dir.return_value = '/var/lib/juju/charms/nova/charm' @@ -138,3 +143,93 @@ class NovaCCHooksTests(CharmTestCase): quantum_url='http://nova-cc-host1:9696', quantum_plugin='nvp', relation_id=None, **FAKE_KS_AUTH_CFG) + + def test_db_joined(self): + self.unit_get.return_value = 'nova.foohost.com' + self.is_relation_made.return_value = False + hooks.db_joined() + self.relation_set.assert_called_with(nova_database='nova', + nova_username='nova', + nova_hostname='nova.foohost.com') + self.unit_get.assert_called_with('private-address') + + def test_postgresql_nova_db_joined(self): + self.is_relation_made.return_value = False + hooks.pgsql_nova_db_joined() + self.relation_set.assert_called_with(database='nova') + + def test_postgresql_neutron_db_joined(self): + self.is_relation_made.return_value = False + hooks.pgsql_neutron_db_joined() + self.relation_set.assert_called_with(database='neutron') + + def test_db_joined_with_postgresql(self): + self.is_relation_made.return_value = True + + with self.assertRaises(Exception) as context: + hooks.db_joined() + self.assertEqual(context.exception.message, + 'Attempting to associate a mysql database when there is already ' + 'associated a postgresql one') + + def test_postgresql_nova_joined_with_db(self): + self.is_relation_made.return_value = True + + with self.assertRaises(Exception) as context: + hooks.pgsql_nova_db_joined() + self.assertEqual(context.exception.message, + 'Attempting to associate a postgresql database when there is already ' + 'associated a mysql one') + + def test_postgresql_neutron_joined_with_db(self): + self.is_relation_made.return_value = True + + with self.assertRaises(Exception) as context: + hooks.pgsql_neutron_db_joined() + self.assertEqual(context.exception.message, + 'Attempting to associate a postgresql database when there is already ' + 'associated a mysql one') + + @patch.object(hooks, 'CONFIGS') + def test_db_changed_missing_relation_data(self, configs): + configs.complete_contexts = MagicMock() + configs.complete_contexts.return_value = [] + hooks.db_changed() + self.log.assert_called_with( + 'shared-db relation incomplete. Peer not ready?' + ) + + @patch.object(hooks, 'CONFIGS') + def test_postgresql_nova_db_changed_missing_relation_data(self, configs): + configs.complete_contexts = MagicMock() + configs.complete_contexts.return_value = [] + hooks.postgresql_nova_db_changed() + self.log.assert_called_with( + 'pgsql-nova-db relation incomplete. Peer not ready?' + ) + + def _shared_db_test(self, configs): + configs.complete_contexts = MagicMock() + configs.complete_contexts.return_value = ['shared-db'] + configs.write = MagicMock() + hooks.db_changed() + + def _postgresql_db_test(self, configs): + configs.complete_contexts = MagicMock() + configs.complete_contexts.return_value = ['pgsql-nova-db'] + configs.write = MagicMock() + hooks.postgresql_nova_db_changed() + + @patch.object(hooks, 'CONFIGS') + def test_db_changed(self, configs): + self._shared_db_test(configs) + self.assertEquals([call('/etc/nova/nova.conf')], + configs.write.call_args_list) + self.migrate_database.assert_called_with() + + @patch.object(hooks, 'CONFIGS') + def test_postgresql_db_changed(self, configs): + self._postgresql_db_test(configs) + self.assertEquals([call('/etc/nova/nova.conf')], + configs.write.call_args_list) + self.migrate_database.assert_called_with() From 2a3d653196d5f770ae624c63adf896e4cca71c9c Mon Sep 17 00:00:00 2001 From: James Page Date: Thu, 27 Mar 2014 22:33:49 +0000 Subject: [PATCH 66/98] Enable lbaas and vpnaas for icehouse --- templates/icehouse/neutron.conf | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/templates/icehouse/neutron.conf b/templates/icehouse/neutron.conf index 9e6c402f..3d1d3f01 100644 --- a/templates/icehouse/neutron.conf +++ b/templates/icehouse/neutron.conf @@ -18,7 +18,8 @@ bind_port = 9696 {% if core_plugin -%} core_plugin = {{ core_plugin }} {% if neutron_plugin in ['ovs', 'ml2'] -%} -service_plugins = neutron.services.l3_router.l3_router_plugin.L3RouterPlugin,neutron.services.metering.metering_plugin.MeteringPlugin +# service_plugins = router,firewall,lbaas,vpnaas,metering +service_plugins = router,metering,lbaas,vpnaas {% endif -%} {% endif -%} @@ -79,5 +80,6 @@ connection = mysql://{{ database_user }}:{{ database_password }}@{{ database_hos connection = sqlite:////var/lib/neutron/neutron.sqlite {% endif -%} -[lbaas] [service_providers] +service_provider=LOADBALANCER:Haproxy:neutron.services.loadbalancer.drivers.haproxy.plugin_driver.HaproxyOnHostPluginDriver:default +service_provider=VPN:openswan:neutron.services.vpn.service_drivers.ipsec.IPsecVPNDriver:default From 13217d10bfb3210a7051b5d4a27954ea3783c5c4 Mon Sep 17 00:00:00 2001 From: James Page Date: Thu, 27 Mar 2014 23:06:30 +0000 Subject: [PATCH 67/98] Enable firewall driver --- templates/icehouse/neutron.conf | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/templates/icehouse/neutron.conf b/templates/icehouse/neutron.conf index 3d1d3f01..45ded184 100644 --- a/templates/icehouse/neutron.conf +++ b/templates/icehouse/neutron.conf @@ -18,8 +18,7 @@ bind_port = 9696 {% if core_plugin -%} core_plugin = {{ core_plugin }} {% if neutron_plugin in ['ovs', 'ml2'] -%} -# service_plugins = router,firewall,lbaas,vpnaas,metering -service_plugins = router,metering,lbaas,vpnaas +service_plugins = router,firewall,lbaas,vpnaas,metering {% endif -%} {% endif -%} From 36a37b420c64653178641b27800ab3ee96ad10f7 Mon Sep 17 00:00:00 2001 From: "yolanda.robla@canonical.com" <> Date: Fri, 28 Mar 2014 14:13:18 +0100 Subject: [PATCH 68/98] not ssl in postgres --- hooks/charmhelpers/contrib/openstack/context.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/hooks/charmhelpers/contrib/openstack/context.py b/hooks/charmhelpers/contrib/openstack/context.py index ba909522..d3ea5ff6 100644 --- a/hooks/charmhelpers/contrib/openstack/context.py +++ b/hooks/charmhelpers/contrib/openstack/context.py @@ -179,9 +179,7 @@ class PostgresqlDBContext(OSContextGenerator): 'database_password': relation_get('password', rid=rid, unit=unit), 'database_type': 'postgresql', } - if context_complete(ctxt): - db_ssl(rdata, ctxt, self.ssl_dir) - return ctxt + return ctxt return {} From 76b768e79fec624e9ad3b9d36267debc3b6a773e Mon Sep 17 00:00:00 2001 From: "yolanda.robla@canonical.com" <> Date: Fri, 28 Mar 2014 14:14:32 +0100 Subject: [PATCH 69/98] not using ssl --- hooks/charmhelpers/contrib/openstack/context.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/hooks/charmhelpers/contrib/openstack/context.py b/hooks/charmhelpers/contrib/openstack/context.py index d3ea5ff6..b564a01a 100644 --- a/hooks/charmhelpers/contrib/openstack/context.py +++ b/hooks/charmhelpers/contrib/openstack/context.py @@ -179,7 +179,8 @@ class PostgresqlDBContext(OSContextGenerator): 'database_password': relation_get('password', rid=rid, unit=unit), 'database_type': 'postgresql', } - return ctxt + if context_complete(ctxt): + return ctxt return {} From 96a03a567c3c8ecc06abd3c3a91746a212db7773 Mon Sep 17 00:00:00 2001 From: "yolanda.robla@canonical.com" <> Date: Fri, 28 Mar 2014 15:17:48 +0100 Subject: [PATCH 70/98] typo in template --- templates/icehouse/neutron.conf | 1 + 1 file changed, 1 insertion(+) diff --git a/templates/icehouse/neutron.conf b/templates/icehouse/neutron.conf index 2f44eaa7..36464be4 100644 --- a/templates/icehouse/neutron.conf +++ b/templates/icehouse/neutron.conf @@ -73,6 +73,7 @@ admin_password = {{ admin_password }} {% endif -%} [database] +{% if database_host -%} connection = {{ database_type }}://{{ database_user }}:{{ database_password }}@{{ database_host }}/{{ database }}{% if database_ssl_ca %}?ssl_ca={{ database_ssl_ca }}{% if database_ssl_cert %}&ssl_cert={{ database_ssl_cert }}&ssl_key={{ database_ssl_key }}{% endif %}{% endif %} {% else -%} connection = sqlite:////var/lib/neutron/neutron.sqlite From f967dc60179ac9fc973958575dee6a17b5d73776 Mon Sep 17 00:00:00 2001 From: "yolanda.robla@canonical.com" <> Date: Mon, 31 Mar 2014 12:56:51 +0200 Subject: [PATCH 71/98] resynced context --- hooks/charmhelpers/contrib/openstack/context.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/hooks/charmhelpers/contrib/openstack/context.py b/hooks/charmhelpers/contrib/openstack/context.py index b564a01a..8429d832 100644 --- a/hooks/charmhelpers/contrib/openstack/context.py +++ b/hooks/charmhelpers/contrib/openstack/context.py @@ -147,10 +147,11 @@ class SharedDBContext(OSContextGenerator): 'database_host': rdata.get('db_host'), 'database': self.database, 'database_user': self.user, - 'database_password': passwd, - 'database_type': 'mysql', + 'database_password': rdata.get(password_setting), + 'database_type': 'mysql' } if context_complete(ctxt): + db_ssl(rdata, ctxt, self.ssl_dir) return ctxt return {} From ddd59369b577d96a47d45a46b9ddd5c033028653 Mon Sep 17 00:00:00 2001 From: James Page Date: Mon, 31 Mar 2014 14:01:44 +0100 Subject: [PATCH 72/98] Use neutron-database config for neutron relation --- hooks/nova_cc_context.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/hooks/nova_cc_context.py b/hooks/nova_cc_context.py index c25d1473..82dec298 100644 --- a/hooks/nova_cc_context.py +++ b/hooks/nova_cc_context.py @@ -172,3 +172,7 @@ class NovaPostgresqlDBContext(context.PostgresqlDBContext): class NeutronPostgresqlDBContext(context.PostgresqlDBContext): interfaces = ['pgsql-neutron-db'] + + def __init__(self): + super(NeutronPostgresqlDBContext, + self).__init__(config('neutron-database')) From c04b70df28571df1735cb8ef2e0cc83120dd8f22 Mon Sep 17 00:00:00 2001 From: James Page Date: Tue, 1 Apr 2014 15:59:00 +0100 Subject: [PATCH 73/98] Rework upgrade approach for renaming/different plugins --- hooks/nova_cc_hooks.py | 3 ++- hooks/nova_cc_utils.py | 8 ++++---- templates/icehouse/neutron.conf | 1 + 3 files changed, 7 insertions(+), 5 deletions(-) diff --git a/hooks/nova_cc_hooks.py b/hooks/nova_cc_hooks.py index 3623afa4..b694daef 100755 --- a/hooks/nova_cc_hooks.py +++ b/hooks/nova_cc_hooks.py @@ -98,8 +98,9 @@ def install(): @hooks.hook('config-changed') @restart_on_change(restart_map(), stopstart=True) def config_changed(): + global CONFIGS if openstack_upgrade_available('nova-common'): - do_openstack_upgrade(configs=CONFIGS) + CONFIGS = do_openstack_upgrade() save_script_rc() configure_https() CONFIGS.write_all() diff --git a/hooks/nova_cc_utils.py b/hooks/nova_cc_utils.py index 5a695328..8a8abba9 100644 --- a/hooks/nova_cc_utils.py +++ b/hooks/nova_cc_utils.py @@ -307,7 +307,7 @@ def save_script_rc(): _save_script_rc(**env_vars) -def do_openstack_upgrade(configs): +def do_openstack_upgrade(): new_src = config('openstack-origin') new_os_rel = get_os_codename_install_source(new_src) log('Performing OpenStack upgrade to %s.' % (new_os_rel)) @@ -322,15 +322,15 @@ def do_openstack_upgrade(configs): apt_upgrade(options=dpkg_opts, fatal=True, dist=True) apt_install(determine_packages(), fatal=True) - # Re-register all configs to accomodate changes in filesname etc - configs.set_release(openstack_release=new_os_rel) + # set CONFIGS to load templates from new release + configs = register_configs() configs.write_all() - # NOTE(jamespage) upgrades that change contexts needs to be resolved still [service_stop(s) for s in services()] if eligible_leader(CLUSTER_RES): migrate_database() [service_start(s) for s in services()] + return configs def volume_service(): diff --git a/templates/icehouse/neutron.conf b/templates/icehouse/neutron.conf index d48de552..aeaa57e8 100644 --- a/templates/icehouse/neutron.conf +++ b/templates/icehouse/neutron.conf @@ -57,3 +57,4 @@ admin_password = {{ admin_password }} [service_providers] service_provider=LOADBALANCER:Haproxy:neutron.services.loadbalancer.drivers.haproxy.plugin_driver.HaproxyOnHostPluginDriver:default service_provider=VPN:openswan:neutron.services.vpn.service_drivers.ipsec.IPsecVPNDriver:default +service_provider=FIREWALL:Iptables:neutron.agent.linux.iptables_firewall.OVSHybridIptablesFirewallDriver:default \ No newline at end of file From a1c245bdf25d6be940dd8cf30ce7560bf73b04d8 Mon Sep 17 00:00:00 2001 From: James Page Date: Tue, 1 Apr 2014 16:40:28 +0100 Subject: [PATCH 74/98] Make utils forget its release on upgrade --- hooks/nova_cc_utils.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/hooks/nova_cc_utils.py b/hooks/nova_cc_utils.py index 8a8abba9..b4862809 100644 --- a/hooks/nova_cc_utils.py +++ b/hooks/nova_cc_utils.py @@ -308,6 +308,9 @@ def save_script_rc(): def do_openstack_upgrade(): + # NOTE(jamespage) horrible hack to make utils forget a cached value + import charmhelpers.contrib.openstack.utils as utils + utils.os_rel = None new_src = config('openstack-origin') new_os_rel = get_os_codename_install_source(new_src) log('Performing OpenStack upgrade to %s.' % (new_os_rel)) From e2a6ec77627e2a7b70fa2f6e52d758c237b0ce66 Mon Sep 17 00:00:00 2001 From: James Page Date: Wed, 2 Apr 2014 11:01:06 +0100 Subject: [PATCH 75/98] Fixup neutron->nova notifications --- hooks/nova_cc_context.py | 30 ++++++++++++++++++++++++++++-- templates/icehouse/neutron.conf | 11 +++++++++++ 2 files changed, 39 insertions(+), 2 deletions(-) diff --git a/hooks/nova_cc_context.py b/hooks/nova_cc_context.py index 82dec298..ebc15740 100644 --- a/hooks/nova_cc_context.py +++ b/hooks/nova_cc_context.py @@ -1,12 +1,17 @@ from charmhelpers.core.hookenv import ( - config, relation_ids, relation_set, log, ERROR) + config, relation_ids, relation_set, log, ERROR, + unit_get) from charmhelpers.fetch import apt_install, filter_installed_packages from charmhelpers.contrib.openstack import context, neutron, utils from charmhelpers.contrib.hahelpers.cluster import ( - determine_apache_port, determine_api_port) + determine_apache_port, + determine_api_port, + https, + is_clustered +) class ApacheSSLContext(context.ApacheSSLContext): @@ -112,6 +117,24 @@ class HAProxyContext(context.HAProxyContext): return ctxt +def canonical_url(vip_setting='vip'): + ''' + Returns the correct HTTP URL to this host given the state of HTTPS + configuration and hacluster. + + :vip_setting: str: Setting in charm config that specifies + VIP address. + ''' + scheme = 'http' + if https(): + scheme = 'https' + if is_clustered(): + addr = config(vip_setting) + else: + addr = unit_get('private-address') + return '%s://%s' % (scheme, addr) + + class NeutronCCContext(context.NeutronContext): interfaces = [] @@ -148,6 +171,8 @@ class NeutronCCContext(context.NeutronContext): ','.join(_config['nvp-controllers'].split()) ctxt['nvp_controllers_list'] = \ _config['nvp-controllers'].split() + # Need URL for nova + ctxt['nova_url'] = "{}:8774".format(canonical_url()) return ctxt @@ -163,6 +188,7 @@ class IdentityServiceContext(context.IdentityServiceContext): ec2_tokens = 'http://%s:%s/v2.0/ec2tokens' % (ctxt['service_host'], ctxt['service_port']) ctxt['keystone_ec2_url'] = ec2_tokens + ctxt['region'] = config('region') return ctxt diff --git a/templates/icehouse/neutron.conf b/templates/icehouse/neutron.conf index aeaa57e8..01f0e4dd 100644 --- a/templates/icehouse/neutron.conf +++ b/templates/icehouse/neutron.conf @@ -29,6 +29,17 @@ neutron_firewall_driver = neutron.agent.linux.iptables_firewall.OVSHybridIptable {% include "parts/rabbitmq" %} +notify_nova_on_port_status_changes = True +notify_nova_on_port_data_changes = True +nova_url = {{ nova_url }} +nova_region_name = {{ region }} +{% if auth_host -%} +nova_admin_username = {{ admin_user }} +nova_admin_tenant_id = {{ admin_tenant_name }} +nova_admin_password = {{ admin_password }} +nova_admin_auth_url = {{ auth_protocol }}://{{ auth_host }}:{{ auth_port }}/v2.0 +{% endif -%} + [quotas] quota_driver = neutron.db.quota_db.DbQuotaDriver {% if neutron_security_groups -%} From c70ffd03ccaec436149d42a22c45f7a596888620 Mon Sep 17 00:00:00 2001 From: James Page Date: Wed, 2 Apr 2014 11:37:43 +0100 Subject: [PATCH 76/98] Fixup url for nova to force v2 --- hooks/nova_cc_context.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/hooks/nova_cc_context.py b/hooks/nova_cc_context.py index ebc15740..9c046064 100644 --- a/hooks/nova_cc_context.py +++ b/hooks/nova_cc_context.py @@ -171,8 +171,7 @@ class NeutronCCContext(context.NeutronContext): ','.join(_config['nvp-controllers'].split()) ctxt['nvp_controllers_list'] = \ _config['nvp-controllers'].split() - # Need URL for nova - ctxt['nova_url'] = "{}:8774".format(canonical_url()) + ctxt['nova_url'] = "{}:8774/v2".format(canonical_url()) return ctxt From 7c38fd0d5b20d3accac7eece164c6913bcd4ffc7 Mon Sep 17 00:00:00 2001 From: James Page Date: Wed, 2 Apr 2014 12:27:25 +0100 Subject: [PATCH 77/98] Put temp fix into charmhelpers for tenant_id propagation --- hooks/charmhelpers/contrib/openstack/context.py | 2 ++ templates/icehouse/neutron.conf | 2 +- 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/hooks/charmhelpers/contrib/openstack/context.py b/hooks/charmhelpers/contrib/openstack/context.py index 8429d832..9de0c5f0 100644 --- a/hooks/charmhelpers/contrib/openstack/context.py +++ b/hooks/charmhelpers/contrib/openstack/context.py @@ -226,6 +226,8 @@ class IdentityServiceContext(OSContextGenerator): 'auth_host': rdata.get('auth_host'), 'auth_port': rdata.get('auth_port'), 'admin_tenant_name': rdata.get('service_tenant'), + 'admin_tenant_id': rdata.get('service_tenant_id', + default=""), 'admin_user': rdata.get('service_username'), 'admin_password': rdata.get('service_password'), 'service_protocol': diff --git a/templates/icehouse/neutron.conf b/templates/icehouse/neutron.conf index 01f0e4dd..078a7bb6 100644 --- a/templates/icehouse/neutron.conf +++ b/templates/icehouse/neutron.conf @@ -35,7 +35,7 @@ nova_url = {{ nova_url }} nova_region_name = {{ region }} {% if auth_host -%} nova_admin_username = {{ admin_user }} -nova_admin_tenant_id = {{ admin_tenant_name }} +nova_admin_tenant_id = {{ admin_tenant_id }} nova_admin_password = {{ admin_password }} nova_admin_auth_url = {{ auth_protocol }}://{{ auth_host }}:{{ auth_port }}/v2.0 {% endif -%} From 73cd8945d4a0642aec681843fc4b41a58b9a33f1 Mon Sep 17 00:00:00 2001 From: James Page Date: Wed, 2 Apr 2014 12:36:47 +0100 Subject: [PATCH 78/98] Resync helpers from icehouse branch --- charm-helpers.yaml | 2 +- hooks/charmhelpers/contrib/openstack/context.py | 3 +-- 2 files changed, 2 insertions(+), 3 deletions(-) diff --git a/charm-helpers.yaml b/charm-helpers.yaml index 35e5834f..64dd7f89 100644 --- a/charm-helpers.yaml +++ b/charm-helpers.yaml @@ -1,4 +1,4 @@ -branch: lp:charm-helpers +branch: lp:~openstack-charmers/charm-helpers/icehouse destination: hooks/charmhelpers include: - core diff --git a/hooks/charmhelpers/contrib/openstack/context.py b/hooks/charmhelpers/contrib/openstack/context.py index 9de0c5f0..10f4f30c 100644 --- a/hooks/charmhelpers/contrib/openstack/context.py +++ b/hooks/charmhelpers/contrib/openstack/context.py @@ -226,8 +226,6 @@ class IdentityServiceContext(OSContextGenerator): 'auth_host': rdata.get('auth_host'), 'auth_port': rdata.get('auth_port'), 'admin_tenant_name': rdata.get('service_tenant'), - 'admin_tenant_id': rdata.get('service_tenant_id', - default=""), 'admin_user': rdata.get('service_username'), 'admin_password': rdata.get('service_password'), 'service_protocol': @@ -236,6 +234,7 @@ class IdentityServiceContext(OSContextGenerator): rdata.get('auth_protocol') or 'http', } if context_complete(ctxt): + ctxt['admin_tenant_id'] = rdata.get('service_tenant_id') return ctxt return {} From e2c651f98cd9d7d65df6a82d65e07422fd2e263a Mon Sep 17 00:00:00 2001 From: James Page Date: Wed, 2 Apr 2014 18:04:22 +0100 Subject: [PATCH 79/98] Resync to master charm-helpers --- charm-helpers.yaml | 2 +- hooks/charmhelpers/contrib/openstack/context.py | 3 +++ 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/charm-helpers.yaml b/charm-helpers.yaml index 64dd7f89..35e5834f 100644 --- a/charm-helpers.yaml +++ b/charm-helpers.yaml @@ -1,4 +1,4 @@ -branch: lp:~openstack-charmers/charm-helpers/icehouse +branch: lp:charm-helpers destination: hooks/charmhelpers include: - core diff --git a/hooks/charmhelpers/contrib/openstack/context.py b/hooks/charmhelpers/contrib/openstack/context.py index 10f4f30c..e92b1cc8 100644 --- a/hooks/charmhelpers/contrib/openstack/context.py +++ b/hooks/charmhelpers/contrib/openstack/context.py @@ -234,6 +234,9 @@ class IdentityServiceContext(OSContextGenerator): rdata.get('auth_protocol') or 'http', } if context_complete(ctxt): + # NOTE(jamespage) this is required for >= icehouse + # so a missing value just indicates keystone needs + # upgrading ctxt['admin_tenant_id'] = rdata.get('service_tenant_id') return ctxt return {} From 76dd6f087469ccbcbfb499177deab0004d7781ea Mon Sep 17 00:00:00 2001 From: Corey Bryant Date: Tue, 8 Apr 2014 12:37:55 +0000 Subject: [PATCH 80/98] Enable upgrade from grizzly to icehouse by stepping to havana --- hooks/nova_cc_utils.py | 43 +++++++++++++++++++++++++++++++++++++----- 1 file changed, 38 insertions(+), 5 deletions(-) diff --git a/hooks/nova_cc_utils.py b/hooks/nova_cc_utils.py index b4862809..abe66a0c 100644 --- a/hooks/nova_cc_utils.py +++ b/hooks/nova_cc_utils.py @@ -307,11 +307,31 @@ def save_script_rc(): _save_script_rc(**env_vars) -def do_openstack_upgrade(): - # NOTE(jamespage) horrible hack to make utils forget a cached value - import charmhelpers.contrib.openstack.utils as utils - utils.os_rel = None - new_src = config('openstack-origin') +def get_step_upgrade_source(new_src): + ''' + Determine if upgrade skips a release and, if so, return source + of skipped release. + ''' + sources = { + #target_src: (cur_pocket, step_src) + 'cloud:precise-icehouse': + ('precise-updates/grizzly', 'cloud:precise-havana'), + 'cloud:precise-icehouse/proposed': + ('precise-proposed/grizzly', 'cloud:precise-havana/proposed') + } + + with open('/etc/apt/sources.list.d/cloud-archive.list', 'r') as f: + line = f.readline() + for target_src, (cur_pocket, step_src) in sources.items(): + if target_src != new_src: + continue + if cur_pocket in line: + return step_src + + return None + + +def _do_openstack_upgrade(new_src): new_os_rel = get_os_codename_install_source(new_src) log('Performing OpenStack upgrade to %s.' % (new_os_rel)) @@ -336,6 +356,19 @@ def do_openstack_upgrade(): return configs +def do_openstack_upgrade(): + # NOTE(jamespage) horrible hack to make utils forget a cached value + import charmhelpers.contrib.openstack.utils as utils + utils.os_rel = None + new_src = config('openstack-origin') + + step_src = get_step_upgrade_source(new_src) + if step_src is not None: + _do_openstack_upgrade(step_src) + + return _do_openstack_upgrade(new_src) + + def volume_service(): '''Specifies correct volume API for specific OS release''' os_vers = os_release('nova-common') From 6d8e90e9aa0d30b7241da7fe5d924c4e7835d067 Mon Sep 17 00:00:00 2001 From: James Page Date: Tue, 8 Apr 2014 21:49:54 +0100 Subject: [PATCH 81/98] Support override of os_release in register_configs --- hooks/nova_cc_utils.py | 14 +++----------- 1 file changed, 3 insertions(+), 11 deletions(-) diff --git a/hooks/nova_cc_utils.py b/hooks/nova_cc_utils.py index 54549bc4..26bc4541 100644 --- a/hooks/nova_cc_utils.py +++ b/hooks/nova_cc_utils.py @@ -237,8 +237,8 @@ def resource_map(): return resource_map -def register_configs(): - release = os_release('nova-common') +def register_configs(release=None): + release = release or os_release('nova-common') configs = templating.OSConfigRenderer(templates_dir=TEMPLATES, openstack_release=release) for cfg, rscs in resource_map().iteritems(): @@ -332,9 +332,6 @@ def get_step_upgrade_source(new_src): def _do_openstack_upgrade(new_src): - # NOTE(jamespage) horrible hack to make utils forget a cached value - import charmhelpers.contrib.openstack.utils as utils - utils.os_rel = None new_os_rel = get_os_codename_install_source(new_src) log('Performing OpenStack upgrade to %s.' % (new_os_rel)) @@ -349,7 +346,7 @@ def _do_openstack_upgrade(new_src): apt_install(determine_packages(), fatal=True) # set CONFIGS to load templates from new release - configs = register_configs() + configs = register_configs(os_release=new_os_rel) configs.write_all() [service_stop(s) for s in services()] @@ -360,15 +357,10 @@ def _do_openstack_upgrade(new_src): def do_openstack_upgrade(): - # NOTE(jamespage) horrible hack to make utils forget a cached value - import charmhelpers.contrib.openstack.utils as utils - utils.os_rel = None new_src = config('openstack-origin') - step_src = get_step_upgrade_source(new_src) if step_src is not None: _do_openstack_upgrade(step_src) - return _do_openstack_upgrade(new_src) From 57bb6e54e125062bbd2957e3272c883481a46da1 Mon Sep 17 00:00:00 2001 From: James Page Date: Tue, 8 Apr 2014 22:03:17 +0100 Subject: [PATCH 82/98] Fixup release keyword --- hooks/nova_cc_utils.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/hooks/nova_cc_utils.py b/hooks/nova_cc_utils.py index 26bc4541..e297700f 100644 --- a/hooks/nova_cc_utils.py +++ b/hooks/nova_cc_utils.py @@ -346,7 +346,7 @@ def _do_openstack_upgrade(new_src): apt_install(determine_packages(), fatal=True) # set CONFIGS to load templates from new release - configs = register_configs(os_release=new_os_rel) + configs = register_configs(release=new_os_rel) configs.write_all() [service_stop(s) for s in services()] From faea438844f1115560e08242c87c9e5002ecc03b Mon Sep 17 00:00:00 2001 From: James Page Date: Tue, 8 Apr 2014 23:23:04 +0100 Subject: [PATCH 83/98] Rework upgrades for neutron --- hooks/nova_cc_utils.py | 67 +++++++++++++++++++++++++++++++++++++++++- 1 file changed, 66 insertions(+), 1 deletion(-) diff --git a/hooks/nova_cc_utils.py b/hooks/nova_cc_utils.py index e297700f..fa5dc412 100644 --- a/hooks/nova_cc_utils.py +++ b/hooks/nova_cc_utils.py @@ -330,8 +330,61 @@ def get_step_upgrade_source(new_src): return None +POLICY_RC_D = """#!/bin/bash + +set -e + +case $1 in + neutron-server|quantum-server|nova-*) + [ $2 = "start" ] && exit 101 + ;; + *) + ;; +esac + +exit 0 +""" + + +def enable_policy_rcd(): + with open('/usr/sbin/policy-rc.d', 'w') as policy: + policy.write(POLICY_RC_D) + os.chmod('/usr/sbin/policy-rc.d', 0o755) + + +def disable_policy_rcd(): + os.unlink('/usr/sbin/policy-rc.d') + + +QUANTUM_DB_MANAGE = "/usr/bin/quantum-db-manage" +NEUTRON_DB_MANAGE = "/usr/bin/neutron-db-manage" + + +def reset_os_release(): + # Ugly hack to make os_release re-read versions + import charmhelpers.contrib.openstack.utils as utils + utils.os_rel = None + + +def neutron_db_manage(actions): + reset_os_release() + net_manager = network_manager() + if net_manager in ['neutron', 'quantum']: + plugin = neutron_plugin() + conf = neutron_plugin_attribute(plugin, 'config', net_manager) + if net_manager == 'quantum': + cmd = QUANTUM_DB_MANAGE + else: + cmd = NEUTRON_DB_MANAGE + subprocess.check_call([ + cmd, '--config-file=/etc/{mgr}/{mgr}.conf'.format(mgr=net_manager), + '--config-file={}'.format(conf)].append(actions) + ) + def _do_openstack_upgrade(new_src): + enable_policy_rcd() + cur_os_rel = os_release('nova-common') new_os_rel = get_os_codename_install_source(new_src) log('Performing OpenStack upgrade to %s.' % (new_os_rel)) @@ -341,18 +394,30 @@ def _do_openstack_upgrade(new_src): '--option', 'Dpkg::Options::=--force-confdef', ] + if cur_os_rel == 'grizzly': + # NOTE(jamespage) pre-stamp grizzly database before upgrade + neutron_db_manage(['stamp', 'grizzly']) + apt_update(fatal=True) apt_upgrade(options=dpkg_opts, fatal=True, dist=True) apt_install(determine_packages(), fatal=True) # set CONFIGS to load templates from new release + reset_os_release() configs = register_configs(release=new_os_rel) configs.write_all() - [service_stop(s) for s in services()] + neutron_db_manage(['upgrade', 'head']) + if new_os_rel == 'icehouse': + # NOTE(jamespage) add migration to ML2 after upgrade to icehouse + pass + if eligible_leader(CLUSTER_RES): migrate_database() [service_start(s) for s in services()] + + disable_policy_rcd() + return configs From 998d0d53df2055208b42b36727aeb2fd8deb9306 Mon Sep 17 00:00:00 2001 From: James Page Date: Tue, 8 Apr 2014 23:25:58 +0100 Subject: [PATCH 84/98] Fixup upgrade extension --- hooks/nova_cc_utils.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/hooks/nova_cc_utils.py b/hooks/nova_cc_utils.py index fa5dc412..e399052b 100644 --- a/hooks/nova_cc_utils.py +++ b/hooks/nova_cc_utils.py @@ -378,7 +378,7 @@ def neutron_db_manage(actions): cmd = NEUTRON_DB_MANAGE subprocess.check_call([ cmd, '--config-file=/etc/{mgr}/{mgr}.conf'.format(mgr=net_manager), - '--config-file={}'.format(conf)].append(actions) + '--config-file={}'.format(conf)].extend(actions) ) From d8aeac21f45dcdd3934385d12c229270bec663af Mon Sep 17 00:00:00 2001 From: James Page Date: Wed, 9 Apr 2014 08:58:21 +0100 Subject: [PATCH 85/98] Use fully qualified objects for service_plugins --- hooks/nova_cc_utils.py | 31 +++++++++++++++++++++++++++++-- templates/icehouse/neutron.conf | 2 +- 2 files changed, 30 insertions(+), 3 deletions(-) diff --git a/hooks/nova_cc_utils.py b/hooks/nova_cc_utils.py index e399052b..43516991 100644 --- a/hooks/nova_cc_utils.py +++ b/hooks/nova_cc_utils.py @@ -382,6 +382,32 @@ def neutron_db_manage(actions): ) +def get_db_connection(): + config = ConfigParser.RawConfigParser() + config.read('/etc/neutron/neutron.conf') + try: + return config.get('database', 'connection') + except: + return None + + +def ml2_migration(): + reset_os_release() + net_manager = network_manager() + if net_manager == 'neutron': + plugin = neutron_plugin() + if plugin == 'ovs': + cmd = [ + 'python', + '/usr/lib/python2.7/dist-packages/neutron' + '/db/migration/migrate_to_ml2.py', + '--tunnel-type', 'gre', + '--release', 'icehouse', + 'openvswitch', get_db_connection() + ] + subprocess.check_call(cmd) + + def _do_openstack_upgrade(new_src): enable_policy_rcd() cur_os_rel = os_release('nova-common') @@ -407,10 +433,11 @@ def _do_openstack_upgrade(new_src): configs = register_configs(release=new_os_rel) configs.write_all() - neutron_db_manage(['upgrade', 'head']) if new_os_rel == 'icehouse': # NOTE(jamespage) add migration to ML2 after upgrade to icehouse - pass + ml2_migration() + + neutron_db_manage(['upgrade', 'head']) if eligible_leader(CLUSTER_RES): migrate_database() diff --git a/templates/icehouse/neutron.conf b/templates/icehouse/neutron.conf index 078a7bb6..2d4c4a02 100644 --- a/templates/icehouse/neutron.conf +++ b/templates/icehouse/neutron.conf @@ -18,7 +18,7 @@ bind_port = 9696 {% if core_plugin -%} core_plugin = {{ core_plugin }} {% if neutron_plugin in ['ovs', 'ml2'] -%} -service_plugins = router,firewall,lbaas,vpnaas,metering +service_plugins = neutron.services.l3_router.l3_router_plugin.L3RouterPlugin,neutron.services.firewall.fwaas_plugin.FirewallPlugin,neutron.services.loadbalancer.plugin.LoadBalancerPlugin,neutron.services.vpn.plugin.VPNDriverPlugin,neutron.services.metering.metering_plugin.MeteringPlugin {% endif -%} {% endif -%} From fe648a0462928cd3a7457f222c7de50020dd4af6 Mon Sep 17 00:00:00 2001 From: James Page Date: Wed, 9 Apr 2014 09:06:06 +0100 Subject: [PATCH 86/98] Always stamp prior to upgrade --- hooks/nova_cc_utils.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/hooks/nova_cc_utils.py b/hooks/nova_cc_utils.py index 43516991..535b37b4 100644 --- a/hooks/nova_cc_utils.py +++ b/hooks/nova_cc_utils.py @@ -420,9 +420,8 @@ def _do_openstack_upgrade(new_src): '--option', 'Dpkg::Options::=--force-confdef', ] - if cur_os_rel == 'grizzly': - # NOTE(jamespage) pre-stamp grizzly database before upgrade - neutron_db_manage(['stamp', 'grizzly']) + # NOTE(jamespage) pre-stamp database before upgrade + neutron_db_manage(['stamp', cur_os_rel]) apt_update(fatal=True) apt_upgrade(options=dpkg_opts, fatal=True, dist=True) From 1413b075f1260e86338aff222b8112025fe164a0 Mon Sep 17 00:00:00 2001 From: James Page Date: Wed, 9 Apr 2014 09:23:23 +0100 Subject: [PATCH 87/98] fixup actions again --- hooks/nova_cc_utils.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/hooks/nova_cc_utils.py b/hooks/nova_cc_utils.py index 535b37b4..a921bd7f 100644 --- a/hooks/nova_cc_utils.py +++ b/hooks/nova_cc_utils.py @@ -378,7 +378,7 @@ def neutron_db_manage(actions): cmd = NEUTRON_DB_MANAGE subprocess.check_call([ cmd, '--config-file=/etc/{mgr}/{mgr}.conf'.format(mgr=net_manager), - '--config-file={}'.format(conf)].extend(actions) + '--config-file={}'.format(conf)] + actions ) From 43e32df48032fbbc2a46083195eb9bb253b84f5e Mon Sep 17 00:00:00 2001 From: James Page Date: Wed, 9 Apr 2014 09:42:14 +0100 Subject: [PATCH 88/98] Only use db-manage when doing the awkward rename in grizzly->havana --- hooks/nova_cc_utils.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/hooks/nova_cc_utils.py b/hooks/nova_cc_utils.py index a921bd7f..80e8347e 100644 --- a/hooks/nova_cc_utils.py +++ b/hooks/nova_cc_utils.py @@ -421,7 +421,8 @@ def _do_openstack_upgrade(new_src): ] # NOTE(jamespage) pre-stamp database before upgrade - neutron_db_manage(['stamp', cur_os_rel]) + if cur_os_rel == 'grizzly': + neutron_db_manage(['stamp', 'grizzly']) apt_update(fatal=True) apt_upgrade(options=dpkg_opts, fatal=True, dist=True) @@ -436,7 +437,9 @@ def _do_openstack_upgrade(new_src): # NOTE(jamespage) add migration to ML2 after upgrade to icehouse ml2_migration() - neutron_db_manage(['upgrade', 'head']) + if cur_os_rel == 'grizzly' and new_os_rel == 'havana': + # NOTE(jamespage) only force an upgrade when going grizzly->havana + neutron_db_manage(['upgrade', 'havana']) if eligible_leader(CLUSTER_RES): migrate_database() From d63968287cae1e6c4339fe46abd568b958362e6f Mon Sep 17 00:00:00 2001 From: James Page Date: Wed, 9 Apr 2014 10:12:01 +0100 Subject: [PATCH 89/98] Gaahhhh this is horrible --- hooks/nova_cc_utils.py | 28 +++++++++++++++++----------- 1 file changed, 17 insertions(+), 11 deletions(-) diff --git a/hooks/nova_cc_utils.py b/hooks/nova_cc_utils.py index 80e8347e..a4e6f4a2 100644 --- a/hooks/nova_cc_utils.py +++ b/hooks/nova_cc_utils.py @@ -367,7 +367,6 @@ def reset_os_release(): def neutron_db_manage(actions): - reset_os_release() net_manager = network_manager() if net_manager in ['neutron', 'quantum']: plugin = neutron_plugin() @@ -420,7 +419,7 @@ def _do_openstack_upgrade(new_src): '--option', 'Dpkg::Options::=--force-confdef', ] - # NOTE(jamespage) pre-stamp database before upgrade + # NOTE(jamespage) pre-stamp neutron database before upgrade from grizzly if cur_os_rel == 'grizzly': neutron_db_manage(['stamp', 'grizzly']) @@ -428,19 +427,26 @@ def _do_openstack_upgrade(new_src): apt_upgrade(options=dpkg_opts, fatal=True, dist=True) apt_install(determine_packages(), fatal=True) - # set CONFIGS to load templates from new release - reset_os_release() - configs = register_configs(release=new_os_rel) - configs.write_all() + if cur_os_rel == 'grizzly': + # NOTE(jamespage) when upgrading from grizzly->havana, config + # files need to be generated prior to performing the db upgrade + reset_os_release() + configs = register_configs(release=new_os_rel) + configs.write_all() + neutron_db_manage(['upgrade', 'head']) + else: + # NOTE(jamespage) upgrade with existing config files as the + # havana->icehouse migration enables new service_plugins which + # create issues with db upgrades + neutron_db_manage(['upgrade', 'head']) + reset_os_release() + configs = register_configs(release=new_os_rel) + configs.write_all() if new_os_rel == 'icehouse': - # NOTE(jamespage) add migration to ML2 after upgrade to icehouse + # NOTE(jamespage) default plugin switch to ml2@icehouse ml2_migration() - if cur_os_rel == 'grizzly' and new_os_rel == 'havana': - # NOTE(jamespage) only force an upgrade when going grizzly->havana - neutron_db_manage(['upgrade', 'havana']) - if eligible_leader(CLUSTER_RES): migrate_database() [service_start(s) for s in services()] From da980995b9eb07ac765bdf641dcd1fed5f5c99d8 Mon Sep 17 00:00:00 2001 From: Corey Bryant Date: Wed, 9 Apr 2014 15:10:26 +0000 Subject: [PATCH 90/98] Add upgrade unit tests --- unit_tests/test_nova_cc_utils.py | 101 ++++++++++++++++++++++++++++++- 1 file changed, 100 insertions(+), 1 deletion(-) diff --git a/unit_tests/test_nova_cc_utils.py b/unit_tests/test_nova_cc_utils.py index d7b6e161..928ddda5 100644 --- a/unit_tests/test_nova_cc_utils.py +++ b/unit_tests/test_nova_cc_utils.py @@ -13,17 +13,29 @@ import nova_cc_utils as utils hookenv.config = _conf TO_PATCH = [ + 'apt_update', + 'apt_upgrade', + 'apt_install', 'config', + 'configure_installation_source', + 'disable_policy_rcd', + 'eligible_leader', + 'enable_policy_rcd', + 'get_os_codename_install_source', 'log', + 'ml2_migration', 'network_manager', + 'neutron_db_manage', 'neutron_plugin', 'neutron_plugin_attribute', 'os_release', + 'register_configs', 'relation_ids', 'remote_unit', '_save_script_rc', 'service_stop', - 'service_start' + 'service_start', + 'services' ] SCRIPTRC_ENV_VARS = { @@ -105,6 +117,12 @@ PLUGIN_ATTRIBUTES = { } +DPKG_OPTS = [ + '--option', 'Dpkg::Options::=--force-confnew', + '--option', 'Dpkg::Options::=--force-confdef', +] + + def fake_plugin_attribute(plugin, attr, net_manager): if plugin in PLUGIN_ATTRIBUTES: try: @@ -458,3 +476,84 @@ class NovaCCUtilsTests(CharmTestCase): _known_hosts.assert_called_with(None) utils.remove_known_host('test', 'bar') _known_hosts.assert_called_with('bar') + + + @patch('subprocess.check_output') + def test_migrate_database(self, check_output): + "Migrate database with nova-manage" + utils.migrate_database() + check_output.assert_called_with(['nova-manage', 'db', 'sync']) + + + @patch.object(utils, 'get_step_upgrade_source') + @patch.object(utils, 'migrate_database') + @patch.object(utils, 'determine_packages') + def test_upgrade_grizzly_icehouse(self, determine_packages, + migrate_database, + get_step_upgrade_source): + "Simulate a call to do_openstack_upgrade() for grizzly->icehouse" + get_step_upgrade_source.return_value = 'cloud:precise-havana' + self.os_release.side_effect = ['grizzly', 'havana'] + self.get_os_codename_install_source.side_effect = ['havana', 'icehouse'] + self.eligible_leader.return_value = True + utils.do_openstack_upgrade() + expected = [call(['stamp', 'grizzly']), call(['upgrade', 'head']), + call(['upgrade', 'head'])] + self.assertEquals(utils.neutron_db_manage.call_args_list, expected) + self.apt_update.assert_called_with(fatal=True) + self.apt_upgrade.assert_called_with(options=DPKG_OPTS, fatal=True, + dist=True) + self.apt_install.assert_called_with(determine_packages(), fatal=True) + expected = [call(release='havana'), call(release='icehouse')] + self.assertEquals(utils.register_configs.call_args_list, expected) + self.assertEquals(utils.ml2_migration.call_count, 1) + self.assertTrue(migrate_database.call_count, 2) + + + @patch.object(utils, 'get_step_upgrade_source') + @patch.object(utils, 'migrate_database') + @patch.object(utils, 'determine_packages') + def test_upgrade_havana_icehouse(self, determine_packages, + migrate_database, + get_step_upgrade_source): + "Simulate a call to do_openstack_upgrade() for havana->icehouse" + get_step_upgrade_source.return_value = None + self.os_release.return_value = 'havana' + self.get_os_codename_install_source.return_value = 'icehouse' + self.eligible_leader.return_value = True + utils.do_openstack_upgrade() + utils.neutron_db_manage.assert_called_with(['upgrade', 'head']) + self.apt_update.assert_called_with(fatal=True) + self.apt_upgrade.assert_called_with(options=DPKG_OPTS, fatal=True, + dist=True) + self.apt_install.assert_called_with(determine_packages(), fatal=True) + utils.register_configs.assert_called_with(release='icehouse') + self.assertEquals(utils.ml2_migration.call_count, 1) + self.assertTrue(migrate_database.call_count, 1) + + + @patch.object(utils, '_do_openstack_upgrade') + def test_ugrade_grizzly_icehouse_source(self, _do_openstack_upgrade): + "Verify get_step_upgrade_source() for grizzly->icehouse" + self.config.side_effect = None + self.config.return_value = 'cloud:precise-icehouse' + with patch_open() as (_open, _file): + _file.read = MagicMock() + _file.readline.return_value = "deb url precise-updates/grizzly main" + utils.do_openstack_upgrade() + expected = [call('cloud:precise-havana'), + call('cloud:precise-icehouse')] + self.assertEquals(_do_openstack_upgrade.call_args_list, expected) + + + @patch.object(utils, '_do_openstack_upgrade') + def test_ugrade_havana_icehouse_source(self, _do_openstack_upgrade): + "Verify get_step_upgrade_source() for havana->icehouse" + self.config.side_effect = None + self.config.return_value = 'cloud:precise-icehouse' + with patch_open() as (_open, _file): + _file.read = MagicMock() + _file.readline.return_value = "deb url precise-updates/havana main" + utils.do_openstack_upgrade() + expected = [call('cloud:precise-icehouse')] + self.assertEquals(_do_openstack_upgrade.call_args_list, expected) From 8778404f5bb81e71a342d97fadde461933665531 Mon Sep 17 00:00:00 2001 From: James Page Date: Wed, 9 Apr 2014 16:22:15 +0100 Subject: [PATCH 91/98] Add logging --- hooks/nova_cc_utils.py | 1 + 1 file changed, 1 insertion(+) diff --git a/hooks/nova_cc_utils.py b/hooks/nova_cc_utils.py index a4e6f4a2..56e39930 100644 --- a/hooks/nova_cc_utils.py +++ b/hooks/nova_cc_utils.py @@ -396,6 +396,7 @@ def ml2_migration(): if net_manager == 'neutron': plugin = neutron_plugin() if plugin == 'ovs': + log('Migrating from openvswitch to ml2 plugin') cmd = [ 'python', '/usr/lib/python2.7/dist-packages/neutron' From 4e0d91c32f97f8afe5f33ebb462d0b540884f920 Mon Sep 17 00:00:00 2001 From: James Page Date: Thu, 10 Apr 2014 17:05:53 +0100 Subject: [PATCH 92/98] Tidy before proposal --- hooks/charmhelpers/contrib/openstack/utils.py | 1 + hooks/nova_cc_hooks.py | 11 ++++++----- unit_tests/test_nova_cc_hooks.py | 12 ++++++------ unit_tests/test_nova_cc_utils.py | 3 ++- 4 files changed, 15 insertions(+), 12 deletions(-) diff --git a/hooks/charmhelpers/contrib/openstack/utils.py b/hooks/charmhelpers/contrib/openstack/utils.py index 3261adf4..ac261fd7 100644 --- a/hooks/charmhelpers/contrib/openstack/utils.py +++ b/hooks/charmhelpers/contrib/openstack/utils.py @@ -65,6 +65,7 @@ SWIFT_CODENAMES = OrderedDict([ ('1.10.0', 'havana'), ('1.9.1', 'havana'), ('1.9.0', 'havana'), + ('1.13.1', 'icehouse'), ('1.13.0', 'icehouse'), ('1.12.0', 'icehouse'), ('1.11.0', 'icehouse'), diff --git a/hooks/nova_cc_hooks.py b/hooks/nova_cc_hooks.py index b694daef..8c268974 100755 --- a/hooks/nova_cc_hooks.py +++ b/hooks/nova_cc_hooks.py @@ -128,7 +128,8 @@ def amqp_changed(): @hooks.hook('shared-db-relation-joined') def db_joined(): - if is_relation_made('pgsql-nova-db') or is_relation_made('pgsql-neutron-db'): + if is_relation_made('pgsql-nova-db') or \ + is_relation_made('pgsql-neutron-db'): # error, postgresql is used e = ('Attempting to associate a mysql database when there is already ' 'associated a postgresql one') @@ -149,8 +150,8 @@ def db_joined(): def pgsql_nova_db_joined(): if is_relation_made('shared-db'): # raise error - e = ('Attempting to associate a postgresql database when there is already ' - 'associated a mysql one') + e = ('Attempting to associate a postgresql database' + ' when there is already associated a mysql one') log(e, level=ERROR) raise Exception(e) @@ -161,8 +162,8 @@ def pgsql_nova_db_joined(): def pgsql_neutron_db_joined(): if is_relation_made('shared-db'): # raise error - e = ('Attempting to associate a postgresql database when there is already ' - 'associated a mysql one') + e = ('Attempting to associate a postgresql database' + ' when there is already associated a mysql one') log(e, level=ERROR) raise Exception(e) diff --git a/unit_tests/test_nova_cc_hooks.py b/unit_tests/test_nova_cc_hooks.py index 9d5787ff..f74fcfcc 100644 --- a/unit_tests/test_nova_cc_hooks.py +++ b/unit_tests/test_nova_cc_hooks.py @@ -169,8 +169,8 @@ class NovaCCHooksTests(CharmTestCase): with self.assertRaises(Exception) as context: hooks.db_joined() self.assertEqual(context.exception.message, - 'Attempting to associate a mysql database when there is already ' - 'associated a postgresql one') + 'Attempting to associate a mysql database when' + ' there is already associated a postgresql one') def test_postgresql_nova_joined_with_db(self): self.is_relation_made.return_value = True @@ -178,8 +178,8 @@ class NovaCCHooksTests(CharmTestCase): with self.assertRaises(Exception) as context: hooks.pgsql_nova_db_joined() self.assertEqual(context.exception.message, - 'Attempting to associate a postgresql database when there is already ' - 'associated a mysql one') + 'Attempting to associate a postgresql database when' + ' there is already associated a mysql one') def test_postgresql_neutron_joined_with_db(self): self.is_relation_made.return_value = True @@ -187,8 +187,8 @@ class NovaCCHooksTests(CharmTestCase): with self.assertRaises(Exception) as context: hooks.pgsql_neutron_db_joined() self.assertEqual(context.exception.message, - 'Attempting to associate a postgresql database when there is already ' - 'associated a mysql one') + 'Attempting to associate a postgresql database when' + ' there is already associated a mysql one') @patch.object(hooks, 'CONFIGS') def test_db_changed_missing_relation_data(self, configs): diff --git a/unit_tests/test_nova_cc_utils.py b/unit_tests/test_nova_cc_utils.py index dd66118e..93cdc60e 100644 --- a/unit_tests/test_nova_cc_utils.py +++ b/unit_tests/test_nova_cc_utils.py @@ -537,7 +537,8 @@ class NovaCCUtilsTests(CharmTestCase): self.config.return_value = 'cloud:precise-icehouse' with patch_open() as (_open, _file): _file.read = MagicMock() - _file.readline.return_value = "deb url precise-updates/grizzly main" + _file.readline.return_value = ("deb url" + " precise-updates/grizzly main") utils.do_openstack_upgrade() expected = [call('cloud:precise-havana'), call('cloud:precise-icehouse')] From 21e62168fe1097899c07a94e5b962f53bcef138a Mon Sep 17 00:00:00 2001 From: James Page Date: Fri, 11 Apr 2014 17:41:42 +0100 Subject: [PATCH 93/98] Fixup bug in shared-db and postgresql changed hooks - neutron uses the db as well --- hooks/nova_cc_hooks.py | 9 ++------- unit_tests/test_nova_cc_hooks.py | 6 ++---- 2 files changed, 4 insertions(+), 11 deletions(-) diff --git a/hooks/nova_cc_hooks.py b/hooks/nova_cc_hooks.py index 8c268974..ce9d7438 100755 --- a/hooks/nova_cc_hooks.py +++ b/hooks/nova_cc_hooks.py @@ -176,12 +176,7 @@ def db_changed(): if 'shared-db' not in CONFIGS.complete_contexts(): log('shared-db relation incomplete. Peer not ready?') return - CONFIGS.write(NOVA_CONF) - - if network_manager() in ['neutron', 'quantum']: - plugin = neutron_plugin() - # DB config might have been moved to main neutron.conf in H? - CONFIGS.write(neutron_plugin_attribute(plugin, 'config')) + CONFIGS.write_all() if eligible_leader(CLUSTER_RES): migrate_database() @@ -196,7 +191,7 @@ def postgresql_nova_db_changed(): if 'pgsql-nova-db' not in CONFIGS.complete_contexts(): log('pgsql-nova-db relation incomplete. Peer not ready?') return - CONFIGS.write(NOVA_CONF) + CONFIGS.write_all() if eligible_leader(CLUSTER_RES): migrate_database() diff --git a/unit_tests/test_nova_cc_hooks.py b/unit_tests/test_nova_cc_hooks.py index f74fcfcc..126a59c1 100644 --- a/unit_tests/test_nova_cc_hooks.py +++ b/unit_tests/test_nova_cc_hooks.py @@ -223,13 +223,11 @@ class NovaCCHooksTests(CharmTestCase): @patch.object(hooks, 'CONFIGS') def test_db_changed(self, configs): self._shared_db_test(configs) - self.assertEquals([call('/etc/nova/nova.conf')], - configs.write.call_args_list) + self.assertTrue(configs.write_all.called) self.migrate_database.assert_called_with() @patch.object(hooks, 'CONFIGS') def test_postgresql_db_changed(self, configs): self._postgresql_db_test(configs) - self.assertEquals([call('/etc/nova/nova.conf')], - configs.write.call_args_list) + self.assertTrue(configs.write_all.called) self.migrate_database.assert_called_with() From 5e77cf6380378736ff4895e39821405172b4b46e Mon Sep 17 00:00:00 2001 From: James Page Date: Sat, 12 Apr 2014 22:25:19 +0100 Subject: [PATCH 94/98] Fixup templates for icehouse --- hooks/nova_cc_context.py | 7 +- templates/icehouse/etc_nova_api-paste.ini | 112 +++++++++++++++++++ templates/icehouse/nova.conf | 130 ++++++++++++++++++++++ 3 files changed, 247 insertions(+), 2 deletions(-) create mode 100644 templates/icehouse/etc_nova_api-paste.ini create mode 100644 templates/icehouse/nova.conf diff --git a/hooks/nova_cc_context.py b/hooks/nova_cc_context.py index 9c046064..e844ad20 100644 --- a/hooks/nova_cc_context.py +++ b/hooks/nova_cc_context.py @@ -184,8 +184,11 @@ class IdentityServiceContext(context.IdentityServiceContext): # the ec2 api needs to know the location of the keystone ec2 # tokens endpoint, set in nova.conf - ec2_tokens = 'http://%s:%s/v2.0/ec2tokens' % (ctxt['service_host'], - ctxt['service_port']) + ec2_tokens = '%s://%s:%s/v2.0/ec2tokens' % ( + ctxt['service_protocol'] or 'http', + ctxt['service_host'], + ctxt['service_port'] + ) ctxt['keystone_ec2_url'] = ec2_tokens ctxt['region'] = config('region') return ctxt diff --git a/templates/icehouse/etc_nova_api-paste.ini b/templates/icehouse/etc_nova_api-paste.ini new file mode 100644 index 00000000..893e866b --- /dev/null +++ b/templates/icehouse/etc_nova_api-paste.ini @@ -0,0 +1,112 @@ +############ +# Metadata # +############ +[composite:metadata] +use = egg:Paste#urlmap +/: meta + +[pipeline:meta] +pipeline = ec2faultwrap logrequest metaapp + +[app:metaapp] +paste.app_factory = nova.api.metadata.handler:MetadataRequestHandler.factory + +####### +# EC2 # +####### + +[composite:ec2] +use = egg:Paste#urlmap +/services/Cloud: ec2cloud + +[composite:ec2cloud] +use = call:nova.api.auth:pipeline_factory +noauth = ec2faultwrap logrequest ec2noauth cloudrequest validator ec2executor +keystone = ec2faultwrap logrequest ec2keystoneauth cloudrequest validator ec2executor + +[filter:ec2faultwrap] +paste.filter_factory = nova.api.ec2:FaultWrapper.factory + +[filter:logrequest] +paste.filter_factory = nova.api.ec2:RequestLogging.factory + +[filter:ec2lockout] +paste.filter_factory = nova.api.ec2:Lockout.factory + +[filter:ec2keystoneauth] +paste.filter_factory = nova.api.ec2:EC2KeystoneAuth.factory + +[filter:ec2noauth] +paste.filter_factory = nova.api.ec2:NoAuth.factory + +[filter:cloudrequest] +controller = nova.api.ec2.cloud.CloudController +paste.filter_factory = nova.api.ec2:Requestify.factory + +[filter:authorizer] +paste.filter_factory = nova.api.ec2:Authorizer.factory + +[filter:validator] +paste.filter_factory = nova.api.ec2:Validator.factory + +[app:ec2executor] +paste.app_factory = nova.api.ec2:Executor.factory + +############# +# OpenStack # +############# + +[composite:osapi_compute] +use = call:nova.api.openstack.urlmap:urlmap_factory +/: oscomputeversions +/v1.1: openstack_compute_api_v2 +/v2: openstack_compute_api_v2 +/v3: openstack_compute_api_v3 + +[composite:openstack_compute_api_v2] +use = call:nova.api.auth:pipeline_factory +noauth = faultwrap sizelimit noauth ratelimit osapi_compute_app_v2 +keystone = faultwrap sizelimit authtoken keystonecontext ratelimit osapi_compute_app_v2 +keystone_nolimit = faultwrap sizelimit authtoken keystonecontext osapi_compute_app_v2 + +[composite:openstack_compute_api_v3] +use = call:nova.api.auth:pipeline_factory_v3 +noauth = faultwrap sizelimit noauth_v3 osapi_compute_app_v3 +keystone = faultwrap sizelimit authtoken keystonecontext osapi_compute_app_v3 + +[filter:faultwrap] +paste.filter_factory = nova.api.openstack:FaultWrapper.factory + +[filter:noauth] +paste.filter_factory = nova.api.openstack.auth:NoAuthMiddleware.factory + +[filter:noauth_v3] +paste.filter_factory = nova.api.openstack.auth:NoAuthMiddlewareV3.factory + +[filter:ratelimit] +paste.filter_factory = nova.api.openstack.compute.limits:RateLimitingMiddleware.factory + +[filter:sizelimit] +paste.filter_factory = nova.api.sizelimit:RequestBodySizeLimiter.factory + +[app:osapi_compute_app_v2] +paste.app_factory = nova.api.openstack.compute:APIRouter.factory + +[app:osapi_compute_app_v3] +paste.app_factory = nova.api.openstack.compute:APIRouterV3.factory + +[pipeline:oscomputeversions] +pipeline = faultwrap oscomputeversionapp + +[app:oscomputeversionapp] +paste.app_factory = nova.api.openstack.compute.versions:Versions.factory + +########## +# Shared # +########## + +[filter:keystonecontext] +paste.filter_factory = nova.api.auth:NovaKeystoneContext.factory + +[filter:authtoken] +paste.filter_factory = keystoneclient.middleware.auth_token:filter_factory diff --git a/templates/icehouse/nova.conf b/templates/icehouse/nova.conf new file mode 100644 index 00000000..67e54f54 --- /dev/null +++ b/templates/icehouse/nova.conf @@ -0,0 +1,130 @@ +############################################################################### +# [ WARNING ] +# Configuration file maintained by Juju. Local changes may be overwritten. +############################################################################### +[DEFAULT] +dhcpbridge_flagfile=/etc/nova/nova.conf +dhcpbridge=/usr/bin/nova-dhcpbridge +logdir=/var/log/nova +state_path=/var/lib/nova +lock_path=/var/lock/nova +force_dhcp_release=True +iscsi_helper=tgtadm +libvirt_use_virtio_for_bridges=True +connection_type=libvirt +root_helper=sudo nova-rootwrap /etc/nova/rootwrap.conf +verbose=True +ec2_private_dns_show_ip=True +api_paste_config=/etc/nova/api-paste.ini +volumes_path=/var/lib/nova/volumes +enabled_apis=ec2,osapi_compute,metadata +auth_strategy=keystone +compute_driver=libvirt.LibvirtDriver +{% if keystone_ec2_url -%} +keystone_ec2_url = {{ keystone_ec2_url }} +{% endif -%} + +{% include "parts/database" %} + +{% include "parts/rabbitmq" %} + +{% if glance_api_servers -%} +glance_api_servers = {{ glance_api_servers }} +{% endif -%} + +{% if rbd_pool -%} +rbd_pool = {{ rbd_pool }} +rbd_user = {{ rbd_user }} +rbd_secret_uuid = {{ rbd_secret_uuid }} +{% endif -%} + +{% if neutron_plugin and neutron_plugin == 'ovs' -%} +libvirt_vif_driver = nova.virt.libvirt.vif.LibvirtGenericVIFDriver +libvirt_user_virtio_for_bridges = True +{% if neutron_security_groups -%} +security_group_api = {{ network_manager }} +nova_firewall_driver = nova.virt.firewall.NoopFirewallDriver +{% endif -%} +{% if external_network -%} +default_floating_pool = {{ external_network }} +{% endif -%} +{% endif -%} + +{% if neutron_plugin and neutron_plugin == 'nvp' -%} +security_group_api = neutron +nova_firewall_driver = nova.virt.firewall.NoopFirewallDriver +{% if external_network -%} +default_floating_pool = {{ external_network }} +{% endif -%} +{% endif -%} + +{% if network_manager_config -%} +{% for key, value in network_manager_config.iteritems() -%} +{{ key }} = {{ value }} +{% endfor -%} +{% endif -%} + +{% if network_manager and network_manager == 'quantum' -%} +network_api_class = nova.network.quantumv2.api.API +quantum_url = {{ neutron_url }} +{% if auth_host -%} +quantum_auth_strategy = keystone +quantum_admin_tenant_name = {{ admin_tenant_name }} +quantum_admin_username = {{ admin_user }} +quantum_admin_password = {{ admin_password }} +quantum_admin_auth_url = {{ auth_protocol }}://{{ auth_host }}:{{ auth_port }}/v2.0 +{% endif -%} +{% elif network_manager and network_manager == 'neutron' -%} +network_api_class = nova.network.neutronv2.api.API +neutron_url = {{ neutron_url }} +{% if auth_host -%} +neutron_auth_strategy = keystone +neutron_admin_tenant_name = {{ admin_tenant_name }} +neutron_admin_username = {{ admin_user }} +neutron_admin_password = {{ admin_password }} +neutron_admin_auth_url = {{ auth_protocol }}://{{ auth_host }}:{{ auth_port }}/v2.0 +{% endif -%} +{% else -%} +network_manager = nova.network.manager.FlatDHCPManager +{% endif -%} + +{% if default_floating_pool -%} +default_floating_pool = {{ default_floating_pool }} +{% endif -%} + +{% if volume_service -%} +volume_api_class=nova.volume.cinder.API +{% endif -%} + +{% if user_config_flags -%} +{% for key, value in user_config_flags.iteritems() -%} +{{ key }} = {{ value }} +{% endfor -%} +{% endif -%} + +{% if listen_ports -%} +{% for key, value in listen_ports.iteritems() -%} +{{ key }} = {{ value }} +{% endfor -%} +{% endif -%} + +{% if sections and 'DEFAULT' in sections -%} +{% for key, value in sections['DEFAULT'] -%} +{{ key }} = {{ value }} +{% endfor -%} +{% endif -%} + +{% if auth_host -%} +[keystone_authtoken] +auth_uri = {{ service_protocol }}://{{ service_host }}:{{ service_port }}/ +auth_host = {{ auth_host }} +auth_port = {{ auth_port }} +auth_protocol = {{ auth_protocol }} +admin_tenant_name = {{ admin_tenant_name }} +admin_user = {{ admin_user }} +admin_password = {{ admin_password }} +{% endif -%} + +[osapi_v3] +enabled=True + From b78fde1b9bc16ff8d948f0c2ba3bae065c7afa5e Mon Sep 17 00:00:00 2001 From: James Page Date: Sat, 12 Apr 2014 22:47:13 +0100 Subject: [PATCH 95/98] Fixup broken compute relation --- templates/icehouse/etc_nova_api-paste.ini | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/templates/icehouse/etc_nova_api-paste.ini b/templates/icehouse/etc_nova_api-paste.ini index 893e866b..aad3d0f3 100644 --- a/templates/icehouse/etc_nova_api-paste.ini +++ b/templates/icehouse/etc_nova_api-paste.ini @@ -110,3 +110,15 @@ paste.filter_factory = nova.api.auth:NovaKeystoneContext.factory [filter:authtoken] paste.filter_factory = keystoneclient.middleware.auth_token:filter_factory +{% if service_host -%} +# NOTE(jamespage) - not used - but required for relation to nova-compute +service_protocol = {{ service_protocol }} +service_host = {{ service_host }} +service_port = {{ service_port }} +auth_host = {{ auth_host }} +auth_port = {{ auth_port }} +auth_protocol = {{ auth_protocol }} +admin_tenant_name = {{ admin_tenant_name }} +admin_user = {{ admin_user }} +admin_password = {{ admin_password }} +{% endif -%} From 1e0fb2389fd7e39101df9de71dfc0657f0134604 Mon Sep 17 00:00:00 2001 From: James Page Date: Mon, 14 Apr 2014 08:42:38 +0100 Subject: [PATCH 96/98] Align with icehouse config for security groups --- templates/icehouse/ml2_conf.ini | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/templates/icehouse/ml2_conf.ini b/templates/icehouse/ml2_conf.ini index 729f8f86..6c1d9089 100644 --- a/templates/icehouse/ml2_conf.ini +++ b/templates/icehouse/ml2_conf.ini @@ -23,7 +23,7 @@ tunnel_types = gre [securitygroup] {% if neutron_security_groups -%} -firewall_driver = neutron.agent.linux.iptables_firewall.OVSHybridIptablesFirewallDriver +enable_security_group = True {% else -%} -firewall_driver = neutron.agent.firewall.NoopFirewallDriver +enable_security_group = False {% endif -%} From d46f5426aef47bbe9f5968ca9f26124ee9c684c1 Mon Sep 17 00:00:00 2001 From: James Page Date: Mon, 14 Apr 2014 10:01:13 +0100 Subject: [PATCH 97/98] Revert previous change --- templates/icehouse/ml2_conf.ini | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/templates/icehouse/ml2_conf.ini b/templates/icehouse/ml2_conf.ini index 6c1d9089..729f8f86 100644 --- a/templates/icehouse/ml2_conf.ini +++ b/templates/icehouse/ml2_conf.ini @@ -23,7 +23,7 @@ tunnel_types = gre [securitygroup] {% if neutron_security_groups -%} -enable_security_group = True +firewall_driver = neutron.agent.linux.iptables_firewall.OVSHybridIptablesFirewallDriver {% else -%} -enable_security_group = False +firewall_driver = neutron.agent.firewall.NoopFirewallDriver {% endif -%} From 253e3c469c595d33f460d6d2f875824477e9f806 Mon Sep 17 00:00:00 2001 From: James Page Date: Mon, 14 Apr 2014 10:10:51 +0100 Subject: [PATCH 98/98] Use enable_security_group --- templates/icehouse/ml2_conf.ini | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/templates/icehouse/ml2_conf.ini b/templates/icehouse/ml2_conf.ini index 729f8f86..53985e24 100644 --- a/templates/icehouse/ml2_conf.ini +++ b/templates/icehouse/ml2_conf.ini @@ -23,7 +23,8 @@ tunnel_types = gre [securitygroup] {% if neutron_security_groups -%} +enable_security_group = True firewall_driver = neutron.agent.linux.iptables_firewall.OVSHybridIptablesFirewallDriver {% else -%} -firewall_driver = neutron.agent.firewall.NoopFirewallDriver +enable_security_group = False {% endif -%}