From a768342daca86183df0fbaf170596512326b99a1 Mon Sep 17 00:00:00 2001 From: David Ames Date: Tue, 14 Jun 2016 16:24:04 -0700 Subject: [PATCH] DNS HA Implement DNS high availability. Pass the correct information to hacluster to register a DNS entry with MAAS 2.0 or greater rather than using a virtual IP. Charm-helpers sync to bring in DNS HA helpers Change-Id: I745271cd85269469b85a9d06fe8af5df8d54ef1c --- .gitignore | 1 + README.md | 34 ++++++ config.yaml | 32 +++++ .../charmhelpers/contrib/hahelpers/cluster.py | 69 +++++++++-- .../contrib/openstack/amulet/deployment.py | 51 ++++---- .../contrib/openstack/ha/__init__.py | 0 .../contrib/openstack/ha/utils.py | 111 ++++++++++++++++++ hooks/charmhelpers/contrib/openstack/ip.py | 11 +- .../contrib/storage/linux/ceph.py | 2 +- hooks/charmhelpers/core/host.py | 56 ++++++++- hooks/cinder_hooks.py | 61 ++++++---- unit_tests/test_cinder_hooks.py | 37 ++++++ 12 files changed, 392 insertions(+), 73 deletions(-) create mode 100644 hooks/charmhelpers/contrib/openstack/ha/__init__.py create mode 100644 hooks/charmhelpers/contrib/openstack/ha/utils.py diff --git a/.gitignore b/.gitignore index f5295367..8b9560ff 100644 --- a/.gitignore +++ b/.gitignore @@ -4,3 +4,4 @@ bin .tox *.sw[nop] *.pyc +.unit-state.db diff --git a/README.md b/README.md index 09a58380..f0eb1885 100644 --- a/README.md +++ b/README.md @@ -121,6 +121,40 @@ overwrite: Whether or not to wipe local storage that of data that may prevent enabled-services: Can be used to separate cinder services between service service units (see previous section) +HA/Clustering +--------------------- + +There are two mutually exclusive high availability options: using virtual +IP(s) or DNS. In both cases, a relationship to hacluster is required which +provides the corosync back end HA functionality. + +To use virtual IP(s) the clustered nodes must be on the same subnet such that +the VIP is a valid IP on the subnet for one of the node's interfaces and each +node has an interface in said subnet. The VIP becomes a highly-available API +endpoint. + +At a minimum, the config option 'vip' must be set in order to use virtual IP +HA. If multiple networks are being used, a VIP should be provided for each +network, separated by spaces. Optionally, vip_iface or vip_cidr may be +specified. + +To use DNS high availability there are several prerequisites. However, DNS HA +does not require the clustered nodes to be on the same subnet. +Currently the DNS HA feature is only available for MAAS 2.0 or greater +environments. MAAS 2.0 requires Juju 2.0 or greater. The clustered nodes must +have static or "reserved" IP addresses registered in MAAS. The DNS hostname(s) +must be pre-registered in MAAS before use with DNS HA. + +At a minimum, the config option 'dns-ha' must be set to true and at least one +of 'os-public-hostname', 'os-internal-hostname' or 'os-internal-hostname' must +be set in order to use DNS HA. One or more of the above hostnames may be set. + +The charm will throw an exception in the following circumstances: +If neither 'vip' nor 'dns-ha' is set and the charm is related to hacluster +If both 'vip' and 'dns-ha' are set as they are mutually exclusive +If 'dns-ha' is set and none of the os-{admin,internal,public}-hostname(s) are +set + Network Space support --------------------- diff --git a/config.yaml b/config.yaml index 459e1410..5b069221 100644 --- a/config.yaml +++ b/config.yaml @@ -173,6 +173,12 @@ options: cloning of images. This option will default to v1 for backwards compatibility older glance services. # HA configuration settings + dns-ha: + type: boolean + default: False + description: | + Use DNS HA with MAAS 2.0. Note if this is set do not set vip + settings below. vip: type: string default: @@ -269,6 +275,32 @@ options: https://cinder.example.com:443/v1/$(tenant_id)s and https://cinder.example.com:443/v2/$(tenant_id)s + os-internal-hostname: + type: string + default: + description: | + The hostname or address of the internal endpoints created for cinder + in the keystone identity provider. + + This value will be used for internal endpoints. For example, an + os-internal-hostname set to 'cinder.internal.example.com' with ssl + enabled will create two internal endpoints for cinder: + + https://cinder.internal.example.com:443/v1/$(tenant_id)s and + https://cinder.internal.example.com:443/v2/$(tenant_id)s + os-admin-hostname: + type: string + default: + description: | + The hostname or address of the admin endpoints created for cinder + in the keystone identity provider. + + This value will be used for admin endpoints. For example, an + os-admin-hostname set to 'cinder.admin.example.com' with ssl enabled will + create two admin endpoints for cinder: + + https://cinder.admin.example.com:443/v1/$(tenant_id)s and + https://cinder.admin.example.com:443/v2/$(tenant_id)s prefer-ipv6: type: boolean default: False diff --git a/hooks/charmhelpers/contrib/hahelpers/cluster.py b/hooks/charmhelpers/contrib/hahelpers/cluster.py index aa0b515d..92325a96 100644 --- a/hooks/charmhelpers/contrib/hahelpers/cluster.py +++ b/hooks/charmhelpers/contrib/hahelpers/cluster.py @@ -41,10 +41,11 @@ from charmhelpers.core.hookenv import ( relation_get, config as config_get, INFO, - ERROR, + DEBUG, WARNING, unit_get, - is_leader as juju_is_leader + is_leader as juju_is_leader, + status_set, ) from charmhelpers.core.decorators import ( retry_on_exception, @@ -60,6 +61,10 @@ class HAIncompleteConfig(Exception): pass +class HAIncorrectConfig(Exception): + pass + + class CRMResourceNotFound(Exception): pass @@ -274,27 +279,71 @@ def get_hacluster_config(exclude_keys=None): Obtains all relevant configuration from charm configuration required for initiating a relation to hacluster: - ha-bindiface, ha-mcastport, vip + ha-bindiface, ha-mcastport, vip, os-internal-hostname, + os-admin-hostname, os-public-hostname param: exclude_keys: list of setting key(s) to be excluded. returns: dict: A dict containing settings keyed by setting name. - raises: HAIncompleteConfig if settings are missing. + raises: HAIncompleteConfig if settings are missing or incorrect. ''' - settings = ['ha-bindiface', 'ha-mcastport', 'vip'] + settings = ['ha-bindiface', 'ha-mcastport', 'vip', 'os-internal-hostname', + 'os-admin-hostname', 'os-public-hostname'] conf = {} for setting in settings: if exclude_keys and setting in exclude_keys: continue conf[setting] = config_get(setting) - missing = [] - [missing.append(s) for s, v in six.iteritems(conf) if v is None] - if missing: - log('Insufficient config data to configure hacluster.', level=ERROR) - raise HAIncompleteConfig + + if not valid_hacluster_config(): + raise HAIncorrectConfig('Insufficient or incorrect config data to ' + 'configure hacluster.') return conf +def valid_hacluster_config(): + ''' + Check that either vip or dns-ha is set. If dns-ha then one of os-*-hostname + must be set. + + Note: ha-bindiface and ha-macastport both have defaults and will always + be set. We only care that either vip or dns-ha is set. + + :returns: boolean: valid config returns true. + raises: HAIncompatibileConfig if settings conflict. + raises: HAIncompleteConfig if settings are missing. + ''' + vip = config_get('vip') + dns = config_get('dns-ha') + if not(bool(vip) ^ bool(dns)): + msg = ('HA: Either vip or dns-ha must be set but not both in order to ' + 'use high availability') + status_set('blocked', msg) + raise HAIncorrectConfig(msg) + + # If dns-ha then one of os-*-hostname must be set + if dns: + dns_settings = ['os-internal-hostname', 'os-admin-hostname', + 'os-public-hostname'] + # At this point it is unknown if one or all of the possible + # network spaces are in HA. Validate at least one is set which is + # the minimum required. + for setting in dns_settings: + if config_get(setting): + log('DNS HA: At least one hostname is set {}: {}' + ''.format(setting, config_get(setting)), + level=DEBUG) + return True + + msg = ('DNS HA: At least one os-*-hostname(s) must be set to use ' + 'DNS HA') + status_set('blocked', msg) + raise HAIncompleteConfig(msg) + + log('VIP HA: VIP is set {}'.format(vip), level=DEBUG) + return True + + def canonical_url(configs, vip_setting='vip'): ''' Returns the correct HTTP URL to this host given the state of HTTPS diff --git a/hooks/charmhelpers/contrib/openstack/amulet/deployment.py b/hooks/charmhelpers/contrib/openstack/amulet/deployment.py index d21c9c78..6b917d0c 100644 --- a/hooks/charmhelpers/contrib/openstack/amulet/deployment.py +++ b/hooks/charmhelpers/contrib/openstack/amulet/deployment.py @@ -43,9 +43,6 @@ class OpenStackAmuletDeployment(AmuletDeployment): self.openstack = openstack self.source = source self.stable = stable - # Note(coreycb): this needs to be changed when new next branches come - # out. - self.current_next = "trusty" def get_logger(self, name="deployment-logger", level=logging.DEBUG): """Get a logger object that will log to stdout.""" @@ -72,38 +69,34 @@ class OpenStackAmuletDeployment(AmuletDeployment): self.log.info('OpenStackAmuletDeployment: determine branch locations') - # Charms outside the lp:~openstack-charmers namespace - base_charms = ['mysql', 'mongodb', 'nrpe'] - - # Force these charms to current series even when using an older series. - # ie. Use trusty/nrpe even when series is precise, as the P charm - # does not possess the necessary external master config and hooks. - force_series_current = ['nrpe'] - - if self.series in ['precise', 'trusty']: - base_series = self.series - else: - base_series = self.current_next + # Charms outside the ~openstack-charmers + base_charms = { + 'mysql': ['precise', 'trusty'], + 'mongodb': ['precise', 'trusty'], + 'nrpe': ['precise', 'trusty'], + } for svc in other_services: - if svc['name'] in force_series_current: - base_series = self.current_next # If a location has been explicitly set, use it if svc.get('location'): continue - if self.stable: - temp = 'lp:charms/{}/{}' - svc['location'] = temp.format(base_series, - svc['name']) + if svc['name'] in base_charms: + # NOTE: not all charms have support for all series we + # want/need to test against, so fix to most recent + # that each base charm supports + target_series = self.series + if self.series not in base_charms[svc['name']]: + target_series = base_charms[svc['name']][-1] + svc['location'] = 'cs:{}/{}'.format(target_series, + svc['name']) + elif self.stable: + svc['location'] = 'cs:{}/{}'.format(self.series, + svc['name']) else: - if svc['name'] in base_charms: - temp = 'lp:charms/{}/{}' - svc['location'] = temp.format(base_series, - svc['name']) - else: - temp = 'lp:~openstack-charmers/charms/{}/{}/next' - svc['location'] = temp.format(self.current_next, - svc['name']) + svc['location'] = 'cs:~openstack-charmers-next/{}/{}'.format( + self.series, + svc['name'] + ) return other_services diff --git a/hooks/charmhelpers/contrib/openstack/ha/__init__.py b/hooks/charmhelpers/contrib/openstack/ha/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/hooks/charmhelpers/contrib/openstack/ha/utils.py b/hooks/charmhelpers/contrib/openstack/ha/utils.py new file mode 100644 index 00000000..34064237 --- /dev/null +++ b/hooks/charmhelpers/contrib/openstack/ha/utils.py @@ -0,0 +1,111 @@ +# Copyright 2014-2016 Canonical Limited. +# +# This file is part of charm-helpers. +# +# charm-helpers is free software: you can redistribute it and/or modify +# it under the terms of the GNU Lesser General Public License version 3 as +# published by the Free Software Foundation. +# +# charm-helpers is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Lesser General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public License +# along with charm-helpers. If not, see . + +# +# Copyright 2016 Canonical Ltd. +# +# Authors: +# Openstack Charmers < +# + +""" +Helpers for high availability. +""" + +import re + +from charmhelpers.core.hookenv import ( + log, + relation_set, + charm_name, + config, + status_set, + DEBUG, +) + +from charmhelpers.contrib.openstack.ip import ( + resolve_address, +) + + +class DNSHAException(Exception): + """Raised when an error occurs setting up DNS HA + """ + + pass + + +def update_dns_ha_resource_params(resources, resource_params, + relation_id=None, + crm_ocf='ocf:maas:dns'): + """ Check for os-*-hostname settings and update resource dictionaries for + the HA relation. + + @param resources: Pointer to dictionary of resources. + Usually instantiated in ha_joined(). + @param resource_params: Pointer to dictionary of resource parameters. + Usually instantiated in ha_joined() + @param relation_id: Relation ID of the ha relation + @param crm_ocf: Corosync Open Cluster Framework resource agent to use for + DNS HA + """ + + settings = ['os-admin-hostname', 'os-internal-hostname', + 'os-public-hostname'] + + # Check which DNS settings are set and update dictionaries + hostname_group = [] + for setting in settings: + hostname = config(setting) + if hostname is None: + log('DNS HA: Hostname setting {} is None. Ignoring.' + ''.format(setting), + DEBUG) + continue + m = re.search('os-(.+?)-hostname', setting) + if m: + networkspace = m.group(1) + else: + msg = ('Unexpected DNS hostname setting: {}. ' + 'Cannot determine network space name' + ''.format(setting)) + status_set('blocked', msg) + raise DNSHAException(msg) + + hostname_key = 'res_{}_{}_hostname'.format(charm_name(), networkspace) + if hostname_key in hostname_group: + log('DNS HA: Resource {}: {} already exists in ' + 'hostname group - skipping'.format(hostname_key, hostname), + DEBUG) + continue + + hostname_group.append(hostname_key) + resources[hostname_key] = crm_ocf + resource_params[hostname_key] = ( + 'params fqdn="{}" ip_address="{}" ' + ''.format(hostname, resolve_address(endpoint_type=networkspace, + override=False))) + + if len(hostname_group) >= 1: + log('DNS HA: Hostname group is set with {} as members. ' + 'Informing the ha relation'.format(' '.join(hostname_group)), + DEBUG) + relation_set(relation_id=relation_id, groups={ + 'grp_{}_hostnames'.format(charm_name()): ' '.join(hostname_group)}) + else: + msg = 'DNS HA: Hostname group has no members.' + status_set('blocked', msg) + raise DNSHAException(msg) diff --git a/hooks/charmhelpers/contrib/openstack/ip.py b/hooks/charmhelpers/contrib/openstack/ip.py index 532a1dc1..7875b997 100644 --- a/hooks/charmhelpers/contrib/openstack/ip.py +++ b/hooks/charmhelpers/contrib/openstack/ip.py @@ -109,7 +109,7 @@ def _get_address_override(endpoint_type=PUBLIC): return addr_override.format(service_name=service_name()) -def resolve_address(endpoint_type=PUBLIC): +def resolve_address(endpoint_type=PUBLIC, override=True): """Return unit address depending on net config. If unit is clustered with vip(s) and has net splits defined, return vip on @@ -119,10 +119,13 @@ def resolve_address(endpoint_type=PUBLIC): split if one is configured, or a Juju 2.0 extra-binding has been used. :param endpoint_type: Network endpoing type + :param override: Accept hostname overrides or not """ - resolved_address = _get_address_override(endpoint_type) - if resolved_address: - return resolved_address + resolved_address = None + if override: + resolved_address = _get_address_override(endpoint_type) + if resolved_address: + return resolved_address vips = config('vip') if vips: diff --git a/hooks/charmhelpers/contrib/storage/linux/ceph.py b/hooks/charmhelpers/contrib/storage/linux/ceph.py index 2528f5cf..b2484e78 100644 --- a/hooks/charmhelpers/contrib/storage/linux/ceph.py +++ b/hooks/charmhelpers/contrib/storage/linux/ceph.py @@ -1231,7 +1231,7 @@ class CephConfContext(object): permitted = self.permitted_sections if permitted: - diff = set(conf.keys()).symmetric_difference(set(permitted)) + diff = set(conf.keys()).difference(set(permitted)) if diff: log("Config-flags contains invalid keys '%s' - they will be " "ignored" % (', '.join(diff)), level=WARNING) diff --git a/hooks/charmhelpers/core/host.py b/hooks/charmhelpers/core/host.py index 64b2df55..e367e450 100644 --- a/hooks/charmhelpers/core/host.py +++ b/hooks/charmhelpers/core/host.py @@ -176,7 +176,7 @@ def init_is_systemd(): def adduser(username, password=None, shell='/bin/bash', system_user=False, - primary_group=None, secondary_groups=None): + primary_group=None, secondary_groups=None, uid=None): """Add a user to the system. Will log but otherwise succeed if the user already exists. @@ -187,15 +187,21 @@ def adduser(username, password=None, shell='/bin/bash', system_user=False, :param bool system_user: Whether to create a login or system user :param str primary_group: Primary group for user; defaults to username :param list secondary_groups: Optional list of additional groups + :param int uid: UID for user being created :returns: The password database entry struct, as returned by `pwd.getpwnam` """ try: user_info = pwd.getpwnam(username) log('user {0} already exists!'.format(username)) + if uid: + user_info = pwd.getpwuid(int(uid)) + log('user with uid {0} already exists!'.format(uid)) except KeyError: log('creating user {0}'.format(username)) cmd = ['useradd'] + if uid: + cmd.extend(['--uid', str(uid)]) if system_user or password is None: cmd.append('--system') else: @@ -230,14 +236,58 @@ def user_exists(username): return user_exists -def add_group(group_name, system_group=False): - """Add a group to the system""" +def uid_exists(uid): + """Check if a uid exists""" + try: + pwd.getpwuid(uid) + uid_exists = True + except KeyError: + uid_exists = False + return uid_exists + + +def group_exists(groupname): + """Check if a group exists""" + try: + grp.getgrnam(groupname) + group_exists = True + except KeyError: + group_exists = False + return group_exists + + +def gid_exists(gid): + """Check if a gid exists""" + try: + grp.getgrgid(gid) + gid_exists = True + except KeyError: + gid_exists = False + return gid_exists + + +def add_group(group_name, system_group=False, gid=None): + """Add a group to the system + + Will log but otherwise succeed if the group already exists. + + :param str group_name: group to create + :param bool system_group: Create system group + :param int gid: GID for user being created + + :returns: The password database entry struct, as returned by `grp.getgrnam` + """ try: group_info = grp.getgrnam(group_name) log('group {0} already exists!'.format(group_name)) + if gid: + group_info = grp.getgrgid(gid) + log('group with gid {0} already exists!'.format(gid)) except KeyError: log('creating group {0}'.format(group_name)) cmd = ['addgroup'] + if gid: + cmd.extend(['--gid', str(gid)]) if system_group: cmd.append('--system') else: diff --git a/hooks/cinder_hooks.py b/hooks/cinder_hooks.py index 87bfc686..5da86ed6 100755 --- a/hooks/cinder_hooks.py +++ b/hooks/cinder_hooks.py @@ -83,6 +83,10 @@ from charmhelpers.contrib.hahelpers.cluster import ( get_hacluster_config, ) +from charmhelpers.contrib.openstack.ha.utils import ( + update_dns_ha_resource_params, +) + from charmhelpers.payload.execd import execd_preinstall from charmhelpers.contrib.network.ip import ( get_iface_for_address, @@ -434,35 +438,40 @@ def ha_joined(relation_id=None): 'res_cinder_haproxy': 'op monitor interval="5s"' } - vip_group = [] - for vip in cluster_config['vip'].split(): - if is_ipv6(vip): - res_cinder_vip = 'ocf:heartbeat:IPv6addr' - vip_params = 'ipv6addr' - else: - res_cinder_vip = 'ocf:heartbeat:IPaddr2' - vip_params = 'ip' + if config('dns-ha'): + update_dns_ha_resource_params(relation_id=relation_id, + resources=resources, + resource_params=resource_params) + else: + vip_group = [] + for vip in cluster_config['vip'].split(): + if is_ipv6(vip): + res_cinder_vip = 'ocf:heartbeat:IPv6addr' + vip_params = 'ipv6addr' + else: + res_cinder_vip = 'ocf:heartbeat:IPaddr2' + vip_params = 'ip' - iface = (get_iface_for_address(vip) or - config('vip_iface')) - netmask = (get_netmask_for_address(vip) or - config('vip_cidr')) + iface = (get_iface_for_address(vip) or + config('vip_iface')) + netmask = (get_netmask_for_address(vip) or + config('vip_cidr')) - if iface is not None: - vip_key = 'res_cinder_{}_vip'.format(iface) - resources[vip_key] = res_cinder_vip - resource_params[vip_key] = ( - 'params {ip}="{vip}" cidr_netmask="{netmask}"' - ' nic="{iface}"'.format(ip=vip_params, - vip=vip, - iface=iface, - netmask=netmask) - ) - vip_group.append(vip_key) + if iface is not None: + vip_key = 'res_cinder_{}_vip'.format(iface) + resources[vip_key] = res_cinder_vip + resource_params[vip_key] = ( + 'params {ip}="{vip}" cidr_netmask="{netmask}"' + ' nic="{iface}"'.format(ip=vip_params, + vip=vip, + iface=iface, + netmask=netmask) + ) + vip_group.append(vip_key) - if len(vip_group) >= 1: - relation_set(relation_id=relation_id, - groups={'grp_cinder_vips': ' '.join(vip_group)}) + if len(vip_group) >= 1: + relation_set(relation_id=relation_id, + groups={'grp_cinder_vips': ' '.join(vip_group)}) init_services = { 'res_cinder_haproxy': 'haproxy' diff --git a/unit_tests/test_cinder_hooks.py b/unit_tests/test_cinder_hooks.py index 7ed7e600..ecf54a47 100644 --- a/unit_tests/test_cinder_hooks.py +++ b/unit_tests/test_cinder_hooks.py @@ -73,6 +73,8 @@ TO_PATCH = [ 'configure_installation_source', 'openstack_upgrade_available', 'os_release', + # charmhelpers.contrib.openstack.openstack.ha.utils + 'update_dns_ha_resource_params', # charmhelpers.contrib.hahelpers.cluster_utils 'is_elected_leader', 'get_hacluster_config', @@ -679,6 +681,41 @@ class TestJoinedHooks(CharmTestCase): self.ensure_ceph_keyring.return_value = True hooks.hooks.execute(['hooks/ceph-relation-changed']) + def test_ha_joined_dns_ha(self): + def _fake_update(resources, resource_params, relation_id=None): + resources.update({'res_cinder_public_hostname': 'ocf:maas:dns'}) + resource_params.update({'res_cinder_public_hostname': + 'params fqdn="keystone.maas" ' + 'ip_address="10.0.0.1"'}) + + self.test_config.set('dns-ha', True) + self.get_hacluster_config.return_value = { + 'vip': None, + 'ha-bindiface': 'em0', + 'ha-mcastport': '8080', + 'os-admin-hostname': None, + 'os-internal-hostname': None, + 'os-public-hostname': 'keystone.maas', + } + args = { + 'relation_id': None, + 'corosync_bindiface': 'em0', + 'corosync_mcastport': '8080', + 'init_services': {'res_cinder_haproxy': 'haproxy'}, + 'resources': {'res_cinder_public_hostname': 'ocf:maas:dns', + 'res_cinder_haproxy': 'lsb:haproxy'}, + 'resource_params': { + 'res_cinder_public_hostname': 'params fqdn="keystone.maas" ' + 'ip_address="10.0.0.1"', + 'res_cinder_haproxy': 'op monitor interval="5s"'}, + 'clones': {'cl_cinder_haproxy': 'res_cinder_haproxy'} + } + self.update_dns_ha_resource_params.side_effect = _fake_update + + hooks.ha_joined() + self.assertTrue(self.update_dns_ha_resource_params.called) + self.relation_set.assert_called_with(**args) + class TestDepartedHooks(CharmTestCase):