diff --git a/.bzrignore b/.bzrignore new file mode 100644 index 00000000..a2c7a097 --- /dev/null +++ b/.bzrignore @@ -0,0 +1,2 @@ +bin +.coverage diff --git a/Makefile b/Makefile index 8d872e13..e101bf32 100644 --- a/Makefile +++ b/Makefile @@ -6,11 +6,15 @@ lint: @charm proof test: - @echo Starting tests... - @$(PYTHON) /usr/bin/nosetests --nologcapture unit_tests + @$(PYTHON) /usr/bin/nosetests --nologcapture --with-coverage unit_tests -sync: - @charm-helper-sync -c charm-helpers.yaml +bin/charm_helpers_sync.py: + @mkdir -p bin + @bzr cat lp:charm-helpers/tools/charm_helpers_sync/charm_helpers_sync.py \ + > bin/charm_helpers_sync.py + +sync: bin/charm_helpers_sync.py + @$(PYTHON) bin/charm_helpers_sync.py -c charm-helpers.yaml publish: lint test bzr push lp:charms/cinder diff --git a/charm-helpers.yaml b/charm-helpers.yaml index b4a77d14..e64ed4ed 100644 --- a/charm-helpers.yaml +++ b/charm-helpers.yaml @@ -10,3 +10,4 @@ include: - cluster - fetch - payload.execd + - contrib.network.ip diff --git a/config.yaml b/config.yaml index 762d3ed7..c816a216 100644 --- a/config.yaml +++ b/config.yaml @@ -102,15 +102,11 @@ options: # HA configuration settings vip: type: string - description: "Virtual IP to use to front cinder API in ha configuration" - vip_iface: - type: string - default: eth0 - description: "Network Interface where to place the Virtual IP" - vip_cidr: - type: int - default: 24 - description: "Netmask that will be used for the Virtual IP" + description: | + Virtual IP(s) to use to front API services in HA configuration. + . + If multiple networks are being used, a VIP should be provided for each + network, separated by spaces. ha-bindiface: type: string default: eth0 @@ -142,4 +138,27 @@ options: config-flags: type: string description: Comma separated list of key=value config flags to be set in cinder.conf. + # Network configuration options + # by default all access is over 'private-address' + os-admin-network: + type: string + description: | + The IP address and netmask of the OpenStack Admin network (e.g., + 192.168.0.0/24) + . + This network will be used for admin endpoints. + os-internal-network: + type: string + description: | + The IP address and netmask of the OpenStack Internal network (e.g., + 192.168.0.0/24) + . + This network will be used for internal endpoints. + os-public-network: + type: string + description: | + The IP address and netmask of the OpenStack Public network (e.g., + 192.168.0.0/24) + . + This network will be used for public endpoints. diff --git a/hooks/charmhelpers/contrib/hahelpers/cluster.py b/hooks/charmhelpers/contrib/hahelpers/cluster.py index 4e1a473d..505de6b2 100644 --- a/hooks/charmhelpers/contrib/hahelpers/cluster.py +++ b/hooks/charmhelpers/contrib/hahelpers/cluster.py @@ -146,12 +146,12 @@ def get_hacluster_config(): Obtains all relevant configuration from charm configuration required for initiating a relation to hacluster: - ha-bindiface, ha-mcastport, vip, vip_iface, vip_cidr + ha-bindiface, ha-mcastport, vip returns: dict: A dict containing settings keyed by setting name. raises: HAIncompleteConfig if settings are missing. ''' - settings = ['ha-bindiface', 'ha-mcastport', 'vip', 'vip_iface', 'vip_cidr'] + settings = ['ha-bindiface', 'ha-mcastport', 'vip'] conf = {} for setting in settings: conf[setting] = config_get(setting) diff --git a/hooks/charmhelpers/contrib/network/__init__.py b/hooks/charmhelpers/contrib/network/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/hooks/charmhelpers/contrib/network/ip.py b/hooks/charmhelpers/contrib/network/ip.py new file mode 100644 index 00000000..0972e91a --- /dev/null +++ b/hooks/charmhelpers/contrib/network/ip.py @@ -0,0 +1,156 @@ +import sys + +from functools import partial + +from charmhelpers.fetch import apt_install +from charmhelpers.core.hookenv import ( + ERROR, log, +) + +try: + import netifaces +except ImportError: + apt_install('python-netifaces') + import netifaces + +try: + import netaddr +except ImportError: + apt_install('python-netaddr') + import netaddr + + +def _validate_cidr(network): + try: + netaddr.IPNetwork(network) + except (netaddr.core.AddrFormatError, ValueError): + raise ValueError("Network (%s) is not in CIDR presentation format" % + network) + + +def get_address_in_network(network, fallback=None, fatal=False): + """ + Get an IPv4 or IPv6 address within the network from the host. + + :param network (str): CIDR presentation format. For example, + '192.168.1.0/24'. + :param fallback (str): If no address is found, return fallback. + :param fatal (boolean): If no address is found, fallback is not + set and fatal is True then exit(1). + + """ + + def not_found_error_out(): + log("No IP address found in network: %s" % network, + level=ERROR) + sys.exit(1) + + if network is None: + if fallback is not None: + return fallback + else: + if fatal: + not_found_error_out() + + _validate_cidr(network) + network = netaddr.IPNetwork(network) + for iface in netifaces.interfaces(): + addresses = netifaces.ifaddresses(iface) + if network.version == 4 and netifaces.AF_INET in addresses: + addr = addresses[netifaces.AF_INET][0]['addr'] + netmask = addresses[netifaces.AF_INET][0]['netmask'] + cidr = netaddr.IPNetwork("%s/%s" % (addr, netmask)) + if cidr in network: + return str(cidr.ip) + if network.version == 6 and netifaces.AF_INET6 in addresses: + for addr in addresses[netifaces.AF_INET6]: + if not addr['addr'].startswith('fe80'): + cidr = netaddr.IPNetwork("%s/%s" % (addr['addr'], + addr['netmask'])) + if cidr in network: + return str(cidr.ip) + + if fallback is not None: + return fallback + + if fatal: + not_found_error_out() + + return None + + +def is_ipv6(address): + '''Determine whether provided address is IPv6 or not''' + try: + address = netaddr.IPAddress(address) + except netaddr.AddrFormatError: + # probably a hostname - so not an address at all! + return False + else: + return address.version == 6 + + +def is_address_in_network(network, address): + """ + Determine whether the provided address is within a network range. + + :param network (str): CIDR presentation format. For example, + '192.168.1.0/24'. + :param address: An individual IPv4 or IPv6 address without a net + mask or subnet prefix. For example, '192.168.1.1'. + :returns boolean: Flag indicating whether address is in network. + """ + try: + network = netaddr.IPNetwork(network) + except (netaddr.core.AddrFormatError, ValueError): + raise ValueError("Network (%s) is not in CIDR presentation format" % + network) + try: + address = netaddr.IPAddress(address) + except (netaddr.core.AddrFormatError, ValueError): + raise ValueError("Address (%s) is not in correct presentation format" % + address) + if address in network: + return True + else: + return False + + +def _get_for_address(address, key): + """Retrieve an attribute of or the physical interface that + the IP address provided could be bound to. + + :param address (str): An individual IPv4 or IPv6 address without a net + mask or subnet prefix. For example, '192.168.1.1'. + :param key: 'iface' for the physical interface name or an attribute + of the configured interface, for example 'netmask'. + :returns str: Requested attribute or None if address is not bindable. + """ + address = netaddr.IPAddress(address) + for iface in netifaces.interfaces(): + addresses = netifaces.ifaddresses(iface) + if address.version == 4 and netifaces.AF_INET in addresses: + addr = addresses[netifaces.AF_INET][0]['addr'] + netmask = addresses[netifaces.AF_INET][0]['netmask'] + cidr = netaddr.IPNetwork("%s/%s" % (addr, netmask)) + if address in cidr: + if key == 'iface': + return iface + else: + return addresses[netifaces.AF_INET][0][key] + if address.version == 6 and netifaces.AF_INET6 in addresses: + for addr in addresses[netifaces.AF_INET6]: + if not addr['addr'].startswith('fe80'): + cidr = netaddr.IPNetwork("%s/%s" % (addr['addr'], + addr['netmask'])) + if address in cidr: + if key == 'iface': + return iface + else: + return addr[key] + return None + + +get_iface_for_address = partial(_get_for_address, key='iface') + +get_netmask_for_address = partial(_get_for_address, key='netmask') diff --git a/hooks/charmhelpers/contrib/openstack/context.py b/hooks/charmhelpers/contrib/openstack/context.py index 7d745cdb..92c41b23 100644 --- a/hooks/charmhelpers/contrib/openstack/context.py +++ b/hooks/charmhelpers/contrib/openstack/context.py @@ -21,6 +21,7 @@ from charmhelpers.core.hookenv import ( relation_get, relation_ids, related_units, + relation_set, unit_get, unit_private_ip, ERROR, @@ -43,6 +44,8 @@ from charmhelpers.contrib.openstack.neutron import ( neutron_plugin_attribute, ) +from charmhelpers.contrib.network.ip import get_address_in_network + CA_CERT_PATH = '/usr/local/share/ca-certificates/keystone_juju_ca_cert.crt' @@ -135,8 +138,26 @@ class SharedDBContext(OSContextGenerator): 'Missing required charm config options. ' '(database name and user)') raise OSContextError + ctxt = {} + # NOTE(jamespage) if mysql charm provides a network upon which + # access to the database should be made, reconfigure relation + # with the service units local address and defer execution + access_network = relation_get('access-network') + if access_network is not None: + if self.relation_prefix is not None: + hostname_key = "{}_hostname".format(self.relation_prefix) + else: + hostname_key = "hostname" + access_hostname = get_address_in_network(access_network, + unit_get('private-address')) + set_hostname = relation_get(attribute=hostname_key, + unit=local_unit()) + if set_hostname != access_hostname: + relation_set(relation_settings={hostname_key: access_hostname}) + return ctxt # Defer any further hook execution for now.... + password_setting = 'password' if self.relation_prefix: password_setting = self.relation_prefix + '_password' @@ -341,10 +362,12 @@ class CephContext(OSContextGenerator): use_syslog = str(config('use-syslog')).lower() for rid in relation_ids('ceph'): for unit in related_units(rid): - mon_hosts.append(relation_get('private-address', rid=rid, - unit=unit)) auth = relation_get('auth', rid=rid, unit=unit) key = relation_get('key', rid=rid, unit=unit) + ceph_addr = \ + relation_get('ceph-public-address', rid=rid, unit=unit) or \ + relation_get('private-address', rid=rid, unit=unit) + mon_hosts.append(ceph_addr) ctxt = { 'mon_hosts': ' '.join(mon_hosts), @@ -378,7 +401,9 @@ class HAProxyContext(OSContextGenerator): cluster_hosts = {} l_unit = local_unit().replace('/', '-') - cluster_hosts[l_unit] = unit_get('private-address') + cluster_hosts[l_unit] = \ + get_address_in_network(config('os-internal-network'), + unit_get('private-address')) for rid in relation_ids('cluster'): for unit in related_units(rid): diff --git a/hooks/charmhelpers/contrib/openstack/ip.py b/hooks/charmhelpers/contrib/openstack/ip.py new file mode 100644 index 00000000..7e7a536f --- /dev/null +++ b/hooks/charmhelpers/contrib/openstack/ip.py @@ -0,0 +1,75 @@ +from charmhelpers.core.hookenv import ( + config, + unit_get, +) + +from charmhelpers.contrib.network.ip import ( + get_address_in_network, + is_address_in_network, + is_ipv6, +) + +from charmhelpers.contrib.hahelpers.cluster import is_clustered + +PUBLIC = 'public' +INTERNAL = 'int' +ADMIN = 'admin' + +_address_map = { + PUBLIC: { + 'config': 'os-public-network', + 'fallback': 'public-address' + }, + INTERNAL: { + 'config': 'os-internal-network', + 'fallback': 'private-address' + }, + ADMIN: { + 'config': 'os-admin-network', + 'fallback': 'private-address' + } +} + + +def canonical_url(configs, endpoint_type=PUBLIC): + ''' + Returns the correct HTTP URL to this host given the state of HTTPS + configuration, hacluster and charm configuration. + + :configs OSTemplateRenderer: A config tempating object to inspect for + a complete https context. + :endpoint_type str: The endpoint type to resolve. + + :returns str: Base URL for services on the current service unit. + ''' + scheme = 'http' + if 'https' in configs.complete_contexts(): + scheme = 'https' + address = resolve_address(endpoint_type) + if is_ipv6(address): + address = "[{}]".format(address) + return '%s://%s' % (scheme, address) + + +def resolve_address(endpoint_type=PUBLIC): + resolved_address = None + if is_clustered(): + if config(_address_map[endpoint_type]['config']) is None: + # Assume vip is simple and pass back directly + resolved_address = config('vip') + else: + for vip in config('vip').split(): + if is_address_in_network( + config(_address_map[endpoint_type]['config']), + vip): + resolved_address = vip + else: + resolved_address = get_address_in_network( + config(_address_map[endpoint_type]['config']), + unit_get(_address_map[endpoint_type]['fallback']) + ) + if resolved_address is None: + raise ValueError('Unable to resolve a suitable IP address' + ' based on charm state and configuration') + else: + return resolved_address diff --git a/hooks/charmhelpers/contrib/openstack/templates/haproxy.cfg b/hooks/charmhelpers/contrib/openstack/templates/haproxy.cfg index 56ed913e..a95eddd1 100644 --- a/hooks/charmhelpers/contrib/openstack/templates/haproxy.cfg +++ b/hooks/charmhelpers/contrib/openstack/templates/haproxy.cfg @@ -27,7 +27,12 @@ listen stats :8888 {% if units -%} {% for service, ports in service_ports.iteritems() -%} -listen {{ service }} 0.0.0.0:{{ ports[0] }} +listen {{ service }}_ipv4 0.0.0.0:{{ ports[0] }} + balance roundrobin + {% for unit, address in units.iteritems() -%} + server {{ unit }} {{ address }}:{{ ports[1] }} check + {% endfor %} +listen {{ service }}_ipv6 :::{{ ports[0] }} balance roundrobin {% for unit, address in units.iteritems() -%} server {{ unit }} {{ address }}:{{ ports[1] }} check diff --git a/hooks/charmhelpers/core/host.py b/hooks/charmhelpers/core/host.py index 8b617a42..d934f940 100644 --- a/hooks/charmhelpers/core/host.py +++ b/hooks/charmhelpers/core/host.py @@ -322,6 +322,10 @@ def cmp_pkgrevno(package, revno, pkgcache=None): import apt_pkg if not pkgcache: apt_pkg.init() + # Force Apt to build its cache in memory. That way we avoid race + # conditions with other applications building the cache in the same + # place. + apt_pkg.config.set("Dir::Cache::pkgcache", "") pkgcache = apt_pkg.Cache() pkg = pkgcache[package] return apt_pkg.version_compare(pkg.current_ver.ver_str, revno) diff --git a/hooks/cinder_contexts.py b/hooks/cinder_contexts.py index 2c5a31ed..98f9949b 100644 --- a/hooks/cinder_contexts.py +++ b/hooks/cinder_contexts.py @@ -105,5 +105,6 @@ class StorageBackendContext(OSContextGenerator): class LoggingConfigContext(OSContextGenerator): + def __call__(self): return {'debug': config('debug'), 'verbose': config('verbose')} diff --git a/hooks/cinder_hooks.py b/hooks/cinder_hooks.py index 78ab9c1b..7a30d4f4 100755 --- a/hooks/cinder_hooks.py +++ b/hooks/cinder_hooks.py @@ -34,7 +34,7 @@ from charmhelpers.core.hookenv import ( service_name, unit_get, log, - ERROR + ERROR, ) from charmhelpers.fetch import apt_install, apt_update @@ -46,13 +46,21 @@ from charmhelpers.contrib.openstack.utils import ( from charmhelpers.contrib.storage.linux.ceph import ensure_ceph_keyring from charmhelpers.contrib.hahelpers.cluster import ( - canonical_url, eligible_leader, is_leader, get_hacluster_config, ) from charmhelpers.payload.execd import execd_preinstall +from charmhelpers.contrib.network.ip import ( + get_iface_for_address, + get_netmask_for_address, + get_address_in_network +) +from charmhelpers.contrib.openstack.ip import ( + canonical_url, + PUBLIC, INTERNAL, ADMIN +) hooks = Hooks() @@ -65,7 +73,7 @@ def install(): conf = config() src = conf['openstack-origin'] if (lsb_release()['DISTRIB_CODENAME'] == 'precise' and - src == 'distro'): + src == 'distro'): src = 'cloud:precise-folsom' configure_installation_source(src) apt_update() @@ -93,6 +101,9 @@ def config_changed(): CONFIGS.write_all() configure_https() + for rid in relation_ids('cluster'): + cluster_joined(relation_id=rid) + @hooks.hook('shared-db-relation-joined') def db_joined(): @@ -175,17 +186,24 @@ def identity_joined(rid=None): if not eligible_leader(CLUSTER_RES): return - conf = config() - - port = conf['api-listening-port'] - url = canonical_url(CONFIGS) + ':%s/v1/$(tenant_id)s' % port - + public_url = '{}:{}/v1/$(tenant_id)s'.format( + canonical_url(CONFIGS, PUBLIC), + config('api-listening-port') + ) + internal_url = '{}:{}/v1/$(tenant_id)s'.format( + canonical_url(CONFIGS, INTERNAL), + config('api-listening-port') + ) + admin_url = '{}:{}/v1/$(tenant_id)s'.format( + canonical_url(CONFIGS, ADMIN), + config('api-listening-port') + ) settings = { - 'region': conf['region'], + 'region': config('region'), 'service': 'cinder', - 'public_url': url, - 'internal_url': url, - 'admin_url': url, + 'public_url': public_url, + 'internal_url': internal_url, + 'admin_url': admin_url, } relation_set(relation_id=rid, **settings) @@ -228,6 +246,14 @@ def ceph_changed(): replicas=_config['ceph-osd-replication-count']) +@hooks.hook('cluster-relation-joined') +def cluster_joined(relation_id=None): + address = get_address_in_network(config('os-internal-network'), + unit_get('private-address')) + relation_set(relation_id=relation_id, + relation_settings={'private-address': address}) + + @hooks.hook('cluster-relation-changed', 'cluster-relation-departed') @restart_on_change(restart_map(), stopstart=True) @@ -238,17 +264,32 @@ def cluster_changed(): @hooks.hook('ha-relation-joined') def ha_joined(): config = get_hacluster_config() + resources = { - 'res_cinder_vip': 'ocf:heartbeat:IPaddr2', 'res_cinder_haproxy': 'lsb:haproxy' } - vip_params = 'params ip="%s" cidr_netmask="%s" nic="%s"' % \ - (config['vip'], config['vip_cidr'], config['vip_iface']) resource_params = { - 'res_cinder_vip': vip_params, 'res_cinder_haproxy': 'op monitor interval="5s"' } + + vip_group = [] + for vip in config['vip'].split(): + iface = get_iface_for_address(vip) + if iface is not None: + vip_key = 'res_cinder_{}_vip'.format(iface) + resources[vip_key] = 'ocf:heartbeat:IPaddr2' + resource_params[vip_key] = ( + 'params ip="{vip}" cidr_netmask="{netmask}"' + ' nic="{iface}"'.format(vip=vip, + iface=iface, + netmask=get_netmask_for_address(vip)) + ) + vip_group.append(vip_key) + + if len(vip_group) > 1: + relation_set(groups={'grp_cinder_vips': ' '.join(vip_group)}) + init_services = { 'res_cinder_haproxy': 'haproxy' } diff --git a/hooks/cinder_utils.py b/hooks/cinder_utils.py index 29cfb0d4..42b1345c 100644 --- a/hooks/cinder_utils.py +++ b/hooks/cinder_utils.py @@ -86,7 +86,7 @@ SCHEDULER_PACKAGES = ['cinder-scheduler'] DEFAULT_LOOPBACK_SIZE = '5G' # Cluster resource used to determine leadership when hacluster'd -CLUSTER_RES = 'res_cinder_vip' +CLUSTER_RES = 'grp_cinder_vips' class CinderCharmError(Exception): @@ -391,7 +391,7 @@ def set_ceph_env_variables(service): with open('/etc/environment', 'a') as out: out.write('CEPH_ARGS="--id %s"\n' % service) with open('/etc/init/cinder-volume.override', 'w') as out: - out.write('env CEPH_ARGS="--id %s"\n' % service) + out.write('env CEPH_ARGS="--id %s"\n' % service) def do_openstack_upgrade(configs): diff --git a/unit_tests/test_cinder_hooks.py b/unit_tests/test_cinder_hooks.py index a2c9b323..d1a9a41a 100644 --- a/unit_tests/test_cinder_hooks.py +++ b/unit_tests/test_cinder_hooks.py @@ -300,6 +300,7 @@ class TestJoinedHooks(CharmTestCase): def test_identity_service_joined(self): 'It properly requests unclustered endpoint via identity-service' self.unit_get.return_value = 'cindernode1' + self.config.side_effect = self.test_config.get self.canonical_url.return_value = 'http://cindernode1' hooks.hooks.execute(['hooks/identity-service-relation-joined']) expected = { diff --git a/unit_tests/test_cluster_hooks.py b/unit_tests/test_cluster_hooks.py index aa7151b9..da978e07 100644 --- a/unit_tests/test_cluster_hooks.py +++ b/unit_tests/test_cluster_hooks.py @@ -51,7 +51,10 @@ TO_PATCH = [ # charmhelpers.contrib.hahelpers.cluster_utils 'eligible_leader', 'get_hacluster_config', - 'is_leader' + 'is_leader', + # charmhelpers.contrib.network.ip + 'get_iface_for_address', + 'get_netmask_for_address' ] @@ -96,19 +99,22 @@ class TestClusterHooks(CharmTestCase): 'vip_cidr': '19', } self.get_hacluster_config.return_value = conf + self.get_iface_for_address.return_value = 'eth101' + self.get_netmask_for_address.return_value = '255.255.224.0' hooks.hooks.execute(['hooks/ha-relation-joined']) ex_args = { 'corosync_mcastport': '37373', 'init_services': {'res_cinder_haproxy': 'haproxy'}, 'resource_params': { - 'res_cinder_vip': - 'params ip="192.168.25.163" cidr_netmask="19" nic="eth101"', + 'res_cinder_eth101_vip': + 'params ip="192.168.25.163" cidr_netmask="255.255.224.0"' + ' nic="eth101"', 'res_cinder_haproxy': 'op monitor interval="5s"' }, 'corosync_bindiface': 'eth100', 'clones': {'cl_cinder_haproxy': 'res_cinder_haproxy'}, 'resources': { - 'res_cinder_vip': 'ocf:heartbeat:IPaddr2', + 'res_cinder_eth101_vip': 'ocf:heartbeat:IPaddr2', 'res_cinder_haproxy': 'lsb:haproxy' } }