From 0d49ad4c85127fc70d9c793495dda8336bab9697 Mon Sep 17 00:00:00 2001 From: James Page Date: Fri, 6 Jun 2014 13:29:32 +0100 Subject: [PATCH] Add support for splitting public and cluster networks --- charm-helpers-sync.yaml | 1 + config.yaml | 10 ++ hooks/ceph.py | 2 +- .../charmhelpers/contrib/network/__init__.py | 0 hooks/charmhelpers/contrib/network/ip.py | 69 +++++++++++ hooks/charmhelpers/core/fstab.py | 114 ++++++++++++++++++ hooks/charmhelpers/core/host.py | 26 +++- hooks/charmhelpers/fetch/bzrurl.py | 3 +- hooks/hooks.py | 41 +++++-- hooks/utils.py | 11 +- templates/ceph.conf | 8 ++ 11 files changed, 263 insertions(+), 22 deletions(-) create mode 100644 hooks/charmhelpers/contrib/network/__init__.py create mode 100644 hooks/charmhelpers/contrib/network/ip.py create mode 100644 hooks/charmhelpers/core/fstab.py diff --git a/charm-helpers-sync.yaml b/charm-helpers-sync.yaml index 0963bbc..afb9e42 100644 --- a/charm-helpers-sync.yaml +++ b/charm-helpers-sync.yaml @@ -7,3 +7,4 @@ include: - utils - payload.execd - contrib.openstack.alternatives + - contrib.network.ip diff --git a/config.yaml b/config.yaml index 6adafb2..d642243 100644 --- a/config.yaml +++ b/config.yaml @@ -115,3 +115,13 @@ options: default: False description: | If set to True, supporting services will log to syslog. + ceph-public-network: + type: string + description: | + The IP address and netmask of the public (front-side) network (e.g., + 192.168.0.0/24) + ceph-cluster-network: + type: string + description: | + The IP address and netmask of the cluster (back-side) network (e.g., + 192.168.0.0/24) diff --git a/hooks/ceph.py b/hooks/ceph.py index cd81cc4..734acc8 100644 --- a/hooks/ceph.py +++ b/hooks/ceph.py @@ -284,7 +284,7 @@ def bootstrap_monitor_cluster(secret): log('bootstrap_monitor_cluster: mon already initialized.') else: # Ceph >= 0.61.3 needs this for ceph-mon fs creation - mkdir('/var/run/ceph', perms=0755) + mkdir('/var/run/ceph', perms=0o755) mkdir(path) # end changes for Ceph >= 0.61.3 try: diff --git a/hooks/charmhelpers/contrib/network/__init__.py b/hooks/charmhelpers/contrib/network/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/hooks/charmhelpers/contrib/network/ip.py b/hooks/charmhelpers/contrib/network/ip.py new file mode 100644 index 0000000..44c7c97 --- /dev/null +++ b/hooks/charmhelpers/contrib/network/ip.py @@ -0,0 +1,69 @@ +import sys + +from charmhelpers.fetch import apt_install +from charmhelpers.core.hookenv import ( + ERROR, log, +) + +try: + import netifaces +except ImportError: + apt_install('python-netifaces') + import netifaces + +try: + import netaddr +except ImportError: + apt_install('python-netaddr') + import netaddr + + +def _validate_cidr(network): + try: + netaddr.IPNetwork(network) + except (netaddr.core.AddrFormatError, ValueError): + raise ValueError("Network (%s) is not in CIDR presentation format" % + network) + + +def get_address_in_network(network, fallback=None, fatal=False): + """ + Get an IPv4 address within the network from the host. + + Args: + network (str): CIDR presentation format. For example, + '192.168.1.0/24'. + fallback (str): If no address is found, return fallback. + fatal (boolean): If no address is found, fallback is not + set and fatal is True then exit(1). + """ + + def not_found_error_out(): + log("No IP address found in network: %s" % network, + level=ERROR) + sys.exit(1) + + if network is None: + if fallback is not None: + return fallback + else: + if fatal: + not_found_error_out() + + _validate_cidr(network) + for iface in netifaces.interfaces(): + addresses = netifaces.ifaddresses(iface) + if netifaces.AF_INET in addresses: + addr = addresses[netifaces.AF_INET][0]['addr'] + netmask = addresses[netifaces.AF_INET][0]['netmask'] + cidr = netaddr.IPNetwork("%s/%s" % (addr, netmask)) + if cidr in netaddr.IPNetwork(network): + return str(cidr.ip) + + if fallback is not None: + return fallback + + if fatal: + not_found_error_out() + + return None diff --git a/hooks/charmhelpers/core/fstab.py b/hooks/charmhelpers/core/fstab.py new file mode 100644 index 0000000..cdd7261 --- /dev/null +++ b/hooks/charmhelpers/core/fstab.py @@ -0,0 +1,114 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- + +__author__ = 'Jorge Niedbalski R. ' + +import os + + +class Fstab(file): + """This class extends file in order to implement a file reader/writer + for file `/etc/fstab` + """ + + class Entry(object): + """Entry class represents a non-comment line on the `/etc/fstab` file + """ + def __init__(self, device, mountpoint, filesystem, + options, d=0, p=0): + self.device = device + self.mountpoint = mountpoint + self.filesystem = filesystem + + if not options: + options = "defaults" + + self.options = options + self.d = d + self.p = p + + def __eq__(self, o): + return str(self) == str(o) + + def __str__(self): + return "{} {} {} {} {} {}".format(self.device, + self.mountpoint, + self.filesystem, + self.options, + self.d, + self.p) + + DEFAULT_PATH = os.path.join(os.path.sep, 'etc', 'fstab') + + def __init__(self, path=None): + if path: + self._path = path + else: + self._path = self.DEFAULT_PATH + file.__init__(self, self._path, 'r+') + + def _hydrate_entry(self, line): + return Fstab.Entry(*filter( + lambda x: x not in ('', None), + line.strip("\n").split(" "))) + + @property + def entries(self): + self.seek(0) + for line in self.readlines(): + try: + if not line.startswith("#"): + yield self._hydrate_entry(line) + except ValueError: + pass + + def get_entry_by_attr(self, attr, value): + for entry in self.entries: + e_attr = getattr(entry, attr) + if e_attr == value: + return entry + return None + + def add_entry(self, entry): + if self.get_entry_by_attr('device', entry.device): + return False + + self.write(str(entry) + '\n') + self.truncate() + return entry + + def remove_entry(self, entry): + self.seek(0) + + lines = self.readlines() + + found = False + for index, line in enumerate(lines): + if not line.startswith("#"): + if self._hydrate_entry(line) == entry: + found = True + break + + if not found: + return False + + lines.remove(line) + + self.seek(0) + self.write(''.join(lines)) + self.truncate() + return True + + @classmethod + def remove_by_mountpoint(cls, mountpoint, path=None): + fstab = cls(path=path) + entry = fstab.get_entry_by_attr('mountpoint', mountpoint) + if entry: + return fstab.remove_entry(entry) + return False + + @classmethod + def add(cls, device, mountpoint, filesystem, options=None, path=None): + return cls(path=path).add_entry(Fstab.Entry(device, + mountpoint, filesystem, + options=options)) diff --git a/hooks/charmhelpers/core/host.py b/hooks/charmhelpers/core/host.py index 186147f..46bfd36 100644 --- a/hooks/charmhelpers/core/host.py +++ b/hooks/charmhelpers/core/host.py @@ -17,6 +17,7 @@ import apt_pkg from collections import OrderedDict from hookenv import log +from fstab import Fstab def service_start(service_name): @@ -35,7 +36,8 @@ def service_restart(service_name): def service_reload(service_name, restart_on_failure=False): - """Reload a system service, optionally falling back to restart if reload fails""" + """Reload a system service, optionally falling back to restart if + reload fails""" service_result = service('reload', service_name) if not service_result and restart_on_failure: service_result = service('restart', service_name) @@ -144,7 +146,19 @@ def write_file(path, content, owner='root', group='root', perms=0444): target.write(content) -def mount(device, mountpoint, options=None, persist=False): +def fstab_remove(mp): + """Remove the given mountpoint entry from /etc/fstab + """ + return Fstab.remove_by_mountpoint(mp) + + +def fstab_add(dev, mp, fs, options=None): + """Adds the given device entry to the /etc/fstab file + """ + return Fstab.add(dev, mp, fs, options=options) + + +def mount(device, mountpoint, options=None, persist=False, filesystem="ext3"): """Mount a filesystem at a particular mountpoint""" cmd_args = ['mount'] if options is not None: @@ -155,9 +169,9 @@ def mount(device, mountpoint, options=None, persist=False): except subprocess.CalledProcessError, e: log('Error mounting {} at {}\n{}'.format(device, mountpoint, e.output)) return False + if persist: - # TODO: update fstab - pass + return fstab_add(device, mountpoint, filesystem, options=options) return True @@ -169,9 +183,9 @@ def umount(mountpoint, persist=False): except subprocess.CalledProcessError, e: log('Error unmounting {}\n{}'.format(mountpoint, e.output)) return False + if persist: - # TODO: update fstab - pass + return fstab_remove(mountpoint) return True diff --git a/hooks/charmhelpers/fetch/bzrurl.py b/hooks/charmhelpers/fetch/bzrurl.py index db5dd9a..0e580e4 100644 --- a/hooks/charmhelpers/fetch/bzrurl.py +++ b/hooks/charmhelpers/fetch/bzrurl.py @@ -39,7 +39,8 @@ class BzrUrlFetchHandler(BaseFetchHandler): def install(self, source): url_parts = self.parse_url(source) branch_name = url_parts.path.strip("/").split("/")[-1] - dest_dir = os.path.join(os.environ.get('CHARM_DIR'), "fetched", branch_name) + dest_dir = os.path.join(os.environ.get('CHARM_DIR'), "fetched", + branch_name) if not os.path.exists(dest_dir): mkdir(dest_dir, perms=0755) try: diff --git a/hooks/hooks.py b/hooks/hooks.py index 11e1cab..a4df35c 100755 --- a/hooks/hooks.py +++ b/hooks/hooks.py @@ -43,6 +43,7 @@ from charmhelpers.contrib.openstack.alternatives import install_alternative from utils import ( render_template, get_host_ip, + get_public_addr, ) hooks = Hooks() @@ -73,7 +74,9 @@ def emit_cephconf(): 'fsid': config('fsid'), 'version': ceph.get_ceph_version(), 'osd_journal_size': config('osd-journal-size'), - 'use_syslog': str(config('use-syslog')).lower() + 'use_syslog': str(config('use-syslog')).lower(), + 'ceph_public_network': config('ceph-public-network'), + 'ceph_cluster_network': config('ceph-cluster-network'), } # Install ceph.conf as an alternative to support # co-existence with other charms that write this file @@ -133,14 +136,13 @@ def config_changed(): def get_mon_hosts(): hosts = [] - hosts.append('{}:6789'.format(get_host_ip())) + hosts.append('{}:6789'.format(get_public_addr())) for relid in relation_ids('mon'): for unit in related_units(relid): - hosts.append( - '{}:6789'.format(get_host_ip(relation_get('private-address', - unit, relid))) - ) + addr = relation_get('ceph_public_addr', unit, relid) or \ + get_host_ip(relation_get('private-address', unit, relid)) + hosts.append('{}:6789'.format(addr)) hosts.sort() return hosts @@ -160,8 +162,15 @@ def get_devices(): return [] +@hooks.hook('mon-relation-joined') +def mon_relation_joined(): + for relid in relation_ids('mon'): + relation_set(relation_id=relid, + ceph_public_addr=get_public_addr()) + + @hooks.hook('mon-relation-departed', - 'mon-relation-joined') + 'mon-relation-changed') def mon_relation(): log('Begin mon-relation hook.') emit_cephconf() @@ -191,7 +200,8 @@ def notify_osds(): relation_set(relation_id=relid, fsid=config('fsid'), osd_bootstrap_key=ceph.get_osd_bootstrap_key(), - auth=config('auth-supported')) + auth=config('auth-supported'), + ceph_public_addr=get_public_addr()) log('End notify_osds.') @@ -202,7 +212,8 @@ def notify_radosgws(): for relid in relation_ids('radosgw'): relation_set(relation_id=relid, radosgw_key=ceph.get_radosgw_key(), - auth=config('auth-supported')) + auth=config('auth-supported'), + ceph_public_addr=get_public_addr()) log('End notify_radosgws.') @@ -216,7 +227,8 @@ def notify_client(): service_name = units[0].split('/')[0] relation_set(relation_id=relid, key=ceph.get_named_key(service_name), - auth=config('auth-supported')) + auth=config('auth-supported'), + ceph_public_addr=get_public_addr()) log('End notify_client.') @@ -242,7 +254,8 @@ def osd_relation(): log('mon cluster in quorum - providing fsid & keys') relation_set(fsid=config('fsid'), osd_bootstrap_key=ceph.get_osd_bootstrap_key(), - auth=config('auth-supported')) + auth=config('auth-supported'), + ceph_public_addr=get_public_addr()) else: log('mon cluster not in quorum - deferring fsid provision') @@ -258,7 +271,8 @@ def radosgw_relation(): if ceph.is_quorum(): log('mon cluster in quorum - providing radosgw with keys') relation_set(radosgw_key=ceph.get_radosgw_key(), - auth=config('auth-supported')) + auth=config('auth-supported'), + ceph_public_addr=get_public_addr()) else: log('mon cluster not in quorum - deferring key provision') @@ -273,7 +287,8 @@ def client_relation(): log('mon cluster in quorum - providing client with keys') service_name = remote_unit().split('/')[0] relation_set(key=ceph.get_named_key(service_name), - auth=config('auth-supported')) + auth=config('auth-supported'), + ceph_public_addr=get_public_addr()) else: log('mon cluster not in quorum - deferring key provision') diff --git a/hooks/utils.py b/hooks/utils.py index c1044a4..b2937f1 100644 --- a/hooks/utils.py +++ b/hooks/utils.py @@ -11,13 +11,16 @@ import socket import re from charmhelpers.core.hookenv import ( unit_get, - cached + cached, + config ) from charmhelpers.fetch import ( apt_install, filter_installed_packages ) +from charmhelpers.contrib.network import ip + TEMPLATES_DIR = 'templates' try: @@ -72,3 +75,9 @@ def get_host_ip(hostname=None): answers = dns.resolver.query(hostname, 'A') if answers: return answers[0].address + + +@cached +def get_public_addr(): + return ip.get_address_in_network( + config('ceph-public-network'), fallback=get_host_ip()) diff --git a/templates/ceph.conf b/templates/ceph.conf index d82ca82..9626d9a 100644 --- a/templates/ceph.conf +++ b/templates/ceph.conf @@ -9,11 +9,19 @@ keyring = /etc/ceph/$cluster.$name.keyring mon host = {{ mon_hosts }} fsid = {{ fsid }} + log to syslog = {{ use_syslog }} err to syslog = {{ use_syslog }} clog to syslog = {{ use_syslog }} mon cluster log to syslog = {{ use_syslog }} +{%- if ceph_public_network is string %} + public network = {{ ceph_public_network }} +{%- endif %} +{%- if ceph_cluster_network is string %} + cluster network = {{ ceph_cluster_network }} +{%- endif %} + [mon] keyring = /var/lib/ceph/mon/$cluster-$id/keyring