diff --git a/.project b/.project new file mode 100644 index 0000000..a265ebb --- /dev/null +++ b/.project @@ -0,0 +1,17 @@ + + + hacluster + + + + + + org.python.pydev.PyDevBuilder + + + + + + org.python.pydev.pythonNature + + diff --git a/.pydevproject b/.pydevproject new file mode 100644 index 0000000..b2c8db1 --- /dev/null +++ b/.pydevproject @@ -0,0 +1,8 @@ + + +python 2.7 +Default + +/hacluster/hooks + + diff --git a/config.yaml b/config.yaml index ce1e461..06744a4 100644 --- a/config.yaml +++ b/config.yaml @@ -7,22 +7,23 @@ options: If multiple clusters are on the same bindnetaddr network, this value can be changed. corosync_pcmk_ver: - default: 0 + default: 1 type: int description: | Service version for the Pacemaker service version. This will tell Corosync how to start pacemaker corosync_key: type: string - default: corosync-key + default: "64RxJNcCkwo8EJYBsaacitUvbQp5AW4YolJi5/2urYZYp2jfLxY+3IUCOaAUJHPle4Yqfy+WBXO0I/6ASSAjj9jaiHVNaxmVhhjcmyBqy2vtPf+m+0VxVjUXlkTyYsODwobeDdO3SIkbIABGfjLTu29yqPTsfbvSYr6skRb9ne0=" description: | This value will become the Corosync authentication key. To generate a suitable value use: . - corosync-keygen + sudo corosync-keygen + sudo cat /etc/corosync/authkey | base64 -w 0 . This configuration element is mandatory and the service will fail on - install if it is not provided. + install if it is not provided. The value must be base64 encoded. stonith_enabled: type: string default: 'False' @@ -36,3 +37,7 @@ options: maas_credentials: type: string description: MAAS credentials (required for STONITH). + cluster_count: + type: int + default: 2 + description: Number of peer units required to bootstrap cluster services. diff --git a/copyright b/copyright index 1632584..a1d1b7b 100644 --- a/copyright +++ b/copyright @@ -15,3 +15,8 @@ License: GPL-3 . You should have received a copy of the GNU General Public License along with this program. If not, see . + + Files: ocf/ceph/* + Copyright: 2012 Florian Haas, hastexo + License: LGPL-2.1 + On Debian based systems, see /usr/share/common-licenses/LGPL-2.1. \ No newline at end of file diff --git a/hooks/hacluster.py b/hooks/hacluster.py new file mode 100644 index 0000000..1e26921 --- /dev/null +++ b/hooks/hacluster.py @@ -0,0 +1,81 @@ + +# +# Copyright 2012 Canonical Ltd. +# +# Authors: +# James Page +# Paul Collins +# + +import os +import subprocess +import socket +import fcntl +import struct +import lib.utils as utils + + +try: + from netaddr import IPNetwork +except ImportError: + utils.install('python-netaddr') + from netaddr import IPNetwork + + +def disable_upstart_services(*services): + for service in services: + with open("/etc/init/{}.override".format(service), "w") as override: + override.write("manual") + + +def enable_upstart_services(*services): + for service in services: + path = '/etc/init/{}.override'.format(service) + if os.path.exists(path): + os.remove(path) + + +def disable_lsb_services(*services): + for service in services: + subprocess.check_call(['update-rc.d', '-f', service, 'remove']) + + +def enable_lsb_services(*services): + for service in services: + subprocess.check_call(['update-rc.d', '-f', service, 'defaults']) + + +def get_iface_ipaddr(iface): + s = socket.socket(socket.AF_INET, socket.SOCK_DGRAM) + return socket.inet_ntoa(fcntl.ioctl( + s.fileno(), + 0x8919, # SIOCGIFADDR + struct.pack('256s', iface[:15]) + )[20:24]) + + +def get_iface_netmask(iface): + s = socket.socket(socket.AF_INET, socket.SOCK_DGRAM) + return socket.inet_ntoa(fcntl.ioctl( + s.fileno(), + 0x891b, # SIOCGIFNETMASK + struct.pack('256s', iface[:15]) + )[20:24]) + + +def get_netmask_cidr(netmask): + netmask = netmask.split('.') + binary_str = '' + for octet in netmask: + binary_str += bin(int(octet))[2:].zfill(8) + return str(len(binary_str.rstrip('0'))) + + +def get_network_address(iface): + if iface: + network = "{}/{}".format(get_iface_ipaddr(iface), + get_netmask_cidr(get_iface_netmask(iface))) + ip = IPNetwork(network) + return str(ip.network) + else: + return None diff --git a/hooks/ha-relation-departed b/hooks/hanode-relation-changed similarity index 100% rename from hooks/ha-relation-departed rename to hooks/hanode-relation-changed diff --git a/hooks/hooks.py b/hooks/hooks.py index 34eb7d7..59d6db3 100755 --- a/hooks/hooks.py +++ b/hooks/hooks.py @@ -11,10 +11,12 @@ import shutil import sys import time import os +from base64 import b64decode import maas as MAAS -import utils +import lib.utils as utils import pcmk +import hacluster def install(): @@ -36,12 +38,12 @@ def get_corosync_conf(): for unit in utils.relation_list(relid): conf = { 'corosync_bindnetaddr': - utils.get_network_address( + hacluster.get_network_address( utils.relation_get('corosync_bindiface', unit, relid) ), 'corosync_mcastport': utils.relation_get('corosync_mcastport', - unit, relid), + unit, relid), 'corosync_mcastaddr': utils.config_get('corosync_mcastaddr'), 'corosync_pcmk_ver': utils.config_get('corosync_pcmk_ver'), } @@ -68,27 +70,27 @@ def emit_base_conf(): with open('/etc/default/corosync', 'w') as corosync_default: corosync_default.write(utils.render_template('corosync', corosync_default_context)) - - # write the authkey corosync_key = utils.config_get('corosync_key') - with open('/etc/corosync/authkey', 'w') as corosync_key_file: - corosync_key_file.write(corosync_key) - os.chmod = ('/etc/corosync/authkey', 0400) + if corosync_key: + # write the authkey + with open('/etc/corosync/authkey', 'w') as corosync_key_file: + corosync_key_file.write(b64decode(corosync_key)) + os.chmod = ('/etc/corosync/authkey', 0400) def config_changed(): utils.juju_log('INFO', 'Begin config-changed hook.') corosync_key = utils.config_get('corosync_key') - if corosync_key == '': + if not corosync_key: utils.juju_log('CRITICAL', 'No Corosync key supplied, cannot proceed') sys.exit(1) if int(utils.config_get('corosync_pcmk_ver')) == 1: - utils.enable_lsb_services('pacemaker') + hacluster.enable_lsb_services('pacemaker') else: - utils.disable_lsb_services('pacemaker') + hacluster.disable_lsb_services('pacemaker') # Create a new config file emit_base_conf() @@ -109,14 +111,6 @@ def upgrade_charm(): utils.juju_log('INFO', 'End upgrade-charm hook.') -def start(): - pass - - -def stop(): - pass - - def restart_corosync(): if int(utils.config_get('corosync_pcmk_ver')) == 1: if utils.running("pacemaker"): @@ -136,17 +130,23 @@ def configure_cluster(): utils.juju_log('INFO', 'HA already configured, not reconfiguring') return - # Check that there's enough nodes in order to perform the - # configuration of the HA cluster - if len(get_cluster_nodes()) < 2: - utils.juju_log('WARNING', 'Not enough nodes in cluster, bailing') - return # Check that we are related to a principle and that # it has already provided the required corosync configuration if not get_corosync_conf(): utils.juju_log('WARNING', 'Unable to configure corosync right now, bailing') return + else: + utils.juju_log('INFO', + 'Ready to form cluster - informing peers') + utils.relation_set(ready=True, + rid=utils.relation_ids('hanode')[0]) + # Check that there's enough nodes in order to perform the + # configuration of the HA cluster + if (len(get_cluster_nodes()) < + int(utils.config_get('cluster_count'))): + utils.juju_log('WARNING', 'Not enough nodes in cluster, bailing') + return relids = utils.relation_ids('ha') if len(relids) == 1: # Should only ever be one of these @@ -231,13 +231,13 @@ def configure_cluster(): for res_name, res_type in resources.iteritems(): # disable the service we are going to put in HA if res_type.split(':')[0] == "lsb": - utils.disable_lsb_services(res_type.split(':')[1]) + hacluster.disable_lsb_services(res_type.split(':')[1]) if utils.running(res_type.split(':')[1]): utils.stop(res_type.split(':')[1]) elif (len(init_services) != 0 and res_name in init_services and init_services[res_name]): - utils.disable_upstart_services(init_services[res_name]) + hacluster.disable_upstart_services(init_services[res_name]) if utils.running(init_services[res_name]): utils.stop(init_services[res_name]) # Put the services in HA, if not already done so @@ -382,42 +382,28 @@ def configure_stonith(): pcmk.commit(cmd) -def ha_relation_departed(): - # TODO: Fin out which node is departing and put it in standby mode. - # If this happens, and a new relation is created in the same machine - # (which already has node), then check whether it is standby and put it - # in online mode. This should be done in ha_relation_joined. - pcmk.standby(utils.get_unit_hostname()) - - def get_cluster_nodes(): hosts = [] - hosts.append('{}:6789'.format(utils.get_host_ip())) - + hosts.append(utils.unit_get('private-address')) for relid in utils.relation_ids('hanode'): for unit in utils.relation_list(relid): - hosts.append( - '{}:6789'.format(utils.get_host_ip( - utils.relation_get('private-address', - unit, relid))) - ) - + if utils.relation_get('ready', + rid=relid, + unit=unit): + hosts.append(utils.relation_get('private-address', + unit, relid)) hosts.sort() return hosts -utils.do_hooks({ - 'install': install, - 'config-changed': config_changed, - 'start': start, - 'stop': stop, - 'upgrade-charm': upgrade_charm, - 'ha-relation-joined': configure_cluster, - 'ha-relation-changed': configure_cluster, - 'ha-relation-departed': ha_relation_departed, - 'hanode-relation-joined': configure_cluster, - #'hanode-relation-departed': hanode_relation_departed, - # TODO: should probably remove nodes from the cluster - }) +hooks = { + 'install': install, + 'config-changed': config_changed, + 'upgrade-charm': upgrade_charm, + 'ha-relation-joined': configure_cluster, + 'ha-relation-changed': configure_cluster, + 'hanode-relation-joined': configure_cluster, + 'hanode-relation-changed': configure_cluster, + } -sys.exit(0) +utils.do_hooks(hooks) diff --git a/hooks/lib/__init__.py b/hooks/lib/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/hooks/lib/cluster_utils.py b/hooks/lib/cluster_utils.py new file mode 100644 index 0000000..b7d00f8 --- /dev/null +++ b/hooks/lib/cluster_utils.py @@ -0,0 +1,130 @@ +# +# Copyright 2012 Canonical Ltd. +# +# This file is sourced from lp:openstack-charm-helpers +# +# Authors: +# James Page +# Adam Gandelman +# + +from lib.utils import ( + juju_log, + relation_ids, + relation_list, + relation_get, + get_unit_hostname, + config_get + ) +import subprocess +import os + + +def is_clustered(): + for r_id in (relation_ids('ha') or []): + for unit in (relation_list(r_id) or []): + clustered = relation_get('clustered', + rid=r_id, + unit=unit) + if clustered: + return True + return False + + +def is_leader(resource): + cmd = [ + "crm", "resource", + "show", resource + ] + try: + status = subprocess.check_output(cmd) + except subprocess.CalledProcessError: + return False + else: + if get_unit_hostname() in status: + return True + else: + return False + + +def peer_units(): + peers = [] + for r_id in (relation_ids('cluster') or []): + for unit in (relation_list(r_id) or []): + peers.append(unit) + return peers + + +def oldest_peer(peers): + local_unit_no = int(os.getenv('JUJU_UNIT_NAME').split('/')[1]) + for peer in peers: + remote_unit_no = int(peer.split('/')[1]) + if remote_unit_no < local_unit_no: + return False + return True + + +def eligible_leader(resource): + if is_clustered(): + if not is_leader(resource): + juju_log('INFO', 'Deferring action to CRM leader.') + return False + else: + peers = peer_units() + if peers and not oldest_peer(peers): + juju_log('INFO', 'Deferring action to oldest service unit.') + return False + return True + + +def https(): + ''' + Determines whether enough data has been provided in configuration + or relation data to configure HTTPS + . + returns: boolean + ''' + if config_get('use-https') == "yes": + return True + if config_get('ssl_cert') and config_get('ssl_key'): + return True + for r_id in relation_ids('identity-service'): + for unit in relation_list(r_id): + if (relation_get('https_keystone', rid=r_id, unit=unit) and + relation_get('ssl_cert', rid=r_id, unit=unit) and + relation_get('ssl_key', rid=r_id, unit=unit) and + relation_get('ca_cert', rid=r_id, unit=unit)): + return True + return False + + +def determine_api_port(public_port): + ''' + Determine correct API server listening port based on + existence of HTTPS reverse proxy and/or haproxy. + + public_port: int: standard public port for given service + + returns: int: the correct listening port for the API service + ''' + i = 0 + if len(peer_units()) > 0 or is_clustered(): + i += 1 + if https(): + i += 1 + return public_port - (i * 10) + + +def determine_haproxy_port(public_port): + ''' + Description: Determine correct proxy listening port based on public IP + + existence of HTTPS reverse proxy. + + public_port: int: standard public port for given service + + returns: int: the correct listening port for the HAProxy service + ''' + i = 0 + if https(): + i += 1 + return public_port - (i * 10) diff --git a/hooks/utils.py b/hooks/lib/utils.py similarity index 59% rename from hooks/utils.py rename to hooks/lib/utils.py index 61f71c2..1033a58 100644 --- a/hooks/utils.py +++ b/hooks/lib/utils.py @@ -1,28 +1,31 @@ - # # Copyright 2012 Canonical Ltd. # +# This file is sourced from lp:openstack-charm-helpers +# # Authors: # James Page # Paul Collins +# Adam Gandelman # +import json import os import subprocess import socket import sys -import fcntl -import struct def do_hooks(hooks): hook = os.path.basename(sys.argv[0]) try: - hooks[hook]() + hook_func = hooks[hook] except KeyError: juju_log('INFO', "This charm doesn't know how to handle '{}'.".format(hook)) + else: + hook_func() def install(*pkgs): @@ -43,12 +46,6 @@ except ImportError: install('python-jinja2') import jinja2 -try: - from netaddr import IPNetwork -except ImportError: - install('python-netaddr') - from netaddr import IPNetwork - try: import dns.resolver except ImportError: @@ -63,19 +60,18 @@ def render_template(template_name, context, template_dir=TEMPLATES_DIR): template = templates.get_template(template_name) return template.render(context) - CLOUD_ARCHIVE = \ """ # Ubuntu Cloud Archive deb http://ubuntu-cloud.archive.canonical.com/ubuntu {} main """ CLOUD_ARCHIVE_POCKETS = { - 'precise-folsom': 'precise-updates/folsom', - 'precise-folsom/updates': 'precise-updates/folsom', - 'precise-folsom/proposed': 'precise-proposed/folsom', - 'precise-grizzly': 'precise-updates/grizzly', - 'precise-grizzly/updates': 'precise-updates/grizzly', - 'precise-grizzly/proposed': 'precise-proposed/grizzly' + 'folsom': 'precise-updates/folsom', + 'folsom/updates': 'precise-updates/folsom', + 'folsom/proposed': 'precise-proposed/folsom', + 'grizzly': 'precise-updates/grizzly', + 'grizzly/updates': 'precise-updates/grizzly', + 'grizzly/proposed': 'precise-proposed/grizzly' } @@ -90,8 +86,11 @@ def configure_source(): ] subprocess.check_call(cmd) if source.startswith('cloud:'): + # CA values should be formatted as cloud:ubuntu-openstack/pocket, eg: + # cloud:precise-folsom/updates or cloud:precise-folsom/proposed install('ubuntu-cloud-keyring') pocket = source.split(':')[1] + pocket = pocket.split('-')[1] with open('/etc/apt/sources.list.d/cloud-archive.list', 'w') as apt: apt.write(CLOUD_ARCHIVE.format(CLOUD_ARCHIVE_POCKETS[pocket])) if source.startswith('deb'): @@ -137,22 +136,49 @@ def juju_log(severity, message): subprocess.check_call(cmd) +cache = {} + + +def cached(func): + def wrapper(*args, **kwargs): + global cache + key = str((func, args, kwargs)) + try: + return cache[key] + except KeyError: + res = func(*args, **kwargs) + cache[key] = res + return res + return wrapper + + +@cached def relation_ids(relation): cmd = [ 'relation-ids', relation ] - return subprocess.check_output(cmd).split() # IGNORE:E1103 + result = str(subprocess.check_output(cmd)).split() + if result == "": + return None + else: + return result +@cached def relation_list(rid): cmd = [ 'relation-list', '-r', rid, ] - return subprocess.check_output(cmd).split() # IGNORE:E1103 + result = str(subprocess.check_output(cmd)).split() + if result == "": + return None + else: + return result +@cached def relation_get(attribute, unit=None, rid=None): cmd = [ 'relation-get', @@ -170,6 +196,29 @@ def relation_get(attribute, unit=None, rid=None): return value +@cached +def relation_get_dict(relation_id=None, remote_unit=None): + """Obtain all relation data as dict by way of JSON""" + cmd = [ + 'relation-get', '--format=json' + ] + if relation_id: + cmd.append('-r') + cmd.append(relation_id) + if remote_unit: + remote_unit_orig = os.getenv('JUJU_REMOTE_UNIT', None) + os.environ['JUJU_REMOTE_UNIT'] = remote_unit + j = subprocess.check_output(cmd) + if remote_unit and remote_unit_orig: + os.environ['JUJU_REMOTE_UNIT'] = remote_unit_orig + d = json.loads(j) + settings = {} + # convert unicode to strings + for k, v in d.iteritems(): + settings[str(k)] = str(v) + return settings + + def relation_set(**kwargs): cmd = [ 'relation-set' @@ -177,63 +226,89 @@ def relation_set(**kwargs): args = [] for k, v in kwargs.items(): if k == 'rid': - cmd.append('-r') - cmd.append(v) + if v: + cmd.append('-r') + cmd.append(v) else: args.append('{}={}'.format(k, v)) cmd += args subprocess.check_call(cmd) +@cached def unit_get(attribute): cmd = [ 'unit-get', attribute ] - return subprocess.check_output(cmd).strip() # IGNORE:E1103 + value = subprocess.check_output(cmd).strip() # IGNORE:E1103 + if value == "": + return None + else: + return value +@cached def config_get(attribute): cmd = [ 'config-get', - attribute + '--format', + 'json', ] - return subprocess.check_output(cmd).strip() # IGNORE:E1103 + out = subprocess.check_output(cmd).strip() # IGNORE:E1103 + cfg = json.loads(out) + + try: + return cfg[attribute] + except KeyError: + return None +@cached def get_unit_hostname(): return socket.gethostname() +@cached def get_host_ip(hostname=unit_get('private-address')): try: # Test to see if already an IPv4 address socket.inet_aton(hostname) return hostname except socket.error: - pass - try: answers = dns.resolver.query(hostname, 'A') if answers: return answers[0].address - except dns.resolver.NXDOMAIN: - pass return None +def _svc_control(service, action): + subprocess.check_call(['service', service, action]) + + def restart(*services): for service in services: - subprocess.check_call(['service', service, 'restart']) + _svc_control(service, 'restart') def stop(*services): for service in services: - subprocess.check_call(['service', service, 'stop']) + _svc_control(service, 'stop') def start(*services): for service in services: - subprocess.check_call(['service', service, 'start']) + _svc_control(service, 'start') + + +def reload(*services): + for service in services: + try: + _svc_control(service, 'reload') + except subprocess.CalledProcessError: + # Reload failed - either service does not support reload + # or it was not running - restart will fixup most things + _svc_control(service, 'restart') def running(service): @@ -249,60 +324,9 @@ def running(service): return False -def disable_upstart_services(*services): - for service in services: - with open("/etc/init/{}.override".format(service), "w") as override: - override.write("manual") - - -def enable_upstart_services(*services): - for service in services: - path = '/etc/init/{}.override'.format(service) - if os.path.exists(path): - os.remove(path) - - -def disable_lsb_services(*services): - for service in services: - subprocess.check_call(['update-rc.d', '-f', service, 'remove']) - - -def enable_lsb_services(*services): - for service in services: - subprocess.check_call(['update-rc.d', '-f', service, 'defaults']) - - -def get_iface_ipaddr(iface): - s = socket.socket(socket.AF_INET, socket.SOCK_DGRAM) - return socket.inet_ntoa(fcntl.ioctl( - s.fileno(), - 0x8919, # SIOCGIFADDR - struct.pack('256s', iface[:15]) - )[20:24]) - - -def get_iface_netmask(iface): - s = socket.socket(socket.AF_INET, socket.SOCK_DGRAM) - return socket.inet_ntoa(fcntl.ioctl( - s.fileno(), - 0x891b, # SIOCGIFNETMASK - struct.pack('256s', iface[:15]) - )[20:24]) - - -def get_netmask_cidr(netmask): - netmask = netmask.split('.') - binary_str = '' - for octet in netmask: - binary_str += bin(int(octet))[2:].zfill(8) - return str(len(binary_str.rstrip('0'))) - - -def get_network_address(iface): - if iface: - network = "{}/{}".format(get_iface_ipaddr(iface), - get_netmask_cidr(get_iface_netmask(iface))) - ip = IPNetwork(network) - return str(ip.network) - else: - return None +def is_relation_made(relation, key='private-address'): + for r_id in (relation_ids(relation) or []): + for unit in (relation_list(r_id) or []): + if relation_get(key, rid=r_id, unit=unit): + return True + return False diff --git a/hooks/maas.py b/hooks/maas.py index 93399bc..4ec398a 100644 --- a/hooks/maas.py +++ b/hooks/maas.py @@ -3,7 +3,7 @@ import apt_pkg as apt import json import subprocess -import utils +import lib.utils as utils MAAS_STABLE_PPA = 'ppa:maas-maintainers/stable ' MAAS_PROFILE_NAME = 'maas-juju-hacluster' diff --git a/hooks/pcmk.py b/hooks/pcmk.py index 4b7a31f..21e53e8 100644 --- a/hooks/pcmk.py +++ b/hooks/pcmk.py @@ -1,4 +1,4 @@ -import utils +import lib.utils as utils import commands import subprocess diff --git a/hooks/start b/hooks/start deleted file mode 120000 index 9416ca6..0000000 --- a/hooks/start +++ /dev/null @@ -1 +0,0 @@ -hooks.py \ No newline at end of file diff --git a/hooks/stop b/hooks/stop deleted file mode 120000 index 9416ca6..0000000 --- a/hooks/stop +++ /dev/null @@ -1 +0,0 @@ -hooks.py \ No newline at end of file