diff --git a/config.yaml b/config.yaml index 2e7d623..f6e3f0e 100644 --- a/config.yaml +++ b/config.yaml @@ -105,3 +105,27 @@ options: keystone-admin-password: type: string description: Keystone admin password + # HA configuration settings + vip: + type: string + description: "Virtual IP to use to front swift-proxy in ha configuration" + vip_iface: + type: string + default: eth0 + description: "Network Interface where to place the Virtual IP" + vip_cidr: + type: int + default: 24 + description: "Netmask that will be used for the Virtual IP" + ha-bindiface: + type: string + default: eth0 + description: | + Default network interface on which HA cluster will bind to communication + with the other members of the HA Cluster. + ha-mcastport: + type: int + default: 5414 + description: | + Default multicast port number that will be used to communicate between + HA Cluster nodes. diff --git a/hooks/lib/openstack_common.py b/hooks/lib/openstack_common.py index 1c65cc1..6a07ac4 100644 --- a/hooks/lib/openstack_common.py +++ b/hooks/lib/openstack_common.py @@ -3,6 +3,7 @@ # Common python helper functions used for OpenStack charms. import subprocess +import os CLOUD_ARCHIVE_URL = "http://ubuntu-cloud.archive.canonical.com/ubuntu" CLOUD_ARCHIVE_KEY_ID = '5EDB1B62EC4926EA' @@ -223,3 +224,24 @@ def configure_installation_source(rel): f.write(src) else: error_out("Invalid openstack-release specified: %s" % rel) + +HAPROXY_CONF = '/etc/haproxy/haproxy.cfg' +HAPROXY_DEFAULT = '/etc/default/haproxy' + +def configure_haproxy(units, service_ports, template_dir=None): + template_dir = template_dir or 'templates' + import jinja2 + context = { + 'units': units, + 'service_ports': service_ports + } + templates = jinja2.Environment( + loader=jinja2.FileSystemLoader(template_dir) + ) + template = templates.get_template( + os.path.basename(HAPROXY_CONF) + ) + with open(HAPROXY_CONF, 'w') as f: + f.write(template.render(context)) + with open(HAPROXY_DEFAULT, 'w') as f: + f.write('ENABLED=1') diff --git a/hooks/swift-hooks.py b/hooks/swift-hooks.py index d255c38..420fe71 100755 --- a/hooks/swift-hooks.py +++ b/hooks/swift-hooks.py @@ -10,6 +10,11 @@ from subprocess import check_call import lib.openstack_common as openstack import swift_utils as swift +extra_pkgs = [ + "haproxy", + "python-jinja2" + ] + def install(): src = utils.config_get('openstack-origin') if src != 'distro': @@ -19,6 +24,7 @@ def install(): pkgs = swift.determine_packages(rel) utils.install(*pkgs) + utils.install(*extra_pkgs) swift.ensure_swift_dir() @@ -57,7 +63,10 @@ def install(): def keystone_joined(relid=None): - hostname = utils.unit_get('private-address') + if is_clustered(): + hostname = utils.config_get('vip') + else: + hostname = utils.unit_get('private-address') port = utils.config_get('bind-port') ssl = utils.config_get('use-https') if ssl == 'yes': @@ -93,17 +102,18 @@ def balance_rings(): shutil.copyfile(os.path.join(swift.SWIFT_CONF_DIR, f), os.path.join(swift.WWW_DIR, f)) - msg = 'Broadcasting notification to all storage nodes that new '\ - 'ring is ready for consumption.' - utils.juju_log('INFO', msg) + if eligible_leader(): + msg = 'Broadcasting notification to all storage nodes that new '\ + 'ring is ready for consumption.' + utils.juju_log('INFO', msg) + www_dir = swift.WWW_DIR.split('/var/www/')[1] + trigger = uuid.uuid4() + swift_hash = swift.get_swift_hash() + # notify storage nodes that there is a new ring to fetch. + for relid in utils.relation_ids('swift-storage'): + utils.relation_set(rid=relid, swift_hash=swift_hash, + www_dir=www_dir, trigger=trigger) - www_dir = swift.WWW_DIR.split('/var/www/')[1] - trigger = uuid.uuid4() - swift_hash = swift.get_swift_hash() - # notify storage nodes that there is a new ring to fetch. - for relid in utils.relation_ids('swift-storage'): - utils.relation_set(rid=relid, swift_hash=swift_hash, - www_dir=www_dir, trigger=trigger) swift.proxy_control('restart') def storage_changed(): @@ -148,6 +158,79 @@ def config_changed(): for relid in relids: keystone_joined(relid) swift.write_proxy_config() + cluster_changed() + + +SERVICE_PORTS = { + "swift": [ + utils.config_get('bind-port'), + int(utils.config_get('bind-port')) - 10 + ] + } + +def cluster_changed(): + cluster_hosts = {} + cluster_hosts[os.getenv('JUJU_UNIT_NAME').replace('/','-')] = \ + utils.util_get('private-address') + for r_id in relation_ids('cluster'): + for unit in relation_list(r_id): + cluster_hosts[unit.replace('/','-')] = \ + utils.relation_get(attribute='private-address', + rid=r_id, + unit=unit) + configure_haproxy(cluster_hosts, + SERVICE_PORTS) + utils.restart('haproxy') + + +def ha_relation_changed(): + clustered = utils.relation_get('clustered') + if clustered and is_leader(): + juju_log('Cluster configured, notifying other services and updating' + 'keystone endpoint configuration') + # Tell all related services to start using + # the VIP and haproxy ports instead + for r_id in relation_ids('identity-service'): + keystone_joined(relid=r_id) + + +def ha_relation_joined(): + # Obtain the config values necessary for the cluster config. These + # include multicast port and interface to bind to. + corosync_bindiface = utils.config_get('ha-bindiface') + corosync_mcastport = utils.config_get('ha-mcastport') + vip = utils.config_get('vip') + vip_cidr = utils.config_get('vip_cidr') + vip_iface = utils.config_get('vip_iface') + if not vip: + utils.juju_log('ERROR', + 'Unable to configure hacluster as vip not provided') + sys.exit(1) + + # Obtain resources + resources = { + 'res_swift_vip': 'ocf:heartbeat:IPaddr2', + 'res_swift_haproxy': 'lsb:haproxy' + } + resource_params = { + 'res_swift_vip': 'params ip="%s" cidr_netmask="%s" nic="%s"' % \ + (vip, vip_cidr, vip_iface), + 'res_swift_haproxy': 'op monitor interval="5s"' + } + init_services = { + 'res_swift_haproxy': 'haproxy' + } + clones = { + 'cl_swift_haproxy': 'res_swift_haproxy' + } + + utils.relation_set(init_services=init_services, + corosync_bindiface=corosync_bindiface, + corosync_mcastport=corosync_mcastport, + resources=resources, + resource_params=resource_params, + clones=clones) + hooks = { 'install': install, @@ -156,6 +239,10 @@ hooks = { 'identity-service-relation-changed': keystone_changed, 'swift-storage-relation-changed': storage_changed, 'swift-storage-relation-broken': storage_broken, + "cluster-relation-joined": cluster_changed, + "cluster-relation-changed": cluster_changed, + "ha-relation-joined": ha_relation_joined, + "ha-relation-changed": ha_relation_changed } utils.do_hooks(hooks) diff --git a/hooks/swift_utils.py b/hooks/swift_utils.py index 15353e6..54a2c50 100644 --- a/hooks/swift_utils.py +++ b/hooks/swift_utils.py @@ -169,7 +169,7 @@ def write_proxy_config(): ctxt = { 'proxy_ip': utils.get_host_ip(), - 'bind_port': bind_port, + 'bind_port': bind_port - 10, # Drop -10 as behind haproxy 'workers': workers, 'operator_roles': utils.config_get('operator-roles') } diff --git a/hooks/utils.py b/hooks/utils.py index 283bbc6..398bf54 100644 --- a/hooks/utils.py +++ b/hooks/utils.py @@ -18,10 +18,12 @@ def do_hooks(hooks): hook = os.path.basename(sys.argv[0]) try: - hooks[hook]() + hook_func = hooks[hook] except KeyError: juju_log('INFO', "This charm doesn't know how to handle '{}'.".format(hook)) + else: + hook_func() def install(*pkgs): @@ -203,23 +205,24 @@ def config_get(attribute): except KeyError: return None + def get_unit_hostname(): return socket.gethostname() def get_host_ip(hostname=unit_get('private-address')): try: - # Test to see if already an IPv4 address - socket.inet_aton(hostname) - return hostname + # Test to see if already an IPv4 address + socket.inet_aton(hostname) + return hostname except socket.error: - try: - answers = dns.resolver.query(hostname, 'A') - if answers: - return answers[0].address - except dns.resolver.NXDOMAIN: - pass - return None + try: + answers = dns.resolver.query(hostname, 'A') + if answers: + return answers[0].address + except dns.resolver.NXDOMAIN: + pass + return None def restart(*services): @@ -235,3 +238,65 @@ def stop(*services): def start(*services): for service in services: subprocess.check_call(['service', service, 'start']) + + +def reload(*services): + for service in services: + subprocess.check_call(['service', service, 'reload']) + + +def is_clustered(): + for r_id in (relation_ids('ha') or []): + for unit in (relation_list(r_id) or []): + clustered = relation_get('clustered', + rid=r_id, + unit=unit) + if clustered: + return True + return False + + +def is_leader(): + cmd = [ + "crm", "resource", + "show", "res_swift_vip" + ] + try: + status = subprocess.check_output(cmd) + except subprocess.CalledProcessError: + return False + else: + if get_unit_hostname() in status: + return True + else: + return False + + +def peer_units(): + peers = [] + for r_id in (relation_ids('cluster') or []): + for unit in (relation_list(r_id) or []): + peers.append(unit) + return peers + + +def oldest_peer(peers): + local_unit_no = os.getenv('JUJU_UNIT_NAME').split('/')[1] + for peer in peers: + remote_unit_no = peer.split('/')[1] + if remote_unit_no < local_unit_no: + return False + return True + + +def eligible_leader(): + if is_clustered(): + if not is_leader(): + juju_log('INFO', 'Deferring action to CRM leader.') + return False + else: + peers = peer_units() + if peers and not oldest_peer(peers): + juju_log('INFO', 'Deferring action to oldest service unit.') + return False + return True diff --git a/metadata.yaml b/metadata.yaml index 3ff4d31..b70bdb1 100644 --- a/metadata.yaml +++ b/metadata.yaml @@ -12,3 +12,9 @@ requires: interface: swift identity-service: interface: keystone + ha: + interface: hacluster + scope: container +peers: + cluster: + interface: swift-ha