First cut of HA support
This commit is contained in:
parent
f19172904f
commit
7bdde70565
24
config.yaml
24
config.yaml
|
@ -105,3 +105,27 @@ options:
|
||||||
keystone-admin-password:
|
keystone-admin-password:
|
||||||
type: string
|
type: string
|
||||||
description: Keystone admin password
|
description: Keystone admin password
|
||||||
|
# HA configuration settings
|
||||||
|
vip:
|
||||||
|
type: string
|
||||||
|
description: "Virtual IP to use to front swift-proxy in ha configuration"
|
||||||
|
vip_iface:
|
||||||
|
type: string
|
||||||
|
default: eth0
|
||||||
|
description: "Network Interface where to place the Virtual IP"
|
||||||
|
vip_cidr:
|
||||||
|
type: int
|
||||||
|
default: 24
|
||||||
|
description: "Netmask that will be used for the Virtual IP"
|
||||||
|
ha-bindiface:
|
||||||
|
type: string
|
||||||
|
default: eth0
|
||||||
|
description: |
|
||||||
|
Default network interface on which HA cluster will bind to communication
|
||||||
|
with the other members of the HA Cluster.
|
||||||
|
ha-mcastport:
|
||||||
|
type: int
|
||||||
|
default: 5414
|
||||||
|
description: |
|
||||||
|
Default multicast port number that will be used to communicate between
|
||||||
|
HA Cluster nodes.
|
||||||
|
|
|
@ -3,6 +3,7 @@
|
||||||
# Common python helper functions used for OpenStack charms.
|
# Common python helper functions used for OpenStack charms.
|
||||||
|
|
||||||
import subprocess
|
import subprocess
|
||||||
|
import os
|
||||||
|
|
||||||
CLOUD_ARCHIVE_URL = "http://ubuntu-cloud.archive.canonical.com/ubuntu"
|
CLOUD_ARCHIVE_URL = "http://ubuntu-cloud.archive.canonical.com/ubuntu"
|
||||||
CLOUD_ARCHIVE_KEY_ID = '5EDB1B62EC4926EA'
|
CLOUD_ARCHIVE_KEY_ID = '5EDB1B62EC4926EA'
|
||||||
|
@ -223,3 +224,24 @@ def configure_installation_source(rel):
|
||||||
f.write(src)
|
f.write(src)
|
||||||
else:
|
else:
|
||||||
error_out("Invalid openstack-release specified: %s" % rel)
|
error_out("Invalid openstack-release specified: %s" % rel)
|
||||||
|
|
||||||
|
HAPROXY_CONF = '/etc/haproxy/haproxy.cfg'
|
||||||
|
HAPROXY_DEFAULT = '/etc/default/haproxy'
|
||||||
|
|
||||||
|
def configure_haproxy(units, service_ports, template_dir=None):
|
||||||
|
template_dir = template_dir or 'templates'
|
||||||
|
import jinja2
|
||||||
|
context = {
|
||||||
|
'units': units,
|
||||||
|
'service_ports': service_ports
|
||||||
|
}
|
||||||
|
templates = jinja2.Environment(
|
||||||
|
loader=jinja2.FileSystemLoader(template_dir)
|
||||||
|
)
|
||||||
|
template = templates.get_template(
|
||||||
|
os.path.basename(HAPROXY_CONF)
|
||||||
|
)
|
||||||
|
with open(HAPROXY_CONF, 'w') as f:
|
||||||
|
f.write(template.render(context))
|
||||||
|
with open(HAPROXY_DEFAULT, 'w') as f:
|
||||||
|
f.write('ENABLED=1')
|
||||||
|
|
|
@ -10,6 +10,11 @@ from subprocess import check_call
|
||||||
import lib.openstack_common as openstack
|
import lib.openstack_common as openstack
|
||||||
import swift_utils as swift
|
import swift_utils as swift
|
||||||
|
|
||||||
|
extra_pkgs = [
|
||||||
|
"haproxy",
|
||||||
|
"python-jinja2"
|
||||||
|
]
|
||||||
|
|
||||||
def install():
|
def install():
|
||||||
src = utils.config_get('openstack-origin')
|
src = utils.config_get('openstack-origin')
|
||||||
if src != 'distro':
|
if src != 'distro':
|
||||||
|
@ -19,6 +24,7 @@ def install():
|
||||||
|
|
||||||
pkgs = swift.determine_packages(rel)
|
pkgs = swift.determine_packages(rel)
|
||||||
utils.install(*pkgs)
|
utils.install(*pkgs)
|
||||||
|
utils.install(*extra_pkgs)
|
||||||
|
|
||||||
swift.ensure_swift_dir()
|
swift.ensure_swift_dir()
|
||||||
|
|
||||||
|
@ -57,7 +63,10 @@ def install():
|
||||||
|
|
||||||
|
|
||||||
def keystone_joined(relid=None):
|
def keystone_joined(relid=None):
|
||||||
hostname = utils.unit_get('private-address')
|
if is_clustered():
|
||||||
|
hostname = utils.config_get('vip')
|
||||||
|
else:
|
||||||
|
hostname = utils.unit_get('private-address')
|
||||||
port = utils.config_get('bind-port')
|
port = utils.config_get('bind-port')
|
||||||
ssl = utils.config_get('use-https')
|
ssl = utils.config_get('use-https')
|
||||||
if ssl == 'yes':
|
if ssl == 'yes':
|
||||||
|
@ -93,17 +102,18 @@ def balance_rings():
|
||||||
shutil.copyfile(os.path.join(swift.SWIFT_CONF_DIR, f),
|
shutil.copyfile(os.path.join(swift.SWIFT_CONF_DIR, f),
|
||||||
os.path.join(swift.WWW_DIR, f))
|
os.path.join(swift.WWW_DIR, f))
|
||||||
|
|
||||||
msg = 'Broadcasting notification to all storage nodes that new '\
|
if eligible_leader():
|
||||||
'ring is ready for consumption.'
|
msg = 'Broadcasting notification to all storage nodes that new '\
|
||||||
utils.juju_log('INFO', msg)
|
'ring is ready for consumption.'
|
||||||
|
utils.juju_log('INFO', msg)
|
||||||
|
www_dir = swift.WWW_DIR.split('/var/www/')[1]
|
||||||
|
trigger = uuid.uuid4()
|
||||||
|
swift_hash = swift.get_swift_hash()
|
||||||
|
# notify storage nodes that there is a new ring to fetch.
|
||||||
|
for relid in utils.relation_ids('swift-storage'):
|
||||||
|
utils.relation_set(rid=relid, swift_hash=swift_hash,
|
||||||
|
www_dir=www_dir, trigger=trigger)
|
||||||
|
|
||||||
www_dir = swift.WWW_DIR.split('/var/www/')[1]
|
|
||||||
trigger = uuid.uuid4()
|
|
||||||
swift_hash = swift.get_swift_hash()
|
|
||||||
# notify storage nodes that there is a new ring to fetch.
|
|
||||||
for relid in utils.relation_ids('swift-storage'):
|
|
||||||
utils.relation_set(rid=relid, swift_hash=swift_hash,
|
|
||||||
www_dir=www_dir, trigger=trigger)
|
|
||||||
swift.proxy_control('restart')
|
swift.proxy_control('restart')
|
||||||
|
|
||||||
def storage_changed():
|
def storage_changed():
|
||||||
|
@ -148,6 +158,79 @@ def config_changed():
|
||||||
for relid in relids:
|
for relid in relids:
|
||||||
keystone_joined(relid)
|
keystone_joined(relid)
|
||||||
swift.write_proxy_config()
|
swift.write_proxy_config()
|
||||||
|
cluster_changed()
|
||||||
|
|
||||||
|
|
||||||
|
SERVICE_PORTS = {
|
||||||
|
"swift": [
|
||||||
|
utils.config_get('bind-port'),
|
||||||
|
int(utils.config_get('bind-port')) - 10
|
||||||
|
]
|
||||||
|
}
|
||||||
|
|
||||||
|
def cluster_changed():
|
||||||
|
cluster_hosts = {}
|
||||||
|
cluster_hosts[os.getenv('JUJU_UNIT_NAME').replace('/','-')] = \
|
||||||
|
utils.util_get('private-address')
|
||||||
|
for r_id in relation_ids('cluster'):
|
||||||
|
for unit in relation_list(r_id):
|
||||||
|
cluster_hosts[unit.replace('/','-')] = \
|
||||||
|
utils.relation_get(attribute='private-address',
|
||||||
|
rid=r_id,
|
||||||
|
unit=unit)
|
||||||
|
configure_haproxy(cluster_hosts,
|
||||||
|
SERVICE_PORTS)
|
||||||
|
utils.restart('haproxy')
|
||||||
|
|
||||||
|
|
||||||
|
def ha_relation_changed():
|
||||||
|
clustered = utils.relation_get('clustered')
|
||||||
|
if clustered and is_leader():
|
||||||
|
juju_log('Cluster configured, notifying other services and updating'
|
||||||
|
'keystone endpoint configuration')
|
||||||
|
# Tell all related services to start using
|
||||||
|
# the VIP and haproxy ports instead
|
||||||
|
for r_id in relation_ids('identity-service'):
|
||||||
|
keystone_joined(relid=r_id)
|
||||||
|
|
||||||
|
|
||||||
|
def ha_relation_joined():
|
||||||
|
# Obtain the config values necessary for the cluster config. These
|
||||||
|
# include multicast port and interface to bind to.
|
||||||
|
corosync_bindiface = utils.config_get('ha-bindiface')
|
||||||
|
corosync_mcastport = utils.config_get('ha-mcastport')
|
||||||
|
vip = utils.config_get('vip')
|
||||||
|
vip_cidr = utils.config_get('vip_cidr')
|
||||||
|
vip_iface = utils.config_get('vip_iface')
|
||||||
|
if not vip:
|
||||||
|
utils.juju_log('ERROR',
|
||||||
|
'Unable to configure hacluster as vip not provided')
|
||||||
|
sys.exit(1)
|
||||||
|
|
||||||
|
# Obtain resources
|
||||||
|
resources = {
|
||||||
|
'res_swift_vip': 'ocf:heartbeat:IPaddr2',
|
||||||
|
'res_swift_haproxy': 'lsb:haproxy'
|
||||||
|
}
|
||||||
|
resource_params = {
|
||||||
|
'res_swift_vip': 'params ip="%s" cidr_netmask="%s" nic="%s"' % \
|
||||||
|
(vip, vip_cidr, vip_iface),
|
||||||
|
'res_swift_haproxy': 'op monitor interval="5s"'
|
||||||
|
}
|
||||||
|
init_services = {
|
||||||
|
'res_swift_haproxy': 'haproxy'
|
||||||
|
}
|
||||||
|
clones = {
|
||||||
|
'cl_swift_haproxy': 'res_swift_haproxy'
|
||||||
|
}
|
||||||
|
|
||||||
|
utils.relation_set(init_services=init_services,
|
||||||
|
corosync_bindiface=corosync_bindiface,
|
||||||
|
corosync_mcastport=corosync_mcastport,
|
||||||
|
resources=resources,
|
||||||
|
resource_params=resource_params,
|
||||||
|
clones=clones)
|
||||||
|
|
||||||
|
|
||||||
hooks = {
|
hooks = {
|
||||||
'install': install,
|
'install': install,
|
||||||
|
@ -156,6 +239,10 @@ hooks = {
|
||||||
'identity-service-relation-changed': keystone_changed,
|
'identity-service-relation-changed': keystone_changed,
|
||||||
'swift-storage-relation-changed': storage_changed,
|
'swift-storage-relation-changed': storage_changed,
|
||||||
'swift-storage-relation-broken': storage_broken,
|
'swift-storage-relation-broken': storage_broken,
|
||||||
|
"cluster-relation-joined": cluster_changed,
|
||||||
|
"cluster-relation-changed": cluster_changed,
|
||||||
|
"ha-relation-joined": ha_relation_joined,
|
||||||
|
"ha-relation-changed": ha_relation_changed
|
||||||
}
|
}
|
||||||
|
|
||||||
utils.do_hooks(hooks)
|
utils.do_hooks(hooks)
|
||||||
|
|
|
@ -169,7 +169,7 @@ def write_proxy_config():
|
||||||
|
|
||||||
ctxt = {
|
ctxt = {
|
||||||
'proxy_ip': utils.get_host_ip(),
|
'proxy_ip': utils.get_host_ip(),
|
||||||
'bind_port': bind_port,
|
'bind_port': bind_port - 10, # Drop -10 as behind haproxy
|
||||||
'workers': workers,
|
'workers': workers,
|
||||||
'operator_roles': utils.config_get('operator-roles')
|
'operator_roles': utils.config_get('operator-roles')
|
||||||
}
|
}
|
||||||
|
|
|
@ -18,10 +18,12 @@ def do_hooks(hooks):
|
||||||
hook = os.path.basename(sys.argv[0])
|
hook = os.path.basename(sys.argv[0])
|
||||||
|
|
||||||
try:
|
try:
|
||||||
hooks[hook]()
|
hook_func = hooks[hook]
|
||||||
except KeyError:
|
except KeyError:
|
||||||
juju_log('INFO',
|
juju_log('INFO',
|
||||||
"This charm doesn't know how to handle '{}'.".format(hook))
|
"This charm doesn't know how to handle '{}'.".format(hook))
|
||||||
|
else:
|
||||||
|
hook_func()
|
||||||
|
|
||||||
|
|
||||||
def install(*pkgs):
|
def install(*pkgs):
|
||||||
|
@ -203,23 +205,24 @@ def config_get(attribute):
|
||||||
except KeyError:
|
except KeyError:
|
||||||
return None
|
return None
|
||||||
|
|
||||||
|
|
||||||
def get_unit_hostname():
|
def get_unit_hostname():
|
||||||
return socket.gethostname()
|
return socket.gethostname()
|
||||||
|
|
||||||
|
|
||||||
def get_host_ip(hostname=unit_get('private-address')):
|
def get_host_ip(hostname=unit_get('private-address')):
|
||||||
try:
|
try:
|
||||||
# Test to see if already an IPv4 address
|
# Test to see if already an IPv4 address
|
||||||
socket.inet_aton(hostname)
|
socket.inet_aton(hostname)
|
||||||
return hostname
|
return hostname
|
||||||
except socket.error:
|
except socket.error:
|
||||||
try:
|
try:
|
||||||
answers = dns.resolver.query(hostname, 'A')
|
answers = dns.resolver.query(hostname, 'A')
|
||||||
if answers:
|
if answers:
|
||||||
return answers[0].address
|
return answers[0].address
|
||||||
except dns.resolver.NXDOMAIN:
|
except dns.resolver.NXDOMAIN:
|
||||||
pass
|
pass
|
||||||
return None
|
return None
|
||||||
|
|
||||||
|
|
||||||
def restart(*services):
|
def restart(*services):
|
||||||
|
@ -235,3 +238,65 @@ def stop(*services):
|
||||||
def start(*services):
|
def start(*services):
|
||||||
for service in services:
|
for service in services:
|
||||||
subprocess.check_call(['service', service, 'start'])
|
subprocess.check_call(['service', service, 'start'])
|
||||||
|
|
||||||
|
|
||||||
|
def reload(*services):
|
||||||
|
for service in services:
|
||||||
|
subprocess.check_call(['service', service, 'reload'])
|
||||||
|
|
||||||
|
|
||||||
|
def is_clustered():
|
||||||
|
for r_id in (relation_ids('ha') or []):
|
||||||
|
for unit in (relation_list(r_id) or []):
|
||||||
|
clustered = relation_get('clustered',
|
||||||
|
rid=r_id,
|
||||||
|
unit=unit)
|
||||||
|
if clustered:
|
||||||
|
return True
|
||||||
|
return False
|
||||||
|
|
||||||
|
|
||||||
|
def is_leader():
|
||||||
|
cmd = [
|
||||||
|
"crm", "resource",
|
||||||
|
"show", "res_swift_vip"
|
||||||
|
]
|
||||||
|
try:
|
||||||
|
status = subprocess.check_output(cmd)
|
||||||
|
except subprocess.CalledProcessError:
|
||||||
|
return False
|
||||||
|
else:
|
||||||
|
if get_unit_hostname() in status:
|
||||||
|
return True
|
||||||
|
else:
|
||||||
|
return False
|
||||||
|
|
||||||
|
|
||||||
|
def peer_units():
|
||||||
|
peers = []
|
||||||
|
for r_id in (relation_ids('cluster') or []):
|
||||||
|
for unit in (relation_list(r_id) or []):
|
||||||
|
peers.append(unit)
|
||||||
|
return peers
|
||||||
|
|
||||||
|
|
||||||
|
def oldest_peer(peers):
|
||||||
|
local_unit_no = os.getenv('JUJU_UNIT_NAME').split('/')[1]
|
||||||
|
for peer in peers:
|
||||||
|
remote_unit_no = peer.split('/')[1]
|
||||||
|
if remote_unit_no < local_unit_no:
|
||||||
|
return False
|
||||||
|
return True
|
||||||
|
|
||||||
|
|
||||||
|
def eligible_leader():
|
||||||
|
if is_clustered():
|
||||||
|
if not is_leader():
|
||||||
|
juju_log('INFO', 'Deferring action to CRM leader.')
|
||||||
|
return False
|
||||||
|
else:
|
||||||
|
peers = peer_units()
|
||||||
|
if peers and not oldest_peer(peers):
|
||||||
|
juju_log('INFO', 'Deferring action to oldest service unit.')
|
||||||
|
return False
|
||||||
|
return True
|
||||||
|
|
|
@ -12,3 +12,9 @@ requires:
|
||||||
interface: swift
|
interface: swift
|
||||||
identity-service:
|
identity-service:
|
||||||
interface: keystone
|
interface: keystone
|
||||||
|
ha:
|
||||||
|
interface: hacluster
|
||||||
|
scope: container
|
||||||
|
peers:
|
||||||
|
cluster:
|
||||||
|
interface: swift-ha
|
||||||
|
|
Loading…
Reference in New Issue