First cut of HA support
This commit is contained in:
parent
f19172904f
commit
7bdde70565
24
config.yaml
24
config.yaml
|
@ -105,3 +105,27 @@ options:
|
|||
keystone-admin-password:
|
||||
type: string
|
||||
description: Keystone admin password
|
||||
# HA configuration settings
|
||||
vip:
|
||||
type: string
|
||||
description: "Virtual IP to use to front swift-proxy in ha configuration"
|
||||
vip_iface:
|
||||
type: string
|
||||
default: eth0
|
||||
description: "Network Interface where to place the Virtual IP"
|
||||
vip_cidr:
|
||||
type: int
|
||||
default: 24
|
||||
description: "Netmask that will be used for the Virtual IP"
|
||||
ha-bindiface:
|
||||
type: string
|
||||
default: eth0
|
||||
description: |
|
||||
Default network interface on which HA cluster will bind to communication
|
||||
with the other members of the HA Cluster.
|
||||
ha-mcastport:
|
||||
type: int
|
||||
default: 5414
|
||||
description: |
|
||||
Default multicast port number that will be used to communicate between
|
||||
HA Cluster nodes.
|
||||
|
|
|
@ -3,6 +3,7 @@
|
|||
# Common python helper functions used for OpenStack charms.
|
||||
|
||||
import subprocess
|
||||
import os
|
||||
|
||||
CLOUD_ARCHIVE_URL = "http://ubuntu-cloud.archive.canonical.com/ubuntu"
|
||||
CLOUD_ARCHIVE_KEY_ID = '5EDB1B62EC4926EA'
|
||||
|
@ -223,3 +224,24 @@ def configure_installation_source(rel):
|
|||
f.write(src)
|
||||
else:
|
||||
error_out("Invalid openstack-release specified: %s" % rel)
|
||||
|
||||
HAPROXY_CONF = '/etc/haproxy/haproxy.cfg'
|
||||
HAPROXY_DEFAULT = '/etc/default/haproxy'
|
||||
|
||||
def configure_haproxy(units, service_ports, template_dir=None):
|
||||
template_dir = template_dir or 'templates'
|
||||
import jinja2
|
||||
context = {
|
||||
'units': units,
|
||||
'service_ports': service_ports
|
||||
}
|
||||
templates = jinja2.Environment(
|
||||
loader=jinja2.FileSystemLoader(template_dir)
|
||||
)
|
||||
template = templates.get_template(
|
||||
os.path.basename(HAPROXY_CONF)
|
||||
)
|
||||
with open(HAPROXY_CONF, 'w') as f:
|
||||
f.write(template.render(context))
|
||||
with open(HAPROXY_DEFAULT, 'w') as f:
|
||||
f.write('ENABLED=1')
|
||||
|
|
|
@ -10,6 +10,11 @@ from subprocess import check_call
|
|||
import lib.openstack_common as openstack
|
||||
import swift_utils as swift
|
||||
|
||||
extra_pkgs = [
|
||||
"haproxy",
|
||||
"python-jinja2"
|
||||
]
|
||||
|
||||
def install():
|
||||
src = utils.config_get('openstack-origin')
|
||||
if src != 'distro':
|
||||
|
@ -19,6 +24,7 @@ def install():
|
|||
|
||||
pkgs = swift.determine_packages(rel)
|
||||
utils.install(*pkgs)
|
||||
utils.install(*extra_pkgs)
|
||||
|
||||
swift.ensure_swift_dir()
|
||||
|
||||
|
@ -57,7 +63,10 @@ def install():
|
|||
|
||||
|
||||
def keystone_joined(relid=None):
|
||||
hostname = utils.unit_get('private-address')
|
||||
if is_clustered():
|
||||
hostname = utils.config_get('vip')
|
||||
else:
|
||||
hostname = utils.unit_get('private-address')
|
||||
port = utils.config_get('bind-port')
|
||||
ssl = utils.config_get('use-https')
|
||||
if ssl == 'yes':
|
||||
|
@ -93,17 +102,18 @@ def balance_rings():
|
|||
shutil.copyfile(os.path.join(swift.SWIFT_CONF_DIR, f),
|
||||
os.path.join(swift.WWW_DIR, f))
|
||||
|
||||
msg = 'Broadcasting notification to all storage nodes that new '\
|
||||
'ring is ready for consumption.'
|
||||
utils.juju_log('INFO', msg)
|
||||
if eligible_leader():
|
||||
msg = 'Broadcasting notification to all storage nodes that new '\
|
||||
'ring is ready for consumption.'
|
||||
utils.juju_log('INFO', msg)
|
||||
www_dir = swift.WWW_DIR.split('/var/www/')[1]
|
||||
trigger = uuid.uuid4()
|
||||
swift_hash = swift.get_swift_hash()
|
||||
# notify storage nodes that there is a new ring to fetch.
|
||||
for relid in utils.relation_ids('swift-storage'):
|
||||
utils.relation_set(rid=relid, swift_hash=swift_hash,
|
||||
www_dir=www_dir, trigger=trigger)
|
||||
|
||||
www_dir = swift.WWW_DIR.split('/var/www/')[1]
|
||||
trigger = uuid.uuid4()
|
||||
swift_hash = swift.get_swift_hash()
|
||||
# notify storage nodes that there is a new ring to fetch.
|
||||
for relid in utils.relation_ids('swift-storage'):
|
||||
utils.relation_set(rid=relid, swift_hash=swift_hash,
|
||||
www_dir=www_dir, trigger=trigger)
|
||||
swift.proxy_control('restart')
|
||||
|
||||
def storage_changed():
|
||||
|
@ -148,6 +158,79 @@ def config_changed():
|
|||
for relid in relids:
|
||||
keystone_joined(relid)
|
||||
swift.write_proxy_config()
|
||||
cluster_changed()
|
||||
|
||||
|
||||
SERVICE_PORTS = {
|
||||
"swift": [
|
||||
utils.config_get('bind-port'),
|
||||
int(utils.config_get('bind-port')) - 10
|
||||
]
|
||||
}
|
||||
|
||||
def cluster_changed():
|
||||
cluster_hosts = {}
|
||||
cluster_hosts[os.getenv('JUJU_UNIT_NAME').replace('/','-')] = \
|
||||
utils.util_get('private-address')
|
||||
for r_id in relation_ids('cluster'):
|
||||
for unit in relation_list(r_id):
|
||||
cluster_hosts[unit.replace('/','-')] = \
|
||||
utils.relation_get(attribute='private-address',
|
||||
rid=r_id,
|
||||
unit=unit)
|
||||
configure_haproxy(cluster_hosts,
|
||||
SERVICE_PORTS)
|
||||
utils.restart('haproxy')
|
||||
|
||||
|
||||
def ha_relation_changed():
|
||||
clustered = utils.relation_get('clustered')
|
||||
if clustered and is_leader():
|
||||
juju_log('Cluster configured, notifying other services and updating'
|
||||
'keystone endpoint configuration')
|
||||
# Tell all related services to start using
|
||||
# the VIP and haproxy ports instead
|
||||
for r_id in relation_ids('identity-service'):
|
||||
keystone_joined(relid=r_id)
|
||||
|
||||
|
||||
def ha_relation_joined():
|
||||
# Obtain the config values necessary for the cluster config. These
|
||||
# include multicast port and interface to bind to.
|
||||
corosync_bindiface = utils.config_get('ha-bindiface')
|
||||
corosync_mcastport = utils.config_get('ha-mcastport')
|
||||
vip = utils.config_get('vip')
|
||||
vip_cidr = utils.config_get('vip_cidr')
|
||||
vip_iface = utils.config_get('vip_iface')
|
||||
if not vip:
|
||||
utils.juju_log('ERROR',
|
||||
'Unable to configure hacluster as vip not provided')
|
||||
sys.exit(1)
|
||||
|
||||
# Obtain resources
|
||||
resources = {
|
||||
'res_swift_vip': 'ocf:heartbeat:IPaddr2',
|
||||
'res_swift_haproxy': 'lsb:haproxy'
|
||||
}
|
||||
resource_params = {
|
||||
'res_swift_vip': 'params ip="%s" cidr_netmask="%s" nic="%s"' % \
|
||||
(vip, vip_cidr, vip_iface),
|
||||
'res_swift_haproxy': 'op monitor interval="5s"'
|
||||
}
|
||||
init_services = {
|
||||
'res_swift_haproxy': 'haproxy'
|
||||
}
|
||||
clones = {
|
||||
'cl_swift_haproxy': 'res_swift_haproxy'
|
||||
}
|
||||
|
||||
utils.relation_set(init_services=init_services,
|
||||
corosync_bindiface=corosync_bindiface,
|
||||
corosync_mcastport=corosync_mcastport,
|
||||
resources=resources,
|
||||
resource_params=resource_params,
|
||||
clones=clones)
|
||||
|
||||
|
||||
hooks = {
|
||||
'install': install,
|
||||
|
@ -156,6 +239,10 @@ hooks = {
|
|||
'identity-service-relation-changed': keystone_changed,
|
||||
'swift-storage-relation-changed': storage_changed,
|
||||
'swift-storage-relation-broken': storage_broken,
|
||||
"cluster-relation-joined": cluster_changed,
|
||||
"cluster-relation-changed": cluster_changed,
|
||||
"ha-relation-joined": ha_relation_joined,
|
||||
"ha-relation-changed": ha_relation_changed
|
||||
}
|
||||
|
||||
utils.do_hooks(hooks)
|
||||
|
|
|
@ -169,7 +169,7 @@ def write_proxy_config():
|
|||
|
||||
ctxt = {
|
||||
'proxy_ip': utils.get_host_ip(),
|
||||
'bind_port': bind_port,
|
||||
'bind_port': bind_port - 10, # Drop -10 as behind haproxy
|
||||
'workers': workers,
|
||||
'operator_roles': utils.config_get('operator-roles')
|
||||
}
|
||||
|
|
|
@ -18,10 +18,12 @@ def do_hooks(hooks):
|
|||
hook = os.path.basename(sys.argv[0])
|
||||
|
||||
try:
|
||||
hooks[hook]()
|
||||
hook_func = hooks[hook]
|
||||
except KeyError:
|
||||
juju_log('INFO',
|
||||
"This charm doesn't know how to handle '{}'.".format(hook))
|
||||
else:
|
||||
hook_func()
|
||||
|
||||
|
||||
def install(*pkgs):
|
||||
|
@ -203,23 +205,24 @@ def config_get(attribute):
|
|||
except KeyError:
|
||||
return None
|
||||
|
||||
|
||||
def get_unit_hostname():
|
||||
return socket.gethostname()
|
||||
|
||||
|
||||
def get_host_ip(hostname=unit_get('private-address')):
|
||||
try:
|
||||
# Test to see if already an IPv4 address
|
||||
socket.inet_aton(hostname)
|
||||
return hostname
|
||||
# Test to see if already an IPv4 address
|
||||
socket.inet_aton(hostname)
|
||||
return hostname
|
||||
except socket.error:
|
||||
try:
|
||||
answers = dns.resolver.query(hostname, 'A')
|
||||
if answers:
|
||||
return answers[0].address
|
||||
except dns.resolver.NXDOMAIN:
|
||||
pass
|
||||
return None
|
||||
try:
|
||||
answers = dns.resolver.query(hostname, 'A')
|
||||
if answers:
|
||||
return answers[0].address
|
||||
except dns.resolver.NXDOMAIN:
|
||||
pass
|
||||
return None
|
||||
|
||||
|
||||
def restart(*services):
|
||||
|
@ -235,3 +238,65 @@ def stop(*services):
|
|||
def start(*services):
|
||||
for service in services:
|
||||
subprocess.check_call(['service', service, 'start'])
|
||||
|
||||
|
||||
def reload(*services):
|
||||
for service in services:
|
||||
subprocess.check_call(['service', service, 'reload'])
|
||||
|
||||
|
||||
def is_clustered():
|
||||
for r_id in (relation_ids('ha') or []):
|
||||
for unit in (relation_list(r_id) or []):
|
||||
clustered = relation_get('clustered',
|
||||
rid=r_id,
|
||||
unit=unit)
|
||||
if clustered:
|
||||
return True
|
||||
return False
|
||||
|
||||
|
||||
def is_leader():
|
||||
cmd = [
|
||||
"crm", "resource",
|
||||
"show", "res_swift_vip"
|
||||
]
|
||||
try:
|
||||
status = subprocess.check_output(cmd)
|
||||
except subprocess.CalledProcessError:
|
||||
return False
|
||||
else:
|
||||
if get_unit_hostname() in status:
|
||||
return True
|
||||
else:
|
||||
return False
|
||||
|
||||
|
||||
def peer_units():
|
||||
peers = []
|
||||
for r_id in (relation_ids('cluster') or []):
|
||||
for unit in (relation_list(r_id) or []):
|
||||
peers.append(unit)
|
||||
return peers
|
||||
|
||||
|
||||
def oldest_peer(peers):
|
||||
local_unit_no = os.getenv('JUJU_UNIT_NAME').split('/')[1]
|
||||
for peer in peers:
|
||||
remote_unit_no = peer.split('/')[1]
|
||||
if remote_unit_no < local_unit_no:
|
||||
return False
|
||||
return True
|
||||
|
||||
|
||||
def eligible_leader():
|
||||
if is_clustered():
|
||||
if not is_leader():
|
||||
juju_log('INFO', 'Deferring action to CRM leader.')
|
||||
return False
|
||||
else:
|
||||
peers = peer_units()
|
||||
if peers and not oldest_peer(peers):
|
||||
juju_log('INFO', 'Deferring action to oldest service unit.')
|
||||
return False
|
||||
return True
|
||||
|
|
|
@ -12,3 +12,9 @@ requires:
|
|||
interface: swift
|
||||
identity-service:
|
||||
interface: keystone
|
||||
ha:
|
||||
interface: hacluster
|
||||
scope: container
|
||||
peers:
|
||||
cluster:
|
||||
interface: swift-ha
|
||||
|
|
Loading…
Reference in New Issue