openstack
/
charm-swift-proxy
Code Issues Proposed changes

First cut of HA support

This commit is contained in:
James Page 2013-02-27 16:07:46 +00:00
parent f19172904f
commit 7bdde70565
6 changed files with 227 additions and 23 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

24
config.yaml
View File

@ -105,3 +105,27 @@ options:
keystone-admin-password:
type: string
description: Keystone admin password
# HA configuration settings
vip:
type: string
description: "Virtual IP to use to front swift-proxy in ha configuration"
vip_iface:
type: string
default: eth0
description: "Network Interface where to place the Virtual IP"
vip_cidr:
type: int
default: 24
description: "Netmask that will be used for the Virtual IP"
ha-bindiface:
type: string
default: eth0
description: |
Default network interface on which HA cluster will bind to communication
with the other members of the HA Cluster.
ha-mcastport:
type: int
default: 5414
description: |
Default multicast port number that will be used to communicate between
HA Cluster nodes.

22
hooks/lib/openstack_common.py
View File

@ -3,6 +3,7 @@
# Common python helper functions used for OpenStack charms.
import subprocess
import os
CLOUD_ARCHIVE_URL = "http://ubuntu-cloud.archive.canonical.com/ubuntu"
CLOUD_ARCHIVE_KEY_ID = '5EDB1B62EC4926EA'
@ -223,3 +224,24 @@ def configure_installation_source(rel):
f.write(src)
else:
error_out("Invalid openstack-release specified: %s" % rel)
HAPROXY_CONF = '/etc/haproxy/haproxy.cfg'
HAPROXY_DEFAULT = '/etc/default/haproxy'
def configure_haproxy(units, service_ports, template_dir=None):
template_dir = template_dir or 'templates'
import jinja2
context = {
'units': units,
'service_ports': service_ports
}
templates = jinja2.Environment(
loader=jinja2.FileSystemLoader(template_dir)
)
template = templates.get_template(
os.path.basename(HAPROXY_CONF)
)
with open(HAPROXY_CONF, 'w') as f:
f.write(template.render(context))
with open(HAPROXY_DEFAULT, 'w') as f:
f.write('ENABLED=1')

109
hooks/swift-hooks.py
View File

@ -10,6 +10,11 @@ from subprocess import check_call
import lib.openstack_common as openstack
import swift_utils as swift
extra_pkgs = [
"haproxy",
"python-jinja2"
]
def install():
src = utils.config_get('openstack-origin')
if src != 'distro':
@ -19,6 +24,7 @@ def install():
pkgs = swift.determine_packages(rel)
utils.install(*pkgs)
utils.install(*extra_pkgs)
swift.ensure_swift_dir()
@ -57,7 +63,10 @@ def install():
def keystone_joined(relid=None):
hostname = utils.unit_get('private-address')
if is_clustered():
hostname = utils.config_get('vip')
else:
hostname = utils.unit_get('private-address')
port = utils.config_get('bind-port')
ssl = utils.config_get('use-https')
if ssl == 'yes':
@ -93,17 +102,18 @@ def balance_rings():
shutil.copyfile(os.path.join(swift.SWIFT_CONF_DIR, f),
os.path.join(swift.WWW_DIR, f))
msg = 'Broadcasting notification to all storage nodes that new '\
'ring is ready for consumption.'
utils.juju_log('INFO', msg)
if eligible_leader():
msg = 'Broadcasting notification to all storage nodes that new '\
'ring is ready for consumption.'
utils.juju_log('INFO', msg)
www_dir = swift.WWW_DIR.split('/var/www/')[1]
trigger = uuid.uuid4()
swift_hash = swift.get_swift_hash()
# notify storage nodes that there is a new ring to fetch.
for relid in utils.relation_ids('swift-storage'):
utils.relation_set(rid=relid, swift_hash=swift_hash,
www_dir=www_dir, trigger=trigger)
www_dir = swift.WWW_DIR.split('/var/www/')[1]
trigger = uuid.uuid4()
swift_hash = swift.get_swift_hash()
# notify storage nodes that there is a new ring to fetch.
for relid in utils.relation_ids('swift-storage'):
utils.relation_set(rid=relid, swift_hash=swift_hash,
www_dir=www_dir, trigger=trigger)
swift.proxy_control('restart')
def storage_changed():
@ -148,6 +158,79 @@ def config_changed():
for relid in relids:
keystone_joined(relid)
swift.write_proxy_config()
cluster_changed()
SERVICE_PORTS = {
"swift": [
utils.config_get('bind-port'),
int(utils.config_get('bind-port')) - 10
]
}
def cluster_changed():
cluster_hosts = {}
cluster_hosts[os.getenv('JUJU_UNIT_NAME').replace('/','-')] = \
utils.util_get('private-address')
for r_id in relation_ids('cluster'):
for unit in relation_list(r_id):
cluster_hosts[unit.replace('/','-')] = \
utils.relation_get(attribute='private-address',
rid=r_id,
unit=unit)
configure_haproxy(cluster_hosts,
SERVICE_PORTS)
utils.restart('haproxy')
def ha_relation_changed():
clustered = utils.relation_get('clustered')
if clustered and is_leader():
juju_log('Cluster configured, notifying other services and updating'
'keystone endpoint configuration')
# Tell all related services to start using
# the VIP and haproxy ports instead
for r_id in relation_ids('identity-service'):
keystone_joined(relid=r_id)
def ha_relation_joined():
# Obtain the config values necessary for the cluster config. These
# include multicast port and interface to bind to.
corosync_bindiface = utils.config_get('ha-bindiface')
corosync_mcastport = utils.config_get('ha-mcastport')
vip = utils.config_get('vip')
vip_cidr = utils.config_get('vip_cidr')
vip_iface = utils.config_get('vip_iface')
if not vip:
utils.juju_log('ERROR',
'Unable to configure hacluster as vip not provided')
sys.exit(1)
# Obtain resources
resources = {
'res_swift_vip': 'ocf:heartbeat:IPaddr2',
'res_swift_haproxy': 'lsb:haproxy'
}
resource_params = {
'res_swift_vip': 'params ip="%s" cidr_netmask="%s" nic="%s"' % \
(vip, vip_cidr, vip_iface),
'res_swift_haproxy': 'op monitor interval="5s"'
}
init_services = {
'res_swift_haproxy': 'haproxy'
}
clones = {
'cl_swift_haproxy': 'res_swift_haproxy'
}
utils.relation_set(init_services=init_services,
corosync_bindiface=corosync_bindiface,
corosync_mcastport=corosync_mcastport,
resources=resources,
resource_params=resource_params,
clones=clones)
hooks = {
'install': install,
@ -156,6 +239,10 @@ hooks = {
'identity-service-relation-changed': keystone_changed,
'swift-storage-relation-changed': storage_changed,
'swift-storage-relation-broken': storage_broken,
"cluster-relation-joined": cluster_changed,
"cluster-relation-changed": cluster_changed,
"ha-relation-joined": ha_relation_joined,
"ha-relation-changed": ha_relation_changed
}
utils.do_hooks(hooks)

2
hooks/swift_utils.py
View File

@ -169,7 +169,7 @@ def write_proxy_config():
ctxt = {
'proxy_ip': utils.get_host_ip(),
'bind_port': bind_port,
'bind_port': bind_port - 10, # Drop -10 as behind haproxy
'workers': workers,
'operator_roles': utils.config_get('operator-roles')
}

87
hooks/utils.py
View File

@ -18,10 +18,12 @@ def do_hooks(hooks):
hook = os.path.basename(sys.argv[0])
try:
hooks[hook]()
hook_func = hooks[hook]
except KeyError:
juju_log('INFO',
"This charm doesn't know how to handle '{}'.".format(hook))
else:
hook_func()
def install(*pkgs):
@ -203,23 +205,24 @@ def config_get(attribute):
except KeyError:
return None
def get_unit_hostname():
return socket.gethostname()
def get_host_ip(hostname=unit_get('private-address')):
try:
# Test to see if already an IPv4 address
socket.inet_aton(hostname)
return hostname
# Test to see if already an IPv4 address
socket.inet_aton(hostname)
return hostname
except socket.error:
try:
answers = dns.resolver.query(hostname, 'A')
if answers:
return answers[0].address
except dns.resolver.NXDOMAIN:
pass
return None
try:
answers = dns.resolver.query(hostname, 'A')
if answers:
return answers[0].address
except dns.resolver.NXDOMAIN:
pass
return None
def restart(*services):
@ -235,3 +238,65 @@ def stop(*services):
def start(*services):
for service in services:
subprocess.check_call(['service', service, 'start'])
def reload(*services):
for service in services:
subprocess.check_call(['service', service, 'reload'])
def is_clustered():
for r_id in (relation_ids('ha') or []):
for unit in (relation_list(r_id) or []):
clustered = relation_get('clustered',
rid=r_id,
unit=unit)
if clustered:
return True
return False
def is_leader():
cmd = [
"crm", "resource",
"show", "res_swift_vip"
]
try:
status = subprocess.check_output(cmd)
except subprocess.CalledProcessError:
return False
else:
if get_unit_hostname() in status:
return True
else:
return False
def peer_units():
peers = []
for r_id in (relation_ids('cluster') or []):
for unit in (relation_list(r_id) or []):
peers.append(unit)
return peers
def oldest_peer(peers):
local_unit_no = os.getenv('JUJU_UNIT_NAME').split('/')[1]
for peer in peers:
remote_unit_no = peer.split('/')[1]
if remote_unit_no < local_unit_no:
return False
return True
def eligible_leader():
if is_clustered():
if not is_leader():
juju_log('INFO', 'Deferring action to CRM leader.')
return False
else:
peers = peer_units()
if peers and not oldest_peer(peers):
juju_log('INFO', 'Deferring action to oldest service unit.')
return False
return True

6
metadata.yaml
View File

@ -12,3 +12,9 @@ requires:
interface: swift
identity-service:
interface: keystone
ha:
interface: hacluster
scope: container
peers:
cluster:
interface: swift-ha