From 14f39ff1336404220a9544b58b95c1d39c59892c Mon Sep 17 00:00:00 2001 From: Edward Hope-Morley Date: Mon, 5 Jan 2015 17:49:38 +0000 Subject: [PATCH 01/48] [hopem,r=] Fixes ssl cert sycnhronisation across peers Closes-Bug: 1317782 --- hooks/charmhelpers/contrib/unison/__init__.py | 35 +- hooks/keystone_context.py | 51 ++- hooks/keystone_hooks.py | 79 +++-- hooks/keystone_ssl.py | 3 + hooks/keystone_utils.py | 328 +++++++++++++++--- unit_tests/test_keystone_hooks.py | 50 +-- unit_tests/test_keystone_utils.py | 4 +- 7 files changed, 440 insertions(+), 110 deletions(-) diff --git a/hooks/charmhelpers/contrib/unison/__init__.py b/hooks/charmhelpers/contrib/unison/__init__.py index f903ac03..261f7cd2 100644 --- a/hooks/charmhelpers/contrib/unison/__init__.py +++ b/hooks/charmhelpers/contrib/unison/__init__.py @@ -228,7 +228,12 @@ def collect_authed_hosts(peer_interface): return hosts -def sync_path_to_host(path, host, user, verbose=False, cmd=None, gid=None): +def sync_path_to_host(path, host, user, verbose=False, cmd=None, gid=None, + fatal=False): + """Sync path to an specific peer host + + Propagates exception if operation fails and fatal=True. + """ cmd = cmd or copy(BASE_CMD) if not verbose: cmd.append('-silent') @@ -245,20 +250,30 @@ def sync_path_to_host(path, host, user, verbose=False, cmd=None, gid=None): run_as_user(user, cmd, gid) except: log('Error syncing remote files') + if fatal: + raise -def sync_to_peer(host, user, paths=None, verbose=False, cmd=None, gid=None): - '''Sync paths to an specific host''' +def sync_to_peer(host, user, paths=None, verbose=False, cmd=None, gid=None, + fatal=False): + """Sync paths to an specific peer host + + Propagates exception if any operation fails and fatal=True. + """ if paths: for p in paths: - sync_path_to_host(p, host, user, verbose, cmd, gid) + sync_path_to_host(p, host, user, verbose, cmd, gid, fatal) -def sync_to_peers(peer_interface, user, paths=None, - verbose=False, cmd=None, gid=None): - '''Sync all hosts to an specific path''' - '''The type of group is integer, it allows user has permissions to ''' - '''operate a directory have a different group id with the user id.''' +def sync_to_peers(peer_interface, user, paths=None, verbose=False, cmd=None, + gid=None, fatal=False): + """Sync all hosts to an specific path + + The type of group is integer, it allows user has permissions to + operate a directory have a different group id with the user id. + + Propagates exception if any operation fails and fatal=True. + """ if paths: for host in collect_authed_hosts(peer_interface): - sync_to_peer(host, user, paths, verbose, cmd, gid) + sync_to_peer(host, user, paths, verbose, cmd, gid, fatal) diff --git a/hooks/keystone_context.py b/hooks/keystone_context.py index c61e70a1..cfb7eb72 100644 --- a/hooks/keystone_context.py +++ b/hooks/keystone_context.py @@ -1,3 +1,5 @@ +import os + from charmhelpers.core.hookenv import config from charmhelpers.core.host import mkdir, write_file @@ -9,9 +11,12 @@ from charmhelpers.contrib.hahelpers.cluster import ( determine_api_port ) -from charmhelpers.contrib.hahelpers.apache import install_ca_cert +from charmhelpers.core.hookenv import ( + log, + INFO, +) -import os +from charmhelpers.contrib.hahelpers.apache import install_ca_cert CA_CERT_PATH = '/usr/local/share/ca-certificates/keystone_juju_ca_cert.crt' @@ -29,20 +34,52 @@ class ApacheSSLContext(context.ApacheSSLContext): return super(ApacheSSLContext, self).__call__() def configure_cert(self, cn): - from keystone_utils import SSH_USER, get_ca + from keystone_utils import ( + SSH_USER, + get_ca, + is_ssl_cert_master, + ensure_permissions, + ) + ssl_dir = os.path.join('/etc/apache2/ssl/', self.service_namespace) - mkdir(path=ssl_dir) + perms = 0o755 + mkdir(path=ssl_dir, owner=SSH_USER, group='keystone', perms=perms) + # Ensure accessible by keystone ssh user and group (for sync) + ensure_permissions(ssl_dir, user=SSH_USER, group='keystone', + perms=perms) + + if not is_ssl_cert_master(): + log("Not leader or cert master so skipping apache cert config", + level=INFO) + return + + log("Creating apache ssl certs in %s" % (ssl_dir), level=INFO) + ca = get_ca(user=SSH_USER) cert, key = ca.get_cert_and_key(common_name=cn) write_file(path=os.path.join(ssl_dir, 'cert_{}'.format(cn)), - content=cert) + content=cert, owner=SSH_USER, group='keystone', perms=0o644) write_file(path=os.path.join(ssl_dir, 'key_{}'.format(cn)), - content=key) + content=key, owner=SSH_USER, group='keystone', perms=0o644) def configure_ca(self): - from keystone_utils import SSH_USER, get_ca + from keystone_utils import ( + SSH_USER, + get_ca, + is_ssl_cert_master, + ensure_permissions, + ) + + if not is_ssl_cert_master(): + log("Not leader or cert master so skipping apache ca config", + level=INFO) + return + ca = get_ca(user=SSH_USER) install_ca_cert(ca.get_ca_bundle()) + # Ensure accessible by keystone ssh user and group (unison) + ensure_permissions(CA_CERT_PATH, user=SSH_USER, group='keystone', + perms=0o0644) def canonical_names(self): addresses = self.get_network_addresses() diff --git a/hooks/keystone_hooks.py b/hooks/keystone_hooks.py index 27b987e9..4ab442a9 100755 --- a/hooks/keystone_hooks.py +++ b/hooks/keystone_hooks.py @@ -1,7 +1,8 @@ #!/usr/bin/python - import hashlib import os +import re +import stat import sys import time @@ -16,6 +17,7 @@ from charmhelpers.core.hookenv import ( is_relation_made, log, local_unit, + WARNING, ERROR, relation_get, relation_ids, @@ -57,11 +59,13 @@ from keystone_utils import ( STORED_PASSWD, setup_ipv6, send_notifications, + check_peer_actions, + CA_CERT_PATH, + ensure_permissions, ) from charmhelpers.contrib.hahelpers.cluster import ( - eligible_leader, - is_leader, + is_elected_leader, get_hacluster_config, ) @@ -109,10 +113,19 @@ def config_changed(): check_call(['chmod', '-R', 'g+wrx', '/var/lib/keystone/']) + # Ensure unison can write to certs dir. + # FIXME: need to a better way around this e.g. move cert to it's own dir + # and give that unison permissions. + path = os.path.dirname(CA_CERT_PATH) + perms = int(oct(stat.S_IMODE(os.stat(path).st_mode) | + (stat.S_IWGRP | stat.S_IXGRP)), base=8) + ensure_permissions(path, group='keystone', perms=perms) + save_script_rc() configure_https() CONFIGS.write_all() - if eligible_leader(CLUSTER_RES): + + if is_elected_leader(CLUSTER_RES): migrate_database() ensure_initial_admin(config) log('Firing identity_changed hook for all related services.') @@ -121,7 +134,9 @@ def config_changed(): for r_id in relation_ids('identity-service'): for unit in relation_list(r_id): identity_changed(relation_id=r_id, - remote_unit=unit) + remote_unit=unit, sync_certs=False) + + synchronize_ca() [cluster_joined(rid) for rid in relation_ids('cluster')] @@ -163,7 +178,7 @@ def db_changed(): log('shared-db relation incomplete. Peer not ready?') else: CONFIGS.write(KEYSTONE_CONF) - if eligible_leader(CLUSTER_RES): + if is_elected_leader(CLUSTER_RES): # Bugs 1353135 & 1187508. Dbs can appear to be ready before the # units acl entry has been added. So, if the db supports passing # a list of permitted units then check if we're in the list. @@ -188,7 +203,7 @@ def pgsql_db_changed(): log('pgsql-db relation incomplete. Peer not ready?') else: CONFIGS.write(KEYSTONE_CONF) - if eligible_leader(CLUSTER_RES): + if is_elected_leader(CLUSTER_RES): migrate_database() ensure_initial_admin(config) # Ensure any existing service entries are updated in the @@ -199,11 +214,27 @@ def pgsql_db_changed(): @hooks.hook('identity-service-relation-changed') -def identity_changed(relation_id=None, remote_unit=None): +def identity_changed(relation_id=None, remote_unit=None, sync_certs=True): notifications = {} - if eligible_leader(CLUSTER_RES): - add_service_to_keystone(relation_id, remote_unit) - synchronize_ca() + if is_elected_leader(CLUSTER_RES): + # Catch database not configured error and defer until db ready + from keystoneclient.apiclient.exceptions import InternalServerError + try: + add_service_to_keystone(relation_id, remote_unit) + except InternalServerError as exc: + key = re.compile("'keystone\..+' doesn't exist") + if re.search(key, exc.message): + log("Keystone database not yet ready (InternalServerError " + "raised) - deferring until *-db relation completes.", + level=WARNING) + return + + log("Unexpected exception occurred", level=ERROR) + raise + + CONFIGS.write_all() + if sync_certs: + synchronize_ca() settings = relation_get(rid=relation_id, unit=remote_unit) service = settings.get('service', None) @@ -257,18 +288,22 @@ def cluster_joined(relation_id=None): 'cluster-relation-departed') @restart_on_change(restart_map(), stopstart=True) def cluster_changed(): + check_peer_actions() + # NOTE(jamespage) re-echo passwords for peer storage - peer_echo(includes=['_passwd', 'identity-service:']) + echo_whitelist = ['_passwd', 'identity-service:', 'ssl-cert-master'] + peer_echo(includes=echo_whitelist) unison.ssh_authorized_peers(user=SSH_USER, group='keystone', peer_interface='cluster', ensure_local_user=True) - synchronize_ca() CONFIGS.write_all() for r_id in relation_ids('identity-service'): for unit in relation_list(r_id): - identity_changed(relation_id=r_id, - remote_unit=unit) + identity_changed(relation_id=r_id, remote_unit=unit, + sync_certs=False) + + synchronize_ca() @hooks.hook('ha-relation-joined') @@ -325,14 +360,16 @@ def ha_joined(): def ha_changed(): clustered = relation_get('clustered') CONFIGS.write_all() - if (clustered is not None and - is_leader(CLUSTER_RES)): + if clustered is not None and is_elected_leader(CLUSTER_RES): ensure_initial_admin(config) log('Cluster configured, notifying other services and updating ' 'keystone endpoint configuration') for rid in relation_ids('identity-service'): for unit in related_units(rid): - identity_changed(relation_id=rid, remote_unit=unit) + identity_changed(relation_id=rid, remote_unit=unit, + sync_certs=False) + + synchronize_ca() @hooks.hook('identity-admin-relation-changed') @@ -375,8 +412,7 @@ def upgrade_charm(): group='keystone', peer_interface='cluster', ensure_local_user=True) - synchronize_ca() - if eligible_leader(CLUSTER_RES): + if is_elected_leader(CLUSTER_RES): log('Cluster leader - ensuring endpoint configuration' ' is up to date') time.sleep(10) @@ -385,8 +421,9 @@ def upgrade_charm(): for r_id in relation_ids('identity-service'): for unit in relation_list(r_id): identity_changed(relation_id=r_id, - remote_unit=unit) + remote_unit=unit, sync_certs=False) CONFIGS.write_all() + synchronize_ca() def main(): diff --git a/hooks/keystone_ssl.py b/hooks/keystone_ssl.py index 45e0029d..6b2f4a85 100644 --- a/hooks/keystone_ssl.py +++ b/hooks/keystone_ssl.py @@ -101,6 +101,9 @@ keyUsage = digitalSignature, keyEncipherment, keyAgreement extendedKeyUsage = serverAuth, clientAuth """ +# Instance can be appended to this list to represent a singleton +CA_SINGLETON = [] + def init_ca(ca_dir, common_name, org_name=ORG_NAME, org_unit_name=ORG_UNIT): print 'Ensuring certificate authority exists at %s.' % ca_dir diff --git a/hooks/keystone_utils.py b/hooks/keystone_utils.py index 08bd3de7..ff14f941 100644 --- a/hooks/keystone_utils.py +++ b/hooks/keystone_utils.py @@ -1,8 +1,12 @@ #!/usr/bin/python +import glob +import grp import subprocess import os +import pwd import uuid import urlparse +import shutil import time from base64 import b64encode @@ -10,11 +14,11 @@ from collections import OrderedDict from copy import deepcopy from charmhelpers.contrib.hahelpers.cluster import( - eligible_leader, + is_elected_leader, determine_api_port, https, is_clustered, - is_elected_leader, + peer_units, ) from charmhelpers.contrib.openstack import context, templating @@ -37,8 +41,17 @@ from charmhelpers.contrib.openstack.utils import ( os_release, save_script_rc as _save_script_rc) +from charmhelpers.core.host import ( + mkdir, + write_file, +) + import charmhelpers.contrib.unison as unison +from charmhelpers.core.decorators import ( + retry_on_exception, +) + from charmhelpers.core.hookenv import ( config, log, @@ -46,8 +59,10 @@ from charmhelpers.core.hookenv import ( relation_get, relation_set, relation_ids, + unit_get, DEBUG, INFO, + WARNING, ) from charmhelpers.fetch import ( @@ -60,6 +75,7 @@ from charmhelpers.fetch import ( from charmhelpers.core.host import ( service_stop, service_start, + service_restart, pwgen, lsb_release ) @@ -108,6 +124,8 @@ HAPROXY_CONF = '/etc/haproxy/haproxy.cfg' APACHE_CONF = '/etc/apache2/sites-available/openstack_https_frontend' APACHE_24_CONF = '/etc/apache2/sites-available/openstack_https_frontend.conf' +APACHE_SSL_DIR = '/etc/apache2/ssl/keystone' +SYNC_FLAGS_DIR = '/var/lib/keystone/juju_sync_flags/' SSL_DIR = '/var/lib/keystone/juju_ssl/' SSL_CA_NAME = 'Ubuntu Cloud' CLUSTER_RES = 'grp_ks_vips' @@ -197,6 +215,13 @@ valid_services = { } +def str_is_true(value): + if value and value.lower() in ['true', 'yes']: + return True + + return False + + def resource_map(): ''' Dynamically generate a map of resources that will be managed for a single @@ -272,7 +297,7 @@ def do_openstack_upgrade(configs): configs.set_release(openstack_release=new_os_rel) configs.write_all() - if eligible_leader(CLUSTER_RES): + if is_elected_leader(CLUSTER_RES): migrate_database() @@ -474,44 +499,57 @@ def grant_role(user, role, tenant): def ensure_initial_admin(config): - """ Ensures the minimum admin stuff exists in whatever database we're + # Allow retry on fail since leader may not be ready yet. + # NOTE(hopem): ks client may not be installed at module import time so we + # use this wrapped approach instead. + from keystoneclient.apiclient.exceptions import InternalServerError + + @retry_on_exception(3, base_delay=3, exc_type=InternalServerError) + def _ensure_initial_admin(config): + """Ensures the minimum admin stuff exists in whatever database we're using. + This and the helper functions it calls are meant to be idempotent and run during install as well as during db-changed. This will maintain the admin tenant, user, role, service entry and endpoint across every datastore we might use. + TODO: Possibly migrate data from one backend to another after it changes? - """ - create_tenant("admin") - create_tenant(config("service-tenant")) + """ + create_tenant("admin") + create_tenant(config("service-tenant")) - passwd = "" - if config("admin-password") != "None": - passwd = config("admin-password") - elif os.path.isfile(STORED_PASSWD): - log("Loading stored passwd from %s" % STORED_PASSWD) - passwd = open(STORED_PASSWD, 'r').readline().strip('\n') - if passwd == "": - log("Generating new passwd for user: %s" % - config("admin-user")) - cmd = ['pwgen', '-c', '16', '1'] - passwd = str(subprocess.check_output(cmd)).strip() - open(STORED_PASSWD, 'w+').writelines("%s\n" % passwd) - # User is managed by ldap backend when using ldap identity - if not (config('identity-backend') == 'ldap' and config('ldap-readonly')): - create_user(config('admin-user'), passwd, tenant='admin') - update_user_password(config('admin-user'), passwd) - create_role(config('admin-role'), config('admin-user'), 'admin') - create_service_entry("keystone", "identity", "Keystone Identity Service") + passwd = "" + if config("admin-password") != "None": + passwd = config("admin-password") + elif os.path.isfile(STORED_PASSWD): + log("Loading stored passwd from %s" % STORED_PASSWD) + passwd = open(STORED_PASSWD, 'r').readline().strip('\n') + if passwd == "": + log("Generating new passwd for user: %s" % + config("admin-user")) + cmd = ['pwgen', '-c', '16', '1'] + passwd = str(subprocess.check_output(cmd)).strip() + open(STORED_PASSWD, 'w+').writelines("%s\n" % passwd) + # User is managed by ldap backend when using ldap identity + if (not (config('identity-backend') == 'ldap' and + config('ldap-readonly'))): + create_user(config('admin-user'), passwd, tenant='admin') + update_user_password(config('admin-user'), passwd) + create_role(config('admin-role'), config('admin-user'), 'admin') + create_service_entry("keystone", "identity", + "Keystone Identity Service") - for region in config('region').split(): - create_keystone_endpoint(public_ip=resolve_address(PUBLIC), - service_port=config("service-port"), - internal_ip=resolve_address(INTERNAL), - admin_ip=resolve_address(ADMIN), - auth_port=config("admin-port"), - region=region) + for region in config('region').split(): + create_keystone_endpoint(public_ip=resolve_address(PUBLIC), + service_port=config("service-port"), + internal_ip=resolve_address(INTERNAL), + admin_ip=resolve_address(ADMIN), + auth_port=config("admin-port"), + region=region) + + return _ensure_initial_admin(config) def endpoint_url(ip, port): @@ -579,20 +617,201 @@ def get_service_password(service_username): return passwd -def synchronize_ca(): - ''' - Broadcast service credentials to peers or consume those that have been - broadcasted by peer, depending on hook context. - ''' - if not eligible_leader(CLUSTER_RES): - return - log('Synchronizing CA to all peers.') - if is_clustered(): - if config('https-service-endpoints') in ['True', 'true']: - unison.sync_to_peers(peer_interface='cluster', - paths=[SSL_DIR], user=SSH_USER, verbose=True) +def ensure_permissions(path, user=None, group=None, perms=None): + """Set chownand chmod for path -CA = [] + Note that -1 for uid or gid result in no change. + """ + if user: + uid = pwd.getpwnam(user).pw_uid + else: + uid = -1 + + if group: + gid = grp.getgrnam(group).gr_gid + else: + gid = -1 + + os.chown(path, uid, gid) + + if perms: + os.chmod(path, perms) + + +def check_peer_actions(): + """Honour service action requests from sync master. + + Check for service action request flags, perform the action then delete the + flag. + """ + restart = relation_get(attribute='restart-services-trigger') + if restart and os.path.isdir(SYNC_FLAGS_DIR): + for flagfile in glob.glob(os.path.join(SYNC_FLAGS_DIR, '*')): + flag = os.path.basename(flagfile) + service = flag.partition('.')[0] + action = flag.partition('.')[2] + + if action == 'restart': + log("Running action='%s' on service '%s'" % + (action, service), level=DEBUG) + service_restart(service) + elif action == 'start': + log("Running action='%s' on service '%s'" % + (action, service), level=DEBUG) + service_start(service) + elif action == 'stop': + log("Running action='%s' on service '%s'" % + (action, service), level=DEBUG) + service_stop(service) + elif flag == 'update-ca-certificates': + log("Running update-ca-certificates", level=DEBUG) + subprocess.check_call(['update-ca-certificates']) + else: + log("Unknown action flag=%s" % (flag), level=WARNING) + + os.remove(flagfile) + + +def create_peer_service_actions(action, services): + """Mark remote services for action. + + Default action is restart. These action will be picked up by peer units + e.g. we may need to restart services on peer units after certs have been + synced. + """ + for service in services: + flagfile = os.path.join(SYNC_FLAGS_DIR, '%s.%s' % + (service.strip(), action)) + log("Creating action %s" % (flagfile), level=DEBUG) + write_file(flagfile, content='', owner=SSH_USER, group='keystone', + perms=0o644) + + +def create_service_action(action): + flagfile = os.path.join(SYNC_FLAGS_DIR, action) + log("Creating action %s" % (flagfile), level=DEBUG) + write_file(flagfile, content='', owner=SSH_USER, group='keystone', + perms=0o644) + + +def is_ssl_cert_master(): + """Return True if this unit is ssl cert master.""" + master = None + for rid in relation_ids('cluster'): + master = relation_get(attribute='ssl-cert-master', rid=rid, + unit=local_unit()) + + if master and master == unit_get('private-address'): + return True + + return False + + +@retry_on_exception(3, base_delay=2, exc_type=subprocess.CalledProcessError) +def unison_sync(paths_to_sync): + """Do unison sync and retry a few times if it fails since peers may not be + ready for sync. + """ + log('Synchronizing CA (%s) to all peers.' % (', '.join(paths_to_sync)), + level=INFO) + keystone_gid = grp.getgrnam('keystone').gr_gid + unison.sync_to_peers(peer_interface='cluster', paths=paths_to_sync, + user=SSH_USER, verbose=True, gid=keystone_gid, + fatal=True) + + +def synchronize_ca(fatal=True): + """Broadcast service credentials to peers. + + By default a failure to sync is fatal and will result in a raised + exception. + + This function uses a relation setting 'ssl-cert-master' to get some + leader stickiness while synchronisation is being carried out. This ensures + that the last host to create and broadcast cetificates has the option to + complete actions before electing the new leader as sync master. + """ + paths_to_sync = [SYNC_FLAGS_DIR] + + if not peer_units(): + log("Not syncing certs since there are no peer units.", level=INFO) + return + + # If no ssl master elected and we are cluster leader, elect this unit. + if is_elected_leader(CLUSTER_RES): + master = relation_get(attribute='ssl-cert-master') + if not master or master == 'unknown': + log("Electing this unit as ssl-cert-master", level=DEBUG) + for rid in relation_ids('cluster'): + relation_set(relation_id=rid, + relation_settings={'ssl-cert-master': + unit_get('private-address'), + 'trigger': str(uuid.uuid4())}) + # Return now and wait for echo before continuing. + return + + if not is_ssl_cert_master(): + log("Not ssl cert master - skipping sync", level=INFO) + return + + if str_is_true(config('https-service-endpoints')): + log("Syncing all endpoint certs since https-service-endpoints=True", + level=DEBUG) + paths_to_sync.append(SSL_DIR) + paths_to_sync.append(APACHE_SSL_DIR) + paths_to_sync.append(CA_CERT_PATH) + elif str_is_true(config('use-https')): + log("Syncing keystone-endpoint certs since use-https=True", + level=DEBUG) + paths_to_sync.append(APACHE_SSL_DIR) + paths_to_sync.append(CA_CERT_PATH) + + if not paths_to_sync: + log("Nothing to sync - skipping", level=DEBUG) + return + + # If we are sync master proceed even if we are not leader since we are + # most likely to have up-to-date certs. If not leader we will re-elect once + # synced. This is done to avoid being affected by leader changing before + # all units have synced certs. + + # Clear any existing flags + if os.path.isdir(SYNC_FLAGS_DIR): + shutil.rmtree(SYNC_FLAGS_DIR) + + mkdir(SYNC_FLAGS_DIR, SSH_USER, 'keystone', 0o775) + + # We need to restart peer apache services to ensure they have picked up + # new ssl keys. + create_peer_service_actions('restart', ['apache2']) + create_service_action('update-ca-certificates') + + try: + unison_sync(paths_to_sync) + except: + if fatal: + raise + else: + log("Sync failed but fatal=False", level=INFO) + return + + trigger = str(uuid.uuid4()) + log("Sending restart-services-trigger=%s to all peers" % (trigger), + level=DEBUG) + settings = {'restart-services-trigger': trigger} + + # Cleanup + shutil.rmtree(SYNC_FLAGS_DIR) + mkdir(SYNC_FLAGS_DIR, SSH_USER, 'keystone', 0o775) + + # If we are the sync master but no longer leader then re-elect master. + if not is_elected_leader(CLUSTER_RES): + log("Re-electing ssl cert master.", level=INFO) + settings['ssl-cert-master'] = 'unknown' + + log("Sync complete - sending peer info", level=DEBUG) + for rid in relation_ids('cluster'): + relation_set(relation_id=rid, **settings) def get_ca(user='keystone', group='keystone'): @@ -600,22 +819,31 @@ def get_ca(user='keystone', group='keystone'): Initialize a new CA object if one hasn't already been loaded. This will create a new CA or load an existing one. """ - if not CA: + if not ssl.CA_SINGLETON: if not os.path.isdir(SSL_DIR): os.mkdir(SSL_DIR) + d_name = '_'.join(SSL_CA_NAME.lower().split(' ')) ca = ssl.JujuCA(name=SSL_CA_NAME, user=user, group=group, ca_dir=os.path.join(SSL_DIR, '%s_intermediate_ca' % d_name), root_ca_dir=os.path.join(SSL_DIR, '%s_root_ca' % d_name)) + # SSL_DIR is synchronized via all peers over unison+ssh, need # to ensure permissions. subprocess.check_output(['chown', '-R', '%s.%s' % (user, group), '%s' % SSL_DIR]) subprocess.check_output(['chmod', '-R', 'g+rwx', '%s' % SSL_DIR]) - CA.append(ca) - return CA[0] + + # Tag this host as the ssl cert master. + if is_clustered() or peer_units(): + peer_store(key='ssl-cert-master', + value=unit_get('private-address')) + + ssl.CA_SINGLETON.append(ca) + + return ssl.CA_SINGLETON[0] def relation_list(rid): @@ -657,7 +885,7 @@ def add_service_to_keystone(relation_id=None, remote_unit=None): relation_data["auth_port"] = config('admin-port') relation_data["service_port"] = config('service-port') relation_data["region"] = config('region') - if config('https-service-endpoints') in ['True', 'true']: + if str_is_true(config('https-service-endpoints')): # Pass CA cert as client will need it to # verify https connections ca = get_ca(user=SSH_USER) @@ -796,7 +1024,7 @@ def add_service_to_keystone(relation_id=None, remote_unit=None): relation_data["auth_protocol"] = "http" relation_data["service_protocol"] = "http" # generate or get a new cert/key for service if set to manage certs. - if config('https-service-endpoints') in ['True', 'true']: + if str_is_true(config('https-service-endpoints')): ca = get_ca(user=SSH_USER) # NOTE(jamespage) may have multiple cns to deal with to iterate https_cns = set(https_cns) diff --git a/unit_tests/test_keystone_hooks.py b/unit_tests/test_keystone_hooks.py index b8cdee0d..48a60e22 100644 --- a/unit_tests/test_keystone_hooks.py +++ b/unit_tests/test_keystone_hooks.py @@ -43,8 +43,7 @@ TO_PATCH = [ # charmhelpers.contrib.openstack.utils 'configure_installation_source', # charmhelpers.contrib.hahelpers.cluster_utils - 'is_leader', - 'eligible_leader', + 'is_elected_leader', 'get_hacluster_config', # keystone_utils 'restart_map', @@ -234,6 +233,7 @@ class KeystoneRelationTests(CharmTestCase): relation_id='identity-service:0', remote_unit='unit/0') + @patch.object(hooks, 'ensure_permissions') @patch.object(hooks, 'cluster_joined') @patch.object(unison, 'ensure_user') @patch.object(unison, 'get_homedir') @@ -242,9 +242,10 @@ class KeystoneRelationTests(CharmTestCase): @patch.object(hooks, 'configure_https') def test_config_changed_no_openstack_upgrade_leader( self, configure_https, identity_changed, - configs, get_homedir, ensure_user, cluster_joined): + configs, get_homedir, ensure_user, cluster_joined, + ensure_permissions): self.openstack_upgrade_available.return_value = False - self.eligible_leader.return_value = True + self.is_elected_leader.return_value = True self.relation_ids.return_value = ['identity-service:0'] self.relation_list.return_value = ['unit/0'] @@ -262,8 +263,10 @@ class KeystoneRelationTests(CharmTestCase): 'Firing identity_changed hook for all related services.') identity_changed.assert_called_with( relation_id='identity-service:0', - remote_unit='unit/0') + remote_unit='unit/0', + sync_certs=False) + @patch.object(hooks, 'ensure_permissions') @patch.object(hooks, 'cluster_joined') @patch.object(unison, 'ensure_user') @patch.object(unison, 'get_homedir') @@ -272,9 +275,10 @@ class KeystoneRelationTests(CharmTestCase): @patch.object(hooks, 'configure_https') def test_config_changed_no_openstack_upgrade_not_leader( self, configure_https, identity_changed, - configs, get_homedir, ensure_user, cluster_joined): + configs, get_homedir, ensure_user, cluster_joined, + ensure_permissions): self.openstack_upgrade_available.return_value = False - self.eligible_leader.return_value = False + self.is_elected_leader.return_value = False hooks.config_changed() ensure_user.assert_called_with(user=self.ssh_user, group='keystone') @@ -288,6 +292,7 @@ class KeystoneRelationTests(CharmTestCase): self.assertFalse(self.ensure_initial_admin.called) self.assertFalse(identity_changed.called) + @patch.object(hooks, 'ensure_permissions') @patch.object(hooks, 'cluster_joined') @patch.object(unison, 'ensure_user') @patch.object(unison, 'get_homedir') @@ -296,9 +301,10 @@ class KeystoneRelationTests(CharmTestCase): @patch.object(hooks, 'configure_https') def test_config_changed_with_openstack_upgrade( self, configure_https, identity_changed, - configs, get_homedir, ensure_user, cluster_joined): + configs, get_homedir, ensure_user, cluster_joined, + ensure_permissions): self.openstack_upgrade_available.return_value = True - self.eligible_leader.return_value = True + self.is_elected_leader.return_value = True self.relation_ids.return_value = ['identity-service:0'] self.relation_list.return_value = ['unit/0'] @@ -318,13 +324,14 @@ class KeystoneRelationTests(CharmTestCase): 'Firing identity_changed hook for all related services.') identity_changed.assert_called_with( relation_id='identity-service:0', - remote_unit='unit/0') + remote_unit='unit/0', + sync_certs=False) @patch.object(hooks, 'hashlib') @patch.object(hooks, 'send_notifications') def test_identity_changed_leader(self, mock_send_notifications, mock_hashlib): - self.eligible_leader.return_value = True + self.is_elected_leader.return_value = True hooks.identity_changed( relation_id='identity-service:0', remote_unit='unit/0') @@ -334,7 +341,7 @@ class KeystoneRelationTests(CharmTestCase): self.assertTrue(self.synchronize_ca.called) def test_identity_changed_no_leader(self): - self.eligible_leader.return_value = False + self.is_elected_leader.return_value = False hooks.identity_changed( relation_id='identity-service:0', remote_unit='unit/0') @@ -349,12 +356,14 @@ class KeystoneRelationTests(CharmTestCase): user=self.ssh_user, group='juju_keystone', peer_interface='cluster', ensure_local_user=True) + @patch.object(hooks, 'check_peer_actions') @patch.object(unison, 'ssh_authorized_peers') @patch.object(hooks, 'CONFIGS') - def test_cluster_changed(self, configs, ssh_authorized_peers): + def test_cluster_changed(self, configs, ssh_authorized_peers, + check_peer_actions): hooks.cluster_changed() - self.peer_echo.assert_called_with(includes=['_passwd', - 'identity-service:']) + whitelist = ['_passwd', 'identity-service:', 'ssl-cert-master'] + self.peer_echo.assert_called_with(includes=whitelist) ssh_authorized_peers.assert_called_with( user=self.ssh_user, group='keystone', peer_interface='cluster', ensure_local_user=True) @@ -411,7 +420,7 @@ class KeystoneRelationTests(CharmTestCase): @patch.object(hooks, 'CONFIGS') def test_ha_relation_changed_not_clustered_not_leader(self, configs): self.relation_get.return_value = False - self.is_leader.return_value = False + self.is_elected_leader.return_value = False hooks.ha_changed() self.assertTrue(configs.write_all.called) @@ -421,7 +430,7 @@ class KeystoneRelationTests(CharmTestCase): def test_ha_relation_changed_clustered_leader( self, configs, identity_changed): self.relation_get.return_value = True - self.is_leader.return_value = True + self.is_elected_leader.return_value = True self.relation_ids.return_value = ['identity-service:0'] self.related_units.return_value = ['unit/0'] @@ -432,7 +441,8 @@ class KeystoneRelationTests(CharmTestCase): 'keystone endpoint configuration') identity_changed.assert_called_with( relation_id='identity-service:0', - remote_unit='unit/0') + remote_unit='unit/0', + sync_certs=False) @patch.object(hooks, 'CONFIGS') def test_configure_https_enable(self, configs): @@ -458,7 +468,7 @@ class KeystoneRelationTests(CharmTestCase): @patch.object(unison, 'ssh_authorized_peers') def test_upgrade_charm_leader(self, ssh_authorized_peers): - self.eligible_leader.return_value = True + self.is_elected_leader.return_value = True self.filter_installed_packages.return_value = [] hooks.upgrade_charm() self.assertTrue(self.apt_install.called) @@ -473,7 +483,7 @@ class KeystoneRelationTests(CharmTestCase): @patch.object(unison, 'ssh_authorized_peers') def test_upgrade_charm_not_leader(self, ssh_authorized_peers): - self.eligible_leader.return_value = False + self.is_elected_leader.return_value = False self.filter_installed_packages.return_value = [] hooks.upgrade_charm() self.assertTrue(self.apt_install.called) diff --git a/unit_tests/test_keystone_utils.py b/unit_tests/test_keystone_utils.py index f504b21e..53a874cd 100644 --- a/unit_tests/test_keystone_utils.py +++ b/unit_tests/test_keystone_utils.py @@ -26,7 +26,7 @@ TO_PATCH = [ 'get_os_codename_install_source', 'grant_role', 'configure_installation_source', - 'eligible_leader', + 'is_elected_leader', 'https', 'is_clustered', 'peer_store_and_set', @@ -113,7 +113,7 @@ class TestKeystoneUtils(CharmTestCase): self, migrate_database, determine_packages, configs): self.test_config.set('openstack-origin', 'precise') determine_packages.return_value = [] - self.eligible_leader.return_value = True + self.is_elected_leader.return_value = True utils.do_openstack_upgrade(configs) From ee78c8bb554f3eda43b6f14dd44d1c78a9a0bd6c Mon Sep 17 00:00:00 2001 From: Edward Hope-Morley Date: Mon, 5 Jan 2015 21:57:25 +0000 Subject: [PATCH 02/48] defer some action to ha rel hook if waiting on 'clustered' --- hooks/keystone_hooks.py | 23 +++++++++++++++++++++++ 1 file changed, 23 insertions(+) diff --git a/hooks/keystone_hooks.py b/hooks/keystone_hooks.py index 4ab442a9..8570c253 100755 --- a/hooks/keystone_hooks.py +++ b/hooks/keystone_hooks.py @@ -17,6 +17,7 @@ from charmhelpers.core.hookenv import ( is_relation_made, log, local_unit, + INFO, WARNING, ERROR, relation_get, @@ -284,6 +285,19 @@ def cluster_joined(relation_id=None): relation_settings={'private-address': private_addr}) +def is_pending_clustered(): + """If we have HA relations but are not yet 'clustered' return True.""" + pending = False + for r_id in (relation_ids('ha') or []): + for unit in (relation_list(r_id) or []): + if relation_get('clustered', rid=r_id, unit=unit): + pending = False + else: + pending = True + + return pending + + @hooks.hook('cluster-relation-changed', 'cluster-relation-departed') @restart_on_change(restart_map(), stopstart=True) @@ -298,6 +312,14 @@ def cluster_changed(): peer_interface='cluster', ensure_local_user=True) CONFIGS.write_all() + + # If we have a pending cluster formation, defer following actions to the ha + # relation hook instead. + if is_pending_clustered(): + log("Waiting for ha to be 'clustered' - deferring identity-updates " + "and cert sync to ha relation", level=INFO) + return + for r_id in relation_ids('identity-service'): for unit in relation_list(r_id): identity_changed(relation_id=r_id, remote_unit=unit, @@ -364,6 +386,7 @@ def ha_changed(): ensure_initial_admin(config) log('Cluster configured, notifying other services and updating ' 'keystone endpoint configuration') + for rid in relation_ids('identity-service'): for unit in related_units(rid): identity_changed(relation_id=rid, remote_unit=unit, From 61b07fc623eedb286b18c5eaaadffa9ea0024e55 Mon Sep 17 00:00:00 2001 From: Edward Hope-Morley Date: Thu, 8 Jan 2015 14:56:31 +0000 Subject: [PATCH 03/48] more cluster relation noise reduction --- hooks/keystone_hooks.py | 16 ++-------------- hooks/keystone_utils.py | 10 ++++++++++ unit_tests/test_keystone_hooks.py | 4 +++- 3 files changed, 15 insertions(+), 15 deletions(-) diff --git a/hooks/keystone_hooks.py b/hooks/keystone_hooks.py index 8570c253..652679fe 100755 --- a/hooks/keystone_hooks.py +++ b/hooks/keystone_hooks.py @@ -63,6 +63,7 @@ from keystone_utils import ( check_peer_actions, CA_CERT_PATH, ensure_permissions, + is_pending_clustered, ) from charmhelpers.contrib.hahelpers.cluster import ( @@ -285,19 +286,6 @@ def cluster_joined(relation_id=None): relation_settings={'private-address': private_addr}) -def is_pending_clustered(): - """If we have HA relations but are not yet 'clustered' return True.""" - pending = False - for r_id in (relation_ids('ha') or []): - for unit in (relation_list(r_id) or []): - if relation_get('clustered', rid=r_id, unit=unit): - pending = False - else: - pending = True - - return pending - - @hooks.hook('cluster-relation-changed', 'cluster-relation-departed') @restart_on_change(restart_map(), stopstart=True) @@ -306,7 +294,6 @@ def cluster_changed(): # NOTE(jamespage) re-echo passwords for peer storage echo_whitelist = ['_passwd', 'identity-service:', 'ssl-cert-master'] - peer_echo(includes=echo_whitelist) unison.ssh_authorized_peers(user=SSH_USER, group='keystone', peer_interface='cluster', @@ -326,6 +313,7 @@ def cluster_changed(): sync_certs=False) synchronize_ca() + peer_echo(includes=echo_whitelist) @hooks.hook('ha-relation-joined') diff --git a/hooks/keystone_utils.py b/hooks/keystone_utils.py index ff14f941..6ee620c9 100644 --- a/hooks/keystone_utils.py +++ b/hooks/keystone_utils.py @@ -1149,3 +1149,13 @@ def send_notifications(data, force=False): level=DEBUG) for rid in rel_ids: relation_set(relation_id=rid, relation_settings=_notifications) + + +def is_pending_clustered(): + """If we have HA relations but are not yet 'clustered' return True.""" + for r_id in (relation_ids('ha') or []): + for unit in (relation_list(r_id) or []): + if not relation_get('clustered', rid=r_id, unit=unit): + return True + + return False diff --git a/unit_tests/test_keystone_hooks.py b/unit_tests/test_keystone_hooks.py index 48a60e22..3829f648 100644 --- a/unit_tests/test_keystone_hooks.py +++ b/unit_tests/test_keystone_hooks.py @@ -356,11 +356,13 @@ class KeystoneRelationTests(CharmTestCase): user=self.ssh_user, group='juju_keystone', peer_interface='cluster', ensure_local_user=True) + @patch.object(hooks, 'is_pending_clustered') @patch.object(hooks, 'check_peer_actions') @patch.object(unison, 'ssh_authorized_peers') @patch.object(hooks, 'CONFIGS') def test_cluster_changed(self, configs, ssh_authorized_peers, - check_peer_actions): + check_peer_actions, is_pending_clustered): + is_pending_clustered.return_value = False hooks.cluster_changed() whitelist = ['_passwd', 'identity-service:', 'ssl-cert-master'] self.peer_echo.assert_called_with(includes=whitelist) From a760082802b75d0577e23c490e9c584175a65301 Mon Sep 17 00:00:00 2001 From: Edward Hope-Morley Date: Sat, 10 Jan 2015 14:56:22 +0000 Subject: [PATCH 04/48] Fixed a few race issues and switched to using decorators --- hooks/keystone_hooks.py | 134 ++++++++++++---------- hooks/keystone_utils.py | 180 +++++++++++++++++++++--------- unit_tests/test_keystone_hooks.py | 172 +++++++++++++++++++++++----- 3 files changed, 346 insertions(+), 140 deletions(-) diff --git a/hooks/keystone_hooks.py b/hooks/keystone_hooks.py index 652679fe..21a476ac 100755 --- a/hooks/keystone_hooks.py +++ b/hooks/keystone_hooks.py @@ -17,6 +17,7 @@ from charmhelpers.core.hookenv import ( is_relation_made, log, local_unit, + DEBUG, INFO, WARNING, ERROR, @@ -24,6 +25,7 @@ from charmhelpers.core.hookenv import ( relation_ids, relation_set, related_units, + remote_unit, unit_get, ) @@ -50,9 +52,8 @@ from keystone_utils import ( ensure_initial_admin, migrate_database, save_script_rc, - synchronize_ca, + synchronize_ca_if_changed, register_configs, - relation_list, restart_map, CLUSTER_RES, KEYSTONE_CONF, @@ -63,12 +64,13 @@ from keystone_utils import ( check_peer_actions, CA_CERT_PATH, ensure_permissions, - is_pending_clustered, + is_ssl_cert_master, ) from charmhelpers.contrib.hahelpers.cluster import ( is_elected_leader, get_hacluster_config, + peer_units, ) from charmhelpers.payload.execd import execd_preinstall @@ -99,12 +101,14 @@ def install(): @hooks.hook('config-changed') @restart_on_change(restart_map()) +@synchronize_ca_if_changed(fatal=False) def config_changed(): if config('prefer-ipv6'): setup_ipv6() sync_db_with_multi_ipv6_addresses(config('database'), config('database-user')) + unison.ensure_user(user=SSH_USER, group='juju_keystone') unison.ensure_user(user=SSH_USER, group='keystone') homedir = unison.get_homedir(SSH_USER) if not os.path.isdir(homedir): @@ -127,20 +131,10 @@ def config_changed(): configure_https() CONFIGS.write_all() - if is_elected_leader(CLUSTER_RES): - migrate_database() - ensure_initial_admin(config) - log('Firing identity_changed hook for all related services.') - # HTTPS may have been set - so fire all identity relations - # again - for r_id in relation_ids('identity-service'): - for unit in relation_list(r_id): - identity_changed(relation_id=r_id, - remote_unit=unit, sync_certs=False) - - synchronize_ca() - - [cluster_joined(rid) for rid in relation_ids('cluster')] + # Update relations since SSL may have been configured. If we have peer + # units we can rely on the sync to do this in cluster relation. + if is_elected_leader(CLUSTER_RES) and not peer_units(): + update_all_identity_relation_units() @hooks.hook('shared-db-relation-joined') @@ -173,8 +167,23 @@ def pgsql_db_joined(): relation_set(database=config('database')) +def update_all_identity_relation_units(): + try: + migrate_database() + except Exception as exc: + log("Database initialisation failed (%s) - db not ready?" % (exc), + level=WARNING) + else: + ensure_initial_admin(config) + log('Firing identity_changed hook for all related services.') + for rid in relation_ids('identity-service'): + for unit in related_units(rid): + identity_changed(relation_id=rid, remote_unit=unit) + + @hooks.hook('shared-db-relation-changed') @restart_on_change(restart_map()) +@synchronize_ca_if_changed() def db_changed(): if 'shared-db' not in CONFIGS.complete_contexts(): log('shared-db relation incomplete. Peer not ready?') @@ -189,34 +198,28 @@ def db_changed(): if allowed_units and local_unit() not in allowed_units.split(): log('Allowed_units list provided and this unit not present') return - migrate_database() - ensure_initial_admin(config) # Ensure any existing service entries are updated in the # new database backend - for rid in relation_ids('identity-service'): - for unit in related_units(rid): - identity_changed(relation_id=rid, remote_unit=unit) + update_all_identity_relation_units() @hooks.hook('pgsql-db-relation-changed') @restart_on_change(restart_map()) +@synchronize_ca_if_changed() def pgsql_db_changed(): if 'pgsql-db' not in CONFIGS.complete_contexts(): log('pgsql-db relation incomplete. Peer not ready?') else: CONFIGS.write(KEYSTONE_CONF) if is_elected_leader(CLUSTER_RES): - migrate_database() - ensure_initial_admin(config) # Ensure any existing service entries are updated in the # new database backend - for rid in relation_ids('identity-service'): - for unit in related_units(rid): - identity_changed(relation_id=rid, remote_unit=unit) + update_all_identity_relation_units() @hooks.hook('identity-service-relation-changed') -def identity_changed(relation_id=None, remote_unit=None, sync_certs=True): +@synchronize_ca_if_changed() +def identity_changed(relation_id=None, remote_unit=None): notifications = {} if is_elected_leader(CLUSTER_RES): # Catch database not configured error and defer until db ready @@ -235,8 +238,6 @@ def identity_changed(relation_id=None, remote_unit=None, sync_certs=True): raise CONFIGS.write_all() - if sync_certs: - synchronize_ca() settings = relation_get(rid=relation_id, unit=remote_unit) service = settings.get('service', None) @@ -286,33 +287,55 @@ def cluster_joined(relation_id=None): relation_settings={'private-address': private_addr}) +@synchronize_ca_if_changed() +def identity_updates_with_ssl_sync(): + CONFIGS.write_all() + update_all_identity_relation_units() + + +@synchronize_ca_if_changed(force=True) +def identity_updates_with_forced_ssl_sync(): + identity_updates_with_ssl_sync() + + @hooks.hook('cluster-relation-changed', 'cluster-relation-departed') @restart_on_change(restart_map(), stopstart=True) def cluster_changed(): check_peer_actions() + # Uncomment the following to print out all cluster relation settings in + # log (debug only). + """ + rels = ["%s:%s" % (k, v) for k, v in relation_get().iteritems()] + tag = '\n[debug:%s]' % (remote_unit()) + log("PEER RELATION SETTINGS (unit=%s): %s" % (remote_unit(), + tag.join(rels)), + level=DEBUG) + """ + # NOTE(jamespage) re-echo passwords for peer storage - echo_whitelist = ['_passwd', 'identity-service:', 'ssl-cert-master'] + echo_whitelist = ['_passwd', 'identity-service:'] unison.ssh_authorized_peers(user=SSH_USER, group='keystone', peer_interface='cluster', ensure_local_user=True) - CONFIGS.write_all() - # If we have a pending cluster formation, defer following actions to the ha - # relation hook instead. - if is_pending_clustered(): - log("Waiting for ha to be 'clustered' - deferring identity-updates " - "and cert sync to ha relation", level=INFO) - return + synced_units = relation_get(attribute='ssl-synced-units', + unit=local_unit()) + if not synced_units or (remote_unit() not in synced_units): + log("Peer '%s' not in list of synced units (%s)" % + (remote_unit(), synced_units), level=INFO) + identity_updates_with_forced_ssl_sync() + else: + identity_updates_with_ssl_sync() - for r_id in relation_ids('identity-service'): - for unit in relation_list(r_id): - identity_changed(relation_id=r_id, remote_unit=unit, - sync_certs=False) + # If we are cert master ignore what other peers have to say + if not is_ssl_cert_master(): + echo_whitelist.append('ssl-cert-master') - synchronize_ca() + # ssl cert sync must be done BEFORE this to reduce the risk of feedback + # loops in cluster relation peer_echo(includes=echo_whitelist) @@ -367,20 +390,16 @@ def ha_joined(): @hooks.hook('ha-relation-changed') @restart_on_change(restart_map()) +@synchronize_ca_if_changed() def ha_changed(): clustered = relation_get('clustered') CONFIGS.write_all() - if clustered is not None and is_elected_leader(CLUSTER_RES): + if clustered and is_elected_leader(CLUSTER_RES): ensure_initial_admin(config) log('Cluster configured, notifying other services and updating ' 'keystone endpoint configuration') - for rid in relation_ids('identity-service'): - for unit in related_units(rid): - identity_changed(relation_id=rid, remote_unit=unit, - sync_certs=False) - - synchronize_ca() + update_all_identity_relation_units() @hooks.hook('identity-admin-relation-changed') @@ -399,6 +418,7 @@ def admin_relation_changed(): relation_set(**relation_data) +@synchronize_ca_if_changed() def configure_https(): ''' Enables SSL API Apache config if appropriate and kicks identity-service @@ -417,6 +437,7 @@ def configure_https(): @hooks.hook('upgrade-charm') @restart_on_change(restart_map(), stopstart=True) +@synchronize_ca_if_changed() def upgrade_charm(): apt_install(filter_installed_packages(determine_packages())) unison.ssh_authorized_peers(user=SSH_USER, @@ -424,17 +445,12 @@ def upgrade_charm(): peer_interface='cluster', ensure_local_user=True) if is_elected_leader(CLUSTER_RES): - log('Cluster leader - ensuring endpoint configuration' - ' is up to date') + log('Cluster leader - ensuring endpoint configuration is up to ' + 'date', level=DEBUG) time.sleep(10) - ensure_initial_admin(config) - # Deal with interface changes for icehouse - for r_id in relation_ids('identity-service'): - for unit in relation_list(r_id): - identity_changed(relation_id=r_id, - remote_unit=unit, sync_certs=False) + update_all_identity_relation_units() + CONFIGS.write_all() - synchronize_ca() def main(): diff --git a/hooks/keystone_utils.py b/hooks/keystone_utils.py index 6ee620c9..8e9fd0a8 100644 --- a/hooks/keystone_utils.py +++ b/hooks/keystone_utils.py @@ -1,13 +1,15 @@ #!/usr/bin/python import glob import grp -import subprocess +import hashlib import os import pwd -import uuid -import urlparse import shutil +import subprocess +import threading import time +import urlparse +import uuid from base64 import b64encode from collections import OrderedDict @@ -56,10 +58,10 @@ from charmhelpers.core.hookenv import ( config, log, local_unit, + related_units, relation_get, relation_set, relation_ids, - unit_get, DEBUG, INFO, WARNING, @@ -130,6 +132,7 @@ SSL_DIR = '/var/lib/keystone/juju_ssl/' SSL_CA_NAME = 'Ubuntu Cloud' CLUSTER_RES = 'grp_ks_vips' SSH_USER = 'juju_keystone' +SSL_SYNC_SEMAPHORE = threading.Semaphore() BASE_RESOURCE_MAP = OrderedDict([ (KEYSTONE_CONF, { @@ -701,7 +704,7 @@ def is_ssl_cert_master(): master = relation_get(attribute='ssl-cert-master', rid=rid, unit=local_unit()) - if master and master == unit_get('private-address'): + if master and master == local_unit(): return True return False @@ -720,6 +723,39 @@ def unison_sync(paths_to_sync): fatal=True) +def is_sync_master(): + if not peer_units(): + log("Not syncing certs since there are no peer units.", level=INFO) + return False + + # If no ssl master elected and we are cluster leader, elect this unit. + if is_elected_leader(CLUSTER_RES): + master = [] + for rid in relation_ids('cluster'): + for unit in related_units(rid): + m = relation_get(rid=rid, unit=unit, + attribute='ssl-cert-master') + if m is not None: + master.append(m) + + master = set(master) + if not master or ('unknown' in master and len(master) == 1): + log("Electing this unit as ssl-cert-master", level=DEBUG) + for rid in relation_ids('cluster'): + settings = {'ssl-cert-master': local_unit(), + 'ssl-synced-units': None} + relation_set(relation_id=rid, relation_settings=settings) + + # Return now and wait for cluster-relation-changed for sync. + return False + + if not is_ssl_cert_master(): + log("Not ssl cert master - skipping sync", level=INFO) + return False + + return True + + def synchronize_ca(fatal=True): """Broadcast service credentials to peers. @@ -733,27 +769,6 @@ def synchronize_ca(fatal=True): """ paths_to_sync = [SYNC_FLAGS_DIR] - if not peer_units(): - log("Not syncing certs since there are no peer units.", level=INFO) - return - - # If no ssl master elected and we are cluster leader, elect this unit. - if is_elected_leader(CLUSTER_RES): - master = relation_get(attribute='ssl-cert-master') - if not master or master == 'unknown': - log("Electing this unit as ssl-cert-master", level=DEBUG) - for rid in relation_ids('cluster'): - relation_set(relation_id=rid, - relation_settings={'ssl-cert-master': - unit_get('private-address'), - 'trigger': str(uuid.uuid4())}) - # Return now and wait for echo before continuing. - return - - if not is_ssl_cert_master(): - log("Not ssl cert master - skipping sync", level=INFO) - return - if str_is_true(config('https-service-endpoints')): log("Syncing all endpoint certs since https-service-endpoints=True", level=DEBUG) @@ -785,7 +800,6 @@ def synchronize_ca(fatal=True): # new ssl keys. create_peer_service_actions('restart', ['apache2']) create_service_action('update-ca-certificates') - try: unison_sync(paths_to_sync) except: @@ -795,23 +809,94 @@ def synchronize_ca(fatal=True): log("Sync failed but fatal=False", level=INFO) return - trigger = str(uuid.uuid4()) - log("Sending restart-services-trigger=%s to all peers" % (trigger), - level=DEBUG) - settings = {'restart-services-trigger': trigger} - # Cleanup shutil.rmtree(SYNC_FLAGS_DIR) mkdir(SYNC_FLAGS_DIR, SSH_USER, 'keystone', 0o775) - # If we are the sync master but no longer leader then re-elect master. - if not is_elected_leader(CLUSTER_RES): - log("Re-electing ssl cert master.", level=INFO) - settings['ssl-cert-master'] = 'unknown' + trigger = str(uuid.uuid4()) + log("Sending restart-services-trigger=%s to all peers" % (trigger), + level=DEBUG) - log("Sync complete - sending peer info", level=DEBUG) - for rid in relation_ids('cluster'): - relation_set(relation_id=rid, **settings) + log("Sync complete", level=DEBUG) + return {'restart-services-trigger': trigger, + 'ssl-synced-units': peer_units()} + + +def update_hash_from_path(hash, path, recurse_depth=10): + """Recurse through path and update the provided hash for every file found. + """ + if not recurse_depth: + log("Max recursion depth (%s) reached for update_hash_from_path() at " + "path='%s' - not going any deeper" % (recurse_depth, path), + level=WARNING) + return sum + + for p in glob.glob("%s/*" % path): + if os.path.isdir(p): + update_hash_from_path(hash, p, recurse_depth=recurse_depth - 1) + else: + with open(p, 'r') as fd: + hash.update(fd.read()) + + +def synchronize_ca_if_changed(force=False, fatal=True): + """Decorator to perform ssl cert sync if decorated function modifies them + in any way. + + If force is True a sync is done regardless. + """ + def inner_synchronize_ca_if_changed1(f): + def inner_synchronize_ca_if_changed2(*args, **kwargs): + if not is_sync_master(): + return f(*args, **kwargs) + + peer_settings = {} + try: + # Ensure we don't do a double sync if we are nested. + if not force and SSL_SYNC_SEMAPHORE.acquire(blocking=0): + hash1 = hashlib.sha256() + for path in [SSL_DIR, APACHE_SSL_DIR, CA_CERT_PATH]: + update_hash_from_path(hash1, path) + + hash1 = hash1.hexdigest() + + ret = f(*args, **kwargs) + + hash2 = hashlib.sha256() + for path in [SSL_DIR, APACHE_SSL_DIR, CA_CERT_PATH]: + update_hash_from_path(hash2, path) + + hash2 = hash2.hexdigest() + if hash1 != hash2: + log("SSL certs have changed - syncing peers", + level=DEBUG) + peer_settings = synchronize_ca(fatal=fatal) + else: + log("SSL certs have not changed - skipping sync", + level=DEBUG) + else: + ret = f(*args, **kwargs) + if force: + log("Doing forced ssl cert sync", level=DEBUG) + peer_settings = synchronize_ca(fatal=fatal) + + # If we are the sync master but no longer leader then re-elect + # master. + if not is_elected_leader(CLUSTER_RES): + log("Re-electing ssl cert master.", level=INFO) + peer_settings['ssl-cert-master'] = 'unknown' + + for rid in relation_ids('cluster'): + relation_set(relation_id=rid, + relation_settings=peer_settings) + + return ret + finally: + SSL_SYNC_SEMAPHORE.release() + + return inner_synchronize_ca_if_changed2 + + return inner_synchronize_ca_if_changed1 def get_ca(user='keystone', group='keystone'): @@ -838,8 +923,11 @@ def get_ca(user='keystone', group='keystone'): # Tag this host as the ssl cert master. if is_clustered() or peer_units(): - peer_store(key='ssl-cert-master', - value=unit_get('private-address')) + for rid in relation_ids('cluster'): + relation_set(relation_id=rid, + relation_settings={'ssl-cert-master': + local_unit(), + 'synced-units': None}) ssl.CA_SINGLETON.append(ca) @@ -1149,13 +1237,3 @@ def send_notifications(data, force=False): level=DEBUG) for rid in rel_ids: relation_set(relation_id=rid, relation_settings=_notifications) - - -def is_pending_clustered(): - """If we have HA relations but are not yet 'clustered' return True.""" - for r_id in (relation_ids('ha') or []): - for unit in (relation_list(r_id) or []): - if not relation_get('clustered', rid=r_id, unit=unit): - return True - - return False diff --git a/unit_tests/test_keystone_hooks.py b/unit_tests/test_keystone_hooks.py index 3829f648..dd080f1d 100644 --- a/unit_tests/test_keystone_hooks.py +++ b/unit_tests/test_keystone_hooks.py @@ -1,6 +1,7 @@ from mock import call, patch, MagicMock import os import json +import uuid from test_utils import CharmTestCase @@ -34,6 +35,7 @@ TO_PATCH = [ 'relation_set', 'relation_get', 'related_units', + 'remote_unit', 'unit_get', 'peer_echo', # charmhelpers.core.host @@ -54,7 +56,7 @@ TO_PATCH = [ 'migrate_database', 'ensure_initial_admin', 'add_service_to_keystone', - 'synchronize_ca', + 'synchronize_ca_if_changed', # other 'check_call', 'execd_preinstall', @@ -158,8 +160,12 @@ class KeystoneRelationTests(CharmTestCase): 'Attempting to associate a postgresql database when there ' 'is already associated a mysql one') + @patch('keystone_utils.log') + @patch('keystone_utils.peer_units') @patch.object(hooks, 'CONFIGS') - def test_db_changed_missing_relation_data(self, configs): + def test_db_changed_missing_relation_data(self, configs, mock_peer_units, + mock_log): + mock_peer_units.return_value = None configs.complete_contexts = MagicMock() configs.complete_contexts.return_value = [] hooks.db_changed() @@ -190,9 +196,13 @@ class KeystoneRelationTests(CharmTestCase): configs.write = MagicMock() hooks.pgsql_db_changed() + @patch('keystone_utils.log') + @patch('keystone_utils.peer_units') @patch.object(hooks, 'CONFIGS') @patch.object(hooks, 'identity_changed') - def test_db_changed_allowed(self, identity_changed, configs): + def test_db_changed_allowed(self, identity_changed, configs, + mock_peer_units, mock_log): + mock_peer_units.return_value = None self.relation_ids.return_value = ['identity-service:0'] self.related_units.return_value = ['unit/0'] @@ -205,9 +215,13 @@ class KeystoneRelationTests(CharmTestCase): relation_id='identity-service:0', remote_unit='unit/0') + @patch('keystone_utils.log') + @patch('keystone_utils.peer_units') @patch.object(hooks, 'CONFIGS') @patch.object(hooks, 'identity_changed') - def test_db_changed_not_allowed(self, identity_changed, configs): + def test_db_changed_not_allowed(self, identity_changed, configs, + mock_peer_units, mock_log): + mock_peer_units.return_value = None self.relation_ids.return_value = ['identity-service:0'] self.related_units.return_value = ['unit/0'] @@ -218,9 +232,13 @@ class KeystoneRelationTests(CharmTestCase): self.assertFalse(self.ensure_initial_admin.called) self.assertFalse(identity_changed.called) + @patch('keystone_utils.log') + @patch('keystone_utils.peer_units') @patch.object(hooks, 'CONFIGS') @patch.object(hooks, 'identity_changed') - def test_postgresql_db_changed(self, identity_changed, configs): + def test_postgresql_db_changed(self, identity_changed, configs, + mock_peer_units, mock_log): + mock_peer_units.return_value = None self.relation_ids.return_value = ['identity-service:0'] self.related_units.return_value = ['unit/0'] @@ -233,6 +251,7 @@ class KeystoneRelationTests(CharmTestCase): relation_id='identity-service:0', remote_unit='unit/0') + @patch('keystone_utils.is_sync_master') @patch.object(hooks, 'ensure_permissions') @patch.object(hooks, 'cluster_joined') @patch.object(unison, 'ensure_user') @@ -243,7 +262,8 @@ class KeystoneRelationTests(CharmTestCase): def test_config_changed_no_openstack_upgrade_leader( self, configure_https, identity_changed, configs, get_homedir, ensure_user, cluster_joined, - ensure_permissions): + ensure_permissions, is_sync_master): + is_sync_master.return_value = False self.openstack_upgrade_available.return_value = False self.is_elected_leader.return_value = True self.relation_ids.return_value = ['identity-service:0'] @@ -263,9 +283,9 @@ class KeystoneRelationTests(CharmTestCase): 'Firing identity_changed hook for all related services.') identity_changed.assert_called_with( relation_id='identity-service:0', - remote_unit='unit/0', - sync_certs=False) + remote_unit='unit/0') + @patch('keystone_utils.is_sync_master') @patch.object(hooks, 'ensure_permissions') @patch.object(hooks, 'cluster_joined') @patch.object(unison, 'ensure_user') @@ -276,7 +296,8 @@ class KeystoneRelationTests(CharmTestCase): def test_config_changed_no_openstack_upgrade_not_leader( self, configure_https, identity_changed, configs, get_homedir, ensure_user, cluster_joined, - ensure_permissions): + ensure_permissions, is_sync_master): + is_sync_master.return_value = False self.openstack_upgrade_available.return_value = False self.is_elected_leader.return_value = False @@ -292,6 +313,7 @@ class KeystoneRelationTests(CharmTestCase): self.assertFalse(self.ensure_initial_admin.called) self.assertFalse(identity_changed.called) + @patch('keystone_utils.is_sync_master') @patch.object(hooks, 'ensure_permissions') @patch.object(hooks, 'cluster_joined') @patch.object(unison, 'ensure_user') @@ -302,7 +324,8 @@ class KeystoneRelationTests(CharmTestCase): def test_config_changed_with_openstack_upgrade( self, configure_https, identity_changed, configs, get_homedir, ensure_user, cluster_joined, - ensure_permissions): + ensure_permissions, is_sync_master): + is_sync_master.return_value = False self.openstack_upgrade_available.return_value = True self.is_elected_leader.return_value = True self.relation_ids.return_value = ['identity-service:0'] @@ -324,13 +347,32 @@ class KeystoneRelationTests(CharmTestCase): 'Firing identity_changed hook for all related services.') identity_changed.assert_called_with( relation_id='identity-service:0', - remote_unit='unit/0', - sync_certs=False) + remote_unit='unit/0') + @patch('keystone_utils.log') + @patch('keystone_utils.peer_units') + @patch('keystone_utils.relation_ids') + @patch('keystone_utils.is_elected_leader') + @patch('keystone_utils.is_sync_master') + @patch('keystone_utils.update_hash_from_path') + @patch('keystone_utils.synchronize_ca') @patch.object(hooks, 'hashlib') @patch.object(hooks, 'send_notifications') def test_identity_changed_leader(self, mock_send_notifications, - mock_hashlib): + mock_hashlib, mock_synchronize_ca, + mock_update_hash_from_path, + mock_is_sync_master, + mock_is_elected_leader, + mock_relation_ids, mock_peer_units, + mock_log): + mock_peer_units.return_value = None + mock_relation_ids.return_value = [] + mock_is_sync_master.return_value = True + mock_is_elected_leader.return_value = True + # Ensure always returns diff + mock_update_hash_from_path.side_effect = \ + lambda hash, *args, **kwargs: hash.update(str(uuid.uuid4())) + self.is_elected_leader.return_value = True hooks.identity_changed( relation_id='identity-service:0', @@ -338,9 +380,12 @@ class KeystoneRelationTests(CharmTestCase): self.add_service_to_keystone.assert_called_with( 'identity-service:0', 'unit/0') - self.assertTrue(self.synchronize_ca.called) + self.assertTrue(mock_synchronize_ca.called) - def test_identity_changed_no_leader(self): + @patch('keystone_utils.log') + @patch('keystone_utils.peer_units') + def test_identity_changed_no_leader(self, mock_peer_units, mock_log): + mock_peer_units.return_value = None self.is_elected_leader.return_value = False hooks.identity_changed( relation_id='identity-service:0', @@ -356,20 +401,34 @@ class KeystoneRelationTests(CharmTestCase): user=self.ssh_user, group='juju_keystone', peer_interface='cluster', ensure_local_user=True) - @patch.object(hooks, 'is_pending_clustered') + @patch('keystone_utils.log') + @patch('keystone_utils.relation_ids') + @patch('keystone_utils.is_elected_leader') + @patch('keystone_utils.is_sync_master') + @patch('keystone_utils.update_hash_from_path') + @patch('keystone_utils.synchronize_ca') @patch.object(hooks, 'check_peer_actions') @patch.object(unison, 'ssh_authorized_peers') @patch.object(hooks, 'CONFIGS') def test_cluster_changed(self, configs, ssh_authorized_peers, - check_peer_actions, is_pending_clustered): - is_pending_clustered.return_value = False + check_peer_actions, + mock_synchronize_ca, mock_update_hash_from_path, + mock_is_sync_master, mock_is_elected_leader, + mock_relation_ids, mock_log): + mock_relation_ids.return_value = [] + mock_is_sync_master.return_value = True + mock_is_elected_leader.return_value = True + # Ensure always returns diff + mock_update_hash_from_path.side_effect = \ + lambda hash, *args, **kwargs: hash.update(str(uuid.uuid4())) + hooks.cluster_changed() whitelist = ['_passwd', 'identity-service:', 'ssl-cert-master'] self.peer_echo.assert_called_with(includes=whitelist) ssh_authorized_peers.assert_called_with( user=self.ssh_user, group='keystone', peer_interface='cluster', ensure_local_user=True) - self.assertTrue(self.synchronize_ca.called) + self.assertTrue(mock_synchronize_ca.called) self.assertTrue(configs.write_all.called) def test_ha_joined(self): @@ -419,18 +478,32 @@ class KeystoneRelationTests(CharmTestCase): } self.relation_set.assert_called_with(**args) + @patch('keystone_utils.log') + @patch('keystone_utils.peer_units') + @patch('keystone_utils.synchronize_ca') @patch.object(hooks, 'CONFIGS') - def test_ha_relation_changed_not_clustered_not_leader(self, configs): + def test_ha_relation_changed_not_clustered_not_leader(self, configs, + mock_synchronize_ca, + mock_peer_units, + mock_log): + mock_peer_units.return_value = None self.relation_get.return_value = False self.is_elected_leader.return_value = False hooks.ha_changed() self.assertTrue(configs.write_all.called) + self.assertTrue(mock_synchronize_ca.called) + @patch('keystone_utils.log') + @patch('keystone_utils.peer_units') + @patch('keystone_utils.synchronize_ca') @patch.object(hooks, 'identity_changed') @patch.object(hooks, 'CONFIGS') - def test_ha_relation_changed_clustered_leader( - self, configs, identity_changed): + def test_ha_relation_changed_clustered_leader(self, configs, + identity_changed, + mock_synchronize_ca, + mock_peer_units, mock_log): + mock_peer_units.return_value = None self.relation_get.return_value = True self.is_elected_leader.return_value = True self.relation_ids.return_value = ['identity-service:0'] @@ -443,11 +516,14 @@ class KeystoneRelationTests(CharmTestCase): 'keystone endpoint configuration') identity_changed.assert_called_with( relation_id='identity-service:0', - remote_unit='unit/0', - sync_certs=False) + remote_unit='unit/0') + self.assertTrue(mock_synchronize_ca.called) + @patch('keystone_utils.log') + @patch('keystone_utils.peer_units') @patch.object(hooks, 'CONFIGS') - def test_configure_https_enable(self, configs): + def test_configure_https_enable(self, configs, mock_peer_units, mock_log): + mock_peer_units.return_value = None configs.complete_contexts = MagicMock() configs.complete_contexts.return_value = ['https'] configs.write = MagicMock() @@ -457,8 +533,11 @@ class KeystoneRelationTests(CharmTestCase): cmd = ['a2ensite', 'openstack_https_frontend'] self.check_call.assert_called_with(cmd) + @patch('keystone_utils.log') + @patch('keystone_utils.peer_units') @patch.object(hooks, 'CONFIGS') - def test_configure_https_disable(self, configs): + def test_configure_https_disable(self, configs, mock_peer_units, mock_log): + mock_peer_units.return_value = None configs.complete_contexts = MagicMock() configs.complete_contexts.return_value = [''] configs.write = MagicMock() @@ -468,8 +547,24 @@ class KeystoneRelationTests(CharmTestCase): cmd = ['a2dissite', 'openstack_https_frontend'] self.check_call.assert_called_with(cmd) + @patch('keystone_utils.relation_ids') + @patch('keystone_utils.is_elected_leader') + @patch('keystone_utils.is_sync_master') + @patch('keystone_utils.update_hash_from_path') + @patch('keystone_utils.synchronize_ca') @patch.object(unison, 'ssh_authorized_peers') - def test_upgrade_charm_leader(self, ssh_authorized_peers): + def test_upgrade_charm_leader(self, ssh_authorized_peers, + mock_synchronize_ca, + mock_update_hash_from_path, + mock_is_sync_master, mock_is_elected_leader, + mock_relation_ids): + mock_relation_ids.return_value = [] + mock_is_sync_master.return_value = True + mock_is_elected_leader.return_value = True + # Ensure always returns diff + mock_update_hash_from_path.side_effect = \ + lambda hash, *args, **kwargs: hash.update(str(uuid.uuid4())) + self.is_elected_leader.return_value = True self.filter_installed_packages.return_value = [] hooks.upgrade_charm() @@ -477,14 +572,31 @@ class KeystoneRelationTests(CharmTestCase): ssh_authorized_peers.assert_called_with( user=self.ssh_user, group='keystone', peer_interface='cluster', ensure_local_user=True) - self.assertTrue(self.synchronize_ca.called) + self.assertTrue(mock_synchronize_ca.called) self.log.assert_called_with( 'Cluster leader - ensuring endpoint configuration' ' is up to date') self.assertTrue(self.ensure_initial_admin.called) + @patch('keystone_utils.relation_ids') + @patch('keystone_utils.is_elected_leader') + @patch('keystone_utils.is_sync_master') + @patch('keystone_utils.update_hash_from_path') + @patch('keystone_utils.synchronize_ca') @patch.object(unison, 'ssh_authorized_peers') - def test_upgrade_charm_not_leader(self, ssh_authorized_peers): + def test_upgrade_charm_not_leader(self, ssh_authorized_peers, + mock_synchronize_ca, + mock_update_hash_from_path, + mock_is_sync_master, + mock_is_elected_leader, + mock_relation_ids): + mock_relation_ids.return_value = [] + mock_is_sync_master.return_value = True + mock_is_elected_leader.return_value = True + # Ensure always returns diff + mock_update_hash_from_path.side_effect = \ + lambda hash, *args, **kwargs: hash.update(str(uuid.uuid4())) + self.is_elected_leader.return_value = False self.filter_installed_packages.return_value = [] hooks.upgrade_charm() @@ -492,6 +604,6 @@ class KeystoneRelationTests(CharmTestCase): ssh_authorized_peers.assert_called_with( user=self.ssh_user, group='keystone', peer_interface='cluster', ensure_local_user=True) - self.assertTrue(self.synchronize_ca.called) + self.assertTrue(mock_synchronize_ca.called) self.assertFalse(self.log.called) self.assertFalse(self.ensure_initial_admin.called) From fa440c145c235aaf34edecefe184a0c0319d512a Mon Sep 17 00:00:00 2001 From: Edward Hope-Morley Date: Mon, 12 Jan 2015 12:45:22 +0000 Subject: [PATCH 05/48] Removed sync-master logi --- hooks/keystone_context.py | 17 ++--- hooks/keystone_hooks.py | 10 +-- hooks/keystone_utils.py | 152 ++++++++++++-------------------------- 3 files changed, 59 insertions(+), 120 deletions(-) diff --git a/hooks/keystone_context.py b/hooks/keystone_context.py index cfb7eb72..12a15f7a 100644 --- a/hooks/keystone_context.py +++ b/hooks/keystone_context.py @@ -8,7 +8,8 @@ from charmhelpers.contrib.openstack import context from charmhelpers.contrib.hahelpers.cluster import ( determine_apache_port, - determine_api_port + determine_api_port, + is_elected_leader, ) from charmhelpers.core.hookenv import ( @@ -37,7 +38,7 @@ class ApacheSSLContext(context.ApacheSSLContext): from keystone_utils import ( SSH_USER, get_ca, - is_ssl_cert_master, + CLUSTER_RES, ensure_permissions, ) @@ -48,9 +49,8 @@ class ApacheSSLContext(context.ApacheSSLContext): ensure_permissions(ssl_dir, user=SSH_USER, group='keystone', perms=perms) - if not is_ssl_cert_master(): - log("Not leader or cert master so skipping apache cert config", - level=INFO) + if not is_elected_leader(CLUSTER_RES): + log("Not leader - skipping apache cert config", level=INFO) return log("Creating apache ssl certs in %s" % (ssl_dir), level=INFO) @@ -66,13 +66,12 @@ class ApacheSSLContext(context.ApacheSSLContext): from keystone_utils import ( SSH_USER, get_ca, - is_ssl_cert_master, + CLUSTER_RES, ensure_permissions, ) - if not is_ssl_cert_master(): - log("Not leader or cert master so skipping apache ca config", - level=INFO) + if not is_elected_leader(CLUSTER_RES): + log("Not leader - skipping apache cert config", level=INFO) return ca = get_ca(user=SSH_USER) diff --git a/hooks/keystone_hooks.py b/hooks/keystone_hooks.py index 21a476ac..82a130f1 100755 --- a/hooks/keystone_hooks.py +++ b/hooks/keystone_hooks.py @@ -64,7 +64,6 @@ from keystone_utils import ( check_peer_actions, CA_CERT_PATH, ensure_permissions, - is_ssl_cert_master, ) from charmhelpers.contrib.hahelpers.cluster import ( @@ -307,7 +306,8 @@ def cluster_changed(): # Uncomment the following to print out all cluster relation settings in # log (debug only). """ - rels = ["%s:%s" % (k, v) for k, v in relation_get().iteritems()] + settings = relation_get() + rels = ["%s:%s" % (k, v) for k, v in settings.iteritems()] tag = '\n[debug:%s]' % (remote_unit()) log("PEER RELATION SETTINGS (unit=%s): %s" % (remote_unit(), tag.join(rels)), @@ -330,9 +330,7 @@ def cluster_changed(): else: identity_updates_with_ssl_sync() - # If we are cert master ignore what other peers have to say - if not is_ssl_cert_master(): - echo_whitelist.append('ssl-cert-master') + echo_whitelist.append('ssl-synced-units') # ssl cert sync must be done BEFORE this to reduce the risk of feedback # loops in cluster relation @@ -372,7 +370,7 @@ def ha_joined(): vip_group.append(vip_key) if len(vip_group) >= 1: - relation_set(groups={'grp_ks_vips': ' '.join(vip_group)}) + relation_set(groups={CLUSTER_RES: ' '.join(vip_group)}) init_services = { 'res_ks_haproxy': 'haproxy' diff --git a/hooks/keystone_utils.py b/hooks/keystone_utils.py index 8e9fd0a8..b5021fba 100644 --- a/hooks/keystone_utils.py +++ b/hooks/keystone_utils.py @@ -4,7 +4,7 @@ import grp import hashlib import os import pwd -import shutil +import re import subprocess import threading import time @@ -19,7 +19,6 @@ from charmhelpers.contrib.hahelpers.cluster import( is_elected_leader, determine_api_port, https, - is_clustered, peer_units, ) @@ -58,7 +57,6 @@ from charmhelpers.core.hookenv import ( config, log, local_unit, - related_units, relation_get, relation_set, relation_ids, @@ -400,7 +398,7 @@ def create_endpoint_template(region, service, publicurl, adminurl, up_to_date = True for k in ['publicurl', 'adminurl', 'internalurl']: - if ep[k] != locals()[k]: + if ep.get(k) != locals()[k]: up_to_date = False if up_to_date: @@ -651,26 +649,37 @@ def check_peer_actions(): if restart and os.path.isdir(SYNC_FLAGS_DIR): for flagfile in glob.glob(os.path.join(SYNC_FLAGS_DIR, '*')): flag = os.path.basename(flagfile) - service = flag.partition('.')[0] - action = flag.partition('.')[2] - - if action == 'restart': - log("Running action='%s' on service '%s'" % - (action, service), level=DEBUG) - service_restart(service) - elif action == 'start': - log("Running action='%s' on service '%s'" % - (action, service), level=DEBUG) - service_start(service) - elif action == 'stop': - log("Running action='%s' on service '%s'" % - (action, service), level=DEBUG) - service_stop(service) - elif flag == 'update-ca-certificates': - log("Running update-ca-certificates", level=DEBUG) - subprocess.check_call(['update-ca-certificates']) + key = re.compile("^(.+)?\.(.+)?\.(.+)") + res = re.search(key, flag) + if res: + source = res. group(1) + service = res. group(2) + action = res. group(3) else: - log("Unknown action flag=%s" % (flag), level=WARNING) + key = re.compile("^(.+)?\.(.+)?") + res = re.search(key, flag) + source = res. group(1) + action = res. group(2) + + # Don't execute actions requested byu this unit. + if local_unit().replace('.', '-') != source: + if action == 'restart': + log("Running action='%s' on service '%s'" % + (action, service), level=DEBUG) + service_restart(service) + elif action == 'start': + log("Running action='%s' on service '%s'" % + (action, service), level=DEBUG) + service_start(service) + elif action == 'stop': + log("Running action='%s' on service '%s'" % + (action, service), level=DEBUG) + service_stop(service) + elif action == 'update-ca-certificates': + log("Running update-ca-certificates", level=DEBUG) + subprocess.check_call(['update-ca-certificates']) + else: + log("Unknown action flag=%s" % (flag), level=WARNING) os.remove(flagfile) @@ -683,33 +692,22 @@ def create_peer_service_actions(action, services): synced. """ for service in services: - flagfile = os.path.join(SYNC_FLAGS_DIR, '%s.%s' % - (service.strip(), action)) + flagfile = os.path.join(SYNC_FLAGS_DIR, '%s.%s.%s' % + (local_unit().replace('/', '-'), + service.strip(), action)) log("Creating action %s" % (flagfile), level=DEBUG) write_file(flagfile, content='', owner=SSH_USER, group='keystone', perms=0o644) def create_service_action(action): + action = "%s.%s" % (local_unit().replace('/', '-'), action) flagfile = os.path.join(SYNC_FLAGS_DIR, action) log("Creating action %s" % (flagfile), level=DEBUG) write_file(flagfile, content='', owner=SSH_USER, group='keystone', perms=0o644) -def is_ssl_cert_master(): - """Return True if this unit is ssl cert master.""" - master = None - for rid in relation_ids('cluster'): - master = relation_get(attribute='ssl-cert-master', rid=rid, - unit=local_unit()) - - if master and master == local_unit(): - return True - - return False - - @retry_on_exception(3, base_delay=2, exc_type=subprocess.CalledProcessError) def unison_sync(paths_to_sync): """Do unison sync and retry a few times if it fails since peers may not be @@ -723,39 +721,6 @@ def unison_sync(paths_to_sync): fatal=True) -def is_sync_master(): - if not peer_units(): - log("Not syncing certs since there are no peer units.", level=INFO) - return False - - # If no ssl master elected and we are cluster leader, elect this unit. - if is_elected_leader(CLUSTER_RES): - master = [] - for rid in relation_ids('cluster'): - for unit in related_units(rid): - m = relation_get(rid=rid, unit=unit, - attribute='ssl-cert-master') - if m is not None: - master.append(m) - - master = set(master) - if not master or ('unknown' in master and len(master) == 1): - log("Electing this unit as ssl-cert-master", level=DEBUG) - for rid in relation_ids('cluster'): - settings = {'ssl-cert-master': local_unit(), - 'ssl-synced-units': None} - relation_set(relation_id=rid, relation_settings=settings) - - # Return now and wait for cluster-relation-changed for sync. - return False - - if not is_ssl_cert_master(): - log("Not ssl cert master - skipping sync", level=INFO) - return False - - return True - - def synchronize_ca(fatal=True): """Broadcast service credentials to peers. @@ -785,16 +750,8 @@ def synchronize_ca(fatal=True): log("Nothing to sync - skipping", level=DEBUG) return - # If we are sync master proceed even if we are not leader since we are - # most likely to have up-to-date certs. If not leader we will re-elect once - # synced. This is done to avoid being affected by leader changing before - # all units have synced certs. - - # Clear any existing flags - if os.path.isdir(SYNC_FLAGS_DIR): - shutil.rmtree(SYNC_FLAGS_DIR) - - mkdir(SYNC_FLAGS_DIR, SSH_USER, 'keystone', 0o775) + if not os.path.isdir(SYNC_FLAGS_DIR): + mkdir(SYNC_FLAGS_DIR, SSH_USER, 'keystone', 0o775) # We need to restart peer apache services to ensure they have picked up # new ssl keys. @@ -809,10 +766,6 @@ def synchronize_ca(fatal=True): log("Sync failed but fatal=False", level=INFO) return - # Cleanup - shutil.rmtree(SYNC_FLAGS_DIR) - mkdir(SYNC_FLAGS_DIR, SSH_USER, 'keystone', 0o775) - trigger = str(uuid.uuid4()) log("Sending restart-services-trigger=%s to all peers" % (trigger), level=DEBUG) @@ -829,7 +782,7 @@ def update_hash_from_path(hash, path, recurse_depth=10): log("Max recursion depth (%s) reached for update_hash_from_path() at " "path='%s' - not going any deeper" % (recurse_depth, path), level=WARNING) - return sum + return for p in glob.glob("%s/*" % path): if os.path.isdir(p): @@ -847,13 +800,16 @@ def synchronize_ca_if_changed(force=False, fatal=True): """ def inner_synchronize_ca_if_changed1(f): def inner_synchronize_ca_if_changed2(*args, **kwargs): - if not is_sync_master(): - return f(*args, **kwargs) - - peer_settings = {} + # Only sync master can do sync. Ensure (a) we are not nested and + # (b) a master is elected and we are it. try: + acquired = SSL_SYNC_SEMAPHORE.acquire(blocking=0) + if not acquired or not is_elected_leader(CLUSTER_RES): + return f(*args, **kwargs) + + peer_settings = {} # Ensure we don't do a double sync if we are nested. - if not force and SSL_SYNC_SEMAPHORE.acquire(blocking=0): + if not force: hash1 = hashlib.sha256() for path in [SSL_DIR, APACHE_SSL_DIR, CA_CERT_PATH]: update_hash_from_path(hash1, path) @@ -880,12 +836,6 @@ def synchronize_ca_if_changed(force=False, fatal=True): log("Doing forced ssl cert sync", level=DEBUG) peer_settings = synchronize_ca(fatal=fatal) - # If we are the sync master but no longer leader then re-elect - # master. - if not is_elected_leader(CLUSTER_RES): - log("Re-electing ssl cert master.", level=INFO) - peer_settings['ssl-cert-master'] = 'unknown' - for rid in relation_ids('cluster'): relation_set(relation_id=rid, relation_settings=peer_settings) @@ -921,14 +871,6 @@ def get_ca(user='keystone', group='keystone'): '%s' % SSL_DIR]) subprocess.check_output(['chmod', '-R', 'g+rwx', '%s' % SSL_DIR]) - # Tag this host as the ssl cert master. - if is_clustered() or peer_units(): - for rid in relation_ids('cluster'): - relation_set(relation_id=rid, - relation_settings={'ssl-cert-master': - local_unit(), - 'synced-units': None}) - ssl.CA_SINGLETON.append(ca) return ssl.CA_SINGLETON[0] From 8ec5688fe446ce5970ceb58371599b8a0b247043 Mon Sep 17 00:00:00 2001 From: Edward Hope-Morley Date: Mon, 12 Jan 2015 17:02:08 +0000 Subject: [PATCH 06/48] sync only fatal=True in cluster relation --- hooks/keystone_hooks.py | 8 ++++---- hooks/keystone_utils.py | 11 ++++++----- 2 files changed, 10 insertions(+), 9 deletions(-) diff --git a/hooks/keystone_hooks.py b/hooks/keystone_hooks.py index 82a130f1..92ca76a1 100755 --- a/hooks/keystone_hooks.py +++ b/hooks/keystone_hooks.py @@ -100,7 +100,7 @@ def install(): @hooks.hook('config-changed') @restart_on_change(restart_map()) -@synchronize_ca_if_changed(fatal=False) +@synchronize_ca_if_changed() def config_changed(): if config('prefer-ipv6'): setup_ipv6() @@ -286,13 +286,13 @@ def cluster_joined(relation_id=None): relation_settings={'private-address': private_addr}) -@synchronize_ca_if_changed() +@synchronize_ca_if_changed(fatal=True) def identity_updates_with_ssl_sync(): CONFIGS.write_all() update_all_identity_relation_units() -@synchronize_ca_if_changed(force=True) +@synchronize_ca_if_changed(force=True, fatal=True) def identity_updates_with_forced_ssl_sync(): identity_updates_with_ssl_sync() @@ -416,7 +416,7 @@ def admin_relation_changed(): relation_set(**relation_data) -@synchronize_ca_if_changed() +@synchronize_ca_if_changed(fatal=True) def configure_https(): ''' Enables SSL API Apache config if appropriate and kicks identity-service diff --git a/hooks/keystone_utils.py b/hooks/keystone_utils.py index b5021fba..8199d641 100644 --- a/hooks/keystone_utils.py +++ b/hooks/keystone_utils.py @@ -721,7 +721,7 @@ def unison_sync(paths_to_sync): fatal=True) -def synchronize_ca(fatal=True): +def synchronize_ca(fatal=False): """Broadcast service credentials to peers. By default a failure to sync is fatal and will result in a raised @@ -792,7 +792,7 @@ def update_hash_from_path(hash, path, recurse_depth=10): hash.update(fd.read()) -def synchronize_ca_if_changed(force=False, fatal=True): +def synchronize_ca_if_changed(force=False, fatal=False): """Decorator to perform ssl cert sync if decorated function modifies them in any way. @@ -836,9 +836,10 @@ def synchronize_ca_if_changed(force=False, fatal=True): log("Doing forced ssl cert sync", level=DEBUG) peer_settings = synchronize_ca(fatal=fatal) - for rid in relation_ids('cluster'): - relation_set(relation_id=rid, - relation_settings=peer_settings) + if peer_settings: + for rid in relation_ids('cluster'): + relation_set(relation_id=rid, + relation_settings=peer_settings) return ret finally: From 4a9602f1c117c5a582d709e966423f2e2bb192f5 Mon Sep 17 00:00:00 2001 From: Edward Hope-Morley Date: Tue, 13 Jan 2015 11:04:56 +0000 Subject: [PATCH 07/48] check syncs and retry if inconsistent --- hooks/keystone_hooks.py | 17 +++-- hooks/keystone_ssl.py | 54 ++++++++++++---- hooks/keystone_utils.py | 138 +++++++++++++++++++++++++++------------- 3 files changed, 147 insertions(+), 62 deletions(-) diff --git a/hooks/keystone_hooks.py b/hooks/keystone_hooks.py index 92ca76a1..c6214851 100755 --- a/hooks/keystone_hooks.py +++ b/hooks/keystone_hooks.py @@ -64,6 +64,7 @@ from keystone_utils import ( check_peer_actions, CA_CERT_PATH, ensure_permissions, + print_rel_debug, ) from charmhelpers.contrib.hahelpers.cluster import ( @@ -219,6 +220,8 @@ def pgsql_db_changed(): @hooks.hook('identity-service-relation-changed') @synchronize_ca_if_changed() def identity_changed(relation_id=None, remote_unit=None): + CONFIGS.write_all() + notifications = {} if is_elected_leader(CLUSTER_RES): # Catch database not configured error and defer until db ready @@ -236,8 +239,6 @@ def identity_changed(relation_id=None, remote_unit=None): log("Unexpected exception occurred", level=ERROR) raise - CONFIGS.write_all() - settings = relation_get(rid=relation_id, unit=remote_unit) service = settings.get('service', None) if service: @@ -332,6 +333,10 @@ def cluster_changed(): echo_whitelist.append('ssl-synced-units') + echo_settings = {k: v for k, v in relation_get().iteritems() + if k in echo_whitelist} + print_rel_debug(echo_settings, None, None, 'cluster_changed', '1') + # ssl cert sync must be done BEFORE this to reduce the risk of feedback # loops in cluster relation peer_echo(includes=echo_whitelist) @@ -390,8 +395,9 @@ def ha_joined(): @restart_on_change(restart_map()) @synchronize_ca_if_changed() def ha_changed(): - clustered = relation_get('clustered') CONFIGS.write_all() + + clustered = relation_get('clustered') if clustered and is_elected_leader(CLUSTER_RES): ensure_initial_admin(config) log('Cluster configured, notifying other services and updating ' @@ -442,14 +448,15 @@ def upgrade_charm(): group='keystone', peer_interface='cluster', ensure_local_user=True) + + CONFIGS.write_all() + if is_elected_leader(CLUSTER_RES): log('Cluster leader - ensuring endpoint configuration is up to ' 'date', level=DEBUG) time.sleep(10) update_all_identity_relation_units() - CONFIGS.write_all() - def main(): try: diff --git a/hooks/keystone_ssl.py b/hooks/keystone_ssl.py index 6b2f4a85..4f6fce11 100644 --- a/hooks/keystone_ssl.py +++ b/hooks/keystone_ssl.py @@ -5,6 +5,13 @@ import shutil import subprocess import tarfile import tempfile +import time + +from charmhelpers.core.hookenv import ( + log, + DEBUG, + WARNING, +) CA_EXPIRY = '365' ORG_NAME = 'Ubuntu' @@ -278,23 +285,42 @@ class JujuCA(object): crt = self._sign_csr(csr, service, common_name) cmd = ['chown', '-R', '%s.%s' % (self.user, self.group), self.ca_dir] subprocess.check_call(cmd) - print 'Signed new CSR, crt @ %s' % crt + log('Signed new CSR, crt @ %s' % crt, level=DEBUG) return crt, key def get_cert_and_key(self, common_name): - print 'Getting certificate and key for %s.' % common_name - key = os.path.join(self.ca_dir, 'certs', '%s.key' % common_name) - crt = os.path.join(self.ca_dir, 'certs', '%s.crt' % common_name) - if os.path.isfile(crt): - print 'Found existing certificate for %s.' % common_name - crt = open(crt, 'r').read() - try: - key = open(key, 'r').read() - except: - print 'Could not load ssl private key for %s from %s' %\ - (common_name, key) - exit(1) - return crt, key + log('Getting certificate and key for %s.' % common_name, level=DEBUG) + keypath = os.path.join(self.ca_dir, 'certs', '%s.key' % common_name) + crtpath = os.path.join(self.ca_dir, 'certs', '%s.crt' % common_name) + if os.path.isfile(crtpath): + log('Found existing certificate for %s.' % common_name, + level=DEBUG) + max_retries = 3 + while True: + mtime = os.path.getmtime(crtpath) + + crt = open(crtpath, 'r').read() + try: + key = open(keypath, 'r').read() + except: + msg = ('Could not load ssl private key for %s from %s' % + (common_name, keypath)) + raise Exception(msg) + + # Ensure we are not reading a file that is being written to + if mtime != os.path.getmtime(crtpath): + max_retries -= 1 + if max_retries == 0: + msg = ("crt contents changed during read - retry " + "failed") + raise Exception(msg) + + log("crt contents changed during read - re-reading", + level=WARNING) + time.sleep(1) + else: + return crt, key + crt, key = self._create_certificate(common_name, common_name) return open(crt, 'r').read(), open(key, 'r').read() diff --git a/hooks/keystone_utils.py b/hooks/keystone_utils.py index 8199d641..b62f0ddb 100644 --- a/hooks/keystone_utils.py +++ b/hooks/keystone_utils.py @@ -63,6 +63,7 @@ from charmhelpers.core.hookenv import ( DEBUG, INFO, WARNING, + ERROR, ) from charmhelpers.fetch import ( @@ -652,14 +653,14 @@ def check_peer_actions(): key = re.compile("^(.+)?\.(.+)?\.(.+)") res = re.search(key, flag) if res: - source = res. group(1) - service = res. group(2) - action = res. group(3) + source = res.group(1) + service = res.group(2) + action = res.group(3) else: key = re.compile("^(.+)?\.(.+)?") res = re.search(key, flag) - source = res. group(1) - action = res. group(2) + source = res.group(1) + action = res.group(2) # Don't execute actions requested byu this unit. if local_unit().replace('.', '-') != source: @@ -676,7 +677,7 @@ def check_peer_actions(): (action, service), level=DEBUG) service_stop(service) elif action == 'update-ca-certificates': - log("Running update-ca-certificates", level=DEBUG) + log("Running %s" % (action), level=DEBUG) subprocess.check_call(['update-ca-certificates']) else: log("Unknown action flag=%s" % (flag), level=WARNING) @@ -700,12 +701,13 @@ def create_peer_service_actions(action, services): perms=0o644) -def create_service_action(action): - action = "%s.%s" % (local_unit().replace('/', '-'), action) - flagfile = os.path.join(SYNC_FLAGS_DIR, action) - log("Creating action %s" % (flagfile), level=DEBUG) - write_file(flagfile, content='', owner=SSH_USER, group='keystone', - perms=0o644) +def create_peer_actions(actions): + for action in actions: + action = "%s.%s" % (local_unit().replace('/', '-'), action) + flagfile = os.path.join(SYNC_FLAGS_DIR, action) + log("Creating action %s" % (flagfile), level=DEBUG) + write_file(flagfile, content='', owner=SSH_USER, group='keystone', + perms=0o644) @retry_on_exception(3, base_delay=2, exc_type=subprocess.CalledProcessError) @@ -756,22 +758,47 @@ def synchronize_ca(fatal=False): # We need to restart peer apache services to ensure they have picked up # new ssl keys. create_peer_service_actions('restart', ['apache2']) - create_service_action('update-ca-certificates') - try: - unison_sync(paths_to_sync) - except: - if fatal: - raise - else: - log("Sync failed but fatal=False", level=INFO) - return + create_peer_actions(['update-ca-certificates']) - trigger = str(uuid.uuid4()) - log("Sending restart-services-trigger=%s to all peers" % (trigger), + retries = 3 + while True: + hash1 = hashlib.sha256() + for path in paths_to_sync: + update_hash_from_path(hash1, path) + + try: + unison_sync(paths_to_sync) + except: + if fatal: + raise + else: + log("Sync failed but fatal=False", level=INFO) + return + + hash2 = hashlib.sha256() + for path in paths_to_sync: + update_hash_from_path(hash2, path) + + # Detect whether someone else has synced to this unit while we did our + # transfer. + if hash1.hexdigest() != hash2.hexdigest(): + retries -= 1 + if retries > 0: + log("SSL dir contents changed during sync - retrying unison " + "sync %s more times" % (retries), level=WARNING) + else: + log("SSL dir contents changed during sync - retries failed", + level=ERROR) + return {} + else: + break + + hash = hash1.hexdigest() + log("Sending restart-services-trigger=%s to all peers" % (hash), level=DEBUG) log("Sync complete", level=DEBUG) - return {'restart-services-trigger': trigger, + return {'restart-services-trigger': hash, 'ssl-synced-units': peer_units()} @@ -802,28 +829,31 @@ def synchronize_ca_if_changed(force=False, fatal=False): def inner_synchronize_ca_if_changed2(*args, **kwargs): # Only sync master can do sync. Ensure (a) we are not nested and # (b) a master is elected and we are it. + acquired = SSL_SYNC_SEMAPHORE.acquire(blocking=0) try: - acquired = SSL_SYNC_SEMAPHORE.acquire(blocking=0) - if not acquired or not is_elected_leader(CLUSTER_RES): + if not acquired: + log("Nested sync - ignoring", level=DEBUG) + return f(*args, **kwargs) + + if not is_elected_leader(CLUSTER_RES): + log("Not leader - ignoring sync", level=DEBUG) return f(*args, **kwargs) peer_settings = {} - # Ensure we don't do a double sync if we are nested. if not force: - hash1 = hashlib.sha256() - for path in [SSL_DIR, APACHE_SSL_DIR, CA_CERT_PATH]: - update_hash_from_path(hash1, path) + ssl_dirs = [SSL_DIR, APACHE_SSL_DIR, CA_CERT_PATH] - hash1 = hash1.hexdigest() + hash1 = hashlib.sha256() + for path in ssl_dirs: + update_hash_from_path(hash1, path) ret = f(*args, **kwargs) hash2 = hashlib.sha256() - for path in [SSL_DIR, APACHE_SSL_DIR, CA_CERT_PATH]: + for path in ssl_dirs: update_hash_from_path(hash2, path) - hash2 = hash2.hexdigest() - if hash1 != hash2: + if hash1.hexdigest() != hash2.hexdigest(): log("SSL certs have changed - syncing peers", level=DEBUG) peer_settings = synchronize_ca(fatal=fatal) @@ -832,9 +862,8 @@ def synchronize_ca_if_changed(force=False, fatal=False): level=DEBUG) else: ret = f(*args, **kwargs) - if force: - log("Doing forced ssl cert sync", level=DEBUG) - peer_settings = synchronize_ca(fatal=fatal) + log("Doing forced ssl cert sync", level=DEBUG) + peer_settings = synchronize_ca(fatal=fatal) if peer_settings: for rid in relation_ids('cluster'): @@ -889,6 +918,22 @@ def relation_list(rid): return result +def print_rel_debug(relation_data, remote_unit, relation_id, tag, name): + debug_settings = relation_get(unit=local_unit(), rid=relation_id) + diff = {k: {'b': debug_settings[k], 'a': v} for k, v in + relation_data.iteritems() + if (k in debug_settings and + relation_data[k] != debug_settings.get(k))} + + unchanged = [k for k in debug_settings.iterkeys() + if k not in relation_data] + + log("[debug:%s:%s:%s:%s] diff=%s" % + (name, tag, remote_unit, relation_id, str(diff)), level=DEBUG) + log("[debug:%s:%s:%s:%s] unchanged=%s" % + (name, tag, remote_unit, relation_id, unchanged), level=DEBUG) + + def add_service_to_keystone(relation_id=None, remote_unit=None): import manager manager = manager.KeystoneManager(endpoint=get_local_endpoint(), @@ -900,7 +945,7 @@ def add_service_to_keystone(relation_id=None, remote_unit=None): https_cns = [] if single.issubset(settings): # other end of relation advertised only one endpoint - if 'None' in [v for k, v in settings.iteritems()]: + if 'None' in settings.itervalues(): # Some backend services advertise no endpoint but require a # hook execution to update auth strategy. relation_data = {} @@ -928,6 +973,10 @@ def add_service_to_keystone(relation_id=None, remote_unit=None): for role in get_requested_roles(settings): log("Creating requested role: %s" % role) create_role(role) + + print_rel_debug(relation_data, remote_unit, relation_id, "1", + 'add-svc-to-ks') + peer_store_and_set(relation_id=relation_id, **relation_data) return @@ -1003,7 +1052,7 @@ def add_service_to_keystone(relation_id=None, remote_unit=None): if prefix: service_username = "%s%s" % (prefix, service_username) - if 'None' in [v for k, v in settings.iteritems()]: + if 'None' in settings.itervalues(): return if not service_username: @@ -1041,10 +1090,10 @@ def add_service_to_keystone(relation_id=None, remote_unit=None): "service_password": service_password, "service_tenant": service_tenant, "service_tenant_id": manager.resolve_tenant_id(service_tenant), - "https_keystone": "False", - "ssl_cert": "", - "ssl_key": "", - "ca_cert": "" + "https_keystone": None, + "ssl_cert": None, + "ssl_key": None, + "ca_cert": None } # Check if https is enabled @@ -1070,6 +1119,9 @@ def add_service_to_keystone(relation_id=None, remote_unit=None): ca_bundle = ca.get_ca_bundle() relation_data['ca_cert'] = b64encode(ca_bundle) relation_data['https_keystone'] = 'True' + + print_rel_debug(relation_data, remote_unit, relation_id, "2", + 'add-svc-to-ks') peer_store_and_set(relation_id=relation_id, **relation_data) From 2fa428e50be63ec0a04325bcba55b0fd953e95c9 Mon Sep 17 00:00:00 2001 From: Edward Hope-Morley Date: Tue, 13 Jan 2015 13:43:07 +0000 Subject: [PATCH 08/48] tests passing and cleanup --- hooks/keystone_hooks.py | 16 --- hooks/keystone_utils.py | 21 ---- unit_tests/test_keystone_hooks.py | 156 +++++++++++++++--------------- unit_tests/test_keystone_utils.py | 8 +- 4 files changed, 82 insertions(+), 119 deletions(-) diff --git a/hooks/keystone_hooks.py b/hooks/keystone_hooks.py index 3f4fcdf7..3397fd22 100755 --- a/hooks/keystone_hooks.py +++ b/hooks/keystone_hooks.py @@ -65,7 +65,6 @@ from keystone_utils import ( check_peer_actions, CA_CERT_PATH, ensure_permissions, - print_rel_debug, ) from charmhelpers.contrib.hahelpers.cluster import ( @@ -308,17 +307,6 @@ def identity_updates_with_forced_ssl_sync(): def cluster_changed(): check_peer_actions() - # Uncomment the following to print out all cluster relation settings in - # log (debug only). - """ - settings = relation_get() - rels = ["%s:%s" % (k, v) for k, v in settings.iteritems()] - tag = '\n[debug:%s]' % (remote_unit()) - log("PEER RELATION SETTINGS (unit=%s): %s" % (remote_unit(), - tag.join(rels)), - level=DEBUG) - """ - # NOTE(jamespage) re-echo passwords for peer storage echo_whitelist = ['_passwd', 'identity-service:'] unison.ssh_authorized_peers(user=SSH_USER, @@ -337,10 +325,6 @@ def cluster_changed(): echo_whitelist.append('ssl-synced-units') - echo_settings = {k: v for k, v in relation_get().iteritems() - if k in echo_whitelist} - print_rel_debug(echo_settings, None, None, 'cluster_changed', '1') - # ssl cert sync must be done BEFORE this to reduce the risk of feedback # loops in cluster relation peer_echo(includes=echo_whitelist) diff --git a/hooks/keystone_utils.py b/hooks/keystone_utils.py index 8b4a41b6..431653f8 100644 --- a/hooks/keystone_utils.py +++ b/hooks/keystone_utils.py @@ -929,22 +929,6 @@ def relation_list(rid): return result -def print_rel_debug(relation_data, remote_unit, relation_id, tag, name): - debug_settings = relation_get(unit=local_unit(), rid=relation_id) - diff = {k: {'b': debug_settings[k], 'a': v} for k, v in - relation_data.iteritems() - if (k in debug_settings and - relation_data[k] != debug_settings.get(k))} - - unchanged = [k for k in debug_settings.iterkeys() - if k not in relation_data] - - log("[debug:%s:%s:%s:%s] diff=%s" % - (name, tag, remote_unit, relation_id, str(diff)), level=DEBUG) - log("[debug:%s:%s:%s:%s] unchanged=%s" % - (name, tag, remote_unit, relation_id, unchanged), level=DEBUG) - - def add_service_to_keystone(relation_id=None, remote_unit=None): import manager manager = manager.KeystoneManager(endpoint=get_local_endpoint(), @@ -985,9 +969,6 @@ def add_service_to_keystone(relation_id=None, remote_unit=None): log("Creating requested role: %s" % role) create_role(role) - print_rel_debug(relation_data, remote_unit, relation_id, "1", - 'add-svc-to-ks') - peer_store_and_set(relation_id=relation_id, **relation_data) return @@ -1131,8 +1112,6 @@ def add_service_to_keystone(relation_id=None, remote_unit=None): relation_data['ca_cert'] = b64encode(ca_bundle) relation_data['https_keystone'] = 'True' - print_rel_debug(relation_data, remote_unit, relation_id, "2", - 'add-svc-to-ks') peer_store_and_set(relation_id=relation_id, **relation_data) diff --git a/unit_tests/test_keystone_hooks.py b/unit_tests/test_keystone_hooks.py index d7dc83f8..d3a1daef 100644 --- a/unit_tests/test_keystone_hooks.py +++ b/unit_tests/test_keystone_hooks.py @@ -31,7 +31,6 @@ TO_PATCH = [ 'local_unit', 'filter_installed_packages', 'relation_ids', - 'relation_list', 'relation_set', 'relation_get', 'related_units', @@ -162,10 +161,13 @@ class KeystoneRelationTests(CharmTestCase): 'is already associated a mysql one') @patch('keystone_utils.log') + @patch('keystone_utils.is_elected_leader') @patch('keystone_utils.peer_units') @patch.object(hooks, 'CONFIGS') def test_db_changed_missing_relation_data(self, configs, mock_peer_units, + mock_is_elected_leader, mock_log): + mock_is_elected_leader.return_value = False mock_peer_units.return_value = None configs.complete_contexts = MagicMock() configs.complete_contexts.return_value = [] @@ -174,8 +176,12 @@ class KeystoneRelationTests(CharmTestCase): 'shared-db relation incomplete. Peer not ready?' ) + @patch('keystone_utils.log') + @patch('keystone_utils.is_elected_leader') @patch.object(hooks, 'CONFIGS') - def test_postgresql_db_changed_missing_relation_data(self, configs): + def test_postgresql_db_changed_missing_relation_data(self, configs, + mock_is_leader, + mock_log): configs.complete_contexts = MagicMock() configs.complete_contexts.return_value = [] hooks.pgsql_db_changed() @@ -198,11 +204,13 @@ class KeystoneRelationTests(CharmTestCase): hooks.pgsql_db_changed() @patch('keystone_utils.log') + @patch('keystone_utils.is_elected_leader') @patch('keystone_utils.peer_units') @patch.object(hooks, 'CONFIGS') @patch.object(hooks, 'identity_changed') def test_db_changed_allowed(self, identity_changed, configs, - mock_peer_units, mock_log): + mock_peer_units, mock_is_elected_leader, + mock_log): mock_peer_units.return_value = None self.relation_ids.return_value = ['identity-service:0'] self.related_units.return_value = ['unit/0'] @@ -217,11 +225,13 @@ class KeystoneRelationTests(CharmTestCase): remote_unit='unit/0') @patch('keystone_utils.log') + @patch('keystone_utils.is_elected_leader') @patch('keystone_utils.peer_units') @patch.object(hooks, 'CONFIGS') @patch.object(hooks, 'identity_changed') def test_db_changed_not_allowed(self, identity_changed, configs, - mock_peer_units, mock_log): + mock_peer_units, mock_is_elected_leader, + mock_log): mock_peer_units.return_value = None self.relation_ids.return_value = ['identity-service:0'] self.related_units.return_value = ['unit/0'] @@ -234,11 +244,13 @@ class KeystoneRelationTests(CharmTestCase): self.assertFalse(identity_changed.called) @patch('keystone_utils.log') + @patch('keystone_utils.is_elected_leader') @patch('keystone_utils.peer_units') @patch.object(hooks, 'CONFIGS') @patch.object(hooks, 'identity_changed') def test_postgresql_db_changed(self, identity_changed, configs, - mock_peer_units, mock_log): + mock_peer_units, mock_is_elected_leader, + mock_log): mock_peer_units.return_value = None self.relation_ids.return_value = ['identity-service:0'] self.related_units.return_value = ['unit/0'] @@ -252,7 +264,9 @@ class KeystoneRelationTests(CharmTestCase): relation_id='identity-service:0', remote_unit='unit/0') - @patch('keystone_utils.is_sync_master') + @patch('keystone_utils.log') + @patch('keystone_utils.is_elected_leader') + @patch.object(hooks, 'peer_units') @patch.object(hooks, 'ensure_permissions') @patch.object(hooks, 'cluster_joined') @patch.object(unison, 'ensure_user') @@ -263,12 +277,15 @@ class KeystoneRelationTests(CharmTestCase): def test_config_changed_no_openstack_upgrade_leader( self, configure_https, identity_changed, configs, get_homedir, ensure_user, cluster_joined, - ensure_permissions, is_sync_master): - is_sync_master.return_value = False + ensure_permissions, mock_peer_units, mock_is_elected_leader, + mock_log): self.openstack_upgrade_available.return_value = False self.is_elected_leader.return_value = True + # avoid having to mock syncer + mock_is_elected_leader.return_value = False + mock_peer_units.return_value = [] self.relation_ids.return_value = ['identity-service:0'] - self.relation_list.return_value = ['unit/0'] + self.related_units.return_value = ['unit/0'] hooks.config_changed() ensure_user.assert_called_with(user=self.ssh_user, group='keystone') @@ -286,7 +303,8 @@ class KeystoneRelationTests(CharmTestCase): relation_id='identity-service:0', remote_unit='unit/0') - @patch('keystone_utils.is_sync_master') + @patch('keystone_utils.log') + @patch('keystone_utils.is_elected_leader') @patch.object(hooks, 'ensure_permissions') @patch.object(hooks, 'cluster_joined') @patch.object(unison, 'ensure_user') @@ -297,10 +315,11 @@ class KeystoneRelationTests(CharmTestCase): def test_config_changed_no_openstack_upgrade_not_leader( self, configure_https, identity_changed, configs, get_homedir, ensure_user, cluster_joined, - ensure_permissions, is_sync_master): - is_sync_master.return_value = False + ensure_permissions, mock_is_elected_leader, + mock_log): self.openstack_upgrade_available.return_value = False self.is_elected_leader.return_value = False + mock_is_elected_leader.return_value = False hooks.config_changed() ensure_user.assert_called_with(user=self.ssh_user, group='keystone') @@ -314,7 +333,9 @@ class KeystoneRelationTests(CharmTestCase): self.assertFalse(self.ensure_initial_admin.called) self.assertFalse(identity_changed.called) - @patch('keystone_utils.is_sync_master') + @patch('keystone_utils.log') + @patch('keystone_utils.is_elected_leader') + @patch.object(hooks, 'peer_units') @patch.object(hooks, 'ensure_permissions') @patch.object(hooks, 'cluster_joined') @patch.object(unison, 'ensure_user') @@ -325,12 +346,15 @@ class KeystoneRelationTests(CharmTestCase): def test_config_changed_with_openstack_upgrade( self, configure_https, identity_changed, configs, get_homedir, ensure_user, cluster_joined, - ensure_permissions, is_sync_master): - is_sync_master.return_value = False + ensure_permissions, mock_peer_units, mock_is_elected_leader, + mock_log): self.openstack_upgrade_available.return_value = True self.is_elected_leader.return_value = True + # avoid having to mock syncer + mock_is_elected_leader.return_value = False + mock_peer_units.return_value = [] self.relation_ids.return_value = ['identity-service:0'] - self.relation_list.return_value = ['unit/0'] + self.related_units.return_value = ['unit/0'] hooks.config_changed() ensure_user.assert_called_with(user=self.ssh_user, group='keystone') @@ -351,42 +375,24 @@ class KeystoneRelationTests(CharmTestCase): remote_unit='unit/0') @patch('keystone_utils.log') - @patch('keystone_utils.peer_units') - @patch('keystone_utils.relation_ids') @patch('keystone_utils.is_elected_leader') - @patch('keystone_utils.is_sync_master') - @patch('keystone_utils.update_hash_from_path') - @patch('keystone_utils.synchronize_ca') @patch.object(hooks, 'hashlib') @patch.object(hooks, 'send_notifications') def test_identity_changed_leader(self, mock_send_notifications, - mock_hashlib, mock_synchronize_ca, - mock_update_hash_from_path, - mock_is_sync_master, - mock_is_elected_leader, - mock_relation_ids, mock_peer_units, + mock_hashlib, mock_is_elected_leader, mock_log): - mock_peer_units.return_value = None - mock_relation_ids.return_value = [] - mock_is_sync_master.return_value = True mock_is_elected_leader.return_value = True - # Ensure always returns diff - mock_update_hash_from_path.side_effect = \ - lambda hash, *args, **kwargs: hash.update(str(uuid.uuid4())) - - self.is_elected_leader.return_value = True hooks.identity_changed( relation_id='identity-service:0', remote_unit='unit/0') self.add_service_to_keystone.assert_called_with( 'identity-service:0', 'unit/0') - self.assertTrue(mock_synchronize_ca.called) @patch('keystone_utils.log') - @patch('keystone_utils.peer_units') - def test_identity_changed_no_leader(self, mock_peer_units, mock_log): - mock_peer_units.return_value = None + @patch('keystone_utils.is_elected_leader') + def test_identity_changed_no_leader(self, mock_is_elected_leader, + mock_log): self.is_elected_leader.return_value = False hooks.identity_changed( relation_id='identity-service:0', @@ -403,33 +409,23 @@ class KeystoneRelationTests(CharmTestCase): peer_interface='cluster', ensure_local_user=True) @patch('keystone_utils.log') - @patch('keystone_utils.relation_ids') @patch('keystone_utils.is_elected_leader') - @patch('keystone_utils.is_sync_master') - @patch('keystone_utils.update_hash_from_path') @patch('keystone_utils.synchronize_ca') @patch.object(hooks, 'check_peer_actions') @patch.object(unison, 'ssh_authorized_peers') @patch.object(hooks, 'CONFIGS') def test_cluster_changed(self, configs, ssh_authorized_peers, - check_peer_actions, - mock_synchronize_ca, mock_update_hash_from_path, - mock_is_sync_master, mock_is_elected_leader, - mock_relation_ids, mock_log): - mock_relation_ids.return_value = [] - mock_is_sync_master.return_value = True - mock_is_elected_leader.return_value = True - # Ensure always returns diff - mock_update_hash_from_path.side_effect = \ - lambda hash, *args, **kwargs: hash.update(str(uuid.uuid4())) - + check_peer_actions, mock_synchronize_ca, + mock_is_elected_leader, + mock_log): + mock_is_elected_leader.return_value = False hooks.cluster_changed() - whitelist = ['_passwd', 'identity-service:', 'ssl-cert-master'] + whitelist = ['_passwd', 'identity-service:', 'ssl-synced-units'] self.peer_echo.assert_called_with(includes=whitelist) ssh_authorized_peers.assert_called_with( user=self.ssh_user, group='keystone', peer_interface='cluster', ensure_local_user=True) - self.assertTrue(mock_synchronize_ca.called) + self.assertFalse(mock_synchronize_ca.called) self.assertTrue(configs.write_all.called) def test_ha_joined(self): @@ -480,31 +476,36 @@ class KeystoneRelationTests(CharmTestCase): self.relation_set.assert_called_with(**args) @patch('keystone_utils.log') + @patch('keystone_utils.is_elected_leader') @patch('keystone_utils.peer_units') @patch('keystone_utils.synchronize_ca') @patch.object(hooks, 'CONFIGS') def test_ha_relation_changed_not_clustered_not_leader(self, configs, mock_synchronize_ca, mock_peer_units, + mock_is_leader, mock_log): mock_peer_units.return_value = None + mock_is_leader.return_value = False self.relation_get.return_value = False self.is_elected_leader.return_value = False hooks.ha_changed() self.assertTrue(configs.write_all.called) - self.assertTrue(mock_synchronize_ca.called) + self.assertFalse(mock_synchronize_ca.called) @patch('keystone_utils.log') + @patch('keystone_utils.is_elected_leader') @patch('keystone_utils.peer_units') - @patch('keystone_utils.synchronize_ca') @patch.object(hooks, 'identity_changed') @patch.object(hooks, 'CONFIGS') def test_ha_relation_changed_clustered_leader(self, configs, identity_changed, - mock_synchronize_ca, - mock_peer_units, mock_log): + mock_peer_units, + mock_is_elected_leader, + mock_log): mock_peer_units.return_value = None + mock_is_elected_leader.return_value = False self.relation_get.return_value = True self.is_elected_leader.return_value = True self.relation_ids.return_value = ['identity-service:0'] @@ -513,18 +514,20 @@ class KeystoneRelationTests(CharmTestCase): hooks.ha_changed() self.assertTrue(configs.write_all.called) self.log.assert_called_with( - 'Cluster configured, notifying other services and updating ' - 'keystone endpoint configuration') + 'Firing identity_changed hook for all related services.') identity_changed.assert_called_with( relation_id='identity-service:0', remote_unit='unit/0') - self.assertTrue(mock_synchronize_ca.called) @patch('keystone_utils.log') + @patch('keystone_utils.is_elected_leader') @patch('keystone_utils.peer_units') @patch.object(hooks, 'CONFIGS') - def test_configure_https_enable(self, configs, mock_peer_units, mock_log): + def test_configure_https_enable(self, configs, mock_peer_units, + mock_is_elected_leader, + mock_log): mock_peer_units.return_value = None + mock_is_elected_leader.return_value = False configs.complete_contexts = MagicMock() configs.complete_contexts.return_value = ['https'] configs.write = MagicMock() @@ -535,10 +538,14 @@ class KeystoneRelationTests(CharmTestCase): self.check_call.assert_called_with(cmd) @patch('keystone_utils.log') + @patch('keystone_utils.is_elected_leader') @patch('keystone_utils.peer_units') @patch.object(hooks, 'CONFIGS') - def test_configure_https_disable(self, configs, mock_peer_units, mock_log): + def test_configure_https_disable(self, configs, mock_peer_units, + mock_is_elected_leader, + mock_log): mock_peer_units.return_value = None + mock_is_elected_leader.return_value = False configs.complete_contexts = MagicMock() configs.complete_contexts.return_value = [''] configs.write = MagicMock() @@ -548,19 +555,19 @@ class KeystoneRelationTests(CharmTestCase): cmd = ['a2dissite', 'openstack_https_frontend'] self.check_call.assert_called_with(cmd) + @patch('keystone_utils.log') @patch('keystone_utils.relation_ids') @patch('keystone_utils.is_elected_leader') - @patch('keystone_utils.is_sync_master') @patch('keystone_utils.update_hash_from_path') @patch('keystone_utils.synchronize_ca') @patch.object(unison, 'ssh_authorized_peers') def test_upgrade_charm_leader(self, ssh_authorized_peers, mock_synchronize_ca, mock_update_hash_from_path, - mock_is_sync_master, mock_is_elected_leader, - mock_relation_ids): + mock_is_elected_leader, + mock_relation_ids, + mock_log): mock_relation_ids.return_value = [] - mock_is_sync_master.return_value = True mock_is_elected_leader.return_value = True # Ensure always returns diff mock_update_hash_from_path.side_effect = \ @@ -575,25 +582,21 @@ class KeystoneRelationTests(CharmTestCase): peer_interface='cluster', ensure_local_user=True) self.assertTrue(mock_synchronize_ca.called) self.log.assert_called_with( - 'Cluster leader - ensuring endpoint configuration' - ' is up to date') + 'Firing identity_changed hook for all related services.') self.assertTrue(self.ensure_initial_admin.called) + @patch('keystone_utils.log') @patch('keystone_utils.relation_ids') @patch('keystone_utils.is_elected_leader') - @patch('keystone_utils.is_sync_master') @patch('keystone_utils.update_hash_from_path') - @patch('keystone_utils.synchronize_ca') @patch.object(unison, 'ssh_authorized_peers') def test_upgrade_charm_not_leader(self, ssh_authorized_peers, - mock_synchronize_ca, mock_update_hash_from_path, - mock_is_sync_master, mock_is_elected_leader, - mock_relation_ids): + mock_relation_ids, + mock_log): mock_relation_ids.return_value = [] - mock_is_sync_master.return_value = True - mock_is_elected_leader.return_value = True + mock_is_elected_leader.return_value = False # Ensure always returns diff mock_update_hash_from_path.side_effect = \ lambda hash, *args, **kwargs: hash.update(str(uuid.uuid4())) @@ -605,6 +608,5 @@ class KeystoneRelationTests(CharmTestCase): ssh_authorized_peers.assert_called_with( user=self.ssh_user, group='keystone', peer_interface='cluster', ensure_local_user=True) - self.assertTrue(mock_synchronize_ca.called) self.assertFalse(self.log.called) self.assertFalse(self.ensure_initial_admin.called) diff --git a/unit_tests/test_keystone_utils.py b/unit_tests/test_keystone_utils.py index 53a874cd..b94f0f26 100644 --- a/unit_tests/test_keystone_utils.py +++ b/unit_tests/test_keystone_utils.py @@ -28,7 +28,6 @@ TO_PATCH = [ 'configure_installation_source', 'is_elected_leader', 'https', - 'is_clustered', 'peer_store_and_set', 'service_stop', 'service_start', @@ -200,7 +199,6 @@ class TestKeystoneUtils(CharmTestCase): self.resolve_address.return_value = '10.0.0.3' self.test_config.set('admin-port', 80) self.test_config.set('service-port', 81) - self.is_clustered.return_value = False self.https.return_value = False self.test_config.set('https-service-endpoints', 'False') self.get_local_endpoint.return_value = 'http://localhost:80/v2.0/' @@ -233,9 +231,9 @@ class TestKeystoneUtils(CharmTestCase): 'auth_port': 80, 'service_username': 'keystone', 'service_password': 'password', 'service_tenant': 'tenant', - 'https_keystone': 'False', - 'ssl_cert': '', 'ssl_key': '', - 'ca_cert': '', 'auth_host': '10.0.0.3', + 'https_keystone': None, + 'ssl_cert': None, 'ssl_key': None, + 'ca_cert': None, 'auth_host': '10.0.0.3', 'service_host': '10.0.0.3', 'auth_protocol': 'http', 'service_protocol': 'http', 'service_tenant_id': 'tenant_id'} From 03b0fd66f89c69c244523ad88505ef6ca25af4d8 Mon Sep 17 00:00:00 2001 From: Edward Hope-Morley Date: Tue, 13 Jan 2015 16:01:25 +0000 Subject: [PATCH 09/48] add leader protection to cluster-changed hook --- hooks/keystone_hooks.py | 63 ++++++++++++++++++------------- hooks/keystone_utils.py | 4 +- unit_tests/test_keystone_hooks.py | 52 ++++++++++--------------- 3 files changed, 56 insertions(+), 63 deletions(-) diff --git a/hooks/keystone_hooks.py b/hooks/keystone_hooks.py index 3397fd22..eeea2923 100755 --- a/hooks/keystone_hooks.py +++ b/hooks/keystone_hooks.py @@ -18,7 +18,6 @@ from charmhelpers.core.hookenv import ( log, local_unit, DEBUG, - INFO, WARNING, ERROR, relation_get, @@ -171,6 +170,7 @@ def pgsql_db_joined(): def update_all_identity_relation_units(): + CONFIGS.write_all() try: migrate_database() except Exception as exc: @@ -184,6 +184,11 @@ def update_all_identity_relation_units(): identity_changed(relation_id=rid, remote_unit=unit) +@synchronize_ca_if_changed(force=True) +def update_all_identity_relation_units_force_sync(): + update_all_identity_relation_units() + + @hooks.hook('shared-db-relation-changed') @restart_on_change(restart_map()) @synchronize_ca_if_changed() @@ -269,36 +274,40 @@ def identity_changed(relation_id=None, remote_unit=None): @hooks.hook('cluster-relation-joined') -def cluster_joined(relation_id=None): +def cluster_joined(): unison.ssh_authorized_peers(user=SSH_USER, group='juju_keystone', peer_interface='cluster', ensure_local_user=True) + + settings = {} + for addr_type in ADDRESS_TYPES: address = get_address_in_network( config('os-{}-network'.format(addr_type)) ) if address: - relation_set( - relation_id=relation_id, - relation_settings={'{}-address'.format(addr_type): address} - ) + settings['{}-address'.format(addr_type)] = address if config('prefer-ipv6'): private_addr = get_ipv6_addr(exc_list=[config('vip')])[0] - relation_set(relation_id=relation_id, - relation_settings={'private-address': private_addr}) + settings['private-address'] = private_addr + + # This will be consumed by -changed for ssl sync + settings['ssl-sync-required-%s' % (remote_unit().replace('/', '-'))] = '1' + + relation_set(relation_settings=settings) -@synchronize_ca_if_changed(fatal=True) -def identity_updates_with_ssl_sync(): - CONFIGS.write_all() - update_all_identity_relation_units() +def get_new_peers(peers): + units = [] + key = re.compile("^ssl-sync-required-(.+)") + for peer in peers: + res = re.search(key, peer) + if res: + units.append(res.group(1)) - -@synchronize_ca_if_changed(force=True, fatal=True) -def identity_updates_with_forced_ssl_sync(): - identity_updates_with_ssl_sync() + return units @hooks.hook('cluster-relation-changed', @@ -314,19 +323,19 @@ def cluster_changed(): peer_interface='cluster', ensure_local_user=True) - synced_units = relation_get(attribute='ssl-synced-units', - unit=local_unit()) - if not synced_units or (remote_unit() not in synced_units): - log("Peer '%s' not in list of synced units (%s)" % - (remote_unit(), synced_units), level=INFO) - identity_updates_with_forced_ssl_sync() + if is_elected_leader(CLUSTER_RES): + new_peers = get_new_peers(peer_units()) + if new_peers: + log("New peers joined and need syncing - %s" % + (', '.join(new_peers)), level=DEBUG) + update_all_identity_relation_units_force_sync() + # Clear + relation_set(relation_settings={p: None for p in new_peers}) + else: + update_all_identity_relation_units() else: - identity_updates_with_ssl_sync() + CONFIGS.write_all() - echo_whitelist.append('ssl-synced-units') - - # ssl cert sync must be done BEFORE this to reduce the risk of feedback - # loops in cluster relation peer_echo(includes=echo_whitelist) diff --git a/hooks/keystone_utils.py b/hooks/keystone_utils.py index 431653f8..a1c8a416 100644 --- a/hooks/keystone_utils.py +++ b/hooks/keystone_utils.py @@ -19,7 +19,6 @@ from charmhelpers.contrib.hahelpers.cluster import( is_elected_leader, determine_api_port, https, - peer_units, ) from charmhelpers.contrib.openstack import context, templating @@ -809,8 +808,7 @@ def synchronize_ca(fatal=False): level=DEBUG) log("Sync complete", level=DEBUG) - return {'restart-services-trigger': hash, - 'ssl-synced-units': peer_units()} + return {'restart-services-trigger': hash} def update_hash_from_path(hash, path, recurse_depth=10): diff --git a/unit_tests/test_keystone_hooks.py b/unit_tests/test_keystone_hooks.py index d3a1daef..02b4232d 100644 --- a/unit_tests/test_keystone_hooks.py +++ b/unit_tests/test_keystone_hooks.py @@ -34,7 +34,6 @@ TO_PATCH = [ 'relation_set', 'relation_get', 'related_units', - 'remote_unit', 'unit_get', 'peer_echo', # charmhelpers.core.host @@ -162,13 +161,11 @@ class KeystoneRelationTests(CharmTestCase): @patch('keystone_utils.log') @patch('keystone_utils.is_elected_leader') - @patch('keystone_utils.peer_units') @patch.object(hooks, 'CONFIGS') - def test_db_changed_missing_relation_data(self, configs, mock_peer_units, + def test_db_changed_missing_relation_data(self, configs, mock_is_elected_leader, mock_log): mock_is_elected_leader.return_value = False - mock_peer_units.return_value = None configs.complete_contexts = MagicMock() configs.complete_contexts.return_value = [] hooks.db_changed() @@ -205,13 +202,11 @@ class KeystoneRelationTests(CharmTestCase): @patch('keystone_utils.log') @patch('keystone_utils.is_elected_leader') - @patch('keystone_utils.peer_units') @patch.object(hooks, 'CONFIGS') @patch.object(hooks, 'identity_changed') def test_db_changed_allowed(self, identity_changed, configs, - mock_peer_units, mock_is_elected_leader, + mock_is_elected_leader, mock_log): - mock_peer_units.return_value = None self.relation_ids.return_value = ['identity-service:0'] self.related_units.return_value = ['unit/0'] @@ -226,13 +221,10 @@ class KeystoneRelationTests(CharmTestCase): @patch('keystone_utils.log') @patch('keystone_utils.is_elected_leader') - @patch('keystone_utils.peer_units') @patch.object(hooks, 'CONFIGS') @patch.object(hooks, 'identity_changed') def test_db_changed_not_allowed(self, identity_changed, configs, - mock_peer_units, mock_is_elected_leader, - mock_log): - mock_peer_units.return_value = None + mock_is_elected_leader, mock_log): self.relation_ids.return_value = ['identity-service:0'] self.related_units.return_value = ['unit/0'] @@ -245,13 +237,10 @@ class KeystoneRelationTests(CharmTestCase): @patch('keystone_utils.log') @patch('keystone_utils.is_elected_leader') - @patch('keystone_utils.peer_units') @patch.object(hooks, 'CONFIGS') @patch.object(hooks, 'identity_changed') def test_postgresql_db_changed(self, identity_changed, configs, - mock_peer_units, mock_is_elected_leader, - mock_log): - mock_peer_units.return_value = None + mock_is_elected_leader, mock_log): self.relation_ids.return_value = ['identity-service:0'] self.related_units.return_value = ['unit/0'] @@ -389,10 +378,12 @@ class KeystoneRelationTests(CharmTestCase): 'identity-service:0', 'unit/0') + @patch.object(hooks, 'remote_unit') @patch('keystone_utils.log') @patch('keystone_utils.is_elected_leader') def test_identity_changed_no_leader(self, mock_is_elected_leader, - mock_log): + mock_log, mock_remote_unit): + mock_remote_unit.return_value = 'unit/0' self.is_elected_leader.return_value = False hooks.identity_changed( relation_id='identity-service:0', @@ -401,13 +392,19 @@ class KeystoneRelationTests(CharmTestCase): self.log.assert_called_with( 'Deferring identity_changed() to service leader.') + @patch.object(hooks, 'remote_unit') + @patch.object(hooks, 'peer_units') @patch.object(unison, 'ssh_authorized_peers') - def test_cluster_joined(self, ssh_authorized_peers): + def test_cluster_joined(self, ssh_authorized_peers, mock_peer_units, + mock_remote_unit): + mock_remote_unit.return_value = 'unit/0' + mock_peer_units.return_value = ['unit/0'] hooks.cluster_joined() ssh_authorized_peers.assert_called_with( user=self.ssh_user, group='juju_keystone', peer_interface='cluster', ensure_local_user=True) + @patch.object(hooks, 'peer_units') @patch('keystone_utils.log') @patch('keystone_utils.is_elected_leader') @patch('keystone_utils.synchronize_ca') @@ -417,10 +414,11 @@ class KeystoneRelationTests(CharmTestCase): def test_cluster_changed(self, configs, ssh_authorized_peers, check_peer_actions, mock_synchronize_ca, mock_is_elected_leader, - mock_log): + mock_log, mock_peer_units): + mock_peer_units.return_value = ['unit/0'] mock_is_elected_leader.return_value = False hooks.cluster_changed() - whitelist = ['_passwd', 'identity-service:', 'ssl-synced-units'] + whitelist = ['_passwd', 'identity-service:'] self.peer_echo.assert_called_with(includes=whitelist) ssh_authorized_peers.assert_called_with( user=self.ssh_user, group='keystone', @@ -477,15 +475,12 @@ class KeystoneRelationTests(CharmTestCase): @patch('keystone_utils.log') @patch('keystone_utils.is_elected_leader') - @patch('keystone_utils.peer_units') @patch('keystone_utils.synchronize_ca') @patch.object(hooks, 'CONFIGS') def test_ha_relation_changed_not_clustered_not_leader(self, configs, mock_synchronize_ca, - mock_peer_units, mock_is_leader, mock_log): - mock_peer_units.return_value = None mock_is_leader.return_value = False self.relation_get.return_value = False self.is_elected_leader.return_value = False @@ -496,15 +491,12 @@ class KeystoneRelationTests(CharmTestCase): @patch('keystone_utils.log') @patch('keystone_utils.is_elected_leader') - @patch('keystone_utils.peer_units') @patch.object(hooks, 'identity_changed') @patch.object(hooks, 'CONFIGS') def test_ha_relation_changed_clustered_leader(self, configs, identity_changed, - mock_peer_units, mock_is_elected_leader, mock_log): - mock_peer_units.return_value = None mock_is_elected_leader.return_value = False self.relation_get.return_value = True self.is_elected_leader.return_value = True @@ -521,12 +513,9 @@ class KeystoneRelationTests(CharmTestCase): @patch('keystone_utils.log') @patch('keystone_utils.is_elected_leader') - @patch('keystone_utils.peer_units') @patch.object(hooks, 'CONFIGS') - def test_configure_https_enable(self, configs, mock_peer_units, - mock_is_elected_leader, + def test_configure_https_enable(self, configs, mock_is_elected_leader, mock_log): - mock_peer_units.return_value = None mock_is_elected_leader.return_value = False configs.complete_contexts = MagicMock() configs.complete_contexts.return_value = ['https'] @@ -539,12 +528,9 @@ class KeystoneRelationTests(CharmTestCase): @patch('keystone_utils.log') @patch('keystone_utils.is_elected_leader') - @patch('keystone_utils.peer_units') @patch.object(hooks, 'CONFIGS') - def test_configure_https_disable(self, configs, mock_peer_units, - mock_is_elected_leader, + def test_configure_https_disable(self, configs, mock_is_elected_leader, mock_log): - mock_peer_units.return_value = None mock_is_elected_leader.return_value = False configs.complete_contexts = MagicMock() configs.complete_contexts.return_value = [''] From c05f6a044764b432627bf2e70722457da00f52b5 Mon Sep 17 00:00:00 2001 From: Edward Hope-Morley Date: Tue, 13 Jan 2015 22:16:46 +0000 Subject: [PATCH 10/48] validate echoed peer data --- hooks/keystone_hooks.py | 52 ++++++++++++++++++++++++++++--- hooks/keystone_utils.py | 15 +++++---- unit_tests/test_keystone_hooks.py | 9 ++++-- unit_tests/test_keystone_utils.py | 6 ++-- 4 files changed, 66 insertions(+), 16 deletions(-) diff --git a/hooks/keystone_hooks.py b/hooks/keystone_hooks.py index eeea2923..5b554e2d 100755 --- a/hooks/keystone_hooks.py +++ b/hooks/keystone_hooks.py @@ -88,6 +88,12 @@ from charmhelpers.contrib.openstack.context import ADDRESS_TYPES from charmhelpers.contrib.charmsupport import nrpe +from charmhelpers.contrib.openstack.ip import ( + resolve_address, + PUBLIC, + ADMIN, +) + hooks = Hooks() CONFIGS = register_configs() @@ -310,14 +316,52 @@ def get_new_peers(peers): return units +def apply_echo_filters(settings, echo_whitelist): + """Filter settings to be peer_echo'ed. + + We may have received some data that we don't want to re-echo so filter + out unwanted keys and provide overrides. + + Returns: + tuple(filtered list of keys to be echoed, overrides for keys omitted) + """ + filtered = [] + overrides = {} + for key in settings.iterkeys(): + for ekey in echo_whitelist: + if ekey in key: + if ekey == 'identity-service:': + auth_host = resolve_address(ADMIN) + service_host = resolve_address(PUBLIC) + if (key.endswith('auth_host') and + settings[key] != auth_host): + overrides[key] = auth_host + continue + elif (key.endswith('service_host') and + settings[key] != service_host): + overrides[key] = service_host + continue + + filtered.append(key) + + return filtered, overrides + + @hooks.hook('cluster-relation-changed', 'cluster-relation-departed') @restart_on_change(restart_map(), stopstart=True) def cluster_changed(): - check_peer_actions() - + settings = relation_get() # NOTE(jamespage) re-echo passwords for peer storage - echo_whitelist = ['_passwd', 'identity-service:'] + echo_whitelist, overrides = \ + apply_echo_filters(settings, ['_passwd', 'identity-service:']) + log("Peer echo overrides: %s" % (overrides), level=DEBUG) + relation_set(**overrides) + if echo_whitelist: + log("Peer echo whitelist: %s" % (echo_whitelist), level=DEBUG) + peer_echo(includes=echo_whitelist) + + check_peer_actions() unison.ssh_authorized_peers(user=SSH_USER, group='keystone', peer_interface='cluster', @@ -336,8 +380,6 @@ def cluster_changed(): else: CONFIGS.write_all() - peer_echo(includes=echo_whitelist) - @hooks.hook('ha-relation-joined') def ha_joined(): diff --git a/hooks/keystone_utils.py b/hooks/keystone_utils.py index a1c8a416..ea947a2f 100644 --- a/hooks/keystone_utils.py +++ b/hooks/keystone_utils.py @@ -672,7 +672,7 @@ def check_peer_actions(): source = res.group(1) action = res.group(2) - # Don't execute actions requested byu this unit. + # Don't execute actions requested by this unit. if local_unit().replace('.', '-') != source: if action == 'restart': log("Running action='%s' on service '%s'" % @@ -692,7 +692,10 @@ def check_peer_actions(): else: log("Unknown action flag=%s" % (flag), level=WARNING) - os.remove(flagfile) + try: + os.remove(flagfile) + except: + pass def create_peer_service_actions(action, services): @@ -1080,10 +1083,10 @@ def add_service_to_keystone(relation_id=None, remote_unit=None): "service_password": service_password, "service_tenant": service_tenant, "service_tenant_id": manager.resolve_tenant_id(service_tenant), - "https_keystone": None, - "ssl_cert": None, - "ssl_key": None, - "ca_cert": None + "https_keystone": "False", + "ssl_cert": "", + "ssl_key": "", + "ca_cert": "" } # Check if https is enabled diff --git a/unit_tests/test_keystone_hooks.py b/unit_tests/test_keystone_hooks.py index 02b4232d..4f971012 100644 --- a/unit_tests/test_keystone_hooks.py +++ b/unit_tests/test_keystone_hooks.py @@ -42,6 +42,8 @@ TO_PATCH = [ 'restart_on_change', # charmhelpers.contrib.openstack.utils 'configure_installation_source', + # charmhelpers.contrib.openstack.ip + 'resolve_address', # charmhelpers.contrib.hahelpers.cluster_utils 'is_elected_leader', 'get_hacluster_config', @@ -417,9 +419,12 @@ class KeystoneRelationTests(CharmTestCase): mock_log, mock_peer_units): mock_peer_units.return_value = ['unit/0'] mock_is_elected_leader.return_value = False + self.is_elected_leader.return_value = False + self.relation_get.return_value = {'foo_passwd': '123', + 'identity-service:16_foo': 'bar'} hooks.cluster_changed() - whitelist = ['_passwd', 'identity-service:'] - self.peer_echo.assert_called_with(includes=whitelist) + self.peer_echo.assert_called_with(includes=['foo_passwd', + 'identity-service:16_foo']) ssh_authorized_peers.assert_called_with( user=self.ssh_user, group='keystone', peer_interface='cluster', ensure_local_user=True) diff --git a/unit_tests/test_keystone_utils.py b/unit_tests/test_keystone_utils.py index b94f0f26..23608f0b 100644 --- a/unit_tests/test_keystone_utils.py +++ b/unit_tests/test_keystone_utils.py @@ -231,9 +231,9 @@ class TestKeystoneUtils(CharmTestCase): 'auth_port': 80, 'service_username': 'keystone', 'service_password': 'password', 'service_tenant': 'tenant', - 'https_keystone': None, - 'ssl_cert': None, 'ssl_key': None, - 'ca_cert': None, 'auth_host': '10.0.0.3', + 'https_keystone': 'False', + 'ssl_cert': '', 'ssl_key': '', + 'ca_cert': '', 'auth_host': '10.0.0.3', 'service_host': '10.0.0.3', 'auth_protocol': 'http', 'service_protocol': 'http', 'service_tenant_id': 'tenant_id'} From 93d095375852f6614333a1a8fd3edaddba3cd4fd Mon Sep 17 00:00:00 2001 From: Edward Hope-Morley Date: Fri, 16 Jan 2015 14:02:29 +0000 Subject: [PATCH 11/48] add sync-master stickiness back into the mix --- hooks/keystone_hooks.py | 30 +++----- hooks/keystone_utils.py | 93 ++++++++++++++++++++++++- unit_tests/test_keystone_hooks.py | 109 ++++++++++++++++-------------- 3 files changed, 161 insertions(+), 71 deletions(-) diff --git a/hooks/keystone_hooks.py b/hooks/keystone_hooks.py index 5b554e2d..1dd92a10 100755 --- a/hooks/keystone_hooks.py +++ b/hooks/keystone_hooks.py @@ -24,7 +24,6 @@ from charmhelpers.core.hookenv import ( relation_ids, relation_set, related_units, - remote_unit, unit_get, ) @@ -64,6 +63,8 @@ from keystone_utils import ( check_peer_actions, CA_CERT_PATH, ensure_permissions, + get_ssl_sync_request_units, + clear_ssl_sync_requests, ) from charmhelpers.contrib.hahelpers.cluster import ( @@ -299,23 +300,12 @@ def cluster_joined(): private_addr = get_ipv6_addr(exc_list=[config('vip')])[0] settings['private-address'] = private_addr - # This will be consumed by -changed for ssl sync - settings['ssl-sync-required-%s' % (remote_unit().replace('/', '-'))] = '1' + # This will be consumed by cluster-relation-changed ssl master + settings['ssl-sync-required-%s' % (local_unit().replace('/', '-'))] = '1' relation_set(relation_settings=settings) -def get_new_peers(peers): - units = [] - key = re.compile("^ssl-sync-required-(.+)") - for peer in peers: - res = re.search(key, peer) - if res: - units.append(res.group(1)) - - return units - - def apply_echo_filters(settings, echo_whitelist): """Filter settings to be peer_echo'ed. @@ -354,7 +344,8 @@ def cluster_changed(): settings = relation_get() # NOTE(jamespage) re-echo passwords for peer storage echo_whitelist, overrides = \ - apply_echo_filters(settings, ['_passwd', 'identity-service:']) + apply_echo_filters(settings, ['_passwd', 'identity-service:', + 'ssl-cert-master']) log("Peer echo overrides: %s" % (overrides), level=DEBUG) relation_set(**overrides) if echo_whitelist: @@ -368,13 +359,12 @@ def cluster_changed(): ensure_local_user=True) if is_elected_leader(CLUSTER_RES): - new_peers = get_new_peers(peer_units()) - if new_peers: + units = get_ssl_sync_request_units(settings.keys()) + if units: log("New peers joined and need syncing - %s" % - (', '.join(new_peers)), level=DEBUG) + (', '.join(units)), level=DEBUG) update_all_identity_relation_units_force_sync() - # Clear - relation_set(relation_settings={p: None for p in new_peers}) + clear_ssl_sync_requests(units) else: update_all_identity_relation_units() else: diff --git a/hooks/keystone_utils.py b/hooks/keystone_utils.py index ea947a2f..bf08c5af 100644 --- a/hooks/keystone_utils.py +++ b/hooks/keystone_utils.py @@ -19,6 +19,8 @@ from charmhelpers.contrib.hahelpers.cluster import( is_elected_leader, determine_api_port, https, + peer_units, + oldest_peer, ) from charmhelpers.contrib.openstack import context, templating @@ -59,6 +61,7 @@ from charmhelpers.core.hookenv import ( relation_get, relation_set, relation_ids, + related_units, DEBUG, INFO, WARNING, @@ -736,6 +739,82 @@ def unison_sync(paths_to_sync): fatal=True) +def get_ssl_sync_request_units(rkeys): + units = [] + key = re.compile("^ssl-sync-required-(.+)") + for rkey in rkeys: + res = re.search(key, rkey) + if res: + units.append(res.group(1)) + + return units + + +def clear_ssl_sync_requests(units): + settings = {} + for unit in units: + settings['ssl-sync-required-%s' % (unit)] = None + + relation_set(settings=settings) + + +def is_ssl_cert_master(): + """Return True if this unit is ssl cert master.""" + master = None + for rid in relation_ids('cluster'): + master = relation_get(attribute='ssl-cert-master', rid=rid, + unit=local_unit()) + + return master == local_unit() + + +def ensure_ssl_cert_master(use_oldest_peer=False): + """Ensure that an ssl cert master has been elected. + + Normally the cluster leader will take control but we allow for this to be + ignored since this could be called before the cluster is ready. + """ + if not peer_units(): + log("Not syncing certs since there are no peer units.", level=INFO) + return False + + if use_oldest_peer: + elect = oldest_peer(peer_units()) + else: + elect = is_elected_leader(CLUSTER_RES) + + if elect: + masters = [] + for rid in relation_ids('cluster'): + for unit in related_units(rid): + m = relation_get(rid=rid, unit=unit, + attribute='ssl-cert-master') + if m is not None: + masters.append(m) + + # We expect all peers to echo this setting + if not masters or 'unknown' in masters: + log("Notifying peers this unit is ssl-cert-master", level=INFO) + for rid in relation_ids('cluster'): + settings = {'ssl-cert-master': local_unit()} + relation_set(relation_id=rid, relation_settings=settings) + + # Return now and wait for cluster-relation-changed (peer_echo) for + # sync. + return False + elif len(set(masters)) != 1 and local_unit() not in masters: + log("Did not get concensus from peers on who is master (%s) - " + "waiting for current master to release before self-electing" % + (masters), level=INFO) + return False + + if not is_ssl_cert_master(): + log("Not ssl cert master - skipping sync", level=INFO) + return False + + return True + + def synchronize_ca(fatal=False): """Broadcast service credentials to peers. @@ -847,7 +926,7 @@ def synchronize_ca_if_changed(force=False, fatal=False): log("Nested sync - ignoring", level=DEBUG) return f(*args, **kwargs) - if not is_elected_leader(CLUSTER_RES): + if not ensure_ssl_cert_master(): log("Not leader - ignoring sync", level=DEBUG) return f(*args, **kwargs) @@ -877,6 +956,12 @@ def synchronize_ca_if_changed(force=False, fatal=False): log("Doing forced ssl cert sync", level=DEBUG) peer_settings = synchronize_ca(fatal=fatal) + # If we are the sync master but not leader, ensure we have + # relinquished master status. + if not is_elected_leader(CLUSTER_RES): + log("Re-electing ssl cert master.", level=INFO) + peer_settings['ssl-cert-master'] = 'unknown' + if peer_settings: for rid in relation_ids('cluster'): relation_set(relation_id=rid, @@ -913,6 +998,12 @@ def get_ca(user='keystone', group='keystone'): '%s' % SSL_DIR]) subprocess.check_output(['chmod', '-R', 'g+rwx', '%s' % SSL_DIR]) + # Ensure a master has been elected and prefer this unit. Note that we + # prefer oldest peer as predicate since this action i normally only + # performed once at deploy time when the oldest peer should be the + # first to be ready. + ensure_ssl_cert_master(use_oldest_peer=True) + ssl.CA_SINGLETON.append(ca) return ssl.CA_SINGLETON[0] diff --git a/unit_tests/test_keystone_hooks.py b/unit_tests/test_keystone_hooks.py index 4f971012..62182701 100644 --- a/unit_tests/test_keystone_hooks.py +++ b/unit_tests/test_keystone_hooks.py @@ -162,12 +162,12 @@ class KeystoneRelationTests(CharmTestCase): 'is already associated a mysql one') @patch('keystone_utils.log') - @patch('keystone_utils.is_elected_leader') + @patch('keystone_utils.ensure_ssl_cert_master') @patch.object(hooks, 'CONFIGS') def test_db_changed_missing_relation_data(self, configs, - mock_is_elected_leader, + mock_ensure_ssl_cert_master, mock_log): - mock_is_elected_leader.return_value = False + mock_ensure_ssl_cert_master.return_value = False configs.complete_contexts = MagicMock() configs.complete_contexts.return_value = [] hooks.db_changed() @@ -176,11 +176,12 @@ class KeystoneRelationTests(CharmTestCase): ) @patch('keystone_utils.log') - @patch('keystone_utils.is_elected_leader') + @patch('keystone_utils.ensure_ssl_cert_master') @patch.object(hooks, 'CONFIGS') def test_postgresql_db_changed_missing_relation_data(self, configs, - mock_is_leader, + mock_ensure_leader, mock_log): + mock_ensure_leader.return_value = False configs.complete_contexts = MagicMock() configs.complete_contexts.return_value = [] hooks.pgsql_db_changed() @@ -203,12 +204,13 @@ class KeystoneRelationTests(CharmTestCase): hooks.pgsql_db_changed() @patch('keystone_utils.log') - @patch('keystone_utils.is_elected_leader') + @patch('keystone_utils.ensure_ssl_cert_master') @patch.object(hooks, 'CONFIGS') @patch.object(hooks, 'identity_changed') def test_db_changed_allowed(self, identity_changed, configs, - mock_is_elected_leader, + mock_ensure_ssl_cert_master, mock_log): + mock_ensure_ssl_cert_master.return_value = False self.relation_ids.return_value = ['identity-service:0'] self.related_units.return_value = ['unit/0'] @@ -222,11 +224,12 @@ class KeystoneRelationTests(CharmTestCase): remote_unit='unit/0') @patch('keystone_utils.log') - @patch('keystone_utils.is_elected_leader') + @patch('keystone_utils.ensure_ssl_cert_master') @patch.object(hooks, 'CONFIGS') @patch.object(hooks, 'identity_changed') def test_db_changed_not_allowed(self, identity_changed, configs, - mock_is_elected_leader, mock_log): + mock_ensure_ssl_cert_master, mock_log): + mock_ensure_ssl_cert_master.return_value = False self.relation_ids.return_value = ['identity-service:0'] self.related_units.return_value = ['unit/0'] @@ -238,11 +241,12 @@ class KeystoneRelationTests(CharmTestCase): self.assertFalse(identity_changed.called) @patch('keystone_utils.log') - @patch('keystone_utils.is_elected_leader') + @patch('keystone_utils.ensure_ssl_cert_master') @patch.object(hooks, 'CONFIGS') @patch.object(hooks, 'identity_changed') def test_postgresql_db_changed(self, identity_changed, configs, - mock_is_elected_leader, mock_log): + mock_ensure_ssl_cert_master, mock_log): + mock_ensure_ssl_cert_master.return_value = False self.relation_ids.return_value = ['identity-service:0'] self.related_units.return_value = ['unit/0'] @@ -256,7 +260,7 @@ class KeystoneRelationTests(CharmTestCase): remote_unit='unit/0') @patch('keystone_utils.log') - @patch('keystone_utils.is_elected_leader') + @patch('keystone_utils.ensure_ssl_cert_master') @patch.object(hooks, 'peer_units') @patch.object(hooks, 'ensure_permissions') @patch.object(hooks, 'cluster_joined') @@ -268,12 +272,12 @@ class KeystoneRelationTests(CharmTestCase): def test_config_changed_no_openstack_upgrade_leader( self, configure_https, identity_changed, configs, get_homedir, ensure_user, cluster_joined, - ensure_permissions, mock_peer_units, mock_is_elected_leader, + ensure_permissions, mock_peer_units, mock_ensure_ssl_cert_master, mock_log): self.openstack_upgrade_available.return_value = False self.is_elected_leader.return_value = True # avoid having to mock syncer - mock_is_elected_leader.return_value = False + mock_ensure_ssl_cert_master.return_value = False mock_peer_units.return_value = [] self.relation_ids.return_value = ['identity-service:0'] self.related_units.return_value = ['unit/0'] @@ -295,7 +299,7 @@ class KeystoneRelationTests(CharmTestCase): remote_unit='unit/0') @patch('keystone_utils.log') - @patch('keystone_utils.is_elected_leader') + @patch('keystone_utils.ensure_ssl_cert_master') @patch.object(hooks, 'ensure_permissions') @patch.object(hooks, 'cluster_joined') @patch.object(unison, 'ensure_user') @@ -306,11 +310,11 @@ class KeystoneRelationTests(CharmTestCase): def test_config_changed_no_openstack_upgrade_not_leader( self, configure_https, identity_changed, configs, get_homedir, ensure_user, cluster_joined, - ensure_permissions, mock_is_elected_leader, + ensure_permissions, mock_ensure_ssl_cert_master, mock_log): self.openstack_upgrade_available.return_value = False self.is_elected_leader.return_value = False - mock_is_elected_leader.return_value = False + mock_ensure_ssl_cert_master.return_value = False hooks.config_changed() ensure_user.assert_called_with(user=self.ssh_user, group='keystone') @@ -325,7 +329,7 @@ class KeystoneRelationTests(CharmTestCase): self.assertFalse(identity_changed.called) @patch('keystone_utils.log') - @patch('keystone_utils.is_elected_leader') + @patch('keystone_utils.ensure_ssl_cert_master') @patch.object(hooks, 'peer_units') @patch.object(hooks, 'ensure_permissions') @patch.object(hooks, 'cluster_joined') @@ -337,12 +341,12 @@ class KeystoneRelationTests(CharmTestCase): def test_config_changed_with_openstack_upgrade( self, configure_https, identity_changed, configs, get_homedir, ensure_user, cluster_joined, - ensure_permissions, mock_peer_units, mock_is_elected_leader, + ensure_permissions, mock_peer_units, mock_ensure_ssl_cert_master, mock_log): self.openstack_upgrade_available.return_value = True self.is_elected_leader.return_value = True # avoid having to mock syncer - mock_is_elected_leader.return_value = False + mock_ensure_ssl_cert_master.return_value = False mock_peer_units.return_value = [] self.relation_ids.return_value = ['identity-service:0'] self.related_units.return_value = ['unit/0'] @@ -366,13 +370,13 @@ class KeystoneRelationTests(CharmTestCase): remote_unit='unit/0') @patch('keystone_utils.log') - @patch('keystone_utils.is_elected_leader') + @patch('keystone_utils.ensure_ssl_cert_master') @patch.object(hooks, 'hashlib') @patch.object(hooks, 'send_notifications') def test_identity_changed_leader(self, mock_send_notifications, - mock_hashlib, mock_is_elected_leader, + mock_hashlib, mock_ensure_ssl_cert_master, mock_log): - mock_is_elected_leader.return_value = True + mock_ensure_ssl_cert_master.return_value = False hooks.identity_changed( relation_id='identity-service:0', remote_unit='unit/0') @@ -380,12 +384,13 @@ class KeystoneRelationTests(CharmTestCase): 'identity-service:0', 'unit/0') - @patch.object(hooks, 'remote_unit') + @patch.object(hooks, 'local_unit') @patch('keystone_utils.log') - @patch('keystone_utils.is_elected_leader') - def test_identity_changed_no_leader(self, mock_is_elected_leader, - mock_log, mock_remote_unit): - mock_remote_unit.return_value = 'unit/0' + @patch('keystone_utils.ensure_ssl_cert_master') + def test_identity_changed_no_leader(self, mock_ensure_ssl_cert_master, + mock_log, mock_local_unit): + mock_ensure_ssl_cert_master.return_value = False + mock_local_unit.return_value = 'unit/0' self.is_elected_leader.return_value = False hooks.identity_changed( relation_id='identity-service:0', @@ -394,12 +399,12 @@ class KeystoneRelationTests(CharmTestCase): self.log.assert_called_with( 'Deferring identity_changed() to service leader.') - @patch.object(hooks, 'remote_unit') + @patch.object(hooks, 'local_unit') @patch.object(hooks, 'peer_units') @patch.object(unison, 'ssh_authorized_peers') def test_cluster_joined(self, ssh_authorized_peers, mock_peer_units, - mock_remote_unit): - mock_remote_unit.return_value = 'unit/0' + mock_local_unit): + mock_local_unit.return_value = 'unit/0' mock_peer_units.return_value = ['unit/0'] hooks.cluster_joined() ssh_authorized_peers.assert_called_with( @@ -408,17 +413,17 @@ class KeystoneRelationTests(CharmTestCase): @patch.object(hooks, 'peer_units') @patch('keystone_utils.log') - @patch('keystone_utils.is_elected_leader') + @patch('keystone_utils.ensure_ssl_cert_master') @patch('keystone_utils.synchronize_ca') @patch.object(hooks, 'check_peer_actions') @patch.object(unison, 'ssh_authorized_peers') @patch.object(hooks, 'CONFIGS') def test_cluster_changed(self, configs, ssh_authorized_peers, check_peer_actions, mock_synchronize_ca, - mock_is_elected_leader, + mock_ensure_ssl_cert_master, mock_log, mock_peer_units): mock_peer_units.return_value = ['unit/0'] - mock_is_elected_leader.return_value = False + mock_ensure_ssl_cert_master.return_value = False self.is_elected_leader.return_value = False self.relation_get.return_value = {'foo_passwd': '123', 'identity-service:16_foo': 'bar'} @@ -479,14 +484,14 @@ class KeystoneRelationTests(CharmTestCase): self.relation_set.assert_called_with(**args) @patch('keystone_utils.log') - @patch('keystone_utils.is_elected_leader') + @patch('keystone_utils.ensure_ssl_cert_master') @patch('keystone_utils.synchronize_ca') @patch.object(hooks, 'CONFIGS') def test_ha_relation_changed_not_clustered_not_leader(self, configs, mock_synchronize_ca, - mock_is_leader, + mock_is_master, mock_log): - mock_is_leader.return_value = False + mock_is_master.return_value = False self.relation_get.return_value = False self.is_elected_leader.return_value = False @@ -495,14 +500,14 @@ class KeystoneRelationTests(CharmTestCase): self.assertFalse(mock_synchronize_ca.called) @patch('keystone_utils.log') - @patch('keystone_utils.is_elected_leader') + @patch('keystone_utils.ensure_ssl_cert_master') @patch.object(hooks, 'identity_changed') @patch.object(hooks, 'CONFIGS') def test_ha_relation_changed_clustered_leader(self, configs, identity_changed, - mock_is_elected_leader, + mock_ensure_ssl_cert_master, mock_log): - mock_is_elected_leader.return_value = False + mock_ensure_ssl_cert_master.return_value = False self.relation_get.return_value = True self.is_elected_leader.return_value = True self.relation_ids.return_value = ['identity-service:0'] @@ -517,11 +522,11 @@ class KeystoneRelationTests(CharmTestCase): remote_unit='unit/0') @patch('keystone_utils.log') - @patch('keystone_utils.is_elected_leader') + @patch('keystone_utils.ensure_ssl_cert_master') @patch.object(hooks, 'CONFIGS') - def test_configure_https_enable(self, configs, mock_is_elected_leader, + def test_configure_https_enable(self, configs, mock_ensure_ssl_cert_master, mock_log): - mock_is_elected_leader.return_value = False + mock_ensure_ssl_cert_master.return_value = False configs.complete_contexts = MagicMock() configs.complete_contexts.return_value = ['https'] configs.write = MagicMock() @@ -532,11 +537,12 @@ class KeystoneRelationTests(CharmTestCase): self.check_call.assert_called_with(cmd) @patch('keystone_utils.log') - @patch('keystone_utils.is_elected_leader') + @patch('keystone_utils.ensure_ssl_cert_master') @patch.object(hooks, 'CONFIGS') - def test_configure_https_disable(self, configs, mock_is_elected_leader, + def test_configure_https_disable(self, configs, + mock_ensure_ssl_cert_master, mock_log): - mock_is_elected_leader.return_value = False + mock_ensure_ssl_cert_master.return_value = False configs.complete_contexts = MagicMock() configs.complete_contexts.return_value = [''] configs.write = MagicMock() @@ -549,17 +555,20 @@ class KeystoneRelationTests(CharmTestCase): @patch('keystone_utils.log') @patch('keystone_utils.relation_ids') @patch('keystone_utils.is_elected_leader') + @patch('keystone_utils.ensure_ssl_cert_master') @patch('keystone_utils.update_hash_from_path') @patch('keystone_utils.synchronize_ca') @patch.object(unison, 'ssh_authorized_peers') def test_upgrade_charm_leader(self, ssh_authorized_peers, mock_synchronize_ca, mock_update_hash_from_path, + mock_ensure_ssl_cert_master, mock_is_elected_leader, mock_relation_ids, mock_log): + mock_is_elected_leader.return_value = False mock_relation_ids.return_value = [] - mock_is_elected_leader.return_value = True + mock_ensure_ssl_cert_master.return_value = True # Ensure always returns diff mock_update_hash_from_path.side_effect = \ lambda hash, *args, **kwargs: hash.update(str(uuid.uuid4())) @@ -578,16 +587,16 @@ class KeystoneRelationTests(CharmTestCase): @patch('keystone_utils.log') @patch('keystone_utils.relation_ids') - @patch('keystone_utils.is_elected_leader') + @patch('keystone_utils.ensure_ssl_cert_master') @patch('keystone_utils.update_hash_from_path') @patch.object(unison, 'ssh_authorized_peers') def test_upgrade_charm_not_leader(self, ssh_authorized_peers, mock_update_hash_from_path, - mock_is_elected_leader, + mock_ensure_ssl_cert_master, mock_relation_ids, mock_log): mock_relation_ids.return_value = [] - mock_is_elected_leader.return_value = False + mock_ensure_ssl_cert_master.return_value = False # Ensure always returns diff mock_update_hash_from_path.side_effect = \ lambda hash, *args, **kwargs: hash.update(str(uuid.uuid4())) From 928a2ef91576837f211d5817678ebca34004984c Mon Sep 17 00:00:00 2001 From: Edward Hope-Morley Date: Mon, 19 Jan 2015 10:45:41 +0000 Subject: [PATCH 12/48] [hopem,r=gnuoy] Set root logger level to DEBUG in /etc/logging.conf if debug is True otherwise keystone logger remains as WARNING. Closes-Bug: 1407317 --- hooks/keystone_context.py | 11 +++++++ hooks/keystone_utils.py | 5 ++++ templates/icehouse/logging.conf | 43 ++++++++++++++++++++++++++++ unit_tests/test_keystone_contexts.py | 10 +++++++ 4 files changed, 69 insertions(+) create mode 100644 templates/icehouse/logging.conf diff --git a/hooks/keystone_context.py b/hooks/keystone_context.py index 8d0d40cd..83653712 100644 --- a/hooks/keystone_context.py +++ b/hooks/keystone_context.py @@ -134,3 +134,14 @@ class KeystoneContext(context.OSContextGenerator): resolve_address(ADMIN), api_port('keystone-admin')).rstrip('v2.0') return ctxt + + +class KeystoneLoggingContext(context.OSContextGenerator): + + def __call__(self): + ctxt = {} + debug = config('debug') + if debug and debug.lower() in ['yes', 'true']: + ctxt['root_level'] = 'DEBUG' + + return ctxt diff --git a/hooks/keystone_utils.py b/hooks/keystone_utils.py index 58123c3e..fda9f4f1 100644 --- a/hooks/keystone_utils.py +++ b/hooks/keystone_utils.py @@ -100,6 +100,7 @@ API_PORTS = { } KEYSTONE_CONF = "/etc/keystone/keystone.conf" +KEYSTONE_LOGGER_CONF = "/etc/keystone/logging.conf" KEYSTONE_CONF_DIR = os.path.dirname(KEYSTONE_CONF) STORED_PASSWD = "/var/lib/keystone/keystone.passwd" STORED_TOKEN = "/var/lib/keystone/keystone.token" @@ -125,6 +126,10 @@ BASE_RESOURCE_MAP = OrderedDict([ context.BindHostContext(), context.WorkerConfigContext()], }), + (KEYSTONE_LOGGER_CONF, { + 'contexts': [keystone_context.KeystoneLoggingContext()], + 'services': BASE_SERVICES, + }), (HAPROXY_CONF, { 'contexts': [context.HAProxyContext(singlenode_mode=True), keystone_context.HAProxyContext()], diff --git a/templates/icehouse/logging.conf b/templates/icehouse/logging.conf new file mode 100644 index 00000000..331ff697 --- /dev/null +++ b/templates/icehouse/logging.conf @@ -0,0 +1,43 @@ +[loggers] +keys=root + +[formatters] +keys=normal,normal_with_name,debug + +[handlers] +keys=production,file,devel + +[logger_root] +{% if root_level -%} +level={{ root_level }} +{% else -%} +level=WARNING +{% endif -%} +handlers=file + +[handler_production] +class=handlers.SysLogHandler +level=ERROR +formatter=normal_with_name +args=(('localhost', handlers.SYSLOG_UDP_PORT), handlers.SysLogHandler.LOG_USER) + +[handler_file] +class=FileHandler +level=DEBUG +formatter=normal_with_name +args=('/var/log/keystone/keystone.log', 'a') + +[handler_devel] +class=StreamHandler +level=NOTSET +formatter=debug +args=(sys.stdout,) + +[formatter_normal] +format=%(asctime)s %(levelname)s %(message)s + +[formatter_normal_with_name] +format=(%(name)s): %(asctime)s %(levelname)s %(message)s + +[formatter_debug] +format=(%(name)s): %(asctime)s %(levelname)s %(module)s %(funcName)s %(message)s diff --git a/unit_tests/test_keystone_contexts.py b/unit_tests/test_keystone_contexts.py index 70b44b34..a10426f4 100644 --- a/unit_tests/test_keystone_contexts.py +++ b/unit_tests/test_keystone_contexts.py @@ -119,3 +119,13 @@ class TestKeystoneContexts(CharmTestCase): msg = "Multiple networks configured but net_type" \ " is None (os-public-network)." mock_log.assert_called_with(msg, level="WARNING") + + @patch.object(context, 'config') + def test_keystone_logger_context(self, mock_config): + ctxt = context.KeystoneLoggingContext() + + mock_config.return_value = None + self.assertEqual({}, ctxt()) + + mock_config.return_value = 'True' + self.assertEqual({'root_level': 'DEBUG'}, ctxt()) From b6def2a72c81736998fdc3fb321fbc0296e5a74c Mon Sep 17 00:00:00 2001 From: Edward Hope-Morley Date: Tue, 20 Jan 2015 17:07:58 +0000 Subject: [PATCH 13/48] updated README --- README.md | 56 +++++++++++++++++++++++++++++++++++++++++++++---------- 1 file changed, 46 insertions(+), 10 deletions(-) diff --git a/README.md b/README.md index 7e2935ff..a36659a4 100644 --- a/README.md +++ b/README.md @@ -1,22 +1,30 @@ -This charm provides Keystone, the Openstack identity service. It's target -platform is Ubuntu Precise + Openstack Essex. This has not been tested -using Oneiric + Diablo. +Overview +======== + +This charm provides Keystone, the Openstack identity service. It's target +platform is (ideally) Ubuntu LTS + Openstack. + +Usage +===== + +The following interfaces are provided: + + - nrpe-external-master: Used generate Nagios checks. -It provides three interfaces. - - identity-service: Openstack API endpoints request an entry in the Keystone service catalog + endpoint template catalog. When a relation is established, Keystone receives: service name, region, public_url, admin_url and internal_url. It first checks that the requested service is listed as a supported service. This list should stay updated to - support current Openstack core services. If the services is supported, - a entry in the service catalog is created, an endpoint template is + support current Openstack core services. If the service is supported, + an entry in the service catalog is created, an endpoint template is created and a admin token is generated. The other end of the relation - recieves the token as well as info on which ports Keystone is listening. + receives the token as well as info on which ports Keystone is listening + on. - keystone-service: This is currently only used by Horizon/dashboard as its interaction with Keystone is different from other Openstack API - servicies. That is, Horizon requests a Keystone role and token exists. + services. That is, Horizon requests a Keystone role and token exists. During a relation, Horizon requests its configured default role and Keystone responds with a token and the auth + admin ports on which Keystone is listening. @@ -26,9 +34,37 @@ It provides three interfaces. provision users, tenants, etc. or that otherwise automate using the Openstack cluster deployment. + - identity-notifications: Used to broadcast messages to any services + listening on the interface. + +Database +-------- + Keystone requires a database. By default, a local sqlite database is used. The charm supports relations to a shared-db via mysql-shared interface. When a new data store is configured, the charm ensures the minimum administrator credentials exist (as configured via charm configuration) -VIP is only required if you plan on multi-unit clusterming. The VIP becomes a highly-available API endpoint. +HA/Clustering +------------- + +VIP is only required if you plan on multi-unit clustering (requires relating +with hacluster charm). The VIP becomes a highly-available API endpoint. + +SSL/HTTPS +--------- + +This charm also supports SSL and HTTPS endpoints. In order to ensure SSL +certificates are only created once and distributed to all units, one unit gets +elected as an ssl-cert-master. One side-effect of this is that as units are +scaled-out the currently elected leader needs to be running in order for nodes +to sync certificates. This 'feature' is to work around the lack of native +leadership election via Juju itself, a feature that is due for release some +time soon but until then we have to rely on this. Also, if a keystone unit does +go down, it must be removed from Juju i.e. + + juju destroy-unit keystone/ + +Otherwise it will be assumed that this unit may come back at some point and +therefore must be know to be in-sync with the rest before continuing. + From 744612e8a6e61b6129351b81f4e427e4fa749086 Mon Sep 17 00:00:00 2001 From: Liam Young Date: Wed, 21 Jan 2015 14:38:54 +0000 Subject: [PATCH 14/48] Move default multicast port to avoid clash --- config.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/config.yaml b/config.yaml index 873ed7ea..f4f157ea 100644 --- a/config.yaml +++ b/config.yaml @@ -158,7 +158,7 @@ options: with the other members of the HA Cluster. ha-mcastport: type: int - default: 5403 + default: 5434 description: | Default multicast port number that will be used to communicate between HA Cluster nodes. From 208fd9ea7e38238b22fa6901eb5aa0b9c344fefe Mon Sep 17 00:00:00 2001 From: Edward Hope-Morley Date: Wed, 21 Jan 2015 16:23:15 +0000 Subject: [PATCH 15/48] ignore ssl actions if not enabled and improve support for non-ssl -> ssl --- hooks/keystone_context.py | 15 +++++----- hooks/keystone_hooks.py | 38 ++++++++++++++++++----- hooks/keystone_utils.py | 50 ++++++++++++++++++------------- unit_tests/test_keystone_hooks.py | 5 +++- 4 files changed, 73 insertions(+), 35 deletions(-) diff --git a/hooks/keystone_context.py b/hooks/keystone_context.py index 0d86c3db..5cedbc4a 100644 --- a/hooks/keystone_context.py +++ b/hooks/keystone_context.py @@ -9,7 +9,6 @@ from charmhelpers.contrib.openstack import context from charmhelpers.contrib.hahelpers.cluster import ( determine_apache_port, determine_api_port, - is_elected_leader, ) from charmhelpers.core.hookenv import ( @@ -38,8 +37,8 @@ class ApacheSSLContext(context.ApacheSSLContext): from keystone_utils import ( SSH_USER, get_ca, - CLUSTER_RES, ensure_permissions, + is_ssl_cert_master, ) ssl_dir = os.path.join('/etc/apache2/ssl/', self.service_namespace) @@ -49,8 +48,9 @@ class ApacheSSLContext(context.ApacheSSLContext): ensure_permissions(ssl_dir, user=SSH_USER, group='keystone', perms=perms) - if not is_elected_leader(CLUSTER_RES): - log("Not leader - skipping apache cert config", level=INFO) + if not is_ssl_cert_master(): + log("Not ssl-cert-master - skipping apache cert config", + level=INFO) return log("Creating apache ssl certs in %s" % (ssl_dir), level=INFO) @@ -66,12 +66,13 @@ class ApacheSSLContext(context.ApacheSSLContext): from keystone_utils import ( SSH_USER, get_ca, - CLUSTER_RES, ensure_permissions, + is_ssl_cert_master, ) - if not is_elected_leader(CLUSTER_RES): - log("Not leader - skipping apache cert config", level=INFO) + if not is_ssl_cert_master(): + log("Not ssl-cert-master - skipping apache cert config", + level=INFO) return ca = get_ca(user=SSH_USER) diff --git a/hooks/keystone_hooks.py b/hooks/keystone_hooks.py index d27ca523..6c6096e6 100755 --- a/hooks/keystone_hooks.py +++ b/hooks/keystone_hooks.py @@ -1,5 +1,6 @@ #!/usr/bin/python import hashlib +import json import os import re import stat @@ -64,7 +65,8 @@ from keystone_utils import ( CA_CERT_PATH, ensure_permissions, get_ssl_sync_request_units, - clear_ssl_sync_requests, + is_str_true, + is_ssl_cert_master, ) from charmhelpers.contrib.hahelpers.cluster import ( @@ -147,6 +149,13 @@ def config_changed(): for rid in relation_ids('identity-admin'): admin_relation_changed(rid) + # Ensure sync request is sent out (needed for upgrade to ssl from non-ssl) + settings = {} + append_ssl_sync_request(settings) + if settings: + for rid in relation_ids('cluster'): + relation_set(relation_id=rid, relation_settings=settings) + @hooks.hook('shared-db-relation-joined') def db_joined(): @@ -281,6 +290,17 @@ def identity_changed(relation_id=None, remote_unit=None): send_notifications(notifications) +def append_ssl_sync_request(settings): + """Add request to be synced to relation settings. + + This will be consumed by cluster-relation-changed ssl master. + """ + if (is_str_true(config('use-https')) or + is_str_true(config('https-service-endpoints'))): + unit = local_unit().replace('/', '-') + settings['ssl-sync-required-%s' % (unit)] = '1' + + @hooks.hook('cluster-relation-joined') def cluster_joined(): unison.ssh_authorized_peers(user=SSH_USER, @@ -301,8 +321,7 @@ def cluster_joined(): private_addr = get_ipv6_addr(exc_list=[config('vip')])[0] settings['private-address'] = private_addr - # This will be consumed by cluster-relation-changed ssl master - settings['ssl-sync-required-%s' % (local_unit().replace('/', '-'))] = '1' + append_ssl_sync_request(settings) relation_set(relation_settings=settings) @@ -359,13 +378,18 @@ def cluster_changed(): peer_interface='cluster', ensure_local_user=True) - if is_elected_leader(CLUSTER_RES): - units = get_ssl_sync_request_units(settings.keys()) - if units: + if is_elected_leader(CLUSTER_RES) or is_ssl_cert_master(): + units = get_ssl_sync_request_units() + synced_units = relation_get(attribute='ssl-synced-units', + unit=local_unit()) + if synced_units: + synced_units = json.loads(synced_units) + diff = set(units).symmetric_difference(set(synced_units)) + + if units and (not synced_units or diff): log("New peers joined and need syncing - %s" % (', '.join(units)), level=DEBUG) update_all_identity_relation_units_force_sync() - clear_ssl_sync_requests(units) else: update_all_identity_relation_units() diff --git a/hooks/keystone_utils.py b/hooks/keystone_utils.py index a17defdc..79fb389c 100644 --- a/hooks/keystone_utils.py +++ b/hooks/keystone_utils.py @@ -2,6 +2,7 @@ import glob import grp import hashlib +import json import os import pwd import re @@ -220,7 +221,7 @@ valid_services = { } -def str_is_true(value): +def is_str_true(value): if value and value.lower() in ['true', 'yes']: return True @@ -766,25 +767,24 @@ def unison_sync(paths_to_sync): fatal=True) -def get_ssl_sync_request_units(rkeys): +def get_ssl_sync_request_units(): + """Get list of units that have requested to be synced. + + NOTE: this must be called from cluster relation context. + """ units = [] - key = re.compile("^ssl-sync-required-(.+)") - for rkey in rkeys: - res = re.search(key, rkey) - if res: - units.append(res.group(1)) + for unit in related_units(): + settings = relation_get(unit=unit) or {} + rkeys = settings.keys() + key = re.compile("^ssl-sync-required-(.+)") + for rkey in rkeys: + res = re.search(key, rkey) + if res: + units.append(res.group(1)) return units -def clear_ssl_sync_requests(units): - settings = {} - for unit in units: - settings['ssl-sync-required-%s' % (unit)] = None - - relation_set(settings=settings) - - def is_ssl_cert_master(): """Return True if this unit is ssl cert master.""" master = None @@ -801,6 +801,12 @@ def ensure_ssl_cert_master(use_oldest_peer=False): Normally the cluster leader will take control but we allow for this to be ignored since this could be called before the cluster is ready. """ + # Don't do anything if we are not in ssl/https mode + if not (is_str_true(config('use-https')) or + is_str_true(config('https-service-endpoints'))): + log("SSL/HTTPS is NOT enabled", level=DEBUG) + return False + if not peer_units(): log("Not syncing certs since there are no peer units.", level=INFO) return False @@ -855,13 +861,13 @@ def synchronize_ca(fatal=False): """ paths_to_sync = [SYNC_FLAGS_DIR] - if str_is_true(config('https-service-endpoints')): + if is_str_true(config('https-service-endpoints')): log("Syncing all endpoint certs since https-service-endpoints=True", level=DEBUG) paths_to_sync.append(SSL_DIR) paths_to_sync.append(APACHE_SSL_DIR) paths_to_sync.append(CA_CERT_PATH) - elif str_is_true(config('use-https')): + elif is_str_true(config('use-https')): log("Syncing keystone-endpoint certs since use-https=True", level=DEBUG) paths_to_sync.append(APACHE_SSL_DIR) @@ -879,6 +885,9 @@ def synchronize_ca(fatal=False): create_peer_service_actions('restart', ['apache2']) create_peer_actions(['update-ca-certificates']) + # Format here needs to match that used when peers request sync + synced_units = [unit.replace('/', '-') for unit in peer_units()] + retries = 3 while True: hash1 = hashlib.sha256() @@ -917,7 +926,8 @@ def synchronize_ca(fatal=False): level=DEBUG) log("Sync complete", level=DEBUG) - return {'restart-services-trigger': hash} + return {'restart-services-trigger': hash, + 'ssl-synced-units': json.dumps(synced_units)} def update_hash_from_path(hash, path, recurse_depth=10): @@ -1075,7 +1085,7 @@ def add_service_to_keystone(relation_id=None, remote_unit=None): relation_data["auth_port"] = config('admin-port') relation_data["service_port"] = config('service-port') relation_data["region"] = config('region') - if str_is_true(config('https-service-endpoints')): + if is_str_true(config('https-service-endpoints')): # Pass CA cert as client will need it to # verify https connections ca = get_ca(user=SSH_USER) @@ -1215,7 +1225,7 @@ def add_service_to_keystone(relation_id=None, remote_unit=None): relation_data["auth_protocol"] = "http" relation_data["service_protocol"] = "http" # generate or get a new cert/key for service if set to manage certs. - if str_is_true(config('https-service-endpoints')): + if is_str_true(config('https-service-endpoints')): ca = get_ca(user=SSH_USER) # NOTE(jamespage) may have multiple cns to deal with to iterate https_cns = set(https_cns) diff --git a/unit_tests/test_keystone_hooks.py b/unit_tests/test_keystone_hooks.py index 8b44e56c..cba69d48 100644 --- a/unit_tests/test_keystone_hooks.py +++ b/unit_tests/test_keystone_hooks.py @@ -416,6 +416,7 @@ class KeystoneRelationTests(CharmTestCase): user=self.ssh_user, group='juju_keystone', peer_interface='cluster', ensure_local_user=True) + @patch.object(hooks, 'is_ssl_cert_master') @patch.object(hooks, 'peer_units') @patch('keystone_utils.log') @patch('keystone_utils.ensure_ssl_cert_master') @@ -426,7 +427,9 @@ class KeystoneRelationTests(CharmTestCase): def test_cluster_changed(self, configs, ssh_authorized_peers, check_peer_actions, mock_synchronize_ca, mock_ensure_ssl_cert_master, - mock_log, mock_peer_units): + mock_log, mock_peer_units, + mock_is_ssl_cert_master): + mock_is_ssl_cert_master.return_value = False mock_peer_units.return_value = ['unit/0'] mock_ensure_ssl_cert_master.return_value = False self.is_elected_leader.return_value = False From 4e4a42657f4dfe35bc7997d0fb156cff8bb665ed Mon Sep 17 00:00:00 2001 From: Liam Young Date: Wed, 21 Jan 2015 16:25:35 +0000 Subject: [PATCH 16/48] Write out config updates early in upgrade_charm() to make sure local endpoint is configured correctly for subsequent calls in the same hook --- hooks/keystone_hooks.py | 5 +++++ unit_tests/test_keystone_hooks.py | 3 +++ 2 files changed, 8 insertions(+) diff --git a/hooks/keystone_hooks.py b/hooks/keystone_hooks.py index 31ab4ba5..24f48b8d 100755 --- a/hooks/keystone_hooks.py +++ b/hooks/keystone_hooks.py @@ -27,6 +27,7 @@ from charmhelpers.core.hookenv import ( from charmhelpers.core.host import ( mkdir, restart_on_change, + service_reload, ) from charmhelpers.fetch import ( @@ -385,6 +386,10 @@ def configure_https(): @restart_on_change(restart_map(), stopstart=True) def upgrade_charm(): apt_install(filter_installed_packages(determine_packages())) + # Migrating to haproxy always-on config needs an early reconfigure to + # fix Bug #1413285 + CONFIGS.write_all() + service_reload('keystone') unison.ssh_authorized_peers(user=SSH_USER, group='keystone', peer_interface='cluster', diff --git a/unit_tests/test_keystone_hooks.py b/unit_tests/test_keystone_hooks.py index 4217789e..7753b2f4 100644 --- a/unit_tests/test_keystone_hooks.py +++ b/unit_tests/test_keystone_hooks.py @@ -40,6 +40,7 @@ TO_PATCH = [ 'apt_install', 'apt_update', 'restart_on_change', + 'service_reload', # charmhelpers.contrib.openstack.utils 'configure_installation_source', # charmhelpers.contrib.hahelpers.cluster_utils @@ -494,6 +495,7 @@ class KeystoneRelationTests(CharmTestCase): self.filter_installed_packages.return_value = [] hooks.upgrade_charm() self.assertTrue(self.apt_install.called) + self.service_reload.assert_called_with('keystone') ssh_authorized_peers.assert_called_with( user=self.ssh_user, group='keystone', peer_interface='cluster', ensure_local_user=True) @@ -509,6 +511,7 @@ class KeystoneRelationTests(CharmTestCase): self.filter_installed_packages.return_value = [] hooks.upgrade_charm() self.assertTrue(self.apt_install.called) + self.service_reload.assert_called_with('keystone') ssh_authorized_peers.assert_called_with( user=self.ssh_user, group='keystone', peer_interface='cluster', ensure_local_user=True) From 66174c78ab2b0598fa7f0fd083f1655d0d73125e Mon Sep 17 00:00:00 2001 From: Liam Young Date: Wed, 21 Jan 2015 16:26:50 +0000 Subject: [PATCH 17/48] [gnuoy] Revert previos commit pushed in error --- hooks/keystone_hooks.py | 5 ----- unit_tests/test_keystone_hooks.py | 3 --- 2 files changed, 8 deletions(-) diff --git a/hooks/keystone_hooks.py b/hooks/keystone_hooks.py index 24f48b8d..31ab4ba5 100755 --- a/hooks/keystone_hooks.py +++ b/hooks/keystone_hooks.py @@ -27,7 +27,6 @@ from charmhelpers.core.hookenv import ( from charmhelpers.core.host import ( mkdir, restart_on_change, - service_reload, ) from charmhelpers.fetch import ( @@ -386,10 +385,6 @@ def configure_https(): @restart_on_change(restart_map(), stopstart=True) def upgrade_charm(): apt_install(filter_installed_packages(determine_packages())) - # Migrating to haproxy always-on config needs an early reconfigure to - # fix Bug #1413285 - CONFIGS.write_all() - service_reload('keystone') unison.ssh_authorized_peers(user=SSH_USER, group='keystone', peer_interface='cluster', diff --git a/unit_tests/test_keystone_hooks.py b/unit_tests/test_keystone_hooks.py index 7753b2f4..4217789e 100644 --- a/unit_tests/test_keystone_hooks.py +++ b/unit_tests/test_keystone_hooks.py @@ -40,7 +40,6 @@ TO_PATCH = [ 'apt_install', 'apt_update', 'restart_on_change', - 'service_reload', # charmhelpers.contrib.openstack.utils 'configure_installation_source', # charmhelpers.contrib.hahelpers.cluster_utils @@ -495,7 +494,6 @@ class KeystoneRelationTests(CharmTestCase): self.filter_installed_packages.return_value = [] hooks.upgrade_charm() self.assertTrue(self.apt_install.called) - self.service_reload.assert_called_with('keystone') ssh_authorized_peers.assert_called_with( user=self.ssh_user, group='keystone', peer_interface='cluster', ensure_local_user=True) @@ -511,7 +509,6 @@ class KeystoneRelationTests(CharmTestCase): self.filter_installed_packages.return_value = [] hooks.upgrade_charm() self.assertTrue(self.apt_install.called) - self.service_reload.assert_called_with('keystone') ssh_authorized_peers.assert_called_with( user=self.ssh_user, group='keystone', peer_interface='cluster', ensure_local_user=True) From 5010dd4f8cd1a2aaf424f39a92d8ab48e5bcfcb5 Mon Sep 17 00:00:00 2001 From: Liam Young Date: Thu, 22 Jan 2015 16:09:29 +0000 Subject: [PATCH 18/48] Propogate corosync config option set in the charm to the hacluster subordinate --- hooks/keystone_hooks.py | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/hooks/keystone_hooks.py b/hooks/keystone_hooks.py index 31ab4ba5..2ba45120 100755 --- a/hooks/keystone_hooks.py +++ b/hooks/keystone_hooks.py @@ -284,7 +284,7 @@ def cluster_changed(): @hooks.hook('ha-relation-joined') -def ha_joined(): +def ha_joined(relation_id=None): cluster_config = get_hacluster_config() resources = { 'res_ks_haproxy': 'lsb:haproxy', @@ -320,7 +320,8 @@ def ha_joined(): vip_group.append(vip_key) if len(vip_group) >= 1: - relation_set(groups={'grp_ks_vips': ' '.join(vip_group)}) + relation_set(relation_id=relation_id, + groups={'grp_ks_vips': ' '.join(vip_group)}) init_services = { 'res_ks_haproxy': 'haproxy' @@ -328,7 +329,8 @@ def ha_joined(): clones = { 'cl_ks_haproxy': 'res_ks_haproxy' } - relation_set(init_services=init_services, + relation_set(relation_id=relation_id, + init_services=init_services, corosync_bindiface=cluster_config['ha-bindiface'], corosync_mcastport=cluster_config['ha-mcastport'], resources=resources, @@ -401,6 +403,8 @@ def upgrade_charm(): for unit in relation_list(r_id): identity_changed(relation_id=r_id, remote_unit=unit) + for r_id in relation_ids('ha'): + ha_joined(relation_id=r_id) CONFIGS.write_all() From 348e7a6afaa7eb85bc4ea9f6174a90a5a685624d Mon Sep 17 00:00:00 2001 From: Liam Young Date: Thu, 22 Jan 2015 16:19:44 +0000 Subject: [PATCH 19/48] Update hacluster setting from config-changed --- hooks/keystone_hooks.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/hooks/keystone_hooks.py b/hooks/keystone_hooks.py index 2ba45120..8a7e4925 100755 --- a/hooks/keystone_hooks.py +++ b/hooks/keystone_hooks.py @@ -135,6 +135,8 @@ def config_changed(): admin_relation_changed(rid) for rid in relation_ids('cluster'): cluster_joined(rid) + for r_id in relation_ids('ha'): + ha_joined(relation_id=r_id) @hooks.hook('shared-db-relation-joined') @@ -403,8 +405,6 @@ def upgrade_charm(): for unit in relation_list(r_id): identity_changed(relation_id=r_id, remote_unit=unit) - for r_id in relation_ids('ha'): - ha_joined(relation_id=r_id) CONFIGS.write_all() From f57638d8fe582884919985882c1ddaa9f6563b27 Mon Sep 17 00:00:00 2001 From: Edward Hope-Morley Date: Thu, 22 Jan 2015 18:44:33 +0000 Subject: [PATCH 20/48] [hopem, r=] Wait until DB ready before performing Keystone api ops. --- hooks/keystone_hooks.py | 23 ++++++----------- hooks/keystone_utils.py | 41 +++++++++++++++++++++++++++++++ unit_tests/test_keystone_hooks.py | 13 +++++++--- unit_tests/test_keystone_utils.py | 24 ++++++++++++++++++ 4 files changed, 82 insertions(+), 19 deletions(-) diff --git a/hooks/keystone_hooks.py b/hooks/keystone_hooks.py index 6c6096e6..b39810a3 100755 --- a/hooks/keystone_hooks.py +++ b/hooks/keystone_hooks.py @@ -2,7 +2,6 @@ import hashlib import json import os -import re import stat import sys import time @@ -67,6 +66,7 @@ from keystone_utils import ( get_ssl_sync_request_units, is_str_true, is_ssl_cert_master, + is_db_ready, ) from charmhelpers.contrib.hahelpers.cluster import ( @@ -219,8 +219,7 @@ def db_changed(): # Bugs 1353135 & 1187508. Dbs can appear to be ready before the # units acl entry has been added. So, if the db supports passing # a list of permitted units then check if we're in the list. - allowed_units = relation_get('allowed_units') - if allowed_units and local_unit() not in allowed_units.split(): + if not is_db_ready(use_current_context=True): log('Allowed_units list provided and this unit not present') return # Ensure any existing service entries are updated in the @@ -249,21 +248,13 @@ def identity_changed(relation_id=None, remote_unit=None): notifications = {} if is_elected_leader(CLUSTER_RES): - # Catch database not configured error and defer until db ready - from keystoneclient.apiclient.exceptions import InternalServerError - try: - add_service_to_keystone(relation_id, remote_unit) - except InternalServerError as exc: - key = re.compile("'keystone\..+' doesn't exist") - if re.search(key, exc.message): - log("Keystone database not yet ready (InternalServerError " - "raised) - deferring until *-db relation completes.", - level=WARNING) - return - log("Unexpected exception occurred", level=ERROR) - raise + if not is_db_ready(): + log("identity-service-relation-changed hook fired before db " + "ready - deferring until db ready", level=WARNING) + return + add_service_to_keystone(relation_id, remote_unit) settings = relation_get(rid=relation_id, unit=remote_unit) service = settings.get('service', None) if service: diff --git a/hooks/keystone_utils.py b/hooks/keystone_utils.py index 4ef33c51..5ac3bfa9 100644 --- a/hooks/keystone_utils.py +++ b/hooks/keystone_utils.py @@ -62,6 +62,7 @@ from charmhelpers.core.hookenv import ( local_unit, relation_get, relation_set, + relation_id, relation_ids, related_units, DEBUG, @@ -1356,3 +1357,43 @@ def send_notifications(data, force=False): level=DEBUG) for rid in rel_ids: relation_set(relation_id=rid, relation_settings=_notifications) + + +def is_db_ready(use_current_context=False, db_rel=None): + """Database relations are expected to provide a list of 'allowed' units to + confirm that the database is ready for use by those units. + + If db relation has provided this information and local unit is a member, + returns True otherwise False. + """ + key = 'allowed_units' + db_rels = ['shared-db', 'pgsql-db'] + if db_rel: + db_rels = [db_rel] + + rel_has_units = False + + if use_current_context: + if not any([relation_id() in relation_ids(r) for r in db_rels]): + raise Exception("use_current_context=True but not in one of %s " + "rel hook contexts (currently in %s)." % + (', '.join(db_rels), relation_id())) + + allowed_units = relation_get(attribute=key) + if allowed_units and local_unit() in allowed_units.split(): + return True + else: + for rel in db_rels: + for rid in relation_ids(rel): + for unit in related_units(rid): + allowed_units = relation_get(rid=rid, unit=unit, + attribute=key) + if allowed_units and local_unit() in allowed_units.split(): + return True + + # If relation has units + return False + + # If neither relation has units then we are probably in sqllite mode return + # True. + return not rel_has_units diff --git a/unit_tests/test_keystone_hooks.py b/unit_tests/test_keystone_hooks.py index cba69d48..33fb3fe9 100644 --- a/unit_tests/test_keystone_hooks.py +++ b/unit_tests/test_keystone_hooks.py @@ -203,13 +203,15 @@ class KeystoneRelationTests(CharmTestCase): configs.write = MagicMock() hooks.pgsql_db_changed() + @patch.object(hooks, 'is_db_ready') @patch('keystone_utils.log') @patch('keystone_utils.ensure_ssl_cert_master') @patch.object(hooks, 'CONFIGS') @patch.object(hooks, 'identity_changed') def test_db_changed_allowed(self, identity_changed, configs, mock_ensure_ssl_cert_master, - mock_log): + mock_log, mock_is_db_ready): + mock_is_db_ready.return_value = True mock_ensure_ssl_cert_master.return_value = False self.relation_ids.return_value = ['identity-service:0'] self.related_units.return_value = ['unit/0'] @@ -223,12 +225,15 @@ class KeystoneRelationTests(CharmTestCase): relation_id='identity-service:0', remote_unit='unit/0') + @patch.object(hooks, 'is_db_ready') @patch('keystone_utils.log') @patch('keystone_utils.ensure_ssl_cert_master') @patch.object(hooks, 'CONFIGS') @patch.object(hooks, 'identity_changed') def test_db_changed_not_allowed(self, identity_changed, configs, - mock_ensure_ssl_cert_master, mock_log): + mock_ensure_ssl_cert_master, mock_log, + mock_is_db_ready): + mock_is_db_ready.return_value = False mock_ensure_ssl_cert_master.return_value = False self.relation_ids.return_value = ['identity-service:0'] self.related_units.return_value = ['unit/0'] @@ -374,13 +379,15 @@ class KeystoneRelationTests(CharmTestCase): remote_unit='unit/0') admin_relation_changed.assert_called_with('identity-service:0') + @patch.object(hooks, 'is_db_ready') @patch('keystone_utils.log') @patch('keystone_utils.ensure_ssl_cert_master') @patch.object(hooks, 'hashlib') @patch.object(hooks, 'send_notifications') def test_identity_changed_leader(self, mock_send_notifications, mock_hashlib, mock_ensure_ssl_cert_master, - mock_log): + mock_log, mock_is_db_ready): + mock_is_db_ready.return_value = True mock_ensure_ssl_cert_master.return_value = False hooks.identity_changed( relation_id='identity-service:0', diff --git a/unit_tests/test_keystone_utils.py b/unit_tests/test_keystone_utils.py index 4e3f3ef7..629d774a 100644 --- a/unit_tests/test_keystone_utils.py +++ b/unit_tests/test_keystone_utils.py @@ -33,6 +33,10 @@ TO_PATCH = [ 'service_start', 'relation_get', 'relation_set', + 'relation_ids', + 'relation_id', + 'local_unit', + 'related_units', 'https', 'is_relation_made', 'peer_store', @@ -345,3 +349,23 @@ class TestKeystoneUtils(CharmTestCase): isfile.return_value = False self.subprocess.check_output.return_value = 'supersecretgen' self.assertEqual(utils.get_admin_passwd(), 'supersecretgen') + + def test_is_db_ready(self): + self.relation_id.return_value = 'shared-db:0' + self.relation_ids.return_value = [self.relation_id.return_value] + self.local_unit.return_value = 'unit/0' + self.relation_get.return_value = 'unit/0' + self.assertTrue(utils.is_db_ready(use_current_context=True)) + + self.relation_ids.return_value = ['acme:0'] + self.assertRaises(utils.is_db_ready, use_current_context=True) + + self.related_units.return_value = ['unit/0'] + self.relation_ids.return_value = [self.relation_id.return_value] + self.assertTrue(utils.is_db_ready()) + + self.relation_get.return_value = 'unit/1' + self.assertFalse(utils.is_db_ready()) + + self.related_units.return_value = [] + self.assertTrue(utils.is_db_ready()) From a3a1f5908fd74da73907facff270c2d09168b924 Mon Sep 17 00:00:00 2001 From: Liam Young Date: Mon, 26 Jan 2015 09:44:47 +0000 Subject: [PATCH 21/48] [gnuoy,trivial] Pre-release charmhelper sync --- hooks/charmhelpers/__init__.py | 16 +++++++++ hooks/charmhelpers/contrib/__init__.py | 15 ++++++++ .../contrib/charmsupport/__init__.py | 15 ++++++++ .../charmhelpers/contrib/charmsupport/nrpe.py | 16 +++++++++ .../contrib/charmsupport/volumes.py | 16 +++++++++ .../contrib/hahelpers/__init__.py | 15 ++++++++ .../charmhelpers/contrib/hahelpers/apache.py | 16 +++++++++ .../charmhelpers/contrib/hahelpers/cluster.py | 22 +++++++++++- .../charmhelpers/contrib/network/__init__.py | 15 ++++++++ hooks/charmhelpers/contrib/network/ip.py | 16 +++++++++ .../contrib/openstack/__init__.py | 15 ++++++++ .../contrib/openstack/alternatives.py | 16 +++++++++ .../contrib/openstack/amulet/__init__.py | 15 ++++++++ .../contrib/openstack/amulet/deployment.py | 16 +++++++++ .../contrib/openstack/amulet/utils.py | 16 +++++++++ .../charmhelpers/contrib/openstack/context.py | 16 +++++++++ hooks/charmhelpers/contrib/openstack/ip.py | 16 +++++++++ .../charmhelpers/contrib/openstack/neutron.py | 16 +++++++++ .../contrib/openstack/templates/__init__.py | 16 +++++++++ .../contrib/openstack/templating.py | 16 +++++++++ hooks/charmhelpers/contrib/openstack/utils.py | 16 +++++++++ .../contrib/peerstorage/__init__.py | 16 +++++++++ hooks/charmhelpers/contrib/python/__init__.py | 15 ++++++++ hooks/charmhelpers/contrib/python/packages.py | 21 ++++++++++- .../charmhelpers/contrib/storage/__init__.py | 15 ++++++++ .../contrib/storage/linux/__init__.py | 15 ++++++++ .../contrib/storage/linux/ceph.py | 16 +++++++++ .../contrib/storage/linux/loopback.py | 16 +++++++++ .../charmhelpers/contrib/storage/linux/lvm.py | 16 +++++++++ .../contrib/storage/linux/utils.py | 16 +++++++++ hooks/charmhelpers/contrib/unison/__init__.py | 16 +++++++++ hooks/charmhelpers/core/__init__.py | 15 ++++++++ hooks/charmhelpers/core/decorators.py | 16 +++++++++ hooks/charmhelpers/core/fstab.py | 16 +++++++++ hooks/charmhelpers/core/hookenv.py | 16 +++++++++ hooks/charmhelpers/core/host.py | 35 ++++++++++++++++--- hooks/charmhelpers/core/services/__init__.py | 16 +++++++++ hooks/charmhelpers/core/services/base.py | 16 +++++++++ hooks/charmhelpers/core/services/helpers.py | 16 +++++++++ hooks/charmhelpers/core/sysctl.py | 16 +++++++++ hooks/charmhelpers/core/templating.py | 16 +++++++++ hooks/charmhelpers/fetch/__init__.py | 16 +++++++++ hooks/charmhelpers/fetch/archiveurl.py | 16 +++++++++ hooks/charmhelpers/fetch/bzrurl.py | 26 +++++++++++++- hooks/charmhelpers/fetch/giturl.py | 20 +++++++++++ hooks/charmhelpers/payload/__init__.py | 16 +++++++++ hooks/charmhelpers/payload/execd.py | 16 +++++++++ tests/charmhelpers/__init__.py | 16 +++++++++ tests/charmhelpers/contrib/__init__.py | 15 ++++++++ tests/charmhelpers/contrib/amulet/__init__.py | 15 ++++++++ .../charmhelpers/contrib/amulet/deployment.py | 16 +++++++++ tests/charmhelpers/contrib/amulet/utils.py | 16 +++++++++ .../contrib/openstack/__init__.py | 15 ++++++++ .../contrib/openstack/amulet/__init__.py | 15 ++++++++ .../contrib/openstack/amulet/deployment.py | 16 +++++++++ .../contrib/openstack/amulet/utils.py | 16 +++++++++ 56 files changed, 919 insertions(+), 7 deletions(-) diff --git a/hooks/charmhelpers/__init__.py b/hooks/charmhelpers/__init__.py index b46e2e23..f72e7f84 100644 --- a/hooks/charmhelpers/__init__.py +++ b/hooks/charmhelpers/__init__.py @@ -1,3 +1,19 @@ +# Copyright 2014-2015 Canonical Limited. +# +# This file is part of charm-helpers. +# +# charm-helpers is free software: you can redistribute it and/or modify +# it under the terms of the GNU Lesser General Public License version 3 as +# published by the Free Software Foundation. +# +# charm-helpers is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Lesser General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public License +# along with charm-helpers. If not, see . + # Bootstrap charm-helpers, installing its dependencies if necessary using # only standard libraries. import subprocess diff --git a/hooks/charmhelpers/contrib/__init__.py b/hooks/charmhelpers/contrib/__init__.py index e69de29b..d1400a02 100644 --- a/hooks/charmhelpers/contrib/__init__.py +++ b/hooks/charmhelpers/contrib/__init__.py @@ -0,0 +1,15 @@ +# Copyright 2014-2015 Canonical Limited. +# +# This file is part of charm-helpers. +# +# charm-helpers is free software: you can redistribute it and/or modify +# it under the terms of the GNU Lesser General Public License version 3 as +# published by the Free Software Foundation. +# +# charm-helpers is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Lesser General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public License +# along with charm-helpers. If not, see . diff --git a/hooks/charmhelpers/contrib/charmsupport/__init__.py b/hooks/charmhelpers/contrib/charmsupport/__init__.py index e69de29b..d1400a02 100644 --- a/hooks/charmhelpers/contrib/charmsupport/__init__.py +++ b/hooks/charmhelpers/contrib/charmsupport/__init__.py @@ -0,0 +1,15 @@ +# Copyright 2014-2015 Canonical Limited. +# +# This file is part of charm-helpers. +# +# charm-helpers is free software: you can redistribute it and/or modify +# it under the terms of the GNU Lesser General Public License version 3 as +# published by the Free Software Foundation. +# +# charm-helpers is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Lesser General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public License +# along with charm-helpers. If not, see . diff --git a/hooks/charmhelpers/contrib/charmsupport/nrpe.py b/hooks/charmhelpers/contrib/charmsupport/nrpe.py index f3a936d0..0fd0a9d8 100644 --- a/hooks/charmhelpers/contrib/charmsupport/nrpe.py +++ b/hooks/charmhelpers/contrib/charmsupport/nrpe.py @@ -1,3 +1,19 @@ +# Copyright 2014-2015 Canonical Limited. +# +# This file is part of charm-helpers. +# +# charm-helpers is free software: you can redistribute it and/or modify +# it under the terms of the GNU Lesser General Public License version 3 as +# published by the Free Software Foundation. +# +# charm-helpers is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Lesser General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public License +# along with charm-helpers. If not, see . + """Compatibility with the nrpe-external-master charm""" # Copyright 2012 Canonical Ltd. # diff --git a/hooks/charmhelpers/contrib/charmsupport/volumes.py b/hooks/charmhelpers/contrib/charmsupport/volumes.py index d61aa47f..320961b9 100644 --- a/hooks/charmhelpers/contrib/charmsupport/volumes.py +++ b/hooks/charmhelpers/contrib/charmsupport/volumes.py @@ -1,3 +1,19 @@ +# Copyright 2014-2015 Canonical Limited. +# +# This file is part of charm-helpers. +# +# charm-helpers is free software: you can redistribute it and/or modify +# it under the terms of the GNU Lesser General Public License version 3 as +# published by the Free Software Foundation. +# +# charm-helpers is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Lesser General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public License +# along with charm-helpers. If not, see . + ''' Functions for managing volumes in juju units. One volume is supported per unit. Subordinates may have their own storage, provided it is on its own partition. diff --git a/hooks/charmhelpers/contrib/hahelpers/__init__.py b/hooks/charmhelpers/contrib/hahelpers/__init__.py index e69de29b..d1400a02 100644 --- a/hooks/charmhelpers/contrib/hahelpers/__init__.py +++ b/hooks/charmhelpers/contrib/hahelpers/__init__.py @@ -0,0 +1,15 @@ +# Copyright 2014-2015 Canonical Limited. +# +# This file is part of charm-helpers. +# +# charm-helpers is free software: you can redistribute it and/or modify +# it under the terms of the GNU Lesser General Public License version 3 as +# published by the Free Software Foundation. +# +# charm-helpers is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Lesser General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public License +# along with charm-helpers. If not, see . diff --git a/hooks/charmhelpers/contrib/hahelpers/apache.py b/hooks/charmhelpers/contrib/hahelpers/apache.py index 6616ffff..00917195 100644 --- a/hooks/charmhelpers/contrib/hahelpers/apache.py +++ b/hooks/charmhelpers/contrib/hahelpers/apache.py @@ -1,3 +1,19 @@ +# Copyright 2014-2015 Canonical Limited. +# +# This file is part of charm-helpers. +# +# charm-helpers is free software: you can redistribute it and/or modify +# it under the terms of the GNU Lesser General Public License version 3 as +# published by the Free Software Foundation. +# +# charm-helpers is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Lesser General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public License +# along with charm-helpers. If not, see . + # # Copyright 2012 Canonical Ltd. # diff --git a/hooks/charmhelpers/contrib/hahelpers/cluster.py b/hooks/charmhelpers/contrib/hahelpers/cluster.py index 912b2fe3..9a2588b6 100644 --- a/hooks/charmhelpers/contrib/hahelpers/cluster.py +++ b/hooks/charmhelpers/contrib/hahelpers/cluster.py @@ -1,3 +1,19 @@ +# Copyright 2014-2015 Canonical Limited. +# +# This file is part of charm-helpers. +# +# charm-helpers is free software: you can redistribute it and/or modify +# it under the terms of the GNU Lesser General Public License version 3 as +# published by the Free Software Foundation. +# +# charm-helpers is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Lesser General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public License +# along with charm-helpers. If not, see . + # # Copyright 2012 Canonical Ltd. # @@ -205,19 +221,23 @@ def determine_apache_port(public_port, singlenode_mode=False): return public_port - (i * 10) -def get_hacluster_config(): +def get_hacluster_config(exclude_keys=None): ''' Obtains all relevant configuration from charm configuration required for initiating a relation to hacluster: ha-bindiface, ha-mcastport, vip + param: exclude_keys: list of setting key(s) to be excluded. returns: dict: A dict containing settings keyed by setting name. raises: HAIncompleteConfig if settings are missing. ''' settings = ['ha-bindiface', 'ha-mcastport', 'vip'] conf = {} for setting in settings: + if exclude_keys and setting in exclude_keys: + continue + conf[setting] = config_get(setting) missing = [] [missing.append(s) for s, v in six.iteritems(conf) if v is None] diff --git a/hooks/charmhelpers/contrib/network/__init__.py b/hooks/charmhelpers/contrib/network/__init__.py index e69de29b..d1400a02 100644 --- a/hooks/charmhelpers/contrib/network/__init__.py +++ b/hooks/charmhelpers/contrib/network/__init__.py @@ -0,0 +1,15 @@ +# Copyright 2014-2015 Canonical Limited. +# +# This file is part of charm-helpers. +# +# charm-helpers is free software: you can redistribute it and/or modify +# it under the terms of the GNU Lesser General Public License version 3 as +# published by the Free Software Foundation. +# +# charm-helpers is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Lesser General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public License +# along with charm-helpers. If not, see . diff --git a/hooks/charmhelpers/contrib/network/ip.py b/hooks/charmhelpers/contrib/network/ip.py index 8dc83165..98b17544 100644 --- a/hooks/charmhelpers/contrib/network/ip.py +++ b/hooks/charmhelpers/contrib/network/ip.py @@ -1,3 +1,19 @@ +# Copyright 2014-2015 Canonical Limited. +# +# This file is part of charm-helpers. +# +# charm-helpers is free software: you can redistribute it and/or modify +# it under the terms of the GNU Lesser General Public License version 3 as +# published by the Free Software Foundation. +# +# charm-helpers is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Lesser General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public License +# along with charm-helpers. If not, see . + import glob import re import subprocess diff --git a/hooks/charmhelpers/contrib/openstack/__init__.py b/hooks/charmhelpers/contrib/openstack/__init__.py index e69de29b..d1400a02 100644 --- a/hooks/charmhelpers/contrib/openstack/__init__.py +++ b/hooks/charmhelpers/contrib/openstack/__init__.py @@ -0,0 +1,15 @@ +# Copyright 2014-2015 Canonical Limited. +# +# This file is part of charm-helpers. +# +# charm-helpers is free software: you can redistribute it and/or modify +# it under the terms of the GNU Lesser General Public License version 3 as +# published by the Free Software Foundation. +# +# charm-helpers is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Lesser General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public License +# along with charm-helpers. If not, see . diff --git a/hooks/charmhelpers/contrib/openstack/alternatives.py b/hooks/charmhelpers/contrib/openstack/alternatives.py index b413259c..ef77caf3 100644 --- a/hooks/charmhelpers/contrib/openstack/alternatives.py +++ b/hooks/charmhelpers/contrib/openstack/alternatives.py @@ -1,3 +1,19 @@ +# Copyright 2014-2015 Canonical Limited. +# +# This file is part of charm-helpers. +# +# charm-helpers is free software: you can redistribute it and/or modify +# it under the terms of the GNU Lesser General Public License version 3 as +# published by the Free Software Foundation. +# +# charm-helpers is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Lesser General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public License +# along with charm-helpers. If not, see . + ''' Helper for managing alternatives for file conflict resolution ''' import subprocess diff --git a/hooks/charmhelpers/contrib/openstack/amulet/__init__.py b/hooks/charmhelpers/contrib/openstack/amulet/__init__.py index e69de29b..d1400a02 100644 --- a/hooks/charmhelpers/contrib/openstack/amulet/__init__.py +++ b/hooks/charmhelpers/contrib/openstack/amulet/__init__.py @@ -0,0 +1,15 @@ +# Copyright 2014-2015 Canonical Limited. +# +# This file is part of charm-helpers. +# +# charm-helpers is free software: you can redistribute it and/or modify +# it under the terms of the GNU Lesser General Public License version 3 as +# published by the Free Software Foundation. +# +# charm-helpers is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Lesser General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public License +# along with charm-helpers. If not, see . diff --git a/hooks/charmhelpers/contrib/openstack/amulet/deployment.py b/hooks/charmhelpers/contrib/openstack/amulet/deployment.py index f3fee074..c50d3ec6 100644 --- a/hooks/charmhelpers/contrib/openstack/amulet/deployment.py +++ b/hooks/charmhelpers/contrib/openstack/amulet/deployment.py @@ -1,3 +1,19 @@ +# Copyright 2014-2015 Canonical Limited. +# +# This file is part of charm-helpers. +# +# charm-helpers is free software: you can redistribute it and/or modify +# it under the terms of the GNU Lesser General Public License version 3 as +# published by the Free Software Foundation. +# +# charm-helpers is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Lesser General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public License +# along with charm-helpers. If not, see . + import six from charmhelpers.contrib.amulet.deployment import ( AmuletDeployment diff --git a/hooks/charmhelpers/contrib/openstack/amulet/utils.py b/hooks/charmhelpers/contrib/openstack/amulet/utils.py index 3e0cc61c..9c3d918a 100644 --- a/hooks/charmhelpers/contrib/openstack/amulet/utils.py +++ b/hooks/charmhelpers/contrib/openstack/amulet/utils.py @@ -1,3 +1,19 @@ +# Copyright 2014-2015 Canonical Limited. +# +# This file is part of charm-helpers. +# +# charm-helpers is free software: you can redistribute it and/or modify +# it under the terms of the GNU Lesser General Public License version 3 as +# published by the Free Software Foundation. +# +# charm-helpers is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Lesser General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public License +# along with charm-helpers. If not, see . + import logging import os import time diff --git a/hooks/charmhelpers/contrib/openstack/context.py b/hooks/charmhelpers/contrib/openstack/context.py index eaa89a67..c7c4cd4a 100644 --- a/hooks/charmhelpers/contrib/openstack/context.py +++ b/hooks/charmhelpers/contrib/openstack/context.py @@ -1,3 +1,19 @@ +# Copyright 2014-2015 Canonical Limited. +# +# This file is part of charm-helpers. +# +# charm-helpers is free software: you can redistribute it and/or modify +# it under the terms of the GNU Lesser General Public License version 3 as +# published by the Free Software Foundation. +# +# charm-helpers is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Lesser General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public License +# along with charm-helpers. If not, see . + import json import os import time diff --git a/hooks/charmhelpers/contrib/openstack/ip.py b/hooks/charmhelpers/contrib/openstack/ip.py index f062c807..9eabed73 100644 --- a/hooks/charmhelpers/contrib/openstack/ip.py +++ b/hooks/charmhelpers/contrib/openstack/ip.py @@ -1,3 +1,19 @@ +# Copyright 2014-2015 Canonical Limited. +# +# This file is part of charm-helpers. +# +# charm-helpers is free software: you can redistribute it and/or modify +# it under the terms of the GNU Lesser General Public License version 3 as +# published by the Free Software Foundation. +# +# charm-helpers is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Lesser General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public License +# along with charm-helpers. If not, see . + from charmhelpers.core.hookenv import ( config, unit_get, diff --git a/hooks/charmhelpers/contrib/openstack/neutron.py b/hooks/charmhelpers/contrib/openstack/neutron.py index 095cc24b..902757fe 100644 --- a/hooks/charmhelpers/contrib/openstack/neutron.py +++ b/hooks/charmhelpers/contrib/openstack/neutron.py @@ -1,3 +1,19 @@ +# Copyright 2014-2015 Canonical Limited. +# +# This file is part of charm-helpers. +# +# charm-helpers is free software: you can redistribute it and/or modify +# it under the terms of the GNU Lesser General Public License version 3 as +# published by the Free Software Foundation. +# +# charm-helpers is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Lesser General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public License +# along with charm-helpers. If not, see . + # Various utilies for dealing with Neutron and the renaming from Quantum. from subprocess import check_output diff --git a/hooks/charmhelpers/contrib/openstack/templates/__init__.py b/hooks/charmhelpers/contrib/openstack/templates/__init__.py index 0b49ad28..75876796 100644 --- a/hooks/charmhelpers/contrib/openstack/templates/__init__.py +++ b/hooks/charmhelpers/contrib/openstack/templates/__init__.py @@ -1,2 +1,18 @@ +# Copyright 2014-2015 Canonical Limited. +# +# This file is part of charm-helpers. +# +# charm-helpers is free software: you can redistribute it and/or modify +# it under the terms of the GNU Lesser General Public License version 3 as +# published by the Free Software Foundation. +# +# charm-helpers is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Lesser General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public License +# along with charm-helpers. If not, see . + # dummy __init__.py to fool syncer into thinking this is a syncable python # module diff --git a/hooks/charmhelpers/contrib/openstack/templating.py b/hooks/charmhelpers/contrib/openstack/templating.py index 33df0675..24cb272b 100644 --- a/hooks/charmhelpers/contrib/openstack/templating.py +++ b/hooks/charmhelpers/contrib/openstack/templating.py @@ -1,3 +1,19 @@ +# Copyright 2014-2015 Canonical Limited. +# +# This file is part of charm-helpers. +# +# charm-helpers is free software: you can redistribute it and/or modify +# it under the terms of the GNU Lesser General Public License version 3 as +# published by the Free Software Foundation. +# +# charm-helpers is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Lesser General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public License +# along with charm-helpers. If not, see . + import os import six diff --git a/hooks/charmhelpers/contrib/openstack/utils.py b/hooks/charmhelpers/contrib/openstack/utils.py index ddd40ce5..26259a03 100644 --- a/hooks/charmhelpers/contrib/openstack/utils.py +++ b/hooks/charmhelpers/contrib/openstack/utils.py @@ -1,5 +1,21 @@ #!/usr/bin/python +# Copyright 2014-2015 Canonical Limited. +# +# This file is part of charm-helpers. +# +# charm-helpers is free software: you can redistribute it and/or modify +# it under the terms of the GNU Lesser General Public License version 3 as +# published by the Free Software Foundation. +# +# charm-helpers is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Lesser General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public License +# along with charm-helpers. If not, see . + # Common python helper functions used for OpenStack charms. from collections import OrderedDict from functools import wraps diff --git a/hooks/charmhelpers/contrib/peerstorage/__init__.py b/hooks/charmhelpers/contrib/peerstorage/__init__.py index 773d72a2..c4af5c2d 100644 --- a/hooks/charmhelpers/contrib/peerstorage/__init__.py +++ b/hooks/charmhelpers/contrib/peerstorage/__init__.py @@ -1,3 +1,19 @@ +# Copyright 2014-2015 Canonical Limited. +# +# This file is part of charm-helpers. +# +# charm-helpers is free software: you can redistribute it and/or modify +# it under the terms of the GNU Lesser General Public License version 3 as +# published by the Free Software Foundation. +# +# charm-helpers is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Lesser General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public License +# along with charm-helpers. If not, see . + import six from charmhelpers.core.hookenv import relation_id as current_relation_id from charmhelpers.core.hookenv import ( diff --git a/hooks/charmhelpers/contrib/python/__init__.py b/hooks/charmhelpers/contrib/python/__init__.py index e69de29b..d1400a02 100644 --- a/hooks/charmhelpers/contrib/python/__init__.py +++ b/hooks/charmhelpers/contrib/python/__init__.py @@ -0,0 +1,15 @@ +# Copyright 2014-2015 Canonical Limited. +# +# This file is part of charm-helpers. +# +# charm-helpers is free software: you can redistribute it and/or modify +# it under the terms of the GNU Lesser General Public License version 3 as +# published by the Free Software Foundation. +# +# charm-helpers is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Lesser General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public License +# along with charm-helpers. If not, see . diff --git a/hooks/charmhelpers/contrib/python/packages.py b/hooks/charmhelpers/contrib/python/packages.py index 78162b1b..d848a120 100644 --- a/hooks/charmhelpers/contrib/python/packages.py +++ b/hooks/charmhelpers/contrib/python/packages.py @@ -1,6 +1,22 @@ #!/usr/bin/env python # coding: utf-8 +# Copyright 2014-2015 Canonical Limited. +# +# This file is part of charm-helpers. +# +# charm-helpers is free software: you can redistribute it and/or modify +# it under the terms of the GNU Lesser General Public License version 3 as +# published by the Free Software Foundation. +# +# charm-helpers is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Lesser General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public License +# along with charm-helpers. If not, see . + __author__ = "Jorge Niedbalski " from charmhelpers.fetch import apt_install, apt_update @@ -35,7 +51,7 @@ def pip_install_requirements(requirements, **options): pip_execute(command) -def pip_install(package, fatal=False, **options): +def pip_install(package, fatal=False, upgrade=False, **options): """Install a python package""" command = ["install"] @@ -43,6 +59,9 @@ def pip_install(package, fatal=False, **options): for option in parse_options(options, available_options): command.append(option) + if upgrade: + command.append('--upgrade') + if isinstance(package, list): command.extend(package) else: diff --git a/hooks/charmhelpers/contrib/storage/__init__.py b/hooks/charmhelpers/contrib/storage/__init__.py index e69de29b..d1400a02 100644 --- a/hooks/charmhelpers/contrib/storage/__init__.py +++ b/hooks/charmhelpers/contrib/storage/__init__.py @@ -0,0 +1,15 @@ +# Copyright 2014-2015 Canonical Limited. +# +# This file is part of charm-helpers. +# +# charm-helpers is free software: you can redistribute it and/or modify +# it under the terms of the GNU Lesser General Public License version 3 as +# published by the Free Software Foundation. +# +# charm-helpers is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Lesser General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public License +# along with charm-helpers. If not, see . diff --git a/hooks/charmhelpers/contrib/storage/linux/__init__.py b/hooks/charmhelpers/contrib/storage/linux/__init__.py index e69de29b..d1400a02 100644 --- a/hooks/charmhelpers/contrib/storage/linux/__init__.py +++ b/hooks/charmhelpers/contrib/storage/linux/__init__.py @@ -0,0 +1,15 @@ +# Copyright 2014-2015 Canonical Limited. +# +# This file is part of charm-helpers. +# +# charm-helpers is free software: you can redistribute it and/or modify +# it under the terms of the GNU Lesser General Public License version 3 as +# published by the Free Software Foundation. +# +# charm-helpers is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Lesser General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public License +# along with charm-helpers. If not, see . diff --git a/hooks/charmhelpers/contrib/storage/linux/ceph.py b/hooks/charmhelpers/contrib/storage/linux/ceph.py index 6ebeab5c..31ea7f9e 100644 --- a/hooks/charmhelpers/contrib/storage/linux/ceph.py +++ b/hooks/charmhelpers/contrib/storage/linux/ceph.py @@ -1,3 +1,19 @@ +# Copyright 2014-2015 Canonical Limited. +# +# This file is part of charm-helpers. +# +# charm-helpers is free software: you can redistribute it and/or modify +# it under the terms of the GNU Lesser General Public License version 3 as +# published by the Free Software Foundation. +# +# charm-helpers is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Lesser General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public License +# along with charm-helpers. If not, see . + # # Copyright 2012 Canonical Ltd. # diff --git a/hooks/charmhelpers/contrib/storage/linux/loopback.py b/hooks/charmhelpers/contrib/storage/linux/loopback.py index a22c3d7b..c296f098 100644 --- a/hooks/charmhelpers/contrib/storage/linux/loopback.py +++ b/hooks/charmhelpers/contrib/storage/linux/loopback.py @@ -1,3 +1,19 @@ +# Copyright 2014-2015 Canonical Limited. +# +# This file is part of charm-helpers. +# +# charm-helpers is free software: you can redistribute it and/or modify +# it under the terms of the GNU Lesser General Public License version 3 as +# published by the Free Software Foundation. +# +# charm-helpers is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Lesser General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public License +# along with charm-helpers. If not, see . + import os import re from subprocess import ( diff --git a/hooks/charmhelpers/contrib/storage/linux/lvm.py b/hooks/charmhelpers/contrib/storage/linux/lvm.py index 0aa65f4f..34b5f71a 100644 --- a/hooks/charmhelpers/contrib/storage/linux/lvm.py +++ b/hooks/charmhelpers/contrib/storage/linux/lvm.py @@ -1,3 +1,19 @@ +# Copyright 2014-2015 Canonical Limited. +# +# This file is part of charm-helpers. +# +# charm-helpers is free software: you can redistribute it and/or modify +# it under the terms of the GNU Lesser General Public License version 3 as +# published by the Free Software Foundation. +# +# charm-helpers is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Lesser General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public License +# along with charm-helpers. If not, see . + from subprocess import ( CalledProcessError, check_call, diff --git a/hooks/charmhelpers/contrib/storage/linux/utils.py b/hooks/charmhelpers/contrib/storage/linux/utils.py index c6a15e14..c8373b72 100644 --- a/hooks/charmhelpers/contrib/storage/linux/utils.py +++ b/hooks/charmhelpers/contrib/storage/linux/utils.py @@ -1,3 +1,19 @@ +# Copyright 2014-2015 Canonical Limited. +# +# This file is part of charm-helpers. +# +# charm-helpers is free software: you can redistribute it and/or modify +# it under the terms of the GNU Lesser General Public License version 3 as +# published by the Free Software Foundation. +# +# charm-helpers is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Lesser General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public License +# along with charm-helpers. If not, see . + import os import re from stat import S_ISBLK diff --git a/hooks/charmhelpers/contrib/unison/__init__.py b/hooks/charmhelpers/contrib/unison/__init__.py index 261f7cd2..f8551d10 100644 --- a/hooks/charmhelpers/contrib/unison/__init__.py +++ b/hooks/charmhelpers/contrib/unison/__init__.py @@ -1,3 +1,19 @@ +# Copyright 2014-2015 Canonical Limited. +# +# This file is part of charm-helpers. +# +# charm-helpers is free software: you can redistribute it and/or modify +# it under the terms of the GNU Lesser General Public License version 3 as +# published by the Free Software Foundation. +# +# charm-helpers is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Lesser General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public License +# along with charm-helpers. If not, see . + # Easy file synchronization among peer units using ssh + unison. # # From *both* peer relation -joined and -changed, add a call to diff --git a/hooks/charmhelpers/core/__init__.py b/hooks/charmhelpers/core/__init__.py index e69de29b..d1400a02 100644 --- a/hooks/charmhelpers/core/__init__.py +++ b/hooks/charmhelpers/core/__init__.py @@ -0,0 +1,15 @@ +# Copyright 2014-2015 Canonical Limited. +# +# This file is part of charm-helpers. +# +# charm-helpers is free software: you can redistribute it and/or modify +# it under the terms of the GNU Lesser General Public License version 3 as +# published by the Free Software Foundation. +# +# charm-helpers is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Lesser General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public License +# along with charm-helpers. If not, see . diff --git a/hooks/charmhelpers/core/decorators.py b/hooks/charmhelpers/core/decorators.py index 029a4ef4..bb05620b 100644 --- a/hooks/charmhelpers/core/decorators.py +++ b/hooks/charmhelpers/core/decorators.py @@ -1,3 +1,19 @@ +# Copyright 2014-2015 Canonical Limited. +# +# This file is part of charm-helpers. +# +# charm-helpers is free software: you can redistribute it and/or modify +# it under the terms of the GNU Lesser General Public License version 3 as +# published by the Free Software Foundation. +# +# charm-helpers is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Lesser General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public License +# along with charm-helpers. If not, see . + # # Copyright 2014 Canonical Ltd. # diff --git a/hooks/charmhelpers/core/fstab.py b/hooks/charmhelpers/core/fstab.py index 0adf0db3..be7de248 100644 --- a/hooks/charmhelpers/core/fstab.py +++ b/hooks/charmhelpers/core/fstab.py @@ -1,6 +1,22 @@ #!/usr/bin/env python # -*- coding: utf-8 -*- +# Copyright 2014-2015 Canonical Limited. +# +# This file is part of charm-helpers. +# +# charm-helpers is free software: you can redistribute it and/or modify +# it under the terms of the GNU Lesser General Public License version 3 as +# published by the Free Software Foundation. +# +# charm-helpers is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Lesser General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public License +# along with charm-helpers. If not, see . + __author__ = 'Jorge Niedbalski R. ' import io diff --git a/hooks/charmhelpers/core/hookenv.py b/hooks/charmhelpers/core/hookenv.py index 69ae4564..cf552b39 100644 --- a/hooks/charmhelpers/core/hookenv.py +++ b/hooks/charmhelpers/core/hookenv.py @@ -1,3 +1,19 @@ +# Copyright 2014-2015 Canonical Limited. +# +# This file is part of charm-helpers. +# +# charm-helpers is free software: you can redistribute it and/or modify +# it under the terms of the GNU Lesser General Public License version 3 as +# published by the Free Software Foundation. +# +# charm-helpers is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Lesser General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public License +# along with charm-helpers. If not, see . + "Interactions with the Juju environment" # Copyright 2013 Canonical Ltd. # diff --git a/hooks/charmhelpers/core/host.py b/hooks/charmhelpers/core/host.py index 5221120c..cf2cbe14 100644 --- a/hooks/charmhelpers/core/host.py +++ b/hooks/charmhelpers/core/host.py @@ -1,3 +1,19 @@ +# Copyright 2014-2015 Canonical Limited. +# +# This file is part of charm-helpers. +# +# charm-helpers is free software: you can redistribute it and/or modify +# it under the terms of the GNU Lesser General Public License version 3 as +# published by the Free Software Foundation. +# +# charm-helpers is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Lesser General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public License +# along with charm-helpers. If not, see . + """Tools for working with the host system""" # Copyright 2012 Canonical Ltd. # @@ -168,10 +184,10 @@ def mkdir(path, owner='root', group='root', perms=0o555, force=False): log("Removing non-directory file {} prior to mkdir()".format(path)) os.unlink(realpath) os.makedirs(realpath, perms) - os.chown(realpath, uid, gid) elif not path_exists: os.makedirs(realpath, perms) - os.chown(realpath, uid, gid) + os.chown(realpath, uid, gid) + os.chmod(realpath, perms) def write_file(path, content, owner='root', group='root', perms=0o444): @@ -389,6 +405,9 @@ def cmp_pkgrevno(package, revno, pkgcache=None): * 0 => Installed revno is the same as supplied arg * -1 => Installed revno is less than supplied arg + This function imports apt_cache function from charmhelpers.fetch if + the pkgcache argument is None. Be sure to add charmhelpers.fetch if + you call this function, or pass an apt_pkg.Cache() instance. ''' import apt_pkg if not pkgcache: @@ -407,13 +426,21 @@ def chdir(d): os.chdir(cur) -def chownr(path, owner, group): +def chownr(path, owner, group, follow_links=True): uid = pwd.getpwnam(owner).pw_uid gid = grp.getgrnam(group).gr_gid + if follow_links: + chown = os.chown + else: + chown = os.lchown for root, dirs, files in os.walk(path): for name in dirs + files: full = os.path.join(root, name) broken_symlink = os.path.lexists(full) and not os.path.exists(full) if not broken_symlink: - os.chown(full, uid, gid) + chown(full, uid, gid) + + +def lchownr(path, owner, group): + chownr(path, owner, group, follow_links=False) diff --git a/hooks/charmhelpers/core/services/__init__.py b/hooks/charmhelpers/core/services/__init__.py index 69dde79a..0928158b 100644 --- a/hooks/charmhelpers/core/services/__init__.py +++ b/hooks/charmhelpers/core/services/__init__.py @@ -1,2 +1,18 @@ +# Copyright 2014-2015 Canonical Limited. +# +# This file is part of charm-helpers. +# +# charm-helpers is free software: you can redistribute it and/or modify +# it under the terms of the GNU Lesser General Public License version 3 as +# published by the Free Software Foundation. +# +# charm-helpers is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Lesser General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public License +# along with charm-helpers. If not, see . + from .base import * # NOQA from .helpers import * # NOQA diff --git a/hooks/charmhelpers/core/services/base.py b/hooks/charmhelpers/core/services/base.py index 87ecb130..c5534e4c 100644 --- a/hooks/charmhelpers/core/services/base.py +++ b/hooks/charmhelpers/core/services/base.py @@ -1,3 +1,19 @@ +# Copyright 2014-2015 Canonical Limited. +# +# This file is part of charm-helpers. +# +# charm-helpers is free software: you can redistribute it and/or modify +# it under the terms of the GNU Lesser General Public License version 3 as +# published by the Free Software Foundation. +# +# charm-helpers is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Lesser General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public License +# along with charm-helpers. If not, see . + import os import re import json diff --git a/hooks/charmhelpers/core/services/helpers.py b/hooks/charmhelpers/core/services/helpers.py index 163a7932..5e3af9da 100644 --- a/hooks/charmhelpers/core/services/helpers.py +++ b/hooks/charmhelpers/core/services/helpers.py @@ -1,3 +1,19 @@ +# Copyright 2014-2015 Canonical Limited. +# +# This file is part of charm-helpers. +# +# charm-helpers is free software: you can redistribute it and/or modify +# it under the terms of the GNU Lesser General Public License version 3 as +# published by the Free Software Foundation. +# +# charm-helpers is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Lesser General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public License +# along with charm-helpers. If not, see . + import os import yaml from charmhelpers.core import hookenv diff --git a/hooks/charmhelpers/core/sysctl.py b/hooks/charmhelpers/core/sysctl.py index 0f299630..d642a371 100644 --- a/hooks/charmhelpers/core/sysctl.py +++ b/hooks/charmhelpers/core/sysctl.py @@ -1,6 +1,22 @@ #!/usr/bin/env python # -*- coding: utf-8 -*- +# Copyright 2014-2015 Canonical Limited. +# +# This file is part of charm-helpers. +# +# charm-helpers is free software: you can redistribute it and/or modify +# it under the terms of the GNU Lesser General Public License version 3 as +# published by the Free Software Foundation. +# +# charm-helpers is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Lesser General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public License +# along with charm-helpers. If not, see . + __author__ = 'Jorge Niedbalski R. ' import yaml diff --git a/hooks/charmhelpers/core/templating.py b/hooks/charmhelpers/core/templating.py index 569eaed6..97669092 100644 --- a/hooks/charmhelpers/core/templating.py +++ b/hooks/charmhelpers/core/templating.py @@ -1,3 +1,19 @@ +# Copyright 2014-2015 Canonical Limited. +# +# This file is part of charm-helpers. +# +# charm-helpers is free software: you can redistribute it and/or modify +# it under the terms of the GNU Lesser General Public License version 3 as +# published by the Free Software Foundation. +# +# charm-helpers is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Lesser General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public License +# along with charm-helpers. If not, see . + import os from charmhelpers.core import host diff --git a/hooks/charmhelpers/fetch/__init__.py b/hooks/charmhelpers/fetch/__init__.py index aceadea4..792e629a 100644 --- a/hooks/charmhelpers/fetch/__init__.py +++ b/hooks/charmhelpers/fetch/__init__.py @@ -1,3 +1,19 @@ +# Copyright 2014-2015 Canonical Limited. +# +# This file is part of charm-helpers. +# +# charm-helpers is free software: you can redistribute it and/or modify +# it under the terms of the GNU Lesser General Public License version 3 as +# published by the Free Software Foundation. +# +# charm-helpers is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Lesser General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public License +# along with charm-helpers. If not, see . + import importlib from tempfile import NamedTemporaryFile import time diff --git a/hooks/charmhelpers/fetch/archiveurl.py b/hooks/charmhelpers/fetch/archiveurl.py index 8a4624b2..d25a0ddd 100644 --- a/hooks/charmhelpers/fetch/archiveurl.py +++ b/hooks/charmhelpers/fetch/archiveurl.py @@ -1,3 +1,19 @@ +# Copyright 2014-2015 Canonical Limited. +# +# This file is part of charm-helpers. +# +# charm-helpers is free software: you can redistribute it and/or modify +# it under the terms of the GNU Lesser General Public License version 3 as +# published by the Free Software Foundation. +# +# charm-helpers is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Lesser General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public License +# along with charm-helpers. If not, see . + import os import hashlib import re diff --git a/hooks/charmhelpers/fetch/bzrurl.py b/hooks/charmhelpers/fetch/bzrurl.py index 8ef48f30..3531315a 100644 --- a/hooks/charmhelpers/fetch/bzrurl.py +++ b/hooks/charmhelpers/fetch/bzrurl.py @@ -1,3 +1,19 @@ +# Copyright 2014-2015 Canonical Limited. +# +# This file is part of charm-helpers. +# +# charm-helpers is free software: you can redistribute it and/or modify +# it under the terms of the GNU Lesser General Public License version 3 as +# published by the Free Software Foundation. +# +# charm-helpers is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Lesser General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public License +# along with charm-helpers. If not, see . + import os from charmhelpers.fetch import ( BaseFetchHandler, @@ -11,10 +27,12 @@ if six.PY3: try: from bzrlib.branch import Branch + from bzrlib import bzrdir, workingtree, errors except ImportError: from charmhelpers.fetch import apt_install apt_install("python-bzrlib") from bzrlib.branch import Branch + from bzrlib import bzrdir, workingtree, errors class BzrUrlFetchHandler(BaseFetchHandler): @@ -34,9 +52,15 @@ class BzrUrlFetchHandler(BaseFetchHandler): if url_parts.scheme == "lp": from bzrlib.plugin import load_plugins load_plugins() + try: + local_branch = bzrdir.BzrDir.create_branch_convenience(dest) + except errors.AlreadyControlDirError: + local_branch = Branch.open(dest) try: remote_branch = Branch.open(source) - remote_branch.bzrdir.sprout(dest).open_branch() + remote_branch.push(local_branch) + tree = workingtree.WorkingTree.open(dest) + tree.update() except Exception as e: raise e diff --git a/hooks/charmhelpers/fetch/giturl.py b/hooks/charmhelpers/fetch/giturl.py index f3aa2821..5376786b 100644 --- a/hooks/charmhelpers/fetch/giturl.py +++ b/hooks/charmhelpers/fetch/giturl.py @@ -1,3 +1,19 @@ +# Copyright 2014-2015 Canonical Limited. +# +# This file is part of charm-helpers. +# +# charm-helpers is free software: you can redistribute it and/or modify +# it under the terms of the GNU Lesser General Public License version 3 as +# published by the Free Software Foundation. +# +# charm-helpers is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Lesser General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public License +# along with charm-helpers. If not, see . + import os from charmhelpers.fetch import ( BaseFetchHandler, @@ -16,6 +32,8 @@ except ImportError: apt_install("python-git") from git import Repo +from git.exc import GitCommandError + class GitUrlFetchHandler(BaseFetchHandler): """Handler for git branches via generic and github URLs""" @@ -46,6 +64,8 @@ class GitUrlFetchHandler(BaseFetchHandler): mkdir(dest_dir, perms=0o755) try: self.clone(source, dest_dir, branch) + except GitCommandError as e: + raise UnhandledSource(e.message) except OSError as e: raise UnhandledSource(e.strerror) return dest_dir diff --git a/hooks/charmhelpers/payload/__init__.py b/hooks/charmhelpers/payload/__init__.py index fc9fbc08..e6f42497 100644 --- a/hooks/charmhelpers/payload/__init__.py +++ b/hooks/charmhelpers/payload/__init__.py @@ -1 +1,17 @@ +# Copyright 2014-2015 Canonical Limited. +# +# This file is part of charm-helpers. +# +# charm-helpers is free software: you can redistribute it and/or modify +# it under the terms of the GNU Lesser General Public License version 3 as +# published by the Free Software Foundation. +# +# charm-helpers is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Lesser General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public License +# along with charm-helpers. If not, see . + "Tools for working with files injected into a charm just before deployment." diff --git a/hooks/charmhelpers/payload/execd.py b/hooks/charmhelpers/payload/execd.py index 6476a75f..4d4d81a6 100644 --- a/hooks/charmhelpers/payload/execd.py +++ b/hooks/charmhelpers/payload/execd.py @@ -1,5 +1,21 @@ #!/usr/bin/env python +# Copyright 2014-2015 Canonical Limited. +# +# This file is part of charm-helpers. +# +# charm-helpers is free software: you can redistribute it and/or modify +# it under the terms of the GNU Lesser General Public License version 3 as +# published by the Free Software Foundation. +# +# charm-helpers is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Lesser General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public License +# along with charm-helpers. If not, see . + import os import sys import subprocess diff --git a/tests/charmhelpers/__init__.py b/tests/charmhelpers/__init__.py index b46e2e23..f72e7f84 100644 --- a/tests/charmhelpers/__init__.py +++ b/tests/charmhelpers/__init__.py @@ -1,3 +1,19 @@ +# Copyright 2014-2015 Canonical Limited. +# +# This file is part of charm-helpers. +# +# charm-helpers is free software: you can redistribute it and/or modify +# it under the terms of the GNU Lesser General Public License version 3 as +# published by the Free Software Foundation. +# +# charm-helpers is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Lesser General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public License +# along with charm-helpers. If not, see . + # Bootstrap charm-helpers, installing its dependencies if necessary using # only standard libraries. import subprocess diff --git a/tests/charmhelpers/contrib/__init__.py b/tests/charmhelpers/contrib/__init__.py index e69de29b..d1400a02 100644 --- a/tests/charmhelpers/contrib/__init__.py +++ b/tests/charmhelpers/contrib/__init__.py @@ -0,0 +1,15 @@ +# Copyright 2014-2015 Canonical Limited. +# +# This file is part of charm-helpers. +# +# charm-helpers is free software: you can redistribute it and/or modify +# it under the terms of the GNU Lesser General Public License version 3 as +# published by the Free Software Foundation. +# +# charm-helpers is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Lesser General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public License +# along with charm-helpers. If not, see . diff --git a/tests/charmhelpers/contrib/amulet/__init__.py b/tests/charmhelpers/contrib/amulet/__init__.py index e69de29b..d1400a02 100644 --- a/tests/charmhelpers/contrib/amulet/__init__.py +++ b/tests/charmhelpers/contrib/amulet/__init__.py @@ -0,0 +1,15 @@ +# Copyright 2014-2015 Canonical Limited. +# +# This file is part of charm-helpers. +# +# charm-helpers is free software: you can redistribute it and/or modify +# it under the terms of the GNU Lesser General Public License version 3 as +# published by the Free Software Foundation. +# +# charm-helpers is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Lesser General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public License +# along with charm-helpers. If not, see . diff --git a/tests/charmhelpers/contrib/amulet/deployment.py b/tests/charmhelpers/contrib/amulet/deployment.py index 3d3ef339..367d6b47 100644 --- a/tests/charmhelpers/contrib/amulet/deployment.py +++ b/tests/charmhelpers/contrib/amulet/deployment.py @@ -1,3 +1,19 @@ +# Copyright 2014-2015 Canonical Limited. +# +# This file is part of charm-helpers. +# +# charm-helpers is free software: you can redistribute it and/or modify +# it under the terms of the GNU Lesser General Public License version 3 as +# published by the Free Software Foundation. +# +# charm-helpers is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Lesser General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public License +# along with charm-helpers. If not, see . + import amulet import os import six diff --git a/tests/charmhelpers/contrib/amulet/utils.py b/tests/charmhelpers/contrib/amulet/utils.py index d333e63b..3464b873 100644 --- a/tests/charmhelpers/contrib/amulet/utils.py +++ b/tests/charmhelpers/contrib/amulet/utils.py @@ -1,3 +1,19 @@ +# Copyright 2014-2015 Canonical Limited. +# +# This file is part of charm-helpers. +# +# charm-helpers is free software: you can redistribute it and/or modify +# it under the terms of the GNU Lesser General Public License version 3 as +# published by the Free Software Foundation. +# +# charm-helpers is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Lesser General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public License +# along with charm-helpers. If not, see . + import ConfigParser import io import logging diff --git a/tests/charmhelpers/contrib/openstack/__init__.py b/tests/charmhelpers/contrib/openstack/__init__.py index e69de29b..d1400a02 100644 --- a/tests/charmhelpers/contrib/openstack/__init__.py +++ b/tests/charmhelpers/contrib/openstack/__init__.py @@ -0,0 +1,15 @@ +# Copyright 2014-2015 Canonical Limited. +# +# This file is part of charm-helpers. +# +# charm-helpers is free software: you can redistribute it and/or modify +# it under the terms of the GNU Lesser General Public License version 3 as +# published by the Free Software Foundation. +# +# charm-helpers is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Lesser General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public License +# along with charm-helpers. If not, see . diff --git a/tests/charmhelpers/contrib/openstack/amulet/__init__.py b/tests/charmhelpers/contrib/openstack/amulet/__init__.py index e69de29b..d1400a02 100644 --- a/tests/charmhelpers/contrib/openstack/amulet/__init__.py +++ b/tests/charmhelpers/contrib/openstack/amulet/__init__.py @@ -0,0 +1,15 @@ +# Copyright 2014-2015 Canonical Limited. +# +# This file is part of charm-helpers. +# +# charm-helpers is free software: you can redistribute it and/or modify +# it under the terms of the GNU Lesser General Public License version 3 as +# published by the Free Software Foundation. +# +# charm-helpers is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Lesser General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public License +# along with charm-helpers. If not, see . diff --git a/tests/charmhelpers/contrib/openstack/amulet/deployment.py b/tests/charmhelpers/contrib/openstack/amulet/deployment.py index f3fee074..c50d3ec6 100644 --- a/tests/charmhelpers/contrib/openstack/amulet/deployment.py +++ b/tests/charmhelpers/contrib/openstack/amulet/deployment.py @@ -1,3 +1,19 @@ +# Copyright 2014-2015 Canonical Limited. +# +# This file is part of charm-helpers. +# +# charm-helpers is free software: you can redistribute it and/or modify +# it under the terms of the GNU Lesser General Public License version 3 as +# published by the Free Software Foundation. +# +# charm-helpers is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Lesser General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public License +# along with charm-helpers. If not, see . + import six from charmhelpers.contrib.amulet.deployment import ( AmuletDeployment diff --git a/tests/charmhelpers/contrib/openstack/amulet/utils.py b/tests/charmhelpers/contrib/openstack/amulet/utils.py index 3e0cc61c..9c3d918a 100644 --- a/tests/charmhelpers/contrib/openstack/amulet/utils.py +++ b/tests/charmhelpers/contrib/openstack/amulet/utils.py @@ -1,3 +1,19 @@ +# Copyright 2014-2015 Canonical Limited. +# +# This file is part of charm-helpers. +# +# charm-helpers is free software: you can redistribute it and/or modify +# it under the terms of the GNU Lesser General Public License version 3 as +# published by the Free Software Foundation. +# +# charm-helpers is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Lesser General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public License +# along with charm-helpers. If not, see . + import logging import os import time From 54a58fca29cb21bd0f32e19fde3ba99a28887407 Mon Sep 17 00:00:00 2001 From: Edward Hope-Morley Date: Tue, 27 Jan 2015 22:21:37 +0000 Subject: [PATCH 22/48] [hopem,r=] Fixes single unit SSL. --- hooks/keystone_context.py | 55 +++++++++++++++++++++++----- hooks/keystone_hooks.py | 12 +++--- hooks/keystone_utils.py | 46 +++++++++++++++-------- unit_tests/test_keystone_contexts.py | 26 ++++++++++++- unit_tests/test_keystone_hooks.py | 6 +-- unit_tests/test_keystone_utils.py | 47 +++++++++++++++++++++++- 6 files changed, 156 insertions(+), 36 deletions(-) diff --git a/hooks/keystone_context.py b/hooks/keystone_context.py index 25d901f8..86cb407d 100644 --- a/hooks/keystone_context.py +++ b/hooks/keystone_context.py @@ -1,8 +1,13 @@ +import hashlib import os from charmhelpers.core.hookenv import config -from charmhelpers.core.host import mkdir, write_file +from charmhelpers.core.host import ( + mkdir, + write_file, + service_restart, +) from charmhelpers.contrib.openstack import context @@ -29,9 +34,31 @@ class ApacheSSLContext(context.ApacheSSLContext): def __call__(self): # late import to work around circular dependency - from keystone_utils import determine_ports + from keystone_utils import ( + determine_ports, + update_hash_from_path, + ) + + ssl_paths = [CA_CERT_PATH, + os.path.join('/etc/apache2/ssl/', + self.service_namespace)] + self.external_ports = determine_ports() - return super(ApacheSSLContext, self).__call__() + before = hashlib.sha256() + for path in ssl_paths: + update_hash_from_path(before, path) + + ret = super(ApacheSSLContext, self).__call__() + + after = hashlib.sha256() + for path in ssl_paths: + update_hash_from_path(after, path) + + # Ensure that apache2 is restarted if these change + if before.hexdigest() != after.hexdigest(): + service_restart('apache2') + + return ret def configure_cert(self, cn): from keystone_utils import ( @@ -39,8 +66,17 @@ class ApacheSSLContext(context.ApacheSSLContext): get_ca, ensure_permissions, is_ssl_cert_master, + is_ssl_enabled, ) + if not is_ssl_enabled(): + return + + if not is_ssl_cert_master(): + log("Not ssl-cert-master - skipping apache cert config until " + "master is elected", level=INFO) + return + ssl_dir = os.path.join('/etc/apache2/ssl/', self.service_namespace) perms = 0o755 mkdir(path=ssl_dir, owner=SSH_USER, group='keystone', perms=perms) @@ -48,11 +84,6 @@ class ApacheSSLContext(context.ApacheSSLContext): ensure_permissions(ssl_dir, user=SSH_USER, group='keystone', perms=perms) - if not is_ssl_cert_master(): - log("Not ssl-cert-master - skipping apache cert config", - level=INFO) - return - log("Creating apache ssl certs in %s" % (ssl_dir), level=INFO) ca = get_ca(user=SSH_USER) @@ -68,11 +99,15 @@ class ApacheSSLContext(context.ApacheSSLContext): get_ca, ensure_permissions, is_ssl_cert_master, + is_ssl_enabled, ) + if not is_ssl_enabled(): + return + if not is_ssl_cert_master(): - log("Not ssl-cert-master - skipping apache cert config", - level=INFO) + log("Not ssl-cert-master - skipping apache ca config until " + "master is elected", level=INFO) return ca = get_ca(user=SSH_USER) diff --git a/hooks/keystone_hooks.py b/hooks/keystone_hooks.py index 39f641c0..2bb49aaa 100755 --- a/hooks/keystone_hooks.py +++ b/hooks/keystone_hooks.py @@ -121,7 +121,7 @@ def config_changed(): unison.ensure_user(user=SSH_USER, group='keystone') homedir = unison.get_homedir(SSH_USER) if not os.path.isdir(homedir): - mkdir(homedir, SSH_USER, 'keystone', 0o775) + mkdir(homedir, SSH_USER, 'juju_keystone', 0o775) if openstack_upgrade_available('keystone'): do_openstack_upgrade(configs=CONFIGS) @@ -354,6 +354,10 @@ def apply_echo_filters(settings, echo_whitelist): 'cluster-relation-departed') @restart_on_change(restart_map(), stopstart=True) def cluster_changed(): + unison.ssh_authorized_peers(user=SSH_USER, + group='juju_keystone', + peer_interface='cluster', + ensure_local_user=True) settings = relation_get() # NOTE(jamespage) re-echo passwords for peer storage echo_whitelist, overrides = \ @@ -366,10 +370,6 @@ def cluster_changed(): peer_echo(includes=echo_whitelist) check_peer_actions() - unison.ssh_authorized_peers(user=SSH_USER, - group='keystone', - peer_interface='cluster', - ensure_local_user=True) if is_elected_leader(CLUSTER_RES) or is_ssl_cert_master(): units = get_ssl_sync_request_units() @@ -499,7 +499,7 @@ def configure_https(): def upgrade_charm(): apt_install(filter_installed_packages(determine_packages())) unison.ssh_authorized_peers(user=SSH_USER, - group='keystone', + group='juju_keystone', peer_interface='cluster', ensure_local_user=True) diff --git a/hooks/keystone_utils.py b/hooks/keystone_utils.py index 5ac3bfa9..2ee1f5cc 100644 --- a/hooks/keystone_utils.py +++ b/hooks/keystone_utils.py @@ -801,6 +801,17 @@ def is_ssl_cert_master(): return master == local_unit() +def is_ssl_enabled(): + # Don't do anything if we are not in ssl/https mode + if (is_str_true(config('use-https')) or + is_str_true(config('https-service-endpoints'))): + log("SSL/HTTPS is enabled", level=DEBUG) + return True + + log("SSL/HTTPS is NOT enabled", level=DEBUG) + return True + + def ensure_ssl_cert_master(use_oldest_peer=False): """Ensure that an ssl cert master has been elected. @@ -808,20 +819,23 @@ def ensure_ssl_cert_master(use_oldest_peer=False): ignored since this could be called before the cluster is ready. """ # Don't do anything if we are not in ssl/https mode - if not (is_str_true(config('use-https')) or - is_str_true(config('https-service-endpoints'))): - log("SSL/HTTPS is NOT enabled", level=DEBUG) - return False - - if not peer_units(): - log("Not syncing certs since there are no peer units.", level=INFO) + if not is_ssl_enabled(): return False + elect = False + peers = peer_units() + master_override = False if use_oldest_peer: - elect = oldest_peer(peer_units()) + elect = oldest_peer(peers) else: elect = is_elected_leader(CLUSTER_RES) + # If no peers we allow this unit to elect itsef as master and do + # sync immediately. + if not peers and not is_ssl_cert_master(): + elect = True + master_override = True + if elect: masters = [] for rid in relation_ids('cluster'): @@ -840,12 +854,12 @@ def ensure_ssl_cert_master(use_oldest_peer=False): # Return now and wait for cluster-relation-changed (peer_echo) for # sync. - return False + return master_override elif len(set(masters)) != 1 and local_unit() not in masters: - log("Did not get concensus from peers on who is master (%s) - " - "waiting for current master to release before self-electing" % - (masters), level=INFO) - return False + log("Did not get consensus from peers on who is ssl-cert-master " + "(%s) - waiting for current master to release before " + "self-electing" % (masters), level=INFO) + return master_override if not is_ssl_cert_master(): log("Not ssl cert master - skipping sync", level=INFO) @@ -864,6 +878,8 @@ def synchronize_ca(fatal=False): leader stickiness while synchronisation is being carried out. This ensures that the last host to create and broadcast cetificates has the option to complete actions before electing the new leader as sync master. + + Returns a dictionary of settings to be set on the cluster relation. """ paths_to_sync = [SYNC_FLAGS_DIR] @@ -881,7 +897,7 @@ def synchronize_ca(fatal=False): if not paths_to_sync: log("Nothing to sync - skipping", level=DEBUG) - return + return {} if not os.path.isdir(SYNC_FLAGS_DIR): mkdir(SYNC_FLAGS_DIR, SSH_USER, 'keystone', 0o775) @@ -907,7 +923,7 @@ def synchronize_ca(fatal=False): raise else: log("Sync failed but fatal=False", level=INFO) - return + return {} hash2 = hashlib.sha256() for path in paths_to_sync: diff --git a/unit_tests/test_keystone_contexts.py b/unit_tests/test_keystone_contexts.py index a10426f4..114048ac 100644 --- a/unit_tests/test_keystone_contexts.py +++ b/unit_tests/test_keystone_contexts.py @@ -16,6 +16,26 @@ class TestKeystoneContexts(CharmTestCase): def setUp(self): super(TestKeystoneContexts, self).setUp(context, TO_PATCH) + @patch.object(context, 'mkdir') + @patch('keystone_utils.determine_ports') + @patch('keystone_utils.is_ssl_cert_master') + @patch('keystone_utils.is_ssl_enabled') + @patch.object(context, 'log') + def test_apache_ssl_context_ssl_not_master(self, + mock_log, + mock_is_ssl_enabled, + mock_is_ssl_cert_master, + mock_determine_ports, + mock_mkdir): + mock_is_ssl_enabled.return_value = True + mock_is_ssl_cert_master.return_value = False + + context.ApacheSSLContext().configure_cert('foo') + context.ApacheSSLContext().configure_ca() + self.assertFalse(mock_mkdir.called) + + @patch('keystone_utils.is_ssl_cert_master') + @patch('keystone_utils.is_ssl_enabled') @patch('charmhelpers.contrib.openstack.context.config') @patch('charmhelpers.contrib.openstack.context.is_clustered') @patch('charmhelpers.contrib.openstack.context.determine_apache_port') @@ -27,7 +47,11 @@ class TestKeystoneContexts(CharmTestCase): mock_determine_api_port, mock_determine_apache_port, mock_is_clustered, - mock_config): + mock_config, + mock_is_ssl_enabled, + mock_is_ssl_cert_master): + mock_is_ssl_enabled.return_value = True + mock_is_ssl_cert_master.return_value = True mock_https.return_value = True mock_unit_get.return_value = '1.2.3.4' mock_determine_api_port.return_value = '12' diff --git a/unit_tests/test_keystone_hooks.py b/unit_tests/test_keystone_hooks.py index eae1f03e..c646e397 100644 --- a/unit_tests/test_keystone_hooks.py +++ b/unit_tests/test_keystone_hooks.py @@ -446,7 +446,7 @@ class KeystoneRelationTests(CharmTestCase): self.peer_echo.assert_called_with(includes=['foo_passwd', 'identity-service:16_foo']) ssh_authorized_peers.assert_called_with( - user=self.ssh_user, group='keystone', + user=self.ssh_user, group='juju_keystone', peer_interface='cluster', ensure_local_user=True) self.assertFalse(mock_synchronize_ca.called) self.assertTrue(configs.write_all.called) @@ -621,7 +621,7 @@ class KeystoneRelationTests(CharmTestCase): hooks.upgrade_charm() self.assertTrue(self.apt_install.called) ssh_authorized_peers.assert_called_with( - user=self.ssh_user, group='keystone', + user=self.ssh_user, group='juju_keystone', peer_interface='cluster', ensure_local_user=True) self.assertTrue(mock_synchronize_ca.called) self.log.assert_called_with( @@ -649,7 +649,7 @@ class KeystoneRelationTests(CharmTestCase): hooks.upgrade_charm() self.assertTrue(self.apt_install.called) ssh_authorized_peers.assert_called_with( - user=self.ssh_user, group='keystone', + user=self.ssh_user, group='juju_keystone', peer_interface='cluster', ensure_local_user=True) self.assertFalse(self.log.called) self.assertFalse(self.ensure_initial_admin.called) diff --git a/unit_tests/test_keystone_utils.py b/unit_tests/test_keystone_utils.py index 629d774a..cc7f250b 100644 --- a/unit_tests/test_keystone_utils.py +++ b/unit_tests/test_keystone_utils.py @@ -7,7 +7,8 @@ os.environ['JUJU_UNIT_NAME'] = 'keystone' with patch('charmhelpers.core.hookenv.config') as config: import keystone_utils as utils -import keystone_hooks as hooks +with patch.object(utils, 'register_configs'): + import keystone_hooks as hooks TO_PATCH = [ 'api_port', @@ -369,3 +370,47 @@ class TestKeystoneUtils(CharmTestCase): self.related_units.return_value = [] self.assertTrue(utils.is_db_ready()) + + @patch.object(utils, 'peer_units') + @patch.object(utils, 'is_elected_leader') + @patch.object(utils, 'oldest_peer') + @patch.object(utils, 'is_ssl_enabled') + def test_ensure_ssl_cert_master(self, mock_is_str_true, mock_oldest_peer, + mock_is_elected_leader, mock_peer_units): + self.relation_ids.return_value = ['cluster:0'] + self.local_unit.return_value = 'unit/0' + + mock_is_str_true.return_value = False + self.assertFalse(utils.ensure_ssl_cert_master()) + self.assertFalse(self.relation_set.called) + + mock_is_elected_leader.return_value = False + self.assertFalse(utils.ensure_ssl_cert_master()) + self.assertFalse(self.relation_set.called) + + mock_is_str_true.return_value = True + mock_is_elected_leader.return_value = False + mock_peer_units.return_value = ['unit/0'] + self.assertFalse(utils.ensure_ssl_cert_master()) + self.assertFalse(self.relation_set.called) + + mock_peer_units.return_value = [] + self.assertTrue(utils.ensure_ssl_cert_master()) + settings = {'ssl-cert-master': 'unit/0'} + self.relation_set.assert_called_with(relation_id='cluster:0', + relation_settings=settings) + self.relation_set.reset_mock() + + self.assertTrue(utils.ensure_ssl_cert_master(use_oldest_peer=True)) + settings = {'ssl-cert-master': 'unit/0'} + self.relation_set.assert_called_with(relation_id='cluster:0', + relation_settings=settings) + self.relation_set.reset_mock() + + mock_peer_units.return_value = ['unit/0'] + self.assertFalse(utils.ensure_ssl_cert_master()) + self.assertFalse(utils.ensure_ssl_cert_master(use_oldest_peer=True)) + settings = {'ssl-cert-master': 'unit/0'} + self.relation_set.assert_called_with(relation_id='cluster:0', + relation_settings=settings) + self.relation_set.reset_mock() From 0fe447683a7d285cd433d0a63d37e3f2789340d4 Mon Sep 17 00:00:00 2001 From: Edward Hope-Morley Date: Tue, 27 Jan 2015 23:56:15 +0000 Subject: [PATCH 23/48] more --- hooks/keystone_context.py | 11 ++++++----- unit_tests/test_keystone_contexts.py | 8 +++++++- 2 files changed, 13 insertions(+), 6 deletions(-) diff --git a/hooks/keystone_context.py b/hooks/keystone_context.py index 86cb407d..6094461e 100644 --- a/hooks/keystone_context.py +++ b/hooks/keystone_context.py @@ -72,11 +72,7 @@ class ApacheSSLContext(context.ApacheSSLContext): if not is_ssl_enabled(): return - if not is_ssl_cert_master(): - log("Not ssl-cert-master - skipping apache cert config until " - "master is elected", level=INFO) - return - + # Ensure ssl dir exists whether master or not ssl_dir = os.path.join('/etc/apache2/ssl/', self.service_namespace) perms = 0o755 mkdir(path=ssl_dir, owner=SSH_USER, group='keystone', perms=perms) @@ -84,6 +80,11 @@ class ApacheSSLContext(context.ApacheSSLContext): ensure_permissions(ssl_dir, user=SSH_USER, group='keystone', perms=perms) + if not is_ssl_cert_master(): + log("Not ssl-cert-master - skipping apache cert config until " + "master is elected", level=INFO) + return + log("Creating apache ssl certs in %s" % (ssl_dir), level=INFO) ca = get_ca(user=SSH_USER) diff --git a/unit_tests/test_keystone_contexts.py b/unit_tests/test_keystone_contexts.py index 114048ac..1e29286c 100644 --- a/unit_tests/test_keystone_contexts.py +++ b/unit_tests/test_keystone_contexts.py @@ -17,6 +17,8 @@ class TestKeystoneContexts(CharmTestCase): super(TestKeystoneContexts, self).setUp(context, TO_PATCH) @patch.object(context, 'mkdir') + @patch('keystone_utils.get_ca') + @patch('keystone_utils.ensure_permissions') @patch('keystone_utils.determine_ports') @patch('keystone_utils.is_ssl_cert_master') @patch('keystone_utils.is_ssl_enabled') @@ -26,13 +28,17 @@ class TestKeystoneContexts(CharmTestCase): mock_is_ssl_enabled, mock_is_ssl_cert_master, mock_determine_ports, + mock_ensure_permissions, + mock_get_ca, mock_mkdir): mock_is_ssl_enabled.return_value = True mock_is_ssl_cert_master.return_value = False context.ApacheSSLContext().configure_cert('foo') context.ApacheSSLContext().configure_ca() - self.assertFalse(mock_mkdir.called) + self.assertTrue(mock_mkdir.called) + self.assertTrue(mock_ensure_permissions.called) + self.assertFalse(mock_get_ca.called) @patch('keystone_utils.is_ssl_cert_master') @patch('keystone_utils.is_ssl_enabled') From 623b173df3b07932a3366b8ccae64bb691dd3692 Mon Sep 17 00:00:00 2001 From: Edward Hope-Morley Date: Wed, 28 Jan 2015 16:54:56 +0000 Subject: [PATCH 24/48] We need to sync all ssl certs for both use-https AND https-service-endpoints. --- hooks/keystone_utils.py | 1 + 1 file changed, 1 insertion(+) diff --git a/hooks/keystone_utils.py b/hooks/keystone_utils.py index 2ee1f5cc..0b8b6fb2 100644 --- a/hooks/keystone_utils.py +++ b/hooks/keystone_utils.py @@ -892,6 +892,7 @@ def synchronize_ca(fatal=False): elif is_str_true(config('use-https')): log("Syncing keystone-endpoint certs since use-https=True", level=DEBUG) + paths_to_sync.append(SSL_DIR) paths_to_sync.append(APACHE_SSL_DIR) paths_to_sync.append(CA_CERT_PATH) From 20c52bdb794215a633ae0cd65a4056185a5c27be Mon Sep 17 00:00:00 2001 From: Corey Bryant Date: Thu, 29 Jan 2015 11:09:05 -0500 Subject: [PATCH 25/48] Switch amulet tests to use stable branches --- tests/basic_deployment.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/basic_deployment.py b/tests/basic_deployment.py index 75c1062d..b66a6cc6 100644 --- a/tests/basic_deployment.py +++ b/tests/basic_deployment.py @@ -19,7 +19,7 @@ u = OpenStackAmuletUtils(ERROR) class KeystoneBasicDeployment(OpenStackAmuletDeployment): """Amulet tests on a basic keystone deployment.""" - def __init__(self, series=None, openstack=None, source=None, stable=False): + def __init__(self, series=None, openstack=None, source=None, stable=True): """Deploy the entire test environment.""" super(KeystoneBasicDeployment, self).__init__(series, openstack, source, stable) self._add_services() From 172c11f7fdd7844e1543beacded6ef7fad99e5e5 Mon Sep 17 00:00:00 2001 From: Edward Hope-Morley Date: Mon, 2 Feb 2015 13:49:05 +0000 Subject: [PATCH 26/48] [hopem,r=] Fixes is_db_ready() logic Closes-Bug: 1417108 --- hooks/keystone_utils.py | 7 +++---- unit_tests/test_keystone_utils.py | 16 ++++++++++++---- 2 files changed, 15 insertions(+), 8 deletions(-) diff --git a/hooks/keystone_utils.py b/hooks/keystone_utils.py index 0b8b6fb2..e804a849 100644 --- a/hooks/keystone_utils.py +++ b/hooks/keystone_utils.py @@ -1408,9 +1408,8 @@ def is_db_ready(use_current_context=False, db_rel=None): if allowed_units and local_unit() in allowed_units.split(): return True - # If relation has units - return False + rel_has_units = True - # If neither relation has units then we are probably in sqllite mode return - # True. + # If neither relation has units then we are probably in sqlite mode so + # return True. return not rel_has_units diff --git a/unit_tests/test_keystone_utils.py b/unit_tests/test_keystone_utils.py index cc7f250b..7e60dfa4 100644 --- a/unit_tests/test_keystone_utils.py +++ b/unit_tests/test_keystone_utils.py @@ -352,20 +352,28 @@ class TestKeystoneUtils(CharmTestCase): self.assertEqual(utils.get_admin_passwd(), 'supersecretgen') def test_is_db_ready(self): + allowed_units = None + + def fake_rel_get(attribute=None, *args, **kwargs): + if attribute == 'allowed_units': + return allowed_units + + self.relation_get.side_effect = fake_rel_get + self.relation_id.return_value = 'shared-db:0' - self.relation_ids.return_value = [self.relation_id.return_value] + self.relation_ids.return_value = ['shared-db:0'] self.local_unit.return_value = 'unit/0' - self.relation_get.return_value = 'unit/0' + allowed_units = 'unit/0' self.assertTrue(utils.is_db_ready(use_current_context=True)) self.relation_ids.return_value = ['acme:0'] self.assertRaises(utils.is_db_ready, use_current_context=True) self.related_units.return_value = ['unit/0'] - self.relation_ids.return_value = [self.relation_id.return_value] + self.relation_ids.return_value = ['shared-db:0', 'shared-db:1'] self.assertTrue(utils.is_db_ready()) - self.relation_get.return_value = 'unit/1' + allowed_units = 'unit/1' self.assertFalse(utils.is_db_ready()) self.related_units.return_value = [] From 12f1eee2603d7c05875723e75f62334b6ba1c8bf Mon Sep 17 00:00:00 2001 From: Edward Hope-Morley Date: Mon, 2 Feb 2015 13:52:05 +0000 Subject: [PATCH 27/48] [hopem,r=] Backport "Fixes is_db_ready() logic" Closes-Bug: 1417108 --- hooks/keystone_utils.py | 7 +++---- unit_tests/test_keystone_utils.py | 16 ++++++++++++---- 2 files changed, 15 insertions(+), 8 deletions(-) diff --git a/hooks/keystone_utils.py b/hooks/keystone_utils.py index 0b8b6fb2..e804a849 100644 --- a/hooks/keystone_utils.py +++ b/hooks/keystone_utils.py @@ -1408,9 +1408,8 @@ def is_db_ready(use_current_context=False, db_rel=None): if allowed_units and local_unit() in allowed_units.split(): return True - # If relation has units - return False + rel_has_units = True - # If neither relation has units then we are probably in sqllite mode return - # True. + # If neither relation has units then we are probably in sqlite mode so + # return True. return not rel_has_units diff --git a/unit_tests/test_keystone_utils.py b/unit_tests/test_keystone_utils.py index cc7f250b..7e60dfa4 100644 --- a/unit_tests/test_keystone_utils.py +++ b/unit_tests/test_keystone_utils.py @@ -352,20 +352,28 @@ class TestKeystoneUtils(CharmTestCase): self.assertEqual(utils.get_admin_passwd(), 'supersecretgen') def test_is_db_ready(self): + allowed_units = None + + def fake_rel_get(attribute=None, *args, **kwargs): + if attribute == 'allowed_units': + return allowed_units + + self.relation_get.side_effect = fake_rel_get + self.relation_id.return_value = 'shared-db:0' - self.relation_ids.return_value = [self.relation_id.return_value] + self.relation_ids.return_value = ['shared-db:0'] self.local_unit.return_value = 'unit/0' - self.relation_get.return_value = 'unit/0' + allowed_units = 'unit/0' self.assertTrue(utils.is_db_ready(use_current_context=True)) self.relation_ids.return_value = ['acme:0'] self.assertRaises(utils.is_db_ready, use_current_context=True) self.related_units.return_value = ['unit/0'] - self.relation_ids.return_value = [self.relation_id.return_value] + self.relation_ids.return_value = ['shared-db:0', 'shared-db:1'] self.assertTrue(utils.is_db_ready()) - self.relation_get.return_value = 'unit/1' + allowed_units = 'unit/1' self.assertFalse(utils.is_db_ready()) self.related_units.return_value = [] From c4c78977e8fbd373a600f15696521dbae2c260db Mon Sep 17 00:00:00 2001 From: Edward Hope-Morley Date: Tue, 3 Feb 2015 14:38:55 +0000 Subject: [PATCH 28/48] fix null setting issue --- hooks/keystone_utils.py | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/hooks/keystone_utils.py b/hooks/keystone_utils.py index 957786fe..c20a81e4 100644 --- a/hooks/keystone_utils.py +++ b/hooks/keystone_utils.py @@ -934,6 +934,8 @@ def synchronize_ca(fatal=False): create_peer_service_actions('restart', ['apache2']) create_peer_actions(['update-ca-certificates']) + cluster_rel_settings = {} + retries = 3 while True: hash1 = hashlib.sha256() @@ -945,6 +947,8 @@ def synchronize_ca(fatal=False): if synced_units: # Format here needs to match that used when peers request sync synced_units = [u.replace('/', '-') for u in synced_units] + cluster_rel_settings['ssl-synced-units'] = \ + json.dumps(synced_units) except: if fatal: raise @@ -973,10 +977,10 @@ def synchronize_ca(fatal=False): hash = hash1.hexdigest() log("Sending restart-services-trigger=%s to all peers" % (hash), level=DEBUG) + cluster_rel_settings['restart-services-trigger'] = hash log("Sync complete", level=DEBUG) - return {'restart-services-trigger': hash, - 'ssl-synced-units': json.dumps(synced_units)} + return cluster_rel_settings def clear_ssl_synced_units(): From 9ee43245c2b64c319d3d1b50306f2f260701144a Mon Sep 17 00:00:00 2001 From: Ryan Beisner Date: Tue, 3 Feb 2015 22:08:46 +0000 Subject: [PATCH 29/48] This branch adds templates for kilo and updates the token section to point to the correct driver. Fixes bug 1417211. --- templates/kilo/keystone.conf | 105 +++++++++++++++++++++++++++++++++++ templates/kilo/logging.conf | 44 +++++++++++++++ 2 files changed, 149 insertions(+) create mode 100644 templates/kilo/keystone.conf create mode 100644 templates/kilo/logging.conf diff --git a/templates/kilo/keystone.conf b/templates/kilo/keystone.conf new file mode 100644 index 00000000..36f1018d --- /dev/null +++ b/templates/kilo/keystone.conf @@ -0,0 +1,105 @@ +# kilo +############################################################################### +# [ WARNING ] +# Configuration file maintained by Juju. Local changes may be overwritten. +############################################################################### +[DEFAULT] +admin_token = {{ token }} +admin_port = {{ admin_port }} +public_port = {{ public_port }} +use_syslog = {{ use_syslog }} +log_config = /etc/keystone/logging.conf +debug = {{ debug }} +verbose = {{ verbose }} +public_endpoint = {{ public_endpoint }} +admin_endpoint = {{ admin_endpoint }} +bind_host = {{ bind_host }} +public_workers = {{ workers }} +admin_workers = {{ workers }} + +[database] +{% if database_host -%} +connection = {{ database_type }}://{{ database_user }}:{{ database_password }}@{{ database_host }}/{{ database }}{% if database_ssl_ca %}?ssl_ca={{ database_ssl_ca }}{% if database_ssl_cert %}&ssl_cert={{ database_ssl_cert }}&ssl_key={{ database_ssl_key }}{% endif %}{% endif %} +{% else -%} +connection = sqlite:////var/lib/keystone/keystone.db +{% endif -%} +idle_timeout = 200 + +[identity] +driver = keystone.identity.backends.{{ identity_backend }}.Identity + +[credential] +driver = keystone.credential.backends.sql.Credential + +[trust] +driver = keystone.trust.backends.sql.Trust + +[os_inherit] + +[catalog] +driver = keystone.catalog.backends.sql.Catalog + +[endpoint_filter] + +[token] +driver = keystone.token.persistence.backends.sql.Token +provider = keystone.token.providers.uuid.Provider + +[cache] + +[policy] +driver = keystone.policy.backends.sql.Policy + +[ec2] +driver = keystone.contrib.ec2.backends.sql.Ec2 + +[assignment] +driver = keystone.assignment.backends.{{ assignment_backend }}.Assignment + +[oauth1] + +[signing] + +[auth] +methods = external,password,token,oauth1 +password = keystone.auth.plugins.password.Password +token = keystone.auth.plugins.token.Token +oauth1 = keystone.auth.plugins.oauth1.OAuth + +[paste_deploy] +config_file = keystone-paste.ini + +[extra_headers] +Distribution = Ubuntu + +[ldap] +{% if identity_backend == 'ldap' -%} +url = {{ ldap_server }} +user = {{ ldap_user }} +password = {{ ldap_password }} +suffix = {{ ldap_suffix }} + +{% if ldap_config_flags -%} +{% for key, value in ldap_config_flags.iteritems() -%} +{{ key }} = {{ value }} +{% endfor -%} +{% endif -%} + +{% if ldap_readonly -%} +user_allow_create = False +user_allow_update = False +user_allow_delete = False + +tenant_allow_create = False +tenant_allow_update = False +tenant_allow_delete = False + +role_allow_create = False +role_allow_update = False +role_allow_delete = False + +group_allow_create = False +group_allow_update = False +group_allow_delete = False +{% endif -%} +{% endif -%} diff --git a/templates/kilo/logging.conf b/templates/kilo/logging.conf new file mode 100644 index 00000000..9f212188 --- /dev/null +++ b/templates/kilo/logging.conf @@ -0,0 +1,44 @@ +# kilo +[loggers] +keys=root + +[formatters] +keys=normal,normal_with_name,debug + +[handlers] +keys=production,file,devel + +[logger_root] +{% if root_level -%} +level={{ root_level }} +{% else -%} +level=WARNING +{% endif -%} +handlers=file + +[handler_production] +class=handlers.SysLogHandler +level=ERROR +formatter=normal_with_name +args=(('localhost', handlers.SYSLOG_UDP_PORT), handlers.SysLogHandler.LOG_USER) + +[handler_file] +class=FileHandler +level=DEBUG +formatter=normal_with_name +args=('/var/log/keystone/keystone.log', 'a') + +[handler_devel] +class=StreamHandler +level=NOTSET +formatter=debug +args=(sys.stdout,) + +[formatter_normal] +format=%(asctime)s %(levelname)s %(message)s + +[formatter_normal_with_name] +format=(%(name)s): %(asctime)s %(levelname)s %(message)s + +[formatter_debug] +format=(%(name)s): %(asctime)s %(levelname)s %(module)s %(funcName)s %(message)s From a327b82b6f1c9bfdd66e06bd113266067b2d027d Mon Sep 17 00:00:00 2001 From: Edward Hope-Morley Date: Thu, 5 Feb 2015 17:48:25 +0000 Subject: [PATCH 30/48] add more is_db_ready() protection --- hooks/keystone_hooks.py | 17 +++++++++++++++-- unit_tests/test_keystone_hooks.py | 27 +++++++++++++++++++++------ 2 files changed, 36 insertions(+), 8 deletions(-) diff --git a/hooks/keystone_hooks.py b/hooks/keystone_hooks.py index 0c8dcf95..3faf8ab1 100755 --- a/hooks/keystone_hooks.py +++ b/hooks/keystone_hooks.py @@ -18,6 +18,7 @@ from charmhelpers.core.hookenv import ( log, local_unit, DEBUG, + INFO, WARNING, ERROR, relation_get, @@ -189,6 +190,12 @@ def pgsql_db_joined(): def update_all_identity_relation_units(): CONFIGS.write_all() + + if not is_db_ready(): + log('Allowed_units list provided and this unit not present', + level=INFO) + return + try: migrate_database() except Exception as exc: @@ -220,8 +227,10 @@ def db_changed(): # units acl entry has been added. So, if the db supports passing # a list of permitted units then check if we're in the list. if not is_db_ready(use_current_context=True): - log('Allowed_units list provided and this unit not present') + log('Allowed_units list provided and this unit not present', + level=INFO) return + # Ensure any existing service entries are updated in the # new database backend update_all_identity_relation_units() @@ -248,7 +257,6 @@ def identity_changed(relation_id=None, remote_unit=None): notifications = {} if is_elected_leader(CLUSTER_RES): - if not is_db_ready(): log("identity-service-relation-changed hook fired before db " "ready - deferring until db ready", level=WARNING) @@ -476,6 +484,11 @@ def ha_changed(): clustered = relation_get('clustered') if clustered and is_elected_leader(CLUSTER_RES): + if not is_db_ready(): + log('Allowed_units list provided and this unit not present', + level=INFO) + return + ensure_initial_admin(config) log('Cluster configured, notifying other services and updating ' 'keystone endpoint configuration') diff --git a/unit_tests/test_keystone_hooks.py b/unit_tests/test_keystone_hooks.py index c646e397..28063f9d 100644 --- a/unit_tests/test_keystone_hooks.py +++ b/unit_tests/test_keystone_hooks.py @@ -247,10 +247,13 @@ class KeystoneRelationTests(CharmTestCase): @patch('keystone_utils.log') @patch('keystone_utils.ensure_ssl_cert_master') + @patch.object(hooks, 'is_db_ready') @patch.object(hooks, 'CONFIGS') @patch.object(hooks, 'identity_changed') def test_postgresql_db_changed(self, identity_changed, configs, + mock_is_db_ready, mock_ensure_ssl_cert_master, mock_log): + mock_is_db_ready.return_value = True mock_ensure_ssl_cert_master.return_value = False self.relation_ids.return_value = ['identity-service:0'] self.related_units.return_value = ['unit/0'] @@ -266,6 +269,7 @@ class KeystoneRelationTests(CharmTestCase): @patch('keystone_utils.log') @patch('keystone_utils.ensure_ssl_cert_master') + @patch.object(hooks, 'is_db_ready') @patch.object(hooks, 'peer_units') @patch.object(hooks, 'ensure_permissions') @patch.object(hooks, 'admin_relation_changed') @@ -279,7 +283,9 @@ class KeystoneRelationTests(CharmTestCase): self, configure_https, identity_changed, configs, get_homedir, ensure_user, cluster_joined, admin_relation_changed, ensure_permissions, mock_peer_units, + mock_is_db_ready, mock_ensure_ssl_cert_master, mock_log): + mock_is_db_ready.return_value = True self.openstack_upgrade_available.return_value = False self.is_elected_leader.return_value = True # avoid having to mock syncer @@ -337,6 +343,7 @@ class KeystoneRelationTests(CharmTestCase): @patch('keystone_utils.log') @patch('keystone_utils.ensure_ssl_cert_master') + @patch.object(hooks, 'is_db_ready') @patch.object(hooks, 'peer_units') @patch.object(hooks, 'ensure_permissions') @patch.object(hooks, 'admin_relation_changed') @@ -346,12 +353,17 @@ class KeystoneRelationTests(CharmTestCase): @patch.object(hooks, 'CONFIGS') @patch.object(hooks, 'identity_changed') @patch.object(hooks, 'configure_https') - def test_config_changed_with_openstack_upgrade( - self, configure_https, identity_changed, - configs, get_homedir, ensure_user, cluster_joined, - admin_relation_changed, - ensure_permissions, mock_peer_units, mock_ensure_ssl_cert_master, - mock_log): + def test_config_changed_with_openstack_upgrade(self, configure_https, + identity_changed, + configs, get_homedir, + ensure_user, cluster_joined, + admin_relation_changed, + ensure_permissions, + mock_peer_units, + mock_is_db_ready, + mock_ensure_ssl_cert_master, + mock_log): + mock_is_db_ready.return_value = True self.openstack_upgrade_available.return_value = True self.is_elected_leader.return_value = True # avoid having to mock syncer @@ -544,12 +556,15 @@ class KeystoneRelationTests(CharmTestCase): @patch('keystone_utils.log') @patch('keystone_utils.ensure_ssl_cert_master') + @patch.object(hooks, 'is_db_ready') @patch.object(hooks, 'identity_changed') @patch.object(hooks, 'CONFIGS') def test_ha_relation_changed_clustered_leader(self, configs, identity_changed, + mock_is_db_ready, mock_ensure_ssl_cert_master, mock_log): + mock_is_db_ready.return_value = True mock_ensure_ssl_cert_master.return_value = False self.relation_get.return_value = True self.is_elected_leader.return_value = True From 12aad4c1323e34fd0746a0822264d5cd66f9dc38 Mon Sep 17 00:00:00 2001 From: Edward Hope-Morley Date: Fri, 6 Feb 2015 11:52:49 +0000 Subject: [PATCH 31/48] synced charm-helpers --- hooks/charmhelpers/core/host.py | 10 +++++----- hooks/charmhelpers/core/templating.py | 6 +++--- hooks/keystone_hooks.py | 1 + 3 files changed, 9 insertions(+), 8 deletions(-) diff --git a/hooks/charmhelpers/core/host.py b/hooks/charmhelpers/core/host.py index cf2cbe14..b771c611 100644 --- a/hooks/charmhelpers/core/host.py +++ b/hooks/charmhelpers/core/host.py @@ -191,11 +191,11 @@ def mkdir(path, owner='root', group='root', perms=0o555, force=False): def write_file(path, content, owner='root', group='root', perms=0o444): - """Create or overwrite a file with the contents of a string""" + """Create or overwrite a file with the contents of a byte string.""" log("Writing file {} {}:{} {:o}".format(path, owner, group, perms)) uid = pwd.getpwnam(owner).pw_uid gid = grp.getgrnam(group).gr_gid - with open(path, 'w') as target: + with open(path, 'wb') as target: os.fchown(target.fileno(), uid, gid) os.fchmod(target.fileno(), perms) target.write(content) @@ -305,11 +305,11 @@ def restart_on_change(restart_map, stopstart=False): ceph_client_changed function. """ def wrap(f): - def wrapped_f(*args): + def wrapped_f(*args, **kwargs): checksums = {} for path in restart_map: checksums[path] = file_hash(path) - f(*args) + f(*args, **kwargs) restarts = [] for path in restart_map: if checksums[path] != file_hash(path): @@ -361,7 +361,7 @@ def list_nics(nic_type): ip_output = (line for line in ip_output if line) for line in ip_output: if line.split()[1].startswith(int_type): - matched = re.search('.*: (bond[0-9]+\.[0-9]+)@.*', line) + matched = re.search('.*: (' + int_type + r'[0-9]+\.[0-9]+)@.*', line) if matched: interface = matched.groups()[0] else: diff --git a/hooks/charmhelpers/core/templating.py b/hooks/charmhelpers/core/templating.py index 97669092..45319998 100644 --- a/hooks/charmhelpers/core/templating.py +++ b/hooks/charmhelpers/core/templating.py @@ -21,7 +21,7 @@ from charmhelpers.core import hookenv def render(source, target, context, owner='root', group='root', - perms=0o444, templates_dir=None): + perms=0o444, templates_dir=None, encoding='UTF-8'): """ Render a template. @@ -64,5 +64,5 @@ def render(source, target, context, owner='root', group='root', level=hookenv.ERROR) raise e content = template.render(context) - host.mkdir(os.path.dirname(target), owner, group) - host.write_file(target, content, owner, group, perms) + host.mkdir(os.path.dirname(target), owner, group, perms=0o755) + host.write_file(target, content.encode(encoding), owner, group, perms) diff --git a/hooks/keystone_hooks.py b/hooks/keystone_hooks.py index 3faf8ab1..4bcc9c48 100755 --- a/hooks/keystone_hooks.py +++ b/hooks/keystone_hooks.py @@ -251,6 +251,7 @@ def pgsql_db_changed(): @hooks.hook('identity-service-relation-changed') +@restart_on_change(restart_map()) @synchronize_ca_if_changed() def identity_changed(relation_id=None, remote_unit=None): CONFIGS.write_all() From 954a47a3d476b90366d6c3024cf1a0a03d318e27 Mon Sep 17 00:00:00 2001 From: Edward Hope-Morley Date: Fri, 6 Feb 2015 12:05:52 +0000 Subject: [PATCH 32/48] more --- hooks/keystone_hooks.py | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/hooks/keystone_hooks.py b/hooks/keystone_hooks.py index 4bcc9c48..ce40d6b3 100755 --- a/hooks/keystone_hooks.py +++ b/hooks/keystone_hooks.py @@ -188,10 +188,9 @@ def pgsql_db_joined(): relation_set(database=config('database')) -def update_all_identity_relation_units(): +def update_all_identity_relation_units(check_db_ready=True): CONFIGS.write_all() - - if not is_db_ready(): + if check_db_ready and not is_db_ready(): log('Allowed_units list provided and this unit not present', level=INFO) return @@ -233,7 +232,7 @@ def db_changed(): # Ensure any existing service entries are updated in the # new database backend - update_all_identity_relation_units() + update_all_identity_relation_units(check_db_ready=False) @hooks.hook('pgsql-db-relation-changed') From d7766ba6a4f5aff4ade45d82375364e29806fe59 Mon Sep 17 00:00:00 2001 From: Edward Hope-Morley Date: Fri, 6 Feb 2015 12:29:53 +0000 Subject: [PATCH 33/48] [hopem,r=] Backport fix from /next -r 116 * improves ssl master election logic * adds more unit tests * fixes config changes logic Closes-Bug: 1415579 --- hooks/charmhelpers/contrib/unison/__init__.py | 10 +- hooks/charmhelpers/core/sysctl.py | 16 +- hooks/keystone_hooks.py | 49 ++++-- hooks/keystone_utils.py | 114 +++++++++----- unit_tests/test_keystone_utils.py | 147 +++++++++++++++--- 5 files changed, 252 insertions(+), 84 deletions(-) diff --git a/hooks/charmhelpers/contrib/unison/__init__.py b/hooks/charmhelpers/contrib/unison/__init__.py index f8551d10..4adcbf61 100644 --- a/hooks/charmhelpers/contrib/unison/__init__.py +++ b/hooks/charmhelpers/contrib/unison/__init__.py @@ -73,6 +73,7 @@ from charmhelpers.core.hookenv import ( relation_set, relation_get, unit_private_ip, + INFO, ERROR, ) @@ -86,7 +87,7 @@ def get_homedir(user): user = pwd.getpwnam(user) return user.pw_dir except KeyError: - log('Could not get homedir for user %s: user exists?', ERROR) + log('Could not get homedir for user %s: user exists?' % (user), ERROR) raise Exception @@ -233,14 +234,15 @@ def collect_authed_hosts(peer_interface): rid=r_id, unit=unit) if not authed_hosts: - log('Peer %s has not authorized *any* hosts yet, skipping.') + log('Peer %s has not authorized *any* hosts yet, skipping.' % + (unit), level=INFO) continue if unit_private_ip() in authed_hosts.split(':'): hosts.append(private_addr) else: - log('Peer %s has not authorized *this* host yet, skipping.') - + log('Peer %s has not authorized *this* host yet, skipping.' % + (unit), level=INFO) return hosts diff --git a/hooks/charmhelpers/core/sysctl.py b/hooks/charmhelpers/core/sysctl.py index d642a371..8e1b9eeb 100644 --- a/hooks/charmhelpers/core/sysctl.py +++ b/hooks/charmhelpers/core/sysctl.py @@ -26,25 +26,31 @@ from subprocess import check_call from charmhelpers.core.hookenv import ( log, DEBUG, + ERROR, ) def create(sysctl_dict, sysctl_file): """Creates a sysctl.conf file from a YAML associative array - :param sysctl_dict: a dict of sysctl options eg { 'kernel.max_pid': 1337 } - :type sysctl_dict: dict + :param sysctl_dict: a YAML-formatted string of sysctl options eg "{ 'kernel.max_pid': 1337 }" + :type sysctl_dict: str :param sysctl_file: path to the sysctl file to be saved :type sysctl_file: str or unicode :returns: None """ - sysctl_dict = yaml.load(sysctl_dict) + try: + sysctl_dict_parsed = yaml.safe_load(sysctl_dict) + except yaml.YAMLError: + log("Error parsing YAML sysctl_dict: {}".format(sysctl_dict), + level=ERROR) + return with open(sysctl_file, "w") as fd: - for key, value in sysctl_dict.items(): + for key, value in sysctl_dict_parsed.items(): fd.write("{}={}\n".format(key, value)) - log("Updating sysctl_file: %s values: %s" % (sysctl_file, sysctl_dict), + log("Updating sysctl_file: %s values: %s" % (sysctl_file, sysctl_dict_parsed), level=DEBUG) check_call(["sysctl", "-p", sysctl_file]) diff --git a/hooks/keystone_hooks.py b/hooks/keystone_hooks.py index 2bb49aaa..0c8dcf95 100755 --- a/hooks/keystone_hooks.py +++ b/hooks/keystone_hooks.py @@ -67,6 +67,7 @@ from keystone_utils import ( is_str_true, is_ssl_cert_master, is_db_ready, + clear_ssl_synced_units, ) from charmhelpers.contrib.hahelpers.cluster import ( @@ -150,11 +151,8 @@ def config_changed(): admin_relation_changed(rid) # Ensure sync request is sent out (needed for upgrade to ssl from non-ssl) - settings = {} - append_ssl_sync_request(settings) - if settings: - for rid in relation_ids('cluster'): - relation_set(relation_id=rid, relation_settings=settings) + send_ssl_sync_request() + for r_id in relation_ids('ha'): ha_joined(relation_id=r_id) @@ -283,15 +281,39 @@ def identity_changed(relation_id=None, remote_unit=None): send_notifications(notifications) -def append_ssl_sync_request(settings): - """Add request to be synced to relation settings. +def send_ssl_sync_request(): + """Set sync request on cluster relation. - This will be consumed by cluster-relation-changed ssl master. + Value set equals number of ssl configs currently enabled so that if they + change, we ensure that certs are synced. This setting is consumed by + cluster-relation-changed ssl master. We also clear the 'synced' set to + guarantee that a sync will occur. + + Note the we do nothing if the setting is already applied. """ - if (is_str_true(config('use-https')) or - is_str_true(config('https-service-endpoints'))): - unit = local_unit().replace('/', '-') - settings['ssl-sync-required-%s' % (unit)] = '1' + unit = local_unit().replace('/', '-') + count = 0 + if is_str_true(config('use-https')): + count += 1 + + if is_str_true(config('https-service-endpoints')): + count += 2 + + if count: + key = 'ssl-sync-required-%s' % (unit) + settings = {key: count} + prev = 0 + rid = None + for rid in relation_ids('cluster'): + for unit in related_units(rid): + _prev = relation_get(rid=rid, unit=unit, attribute=key) or 0 + if _prev and _prev > prev: + prev = _prev + + if rid and prev < count: + clear_ssl_synced_units() + log("Setting %s=%s" % (key, count), level=DEBUG) + relation_set(relation_id=rid, relation_settings=settings) @hooks.hook('cluster-relation-joined') @@ -314,9 +336,8 @@ def cluster_joined(): private_addr = get_ipv6_addr(exc_list=[config('vip')])[0] settings['private-address'] = private_addr - append_ssl_sync_request(settings) - relation_set(relation_settings=settings) + send_ssl_sync_request() def apply_echo_filters(settings, echo_whitelist): diff --git a/hooks/keystone_utils.py b/hooks/keystone_utils.py index e804a849..d62784d0 100644 --- a/hooks/keystone_utils.py +++ b/hooks/keystone_utils.py @@ -21,7 +21,6 @@ from charmhelpers.contrib.hahelpers.cluster import( determine_api_port, https, peer_units, - oldest_peer, ) from charmhelpers.contrib.openstack import context, templating @@ -764,14 +763,27 @@ def create_peer_actions(actions): def unison_sync(paths_to_sync): """Do unison sync and retry a few times if it fails since peers may not be ready for sync. + + Returns list of synced units or None if one or more peers was not synced. """ log('Synchronizing CA (%s) to all peers.' % (', '.join(paths_to_sync)), level=INFO) keystone_gid = grp.getgrnam('keystone').gr_gid + + # NOTE(dosaboy): This will sync to all peers who have already provided + # their ssh keys. If any existing peers have not provided their keys yet, + # they will be silently ignored. unison.sync_to_peers(peer_interface='cluster', paths=paths_to_sync, user=SSH_USER, verbose=True, gid=keystone_gid, fatal=True) + synced_units = peer_units() + if len(unison.collect_authed_hosts('cluster')) != len(synced_units): + log("Not all peer units synced due to missing public keys", level=INFO) + return None + else: + return synced_units + def get_ssl_sync_request_units(): """Get list of units that have requested to be synced. @@ -791,14 +803,22 @@ def get_ssl_sync_request_units(): return units -def is_ssl_cert_master(): +def is_ssl_cert_master(votes=None): """Return True if this unit is ssl cert master.""" master = None for rid in relation_ids('cluster'): master = relation_get(attribute='ssl-cert-master', rid=rid, unit=local_unit()) - return master == local_unit() + if master == local_unit(): + votes = votes or get_ssl_cert_master_votes() + if not peer_units() or (len(votes) == 1 and master in votes): + return True + + log("Did not get consensus from peers on who is ssl-cert-master " + "(%s)" % (votes), level=INFO) + + return False def is_ssl_enabled(): @@ -812,7 +832,21 @@ def is_ssl_enabled(): return True -def ensure_ssl_cert_master(use_oldest_peer=False): +def get_ssl_cert_master_votes(): + """Returns a list of unique votes.""" + votes = [] + # Gather election results from peers. These will need to be consistent. + for rid in relation_ids('cluster'): + for unit in related_units(rid): + m = relation_get(rid=rid, unit=unit, + attribute='ssl-cert-master') + if m is not None: + votes.append(m) + + return list(set(votes)) + + +def ensure_ssl_cert_master(): """Ensure that an ssl cert master has been elected. Normally the cluster leader will take control but we allow for this to be @@ -822,31 +856,19 @@ def ensure_ssl_cert_master(use_oldest_peer=False): if not is_ssl_enabled(): return False - elect = False - peers = peer_units() master_override = False - if use_oldest_peer: - elect = oldest_peer(peers) - else: - elect = is_elected_leader(CLUSTER_RES) + elect = is_elected_leader(CLUSTER_RES) # If no peers we allow this unit to elect itsef as master and do # sync immediately. - if not peers and not is_ssl_cert_master(): + if not peer_units(): elect = True master_override = True if elect: - masters = [] - for rid in relation_ids('cluster'): - for unit in related_units(rid): - m = relation_get(rid=rid, unit=unit, - attribute='ssl-cert-master') - if m is not None: - masters.append(m) - + votes = get_ssl_cert_master_votes() # We expect all peers to echo this setting - if not masters or 'unknown' in masters: + if not votes or 'unknown' in votes: log("Notifying peers this unit is ssl-cert-master", level=INFO) for rid in relation_ids('cluster'): settings = {'ssl-cert-master': local_unit()} @@ -855,10 +877,11 @@ def ensure_ssl_cert_master(use_oldest_peer=False): # Return now and wait for cluster-relation-changed (peer_echo) for # sync. return master_override - elif len(set(masters)) != 1 and local_unit() not in masters: - log("Did not get consensus from peers on who is ssl-cert-master " - "(%s) - waiting for current master to release before " - "self-electing" % (masters), level=INFO) + elif not is_ssl_cert_master(votes): + if not master_override: + log("Conscensus not reached - current master will need to " + "release", level=INFO) + return master_override if not is_ssl_cert_master(): @@ -887,15 +910,18 @@ def synchronize_ca(fatal=False): log("Syncing all endpoint certs since https-service-endpoints=True", level=DEBUG) paths_to_sync.append(SSL_DIR) - paths_to_sync.append(APACHE_SSL_DIR) paths_to_sync.append(CA_CERT_PATH) - elif is_str_true(config('use-https')): + + if is_str_true(config('use-https')): log("Syncing keystone-endpoint certs since use-https=True", level=DEBUG) paths_to_sync.append(SSL_DIR) paths_to_sync.append(APACHE_SSL_DIR) paths_to_sync.append(CA_CERT_PATH) + # Ensure unique + paths_to_sync = list(set(paths_to_sync)) + if not paths_to_sync: log("Nothing to sync - skipping", level=DEBUG) return {} @@ -908,8 +934,7 @@ def synchronize_ca(fatal=False): create_peer_service_actions('restart', ['apache2']) create_peer_actions(['update-ca-certificates']) - # Format here needs to match that used when peers request sync - synced_units = [unit.replace('/', '-') for unit in peer_units()] + cluster_rel_settings = {} retries = 3 while True: @@ -918,7 +943,12 @@ def synchronize_ca(fatal=False): update_hash_from_path(hash1, path) try: - unison_sync(paths_to_sync) + synced_units = unison_sync(paths_to_sync) + if synced_units: + # Format here needs to match that used when peers request sync + synced_units = [u.replace('/', '-') for u in synced_units] + cluster_rel_settings['ssl-synced-units'] = \ + json.dumps(synced_units) except: if fatal: raise @@ -947,10 +977,22 @@ def synchronize_ca(fatal=False): hash = hash1.hexdigest() log("Sending restart-services-trigger=%s to all peers" % (hash), level=DEBUG) + cluster_rel_settings['restart-services-trigger'] = hash log("Sync complete", level=DEBUG) - return {'restart-services-trigger': hash, - 'ssl-synced-units': json.dumps(synced_units)} + return cluster_rel_settings + + +def clear_ssl_synced_units(): + """Clear the 'synced' units record on the cluster relation. + + If new unit sync reauests are set this will ensure that a sync occurs when + the sync master receives the requests. + """ + log("Clearing ssl sync units", level=DEBUG) + for rid in relation_ids('cluster'): + relation_set(relation_id=rid, + relation_settings={'ssl-synced-units': None}) def update_hash_from_path(hash, path, recurse_depth=10): @@ -1058,11 +1100,11 @@ def get_ca(user='keystone', group='keystone'): '%s' % SSL_DIR]) subprocess.check_output(['chmod', '-R', 'g+rwx', '%s' % SSL_DIR]) - # Ensure a master has been elected and prefer this unit. Note that we - # prefer oldest peer as predicate since this action i normally only - # performed once at deploy time when the oldest peer should be the - # first to be ready. - ensure_ssl_cert_master(use_oldest_peer=True) + # Ensure a master is elected. This should cover the following cases: + # * single unit == 'oldest' unit is elected as master + # * multi unit + not clustered == 'oldest' unit is elcted as master + # * multi unit + clustered == cluster leader is elected as master + ensure_ssl_cert_master() ssl.CA_SINGLETON.append(ca) diff --git a/unit_tests/test_keystone_utils.py b/unit_tests/test_keystone_utils.py index 7e60dfa4..ac71496c 100644 --- a/unit_tests/test_keystone_utils.py +++ b/unit_tests/test_keystone_utils.py @@ -28,6 +28,7 @@ TO_PATCH = [ 'grant_role', 'configure_installation_source', 'is_elected_leader', + 'is_ssl_cert_master', 'https', 'peer_store_and_set', 'service_stop', @@ -380,45 +381,141 @@ class TestKeystoneUtils(CharmTestCase): self.assertTrue(utils.is_db_ready()) @patch.object(utils, 'peer_units') - @patch.object(utils, 'is_elected_leader') - @patch.object(utils, 'oldest_peer') @patch.object(utils, 'is_ssl_enabled') - def test_ensure_ssl_cert_master(self, mock_is_str_true, mock_oldest_peer, - mock_is_elected_leader, mock_peer_units): + def test_ensure_ssl_cert_master_no_ssl(self, mock_is_ssl_enabled, + mock_peer_units): + mock_is_ssl_enabled.return_value = False + self.assertFalse(utils.ensure_ssl_cert_master()) + self.assertFalse(self.relation_set.called) + + @patch.object(utils, 'peer_units') + @patch.object(utils, 'is_ssl_enabled') + def test_ensure_ssl_cert_master_ssl_no_peers(self, mock_is_ssl_enabled, + mock_peer_units): + def mock_rel_get(unit=None, **kwargs): + return None + + self.relation_get.side_effect = mock_rel_get + mock_is_ssl_enabled.return_value = True self.relation_ids.return_value = ['cluster:0'] self.local_unit.return_value = 'unit/0' - - mock_is_str_true.return_value = False - self.assertFalse(utils.ensure_ssl_cert_master()) - self.assertFalse(self.relation_set.called) - - mock_is_elected_leader.return_value = False - self.assertFalse(utils.ensure_ssl_cert_master()) - self.assertFalse(self.relation_set.called) - - mock_is_str_true.return_value = True - mock_is_elected_leader.return_value = False - mock_peer_units.return_value = ['unit/0'] - self.assertFalse(utils.ensure_ssl_cert_master()) - self.assertFalse(self.relation_set.called) - + self.related_units.return_value = [] mock_peer_units.return_value = [] + # This should get ignored since we are overriding + self.is_ssl_cert_master.return_value = False + self.is_elected_leader.return_value = False self.assertTrue(utils.ensure_ssl_cert_master()) settings = {'ssl-cert-master': 'unit/0'} self.relation_set.assert_called_with(relation_id='cluster:0', relation_settings=settings) - self.relation_set.reset_mock() - self.assertTrue(utils.ensure_ssl_cert_master(use_oldest_peer=True)) + @patch.object(utils, 'peer_units') + @patch.object(utils, 'is_ssl_enabled') + def test_ensure_ssl_cert_master_ssl_master_no_peers(self, + mock_is_ssl_enabled, + mock_peer_units): + def mock_rel_get(unit=None, **kwargs): + if unit == 'unit/0': + return 'unit/0' + + return None + + self.relation_get.side_effect = mock_rel_get + mock_is_ssl_enabled.return_value = True + self.relation_ids.return_value = ['cluster:0'] + self.local_unit.return_value = 'unit/0' + self.related_units.return_value = [] + mock_peer_units.return_value = [] + # This should get ignored since we are overriding + self.is_ssl_cert_master.return_value = False + self.is_elected_leader.return_value = False + self.assertTrue(utils.ensure_ssl_cert_master()) settings = {'ssl-cert-master': 'unit/0'} self.relation_set.assert_called_with(relation_id='cluster:0', relation_settings=settings) - self.relation_set.reset_mock() - mock_peer_units.return_value = ['unit/0'] + @patch.object(utils, 'peer_units') + @patch.object(utils, 'is_ssl_enabled') + def test_ensure_ssl_cert_master_ssl_not_leader(self, mock_is_ssl_enabled, + mock_peer_units): + mock_is_ssl_enabled.return_value = True + self.relation_ids.return_value = ['cluster:0'] + self.local_unit.return_value = 'unit/0' + mock_peer_units.return_value = ['unit/1'] + self.is_ssl_cert_master.return_value = False + self.is_elected_leader.return_value = False + self.assertFalse(utils.ensure_ssl_cert_master()) + self.assertFalse(self.relation_set.called) + + @patch.object(utils, 'peer_units') + @patch.object(utils, 'is_ssl_enabled') + def test_ensure_ssl_cert_master_is_leader_new_peer(self, + mock_is_ssl_enabled, + mock_peer_units): + def mock_rel_get(unit=None, **kwargs): + if unit == 'unit/0': + return 'unit/0' + + return 'unknown' + + self.relation_get.side_effect = mock_rel_get + mock_is_ssl_enabled.return_value = True + self.relation_ids.return_value = ['cluster:0'] + self.local_unit.return_value = 'unit/0' + mock_peer_units.return_value = ['unit/1'] + self.related_units.return_value = ['unit/1'] + self.is_ssl_cert_master.return_value = False + self.is_elected_leader.return_value = True self.assertFalse(utils.ensure_ssl_cert_master()) - self.assertFalse(utils.ensure_ssl_cert_master(use_oldest_peer=True)) settings = {'ssl-cert-master': 'unit/0'} self.relation_set.assert_called_with(relation_id='cluster:0', relation_settings=settings) - self.relation_set.reset_mock() + + @patch.object(utils, 'peer_units') + @patch.object(utils, 'is_ssl_enabled') + def test_ensure_ssl_cert_master_is_leader_no_new_peer(self, + mock_is_ssl_enabled, + mock_peer_units): + def mock_rel_get(unit=None, **kwargs): + if unit == 'unit/0': + return 'unit/0' + + return 'unit/0' + + self.relation_get.side_effect = mock_rel_get + mock_is_ssl_enabled.return_value = True + self.relation_ids.return_value = ['cluster:0'] + self.local_unit.return_value = 'unit/0' + mock_peer_units.return_value = ['unit/1'] + self.related_units.return_value = ['unit/1'] + self.is_ssl_cert_master.return_value = False + self.is_elected_leader.return_value = True + self.assertFalse(utils.ensure_ssl_cert_master()) + self.assertFalse(self.relation_set.called) + + @patch.object(utils, 'peer_units') + @patch.object(utils, 'is_ssl_enabled') + def test_ensure_ssl_cert_master_is_leader_bad_votes(self, + mock_is_ssl_enabled, + mock_peer_units): + counter = {0: 0} + + def mock_rel_get(unit=None, **kwargs): + """Returns a mix of votes.""" + if unit == 'unit/0': + return 'unit/0' + + ret = 'unit/%d' % (counter[0]) + counter[0] += 1 + return ret + + self.relation_get.side_effect = mock_rel_get + mock_is_ssl_enabled.return_value = True + self.relation_ids.return_value = ['cluster:0'] + self.local_unit.return_value = 'unit/0' + mock_peer_units.return_value = ['unit/1'] + self.related_units.return_value = ['unit/1'] + self.is_ssl_cert_master.return_value = False + self.is_elected_leader.return_value = True + self.assertFalse(utils.ensure_ssl_cert_master()) + self.assertFalse(self.relation_set.called) From 1915bb6852f220abc49242a231bd3e5adf921f8a Mon Sep 17 00:00:00 2001 From: Edward Hope-Morley Date: Mon, 16 Feb 2015 11:25:45 +0000 Subject: [PATCH 34/48] [hopem,r=] Synced charmhelpers and now using bool_from_string --- .../contrib/openstack/amulet/deployment.py | 7 +- hooks/charmhelpers/contrib/python/packages.py | 4 +- hooks/charmhelpers/core/fstab.py | 4 +- hooks/charmhelpers/core/host.py | 10 +- hooks/charmhelpers/core/strutils.py | 38 ++ hooks/charmhelpers/core/sysctl.py | 4 +- hooks/charmhelpers/core/templating.py | 6 +- hooks/charmhelpers/core/unitdata.py | 477 ++++++++++++++++++ hooks/charmhelpers/fetch/archiveurl.py | 20 +- hooks/charmhelpers/fetch/giturl.py | 2 +- hooks/keystone_hooks.py | 9 +- hooks/keystone_utils.py | 23 +- .../contrib/openstack/amulet/deployment.py | 7 +- 13 files changed, 566 insertions(+), 45 deletions(-) create mode 100644 hooks/charmhelpers/core/strutils.py create mode 100644 hooks/charmhelpers/core/unitdata.py diff --git a/hooks/charmhelpers/contrib/openstack/amulet/deployment.py b/hooks/charmhelpers/contrib/openstack/amulet/deployment.py index c50d3ec6..0cfeaa4c 100644 --- a/hooks/charmhelpers/contrib/openstack/amulet/deployment.py +++ b/hooks/charmhelpers/contrib/openstack/amulet/deployment.py @@ -71,16 +71,19 @@ class OpenStackAmuletDeployment(AmuletDeployment): services.append(this_service) use_source = ['mysql', 'mongodb', 'rabbitmq-server', 'ceph', 'ceph-osd', 'ceph-radosgw'] + # Openstack subordinate charms do not expose an origin option as that + # is controlled by the principle + ignore = ['neutron-openvswitch'] if self.openstack: for svc in services: - if svc['name'] not in use_source: + if svc['name'] not in use_source + ignore: config = {'openstack-origin': self.openstack} self.d.configure(svc['name'], config) if self.source: for svc in services: - if svc['name'] in use_source: + if svc['name'] in use_source and svc['name'] not in ignore: config = {'source': self.source} self.d.configure(svc['name'], config) diff --git a/hooks/charmhelpers/contrib/python/packages.py b/hooks/charmhelpers/contrib/python/packages.py index d848a120..8659516b 100644 --- a/hooks/charmhelpers/contrib/python/packages.py +++ b/hooks/charmhelpers/contrib/python/packages.py @@ -17,8 +17,6 @@ # You should have received a copy of the GNU Lesser General Public License # along with charm-helpers. If not, see . -__author__ = "Jorge Niedbalski " - from charmhelpers.fetch import apt_install, apt_update from charmhelpers.core.hookenv import log @@ -29,6 +27,8 @@ except ImportError: apt_install('python-pip') from pip import main as pip_execute +__author__ = "Jorge Niedbalski " + def parse_options(given, available): """Given a set of options, check if available""" diff --git a/hooks/charmhelpers/core/fstab.py b/hooks/charmhelpers/core/fstab.py index be7de248..9cdcc886 100644 --- a/hooks/charmhelpers/core/fstab.py +++ b/hooks/charmhelpers/core/fstab.py @@ -17,11 +17,11 @@ # You should have received a copy of the GNU Lesser General Public License # along with charm-helpers. If not, see . -__author__ = 'Jorge Niedbalski R. ' - import io import os +__author__ = 'Jorge Niedbalski R. ' + class Fstab(io.FileIO): """This class extends file in order to implement a file reader/writer diff --git a/hooks/charmhelpers/core/host.py b/hooks/charmhelpers/core/host.py index cf2cbe14..b771c611 100644 --- a/hooks/charmhelpers/core/host.py +++ b/hooks/charmhelpers/core/host.py @@ -191,11 +191,11 @@ def mkdir(path, owner='root', group='root', perms=0o555, force=False): def write_file(path, content, owner='root', group='root', perms=0o444): - """Create or overwrite a file with the contents of a string""" + """Create or overwrite a file with the contents of a byte string.""" log("Writing file {} {}:{} {:o}".format(path, owner, group, perms)) uid = pwd.getpwnam(owner).pw_uid gid = grp.getgrnam(group).gr_gid - with open(path, 'w') as target: + with open(path, 'wb') as target: os.fchown(target.fileno(), uid, gid) os.fchmod(target.fileno(), perms) target.write(content) @@ -305,11 +305,11 @@ def restart_on_change(restart_map, stopstart=False): ceph_client_changed function. """ def wrap(f): - def wrapped_f(*args): + def wrapped_f(*args, **kwargs): checksums = {} for path in restart_map: checksums[path] = file_hash(path) - f(*args) + f(*args, **kwargs) restarts = [] for path in restart_map: if checksums[path] != file_hash(path): @@ -361,7 +361,7 @@ def list_nics(nic_type): ip_output = (line for line in ip_output if line) for line in ip_output: if line.split()[1].startswith(int_type): - matched = re.search('.*: (bond[0-9]+\.[0-9]+)@.*', line) + matched = re.search('.*: (' + int_type + r'[0-9]+\.[0-9]+)@.*', line) if matched: interface = matched.groups()[0] else: diff --git a/hooks/charmhelpers/core/strutils.py b/hooks/charmhelpers/core/strutils.py new file mode 100644 index 00000000..668753ba --- /dev/null +++ b/hooks/charmhelpers/core/strutils.py @@ -0,0 +1,38 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- + +# Copyright 2014-2015 Canonical Limited. +# +# This file is part of charm-helpers. +# +# charm-helpers is free software: you can redistribute it and/or modify +# it under the terms of the GNU Lesser General Public License version 3 as +# published by the Free Software Foundation. +# +# charm-helpers is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Lesser General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public License +# along with charm-helpers. If not, see . + + +def bool_from_string(value): + """Interpret string value as boolean. + + Returns True if value translates to True otherwise False. + """ + if isinstance(value, str): + value = value.lower() + else: + msg = "Unable to interpret non-string value '%s' as boolean" % (value) + raise ValueError(msg) + + if value in ['y', 'yes', 'true', 't']: + return True + elif value in ['n', 'no', 'false', 'f']: + return False + + msg = "Unable to interpret string value '%s' as boolean" % (value) + raise ValueError(msg) diff --git a/hooks/charmhelpers/core/sysctl.py b/hooks/charmhelpers/core/sysctl.py index 8e1b9eeb..21cc8ab2 100644 --- a/hooks/charmhelpers/core/sysctl.py +++ b/hooks/charmhelpers/core/sysctl.py @@ -17,8 +17,6 @@ # You should have received a copy of the GNU Lesser General Public License # along with charm-helpers. If not, see . -__author__ = 'Jorge Niedbalski R. ' - import yaml from subprocess import check_call @@ -29,6 +27,8 @@ from charmhelpers.core.hookenv import ( ERROR, ) +__author__ = 'Jorge Niedbalski R. ' + def create(sysctl_dict, sysctl_file): """Creates a sysctl.conf file from a YAML associative array diff --git a/hooks/charmhelpers/core/templating.py b/hooks/charmhelpers/core/templating.py index 97669092..45319998 100644 --- a/hooks/charmhelpers/core/templating.py +++ b/hooks/charmhelpers/core/templating.py @@ -21,7 +21,7 @@ from charmhelpers.core import hookenv def render(source, target, context, owner='root', group='root', - perms=0o444, templates_dir=None): + perms=0o444, templates_dir=None, encoding='UTF-8'): """ Render a template. @@ -64,5 +64,5 @@ def render(source, target, context, owner='root', group='root', level=hookenv.ERROR) raise e content = template.render(context) - host.mkdir(os.path.dirname(target), owner, group) - host.write_file(target, content, owner, group, perms) + host.mkdir(os.path.dirname(target), owner, group, perms=0o755) + host.write_file(target, content.encode(encoding), owner, group, perms) diff --git a/hooks/charmhelpers/core/unitdata.py b/hooks/charmhelpers/core/unitdata.py new file mode 100644 index 00000000..3000134a --- /dev/null +++ b/hooks/charmhelpers/core/unitdata.py @@ -0,0 +1,477 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- +# +# Copyright 2014-2015 Canonical Limited. +# +# This file is part of charm-helpers. +# +# charm-helpers is free software: you can redistribute it and/or modify +# it under the terms of the GNU Lesser General Public License version 3 as +# published by the Free Software Foundation. +# +# charm-helpers is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Lesser General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public License +# along with charm-helpers. If not, see . +# +# +# Authors: +# Kapil Thangavelu +# +""" +Intro +----- + +A simple way to store state in units. This provides a key value +storage with support for versioned, transactional operation, +and can calculate deltas from previous values to simplify unit logic +when processing changes. + + +Hook Integration +---------------- + +There are several extant frameworks for hook execution, including + + - charmhelpers.core.hookenv.Hooks + - charmhelpers.core.services.ServiceManager + +The storage classes are framework agnostic, one simple integration is +via the HookData contextmanager. It will record the current hook +execution environment (including relation data, config data, etc.), +setup a transaction and allow easy access to the changes from +previously seen values. One consequence of the integration is the +reservation of particular keys ('rels', 'unit', 'env', 'config', +'charm_revisions') for their respective values. + +Here's a fully worked integration example using hookenv.Hooks:: + + from charmhelper.core import hookenv, unitdata + + hook_data = unitdata.HookData() + db = unitdata.kv() + hooks = hookenv.Hooks() + + @hooks.hook + def config_changed(): + # Print all changes to configuration from previously seen + # values. + for changed, (prev, cur) in hook_data.conf.items(): + print('config changed', changed, + 'previous value', prev, + 'current value', cur) + + # Get some unit specific bookeeping + if not db.get('pkg_key'): + key = urllib.urlopen('https://example.com/pkg_key').read() + db.set('pkg_key', key) + + # Directly access all charm config as a mapping. + conf = db.getrange('config', True) + + # Directly access all relation data as a mapping + rels = db.getrange('rels', True) + + if __name__ == '__main__': + with hook_data(): + hook.execute() + + +A more basic integration is via the hook_scope context manager which simply +manages transaction scope (and records hook name, and timestamp):: + + >>> from unitdata import kv + >>> db = kv() + >>> with db.hook_scope('install'): + ... # do work, in transactional scope. + ... db.set('x', 1) + >>> db.get('x') + 1 + + +Usage +----- + +Values are automatically json de/serialized to preserve basic typing +and complex data struct capabilities (dicts, lists, ints, booleans, etc). + +Individual values can be manipulated via get/set:: + + >>> kv.set('y', True) + >>> kv.get('y') + True + + # We can set complex values (dicts, lists) as a single key. + >>> kv.set('config', {'a': 1, 'b': True'}) + + # Also supports returning dictionaries as a record which + # provides attribute access. + >>> config = kv.get('config', record=True) + >>> config.b + True + + +Groups of keys can be manipulated with update/getrange:: + + >>> kv.update({'z': 1, 'y': 2}, prefix="gui.") + >>> kv.getrange('gui.', strip=True) + {'z': 1, 'y': 2} + +When updating values, its very helpful to understand which values +have actually changed and how have they changed. The storage +provides a delta method to provide for this:: + + >>> data = {'debug': True, 'option': 2} + >>> delta = kv.delta(data, 'config.') + >>> delta.debug.previous + None + >>> delta.debug.current + True + >>> delta + {'debug': (None, True), 'option': (None, 2)} + +Note the delta method does not persist the actual change, it needs to +be explicitly saved via 'update' method:: + + >>> kv.update(data, 'config.') + +Values modified in the context of a hook scope retain historical values +associated to the hookname. + + >>> with db.hook_scope('config-changed'): + ... db.set('x', 42) + >>> db.gethistory('x') + [(1, u'x', 1, u'install', u'2015-01-21T16:49:30.038372'), + (2, u'x', 42, u'config-changed', u'2015-01-21T16:49:30.038786')] + +""" + +import collections +import contextlib +import datetime +import json +import os +import pprint +import sqlite3 +import sys + +__author__ = 'Kapil Thangavelu ' + + +class Storage(object): + """Simple key value database for local unit state within charms. + + Modifications are automatically committed at hook exit. That's + currently regardless of exit code. + + To support dicts, lists, integer, floats, and booleans values + are automatically json encoded/decoded. + """ + def __init__(self, path=None): + self.db_path = path + if path is None: + self.db_path = os.path.join( + os.environ.get('CHARM_DIR', ''), '.unit-state.db') + self.conn = sqlite3.connect('%s' % self.db_path) + self.cursor = self.conn.cursor() + self.revision = None + self._closed = False + self._init() + + def close(self): + if self._closed: + return + self.flush(False) + self.cursor.close() + self.conn.close() + self._closed = True + + def _scoped_query(self, stmt, params=None): + if params is None: + params = [] + return stmt, params + + def get(self, key, default=None, record=False): + self.cursor.execute( + *self._scoped_query( + 'select data from kv where key=?', [key])) + result = self.cursor.fetchone() + if not result: + return default + if record: + return Record(json.loads(result[0])) + return json.loads(result[0]) + + def getrange(self, key_prefix, strip=False): + stmt = "select key, data from kv where key like '%s%%'" % key_prefix + self.cursor.execute(*self._scoped_query(stmt)) + result = self.cursor.fetchall() + + if not result: + return None + if not strip: + key_prefix = '' + return dict([ + (k[len(key_prefix):], json.loads(v)) for k, v in result]) + + def update(self, mapping, prefix=""): + for k, v in mapping.items(): + self.set("%s%s" % (prefix, k), v) + + def unset(self, key): + self.cursor.execute('delete from kv where key=?', [key]) + if self.revision and self.cursor.rowcount: + self.cursor.execute( + 'insert into kv_revisions values (?, ?, ?)', + [key, self.revision, json.dumps('DELETED')]) + + def set(self, key, value): + serialized = json.dumps(value) + + self.cursor.execute( + 'select data from kv where key=?', [key]) + exists = self.cursor.fetchone() + + # Skip mutations to the same value + if exists: + if exists[0] == serialized: + return value + + if not exists: + self.cursor.execute( + 'insert into kv (key, data) values (?, ?)', + (key, serialized)) + else: + self.cursor.execute(''' + update kv + set data = ? + where key = ?''', [serialized, key]) + + # Save + if not self.revision: + return value + + self.cursor.execute( + 'select 1 from kv_revisions where key=? and revision=?', + [key, self.revision]) + exists = self.cursor.fetchone() + + if not exists: + self.cursor.execute( + '''insert into kv_revisions ( + revision, key, data) values (?, ?, ?)''', + (self.revision, key, serialized)) + else: + self.cursor.execute( + ''' + update kv_revisions + set data = ? + where key = ? + and revision = ?''', + [serialized, key, self.revision]) + + return value + + def delta(self, mapping, prefix): + """ + return a delta containing values that have changed. + """ + previous = self.getrange(prefix, strip=True) + if not previous: + pk = set() + else: + pk = set(previous.keys()) + ck = set(mapping.keys()) + delta = DeltaSet() + + # added + for k in ck.difference(pk): + delta[k] = Delta(None, mapping[k]) + + # removed + for k in pk.difference(ck): + delta[k] = Delta(previous[k], None) + + # changed + for k in pk.intersection(ck): + c = mapping[k] + p = previous[k] + if c != p: + delta[k] = Delta(p, c) + + return delta + + @contextlib.contextmanager + def hook_scope(self, name=""): + """Scope all future interactions to the current hook execution + revision.""" + assert not self.revision + self.cursor.execute( + 'insert into hooks (hook, date) values (?, ?)', + (name or sys.argv[0], + datetime.datetime.utcnow().isoformat())) + self.revision = self.cursor.lastrowid + try: + yield self.revision + self.revision = None + except: + self.flush(False) + self.revision = None + raise + else: + self.flush() + + def flush(self, save=True): + if save: + self.conn.commit() + elif self._closed: + return + else: + self.conn.rollback() + + def _init(self): + self.cursor.execute(''' + create table if not exists kv ( + key text, + data text, + primary key (key) + )''') + self.cursor.execute(''' + create table if not exists kv_revisions ( + key text, + revision integer, + data text, + primary key (key, revision) + )''') + self.cursor.execute(''' + create table if not exists hooks ( + version integer primary key autoincrement, + hook text, + date text + )''') + self.conn.commit() + + def gethistory(self, key, deserialize=False): + self.cursor.execute( + ''' + select kv.revision, kv.key, kv.data, h.hook, h.date + from kv_revisions kv, + hooks h + where kv.key=? + and kv.revision = h.version + ''', [key]) + if deserialize is False: + return self.cursor.fetchall() + return map(_parse_history, self.cursor.fetchall()) + + def debug(self, fh=sys.stderr): + self.cursor.execute('select * from kv') + pprint.pprint(self.cursor.fetchall(), stream=fh) + self.cursor.execute('select * from kv_revisions') + pprint.pprint(self.cursor.fetchall(), stream=fh) + + +def _parse_history(d): + return (d[0], d[1], json.loads(d[2]), d[3], + datetime.datetime.strptime(d[-1], "%Y-%m-%dT%H:%M:%S.%f")) + + +class HookData(object): + """Simple integration for existing hook exec frameworks. + + Records all unit information, and stores deltas for processing + by the hook. + + Sample:: + + from charmhelper.core import hookenv, unitdata + + changes = unitdata.HookData() + db = unitdata.kv() + hooks = hookenv.Hooks() + + @hooks.hook + def config_changed(): + # View all changes to configuration + for changed, (prev, cur) in changes.conf.items(): + print('config changed', changed, + 'previous value', prev, + 'current value', cur) + + # Get some unit specific bookeeping + if not db.get('pkg_key'): + key = urllib.urlopen('https://example.com/pkg_key').read() + db.set('pkg_key', key) + + if __name__ == '__main__': + with changes(): + hook.execute() + + """ + def __init__(self): + self.kv = kv() + self.conf = None + self.rels = None + + @contextlib.contextmanager + def __call__(self): + from charmhelpers.core import hookenv + hook_name = hookenv.hook_name() + + with self.kv.hook_scope(hook_name): + self._record_charm_version(hookenv.charm_dir()) + delta_config, delta_relation = self._record_hook(hookenv) + yield self.kv, delta_config, delta_relation + + def _record_charm_version(self, charm_dir): + # Record revisions.. charm revisions are meaningless + # to charm authors as they don't control the revision. + # so logic dependnent on revision is not particularly + # useful, however it is useful for debugging analysis. + charm_rev = open( + os.path.join(charm_dir, 'revision')).read().strip() + charm_rev = charm_rev or '0' + revs = self.kv.get('charm_revisions', []) + if charm_rev not in revs: + revs.append(charm_rev.strip() or '0') + self.kv.set('charm_revisions', revs) + + def _record_hook(self, hookenv): + data = hookenv.execution_environment() + self.conf = conf_delta = self.kv.delta(data['conf'], 'config') + self.rels = rels_delta = self.kv.delta(data['rels'], 'rels') + self.kv.set('env', data['env']) + self.kv.set('unit', data['unit']) + self.kv.set('relid', data.get('relid')) + return conf_delta, rels_delta + + +class Record(dict): + + __slots__ = () + + def __getattr__(self, k): + if k in self: + return self[k] + raise AttributeError(k) + + +class DeltaSet(Record): + + __slots__ = () + + +Delta = collections.namedtuple('Delta', ['previous', 'current']) + + +_KV = None + + +def kv(): + global _KV + if _KV is None: + _KV = Storage() + return _KV diff --git a/hooks/charmhelpers/fetch/archiveurl.py b/hooks/charmhelpers/fetch/archiveurl.py index d25a0ddd..8dfce505 100644 --- a/hooks/charmhelpers/fetch/archiveurl.py +++ b/hooks/charmhelpers/fetch/archiveurl.py @@ -18,6 +18,16 @@ import os import hashlib import re +from charmhelpers.fetch import ( + BaseFetchHandler, + UnhandledSource +) +from charmhelpers.payload.archive import ( + get_archive_handler, + extract, +) +from charmhelpers.core.host import mkdir, check_hash + import six if six.PY3: from urllib.request import ( @@ -35,16 +45,6 @@ else: ) from urlparse import urlparse, urlunparse, parse_qs -from charmhelpers.fetch import ( - BaseFetchHandler, - UnhandledSource -) -from charmhelpers.payload.archive import ( - get_archive_handler, - extract, -) -from charmhelpers.core.host import mkdir, check_hash - def splituser(host): '''urllib.splituser(), but six's support of this seems broken''' diff --git a/hooks/charmhelpers/fetch/giturl.py b/hooks/charmhelpers/fetch/giturl.py index 5376786b..93aae87b 100644 --- a/hooks/charmhelpers/fetch/giturl.py +++ b/hooks/charmhelpers/fetch/giturl.py @@ -32,7 +32,7 @@ except ImportError: apt_install("python-git") from git import Repo -from git.exc import GitCommandError +from git.exc import GitCommandError # noqa E402 class GitUrlFetchHandler(BaseFetchHandler): diff --git a/hooks/keystone_hooks.py b/hooks/keystone_hooks.py index 0c8dcf95..1a861b7f 100755 --- a/hooks/keystone_hooks.py +++ b/hooks/keystone_hooks.py @@ -32,6 +32,10 @@ from charmhelpers.core.host import ( restart_on_change, ) +from charmhelpers.core.strutils import ( + bool_from_string, +) + from charmhelpers.fetch import ( apt_install, apt_update, filter_installed_packages @@ -64,7 +68,6 @@ from keystone_utils import ( CA_CERT_PATH, ensure_permissions, get_ssl_sync_request_units, - is_str_true, is_ssl_cert_master, is_db_ready, clear_ssl_synced_units, @@ -293,10 +296,10 @@ def send_ssl_sync_request(): """ unit = local_unit().replace('/', '-') count = 0 - if is_str_true(config('use-https')): + if bool_from_string(config('use-https')): count += 1 - if is_str_true(config('https-service-endpoints')): + if bool_from_string(config('https-service-endpoints')): count += 2 if count: diff --git a/hooks/keystone_utils.py b/hooks/keystone_utils.py index d62784d0..4fd302f0 100644 --- a/hooks/keystone_utils.py +++ b/hooks/keystone_utils.py @@ -48,6 +48,10 @@ from charmhelpers.core.host import ( write_file, ) +from charmhelpers.core.strutils import ( + bool_from_string, +) + import charmhelpers.contrib.unison as unison from charmhelpers.core.decorators import ( @@ -226,13 +230,6 @@ valid_services = { } -def is_str_true(value): - if value and value.lower() in ['true', 'yes']: - return True - - return False - - def resource_map(): ''' Dynamically generate a map of resources that will be managed for a single @@ -823,8 +820,8 @@ def is_ssl_cert_master(votes=None): def is_ssl_enabled(): # Don't do anything if we are not in ssl/https mode - if (is_str_true(config('use-https')) or - is_str_true(config('https-service-endpoints'))): + if (bool_from_string(config('use-https')) or + bool_from_string(config('https-service-endpoints'))): log("SSL/HTTPS is enabled", level=DEBUG) return True @@ -906,13 +903,13 @@ def synchronize_ca(fatal=False): """ paths_to_sync = [SYNC_FLAGS_DIR] - if is_str_true(config('https-service-endpoints')): + if bool_from_string(config('https-service-endpoints')): log("Syncing all endpoint certs since https-service-endpoints=True", level=DEBUG) paths_to_sync.append(SSL_DIR) paths_to_sync.append(CA_CERT_PATH) - if is_str_true(config('use-https')): + if bool_from_string(config('use-https')): log("Syncing keystone-endpoint certs since use-https=True", level=DEBUG) paths_to_sync.append(SSL_DIR) @@ -1150,7 +1147,7 @@ def add_service_to_keystone(relation_id=None, remote_unit=None): relation_data["auth_port"] = config('admin-port') relation_data["service_port"] = config('service-port') relation_data["region"] = config('region') - if is_str_true(config('https-service-endpoints')): + if bool_from_string(config('https-service-endpoints')): # Pass CA cert as client will need it to # verify https connections ca = get_ca(user=SSH_USER) @@ -1290,7 +1287,7 @@ def add_service_to_keystone(relation_id=None, remote_unit=None): relation_data["auth_protocol"] = "http" relation_data["service_protocol"] = "http" # generate or get a new cert/key for service if set to manage certs. - if is_str_true(config('https-service-endpoints')): + if bool_from_string(config('https-service-endpoints')): ca = get_ca(user=SSH_USER) # NOTE(jamespage) may have multiple cns to deal with to iterate https_cns = set(https_cns) diff --git a/tests/charmhelpers/contrib/openstack/amulet/deployment.py b/tests/charmhelpers/contrib/openstack/amulet/deployment.py index c50d3ec6..0cfeaa4c 100644 --- a/tests/charmhelpers/contrib/openstack/amulet/deployment.py +++ b/tests/charmhelpers/contrib/openstack/amulet/deployment.py @@ -71,16 +71,19 @@ class OpenStackAmuletDeployment(AmuletDeployment): services.append(this_service) use_source = ['mysql', 'mongodb', 'rabbitmq-server', 'ceph', 'ceph-osd', 'ceph-radosgw'] + # Openstack subordinate charms do not expose an origin option as that + # is controlled by the principle + ignore = ['neutron-openvswitch'] if self.openstack: for svc in services: - if svc['name'] not in use_source: + if svc['name'] not in use_source + ignore: config = {'openstack-origin': self.openstack} self.d.configure(svc['name'], config) if self.source: for svc in services: - if svc['name'] in use_source: + if svc['name'] in use_source and svc['name'] not in ignore: config = {'source': self.source} self.d.configure(svc['name'], config) From b7d308bc3822409090b3be94da444863a150c7d7 Mon Sep 17 00:00:00 2001 From: Edward Hope-Morley Date: Mon, 16 Feb 2015 11:38:34 +0000 Subject: [PATCH 35/48] more --- hooks/keystone_context.py | 17 +++++++++++++---- 1 file changed, 13 insertions(+), 4 deletions(-) diff --git a/hooks/keystone_context.py b/hooks/keystone_context.py index 6094461e..2d40ae99 100644 --- a/hooks/keystone_context.py +++ b/hooks/keystone_context.py @@ -21,6 +21,10 @@ from charmhelpers.core.hookenv import ( INFO, ) +from charmhelpers.core.strutils import ( + bool_from_string, +) + from charmhelpers.contrib.hahelpers.apache import install_ca_cert CA_CERT_PATH = '/usr/local/share/ca-certificates/keystone_juju_ca_cert.crt' @@ -179,8 +183,12 @@ class KeystoneContext(context.OSContextGenerator): singlenode_mode=True) ctxt['public_port'] = determine_api_port(api_port('keystone-public'), singlenode_mode=True) - ctxt['debug'] = config('debug') in ['yes', 'true', 'True'] - ctxt['verbose'] = config('verbose') in ['yes', 'true', 'True'] + + debug = config('debug') + ctxt['debug'] = debug and bool_from_string(debug) + verbose = config('verbose') + ctxt['verbose'] = verbose and bool_from_string(verbose) + ctxt['identity_backend'] = config('identity-backend') ctxt['assignment_backend'] = config('assignment-backend') if config('identity-backend') == 'ldap': @@ -194,7 +202,8 @@ class KeystoneContext(context.OSContextGenerator): flags = context.config_flags_parser(ldap_flags) ctxt['ldap_config_flags'] = flags - if config('enable-pki') not in ['false', 'False', 'no', 'No']: + enable_pki = config('enable-pki') + if enable_pki and bool_from_string(enable_pki): ctxt['signing'] = True # Base endpoint URL's which are used in keystone responses @@ -214,7 +223,7 @@ class KeystoneLoggingContext(context.OSContextGenerator): def __call__(self): ctxt = {} debug = config('debug') - if debug and debug.lower() in ['yes', 'true']: + if debug and bool_from_string(debug): ctxt['root_level'] = 'DEBUG' return ctxt From fceb1ca3f0f7f8e7bd51b882ad99ea42fcbed776 Mon Sep 17 00:00:00 2001 From: Edward Hope-Morley Date: Mon, 16 Feb 2015 12:06:32 +0000 Subject: [PATCH 36/48] re-sync ch --- hooks/charmhelpers/core/strutils.py | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/hooks/charmhelpers/core/strutils.py b/hooks/charmhelpers/core/strutils.py index 668753ba..efc4402e 100644 --- a/hooks/charmhelpers/core/strutils.py +++ b/hooks/charmhelpers/core/strutils.py @@ -17,18 +17,22 @@ # You should have received a copy of the GNU Lesser General Public License # along with charm-helpers. If not, see . +import six + def bool_from_string(value): """Interpret string value as boolean. Returns True if value translates to True otherwise False. """ - if isinstance(value, str): - value = value.lower() + if isinstance(value, six.string_types): + value = six.text_type(value) else: msg = "Unable to interpret non-string value '%s' as boolean" % (value) raise ValueError(msg) + value = value.strip().lower() + if value in ['y', 'yes', 'true', 't']: return True elif value in ['n', 'no', 'false', 'f']: From f40c92f81bc8a34989918f7b15841dea10353f7f Mon Sep 17 00:00:00 2001 From: Edward Hope-Morley Date: Mon, 16 Feb 2015 14:48:02 +0000 Subject: [PATCH 37/48] [trivial] keystone_utils code cleanup (no functional changes) --- hooks/keystone_utils.py | 80 +++++++++++++++++++++-------------------- 1 file changed, 41 insertions(+), 39 deletions(-) diff --git a/hooks/keystone_utils.py b/hooks/keystone_utils.py index 4fd302f0..26d6eb74 100644 --- a/hooks/keystone_utils.py +++ b/hooks/keystone_utils.py @@ -231,10 +231,9 @@ valid_services = { def resource_map(): - ''' - Dynamically generate a map of resources that will be managed for a single - hook execution. - ''' + """Dynamically generate a map of resources that will be managed for a + single hook execution. + """ resource_map = deepcopy(BASE_RESOURCE_MAP) if os.path.exists('/etc/apache2/conf-available'): @@ -260,7 +259,7 @@ def restart_map(): def services(): - ''' Returns a list of services associate with this charm ''' + """Returns a list of services associate with this charm""" _services = [] for v in restart_map().values(): _services = _services + v @@ -268,7 +267,7 @@ def services(): def determine_ports(): - '''Assemble a list of API ports for services we are managing''' + """Assemble a list of API ports for services we are managing""" ports = [config('admin-port'), config('service-port')] return list(set(ports)) @@ -319,7 +318,7 @@ def do_openstack_upgrade(configs): def migrate_database(): - '''Runs keystone-manage to initialize a new database or migrate existing''' + """Runs keystone-manage to initialize a new database or migrate existing""" log('Migrating the keystone database.', level=INFO) service_stop('keystone') # NOTE(jamespage) > icehouse creates a log file as root so use @@ -334,7 +333,7 @@ def migrate_database(): # OLD def get_local_endpoint(): - """ Returns the URL for the local end-point bypassing haproxy/ssl """ + """Returns the URL for the local end-point bypassing haproxy/ssl""" if config('prefer-ipv6'): ipv6_addr = get_ipv6_addr(exc_list=[config('vip')])[0] endpoint_url = 'http://[%s]:{}/v2.0/' % ipv6_addr @@ -435,7 +434,7 @@ def create_endpoint_template(region, service, publicurl, adminurl, def create_tenant(name): - """ creates a tenant if it does not already exist """ + """Creates a tenant if it does not already exist""" import manager manager = manager.KeystoneManager(endpoint=get_local_endpoint(), token=get_admin_token()) @@ -449,7 +448,7 @@ def create_tenant(name): def create_user(name, password, tenant): - """ creates a user if it doesn't already exist, as a member of tenant """ + """Creates a user if it doesn't already exist, as a member of tenant""" import manager manager = manager.KeystoneManager(endpoint=get_local_endpoint(), token=get_admin_token()) @@ -468,7 +467,7 @@ def create_user(name, password, tenant): def create_role(name, user=None, tenant=None): - """ creates a role if it doesn't already exist. grants role to user """ + """Creates a role if it doesn't already exist. grants role to user""" import manager manager = manager.KeystoneManager(endpoint=get_local_endpoint(), token=get_admin_token()) @@ -495,7 +494,7 @@ def create_role(name, user=None, tenant=None): def grant_role(user, role, tenant): - """grant user+tenant a specific role""" + """Grant user and tenant a specific role""" import manager manager = manager.KeystoneManager(endpoint=get_local_endpoint(), token=get_admin_token()) @@ -642,7 +641,7 @@ def load_stored_passwords(path=SERVICE_PASSWD_PATH): def _migrate_service_passwords(): - ''' Migrate on-disk service passwords to peer storage ''' + """Migrate on-disk service passwords to peer storage""" if os.path.exists(SERVICE_PASSWD_PATH): log('Migrating on-disk stored passwords to peer storage') creds = load_stored_passwords() @@ -819,7 +818,6 @@ def is_ssl_cert_master(votes=None): def is_ssl_enabled(): - # Don't do anything if we are not in ssl/https mode if (bool_from_string(config('use-https')) or bool_from_string(config('https-service-endpoints'))): log("SSL/HTTPS is enabled", level=DEBUG) @@ -1076,8 +1074,8 @@ def synchronize_ca_if_changed(force=False, fatal=False): def get_ca(user='keystone', group='keystone'): - """ - Initialize a new CA object if one hasn't already been loaded. + """Initialize a new CA object if one hasn't already been loaded. + This will create a new CA or load an existing one. """ if not ssl.CA_SINGLETON: @@ -1129,6 +1127,12 @@ def add_service_to_keystone(relation_id=None, remote_unit=None): single = set(['service', 'region', 'public_url', 'admin_url', 'internal_url']) https_cns = [] + + if https(): + protocol = 'https' + else: + protocol = 'http' + if single.issubset(settings): # other end of relation advertised only one endpoint if 'None' in settings.itervalues(): @@ -1138,22 +1142,22 @@ def add_service_to_keystone(relation_id=None, remote_unit=None): # Check if clustered and use vip + haproxy ports if so relation_data["auth_host"] = resolve_address(ADMIN) relation_data["service_host"] = resolve_address(PUBLIC) - if https(): - relation_data["auth_protocol"] = "https" - relation_data["service_protocol"] = "https" - else: - relation_data["auth_protocol"] = "http" - relation_data["service_protocol"] = "http" + relation_data["auth_protocol"] = protocol + relation_data["service_protocol"] = protocol relation_data["auth_port"] = config('admin-port') relation_data["service_port"] = config('service-port') relation_data["region"] = config('region') - if bool_from_string(config('https-service-endpoints')): + + https_service_endpoints = config('https-service-endpoints') + if (https_service_endpoints and + bool_from_string(https_service_endpoints)): # Pass CA cert as client will need it to # verify https connections ca = get_ca(user=SSH_USER) ca_bundle = ca.get_ca_bundle() relation_data['https_keystone'] = 'True' relation_data['ca_cert'] = b64encode(ca_bundle) + # Allow the remote service to request creation of any additional # roles. Currently used by Horizon for role in get_requested_roles(settings): @@ -1181,8 +1185,8 @@ def add_service_to_keystone(relation_id=None, remote_unit=None): # NOTE(jamespage) internal IP for backwards compat for SSL certs internal_cn = urlparse.urlparse(settings['internal_url']).hostname https_cns.append(internal_cn) - https_cns.append( - urlparse.urlparse(settings['public_url']).hostname) + public_cn = urlparse.urlparse(settings['public_url']).hostname + https_cns.append(public_cn) https_cns.append(urlparse.urlparse(settings['admin_url']).hostname) else: # assemble multiple endpoints from relation data. service name @@ -1208,6 +1212,7 @@ def add_service_to_keystone(relation_id=None, remote_unit=None): if ep not in endpoints: endpoints[ep] = {} endpoints[ep][x] = v + services = [] https_cn = None for ep in endpoints: @@ -1228,6 +1233,7 @@ def add_service_to_keystone(relation_id=None, remote_unit=None): https_cns.append(internal_cn) https_cns.append(urlparse.urlparse(ep['public_url']).hostname) https_cns.append(urlparse.urlparse(ep['admin_url']).hostname) + service_username = '_'.join(services) # If an admin username prefix is provided, ensure all services use it. @@ -1253,8 +1259,7 @@ def add_service_to_keystone(relation_id=None, remote_unit=None): # Currently used by Swift and Ceilometer. for role in get_requested_roles(settings): log("Creating requested role: %s" % role) - create_role(role, service_username, - config('service-tenant')) + create_role(role, service_username, config('service-tenant')) # As of https://review.openstack.org/#change,4675, all nodes hosting # an endpoint(s) needs a service username and password assigned to @@ -1276,18 +1281,14 @@ def add_service_to_keystone(relation_id=None, remote_unit=None): "https_keystone": "False", "ssl_cert": "", "ssl_key": "", - "ca_cert": "" + "ca_cert": "", + "auth_protocol": protocol, + "service_protocol": protocol, } - # Check if https is enabled - if https(): - relation_data["auth_protocol"] = "https" - relation_data["service_protocol"] = "https" - else: - relation_data["auth_protocol"] = "http" - relation_data["service_protocol"] = "http" # generate or get a new cert/key for service if set to manage certs. - if bool_from_string(config('https-service-endpoints')): + https_service_endpoints = config('https-service-endpoints') + if https_service_endpoints and bool_from_string(https_service_endpoints): ca = get_ca(user=SSH_USER) # NOTE(jamespage) may have multiple cns to deal with to iterate https_cns = set(https_cns) @@ -1295,6 +1296,7 @@ def add_service_to_keystone(relation_id=None, remote_unit=None): cert, key = ca.get_cert_and_key(common_name=https_cn) relation_data['ssl_cert_{}'.format(https_cn)] = b64encode(cert) relation_data['ssl_key_{}'.format(https_cn)] = b64encode(key) + # NOTE(jamespage) for backwards compatibility cert, key = ca.get_cert_and_key(common_name=internal_cn) relation_data['ssl_cert'] = b64encode(cert) @@ -1303,8 +1305,7 @@ def add_service_to_keystone(relation_id=None, remote_unit=None): relation_data['ca_cert'] = b64encode(ca_bundle) relation_data['https_keystone'] = 'True' - peer_store_and_set(relation_id=relation_id, - **relation_data) + peer_store_and_set(relation_id=relation_id, **relation_data) def ensure_valid_service(service): @@ -1325,7 +1326,7 @@ def add_endpoint(region, service, publicurl, adminurl, internalurl): def get_requested_roles(settings): - ''' Retrieve any valid requested_roles from dict settings ''' + """Retrieve any valid requested_roles from dict settings""" if ('requested_roles' in settings and settings['requested_roles'] not in ['None', None]): return settings['requested_roles'].split(',') @@ -1334,6 +1335,7 @@ def get_requested_roles(settings): def setup_ipv6(): + """Check ipv6-mode validity and setup dependencies""" ubuntu_rel = lsb_release()['DISTRIB_CODENAME'].lower() if ubuntu_rel < "trusty": raise Exception("IPv6 is not supported in the charms for Ubuntu " From fb91126bc3e36e9e667eda21bbe960788dc7e398 Mon Sep 17 00:00:00 2001 From: Edward Hope-Morley Date: Mon, 16 Feb 2015 20:58:06 +0000 Subject: [PATCH 38/48] [hopem,r=] Ensure db is migrated in identity-relation hooks and also avoiding unecessarily re-migrating unless an upgrade is being performed. --- hooks/keystone_hooks.py | 22 ++++++++++------------ hooks/keystone_utils.py | 27 +++++++++++++++++++++++++-- 2 files changed, 35 insertions(+), 14 deletions(-) diff --git a/hooks/keystone_hooks.py b/hooks/keystone_hooks.py index 8daa9d60..bc4938d8 100755 --- a/hooks/keystone_hooks.py +++ b/hooks/keystone_hooks.py @@ -149,6 +149,7 @@ def config_changed(): # Update relations since SSL may have been configured. If we have peer # units we can rely on the sync to do this in cluster relation. if is_elected_leader(CLUSTER_RES) and not peer_units(): + migrate_database(force=True) update_all_identity_relation_units() for rid in relation_ids('identity-admin'): @@ -198,17 +199,12 @@ def update_all_identity_relation_units(check_db_ready=True): level=INFO) return - try: - migrate_database() - except Exception as exc: - log("Database initialisation failed (%s) - db not ready?" % (exc), - level=WARNING) - else: - ensure_initial_admin(config) - log('Firing identity_changed hook for all related services.') - for rid in relation_ids('identity-service'): - for unit in related_units(rid): - identity_changed(relation_id=rid, remote_unit=unit) + migrate_database() + ensure_initial_admin(config) + log('Firing identity_changed hook for all related services.') + for rid in relation_ids('identity-service'): + for unit in related_units(rid): + identity_changed(relation_id=rid, remote_unit=unit) @synchronize_ca_if_changed(force=True) @@ -265,6 +261,7 @@ def identity_changed(relation_id=None, remote_unit=None): "ready - deferring until db ready", level=WARNING) return + migrate_database() add_service_to_keystone(relation_id, remote_unit) settings = relation_get(rid=relation_id, unit=remote_unit) service = settings.get('service', None) @@ -394,7 +391,7 @@ def cluster_changed(): # NOTE(jamespage) re-echo passwords for peer storage echo_whitelist, overrides = \ apply_echo_filters(settings, ['_passwd', 'identity-service:', - 'ssl-cert-master']) + 'ssl-cert-master', 'db-initialised']) log("Peer echo overrides: %s" % (overrides), level=DEBUG) relation_set(**overrides) if echo_whitelist: @@ -546,6 +543,7 @@ def upgrade_charm(): if is_elected_leader(CLUSTER_RES): log('Cluster leader - ensuring endpoint configuration is up to ' 'date', level=DEBUG) + migrate_database(force=True) time.sleep(10) update_all_identity_relation_units() diff --git a/hooks/keystone_utils.py b/hooks/keystone_utils.py index 26d6eb74..dded48f2 100644 --- a/hooks/keystone_utils.py +++ b/hooks/keystone_utils.py @@ -317,8 +317,30 @@ def do_openstack_upgrade(configs): migrate_database() -def migrate_database(): +def set_db_initialised(): + for rid in relation_ids('cluster'): + relation_set(relation_settings={'db-initialised': 'True'}, + relation_id=rid) + + +def is_db_initialised(): + for rid in relation_ids('cluster'): + units = related_units(rid) + [local_unit()] + for unit in units: + db_initialised = relation_get(attribute='db-initialised', + unit=unit, rid=rid) + if db_initialised: + return True + + return False + + +def migrate_database(force=False): """Runs keystone-manage to initialize a new database or migrate existing""" + if not force and is_db_initialised(): + log('Keystone DB already migrated - skipping', level=INFO) + return + log('Migrating the keystone database.', level=INFO) service_stop('keystone') # NOTE(jamespage) > icehouse creates a log file as root so use @@ -328,10 +350,11 @@ def migrate_database(): subprocess.check_output(cmd) service_start('keystone') time.sleep(10) - + set_db_initialised() # OLD + def get_local_endpoint(): """Returns the URL for the local end-point bypassing haproxy/ssl""" if config('prefer-ipv6'): From 717d6776b9471f969e6eb0962800f7fd0e048d19 Mon Sep 17 00:00:00 2001 From: Edward Hope-Morley Date: Mon, 16 Feb 2015 22:59:25 +0000 Subject: [PATCH 39/48] always ensure db_ready before migration --- hooks/keystone_hooks.py | 25 +++++++++++++++++++------ hooks/keystone_utils.py | 11 ++++++++--- 2 files changed, 27 insertions(+), 9 deletions(-) diff --git a/hooks/keystone_hooks.py b/hooks/keystone_hooks.py index bc4938d8..ec979fb3 100755 --- a/hooks/keystone_hooks.py +++ b/hooks/keystone_hooks.py @@ -146,11 +146,17 @@ def config_changed(): update_nrpe_config() CONFIGS.write_all() - # Update relations since SSL may have been configured. If we have peer - # units we can rely on the sync to do this in cluster relation. - if is_elected_leader(CLUSTER_RES) and not peer_units(): - migrate_database(force=True) - update_all_identity_relation_units() + if is_elected_leader(CLUSTER_RES): + if not is_db_ready(): + log("Database not ready - skipping db migration and " + "identity-relation updates", level=INFO) + else: + migrate_database(force=True) + # Update relations since SSL may have been configured. If we have + # peer units we can rely on the sync to do this in cluster + # relation. + if not peer_units(): + update_all_identity_relation_units() for rid in relation_ids('identity-admin'): admin_relation_changed(rid) @@ -230,7 +236,7 @@ def db_changed(): return # Ensure any existing service entries are updated in the - # new database backend + # new database backend. Also avoid duplicate db ready check. update_all_identity_relation_units(check_db_ready=False) @@ -489,6 +495,7 @@ def ha_changed(): level=INFO) return + migrate_database() ensure_initial_admin(config) log('Cluster configured, notifying other services and updating ' 'keystone endpoint configuration') @@ -543,6 +550,12 @@ def upgrade_charm(): if is_elected_leader(CLUSTER_RES): log('Cluster leader - ensuring endpoint configuration is up to ' 'date', level=DEBUG) + + if not is_db_ready(): + log("Database not ready - deferring to shared-db relation", + level=INFO) + return + migrate_database(force=True) time.sleep(10) update_all_identity_relation_units() diff --git a/hooks/keystone_utils.py b/hooks/keystone_utils.py index dded48f2..5139b20a 100644 --- a/hooks/keystone_utils.py +++ b/hooks/keystone_utils.py @@ -314,7 +314,12 @@ def do_openstack_upgrade(configs): configs.write_all() if is_elected_leader(CLUSTER_RES): - migrate_database() + if is_db_ready(): + migrate_database(force=True) + else: + log("Database not ready - deferring to shared-db relation", + level=INFO) + return def set_db_initialised(): @@ -341,7 +346,7 @@ def migrate_database(force=False): log('Keystone DB already migrated - skipping', level=INFO) return - log('Migrating the keystone database.', level=INFO) + log('Migrating the keystone database. (force=%s)' % force, level=INFO) service_stop('keystone') # NOTE(jamespage) > icehouse creates a log file as root so use # sudo to execute as keystone otherwise keystone won't start @@ -349,8 +354,8 @@ def migrate_database(force=False): cmd = ['sudo', '-u', 'keystone', 'keystone-manage', 'db_sync'] subprocess.check_output(cmd) service_start('keystone') - time.sleep(10) set_db_initialised() + time.sleep(10) # OLD From e092465fe4fbd748db95c47e988150a31417099d Mon Sep 17 00:00:00 2001 From: Edward Hope-Morley Date: Mon, 16 Feb 2015 23:56:01 +0000 Subject: [PATCH 40/48] more --- hooks/keystone_hooks.py | 61 ++++++++++++++----------------- hooks/keystone_utils.py | 14 +++---- unit_tests/test_keystone_hooks.py | 35 ++++++++++++++---- 3 files changed, 61 insertions(+), 49 deletions(-) diff --git a/hooks/keystone_hooks.py b/hooks/keystone_hooks.py index ec979fb3..dcc41907 100755 --- a/hooks/keystone_hooks.py +++ b/hooks/keystone_hooks.py @@ -4,7 +4,6 @@ import json import os import stat import sys -import time from subprocess import check_call @@ -72,6 +71,7 @@ from keystone_utils import ( is_ssl_cert_master, is_db_ready, clear_ssl_synced_units, + is_db_initialised, ) from charmhelpers.contrib.hahelpers.cluster import ( @@ -146,17 +146,10 @@ def config_changed(): update_nrpe_config() CONFIGS.write_all() - if is_elected_leader(CLUSTER_RES): - if not is_db_ready(): - log("Database not ready - skipping db migration and " - "identity-relation updates", level=INFO) - else: - migrate_database(force=True) - # Update relations since SSL may have been configured. If we have - # peer units we can rely on the sync to do this in cluster - # relation. - if not peer_units(): - update_all_identity_relation_units() + # Update relations since SSL may have been configured. If we have peer + # units we can rely on the sync to do this in cluster relation. + if is_elected_leader(CLUSTER_RES) and not peer_units(): + update_all_identity_relation_units() for rid in relation_ids('identity-admin'): admin_relation_changed(rid) @@ -205,8 +198,14 @@ def update_all_identity_relation_units(check_db_ready=True): level=INFO) return - migrate_database() - ensure_initial_admin(config) + if not is_db_initialised(): + log("Database not yet initialised - deferring identity-relation " + "updates", level=INFO) + return + + if is_elected_leader(CLUSTER_RES): + ensure_initial_admin(config) + log('Firing identity_changed hook for all related services.') for rid in relation_ids('identity-service'): for unit in related_units(rid): @@ -235,6 +234,8 @@ def db_changed(): level=INFO) return + migrate_database() + # Ensure any existing service entries are updated in the # new database backend. Also avoid duplicate db ready check. update_all_identity_relation_units(check_db_ready=False) @@ -249,9 +250,15 @@ def pgsql_db_changed(): else: CONFIGS.write(KEYSTONE_CONF) if is_elected_leader(CLUSTER_RES): + if not is_db_ready(use_current_context=True): + log('Allowed_units list provided and this unit not present', + level=INFO) + return + + migrate_database() # Ensure any existing service entries are updated in the - # new database backend - update_all_identity_relation_units() + # new database backend. Also avoid duplicate db ready check. + update_all_identity_relation_units(check_db_ready=False) @hooks.hook('identity-service-relation-changed') @@ -267,7 +274,11 @@ def identity_changed(relation_id=None, remote_unit=None): "ready - deferring until db ready", level=WARNING) return - migrate_database() + if not is_db_initialised(): + log("Database not yet initialised - deferring identity-relation " + "updates", level=INFO) + return + add_service_to_keystone(relation_id, remote_unit) settings = relation_get(rid=relation_id, unit=remote_unit) service = settings.get('service', None) @@ -490,16 +501,8 @@ def ha_changed(): clustered = relation_get('clustered') if clustered and is_elected_leader(CLUSTER_RES): - if not is_db_ready(): - log('Allowed_units list provided and this unit not present', - level=INFO) - return - - migrate_database() - ensure_initial_admin(config) log('Cluster configured, notifying other services and updating ' 'keystone endpoint configuration') - update_all_identity_relation_units() @@ -550,14 +553,6 @@ def upgrade_charm(): if is_elected_leader(CLUSTER_RES): log('Cluster leader - ensuring endpoint configuration is up to ' 'date', level=DEBUG) - - if not is_db_ready(): - log("Database not ready - deferring to shared-db relation", - level=INFO) - return - - migrate_database(force=True) - time.sleep(10) update_all_identity_relation_units() diff --git a/hooks/keystone_utils.py b/hooks/keystone_utils.py index 5139b20a..12a04fa2 100644 --- a/hooks/keystone_utils.py +++ b/hooks/keystone_utils.py @@ -315,7 +315,7 @@ def do_openstack_upgrade(configs): if is_elected_leader(CLUSTER_RES): if is_db_ready(): - migrate_database(force=True) + migrate_database() else: log("Database not ready - deferring to shared-db relation", level=INFO) @@ -335,18 +335,16 @@ def is_db_initialised(): db_initialised = relation_get(attribute='db-initialised', unit=unit, rid=rid) if db_initialised: + log("Database is initialised", level=DEBUG) return True + log("Database is NOT initialised", level=DEBUG) return False -def migrate_database(force=False): +def migrate_database(): """Runs keystone-manage to initialize a new database or migrate existing""" - if not force and is_db_initialised(): - log('Keystone DB already migrated - skipping', level=INFO) - return - - log('Migrating the keystone database. (force=%s)' % force, level=INFO) + log('Migrating the keystone database.', level=INFO) service_stop('keystone') # NOTE(jamespage) > icehouse creates a log file as root so use # sudo to execute as keystone otherwise keystone won't start @@ -354,8 +352,8 @@ def migrate_database(force=False): cmd = ['sudo', '-u', 'keystone', 'keystone-manage', 'db_sync'] subprocess.check_output(cmd) service_start('keystone') - set_db_initialised() time.sleep(10) + set_db_initialised() # OLD diff --git a/unit_tests/test_keystone_hooks.py b/unit_tests/test_keystone_hooks.py index 28063f9d..12cc3f48 100644 --- a/unit_tests/test_keystone_hooks.py +++ b/unit_tests/test_keystone_hooks.py @@ -63,7 +63,6 @@ TO_PATCH = [ 'execd_preinstall', 'mkdir', 'os', - 'time', # ip 'get_iface_for_address', 'get_netmask_for_address', @@ -203,6 +202,7 @@ class KeystoneRelationTests(CharmTestCase): configs.write = MagicMock() hooks.pgsql_db_changed() + @patch.object(hooks, 'is_db_initialised') @patch.object(hooks, 'is_db_ready') @patch('keystone_utils.log') @patch('keystone_utils.ensure_ssl_cert_master') @@ -210,7 +210,9 @@ class KeystoneRelationTests(CharmTestCase): @patch.object(hooks, 'identity_changed') def test_db_changed_allowed(self, identity_changed, configs, mock_ensure_ssl_cert_master, - mock_log, mock_is_db_ready): + mock_log, mock_is_db_ready, + mock_is_db_initialised): + mock_is_db_initialised.return_value = True mock_is_db_ready.return_value = True mock_ensure_ssl_cert_master.return_value = False self.relation_ids.return_value = ['identity-service:0'] @@ -247,12 +249,14 @@ class KeystoneRelationTests(CharmTestCase): @patch('keystone_utils.log') @patch('keystone_utils.ensure_ssl_cert_master') + @patch.object(hooks, 'is_db_initialised') @patch.object(hooks, 'is_db_ready') @patch.object(hooks, 'CONFIGS') @patch.object(hooks, 'identity_changed') def test_postgresql_db_changed(self, identity_changed, configs, - mock_is_db_ready, + mock_is_db_ready, mock_is_db_initialised, mock_ensure_ssl_cert_master, mock_log): + mock_is_db_initialised.return_value = True mock_is_db_ready.return_value = True mock_ensure_ssl_cert_master.return_value = False self.relation_ids.return_value = ['identity-service:0'] @@ -269,6 +273,7 @@ class KeystoneRelationTests(CharmTestCase): @patch('keystone_utils.log') @patch('keystone_utils.ensure_ssl_cert_master') + @patch.object(hooks, 'is_db_initialised') @patch.object(hooks, 'is_db_ready') @patch.object(hooks, 'peer_units') @patch.object(hooks, 'ensure_permissions') @@ -283,8 +288,9 @@ class KeystoneRelationTests(CharmTestCase): self, configure_https, identity_changed, configs, get_homedir, ensure_user, cluster_joined, admin_relation_changed, ensure_permissions, mock_peer_units, - mock_is_db_ready, + mock_is_db_ready, mock_is_db_initialised, mock_ensure_ssl_cert_master, mock_log): + mock_is_db_initialised.return_value = True mock_is_db_ready.return_value = True self.openstack_upgrade_available.return_value = False self.is_elected_leader.return_value = True @@ -302,7 +308,6 @@ class KeystoneRelationTests(CharmTestCase): configure_https.assert_called_with() self.assertTrue(configs.write_all.called) - self.migrate_database.assert_called_with() self.assertTrue(self.ensure_initial_admin.called) self.log.assert_called_with( 'Firing identity_changed hook for all related services.') @@ -343,6 +348,7 @@ class KeystoneRelationTests(CharmTestCase): @patch('keystone_utils.log') @patch('keystone_utils.ensure_ssl_cert_master') + @patch.object(hooks, 'is_db_initialised') @patch.object(hooks, 'is_db_ready') @patch.object(hooks, 'peer_units') @patch.object(hooks, 'ensure_permissions') @@ -361,9 +367,11 @@ class KeystoneRelationTests(CharmTestCase): ensure_permissions, mock_peer_units, mock_is_db_ready, + mock_is_db_initialised, mock_ensure_ssl_cert_master, mock_log): mock_is_db_ready.return_value = True + mock_is_db_initialised.return_value = True self.openstack_upgrade_available.return_value = True self.is_elected_leader.return_value = True # avoid having to mock syncer @@ -382,7 +390,6 @@ class KeystoneRelationTests(CharmTestCase): configure_https.assert_called_with() self.assertTrue(configs.write_all.called) - self.migrate_database.assert_called_with() self.assertTrue(self.ensure_initial_admin.called) self.log.assert_called_with( 'Firing identity_changed hook for all related services.') @@ -391,6 +398,7 @@ class KeystoneRelationTests(CharmTestCase): remote_unit='unit/0') admin_relation_changed.assert_called_with('identity-service:0') + @patch.object(hooks, 'is_db_initialised') @patch.object(hooks, 'is_db_ready') @patch('keystone_utils.log') @patch('keystone_utils.ensure_ssl_cert_master') @@ -398,7 +406,9 @@ class KeystoneRelationTests(CharmTestCase): @patch.object(hooks, 'send_notifications') def test_identity_changed_leader(self, mock_send_notifications, mock_hashlib, mock_ensure_ssl_cert_master, - mock_log, mock_is_db_ready): + mock_log, mock_is_db_ready, + mock_is_db_initialised): + mock_is_db_initialised.return_value = True mock_is_db_ready.return_value = True mock_ensure_ssl_cert_master.return_value = False hooks.identity_changed( @@ -557,13 +567,16 @@ class KeystoneRelationTests(CharmTestCase): @patch('keystone_utils.log') @patch('keystone_utils.ensure_ssl_cert_master') @patch.object(hooks, 'is_db_ready') + @patch.object(hooks, 'is_db_initialised') @patch.object(hooks, 'identity_changed') @patch.object(hooks, 'CONFIGS') def test_ha_relation_changed_clustered_leader(self, configs, identity_changed, + mock_is_db_initialised, mock_is_db_ready, mock_ensure_ssl_cert_master, mock_log): + mock_is_db_initialised.return_value = True mock_is_db_ready.return_value = True mock_ensure_ssl_cert_master.return_value = False self.relation_get.return_value = True @@ -610,6 +623,8 @@ class KeystoneRelationTests(CharmTestCase): cmd = ['a2dissite', 'openstack_https_frontend'] self.check_call.assert_called_with(cmd) + @patch.object(hooks, 'is_db_ready') + @patch.object(hooks, 'is_db_initialised') @patch('keystone_utils.log') @patch('keystone_utils.relation_ids') @patch('keystone_utils.is_elected_leader') @@ -623,7 +638,11 @@ class KeystoneRelationTests(CharmTestCase): mock_ensure_ssl_cert_master, mock_is_elected_leader, mock_relation_ids, - mock_log): + mock_log, + mock_is_db_ready, + mock_is_db_initialised): + mock_is_db_initialised.return_value = True + mock_is_db_ready.return_value = True mock_is_elected_leader.return_value = False mock_relation_ids.return_value = [] mock_ensure_ssl_cert_master.return_value = True From 14f376b310f6f2a4bccdad776c940a39cd0e38cd Mon Sep 17 00:00:00 2001 From: Brad Marshall Date: Thu, 19 Feb 2015 14:16:18 +1000 Subject: [PATCH 41/48] [bradm] Sync charmhelpers --- .../charmhelpers/contrib/charmsupport/nrpe.py | 46 +- .../charmhelpers/contrib/hahelpers/cluster.py | 6 +- .../contrib/openstack/amulet/deployment.py | 7 +- .../contrib/openstack/files/__init__.py | 18 + .../contrib/openstack/files/check_haproxy.sh | 32 ++ .../files/check_haproxy_queue_depth.sh | 30 ++ hooks/charmhelpers/contrib/openstack/ip.py | 37 ++ hooks/charmhelpers/contrib/openstack/utils.py | 1 + hooks/charmhelpers/contrib/python/packages.py | 4 +- hooks/charmhelpers/core/fstab.py | 8 +- hooks/charmhelpers/core/host.py | 10 +- hooks/charmhelpers/core/strutils.py | 42 ++ hooks/charmhelpers/core/sysctl.py | 4 +- hooks/charmhelpers/core/templating.py | 6 +- hooks/charmhelpers/core/unitdata.py | 477 ++++++++++++++++++ hooks/charmhelpers/fetch/archiveurl.py | 20 +- hooks/charmhelpers/fetch/giturl.py | 2 +- .../contrib/openstack/amulet/deployment.py | 7 +- 18 files changed, 719 insertions(+), 38 deletions(-) create mode 100644 hooks/charmhelpers/contrib/openstack/files/__init__.py create mode 100755 hooks/charmhelpers/contrib/openstack/files/check_haproxy.sh create mode 100755 hooks/charmhelpers/contrib/openstack/files/check_haproxy_queue_depth.sh create mode 100644 hooks/charmhelpers/core/strutils.py create mode 100644 hooks/charmhelpers/core/unitdata.py diff --git a/hooks/charmhelpers/contrib/charmsupport/nrpe.py b/hooks/charmhelpers/contrib/charmsupport/nrpe.py index 0fd0a9d8..8229f6b5 100644 --- a/hooks/charmhelpers/contrib/charmsupport/nrpe.py +++ b/hooks/charmhelpers/contrib/charmsupport/nrpe.py @@ -24,6 +24,8 @@ import subprocess import pwd import grp import os +import glob +import shutil import re import shlex import yaml @@ -161,7 +163,7 @@ define service {{ log('Check command not found: {}'.format(parts[0])) return '' - def write(self, nagios_context, hostname, nagios_servicegroups=None): + def write(self, nagios_context, hostname, nagios_servicegroups): nrpe_check_file = '/etc/nagios/nrpe.d/{}.cfg'.format( self.command) with open(nrpe_check_file, 'w') as nrpe_check_config: @@ -177,14 +179,11 @@ define service {{ nagios_servicegroups) def write_service_config(self, nagios_context, hostname, - nagios_servicegroups=None): + nagios_servicegroups): for f in os.listdir(NRPE.nagios_exportdir): if re.search('.*{}.cfg'.format(self.command), f): os.remove(os.path.join(NRPE.nagios_exportdir, f)) - if not nagios_servicegroups: - nagios_servicegroups = nagios_context - templ_vars = { 'nagios_hostname': hostname, 'nagios_servicegroup': nagios_servicegroups, @@ -214,7 +213,7 @@ class NRPE(object): if 'nagios_servicegroups' in self.config: self.nagios_servicegroups = self.config['nagios_servicegroups'] else: - self.nagios_servicegroups = 'juju' + self.nagios_servicegroups = self.nagios_context self.unit_name = local_unit().replace('/', '-') if hostname: self.hostname = hostname @@ -322,3 +321,38 @@ def add_init_service_checks(nrpe, services, unit_name): check_cmd='check_status_file.py -f ' '/var/lib/nagios/service-check-%s.txt' % svc, ) + + +def copy_nrpe_checks(): + """ + Copy the nrpe checks into place + + """ + NAGIOS_PLUGINS = '/usr/local/lib/nagios/plugins' + nrpe_files_dir = os.path.join(os.getenv('CHARM_DIR'), 'hooks', + 'charmhelpers', 'contrib', 'openstack', + 'files') + + if not os.path.exists(NAGIOS_PLUGINS): + os.makedirs(NAGIOS_PLUGINS) + for fname in glob.glob(os.path.join(nrpe_files_dir, "check_*")): + if os.path.isfile(fname): + shutil.copy2(fname, + os.path.join(NAGIOS_PLUGINS, os.path.basename(fname))) + + +def add_haproxy_checks(nrpe, unit_name): + """ + Add checks for each service in list + + :param NRPE nrpe: NRPE object to add check to + :param str unit_name: Unit name to use in check description + """ + nrpe.add_check( + shortname='haproxy_servers', + description='Check HAProxy {%s}' % unit_name, + check_cmd='check_haproxy.sh') + nrpe.add_check( + shortname='haproxy_queue', + description='Check HAProxy queue depth {%s}' % unit_name, + check_cmd='check_haproxy_queue_depth.sh') diff --git a/hooks/charmhelpers/contrib/hahelpers/cluster.py b/hooks/charmhelpers/contrib/hahelpers/cluster.py index 9a2588b6..9333efc3 100644 --- a/hooks/charmhelpers/contrib/hahelpers/cluster.py +++ b/hooks/charmhelpers/contrib/hahelpers/cluster.py @@ -48,6 +48,9 @@ from charmhelpers.core.hookenv import ( from charmhelpers.core.decorators import ( retry_on_exception, ) +from charmhelpers.core.strutils import ( + bool_from_string, +) class HAIncompleteConfig(Exception): @@ -164,7 +167,8 @@ def https(): . returns: boolean ''' - if config_get('use-https') == "yes": + use_https = config_get('use-https') + if use_https and bool_from_string(use_https): return True if config_get('ssl_cert') and config_get('ssl_key'): return True diff --git a/hooks/charmhelpers/contrib/openstack/amulet/deployment.py b/hooks/charmhelpers/contrib/openstack/amulet/deployment.py index c50d3ec6..0cfeaa4c 100644 --- a/hooks/charmhelpers/contrib/openstack/amulet/deployment.py +++ b/hooks/charmhelpers/contrib/openstack/amulet/deployment.py @@ -71,16 +71,19 @@ class OpenStackAmuletDeployment(AmuletDeployment): services.append(this_service) use_source = ['mysql', 'mongodb', 'rabbitmq-server', 'ceph', 'ceph-osd', 'ceph-radosgw'] + # Openstack subordinate charms do not expose an origin option as that + # is controlled by the principle + ignore = ['neutron-openvswitch'] if self.openstack: for svc in services: - if svc['name'] not in use_source: + if svc['name'] not in use_source + ignore: config = {'openstack-origin': self.openstack} self.d.configure(svc['name'], config) if self.source: for svc in services: - if svc['name'] in use_source: + if svc['name'] in use_source and svc['name'] not in ignore: config = {'source': self.source} self.d.configure(svc['name'], config) diff --git a/hooks/charmhelpers/contrib/openstack/files/__init__.py b/hooks/charmhelpers/contrib/openstack/files/__init__.py new file mode 100644 index 00000000..75876796 --- /dev/null +++ b/hooks/charmhelpers/contrib/openstack/files/__init__.py @@ -0,0 +1,18 @@ +# Copyright 2014-2015 Canonical Limited. +# +# This file is part of charm-helpers. +# +# charm-helpers is free software: you can redistribute it and/or modify +# it under the terms of the GNU Lesser General Public License version 3 as +# published by the Free Software Foundation. +# +# charm-helpers is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Lesser General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public License +# along with charm-helpers. If not, see . + +# dummy __init__.py to fool syncer into thinking this is a syncable python +# module diff --git a/hooks/charmhelpers/contrib/openstack/files/check_haproxy.sh b/hooks/charmhelpers/contrib/openstack/files/check_haproxy.sh new file mode 100755 index 00000000..eb8527f5 --- /dev/null +++ b/hooks/charmhelpers/contrib/openstack/files/check_haproxy.sh @@ -0,0 +1,32 @@ +#!/bin/bash +#-------------------------------------------- +# This file is managed by Juju +#-------------------------------------------- +# +# Copyright 2009,2012 Canonical Ltd. +# Author: Tom Haddon + +CRITICAL=0 +NOTACTIVE='' +LOGFILE=/var/log/nagios/check_haproxy.log +AUTH=$(grep -r "stats auth" /etc/haproxy | head -1 | awk '{print $4}') + +for appserver in $(grep ' server' /etc/haproxy/haproxy.cfg | awk '{print $2'}); +do + output=$(/usr/lib/nagios/plugins/check_http -a ${AUTH} -I 127.0.0.1 -p 8888 --regex="class=\"(active|backup)(2|3).*${appserver}" -e ' 200 OK') + if [ $? != 0 ]; then + date >> $LOGFILE + echo $output >> $LOGFILE + /usr/lib/nagios/plugins/check_http -a ${AUTH} -I 127.0.0.1 -p 8888 -v | grep $appserver >> $LOGFILE 2>&1 + CRITICAL=1 + NOTACTIVE="${NOTACTIVE} $appserver" + fi +done + +if [ $CRITICAL = 1 ]; then + echo "CRITICAL:${NOTACTIVE}" + exit 2 +fi + +echo "OK: All haproxy instances looking good" +exit 0 diff --git a/hooks/charmhelpers/contrib/openstack/files/check_haproxy_queue_depth.sh b/hooks/charmhelpers/contrib/openstack/files/check_haproxy_queue_depth.sh new file mode 100755 index 00000000..3ebb5329 --- /dev/null +++ b/hooks/charmhelpers/contrib/openstack/files/check_haproxy_queue_depth.sh @@ -0,0 +1,30 @@ +#!/bin/bash +#-------------------------------------------- +# This file is managed by Juju +#-------------------------------------------- +# +# Copyright 2009,2012 Canonical Ltd. +# Author: Tom Haddon + +# These should be config options at some stage +CURRQthrsh=0 +MAXQthrsh=100 + +AUTH=$(grep -r "stats auth" /etc/haproxy | head -1 | awk '{print $4}') + +HAPROXYSTATS=$(/usr/lib/nagios/plugins/check_http -a ${AUTH} -I 127.0.0.1 -p 8888 -u '/;csv' -v) + +for BACKEND in $(echo $HAPROXYSTATS| xargs -n1 | grep BACKEND | awk -F , '{print $1}') +do + CURRQ=$(echo "$HAPROXYSTATS" | grep $BACKEND | grep BACKEND | cut -d , -f 3) + MAXQ=$(echo "$HAPROXYSTATS" | grep $BACKEND | grep BACKEND | cut -d , -f 4) + + if [[ $CURRQ -gt $CURRQthrsh || $MAXQ -gt $MAXQthrsh ]] ; then + echo "CRITICAL: queue depth for $BACKEND - CURRENT:$CURRQ MAX:$MAXQ" + exit 2 + fi +done + +echo "OK: All haproxy queue depths looking good" +exit 0 + diff --git a/hooks/charmhelpers/contrib/openstack/ip.py b/hooks/charmhelpers/contrib/openstack/ip.py index 9eabed73..29bbddcb 100644 --- a/hooks/charmhelpers/contrib/openstack/ip.py +++ b/hooks/charmhelpers/contrib/openstack/ip.py @@ -26,6 +26,8 @@ from charmhelpers.contrib.network.ip import ( ) from charmhelpers.contrib.hahelpers.cluster import is_clustered +from functools import partial + PUBLIC = 'public' INTERNAL = 'int' ADMIN = 'admin' @@ -107,3 +109,38 @@ def resolve_address(endpoint_type=PUBLIC): "clustered=%s)" % (net_type, clustered)) return resolved_address + + +def endpoint_url(configs, url_template, port, endpoint_type=PUBLIC, + override=None): + """Returns the correct endpoint URL to advertise to Keystone. + + This method provides the correct endpoint URL which should be advertised to + the keystone charm for endpoint creation. This method allows for the url to + be overridden to force a keystone endpoint to have specific URL for any of + the defined scopes (admin, internal, public). + + :param configs: OSTemplateRenderer config templating object to inspect + for a complete https context. + :param url_template: str format string for creating the url template. Only + two values will be passed - the scheme+hostname + returned by the canonical_url and the port. + :param endpoint_type: str endpoint type to resolve. + :param override: str the name of the config option which overrides the + endpoint URL defined by the charm itself. None will + disable any overrides (default). + """ + if override: + # Return any user-defined overrides for the keystone endpoint URL. + user_value = config(override) + if user_value: + return user_value.strip() + + return url_template % (canonical_url(configs, endpoint_type), port) + + +public_endpoint = partial(endpoint_url, endpoint_type=PUBLIC) + +internal_endpoint = partial(endpoint_url, endpoint_type=INTERNAL) + +admin_endpoint = partial(endpoint_url, endpoint_type=ADMIN) diff --git a/hooks/charmhelpers/contrib/openstack/utils.py b/hooks/charmhelpers/contrib/openstack/utils.py index 26259a03..af2b3596 100644 --- a/hooks/charmhelpers/contrib/openstack/utils.py +++ b/hooks/charmhelpers/contrib/openstack/utils.py @@ -103,6 +103,7 @@ SWIFT_CODENAMES = OrderedDict([ ('2.1.0', 'juno'), ('2.2.0', 'juno'), ('2.2.1', 'kilo'), + ('2.2.2', 'kilo'), ]) DEFAULT_LOOPBACK_SIZE = '5G' diff --git a/hooks/charmhelpers/contrib/python/packages.py b/hooks/charmhelpers/contrib/python/packages.py index d848a120..8659516b 100644 --- a/hooks/charmhelpers/contrib/python/packages.py +++ b/hooks/charmhelpers/contrib/python/packages.py @@ -17,8 +17,6 @@ # You should have received a copy of the GNU Lesser General Public License # along with charm-helpers. If not, see . -__author__ = "Jorge Niedbalski " - from charmhelpers.fetch import apt_install, apt_update from charmhelpers.core.hookenv import log @@ -29,6 +27,8 @@ except ImportError: apt_install('python-pip') from pip import main as pip_execute +__author__ = "Jorge Niedbalski " + def parse_options(given, available): """Given a set of options, check if available""" diff --git a/hooks/charmhelpers/core/fstab.py b/hooks/charmhelpers/core/fstab.py index be7de248..3056fbac 100644 --- a/hooks/charmhelpers/core/fstab.py +++ b/hooks/charmhelpers/core/fstab.py @@ -17,11 +17,11 @@ # You should have received a copy of the GNU Lesser General Public License # along with charm-helpers. If not, see . -__author__ = 'Jorge Niedbalski R. ' - import io import os +__author__ = 'Jorge Niedbalski R. ' + class Fstab(io.FileIO): """This class extends file in order to implement a file reader/writer @@ -77,7 +77,7 @@ class Fstab(io.FileIO): for line in self.readlines(): line = line.decode('us-ascii') try: - if line.strip() and not line.startswith("#"): + if line.strip() and not line.strip().startswith("#"): yield self._hydrate_entry(line) except ValueError: pass @@ -104,7 +104,7 @@ class Fstab(io.FileIO): found = False for index, line in enumerate(lines): - if not line.startswith("#"): + if line.strip() and not line.strip().startswith("#"): if self._hydrate_entry(line) == entry: found = True break diff --git a/hooks/charmhelpers/core/host.py b/hooks/charmhelpers/core/host.py index cf2cbe14..b771c611 100644 --- a/hooks/charmhelpers/core/host.py +++ b/hooks/charmhelpers/core/host.py @@ -191,11 +191,11 @@ def mkdir(path, owner='root', group='root', perms=0o555, force=False): def write_file(path, content, owner='root', group='root', perms=0o444): - """Create or overwrite a file with the contents of a string""" + """Create or overwrite a file with the contents of a byte string.""" log("Writing file {} {}:{} {:o}".format(path, owner, group, perms)) uid = pwd.getpwnam(owner).pw_uid gid = grp.getgrnam(group).gr_gid - with open(path, 'w') as target: + with open(path, 'wb') as target: os.fchown(target.fileno(), uid, gid) os.fchmod(target.fileno(), perms) target.write(content) @@ -305,11 +305,11 @@ def restart_on_change(restart_map, stopstart=False): ceph_client_changed function. """ def wrap(f): - def wrapped_f(*args): + def wrapped_f(*args, **kwargs): checksums = {} for path in restart_map: checksums[path] = file_hash(path) - f(*args) + f(*args, **kwargs) restarts = [] for path in restart_map: if checksums[path] != file_hash(path): @@ -361,7 +361,7 @@ def list_nics(nic_type): ip_output = (line for line in ip_output if line) for line in ip_output: if line.split()[1].startswith(int_type): - matched = re.search('.*: (bond[0-9]+\.[0-9]+)@.*', line) + matched = re.search('.*: (' + int_type + r'[0-9]+\.[0-9]+)@.*', line) if matched: interface = matched.groups()[0] else: diff --git a/hooks/charmhelpers/core/strutils.py b/hooks/charmhelpers/core/strutils.py new file mode 100644 index 00000000..efc4402e --- /dev/null +++ b/hooks/charmhelpers/core/strutils.py @@ -0,0 +1,42 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- + +# Copyright 2014-2015 Canonical Limited. +# +# This file is part of charm-helpers. +# +# charm-helpers is free software: you can redistribute it and/or modify +# it under the terms of the GNU Lesser General Public License version 3 as +# published by the Free Software Foundation. +# +# charm-helpers is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Lesser General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public License +# along with charm-helpers. If not, see . + +import six + + +def bool_from_string(value): + """Interpret string value as boolean. + + Returns True if value translates to True otherwise False. + """ + if isinstance(value, six.string_types): + value = six.text_type(value) + else: + msg = "Unable to interpret non-string value '%s' as boolean" % (value) + raise ValueError(msg) + + value = value.strip().lower() + + if value in ['y', 'yes', 'true', 't']: + return True + elif value in ['n', 'no', 'false', 'f']: + return False + + msg = "Unable to interpret string value '%s' as boolean" % (value) + raise ValueError(msg) diff --git a/hooks/charmhelpers/core/sysctl.py b/hooks/charmhelpers/core/sysctl.py index 8e1b9eeb..21cc8ab2 100644 --- a/hooks/charmhelpers/core/sysctl.py +++ b/hooks/charmhelpers/core/sysctl.py @@ -17,8 +17,6 @@ # You should have received a copy of the GNU Lesser General Public License # along with charm-helpers. If not, see . -__author__ = 'Jorge Niedbalski R. ' - import yaml from subprocess import check_call @@ -29,6 +27,8 @@ from charmhelpers.core.hookenv import ( ERROR, ) +__author__ = 'Jorge Niedbalski R. ' + def create(sysctl_dict, sysctl_file): """Creates a sysctl.conf file from a YAML associative array diff --git a/hooks/charmhelpers/core/templating.py b/hooks/charmhelpers/core/templating.py index 97669092..45319998 100644 --- a/hooks/charmhelpers/core/templating.py +++ b/hooks/charmhelpers/core/templating.py @@ -21,7 +21,7 @@ from charmhelpers.core import hookenv def render(source, target, context, owner='root', group='root', - perms=0o444, templates_dir=None): + perms=0o444, templates_dir=None, encoding='UTF-8'): """ Render a template. @@ -64,5 +64,5 @@ def render(source, target, context, owner='root', group='root', level=hookenv.ERROR) raise e content = template.render(context) - host.mkdir(os.path.dirname(target), owner, group) - host.write_file(target, content, owner, group, perms) + host.mkdir(os.path.dirname(target), owner, group, perms=0o755) + host.write_file(target, content.encode(encoding), owner, group, perms) diff --git a/hooks/charmhelpers/core/unitdata.py b/hooks/charmhelpers/core/unitdata.py new file mode 100644 index 00000000..3000134a --- /dev/null +++ b/hooks/charmhelpers/core/unitdata.py @@ -0,0 +1,477 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- +# +# Copyright 2014-2015 Canonical Limited. +# +# This file is part of charm-helpers. +# +# charm-helpers is free software: you can redistribute it and/or modify +# it under the terms of the GNU Lesser General Public License version 3 as +# published by the Free Software Foundation. +# +# charm-helpers is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Lesser General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public License +# along with charm-helpers. If not, see . +# +# +# Authors: +# Kapil Thangavelu +# +""" +Intro +----- + +A simple way to store state in units. This provides a key value +storage with support for versioned, transactional operation, +and can calculate deltas from previous values to simplify unit logic +when processing changes. + + +Hook Integration +---------------- + +There are several extant frameworks for hook execution, including + + - charmhelpers.core.hookenv.Hooks + - charmhelpers.core.services.ServiceManager + +The storage classes are framework agnostic, one simple integration is +via the HookData contextmanager. It will record the current hook +execution environment (including relation data, config data, etc.), +setup a transaction and allow easy access to the changes from +previously seen values. One consequence of the integration is the +reservation of particular keys ('rels', 'unit', 'env', 'config', +'charm_revisions') for their respective values. + +Here's a fully worked integration example using hookenv.Hooks:: + + from charmhelper.core import hookenv, unitdata + + hook_data = unitdata.HookData() + db = unitdata.kv() + hooks = hookenv.Hooks() + + @hooks.hook + def config_changed(): + # Print all changes to configuration from previously seen + # values. + for changed, (prev, cur) in hook_data.conf.items(): + print('config changed', changed, + 'previous value', prev, + 'current value', cur) + + # Get some unit specific bookeeping + if not db.get('pkg_key'): + key = urllib.urlopen('https://example.com/pkg_key').read() + db.set('pkg_key', key) + + # Directly access all charm config as a mapping. + conf = db.getrange('config', True) + + # Directly access all relation data as a mapping + rels = db.getrange('rels', True) + + if __name__ == '__main__': + with hook_data(): + hook.execute() + + +A more basic integration is via the hook_scope context manager which simply +manages transaction scope (and records hook name, and timestamp):: + + >>> from unitdata import kv + >>> db = kv() + >>> with db.hook_scope('install'): + ... # do work, in transactional scope. + ... db.set('x', 1) + >>> db.get('x') + 1 + + +Usage +----- + +Values are automatically json de/serialized to preserve basic typing +and complex data struct capabilities (dicts, lists, ints, booleans, etc). + +Individual values can be manipulated via get/set:: + + >>> kv.set('y', True) + >>> kv.get('y') + True + + # We can set complex values (dicts, lists) as a single key. + >>> kv.set('config', {'a': 1, 'b': True'}) + + # Also supports returning dictionaries as a record which + # provides attribute access. + >>> config = kv.get('config', record=True) + >>> config.b + True + + +Groups of keys can be manipulated with update/getrange:: + + >>> kv.update({'z': 1, 'y': 2}, prefix="gui.") + >>> kv.getrange('gui.', strip=True) + {'z': 1, 'y': 2} + +When updating values, its very helpful to understand which values +have actually changed and how have they changed. The storage +provides a delta method to provide for this:: + + >>> data = {'debug': True, 'option': 2} + >>> delta = kv.delta(data, 'config.') + >>> delta.debug.previous + None + >>> delta.debug.current + True + >>> delta + {'debug': (None, True), 'option': (None, 2)} + +Note the delta method does not persist the actual change, it needs to +be explicitly saved via 'update' method:: + + >>> kv.update(data, 'config.') + +Values modified in the context of a hook scope retain historical values +associated to the hookname. + + >>> with db.hook_scope('config-changed'): + ... db.set('x', 42) + >>> db.gethistory('x') + [(1, u'x', 1, u'install', u'2015-01-21T16:49:30.038372'), + (2, u'x', 42, u'config-changed', u'2015-01-21T16:49:30.038786')] + +""" + +import collections +import contextlib +import datetime +import json +import os +import pprint +import sqlite3 +import sys + +__author__ = 'Kapil Thangavelu ' + + +class Storage(object): + """Simple key value database for local unit state within charms. + + Modifications are automatically committed at hook exit. That's + currently regardless of exit code. + + To support dicts, lists, integer, floats, and booleans values + are automatically json encoded/decoded. + """ + def __init__(self, path=None): + self.db_path = path + if path is None: + self.db_path = os.path.join( + os.environ.get('CHARM_DIR', ''), '.unit-state.db') + self.conn = sqlite3.connect('%s' % self.db_path) + self.cursor = self.conn.cursor() + self.revision = None + self._closed = False + self._init() + + def close(self): + if self._closed: + return + self.flush(False) + self.cursor.close() + self.conn.close() + self._closed = True + + def _scoped_query(self, stmt, params=None): + if params is None: + params = [] + return stmt, params + + def get(self, key, default=None, record=False): + self.cursor.execute( + *self._scoped_query( + 'select data from kv where key=?', [key])) + result = self.cursor.fetchone() + if not result: + return default + if record: + return Record(json.loads(result[0])) + return json.loads(result[0]) + + def getrange(self, key_prefix, strip=False): + stmt = "select key, data from kv where key like '%s%%'" % key_prefix + self.cursor.execute(*self._scoped_query(stmt)) + result = self.cursor.fetchall() + + if not result: + return None + if not strip: + key_prefix = '' + return dict([ + (k[len(key_prefix):], json.loads(v)) for k, v in result]) + + def update(self, mapping, prefix=""): + for k, v in mapping.items(): + self.set("%s%s" % (prefix, k), v) + + def unset(self, key): + self.cursor.execute('delete from kv where key=?', [key]) + if self.revision and self.cursor.rowcount: + self.cursor.execute( + 'insert into kv_revisions values (?, ?, ?)', + [key, self.revision, json.dumps('DELETED')]) + + def set(self, key, value): + serialized = json.dumps(value) + + self.cursor.execute( + 'select data from kv where key=?', [key]) + exists = self.cursor.fetchone() + + # Skip mutations to the same value + if exists: + if exists[0] == serialized: + return value + + if not exists: + self.cursor.execute( + 'insert into kv (key, data) values (?, ?)', + (key, serialized)) + else: + self.cursor.execute(''' + update kv + set data = ? + where key = ?''', [serialized, key]) + + # Save + if not self.revision: + return value + + self.cursor.execute( + 'select 1 from kv_revisions where key=? and revision=?', + [key, self.revision]) + exists = self.cursor.fetchone() + + if not exists: + self.cursor.execute( + '''insert into kv_revisions ( + revision, key, data) values (?, ?, ?)''', + (self.revision, key, serialized)) + else: + self.cursor.execute( + ''' + update kv_revisions + set data = ? + where key = ? + and revision = ?''', + [serialized, key, self.revision]) + + return value + + def delta(self, mapping, prefix): + """ + return a delta containing values that have changed. + """ + previous = self.getrange(prefix, strip=True) + if not previous: + pk = set() + else: + pk = set(previous.keys()) + ck = set(mapping.keys()) + delta = DeltaSet() + + # added + for k in ck.difference(pk): + delta[k] = Delta(None, mapping[k]) + + # removed + for k in pk.difference(ck): + delta[k] = Delta(previous[k], None) + + # changed + for k in pk.intersection(ck): + c = mapping[k] + p = previous[k] + if c != p: + delta[k] = Delta(p, c) + + return delta + + @contextlib.contextmanager + def hook_scope(self, name=""): + """Scope all future interactions to the current hook execution + revision.""" + assert not self.revision + self.cursor.execute( + 'insert into hooks (hook, date) values (?, ?)', + (name or sys.argv[0], + datetime.datetime.utcnow().isoformat())) + self.revision = self.cursor.lastrowid + try: + yield self.revision + self.revision = None + except: + self.flush(False) + self.revision = None + raise + else: + self.flush() + + def flush(self, save=True): + if save: + self.conn.commit() + elif self._closed: + return + else: + self.conn.rollback() + + def _init(self): + self.cursor.execute(''' + create table if not exists kv ( + key text, + data text, + primary key (key) + )''') + self.cursor.execute(''' + create table if not exists kv_revisions ( + key text, + revision integer, + data text, + primary key (key, revision) + )''') + self.cursor.execute(''' + create table if not exists hooks ( + version integer primary key autoincrement, + hook text, + date text + )''') + self.conn.commit() + + def gethistory(self, key, deserialize=False): + self.cursor.execute( + ''' + select kv.revision, kv.key, kv.data, h.hook, h.date + from kv_revisions kv, + hooks h + where kv.key=? + and kv.revision = h.version + ''', [key]) + if deserialize is False: + return self.cursor.fetchall() + return map(_parse_history, self.cursor.fetchall()) + + def debug(self, fh=sys.stderr): + self.cursor.execute('select * from kv') + pprint.pprint(self.cursor.fetchall(), stream=fh) + self.cursor.execute('select * from kv_revisions') + pprint.pprint(self.cursor.fetchall(), stream=fh) + + +def _parse_history(d): + return (d[0], d[1], json.loads(d[2]), d[3], + datetime.datetime.strptime(d[-1], "%Y-%m-%dT%H:%M:%S.%f")) + + +class HookData(object): + """Simple integration for existing hook exec frameworks. + + Records all unit information, and stores deltas for processing + by the hook. + + Sample:: + + from charmhelper.core import hookenv, unitdata + + changes = unitdata.HookData() + db = unitdata.kv() + hooks = hookenv.Hooks() + + @hooks.hook + def config_changed(): + # View all changes to configuration + for changed, (prev, cur) in changes.conf.items(): + print('config changed', changed, + 'previous value', prev, + 'current value', cur) + + # Get some unit specific bookeeping + if not db.get('pkg_key'): + key = urllib.urlopen('https://example.com/pkg_key').read() + db.set('pkg_key', key) + + if __name__ == '__main__': + with changes(): + hook.execute() + + """ + def __init__(self): + self.kv = kv() + self.conf = None + self.rels = None + + @contextlib.contextmanager + def __call__(self): + from charmhelpers.core import hookenv + hook_name = hookenv.hook_name() + + with self.kv.hook_scope(hook_name): + self._record_charm_version(hookenv.charm_dir()) + delta_config, delta_relation = self._record_hook(hookenv) + yield self.kv, delta_config, delta_relation + + def _record_charm_version(self, charm_dir): + # Record revisions.. charm revisions are meaningless + # to charm authors as they don't control the revision. + # so logic dependnent on revision is not particularly + # useful, however it is useful for debugging analysis. + charm_rev = open( + os.path.join(charm_dir, 'revision')).read().strip() + charm_rev = charm_rev or '0' + revs = self.kv.get('charm_revisions', []) + if charm_rev not in revs: + revs.append(charm_rev.strip() or '0') + self.kv.set('charm_revisions', revs) + + def _record_hook(self, hookenv): + data = hookenv.execution_environment() + self.conf = conf_delta = self.kv.delta(data['conf'], 'config') + self.rels = rels_delta = self.kv.delta(data['rels'], 'rels') + self.kv.set('env', data['env']) + self.kv.set('unit', data['unit']) + self.kv.set('relid', data.get('relid')) + return conf_delta, rels_delta + + +class Record(dict): + + __slots__ = () + + def __getattr__(self, k): + if k in self: + return self[k] + raise AttributeError(k) + + +class DeltaSet(Record): + + __slots__ = () + + +Delta = collections.namedtuple('Delta', ['previous', 'current']) + + +_KV = None + + +def kv(): + global _KV + if _KV is None: + _KV = Storage() + return _KV diff --git a/hooks/charmhelpers/fetch/archiveurl.py b/hooks/charmhelpers/fetch/archiveurl.py index d25a0ddd..8dfce505 100644 --- a/hooks/charmhelpers/fetch/archiveurl.py +++ b/hooks/charmhelpers/fetch/archiveurl.py @@ -18,6 +18,16 @@ import os import hashlib import re +from charmhelpers.fetch import ( + BaseFetchHandler, + UnhandledSource +) +from charmhelpers.payload.archive import ( + get_archive_handler, + extract, +) +from charmhelpers.core.host import mkdir, check_hash + import six if six.PY3: from urllib.request import ( @@ -35,16 +45,6 @@ else: ) from urlparse import urlparse, urlunparse, parse_qs -from charmhelpers.fetch import ( - BaseFetchHandler, - UnhandledSource -) -from charmhelpers.payload.archive import ( - get_archive_handler, - extract, -) -from charmhelpers.core.host import mkdir, check_hash - def splituser(host): '''urllib.splituser(), but six's support of this seems broken''' diff --git a/hooks/charmhelpers/fetch/giturl.py b/hooks/charmhelpers/fetch/giturl.py index 5376786b..93aae87b 100644 --- a/hooks/charmhelpers/fetch/giturl.py +++ b/hooks/charmhelpers/fetch/giturl.py @@ -32,7 +32,7 @@ except ImportError: apt_install("python-git") from git import Repo -from git.exc import GitCommandError +from git.exc import GitCommandError # noqa E402 class GitUrlFetchHandler(BaseFetchHandler): diff --git a/tests/charmhelpers/contrib/openstack/amulet/deployment.py b/tests/charmhelpers/contrib/openstack/amulet/deployment.py index c50d3ec6..0cfeaa4c 100644 --- a/tests/charmhelpers/contrib/openstack/amulet/deployment.py +++ b/tests/charmhelpers/contrib/openstack/amulet/deployment.py @@ -71,16 +71,19 @@ class OpenStackAmuletDeployment(AmuletDeployment): services.append(this_service) use_source = ['mysql', 'mongodb', 'rabbitmq-server', 'ceph', 'ceph-osd', 'ceph-radosgw'] + # Openstack subordinate charms do not expose an origin option as that + # is controlled by the principle + ignore = ['neutron-openvswitch'] if self.openstack: for svc in services: - if svc['name'] not in use_source: + if svc['name'] not in use_source + ignore: config = {'openstack-origin': self.openstack} self.d.configure(svc['name'], config) if self.source: for svc in services: - if svc['name'] in use_source: + if svc['name'] in use_source and svc['name'] not in ignore: config = {'source': self.source} self.d.configure(svc['name'], config) From ab234ca1fde257d2dc2192e3270bce8fa2f9c006 Mon Sep 17 00:00:00 2001 From: Brad Marshall Date: Thu, 19 Feb 2015 14:17:25 +1000 Subject: [PATCH 42/48] [bradm] Add haproxy nrpe checks --- hooks/keystone_hooks.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/hooks/keystone_hooks.py b/hooks/keystone_hooks.py index 0c8dcf95..ed47be64 100755 --- a/hooks/keystone_hooks.py +++ b/hooks/keystone_hooks.py @@ -542,7 +542,9 @@ def update_nrpe_config(): hostname = nrpe.get_nagios_hostname() current_unit = nrpe.get_nagios_unit_name() nrpe_setup = nrpe.NRPE(hostname=hostname) + nrpe.copy_nrpe_checks() nrpe.add_init_service_checks(nrpe_setup, services(), current_unit) + nrpe.add_haproxy_checks(nrpe_setup, current_unit) nrpe_setup.write() From a08ddba72512134bbcaeaecdf8e7b9ff73586c32 Mon Sep 17 00:00:00 2001 From: Brad Marshall Date: Thu, 19 Feb 2015 14:18:24 +1000 Subject: [PATCH 43/48] [bradm] Add nagios_servicegroups config option --- config.yaml | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/config.yaml b/config.yaml index f4f157ea..b0ce3732 100644 --- a/config.yaml +++ b/config.yaml @@ -248,4 +248,10 @@ options: juju-myservice-0 If you're running multiple environments with the same services in them this allows you to differentiate between them. + nagios_servicegroups: + default: "" + type: string + description: | + A comma-separated list of nagios servicegroups. + If left empty, the nagios_context will be used as the servicegroup From d6209c185eaa79335b03a2e6354ceed60687206b Mon Sep 17 00:00:00 2001 From: Edward Hope-Morley Date: Thu, 19 Feb 2015 11:26:28 +0000 Subject: [PATCH 44/48] [hopem,r=] Adds missing logging.conf template for Essex Closes-Bug: 1423513 --- templates/essex/logging.conf | 39 ++++++++++++++++++++++++++++++++++++ 1 file changed, 39 insertions(+) create mode 100644 templates/essex/logging.conf diff --git a/templates/essex/logging.conf b/templates/essex/logging.conf new file mode 100644 index 00000000..7a538ae8 --- /dev/null +++ b/templates/essex/logging.conf @@ -0,0 +1,39 @@ +[loggers] +keys=root + +[formatters] +keys=normal,normal_with_name,debug + +[handlers] +keys=production,file,devel + +[logger_root] +level=WARNING +handlers=file + +[handler_production] +class=handlers.SysLogHandler +level=ERROR +formatter=normal_with_name +args=(('localhost', handlers.SYSLOG_UDP_PORT), handlers.SysLogHandler.LOG_USER) + +[handler_file] +class=FileHandler +level=DEBUG +formatter=normal_with_name +args=('/var/log/keystone/keystone.log', 'a') + +[handler_devel] +class=StreamHandler +level=NOTSET +formatter=debug +args=(sys.stdout,) + +[formatter_normal] +format=%(asctime)s %(levelname)s %(message)s + +[formatter_normal_with_name] +format=(%(name)s): %(asctime)s %(levelname)s %(message)s + +[formatter_debug] +format=(%(name)s): %(asctime)s %(levelname)s %(module)s %(funcName)s %(message)s From 03b2e7dc545ecdca05c77ef36914aa8ba8ff514a Mon Sep 17 00:00:00 2001 From: Brad Marshall Date: Fri, 20 Feb 2015 10:24:14 +1000 Subject: [PATCH 45/48] [bradm] Handle case of empty nagios_servicegroups setting --- .../charmhelpers/contrib/charmsupport/nrpe.py | 2 +- tests/charmhelpers/contrib/amulet/utils.py | 124 +++++++++++++++++- 2 files changed, 123 insertions(+), 3 deletions(-) diff --git a/hooks/charmhelpers/contrib/charmsupport/nrpe.py b/hooks/charmhelpers/contrib/charmsupport/nrpe.py index 8229f6b5..9d961cfb 100644 --- a/hooks/charmhelpers/contrib/charmsupport/nrpe.py +++ b/hooks/charmhelpers/contrib/charmsupport/nrpe.py @@ -210,7 +210,7 @@ class NRPE(object): super(NRPE, self).__init__() self.config = config() self.nagios_context = self.config['nagios_context'] - if 'nagios_servicegroups' in self.config: + if 'nagios_servicegroups' in self.config and self.config['nagios_servicegroups']: self.nagios_servicegroups = self.config['nagios_servicegroups'] else: self.nagios_servicegroups = self.nagios_context diff --git a/tests/charmhelpers/contrib/amulet/utils.py b/tests/charmhelpers/contrib/amulet/utils.py index 3464b873..65219d33 100644 --- a/tests/charmhelpers/contrib/amulet/utils.py +++ b/tests/charmhelpers/contrib/amulet/utils.py @@ -169,8 +169,13 @@ class AmuletUtils(object): cmd = 'pgrep -o -f {}'.format(service) else: cmd = 'pgrep -o {}'.format(service) - proc_dir = '/proc/{}'.format(sentry_unit.run(cmd)[0].strip()) - return self._get_dir_mtime(sentry_unit, proc_dir) + cmd = cmd + ' | grep -v pgrep || exit 0' + cmd_out = sentry_unit.run(cmd) + self.log.debug('CMDout: ' + str(cmd_out)) + if cmd_out[0]: + self.log.debug('Pid for %s %s' % (service, str(cmd_out[0]))) + proc_dir = '/proc/{}'.format(cmd_out[0].strip()) + return self._get_dir_mtime(sentry_unit, proc_dir) def service_restarted(self, sentry_unit, service, filename, pgrep_full=False, sleep_time=20): @@ -187,6 +192,121 @@ class AmuletUtils(object): else: return False + def service_restarted_since(self, sentry_unit, mtime, service, + pgrep_full=False, sleep_time=20, + retry_count=2): + """Check if service was been started after a given time. + + Args: + sentry_unit (sentry): The sentry unit to check for the service on + mtime (float): The epoch time to check against + service (string): service name to look for in process table + pgrep_full (boolean): Use full command line search mode with pgrep + sleep_time (int): Seconds to sleep before looking for process + retry_count (int): If service is not found, how many times to retry + + Returns: + bool: True if service found and its start time it newer than mtime, + False if service is older than mtime or if service was + not found. + """ + self.log.debug('Checking %s restarted since %s' % (service, mtime)) + time.sleep(sleep_time) + proc_start_time = self._get_proc_start_time(sentry_unit, service, + pgrep_full) + while retry_count > 0 and not proc_start_time: + self.log.debug('No pid file found for service %s, will retry %i ' + 'more times' % (service, retry_count)) + time.sleep(30) + proc_start_time = self._get_proc_start_time(sentry_unit, service, + pgrep_full) + retry_count = retry_count - 1 + + if not proc_start_time: + self.log.warn('No proc start time found, assuming service did ' + 'not start') + return False + if proc_start_time >= mtime: + self.log.debug('proc start time is newer than provided mtime' + '(%s >= %s)' % (proc_start_time, mtime)) + return True + else: + self.log.warn('proc start time (%s) is older than provided mtime ' + '(%s), service did not restart' % (proc_start_time, + mtime)) + return False + + def config_updated_since(self, sentry_unit, filename, mtime, + sleep_time=20): + """Check if file was modified after a given time. + + Args: + sentry_unit (sentry): The sentry unit to check the file mtime on + filename (string): The file to check mtime of + mtime (float): The epoch time to check against + sleep_time (int): Seconds to sleep before looking for process + + Returns: + bool: True if file was modified more recently than mtime, False if + file was modified before mtime, + """ + self.log.debug('Checking %s updated since %s' % (filename, mtime)) + time.sleep(sleep_time) + file_mtime = self._get_file_mtime(sentry_unit, filename) + if file_mtime >= mtime: + self.log.debug('File mtime is newer than provided mtime ' + '(%s >= %s)' % (file_mtime, mtime)) + return True + else: + self.log.warn('File mtime %s is older than provided mtime %s' + % (file_mtime, mtime)) + return False + + def validate_service_config_changed(self, sentry_unit, mtime, service, + filename, pgrep_full=False, + sleep_time=20, retry_count=2): + """Check service and file were updated after mtime + + Args: + sentry_unit (sentry): The sentry unit to check for the service on + mtime (float): The epoch time to check against + service (string): service name to look for in process table + filename (string): The file to check mtime of + pgrep_full (boolean): Use full command line search mode with pgrep + sleep_time (int): Seconds to sleep before looking for process + retry_count (int): If service is not found, how many times to retry + + Typical Usage: + u = OpenStackAmuletUtils(ERROR) + ... + mtime = u.get_sentry_time(self.cinder_sentry) + self.d.configure('cinder', {'verbose': 'True', 'debug': 'True'}) + if not u.validate_service_config_changed(self.cinder_sentry, + mtime, + 'cinder-api', + '/etc/cinder/cinder.conf') + amulet.raise_status(amulet.FAIL, msg='update failed') + Returns: + bool: True if both service and file where updated/restarted after + mtime, False if service is older than mtime or if service was + not found or if filename was modified before mtime. + """ + self.log.debug('Checking %s restarted since %s' % (service, mtime)) + time.sleep(sleep_time) + service_restart = self.service_restarted_since(sentry_unit, mtime, + service, + pgrep_full=pgrep_full, + sleep_time=0, + retry_count=retry_count) + config_update = self.config_updated_since(sentry_unit, filename, mtime, + sleep_time=0) + return service_restart and config_update + + def get_sentry_time(self, sentry_unit): + """Return current epoch time on a sentry""" + cmd = "date +'%s'" + return float(sentry_unit.run(cmd)[0]) + def relation_error(self, name, data): return 'unexpected relation data in {} - {}'.format(name, data) From 16fd18d0fc9c9e07c353b441ccdaa11196395117 Mon Sep 17 00:00:00 2001 From: Edward Hope-Morley Date: Tue, 24 Feb 2015 11:05:07 +0000 Subject: [PATCH 46/48] [trivial] charmhelpers sync --- .../charmhelpers/contrib/charmsupport/nrpe.py | 48 ++++++- .../charmhelpers/contrib/hahelpers/cluster.py | 6 +- .../contrib/openstack/files/__init__.py | 18 +++ .../contrib/openstack/files/check_haproxy.sh | 32 +++++ .../files/check_haproxy_queue_depth.sh | 30 +++++ hooks/charmhelpers/contrib/openstack/ip.py | 37 ++++++ hooks/charmhelpers/contrib/openstack/utils.py | 1 + hooks/charmhelpers/core/fstab.py | 4 +- tests/charmhelpers/contrib/amulet/utils.py | 124 +++++++++++++++++- 9 files changed, 288 insertions(+), 12 deletions(-) create mode 100644 hooks/charmhelpers/contrib/openstack/files/__init__.py create mode 100755 hooks/charmhelpers/contrib/openstack/files/check_haproxy.sh create mode 100755 hooks/charmhelpers/contrib/openstack/files/check_haproxy_queue_depth.sh diff --git a/hooks/charmhelpers/contrib/charmsupport/nrpe.py b/hooks/charmhelpers/contrib/charmsupport/nrpe.py index 0fd0a9d8..9d961cfb 100644 --- a/hooks/charmhelpers/contrib/charmsupport/nrpe.py +++ b/hooks/charmhelpers/contrib/charmsupport/nrpe.py @@ -24,6 +24,8 @@ import subprocess import pwd import grp import os +import glob +import shutil import re import shlex import yaml @@ -161,7 +163,7 @@ define service {{ log('Check command not found: {}'.format(parts[0])) return '' - def write(self, nagios_context, hostname, nagios_servicegroups=None): + def write(self, nagios_context, hostname, nagios_servicegroups): nrpe_check_file = '/etc/nagios/nrpe.d/{}.cfg'.format( self.command) with open(nrpe_check_file, 'w') as nrpe_check_config: @@ -177,14 +179,11 @@ define service {{ nagios_servicegroups) def write_service_config(self, nagios_context, hostname, - nagios_servicegroups=None): + nagios_servicegroups): for f in os.listdir(NRPE.nagios_exportdir): if re.search('.*{}.cfg'.format(self.command), f): os.remove(os.path.join(NRPE.nagios_exportdir, f)) - if not nagios_servicegroups: - nagios_servicegroups = nagios_context - templ_vars = { 'nagios_hostname': hostname, 'nagios_servicegroup': nagios_servicegroups, @@ -211,10 +210,10 @@ class NRPE(object): super(NRPE, self).__init__() self.config = config() self.nagios_context = self.config['nagios_context'] - if 'nagios_servicegroups' in self.config: + if 'nagios_servicegroups' in self.config and self.config['nagios_servicegroups']: self.nagios_servicegroups = self.config['nagios_servicegroups'] else: - self.nagios_servicegroups = 'juju' + self.nagios_servicegroups = self.nagios_context self.unit_name = local_unit().replace('/', '-') if hostname: self.hostname = hostname @@ -322,3 +321,38 @@ def add_init_service_checks(nrpe, services, unit_name): check_cmd='check_status_file.py -f ' '/var/lib/nagios/service-check-%s.txt' % svc, ) + + +def copy_nrpe_checks(): + """ + Copy the nrpe checks into place + + """ + NAGIOS_PLUGINS = '/usr/local/lib/nagios/plugins' + nrpe_files_dir = os.path.join(os.getenv('CHARM_DIR'), 'hooks', + 'charmhelpers', 'contrib', 'openstack', + 'files') + + if not os.path.exists(NAGIOS_PLUGINS): + os.makedirs(NAGIOS_PLUGINS) + for fname in glob.glob(os.path.join(nrpe_files_dir, "check_*")): + if os.path.isfile(fname): + shutil.copy2(fname, + os.path.join(NAGIOS_PLUGINS, os.path.basename(fname))) + + +def add_haproxy_checks(nrpe, unit_name): + """ + Add checks for each service in list + + :param NRPE nrpe: NRPE object to add check to + :param str unit_name: Unit name to use in check description + """ + nrpe.add_check( + shortname='haproxy_servers', + description='Check HAProxy {%s}' % unit_name, + check_cmd='check_haproxy.sh') + nrpe.add_check( + shortname='haproxy_queue', + description='Check HAProxy queue depth {%s}' % unit_name, + check_cmd='check_haproxy_queue_depth.sh') diff --git a/hooks/charmhelpers/contrib/hahelpers/cluster.py b/hooks/charmhelpers/contrib/hahelpers/cluster.py index 9a2588b6..9333efc3 100644 --- a/hooks/charmhelpers/contrib/hahelpers/cluster.py +++ b/hooks/charmhelpers/contrib/hahelpers/cluster.py @@ -48,6 +48,9 @@ from charmhelpers.core.hookenv import ( from charmhelpers.core.decorators import ( retry_on_exception, ) +from charmhelpers.core.strutils import ( + bool_from_string, +) class HAIncompleteConfig(Exception): @@ -164,7 +167,8 @@ def https(): . returns: boolean ''' - if config_get('use-https') == "yes": + use_https = config_get('use-https') + if use_https and bool_from_string(use_https): return True if config_get('ssl_cert') and config_get('ssl_key'): return True diff --git a/hooks/charmhelpers/contrib/openstack/files/__init__.py b/hooks/charmhelpers/contrib/openstack/files/__init__.py new file mode 100644 index 00000000..75876796 --- /dev/null +++ b/hooks/charmhelpers/contrib/openstack/files/__init__.py @@ -0,0 +1,18 @@ +# Copyright 2014-2015 Canonical Limited. +# +# This file is part of charm-helpers. +# +# charm-helpers is free software: you can redistribute it and/or modify +# it under the terms of the GNU Lesser General Public License version 3 as +# published by the Free Software Foundation. +# +# charm-helpers is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Lesser General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public License +# along with charm-helpers. If not, see . + +# dummy __init__.py to fool syncer into thinking this is a syncable python +# module diff --git a/hooks/charmhelpers/contrib/openstack/files/check_haproxy.sh b/hooks/charmhelpers/contrib/openstack/files/check_haproxy.sh new file mode 100755 index 00000000..eb8527f5 --- /dev/null +++ b/hooks/charmhelpers/contrib/openstack/files/check_haproxy.sh @@ -0,0 +1,32 @@ +#!/bin/bash +#-------------------------------------------- +# This file is managed by Juju +#-------------------------------------------- +# +# Copyright 2009,2012 Canonical Ltd. +# Author: Tom Haddon + +CRITICAL=0 +NOTACTIVE='' +LOGFILE=/var/log/nagios/check_haproxy.log +AUTH=$(grep -r "stats auth" /etc/haproxy | head -1 | awk '{print $4}') + +for appserver in $(grep ' server' /etc/haproxy/haproxy.cfg | awk '{print $2'}); +do + output=$(/usr/lib/nagios/plugins/check_http -a ${AUTH} -I 127.0.0.1 -p 8888 --regex="class=\"(active|backup)(2|3).*${appserver}" -e ' 200 OK') + if [ $? != 0 ]; then + date >> $LOGFILE + echo $output >> $LOGFILE + /usr/lib/nagios/plugins/check_http -a ${AUTH} -I 127.0.0.1 -p 8888 -v | grep $appserver >> $LOGFILE 2>&1 + CRITICAL=1 + NOTACTIVE="${NOTACTIVE} $appserver" + fi +done + +if [ $CRITICAL = 1 ]; then + echo "CRITICAL:${NOTACTIVE}" + exit 2 +fi + +echo "OK: All haproxy instances looking good" +exit 0 diff --git a/hooks/charmhelpers/contrib/openstack/files/check_haproxy_queue_depth.sh b/hooks/charmhelpers/contrib/openstack/files/check_haproxy_queue_depth.sh new file mode 100755 index 00000000..3ebb5329 --- /dev/null +++ b/hooks/charmhelpers/contrib/openstack/files/check_haproxy_queue_depth.sh @@ -0,0 +1,30 @@ +#!/bin/bash +#-------------------------------------------- +# This file is managed by Juju +#-------------------------------------------- +# +# Copyright 2009,2012 Canonical Ltd. +# Author: Tom Haddon + +# These should be config options at some stage +CURRQthrsh=0 +MAXQthrsh=100 + +AUTH=$(grep -r "stats auth" /etc/haproxy | head -1 | awk '{print $4}') + +HAPROXYSTATS=$(/usr/lib/nagios/plugins/check_http -a ${AUTH} -I 127.0.0.1 -p 8888 -u '/;csv' -v) + +for BACKEND in $(echo $HAPROXYSTATS| xargs -n1 | grep BACKEND | awk -F , '{print $1}') +do + CURRQ=$(echo "$HAPROXYSTATS" | grep $BACKEND | grep BACKEND | cut -d , -f 3) + MAXQ=$(echo "$HAPROXYSTATS" | grep $BACKEND | grep BACKEND | cut -d , -f 4) + + if [[ $CURRQ -gt $CURRQthrsh || $MAXQ -gt $MAXQthrsh ]] ; then + echo "CRITICAL: queue depth for $BACKEND - CURRENT:$CURRQ MAX:$MAXQ" + exit 2 + fi +done + +echo "OK: All haproxy queue depths looking good" +exit 0 + diff --git a/hooks/charmhelpers/contrib/openstack/ip.py b/hooks/charmhelpers/contrib/openstack/ip.py index 9eabed73..29bbddcb 100644 --- a/hooks/charmhelpers/contrib/openstack/ip.py +++ b/hooks/charmhelpers/contrib/openstack/ip.py @@ -26,6 +26,8 @@ from charmhelpers.contrib.network.ip import ( ) from charmhelpers.contrib.hahelpers.cluster import is_clustered +from functools import partial + PUBLIC = 'public' INTERNAL = 'int' ADMIN = 'admin' @@ -107,3 +109,38 @@ def resolve_address(endpoint_type=PUBLIC): "clustered=%s)" % (net_type, clustered)) return resolved_address + + +def endpoint_url(configs, url_template, port, endpoint_type=PUBLIC, + override=None): + """Returns the correct endpoint URL to advertise to Keystone. + + This method provides the correct endpoint URL which should be advertised to + the keystone charm for endpoint creation. This method allows for the url to + be overridden to force a keystone endpoint to have specific URL for any of + the defined scopes (admin, internal, public). + + :param configs: OSTemplateRenderer config templating object to inspect + for a complete https context. + :param url_template: str format string for creating the url template. Only + two values will be passed - the scheme+hostname + returned by the canonical_url and the port. + :param endpoint_type: str endpoint type to resolve. + :param override: str the name of the config option which overrides the + endpoint URL defined by the charm itself. None will + disable any overrides (default). + """ + if override: + # Return any user-defined overrides for the keystone endpoint URL. + user_value = config(override) + if user_value: + return user_value.strip() + + return url_template % (canonical_url(configs, endpoint_type), port) + + +public_endpoint = partial(endpoint_url, endpoint_type=PUBLIC) + +internal_endpoint = partial(endpoint_url, endpoint_type=INTERNAL) + +admin_endpoint = partial(endpoint_url, endpoint_type=ADMIN) diff --git a/hooks/charmhelpers/contrib/openstack/utils.py b/hooks/charmhelpers/contrib/openstack/utils.py index 26259a03..af2b3596 100644 --- a/hooks/charmhelpers/contrib/openstack/utils.py +++ b/hooks/charmhelpers/contrib/openstack/utils.py @@ -103,6 +103,7 @@ SWIFT_CODENAMES = OrderedDict([ ('2.1.0', 'juno'), ('2.2.0', 'juno'), ('2.2.1', 'kilo'), + ('2.2.2', 'kilo'), ]) DEFAULT_LOOPBACK_SIZE = '5G' diff --git a/hooks/charmhelpers/core/fstab.py b/hooks/charmhelpers/core/fstab.py index 9cdcc886..3056fbac 100644 --- a/hooks/charmhelpers/core/fstab.py +++ b/hooks/charmhelpers/core/fstab.py @@ -77,7 +77,7 @@ class Fstab(io.FileIO): for line in self.readlines(): line = line.decode('us-ascii') try: - if line.strip() and not line.startswith("#"): + if line.strip() and not line.strip().startswith("#"): yield self._hydrate_entry(line) except ValueError: pass @@ -104,7 +104,7 @@ class Fstab(io.FileIO): found = False for index, line in enumerate(lines): - if not line.startswith("#"): + if line.strip() and not line.strip().startswith("#"): if self._hydrate_entry(line) == entry: found = True break diff --git a/tests/charmhelpers/contrib/amulet/utils.py b/tests/charmhelpers/contrib/amulet/utils.py index 3464b873..65219d33 100644 --- a/tests/charmhelpers/contrib/amulet/utils.py +++ b/tests/charmhelpers/contrib/amulet/utils.py @@ -169,8 +169,13 @@ class AmuletUtils(object): cmd = 'pgrep -o -f {}'.format(service) else: cmd = 'pgrep -o {}'.format(service) - proc_dir = '/proc/{}'.format(sentry_unit.run(cmd)[0].strip()) - return self._get_dir_mtime(sentry_unit, proc_dir) + cmd = cmd + ' | grep -v pgrep || exit 0' + cmd_out = sentry_unit.run(cmd) + self.log.debug('CMDout: ' + str(cmd_out)) + if cmd_out[0]: + self.log.debug('Pid for %s %s' % (service, str(cmd_out[0]))) + proc_dir = '/proc/{}'.format(cmd_out[0].strip()) + return self._get_dir_mtime(sentry_unit, proc_dir) def service_restarted(self, sentry_unit, service, filename, pgrep_full=False, sleep_time=20): @@ -187,6 +192,121 @@ class AmuletUtils(object): else: return False + def service_restarted_since(self, sentry_unit, mtime, service, + pgrep_full=False, sleep_time=20, + retry_count=2): + """Check if service was been started after a given time. + + Args: + sentry_unit (sentry): The sentry unit to check for the service on + mtime (float): The epoch time to check against + service (string): service name to look for in process table + pgrep_full (boolean): Use full command line search mode with pgrep + sleep_time (int): Seconds to sleep before looking for process + retry_count (int): If service is not found, how many times to retry + + Returns: + bool: True if service found and its start time it newer than mtime, + False if service is older than mtime or if service was + not found. + """ + self.log.debug('Checking %s restarted since %s' % (service, mtime)) + time.sleep(sleep_time) + proc_start_time = self._get_proc_start_time(sentry_unit, service, + pgrep_full) + while retry_count > 0 and not proc_start_time: + self.log.debug('No pid file found for service %s, will retry %i ' + 'more times' % (service, retry_count)) + time.sleep(30) + proc_start_time = self._get_proc_start_time(sentry_unit, service, + pgrep_full) + retry_count = retry_count - 1 + + if not proc_start_time: + self.log.warn('No proc start time found, assuming service did ' + 'not start') + return False + if proc_start_time >= mtime: + self.log.debug('proc start time is newer than provided mtime' + '(%s >= %s)' % (proc_start_time, mtime)) + return True + else: + self.log.warn('proc start time (%s) is older than provided mtime ' + '(%s), service did not restart' % (proc_start_time, + mtime)) + return False + + def config_updated_since(self, sentry_unit, filename, mtime, + sleep_time=20): + """Check if file was modified after a given time. + + Args: + sentry_unit (sentry): The sentry unit to check the file mtime on + filename (string): The file to check mtime of + mtime (float): The epoch time to check against + sleep_time (int): Seconds to sleep before looking for process + + Returns: + bool: True if file was modified more recently than mtime, False if + file was modified before mtime, + """ + self.log.debug('Checking %s updated since %s' % (filename, mtime)) + time.sleep(sleep_time) + file_mtime = self._get_file_mtime(sentry_unit, filename) + if file_mtime >= mtime: + self.log.debug('File mtime is newer than provided mtime ' + '(%s >= %s)' % (file_mtime, mtime)) + return True + else: + self.log.warn('File mtime %s is older than provided mtime %s' + % (file_mtime, mtime)) + return False + + def validate_service_config_changed(self, sentry_unit, mtime, service, + filename, pgrep_full=False, + sleep_time=20, retry_count=2): + """Check service and file were updated after mtime + + Args: + sentry_unit (sentry): The sentry unit to check for the service on + mtime (float): The epoch time to check against + service (string): service name to look for in process table + filename (string): The file to check mtime of + pgrep_full (boolean): Use full command line search mode with pgrep + sleep_time (int): Seconds to sleep before looking for process + retry_count (int): If service is not found, how many times to retry + + Typical Usage: + u = OpenStackAmuletUtils(ERROR) + ... + mtime = u.get_sentry_time(self.cinder_sentry) + self.d.configure('cinder', {'verbose': 'True', 'debug': 'True'}) + if not u.validate_service_config_changed(self.cinder_sentry, + mtime, + 'cinder-api', + '/etc/cinder/cinder.conf') + amulet.raise_status(amulet.FAIL, msg='update failed') + Returns: + bool: True if both service and file where updated/restarted after + mtime, False if service is older than mtime or if service was + not found or if filename was modified before mtime. + """ + self.log.debug('Checking %s restarted since %s' % (service, mtime)) + time.sleep(sleep_time) + service_restart = self.service_restarted_since(sentry_unit, mtime, + service, + pgrep_full=pgrep_full, + sleep_time=0, + retry_count=retry_count) + config_update = self.config_updated_since(sentry_unit, filename, mtime, + sleep_time=0) + return service_restart and config_update + + def get_sentry_time(self, sentry_unit): + """Return current epoch time on a sentry""" + cmd = "date +'%s'" + return float(sentry_unit.run(cmd)[0]) + def relation_error(self, name, data): return 'unexpected relation data in {} - {}'.format(name, data) From 9ae83731b549eacff06082c4d37b9f024b2bc7bb Mon Sep 17 00:00:00 2001 From: James Page Date: Wed, 4 Mar 2015 09:51:12 +0000 Subject: [PATCH 47/48] Automated resync of charm-helpers --- hooks/charmhelpers/contrib/network/ip.py | 85 ++++++++++++++++++- .../charmhelpers/contrib/openstack/context.py | 35 +++++++- hooks/charmhelpers/contrib/openstack/utils.py | 78 ++--------------- hooks/charmhelpers/core/services/helpers.py | 16 +++- 4 files changed, 136 insertions(+), 78 deletions(-) diff --git a/hooks/charmhelpers/contrib/network/ip.py b/hooks/charmhelpers/contrib/network/ip.py index 98b17544..fff6d5ca 100644 --- a/hooks/charmhelpers/contrib/network/ip.py +++ b/hooks/charmhelpers/contrib/network/ip.py @@ -17,13 +17,16 @@ import glob import re import subprocess +import six +import socket from functools import partial from charmhelpers.core.hookenv import unit_get from charmhelpers.fetch import apt_install from charmhelpers.core.hookenv import ( - log + log, + WARNING, ) try: @@ -365,3 +368,83 @@ def is_bridge_member(nic): return True return False + + +def is_ip(address): + """ + Returns True if address is a valid IP address. + """ + try: + # Test to see if already an IPv4 address + socket.inet_aton(address) + return True + except socket.error: + return False + + +def ns_query(address): + try: + import dns.resolver + except ImportError: + apt_install('python-dnspython') + import dns.resolver + + if isinstance(address, dns.name.Name): + rtype = 'PTR' + elif isinstance(address, six.string_types): + rtype = 'A' + else: + return None + + answers = dns.resolver.query(address, rtype) + if answers: + return str(answers[0]) + return None + + +def get_host_ip(hostname, fallback=None): + """ + Resolves the IP for a given hostname, or returns + the input if it is already an IP. + """ + if is_ip(hostname): + return hostname + + ip_addr = ns_query(hostname) + if not ip_addr: + try: + ip_addr = socket.gethostbyname(hostname) + except: + log("Failed to resolve hostname '%s'" % (hostname), + level=WARNING) + return fallback + return ip_addr + + +def get_hostname(address, fqdn=True): + """ + Resolves hostname for given IP, or returns the input + if it is already a hostname. + """ + if is_ip(address): + try: + import dns.reversename + except ImportError: + apt_install("python-dnspython") + import dns.reversename + + rev = dns.reversename.from_address(address) + result = ns_query(rev) + if not result: + return None + else: + result = address + + if fqdn: + # strip trailing . + if result.endswith('.'): + return result[:-1] + else: + return result + else: + return result.split('.')[0] diff --git a/hooks/charmhelpers/contrib/openstack/context.py b/hooks/charmhelpers/contrib/openstack/context.py index d268ea8f..2d9a95cd 100644 --- a/hooks/charmhelpers/contrib/openstack/context.py +++ b/hooks/charmhelpers/contrib/openstack/context.py @@ -21,6 +21,7 @@ from base64 import b64decode from subprocess import check_call import six +import yaml from charmhelpers.fetch import ( apt_install, @@ -104,9 +105,41 @@ def context_complete(ctxt): def config_flags_parser(config_flags): """Parses config flags string into dict. + This parsing method supports a few different formats for the config + flag values to be parsed: + + 1. A string in the simple format of key=value pairs, with the possibility + of specifying multiple key value pairs within the same string. For + example, a string in the format of 'key1=value1, key2=value2' will + return a dict of: + {'key1': 'value1', + 'key2': 'value2'}. + + 2. A string in the above format, but supporting a comma-delimited list + of values for the same key. For example, a string in the format of + 'key1=value1, key2=value3,value4,value5' will return a dict of: + {'key1', 'value1', + 'key2', 'value2,value3,value4'} + + 3. A string containing a colon character (:) prior to an equal + character (=) will be treated as yaml and parsed as such. This can be + used to specify more complex key value pairs. For example, + a string in the format of 'key1: subkey1=value1, subkey2=value2' will + return a dict of: + {'key1', 'subkey1=value1, subkey2=value2'} + The provided config_flags string may be a list of comma-separated values which themselves may be comma-separated list of values. """ + # If we find a colon before an equals sign then treat it as yaml. + # Note: limit it to finding the colon first since this indicates assignment + # for inline yaml. + colon = config_flags.find(':') + equals = config_flags.find('=') + if colon > 0: + if colon < equals or equals < 0: + return yaml.safe_load(config_flags) + if config_flags.find('==') >= 0: log("config_flags is not in expected format (key=value)", level=ERROR) raise OSContextError @@ -191,7 +224,7 @@ class SharedDBContext(OSContextGenerator): unit=local_unit()) if set_hostname != access_hostname: relation_set(relation_settings={hostname_key: access_hostname}) - return ctxt # Defer any further hook execution for now.... + return None # Defer any further hook execution for now.... password_setting = 'password' if self.relation_prefix: diff --git a/hooks/charmhelpers/contrib/openstack/utils.py b/hooks/charmhelpers/contrib/openstack/utils.py index af2b3596..4f110c63 100644 --- a/hooks/charmhelpers/contrib/openstack/utils.py +++ b/hooks/charmhelpers/contrib/openstack/utils.py @@ -23,12 +23,13 @@ from functools import wraps import subprocess import json import os -import socket import sys import six import yaml +from charmhelpers.contrib.network import ip + from charmhelpers.core.hookenv import ( config, log as juju_log, @@ -421,77 +422,10 @@ def clean_storage(block_device): else: zap_disk(block_device) - -def is_ip(address): - """ - Returns True if address is a valid IP address. - """ - try: - # Test to see if already an IPv4 address - socket.inet_aton(address) - return True - except socket.error: - return False - - -def ns_query(address): - try: - import dns.resolver - except ImportError: - apt_install('python-dnspython') - import dns.resolver - - if isinstance(address, dns.name.Name): - rtype = 'PTR' - elif isinstance(address, six.string_types): - rtype = 'A' - else: - return None - - answers = dns.resolver.query(address, rtype) - if answers: - return str(answers[0]) - return None - - -def get_host_ip(hostname): - """ - Resolves the IP for a given hostname, or returns - the input if it is already an IP. - """ - if is_ip(hostname): - return hostname - - return ns_query(hostname) - - -def get_hostname(address, fqdn=True): - """ - Resolves hostname for given IP, or returns the input - if it is already a hostname. - """ - if is_ip(address): - try: - import dns.reversename - except ImportError: - apt_install('python-dnspython') - import dns.reversename - - rev = dns.reversename.from_address(address) - result = ns_query(rev) - if not result: - return None - else: - result = address - - if fqdn: - # strip trailing . - if result.endswith('.'): - return result[:-1] - else: - return result - else: - return result.split('.')[0] +is_ip = ip.is_ip +ns_query = ip.ns_query +get_host_ip = ip.get_host_ip +get_hostname = ip.get_hostname def get_matchmaker_map(mm_file='/etc/oslo/matchmaker_ring.json'): diff --git a/hooks/charmhelpers/core/services/helpers.py b/hooks/charmhelpers/core/services/helpers.py index 5e3af9da..15b21664 100644 --- a/hooks/charmhelpers/core/services/helpers.py +++ b/hooks/charmhelpers/core/services/helpers.py @@ -45,12 +45,14 @@ class RelationContext(dict): """ name = None interface = None - required_keys = [] def __init__(self, name=None, additional_required_keys=None): + if not hasattr(self, 'required_keys'): + self.required_keys = [] + if name is not None: self.name = name - if additional_required_keys is not None: + if additional_required_keys: self.required_keys.extend(additional_required_keys) self.get_data() @@ -134,7 +136,10 @@ class MysqlRelation(RelationContext): """ name = 'db' interface = 'mysql' - required_keys = ['host', 'user', 'password', 'database'] + + def __init__(self, *args, **kwargs): + self.required_keys = ['host', 'user', 'password', 'database'] + super(HttpRelation).__init__(self, *args, **kwargs) class HttpRelation(RelationContext): @@ -146,7 +151,10 @@ class HttpRelation(RelationContext): """ name = 'website' interface = 'http' - required_keys = ['host', 'port'] + + def __init__(self, *args, **kwargs): + self.required_keys = ['host', 'port'] + super(HttpRelation).__init__(self, *args, **kwargs) def provide_data(self): return { From 277fbae84d06eec53dc48a36debb97135826e51a Mon Sep 17 00:00:00 2001 From: Edward Hope-Morley Date: Tue, 10 Mar 2015 12:02:11 +0000 Subject: [PATCH 48/48] [hopem,r=] Fixes disable ssl. Allows disable of use-https and https-service-endpoints. Use '__null__' value to flush out peer relation settings that need to be unset when forwared to other relations. This will fix ssl disable by ensuring that peer settings are correctly forwarded to endpoint relations. Closes-Bug: 1427906 --- hooks/keystone_hooks.py | 85 +++++++++++-------------------- hooks/keystone_utils.py | 37 ++++++++++++-- tests/basic_deployment.py | 1 - unit_tests/test_keystone_hooks.py | 11 ++-- unit_tests/test_keystone_utils.py | 21 +++++--- 5 files changed, 84 insertions(+), 71 deletions(-) diff --git a/hooks/keystone_hooks.py b/hooks/keystone_hooks.py index 0e45f3d7..b8536ef4 100755 --- a/hooks/keystone_hooks.py +++ b/hooks/keystone_hooks.py @@ -72,6 +72,7 @@ from keystone_utils import ( is_db_ready, clear_ssl_synced_units, is_db_initialised, + filter_null, ) from charmhelpers.contrib.hahelpers.cluster import ( @@ -87,7 +88,6 @@ from charmhelpers.contrib.peerstorage import ( ) from charmhelpers.contrib.openstack.ip import ( ADMIN, - PUBLIC, resolve_address, ) from charmhelpers.contrib.network.ip import ( @@ -154,7 +154,7 @@ def config_changed(): for rid in relation_ids('identity-admin'): admin_relation_changed(rid) - # Ensure sync request is sent out (needed for upgrade to ssl from non-ssl) + # Ensure sync request is sent out (needed for any/all ssl change) send_ssl_sync_request() for r_id in relation_ids('ha'): @@ -297,6 +297,8 @@ def identity_changed(relation_id=None, remote_unit=None): # with the info dies the settings die with it Bug# 1355848 for rel_id in relation_ids('identity-service'): peerdb_settings = peer_retrieve_by_prefix(rel_id) + # Ensure the null'd settings are unset in the relation. + peerdb_settings = filter_null(peerdb_settings) if 'service_password' in peerdb_settings: relation_set(relation_id=rel_id, **peerdb_settings) log('Deferring identity_changed() to service leader.') @@ -323,22 +325,31 @@ def send_ssl_sync_request(): if bool_from_string(config('https-service-endpoints')): count += 2 - if count: - key = 'ssl-sync-required-%s' % (unit) - settings = {key: count} - prev = 0 - rid = None - for rid in relation_ids('cluster'): - for unit in related_units(rid): - _prev = relation_get(rid=rid, unit=unit, attribute=key) or 0 - if _prev and _prev > prev: - prev = _prev + key = 'ssl-sync-required-%s' % (unit) + settings = {key: count} - if rid and prev < count: - clear_ssl_synced_units() - log("Setting %s=%s" % (key, count), level=DEBUG) + # If all ssl is disabled ensure this is set to 0 so that cluster hook runs + # and endpoints are updated. + if not count: + log("Setting %s=%s" % (key, count), level=DEBUG) + for rid in relation_ids('cluster'): relation_set(relation_id=rid, relation_settings=settings) + return + + prev = 0 + rid = None + for rid in relation_ids('cluster'): + for unit in related_units(rid): + _prev = relation_get(rid=rid, unit=unit, attribute=key) or 0 + if _prev and _prev > prev: + prev = _prev + + if rid and prev < count: + clear_ssl_synced_units() + log("Setting %s=%s" % (key, count), level=DEBUG) + relation_set(relation_id=rid, relation_settings=settings) + @hooks.hook('cluster-relation-joined') def cluster_joined(): @@ -364,37 +375,6 @@ def cluster_joined(): send_ssl_sync_request() -def apply_echo_filters(settings, echo_whitelist): - """Filter settings to be peer_echo'ed. - - We may have received some data that we don't want to re-echo so filter - out unwanted keys and provide overrides. - - Returns: - tuple(filtered list of keys to be echoed, overrides for keys omitted) - """ - filtered = [] - overrides = {} - for key in settings.iterkeys(): - for ekey in echo_whitelist: - if ekey in key: - if ekey == 'identity-service:': - auth_host = resolve_address(ADMIN) - service_host = resolve_address(PUBLIC) - if (key.endswith('auth_host') and - settings[key] != auth_host): - overrides[key] = auth_host - continue - elif (key.endswith('service_host') and - settings[key] != service_host): - overrides[key] = service_host - continue - - filtered.append(key) - - return filtered, overrides - - @hooks.hook('cluster-relation-changed', 'cluster-relation-departed') @restart_on_change(restart_map(), stopstart=True) @@ -403,16 +383,11 @@ def cluster_changed(): group='juju_keystone', peer_interface='cluster', ensure_local_user=True) - settings = relation_get() # NOTE(jamespage) re-echo passwords for peer storage - echo_whitelist, overrides = \ - apply_echo_filters(settings, ['_passwd', 'identity-service:', - 'ssl-cert-master', 'db-initialised']) - log("Peer echo overrides: %s" % (overrides), level=DEBUG) - relation_set(**overrides) - if echo_whitelist: - log("Peer echo whitelist: %s" % (echo_whitelist), level=DEBUG) - peer_echo(includes=echo_whitelist) + echo_whitelist = ['_passwd', 'identity-service:', 'ssl-cert-master', + 'db-initialised'] + log("Peer echo whitelist: %s" % (echo_whitelist), level=DEBUG) + peer_echo(includes=echo_whitelist) check_peer_actions() diff --git a/hooks/keystone_utils.py b/hooks/keystone_utils.py index 12a04fa2..cad138ce 100644 --- a/hooks/keystone_utils.py +++ b/hooks/keystone_utils.py @@ -230,6 +230,25 @@ valid_services = { } +def filter_null(settings, null='__null__'): + """Replace null values with None in provided settings dict. + + When storing values in the peer relation, it might be necessary at some + future point to flush these values. We therefore need to use a real + (non-None or empty string) value to represent an unset settings. This value + then needs to be converted to None when applying to a non-cluster relation + so that the value is actually unset. + """ + filtered = {} + for k, v in settings.iteritems(): + if v == null: + filtered[k] = None + else: + filtered[k] = v + + return filtered + + def resource_map(): """Dynamically generate a map of resources that will be managed for a single hook execution. @@ -1294,6 +1313,9 @@ def add_service_to_keystone(relation_id=None, remote_unit=None): # we return a token, information about our API endpoints, and the generated # service credentials service_tenant = config('service-tenant') + + # NOTE(dosaboy): we use __null__ to represent settings that are to be + # routed to relations via the cluster relation and set to None. relation_data = { "admin_token": token, "service_host": resolve_address(PUBLIC), @@ -1304,10 +1326,10 @@ def add_service_to_keystone(relation_id=None, remote_unit=None): "service_password": service_password, "service_tenant": service_tenant, "service_tenant_id": manager.resolve_tenant_id(service_tenant), - "https_keystone": "False", - "ssl_cert": "", - "ssl_key": "", - "ca_cert": "", + "https_keystone": '__null__', + "ssl_cert": '__null__', + "ssl_key": '__null__', + "ca_cert": '__null__', "auth_protocol": protocol, "service_protocol": protocol, } @@ -1331,7 +1353,12 @@ def add_service_to_keystone(relation_id=None, remote_unit=None): relation_data['ca_cert'] = b64encode(ca_bundle) relation_data['https_keystone'] = 'True' - peer_store_and_set(relation_id=relation_id, **relation_data) + # NOTE(dosaboy): '__null__' settings are for peer relation only so that + # settings can flushed so we filter them out for non-peer relation. + filtered = filter_null(relation_data) + relation_set(relation_id=relation_id, **filtered) + for rid in relation_ids('cluster'): + relation_set(relation_id=rid, **relation_data) def ensure_valid_service(service): diff --git a/tests/basic_deployment.py b/tests/basic_deployment.py index 75c1062d..21b58f89 100644 --- a/tests/basic_deployment.py +++ b/tests/basic_deployment.py @@ -253,7 +253,6 @@ class KeystoneBasicDeployment(OpenStackAmuletDeployment): 'auth_port': '35357', 'auth_protocol': 'http', 'private-address': u.valid_ip, - 'https_keystone': 'False', 'auth_host': u.valid_ip, 'service_username': 'cinder', 'service_tenant_id': u.not_null, diff --git a/unit_tests/test_keystone_hooks.py b/unit_tests/test_keystone_hooks.py index 12cc3f48..1e4ef54c 100644 --- a/unit_tests/test_keystone_hooks.py +++ b/unit_tests/test_keystone_hooks.py @@ -273,6 +273,7 @@ class KeystoneRelationTests(CharmTestCase): @patch('keystone_utils.log') @patch('keystone_utils.ensure_ssl_cert_master') + @patch.object(hooks, 'send_ssl_sync_request') @patch.object(hooks, 'is_db_initialised') @patch.object(hooks, 'is_db_ready') @patch.object(hooks, 'peer_units') @@ -289,6 +290,7 @@ class KeystoneRelationTests(CharmTestCase): configs, get_homedir, ensure_user, cluster_joined, admin_relation_changed, ensure_permissions, mock_peer_units, mock_is_db_ready, mock_is_db_initialised, + mock_send_ssl_sync_request, mock_ensure_ssl_cert_master, mock_log): mock_is_db_initialised.return_value = True mock_is_db_ready.return_value = True @@ -348,6 +350,7 @@ class KeystoneRelationTests(CharmTestCase): @patch('keystone_utils.log') @patch('keystone_utils.ensure_ssl_cert_master') + @patch.object(hooks, 'send_ssl_sync_request') @patch.object(hooks, 'is_db_initialised') @patch.object(hooks, 'is_db_ready') @patch.object(hooks, 'peer_units') @@ -368,6 +371,7 @@ class KeystoneRelationTests(CharmTestCase): mock_peer_units, mock_is_db_ready, mock_is_db_initialised, + mock_send_ssl_sync_request, mock_ensure_ssl_cert_master, mock_log): mock_is_db_ready.return_value = True @@ -462,11 +466,10 @@ class KeystoneRelationTests(CharmTestCase): mock_peer_units.return_value = ['unit/0'] mock_ensure_ssl_cert_master.return_value = False self.is_elected_leader.return_value = False - self.relation_get.return_value = {'foo_passwd': '123', - 'identity-service:16_foo': 'bar'} hooks.cluster_changed() - self.peer_echo.assert_called_with(includes=['foo_passwd', - 'identity-service:16_foo']) + whitelist = ['_passwd', 'identity-service:', 'ssl-cert-master', + 'db-initialised'] + self.peer_echo.assert_called_with(includes=whitelist) ssh_authorized_peers.assert_called_with( user=self.ssh_user, group='juju_keystone', peer_interface='cluster', ensure_local_user=True) diff --git a/unit_tests/test_keystone_utils.py b/unit_tests/test_keystone_utils.py index ac71496c..4691d0fa 100644 --- a/unit_tests/test_keystone_utils.py +++ b/unit_tests/test_keystone_utils.py @@ -210,6 +210,7 @@ class TestKeystoneUtils(CharmTestCase): self.https.return_value = False self.test_config.set('https-service-endpoints', 'False') self.get_local_endpoint.return_value = 'http://localhost:80/v2.0/' + self.relation_ids.return_value = ['cluster/0'] mock_keystone = MagicMock() mock_keystone.resolve_tenant_id.return_value = 'tenant_id' @@ -239,15 +240,23 @@ class TestKeystoneUtils(CharmTestCase): 'auth_port': 80, 'service_username': 'keystone', 'service_password': 'password', 'service_tenant': 'tenant', - 'https_keystone': 'False', - 'ssl_cert': '', 'ssl_key': '', - 'ca_cert': '', 'auth_host': '10.0.0.3', + 'https_keystone': '__null__', + 'ssl_cert': '__null__', 'ssl_key': '__null__', + 'ca_cert': '__null__', 'auth_host': '10.0.0.3', 'service_host': '10.0.0.3', 'auth_protocol': 'http', 'service_protocol': 'http', 'service_tenant_id': 'tenant_id'} - self.peer_store_and_set.assert_called_with( - relation_id=relation_id, - **relation_data) + + filtered = {} + for k, v in relation_data.iteritems(): + if v == '__null__': + filtered[k] = None + else: + filtered[k] = v + + call1 = call(relation_id=relation_id, **filtered) + call2 = call(relation_id='cluster/0', **relation_data) + self.relation_set.assert_has_calls([call1, call2]) @patch.object(utils, 'ensure_valid_service') @patch.object(utils, 'add_endpoint')