From 0c0c5ff9c09a25de6f2e3c077e8df027aff8c44e Mon Sep 17 00:00:00 2001 From: James Page Date: Wed, 26 Feb 2014 16:54:26 +0000 Subject: [PATCH 01/25] Re-instate a load of lost function --- hooks/charmhelpers/contrib/__init__.py | 0 .../contrib/hahelpers/__init__.py | 0 .../contrib/openstack/__init__.py | 0 .../charmhelpers/contrib/openstack/context.py | 25 +++++- .../charmhelpers/contrib/storage/__init__.py | 0 .../contrib/storage/linux/__init__.py | 0 hooks/keystone_hooks.py | 89 ++++++++++--------- hooks/keystone_utils.py | 22 ++++- 8 files changed, 85 insertions(+), 51 deletions(-) create mode 100644 hooks/charmhelpers/contrib/__init__.py create mode 100644 hooks/charmhelpers/contrib/hahelpers/__init__.py create mode 100644 hooks/charmhelpers/contrib/openstack/__init__.py create mode 100644 hooks/charmhelpers/contrib/storage/__init__.py create mode 100644 hooks/charmhelpers/contrib/storage/linux/__init__.py diff --git a/hooks/charmhelpers/contrib/__init__.py b/hooks/charmhelpers/contrib/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/hooks/charmhelpers/contrib/hahelpers/__init__.py b/hooks/charmhelpers/contrib/hahelpers/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/hooks/charmhelpers/contrib/openstack/__init__.py b/hooks/charmhelpers/contrib/openstack/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/hooks/charmhelpers/contrib/openstack/context.py b/hooks/charmhelpers/contrib/openstack/context.py index 81ac12b6..5785b6a1 100644 --- a/hooks/charmhelpers/contrib/openstack/context.py +++ b/hooks/charmhelpers/contrib/openstack/context.py @@ -29,6 +29,7 @@ from charmhelpers.contrib.hahelpers.cluster import ( determine_apache_port, determine_api_port, https, + is_clustered ) from charmhelpers.contrib.hahelpers.apache import ( @@ -240,17 +241,19 @@ class CephContext(OSContextGenerator): '''This generates context for /etc/ceph/ceph.conf templates''' if not relation_ids('ceph'): return {} + log('Generating template context for ceph') + mon_hosts = [] auth = None key = None + use_syslog = str(config('use-syslog')).lower() for rid in relation_ids('ceph'): for unit in related_units(rid): mon_hosts.append(relation_get('private-address', rid=rid, unit=unit)) auth = relation_get('auth', rid=rid, unit=unit) key = relation_get('key', rid=rid, unit=unit) - use_syslog = str(config('use-syslog')).lower() ctxt = { 'mon_hosts': ' '.join(mon_hosts), @@ -393,7 +396,7 @@ class ApacheSSLContext(OSContextGenerator): return ctxt -class NeutronContext(object): +class NeutronContext(OSContextGenerator): interfaces = [] @property @@ -454,6 +457,22 @@ class NeutronContext(object): return nvp_ctxt + def neutron_ctxt(self): + if https(): + proto = 'https' + else: + proto = 'http' + if is_clustered(): + host = config('vip') + else: + host = unit_get('private-address') + url = '%s://%s:%s' % (proto, host, '9292') + ctxt = { + 'network_manager': self.network_manager, + 'neutron_url': url, + } + return ctxt + def __call__(self): self._ensure_packages() @@ -463,7 +482,7 @@ class NeutronContext(object): if not self.plugin: return {} - ctxt = {'network_manager': self.network_manager} + ctxt = self.neutron_ctxt() if self.plugin == 'ovs': ctxt.update(self.ovs_ctxt()) diff --git a/hooks/charmhelpers/contrib/storage/__init__.py b/hooks/charmhelpers/contrib/storage/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/hooks/charmhelpers/contrib/storage/linux/__init__.py b/hooks/charmhelpers/contrib/storage/linux/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/hooks/keystone_hooks.py b/hooks/keystone_hooks.py index 4d327d88..8fcee361 100755 --- a/hooks/keystone_hooks.py +++ b/hooks/keystone_hooks.py @@ -2,6 +2,7 @@ import os import sys +import time from subprocess import check_call @@ -21,11 +22,14 @@ from charmhelpers.core.hookenv import ( from charmhelpers.core.host import ( mkdir, restart_on_change, - service_restart + service_restart, + service_stop, + service_start ) from charmhelpers.fetch import ( - apt_install, apt_update + apt_install, apt_update, + filter_installed_packages ) from charmhelpers.contrib.openstack.utils import ( @@ -60,6 +64,7 @@ from charmhelpers.payload.execd import execd_preinstall hooks = Hooks() CONFIGS = register_configs() + @hooks.hook() def install(): execd_preinstall() @@ -67,21 +72,25 @@ def install(): apt_update() apt_install(determine_packages(), fatal=True) + @hooks.hook('config-changed') -@restart_on_change(restart_map(), stopstart=True) def config_changed(): - unison.ensure_user(user=SSH_USER, group='juju_keystone') + unison.ensure_user(user=SSH_USER, group='keystone') homedir = unison.get_homedir(SSH_USER) if not os.path.isdir(homedir): - mkdir(homedir, SSH_USER, 'juju_keystone', 0775) - check_call(['chmod', '-R', 'g+wrx', '/var/lib/keystone/']) + mkdir(homedir, SSH_USER, 'keystone', 0775) + if openstack_upgrade_available('keystone'): do_openstack_upgrade(configs=CONFIGS) - check_call(['chmod', '-R', 'g+wrx', '/var/lib/keystone/']) + + check_call(['chmod', '-R', 'g+wrx', '/var/lib/keystone/']) + save_script_rc() configure_https() CONFIGS.write_all() service_restart('keystone') + time.sleep(10) + if eligible_leader(CLUSTER_RES): migrate_database() ensure_initial_admin(config) @@ -89,9 +98,9 @@ def config_changed(): # HTTPS may have been set - so fire all identity relations # again for r_id in relation_ids('identity-service'): - for unit in relation_list(r_id): - identity_changed(relation_id=r_id, - remote_unit=unit) + for unit in relation_list(r_id): + identity_changed(relation_id=r_id, + remote_unit=unit) @hooks.hook('shared-db-relation-joined') @@ -106,12 +115,11 @@ def db_joined(): def db_changed(): if 'shared-db' not in CONFIGS.complete_contexts(): log('shared-db relation incomplete. Peer not ready?') - return - CONFIGS.write(KEYSTONE_CONF) - service_restart('keystone') - if eligible_leader(CLUSTER_RES): - migrate_database() - ensure_initial_admin(config) + else: + CONFIGS.write(KEYSTONE_CONF) + if eligible_leader(CLUSTER_RES): + migrate_database() + ensure_initial_admin(config) @hooks.hook('identity-service-relation-joined') @@ -121,16 +129,12 @@ def identity_joined(): @hooks.hook('identity-service-relation-changed') -@restart_on_change(restart_map()) -def identity_changed(): - if not eligible_leader(CLUSTER_RES): - log('Deferring identity_changed() to service leader.') - #if 'identity-service' not in CONFIGS.complete_contexts(): - # return +def identity_changed(relation_id=None, remote_unit=None): if eligible_leader(CLUSTER_RES): - add_service_to_keystone() + add_service_to_keystone(relation_id, remote_unit) synchronize_service_credentials() - + else: + log('Deferring identity_changed() to service leader.') @hooks.hook('cluster-relation-joined') @@ -146,7 +150,7 @@ def cluster_joined(): @restart_on_change(restart_map(), stopstart=True) def cluster_changed(): unison.ssh_authorized_peers(user=SSH_USER, - group='juju_keystone', + group='keystone', peer_interface='cluster', ensure_local_user=True) synchronize_service_credentials() @@ -183,18 +187,15 @@ def ha_joined(): @hooks.hook('ha-relation-changed') def ha_changed(): clustered = relation_get('clustered') - if not clustered or clustered in [None, 'None', '']: - log('ha_changed: hacluster subordinate not fully clustered.') - return - if not is_leader(CLUSTER_RES): - log('ha_changed: hacluster complete but we are not leader.') - return - ensure_initial_admin(config) - log('Cluster configured, notifying other services and updating ' - 'keystone endpoint configuration') - for rid in relation_ids('identity-service'): - identity_joined(rid=rid) - CONFIGS.write_all() + if (clustered is not None and + is_leader(CLUSTER_RES)): + ensure_initial_admin(config) + log('Cluster configured, notifying other services and updating ' + 'keystone endpoint configuration') + for rid in relation_ids('identity-service'): + relation_set(rid=rid, + auth_host=config('vip'), + service_host=config('vip')) def configure_https(): @@ -212,16 +213,16 @@ def configure_https(): cmd = ['a2dissite', 'openstack_https_frontend'] check_call(cmd) - for rid in relation_ids('identity-service'): - identity_joined(rid=rid) - @hooks.hook('upgrade-charm') +@restart_on_change(restart_map(), stopstart=True) def upgrade_charm(): - if openstack_upgrade_available('keystone'): - do_openstack_upgrade(configs=CONFIGS) - save_script_rc() - configure_https() + apt_install(filter_installed_packages(determine_packages())) + cluster_changed() + if eligible_leader(CLUSTER_RES): + log('Cluster leader - ensuring endpoint configuration' + ' is up to date') + ensure_initial_admin(config) CONFIGS.write_all() diff --git a/hooks/keystone_utils.py b/hooks/keystone_utils.py index 220c3087..b77372d7 100755 --- a/hooks/keystone_utils.py +++ b/hooks/keystone_utils.py @@ -2,6 +2,7 @@ import subprocess import os import urlparse +import time from base64 import b64encode from collections import OrderedDict @@ -39,6 +40,11 @@ from charmhelpers.fetch import ( apt_update, ) +from charmhelpers.core.host import ( + service_stop, + service_start, +) + import keystone_context import keystone_ssl as ssl @@ -155,6 +161,7 @@ valid_services = { } } + def resource_map(): ''' Dynamically generate a map of resources that will be managed for a single @@ -233,11 +240,15 @@ def do_openstack_upgrade(configs): if eligible_leader(CLUSTER_RES): migrate_database() + def migrate_database(): '''Runs keystone-manage to initialize a new database or migrate existing''' log('Migrating the keystone database.', level=INFO) + service_stop('keystone') cmd = ['keystone-manage', 'db_sync'] subprocess.check_output(cmd) + service_start('keystone') + time.sleep(10) ## OLD @@ -578,8 +589,9 @@ def relation_list(rid): else: return result -def add_service_to_keystone(): - settings = relation_get() + +def add_service_to_keystone(relation_id=None, remote_unit=None): + settings = relation_get(rid=relation_id, unit=remote_unit) # the minimum settings needed per endpoint single = set(['service', 'region', 'public_url', 'admin_url', 'internal_url']) @@ -610,7 +622,8 @@ def add_service_to_keystone(): for role in get_requested_roles(settings): log("Creating requested role: %s" % role) create_role(role) - relation_set(**relation_data) + relation_set(relation_id=relation_id, + **relation_data) return else: ensure_valid_service(settings['service']) @@ -724,7 +737,8 @@ def add_service_to_keystone(): if is_clustered(): unison.sync_to_peers(peer_interface='cluster', paths=[SSL_DIR], user=SSH_USER, verbose=True) - relation_set(**relation_data) + relation_set(relation_id=relation_id, + **relation_data) def ensure_valid_service(service): From 0f8869029d074aa1b21bb132ae076c1e825f2917 Mon Sep 17 00:00:00 2001 From: James Page Date: Wed, 26 Feb 2014 17:05:40 +0000 Subject: [PATCH 02/25] Restart if need be --- hooks/keystone_hooks.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/hooks/keystone_hooks.py b/hooks/keystone_hooks.py index 8fcee361..00c07977 100755 --- a/hooks/keystone_hooks.py +++ b/hooks/keystone_hooks.py @@ -23,8 +23,6 @@ from charmhelpers.core.host import ( mkdir, restart_on_change, service_restart, - service_stop, - service_start ) from charmhelpers.fetch import ( @@ -74,6 +72,7 @@ def install(): @hooks.hook('config-changed') +@restart_on_change(restart_map()) def config_changed(): unison.ensure_user(user=SSH_USER, group='keystone') homedir = unison.get_homedir(SSH_USER) From 327ea12707c3935bd32baa539a5f21bdb995c5c5 Mon Sep 17 00:00:00 2001 From: James Page Date: Wed, 26 Feb 2014 17:10:31 +0000 Subject: [PATCH 03/25] stop using old hostname config --- hooks/keystone_utils.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/hooks/keystone_utils.py b/hooks/keystone_utils.py index b77372d7..54445f43 100755 --- a/hooks/keystone_utils.py +++ b/hooks/keystone_utils.py @@ -606,8 +606,8 @@ def add_service_to_keystone(relation_id=None, remote_unit=None): relation_data["auth_host"] = config('vip') relation_data["service_host"] = config('vip') else: - relation_data["auth_host"] = config('hostname') - relation_data["service_host"] = config('hostname') + relation_data["auth_host"] = unit_private_ip() + relation_data["service_host"] = unit_private_ip() relation_data["auth_port"] = config('admin-port') relation_data["service_port"] = config('service-port') if config('https-service-endpoints') in ['True', 'true']: From c32b14764ec7d8f1cbc49914fc9227ffde5d8f6e Mon Sep 17 00:00:00 2001 From: James Page Date: Wed, 26 Feb 2014 17:17:13 +0000 Subject: [PATCH 04/25] Drop more use of hostname config --- hooks/keystone_utils.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/hooks/keystone_utils.py b/hooks/keystone_utils.py index 54445f43..db5c8219 100755 --- a/hooks/keystone_utils.py +++ b/hooks/keystone_utils.py @@ -707,9 +707,9 @@ def add_service_to_keystone(relation_id=None, remote_unit=None): # service credentials relation_data = { "admin_token": token, - "service_host": config("hostname"), + "service_host": unit_private_ip(), "service_port": config("service-port"), - "auth_host": config("hostname"), + "auth_host": unit_private_ip(), "auth_port": config("admin-port"), "service_username": service_username, "service_password": service_password, From 90e376f3569b779934822d781e5d11c0d4872e0e Mon Sep 17 00:00:00 2001 From: James Page Date: Wed, 26 Feb 2014 17:28:37 +0000 Subject: [PATCH 05/25] Restore logging options --- hooks/keystone_context.py | 3 +++ templates/essex/etc_keystone_keystone.conf | 3 +++ templates/folsom/etc_keystone_keystone.conf | 3 +++ templates/grizzly/etc_keystone_keystone.conf | 3 +++ templates/havana/etc_keystone_keystone.conf | 3 +++ 5 files changed, 15 insertions(+) diff --git a/hooks/keystone_context.py b/hooks/keystone_context.py index dbed890d..3badbe78 100644 --- a/hooks/keystone_context.py +++ b/hooks/keystone_context.py @@ -17,6 +17,7 @@ import os CA_CERT_PATH = '/usr/local/share/ca-certificates/keystone_juju_ca_cert.crt' + class ApacheSSLContext(context.ApacheSSLContext): interfaces = ['https'] @@ -98,6 +99,8 @@ class KeystoneContext(context.OSContextGenerator): ctxt['token'] = set_admin_token() ctxt['admin_port'] = determine_api_port(api_port('keystone-admin')) ctxt['public_port'] = determine_api_port(api_port('keystone-public')) + ctxt['debug'] = config('debug') in ['yes', 'true', 'True'] + ctxt['verbose'] = config('verbose') in ['yes', 'true', 'True'] if config('enable-pki') not in ['false', 'False', 'no', 'No']: ctxt['signing'] = True return ctxt diff --git a/templates/essex/etc_keystone_keystone.conf b/templates/essex/etc_keystone_keystone.conf index 502a2c8d..a5f94780 100644 --- a/templates/essex/etc_keystone_keystone.conf +++ b/templates/essex/etc_keystone_keystone.conf @@ -9,6 +9,9 @@ admin_port = {{ admin_port }} public_port = {{ public_port }} use_syslog = {{ use_syslog }} log_config = /etc/keystone/logging.conf +debug = {{ debug }} +verbose = {{ verbose }} + [sql] {% if database_host -%} connection = mysql://{{ database_user }}:{{ database_password }}@{{ database_host }}/{{ database }} diff --git a/templates/folsom/etc_keystone_keystone.conf b/templates/folsom/etc_keystone_keystone.conf index 3834adde..220b10dc 100644 --- a/templates/folsom/etc_keystone_keystone.conf +++ b/templates/folsom/etc_keystone_keystone.conf @@ -9,6 +9,9 @@ admin_port = {{ admin_port }} public_port = {{ public_port }} use_syslog = {{ use_syslog }} log_config = /etc/keystone/logging.conf +debug = {{ debug }} +verbose = {{ verbose }} + [sql] {% if database_host -%} connection = mysql://{{ database_user }}:{{ database_password }}@{{ database_host }}/{{ database }} diff --git a/templates/grizzly/etc_keystone_keystone.conf b/templates/grizzly/etc_keystone_keystone.conf index ef717452..a5c821be 100644 --- a/templates/grizzly/etc_keystone_keystone.conf +++ b/templates/grizzly/etc_keystone_keystone.conf @@ -9,6 +9,9 @@ admin_port = {{ admin_port }} public_port = {{ public_port }} use_syslog = {{ use_syslog }} log_config = /etc/keystone/logging.conf +debug = {{ debug }} +verbose = {{ verbose }} + [sql] {% if database_host -%} connection = mysql://{{ database_user }}:{{ database_password }}@{{ database_host }}/{{ database }} diff --git a/templates/havana/etc_keystone_keystone.conf b/templates/havana/etc_keystone_keystone.conf index 76ff171d..4c30e04b 100644 --- a/templates/havana/etc_keystone_keystone.conf +++ b/templates/havana/etc_keystone_keystone.conf @@ -9,6 +9,9 @@ admin_port = {{ admin_port }} public_port = {{ public_port }} use_syslog = {{ use_syslog }} log_config = /etc/keystone/logging.conf +debug = {{ debug }} +verbose = {{ verbose }} + [sql] {% if database_host -%} connection = mysql://{{ database_user }}:{{ database_password }}@{{ database_host }}/{{ database }} From 2bc62e8235669db6ddf05d9cf70cae46f98a044d Mon Sep 17 00:00:00 2001 From: James Page Date: Wed, 26 Feb 2014 17:30:34 +0000 Subject: [PATCH 06/25] Add sleep for upgrade hook if leader --- hooks/keystone_hooks.py | 1 + 1 file changed, 1 insertion(+) diff --git a/hooks/keystone_hooks.py b/hooks/keystone_hooks.py index 00c07977..6fe66918 100755 --- a/hooks/keystone_hooks.py +++ b/hooks/keystone_hooks.py @@ -221,6 +221,7 @@ def upgrade_charm(): if eligible_leader(CLUSTER_RES): log('Cluster leader - ensuring endpoint configuration' ' is up to date') + time.sleep(10) ensure_initial_admin(config) CONFIGS.write_all() From d96ad889549f394f86927b1c840c7596629a15a6 Mon Sep 17 00:00:00 2001 From: James Page Date: Wed, 26 Feb 2014 17:57:08 +0000 Subject: [PATCH 07/25] Put temp fix into charmhelpers unison --- hooks/charmhelpers/contrib/unison/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/hooks/charmhelpers/contrib/unison/__init__.py b/hooks/charmhelpers/contrib/unison/__init__.py index c9cd0b10..06956085 100644 --- a/hooks/charmhelpers/contrib/unison/__init__.py +++ b/hooks/charmhelpers/contrib/unison/__init__.py @@ -253,5 +253,5 @@ def sync_to_peer(host, user, paths=[], verbose=False): def sync_to_peers(peer_interface, user, paths=[], verbose=False): '''Sync all hosts to an specific path''' - for host in collect_authed_hosts(): + for host in collect_authed_hosts(peer_interface): sync_to_peer(host, user, paths, verbose) From fe09bc6ef9535cd19e004b79f7acdd75a6cd5002 Mon Sep 17 00:00:00 2001 From: James Page Date: Wed, 26 Feb 2014 18:48:58 +0000 Subject: [PATCH 08/25] Add standard options for ssl_cert and key --- config.yaml | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/config.yaml b/config.yaml index f6011b66..9b37eff0 100644 --- a/config.yaml +++ b/config.yaml @@ -126,3 +126,14 @@ options: default: "no" type: string description: "Use SSL for Keystone itself. Set to 'yes' to enable it." + ssl_cert: + type: string + description: | + SSL certificate to install and use for API ports. Setting this value + and ssl_key will enable reverse proxying, point Keystone's entry in the + Keystone catalog to use https, and override any certficiate and key + issued by Keystone (if it is configured to do so). + ssl_key: + type: string + description: SSL key to use with certificate specified as ssl_cert. + From 82a9739edc6e4aa5f083714ecb3b8a1694deb5dd Mon Sep 17 00:00:00 2001 From: James Page Date: Wed, 26 Feb 2014 18:54:04 +0000 Subject: [PATCH 09/25] Set protocols on identity-service relation --- hooks/keystone_utils.py | 13 ++++++++++++- 1 file changed, 12 insertions(+), 1 deletion(-) diff --git a/hooks/keystone_utils.py b/hooks/keystone_utils.py index db5c8219..01fd68fb 100755 --- a/hooks/keystone_utils.py +++ b/hooks/keystone_utils.py @@ -608,6 +608,12 @@ def add_service_to_keystone(relation_id=None, remote_unit=None): else: relation_data["auth_host"] = unit_private_ip() relation_data["service_host"] = unit_private_ip() + if https(): + relation_data["auth_protocol"] = "https" + relation_data["service_protocol"] = "https" + else: + relation_data["auth_protocol"] = "http" + relation_data["service_protocol"] = "http" relation_data["auth_port"] = config('admin-port') relation_data["service_port"] = config('service-port') if config('https-service-endpoints') in ['True', 'true']: @@ -724,7 +730,12 @@ def add_service_to_keystone(relation_id=None, remote_unit=None): if is_clustered(): relation_data["auth_host"] = config('vip') relation_data["service_host"] = config('vip') - + if https(): + relation_data["auth_protocol"] = "https" + relation_data["service_protocol"] = "https" + else: + relation_data["auth_protocol"] = "http" + relation_data["service_protocol"] = "http" # generate or get a new cert/key for service if set to manage certs. if config('https-service-endpoints') in ['True', 'true']: ca = get_ca(user=SSH_USER) From 05f2008e9527ae09c52633071ab5ed410252f8b9 Mon Sep 17 00:00:00 2001 From: James Page Date: Thu, 27 Feb 2014 10:24:33 +0000 Subject: [PATCH 10/25] Resync with ssl-everywhere charm-helpers, add db ssl data to connection strings --- charm-helpers.yaml | 2 +- .../charmhelpers/contrib/hahelpers/apache.py | 17 ++-- .../charmhelpers/contrib/openstack/context.py | 86 ++++++++++++++----- hooks/charmhelpers/contrib/unison/__init__.py | 2 +- templates/essex/etc_keystone_keystone.conf | 2 +- templates/folsom/etc_keystone_keystone.conf | 2 +- templates/grizzly/etc_keystone_keystone.conf | 2 +- templates/havana/etc_keystone_keystone.conf | 2 +- 8 files changed, 80 insertions(+), 35 deletions(-) diff --git a/charm-helpers.yaml b/charm-helpers.yaml index 2ba71da0..9bb0ecc1 100644 --- a/charm-helpers.yaml +++ b/charm-helpers.yaml @@ -1,4 +1,4 @@ -branch: lp:charm-helpers +branch: lp:~openstack-charmers/charm-helpers/ssl-everywhere destination: hooks/charmhelpers include: - core diff --git a/hooks/charmhelpers/contrib/hahelpers/apache.py b/hooks/charmhelpers/contrib/hahelpers/apache.py index 3208a85c..8d5fb8ba 100644 --- a/hooks/charmhelpers/contrib/hahelpers/apache.py +++ b/hooks/charmhelpers/contrib/hahelpers/apache.py @@ -39,14 +39,15 @@ def get_cert(): def get_ca_cert(): - ca_cert = None - log("Inspecting identity-service relations for CA SSL certificate.", - level=INFO) - for r_id in relation_ids('identity-service'): - for unit in relation_list(r_id): - if not ca_cert: - ca_cert = relation_get('ca_cert', - rid=r_id, unit=unit) + ca_cert = config_get('ssl_ca') + if ca_cert is None: + log("Inspecting identity-service relations for CA SSL certificate.", + level=INFO) + for r_id in relation_ids('identity-service'): + for unit in relation_list(r_id): + if ca_cert is None: + ca_cert = relation_get('ca_cert', + rid=r_id, unit=unit) return ca_cert diff --git a/hooks/charmhelpers/contrib/openstack/context.py b/hooks/charmhelpers/contrib/openstack/context.py index 5785b6a1..11ddc0b0 100644 --- a/hooks/charmhelpers/contrib/openstack/context.py +++ b/hooks/charmhelpers/contrib/openstack/context.py @@ -1,5 +1,6 @@ import json import os +import time from base64 import b64decode @@ -113,7 +114,8 @@ class OSContextGenerator(object): class SharedDBContext(OSContextGenerator): interfaces = ['shared-db'] - def __init__(self, database=None, user=None, relation_prefix=None): + def __init__(self, + database=None, user=None, relation_prefix=None, ssl_dir=None): ''' Allows inspecting relation for settings prefixed with relation_prefix. This is useful for parsing access for multiple databases returned via @@ -122,6 +124,7 @@ class SharedDBContext(OSContextGenerator): self.relation_prefix = relation_prefix self.database = database self.user = user + self.ssl_dir = ssl_dir def __call__(self): self.database = self.database or config('database') @@ -139,19 +142,44 @@ class SharedDBContext(OSContextGenerator): for rid in relation_ids('shared-db'): for unit in related_units(rid): - passwd = relation_get(password_setting, rid=rid, unit=unit) + rdata = relation_get(rid=rid, unit=unit) ctxt = { - 'database_host': relation_get('db_host', rid=rid, - unit=unit), + 'database_host': rdata.get('db_host'), 'database': self.database, 'database_user': self.user, - 'database_password': passwd, + 'database_password': rdata.get(password_setting) } if context_complete(ctxt): + db_ssl(rdata, ctxt, self.ssl_dir) return ctxt return {} +def db_ssl(rdata, ctxt, ssl_dir): + if 'ssl_ca' in rdata and ssl_dir: + ca_path = os.path.join(ssl_dir, 'db-client.ca') + with open(ca_path, 'w') as fh: + fh.write(b64decode(rdata['ssl_ca'])) + ctxt['database_ssl_ca'] = ca_path + elif 'ssl_ca' in rdata: + log("Charm not setup for ssl support but ssl ca found") + return ctxt + if 'ssl_cert' in rdata: + cert_path = os.path.join( + ssl_dir, 'db-client.cert') + if not os.path.exists(cert_path): + log("Waiting 1m for ssl client cert validity") + time.sleep(60) + with open(cert_path, 'w') as fh: + fh.write(b64decode(rdata['ssl_cert'])) + ctxt['database_ssl_cert'] = cert_path + key_path = os.path.join(ssl_dir, 'db-client.key') + with open(key_path, 'w') as fh: + fh.write(b64decode(rdata['ssl_key'])) + ctxt['database_ssl_key'] = key_path + return ctxt + + class IdentityServiceContext(OSContextGenerator): interfaces = ['identity-service'] @@ -161,22 +189,19 @@ class IdentityServiceContext(OSContextGenerator): for rid in relation_ids('identity-service'): for unit in related_units(rid): + rdata = relation_get(rid=rid, unit=unit) ctxt = { - 'service_port': relation_get('service_port', rid=rid, - unit=unit), - 'service_host': relation_get('service_host', rid=rid, - unit=unit), - 'auth_host': relation_get('auth_host', rid=rid, unit=unit), - 'auth_port': relation_get('auth_port', rid=rid, unit=unit), - 'admin_tenant_name': relation_get('service_tenant', - rid=rid, unit=unit), - 'admin_user': relation_get('service_username', rid=rid, - unit=unit), - 'admin_password': relation_get('service_password', rid=rid, - unit=unit), - # XXX: Hard-coded http. - 'service_protocol': 'http', - 'auth_protocol': 'http', + 'service_port': rdata.get('service_port'), + 'service_host': rdata.get('service_host'), + 'auth_host': rdata.get('auth_host'), + 'auth_port': rdata.get('auth_port'), + 'admin_tenant_name': rdata.get('service_tenant'), + 'admin_user': rdata.get('service_username'), + 'admin_password': rdata.get('service_password'), + 'service_protocol': + rdata.get('service_protocol') or 'http', + 'auth_protocol': + rdata.get('auth_protocol') or 'http', } if context_complete(ctxt): return ctxt @@ -186,6 +211,9 @@ class IdentityServiceContext(OSContextGenerator): class AMQPContext(OSContextGenerator): interfaces = ['amqp'] + def __init__(self, ssl_dir=None): + self.ssl_dir = ssl_dir + def __call__(self): log('Generating template context for amqp') conf = config() @@ -196,7 +224,6 @@ class AMQPContext(OSContextGenerator): log('Could not generate shared_db context. ' 'Missing required charm config options: %s.' % e) raise OSContextError - ctxt = {} for rid in relation_ids('amqp'): for unit in related_units(rid): @@ -213,7 +240,24 @@ class AMQPContext(OSContextGenerator): unit=unit), 'rabbitmq_virtual_host': vhost, }) + ssl_port = relation_get('ssl_port', rid=rid, unit=unit) + if ssl_port: + ctxt['rabbit_ssl_port'] = ssl_port + ssl_ca = relation_get('ssl_ca', rid=rid, unit=unit) + if ssl_ca: + ctxt['rabbit_ssl_ca'] = ssl_ca + if context_complete(ctxt): + if 'rabbit_ssl_ca' in ctxt: + if not self.ssl_dir: + log(("Charm not setup for ssl support " + "but ssl ca found")) + break + ca_path = os.path.join( + self.ssl_dir, 'rabbit-client-ca.pem') + with open(ca_path, 'w') as fh: + fh.write(b64decode(ctxt['rabbit_ssl_ca'])) + ctxt['rabbit_ssl_ca'] = ca_path # Sufficient information found = break out! break # Used for active/active rabbitmq >= grizzly diff --git a/hooks/charmhelpers/contrib/unison/__init__.py b/hooks/charmhelpers/contrib/unison/__init__.py index 06956085..c9cd0b10 100644 --- a/hooks/charmhelpers/contrib/unison/__init__.py +++ b/hooks/charmhelpers/contrib/unison/__init__.py @@ -253,5 +253,5 @@ def sync_to_peer(host, user, paths=[], verbose=False): def sync_to_peers(peer_interface, user, paths=[], verbose=False): '''Sync all hosts to an specific path''' - for host in collect_authed_hosts(peer_interface): + for host in collect_authed_hosts(): sync_to_peer(host, user, paths, verbose) diff --git a/templates/essex/etc_keystone_keystone.conf b/templates/essex/etc_keystone_keystone.conf index a5f94780..9580f959 100644 --- a/templates/essex/etc_keystone_keystone.conf +++ b/templates/essex/etc_keystone_keystone.conf @@ -14,7 +14,7 @@ verbose = {{ verbose }} [sql] {% if database_host -%} -connection = mysql://{{ database_user }}:{{ database_password }}@{{ database_host }}/{{ database }} +connection = mysql://{{ database_user }}:{{ database_password }}@{{ database_host }}/{{ database }}{% if database_ssl_ca %}?ssl_ca={{ database_ssl_ca }}{% if database_ssl_cert %}&ssl_cert={{ database_ssl_cert }}&ssl_key={{ database_ssl_key }}{% endif %}{% endif %} {% else -%} connection = sqlite:////var/lib/keystone/keystone.db {% endif -%} diff --git a/templates/folsom/etc_keystone_keystone.conf b/templates/folsom/etc_keystone_keystone.conf index 220b10dc..8d1c560c 100644 --- a/templates/folsom/etc_keystone_keystone.conf +++ b/templates/folsom/etc_keystone_keystone.conf @@ -14,7 +14,7 @@ verbose = {{ verbose }} [sql] {% if database_host -%} -connection = mysql://{{ database_user }}:{{ database_password }}@{{ database_host }}/{{ database }} +connection = mysql://{{ database_user }}:{{ database_password }}@{{ database_host }}/{{ database }}{% if database_ssl_ca %}?ssl_ca={{ database_ssl_ca }}{% if database_ssl_cert %}&ssl_cert={{ database_ssl_cert }}&ssl_key={{ database_ssl_key }}{% endif %}{% endif %} {% else -%} connection = sqlite:////var/lib/keystone/keystone.db {% endif -%} diff --git a/templates/grizzly/etc_keystone_keystone.conf b/templates/grizzly/etc_keystone_keystone.conf index a5c821be..0ffb2bfa 100644 --- a/templates/grizzly/etc_keystone_keystone.conf +++ b/templates/grizzly/etc_keystone_keystone.conf @@ -14,7 +14,7 @@ verbose = {{ verbose }} [sql] {% if database_host -%} -connection = mysql://{{ database_user }}:{{ database_password }}@{{ database_host }}/{{ database }} +connection = mysql://{{ database_user }}:{{ database_password }}@{{ database_host }}/{{ database }}{% if database_ssl_ca %}?ssl_ca={{ database_ssl_ca }}{% if database_ssl_cert %}&ssl_cert={{ database_ssl_cert }}&ssl_key={{ database_ssl_key }}{% endif %}{% endif %} {% else -%} connection = sqlite:////var/lib/keystone/keystone.db {% endif -%} diff --git a/templates/havana/etc_keystone_keystone.conf b/templates/havana/etc_keystone_keystone.conf index 4c30e04b..ca28d9b0 100644 --- a/templates/havana/etc_keystone_keystone.conf +++ b/templates/havana/etc_keystone_keystone.conf @@ -14,7 +14,7 @@ verbose = {{ verbose }} [sql] {% if database_host -%} -connection = mysql://{{ database_user }}:{{ database_password }}@{{ database_host }}/{{ database }} +connection = mysql://{{ database_user }}:{{ database_password }}@{{ database_host }}/{{ database }}{% if database_ssl_ca %}?ssl_ca={{ database_ssl_ca }}{% if database_ssl_cert %}&ssl_cert={{ database_ssl_cert }}&ssl_key={{ database_ssl_key }}{% endif %}{% endif %} {% else -%} connection = sqlite:////var/lib/keystone/keystone.db {% endif -%} From e2e5fb98da029504c41aa6cbf028d8851bcddbc4 Mon Sep 17 00:00:00 2001 From: James Page Date: Thu, 27 Feb 2014 10:34:15 +0000 Subject: [PATCH 11/25] Reorder calls to fixup hook failure --- hooks/keystone_utils.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/hooks/keystone_utils.py b/hooks/keystone_utils.py index 01fd68fb..edebf918 100755 --- a/hooks/keystone_utils.py +++ b/hooks/keystone_utils.py @@ -672,8 +672,8 @@ def add_service_to_keystone(relation_id=None, remote_unit=None): # by ensuring each possible endpiont has appropriate fields # ['service', 'region', 'public_url', 'admin_url', 'internal_url'] if single.issubset(endpoints[ep]): - ensure_valid_service(ep['service']) ep = endpoints[ep] + ensure_valid_service(ep['service']) add_endpoint(region=ep['region'], service=ep['service'], publicurl=ep['public_url'], adminurl=ep['admin_url'], From cbdd573f12dd2ef777362756c7c59a8b91d8cac8 Mon Sep 17 00:00:00 2001 From: James Page Date: Thu, 27 Feb 2014 10:41:06 +0000 Subject: [PATCH 12/25] Temp patch for charm-helpers to switch use-https to boolean option --- config.yaml | 4 ++-- hooks/charmhelpers/contrib/hahelpers/cluster.py | 3 ++- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/config.yaml b/config.yaml index 9b37eff0..5874f767 100644 --- a/config.yaml +++ b/config.yaml @@ -123,8 +123,8 @@ options: type: string description: "Manage SSL certificates for all service endpoints." use-https: - default: "no" - type: string + default: False + type: boolean description: "Use SSL for Keystone itself. Set to 'yes' to enable it." ssl_cert: type: string diff --git a/hooks/charmhelpers/contrib/hahelpers/cluster.py b/hooks/charmhelpers/contrib/hahelpers/cluster.py index bf832f7d..ceabcb25 100644 --- a/hooks/charmhelpers/contrib/hahelpers/cluster.py +++ b/hooks/charmhelpers/contrib/hahelpers/cluster.py @@ -91,7 +91,8 @@ def https(): . returns: boolean ''' - if config_get('use-https') == "yes": + if (config_get('use-https') is True or + config_get('use-https') in ["yes", 'Yes', 'True', 'true']): return True if config_get('ssl_cert') and config_get('ssl_key'): return True From e0f2b9017f487fc821bb852c0f90e110996ae04e Mon Sep 17 00:00:00 2001 From: James Page Date: Thu, 27 Feb 2014 10:47:40 +0000 Subject: [PATCH 13/25] Add ssl_ca config option --- config.yaml | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/config.yaml b/config.yaml index 5874f767..2afb17e2 100644 --- a/config.yaml +++ b/config.yaml @@ -136,4 +136,8 @@ options: ssl_key: type: string description: SSL key to use with certificate specified as ssl_cert. - + ssl_ca: + type: string + description: | + SSL CA to use with the certificate and key provided - this is only + required if you are providing a privately signed ssl_cert and ssl_key. From f632a29cea9697b2194f50073f04c51e95143246 Mon Sep 17 00:00:00 2001 From: James Page Date: Thu, 27 Feb 2014 10:55:38 +0000 Subject: [PATCH 14/25] Fixup SSL mysql handling --- hooks/keystone_utils.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/hooks/keystone_utils.py b/hooks/keystone_utils.py index edebf918..ab988a32 100755 --- a/hooks/keystone_utils.py +++ b/hooks/keystone_utils.py @@ -72,6 +72,7 @@ API_PORTS = { } KEYSTONE_CONF = "/etc/keystone/keystone.conf" +KEYSTONE_CONF_DIR = os.path.dirname(KEYSTONE_CONF) STORED_PASSWD = "/var/lib/keystone/keystone.passwd" STORED_TOKEN = "/var/lib/keystone/keystone.token" SERVICE_PASSWD_PATH = '/var/lib/keystone/services.passwd' @@ -89,7 +90,7 @@ BASE_RESOURCE_MAP = OrderedDict([ (KEYSTONE_CONF, { 'services': BASE_SERVICES, 'contexts': [keystone_context.KeystoneContext(), - context.SharedDBContext(), + context.SharedDBContext(ssl_dir=KEYSTONE_CONF_DIR), context.SyslogContext(), keystone_context.HAProxyContext()], }), From 1e1b35b197bfbf0920b1216f33b04e30735a73af Mon Sep 17 00:00:00 2001 From: James Page Date: Thu, 27 Feb 2014 10:56:27 +0000 Subject: [PATCH 15/25] Drop surplus restart --- hooks/keystone_hooks.py | 3 --- 1 file changed, 3 deletions(-) diff --git a/hooks/keystone_hooks.py b/hooks/keystone_hooks.py index 6fe66918..074b1896 100755 --- a/hooks/keystone_hooks.py +++ b/hooks/keystone_hooks.py @@ -87,9 +87,6 @@ def config_changed(): save_script_rc() configure_https() CONFIGS.write_all() - service_restart('keystone') - time.sleep(10) - if eligible_leader(CLUSTER_RES): migrate_database() ensure_initial_admin(config) From a7c8d88dbbd8e52bea4b66340c3e61b79a75e50f Mon Sep 17 00:00:00 2001 From: Ante Karamatic Date: Sat, 1 Mar 2014 18:39:21 +0100 Subject: [PATCH 16/25] Fix unison: https://code.launchpad.net/~ivoks/charm-helpers/fix-unison/+merge/208937 --- hooks/charmhelpers/contrib/unison/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/hooks/charmhelpers/contrib/unison/__init__.py b/hooks/charmhelpers/contrib/unison/__init__.py index c9cd0b10..06956085 100644 --- a/hooks/charmhelpers/contrib/unison/__init__.py +++ b/hooks/charmhelpers/contrib/unison/__init__.py @@ -253,5 +253,5 @@ def sync_to_peer(host, user, paths=[], verbose=False): def sync_to_peers(peer_interface, user, paths=[], verbose=False): '''Sync all hosts to an specific path''' - for host in collect_authed_hosts(): + for host in collect_authed_hosts(peer_interface): sync_to_peer(host, user, paths, verbose) From 38bfc0e54ffdf92588db30ac626e06de3b2fe50d Mon Sep 17 00:00:00 2001 From: Ante Karamatic Date: Sat, 1 Mar 2014 19:09:28 +0100 Subject: [PATCH 17/25] Use tcp instead of http for LB --- templates/haproxy.cfg | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/templates/haproxy.cfg b/templates/haproxy.cfg index fccf1c9d..548766f8 100644 --- a/templates/haproxy.cfg +++ b/templates/haproxy.cfg @@ -8,8 +8,8 @@ global defaults log global - mode http - option httplog + mode tcp + option tcplog option dontlognull retries 3 timeout queue 1000 @@ -19,6 +19,7 @@ defaults listen stats :8888 mode http + option httplog stats enable stats hide-version stats realm Haproxy\ Statistics @@ -29,7 +30,6 @@ listen stats :8888 {% for service, ports in service_ports.iteritems() -%} listen {{ service }} 0.0.0.0:{{ ports[0] }} balance roundrobin - option tcplog {% for unit, address in units.iteritems() -%} server {{ unit }} {{ address }}:{{ ports[1] }} check {% endfor %} From 5772e674674c273a939e618012875be82408f3b3 Mon Sep 17 00:00:00 2001 From: Ante Karamatic Date: Sun, 2 Mar 2014 09:56:04 +0100 Subject: [PATCH 18/25] Use VIP for Apache on ha-changed relation --- hooks/charmhelpers/contrib/openstack/context.py | 3 +++ hooks/keystone_hooks.py | 1 + revision | 2 +- 3 files changed, 5 insertions(+), 1 deletion(-) diff --git a/hooks/charmhelpers/contrib/openstack/context.py b/hooks/charmhelpers/contrib/openstack/context.py index 11ddc0b0..b0233445 100644 --- a/hooks/charmhelpers/contrib/openstack/context.py +++ b/hooks/charmhelpers/contrib/openstack/context.py @@ -432,6 +432,9 @@ class ApacheSSLContext(OSContextGenerator): 'private_address': unit_get('private-address'), 'endpoints': [] } + vip = config('vip') + if is_clustered() and vip: + ctxt['private_address'] = vip for api_port in self.external_ports: ext_port = determine_apache_port(api_port) int_port = determine_api_port(api_port) diff --git a/hooks/keystone_hooks.py b/hooks/keystone_hooks.py index 074b1896..f84010c0 100755 --- a/hooks/keystone_hooks.py +++ b/hooks/keystone_hooks.py @@ -186,6 +186,7 @@ def ha_changed(): if (clustered is not None and is_leader(CLUSTER_RES)): ensure_initial_admin(config) + CONFIGS.write_all() log('Cluster configured, notifying other services and updating ' 'keystone endpoint configuration') for rid in relation_ids('identity-service'): diff --git a/revision b/revision index dcb6b5ba..71d936fd 100644 --- a/revision +++ b/revision @@ -1 +1 @@ -230 +231 From 09ddcbc3135099f266735649b1e7f2a8280dc356 Mon Sep 17 00:00:00 2001 From: Ante Karamatic Date: Sun, 2 Mar 2014 10:16:17 +0100 Subject: [PATCH 19/25] Redux --- hooks/charmhelpers/contrib/openstack/context.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/hooks/charmhelpers/contrib/openstack/context.py b/hooks/charmhelpers/contrib/openstack/context.py index b0233445..249e896c 100644 --- a/hooks/charmhelpers/contrib/openstack/context.py +++ b/hooks/charmhelpers/contrib/openstack/context.py @@ -432,9 +432,8 @@ class ApacheSSLContext(OSContextGenerator): 'private_address': unit_get('private-address'), 'endpoints': [] } - vip = config('vip') - if is_clustered() and vip: - ctxt['private_address'] = vip + if is_clustered(): + ctxt['private_address'] = config('vip') for api_port in self.external_ports: ext_port = determine_apache_port(api_port) int_port = determine_api_port(api_port) From f258309df213c83e4ab95e678f78e709b77c5126 Mon Sep 17 00:00:00 2001 From: James Page Date: Mon, 3 Mar 2014 09:09:17 +0000 Subject: [PATCH 20/25] Resync helpers --- hooks/charmhelpers/contrib/hahelpers/cluster.py | 3 +-- hooks/charmhelpers/contrib/openstack/context.py | 2 +- hooks/charmhelpers/contrib/openstack/neutron.py | 17 +++++++++++++---- 3 files changed, 15 insertions(+), 7 deletions(-) diff --git a/hooks/charmhelpers/contrib/hahelpers/cluster.py b/hooks/charmhelpers/contrib/hahelpers/cluster.py index ceabcb25..bf832f7d 100644 --- a/hooks/charmhelpers/contrib/hahelpers/cluster.py +++ b/hooks/charmhelpers/contrib/hahelpers/cluster.py @@ -91,8 +91,7 @@ def https(): . returns: boolean ''' - if (config_get('use-https') is True or - config_get('use-https') in ["yes", 'Yes', 'True', 'true']): + if config_get('use-https') == "yes": return True if config_get('ssl_cert') and config_get('ssl_key'): return True diff --git a/hooks/charmhelpers/contrib/openstack/context.py b/hooks/charmhelpers/contrib/openstack/context.py index 249e896c..2403ce16 100644 --- a/hooks/charmhelpers/contrib/openstack/context.py +++ b/hooks/charmhelpers/contrib/openstack/context.py @@ -512,7 +512,7 @@ class NeutronContext(OSContextGenerator): host = config('vip') else: host = unit_get('private-address') - url = '%s://%s:%s' % (proto, host, '9292') + url = '%s://%s:%s' % (proto, host, '9696') ctxt = { 'network_manager': self.network_manager, 'neutron_url': url, diff --git a/hooks/charmhelpers/contrib/openstack/neutron.py b/hooks/charmhelpers/contrib/openstack/neutron.py index 8d32bd00..a11c08b1 100644 --- a/hooks/charmhelpers/contrib/openstack/neutron.py +++ b/hooks/charmhelpers/contrib/openstack/neutron.py @@ -17,8 +17,11 @@ def headers_package(): kver = check_output(['uname', '-r']).strip() return 'linux-headers-%s' % kver +QUANTUM_CONF_DIR = '/etc/quantum' # legacy + + def quantum_plugins(): from charmhelpers.contrib.openstack import context return { @@ -30,7 +33,8 @@ def quantum_plugins(): 'contexts': [ context.SharedDBContext(user=config('neutron-database-user'), database=config('neutron-database'), - relation_prefix='neutron')], + relation_prefix='neutron', + ssl_dir=QUANTUM_CONF_DIR)], 'services': ['quantum-plugin-openvswitch-agent'], 'packages': [[headers_package(), 'openvswitch-datapath-dkms'], ['quantum-plugin-openvswitch-agent']], @@ -45,7 +49,8 @@ def quantum_plugins(): 'contexts': [ context.SharedDBContext(user=config('neutron-database-user'), database=config('neutron-database'), - relation_prefix='neutron')], + relation_prefix='neutron', + ssl_dir=QUANTUM_CONF_DIR)], 'services': [], 'packages': [], 'server_packages': ['quantum-server', @@ -54,6 +59,8 @@ def quantum_plugins(): } } +NEUTRON_CONF_DIR = '/etc/neutron' + def neutron_plugins(): from charmhelpers.contrib.openstack import context @@ -66,7 +73,8 @@ def neutron_plugins(): 'contexts': [ context.SharedDBContext(user=config('neutron-database-user'), database=config('neutron-database'), - relation_prefix='neutron')], + relation_prefix='neutron', + ssl_dir=NEUTRON_CONF_DIR)], 'services': ['neutron-plugin-openvswitch-agent'], 'packages': [[headers_package(), 'openvswitch-datapath-dkms'], ['quantum-plugin-openvswitch-agent']], @@ -81,7 +89,8 @@ def neutron_plugins(): 'contexts': [ context.SharedDBContext(user=config('neutron-database-user'), database=config('neutron-database'), - relation_prefix='neutron')], + relation_prefix='neutron', + ssl_dir=NEUTRON_CONF_DIR)], 'services': [], 'packages': [], 'server_packages': ['neutron-server', From 90888b24777aa5dadcdc3b9055d80447b1c02118 Mon Sep 17 00:00:00 2001 From: James Page Date: Mon, 3 Mar 2014 09:13:00 +0000 Subject: [PATCH 21/25] Restart on change and write_all configs for all ha_changed hooked executions --- hooks/keystone_hooks.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/hooks/keystone_hooks.py b/hooks/keystone_hooks.py index f84010c0..1599e36d 100755 --- a/hooks/keystone_hooks.py +++ b/hooks/keystone_hooks.py @@ -181,12 +181,13 @@ def ha_joined(): @hooks.hook('ha-relation-changed') +@restart_on_change(restart_map()) def ha_changed(): clustered = relation_get('clustered') + CONFIGS.write_all() if (clustered is not None and is_leader(CLUSTER_RES)): ensure_initial_admin(config) - CONFIGS.write_all() log('Cluster configured, notifying other services and updating ' 'keystone endpoint configuration') for rid in relation_ids('identity-service'): From 9fc294e196c69a726457fbf9fe791968b8000e5a Mon Sep 17 00:00:00 2001 From: James Page Date: Mon, 3 Mar 2014 09:14:09 +0000 Subject: [PATCH 22/25] Pep8 formatting --- hooks/keystone_hooks.py | 5 ++--- hooks/keystone_ssl.py | 7 ++++--- hooks/keystone_utils.py | 34 +++++++++++++++++----------------- hooks/manager.py | 1 + 4 files changed, 24 insertions(+), 23 deletions(-) diff --git a/hooks/keystone_hooks.py b/hooks/keystone_hooks.py index 1599e36d..2ab440c1 100755 --- a/hooks/keystone_hooks.py +++ b/hooks/keystone_hooks.py @@ -22,7 +22,6 @@ from charmhelpers.core.hookenv import ( from charmhelpers.core.host import ( mkdir, restart_on_change, - service_restart, ) from charmhelpers.fetch import ( @@ -77,7 +76,7 @@ def config_changed(): unison.ensure_user(user=SSH_USER, group='keystone') homedir = unison.get_homedir(SSH_USER) if not os.path.isdir(homedir): - mkdir(homedir, SSH_USER, 'keystone', 0775) + mkdir(homedir, SSH_USER, 'keystone', 0o775) if openstack_upgrade_available('keystone'): do_openstack_upgrade(configs=CONFIGS) @@ -186,7 +185,7 @@ def ha_changed(): clustered = relation_get('clustered') CONFIGS.write_all() if (clustered is not None and - is_leader(CLUSTER_RES)): + is_leader(CLUSTER_RES)): ensure_initial_admin(config) log('Cluster configured, notifying other services and updating ' 'keystone endpoint configuration') diff --git a/hooks/keystone_ssl.py b/hooks/keystone_ssl.py index b39b227e..45e0029d 100755 --- a/hooks/keystone_ssl.py +++ b/hooks/keystone_ssl.py @@ -113,7 +113,7 @@ def init_ca(ca_dir, common_name, org_name=ORG_NAME, org_unit_name=ORG_UNIT): if not os.path.exists(d): print 'Creating %s.' % d os.mkdir(d) - os.chmod(os.path.join(ca_dir, 'private'), 0710) + os.chmod(os.path.join(ca_dir, 'private'), 0o710) if not os.path.isfile(os.path.join(ca_dir, 'serial')): with open(os.path.join(ca_dir, 'serial'), 'wb') as out: @@ -161,7 +161,7 @@ def intermediate_ca_csr_key(ca_dir): def sign_int_csr(ca_dir, csr, common_name): print 'Signing certificate request %s.' % csr crt = os.path.join(ca_dir, 'certs', - '%s.crt' % os.path.basename(csr).split('.')[0]) + '%s.crt' % os.path.basename(csr).split('.')[0]) subj = '/O=%s/OU=%s/CN=%s' % (ORG_NAME, ORG_UNIT, common_name) cmd = ['openssl', 'ca', '-batch', '-config', os.path.join(ca_dir, 'ca.cnf'), @@ -238,6 +238,7 @@ def tar_directory(path): class JujuCA(object): + def __init__(self, name, ca_dir, root_ca_dir, user, group): root_crt, root_key = init_root_ca(root_ca_dir, '%s Certificate Authority' % name) @@ -288,7 +289,7 @@ class JujuCA(object): key = open(key, 'r').read() except: print 'Could not load ssl private key for %s from %s' %\ - (common_name, key) + (common_name, key) exit(1) return crt, key crt, key = self._create_certificate(common_name, common_name) diff --git a/hooks/keystone_utils.py b/hooks/keystone_utils.py index ab988a32..79eaa6c3 100755 --- a/hooks/keystone_utils.py +++ b/hooks/keystone_utils.py @@ -13,7 +13,7 @@ from charmhelpers.contrib.hahelpers.cluster import( determine_api_port, https, is_clustered - ) +) from charmhelpers.contrib.openstack import context, templating @@ -252,13 +252,13 @@ def migrate_database(): time.sleep(10) -## OLD +# OLD def get_local_endpoint(): """ Returns the URL for the local end-point bypassing haproxy/ssl """ local_endpoint = 'http://localhost:{}/v2.0/'.format( determine_api_port(api_port('keystone-admin')) - ) + ) return local_endpoint @@ -317,7 +317,7 @@ def create_service_entry(service_name, service_type, service_desc, owner=None): log("Created new service entry '%s'" % service_name) -def create_endpoint_template(region, service, publicurl, adminurl, +def create_endpoint_template(region, service, publicurl, adminurl, internalurl): """ Create a new endpoint template for service if one does not already exist matching name *and* region """ @@ -328,7 +328,7 @@ def create_endpoint_template(region, service, publicurl, adminurl, for ep in [e._info for e in manager.api.endpoints.list()]: if ep['service_id'] == service_id and ep['region'] == region: log("Endpoint template already exists for '%s' in '%s'" - % (service, region)) + % (service, region)) up_to_date = True for k in ['publicurl', 'adminurl', 'internalurl']: @@ -405,7 +405,7 @@ def create_role(name, user=None, tenant=None): if None in [user_id, role_id, tenant_id]: error_out("Could not resolve [%s, %s, %s]" % - (user_id, role_id, tenant_id)) + (user_id, role_id, tenant_id)) grant_role(user, name, tenant) @@ -415,7 +415,7 @@ def grant_role(user, role, tenant): import manager manager = manager.KeystoneManager(endpoint=get_local_endpoint(), token=get_admin_token()) - log("Granting user '%s' role '%s' on tenant '%s'" % \ + log("Granting user '%s' role '%s' on tenant '%s'" % (user, role, tenant)) user_id = manager.resolve_user_id(user) role_id = manager.resolve_role_id(role) @@ -426,10 +426,10 @@ def grant_role(user, role, tenant): manager.api.roles.add_user_role(user=user_id, role=role_id, tenant=tenant_id) - log("Granted user '%s' role '%s' on tenant '%s'" % \ + log("Granted user '%s' role '%s' on tenant '%s'" % (user, role, tenant)) else: - log("User '%s' already has role '%s' on tenant '%s'" % \ + log("User '%s' already has role '%s' on tenant '%s'" % (user, role, tenant)) @@ -453,8 +453,8 @@ def ensure_initial_admin(config): log("Loading stored passwd from %s" % STORED_PASSWD) passwd = open(STORED_PASSWD, 'r').readline().strip('\n') if passwd == "": - log("Generating new passwd for user: %s" % \ - config("admin-user")) + log("Generating new passwd for user: %s" % + config("admin-user")) cmd = ['pwgen', '-c', '16', '1'] passwd = str(subprocess.check_output(cmd)).strip() open(STORED_PASSWD, 'w+').writelines("%s\n" % passwd) @@ -506,8 +506,8 @@ def update_user_password(username, password): error_out("Could not resolve user id for '%s'" % username) manager.api.users.update_password(user=user_id, password=password) - log("Successfully updated password for user '%s'" % \ - username) + log("Successfully updated password for user '%s'" % + username) def load_stored_passwords(path=SERVICE_PASSWD_PATH): @@ -545,7 +545,7 @@ def synchronize_service_credentials(): broadcasted by peer, depending on hook context. ''' if (not eligible_leader(CLUSTER_RES) or - not os.path.isfile(SERVICE_PASSWD_PATH)): + not os.path.isfile(SERVICE_PASSWD_PATH)): return log('Synchronizing service passwords to all peers.') if is_clustered(): @@ -569,7 +569,7 @@ def get_ca(user='keystone', group='keystone'): ca_dir=os.path.join(SSL_DIR, '%s_intermediate_ca' % d_name), root_ca_dir=os.path.join(SSL_DIR, - '%s_root_ca' % d_name)) + '%s_root_ca' % d_name)) # SSL_DIR is synchronized via all peers over unison+ssh, need # to ensure permissions. subprocess.check_output(['chown', '-R', '%s.%s' % (user, group), @@ -583,7 +583,7 @@ def relation_list(rid): cmd = [ 'relation-list', '-r', rid, - ] + ] result = str(subprocess.check_output(cmd)).split() if result == "": return None @@ -773,7 +773,7 @@ def add_endpoint(region, service, publicurl, adminurl, internalurl): def get_requested_roles(settings): ''' Retrieve any valid requested_roles from dict settings ''' if ('requested_roles' in settings and - settings['requested_roles'] not in ['None', None]): + settings['requested_roles'] not in ['None', None]): return settings['requested_roles'].split(',') else: return [] diff --git a/hooks/manager.py b/hooks/manager.py index e7058820..8c8968d3 100644 --- a/hooks/manager.py +++ b/hooks/manager.py @@ -3,6 +3,7 @@ from keystoneclient.v2_0 import client class KeystoneManager(object): + def __init__(self, endpoint, token): self.api = client.Client(endpoint=endpoint, token=token) From 45ca9775cd265894e1d5a478a75cf6df391a5a2f Mon Sep 17 00:00:00 2001 From: Ante Karamatic Date: Mon, 3 Mar 2014 20:39:38 +0100 Subject: [PATCH 23/25] Judging by hooks/charmhelpers/contrib/hahelpers/cluster.py, use-https is a string, not a bool. --- config.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/config.yaml b/config.yaml index 2afb17e2..a2ce62ac 100644 --- a/config.yaml +++ b/config.yaml @@ -123,8 +123,8 @@ options: type: string description: "Manage SSL certificates for all service endpoints." use-https: - default: False - type: boolean + default: "no" + type: string description: "Use SSL for Keystone itself. Set to 'yes' to enable it." ssl_cert: type: string From 41e6992ea703ccfdb37ff9f9c63e0e0ff2162c73 Mon Sep 17 00:00:00 2001 From: Kapil Thangavelu Date: Tue, 4 Mar 2014 08:56:18 -0500 Subject: [PATCH 24/25] keystone ssl ca sync whenver we sync passwords --- hooks/keystone_ssl.py | 2 ++ hooks/keystone_utils.py | 6 +++--- 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/hooks/keystone_ssl.py b/hooks/keystone_ssl.py index 45e0029d..1cbdfad7 100755 --- a/hooks/keystone_ssl.py +++ b/hooks/keystone_ssl.py @@ -1,10 +1,12 @@ #!/usr/bin/python +import base64 import os import shutil import subprocess import tarfile import tempfile +import zipfile CA_EXPIRY = '365' ORG_NAME = 'Ubuntu' diff --git a/hooks/keystone_utils.py b/hooks/keystone_utils.py index 79eaa6c3..19cdb4a9 100755 --- a/hooks/keystone_utils.py +++ b/hooks/keystone_utils.py @@ -552,6 +552,9 @@ def synchronize_service_credentials(): unison.sync_to_peers(peer_interface='cluster', paths=[SERVICE_PASSWD_PATH], user=SSH_USER, verbose=True) + if config('http-service-endpoints') in ['True', 'true']: + unison.sync_to_peers(peer_interface='cluster', + paths=[SSL_DIR], user=SSH_USER, verbose=True) CA = [] @@ -746,9 +749,6 @@ def add_service_to_keystone(relation_id=None, remote_unit=None): relation_data['ssl_key'] = b64encode(key) relation_data['ca_cert'] = b64encode(ca_bundle) relation_data['https_keystone'] = 'True' - if is_clustered(): - unison.sync_to_peers(peer_interface='cluster', - paths=[SSL_DIR], user=SSH_USER, verbose=True) relation_set(relation_id=relation_id, **relation_data) From c15ac4415082d849cea3281b7e9334246ef899f4 Mon Sep 17 00:00:00 2001 From: James Page Date: Thu, 27 Mar 2014 09:36:34 +0000 Subject: [PATCH 25/25] Fixup incorrect use of config --- hooks/keystone_utils.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/hooks/keystone_utils.py b/hooks/keystone_utils.py index 19cdb4a9..888c882c 100755 --- a/hooks/keystone_utils.py +++ b/hooks/keystone_utils.py @@ -552,7 +552,7 @@ def synchronize_service_credentials(): unison.sync_to_peers(peer_interface='cluster', paths=[SERVICE_PASSWD_PATH], user=SSH_USER, verbose=True) - if config('http-service-endpoints') in ['True', 'true']: + if config('https-service-endpoints') in ['True', 'true']: unison.sync_to_peers(peer_interface='cluster', paths=[SSL_DIR], user=SSH_USER, verbose=True)