From 70e61d50fd9d42e46f94e232d339c3843f205b10 Mon Sep 17 00:00:00 2001 From: "yolanda.robla@canonical.com" <> Date: Thu, 20 Mar 2014 10:31:25 +0100 Subject: [PATCH 01/19] added database type --- hooks/nova_cc_context.py | 11 +++++++++++ hooks/nova_cc_utils.py | 2 +- templates/essex/nova.conf | 2 +- templates/folsom/nova.conf | 2 +- templates/folsom/ovs_quantum_plugin.ini | 2 +- templates/havana/nvp.ini | 2 +- templates/havana/ovs_neutron_plugin.ini | 2 +- 7 files changed, 17 insertions(+), 6 deletions(-) diff --git a/hooks/nova_cc_context.py b/hooks/nova_cc_context.py index d6cafccc..59ddf4bc 100644 --- a/hooks/nova_cc_context.py +++ b/hooks/nova_cc_context.py @@ -163,3 +163,14 @@ class IdentityServiceContext(context.IdentityServiceContext): ctxt['service_port']) ctxt['keystone_ec2_url'] = ec2_tokens return ctxt + +class SharedDBContext(context.SharedDBContext): + + interfaces = ['shared-db'] + + def __call__(self): + # add database type + ctxt = super(SharedDBContext, self).__call__() + ctxt['database_type'] = 'mysql' + return ctxt + diff --git a/hooks/nova_cc_utils.py b/hooks/nova_cc_utils.py index 9eb15113..412e895e 100644 --- a/hooks/nova_cc_utils.py +++ b/hooks/nova_cc_utils.py @@ -84,7 +84,7 @@ BASE_RESOURCE_MAP = OrderedDict([ (NOVA_CONF, { 'services': BASE_SERVICES, 'contexts': [context.AMQPContext(), - context.SharedDBContext(relation_prefix='nova'), + nova_cc_context.SharedDBContext(relation_prefix='nova'), context.ImageServiceContext(), context.OSConfigFlagContext(), context.SubordinateConfigContext( diff --git a/templates/essex/nova.conf b/templates/essex/nova.conf index d0bb3725..bdb0dacd 100644 --- a/templates/essex/nova.conf +++ b/templates/essex/nova.conf @@ -16,7 +16,7 @@ --verbose --ec2_private_dns_show_ip {% if database_host -%} ---sql_connection=mysql://{{ database_user }}:{{ database_password }}@{{ database_host }}/{{ database }} +--sql_connection={{ database_type }}://{{ database_user }}:{{ database_password }}@{{ database_host }}/{{ database }} {% endif -%} {% if rabbitmq_host -%} --rabbit_host={{ rabbitmq_host }} diff --git a/templates/folsom/nova.conf b/templates/folsom/nova.conf index 7c620918..f41408eb 100644 --- a/templates/folsom/nova.conf +++ b/templates/folsom/nova.conf @@ -25,7 +25,7 @@ keystone_ec2_url = {{ keystone_ec2_url }} {% endif -%} {% if database_host -%} -sql_connection = mysql://{{ database_user }}:{{ database_password }}@{{ database_host }}/{{ database }} +sql_connection = {{ database_type }}://{{ database_user }}:{{ database_password }}@{{ database_host }}/{{ database }} {% endif -%} {% if rabbitmq_host -%} diff --git a/templates/folsom/ovs_quantum_plugin.ini b/templates/folsom/ovs_quantum_plugin.ini index a80cca3a..7c1c39c6 100644 --- a/templates/folsom/ovs_quantum_plugin.ini +++ b/templates/folsom/ovs_quantum_plugin.ini @@ -11,7 +11,7 @@ local_ip = {{ local_ip }} [DATABASE] {% if database_host -%} -sql_connection = mysql://{{ database_user }}:{{ database_password }}@{{ database_host }}/{{ database }}?quantum?charset=utf8 +sql_connection = {{ database_type }}://{{ database_user }}:{{ database_password }}@{{ database_host }}/{{ database }}?quantum?charset=utf8 reconnect_interval = 2 {% else -%} connection = sqlite:////var/lib/quantum/quantum.sqlite diff --git a/templates/havana/nvp.ini b/templates/havana/nvp.ini index a6225421..2876cc1d 100644 --- a/templates/havana/nvp.ini +++ b/templates/havana/nvp.ini @@ -12,7 +12,7 @@ default_l3_gw_service_uuid = {{ nvp_l3_uuid }} [database] {% if database_host -%} -connection = mysql://{{ database_user }}:{{ database_password }}@{{ database_host }}/{{ database }} +connection = {{ database_type }}://{{ database_user }}:{{ database_password }}@{{ database_host }}/{{ database }} {% else -%} connection = sqlite:////var/lib/neutron/neutron.sqlite {% endif -%} diff --git a/templates/havana/ovs_neutron_plugin.ini b/templates/havana/ovs_neutron_plugin.ini index 73e5a205..ca12b082 100644 --- a/templates/havana/ovs_neutron_plugin.ini +++ b/templates/havana/ovs_neutron_plugin.ini @@ -6,7 +6,7 @@ local_ip = {{ local_ip }} [database] {% if database_host -%} -connection = mysql://{{ database_user }}:{{ database_password }}@{{ database_host }}/{{ database }} +connection = {{ database_type }}://{{ database_user }}:{{ database_password }}@{{ database_host }}/{{ database }} {% else -%} connection = sqlite:////var/lib/neutron/neutron.sqlite {% endif -%} From 499930bcd8bf6e4b2af8952b479cf3fd71643357 Mon Sep 17 00:00:00 2001 From: "yolanda.robla@canonical.com" <> Date: Thu, 20 Mar 2014 11:18:23 +0100 Subject: [PATCH 02/19] added postgresql support --- .../charmhelpers/contrib/openstack/context.py | 29 ++++++++++ .../contrib/storage/linux/utils.py | 3 +- hooks/charmhelpers/fetch/__init__.py | 4 ++ hooks/nova_cc_context.py | 6 ++ hooks/nova_cc_hooks.py | 57 ++++++++++++++++++- hooks/nova_cc_utils.py | 2 + hooks/pgsql-neutron-db-relation-broken | 1 + hooks/pgsql-neutron-db-relation-changed | 1 + hooks/pgsql-neutron-db-relation-joined | 1 + hooks/pgsql-nova-db-relation-broken | 1 + hooks/pgsql-nova-db-relation-changed | 1 + hooks/pgsql-nova-db-relation-joined | 1 + metadata.yaml | 4 ++ revision | 2 +- 14 files changed, 110 insertions(+), 3 deletions(-) create mode 120000 hooks/pgsql-neutron-db-relation-broken create mode 120000 hooks/pgsql-neutron-db-relation-changed create mode 120000 hooks/pgsql-neutron-db-relation-joined create mode 120000 hooks/pgsql-nova-db-relation-broken create mode 120000 hooks/pgsql-nova-db-relation-changed create mode 120000 hooks/pgsql-nova-db-relation-joined diff --git a/hooks/charmhelpers/contrib/openstack/context.py b/hooks/charmhelpers/contrib/openstack/context.py index a2d4636b..6cc3c971 100644 --- a/hooks/charmhelpers/contrib/openstack/context.py +++ b/hooks/charmhelpers/contrib/openstack/context.py @@ -152,6 +152,35 @@ class SharedDBContext(OSContextGenerator): return {} +class PostgresqlDBContext(OSContextGenerator): + interfaces = ['pgsql-db'] + + def __init__(self, database=None): + self.database = database + + def __call__(self): + self.database = self.database or config('database') + if self.database is None: + log('Could not generate shared_db context. ' + 'Missing required charm config options. ' + '(database name)') + raise OSContextError + ctxt = {} + + for rid in relation_ids(self.interfaces[0]): + for unit in related_units(rid): + ctxt = { + 'database_host': relation_get('host', rid=rid, unit=unit), + 'database': self.database, + 'database_user': relation_get('user', rid=rid, unit=unit), + 'database_password': relation_get('password', rid=rid, unit=unit), + 'database_type': 'postgresql', + } + if context_complete(ctxt): + return ctxt + return {} + + class IdentityServiceContext(OSContextGenerator): interfaces = ['identity-service'] diff --git a/hooks/charmhelpers/contrib/storage/linux/utils.py b/hooks/charmhelpers/contrib/storage/linux/utils.py index c40218f0..5349c3ea 100644 --- a/hooks/charmhelpers/contrib/storage/linux/utils.py +++ b/hooks/charmhelpers/contrib/storage/linux/utils.py @@ -22,4 +22,5 @@ def zap_disk(block_device): :param block_device: str: Full path of block device to clean. ''' - check_call(['sgdisk', '--zap-all', '--mbrtogpt', block_device]) + check_call(['sgdisk', '--zap-all', '--clear', + '--mbrtogpt', block_device]) diff --git a/hooks/charmhelpers/fetch/__init__.py b/hooks/charmhelpers/fetch/__init__.py index 07bb707d..c05e0335 100644 --- a/hooks/charmhelpers/fetch/__init__.py +++ b/hooks/charmhelpers/fetch/__init__.py @@ -135,6 +135,10 @@ def apt_hold(packages, fatal=False): def add_source(source, key=None): + if source is None: + log('Source is not present. Skipping') + return + if (source.startswith('ppa:') or source.startswith('http') or source.startswith('deb ') or diff --git a/hooks/nova_cc_context.py b/hooks/nova_cc_context.py index 59ddf4bc..382a5f81 100644 --- a/hooks/nova_cc_context.py +++ b/hooks/nova_cc_context.py @@ -174,3 +174,9 @@ class SharedDBContext(context.SharedDBContext): ctxt['database_type'] = 'mysql' return ctxt +class NovaPostgresqlDBContext(context.PostgresqlDBContext): + interfaces = ['pgsql-nova-db'] + + +class NeutronPostgresqlDBContext(context.PostgresqlDBContext): + interfaces = ['pgsql-neutron-db'] diff --git a/hooks/nova_cc_hooks.py b/hooks/nova_cc_hooks.py index 6dfab01b..f4e2c40f 100755 --- a/hooks/nova_cc_hooks.py +++ b/hooks/nova_cc_hooks.py @@ -13,6 +13,7 @@ from charmhelpers.core.hookenv import ( UnregisteredHookError, config, charm_dir, + is_relation_made, log, relation_get, relation_ids, @@ -29,6 +30,8 @@ from charmhelpers.fetch import ( apt_install, apt_update ) +from charmhelpers.contrib.openstack import context + from charmhelpers.contrib.openstack.utils import ( configure_installation_source, openstack_upgrade_available, @@ -124,6 +127,12 @@ def amqp_changed(): @hooks.hook('shared-db-relation-joined') def db_joined(): + if is_relation_made('pgsql-nova-db') or is_relation_made('pgsql-neutron-db'): + # error, postgresql is used + e = ('Attempting to associate a mysql database when there is already ' + 'associated a postgresql one') + raise context.OSContextError(e) + relation_set(nova_database=config('database'), nova_username=config('database-user'), nova_hostname=unit_get('private-address')) @@ -134,6 +143,28 @@ def db_joined(): neutron_hostname=unit_get('private-address')) +@hooks.hook('pgsql-nova-db-relation-joined') +def pgsql_nova_db_joined(): + if is_relation_made('shared-db'): + # raise error + e = ('Attempting to associate a postgresql database when there is already ' + 'associated a mysql one') + raise context.OSContextError(e) + + relation_set(database=config('database')), + + +@hooks.hook('pgsql-neutron-db-relation-joined') +def pgsql_neutron_db_joined(): + if is_relation_made('shared-db'): + # raise error + e = ('Attempting to associate a postgresql database when there is already ' + 'associated a mysql one') + raise context.OSContextError(e) + + relation_set(database=config('neutron-database')), + + @hooks.hook('shared-db-relation-changed') @restart_on_change(restart_map()) def db_changed(): @@ -154,6 +185,28 @@ def db_changed(): for rid in relation_ids('cloud-compute')] +@hooks.hook('pgsql-nova-db-relation-changed') +@hooks.hook('pgsql-neutron-db-relation-changed') +@restart_on_change(restart_map()) +def db_changed(): + if 'pgsql-nova-db' not in CONFIGS.complete_contexts() or \ + 'pgsql-neutron-db' not in CONFIGS.complete_contexts(): + log('pgsql-*-db relation incomplete. Peer not ready?') + return + CONFIGS.write(NOVA_CONF) + + if network_manager() in ['neutron', 'quantum']: + plugin = neutron_plugin() + # DB config might have been moved to main neutron.conf in H? + CONFIGS.write(neutron_plugin_attribute(plugin, 'config')) + + if eligible_leader(CLUSTER_RES): + migrate_database() + log('Triggering remote cloud-compute restarts.') + [compute_joined(rid=rid, remote_restart=True) + for rid in relation_ids('cloud-compute')] + + @hooks.hook('image-service-relation-changed') @restart_on_change(restart_map()) def image_service_changed(): @@ -385,7 +438,9 @@ def ha_changed(): 'identity-service-relation-broken', 'image-service-relation-broken', 'nova-volume-service-relation-broken', - 'shared-db-relation-broken' + 'shared-db-relation-broken', + 'pgsql-nova-db-relation-broken', + 'pgsql-neutron-db-relation-broken', 'quantum-network-service-relation-broken') def relation_broken(): CONFIGS.write_all() diff --git a/hooks/nova_cc_utils.py b/hooks/nova_cc_utils.py index 412e895e..f5190ed1 100644 --- a/hooks/nova_cc_utils.py +++ b/hooks/nova_cc_utils.py @@ -85,6 +85,8 @@ BASE_RESOURCE_MAP = OrderedDict([ 'services': BASE_SERVICES, 'contexts': [context.AMQPContext(), nova_cc_context.SharedDBContext(relation_prefix='nova'), + nova_cc_context.NovaPostgresqlDBContext(), + nova_cc_context.NeutronPostgresqlDBContext(), context.ImageServiceContext(), context.OSConfigFlagContext(), context.SubordinateConfigContext( diff --git a/hooks/pgsql-neutron-db-relation-broken b/hooks/pgsql-neutron-db-relation-broken new file mode 120000 index 00000000..f6702415 --- /dev/null +++ b/hooks/pgsql-neutron-db-relation-broken @@ -0,0 +1 @@ +nova_cc_hooks.py \ No newline at end of file diff --git a/hooks/pgsql-neutron-db-relation-changed b/hooks/pgsql-neutron-db-relation-changed new file mode 120000 index 00000000..f6702415 --- /dev/null +++ b/hooks/pgsql-neutron-db-relation-changed @@ -0,0 +1 @@ +nova_cc_hooks.py \ No newline at end of file diff --git a/hooks/pgsql-neutron-db-relation-joined b/hooks/pgsql-neutron-db-relation-joined new file mode 120000 index 00000000..f6702415 --- /dev/null +++ b/hooks/pgsql-neutron-db-relation-joined @@ -0,0 +1 @@ +nova_cc_hooks.py \ No newline at end of file diff --git a/hooks/pgsql-nova-db-relation-broken b/hooks/pgsql-nova-db-relation-broken new file mode 120000 index 00000000..f6702415 --- /dev/null +++ b/hooks/pgsql-nova-db-relation-broken @@ -0,0 +1 @@ +nova_cc_hooks.py \ No newline at end of file diff --git a/hooks/pgsql-nova-db-relation-changed b/hooks/pgsql-nova-db-relation-changed new file mode 120000 index 00000000..f6702415 --- /dev/null +++ b/hooks/pgsql-nova-db-relation-changed @@ -0,0 +1 @@ +nova_cc_hooks.py \ No newline at end of file diff --git a/hooks/pgsql-nova-db-relation-joined b/hooks/pgsql-nova-db-relation-joined new file mode 120000 index 00000000..f6702415 --- /dev/null +++ b/hooks/pgsql-nova-db-relation-joined @@ -0,0 +1 @@ +nova_cc_hooks.py \ No newline at end of file diff --git a/metadata.yaml b/metadata.yaml index 366bdb75..5907e5d7 100644 --- a/metadata.yaml +++ b/metadata.yaml @@ -12,6 +12,10 @@ provides: requires: shared-db: interface: mysql-shared + pgsql-nova-db: + interface: pgsql + pgsql-neutron-db: + interface: pgsql amqp: interface: rabbitmq image-service: diff --git a/revision b/revision index 9346fabb..66953656 100644 --- a/revision +++ b/revision @@ -1 +1 @@ -314 +315 From f0214bc3c1dd83c234e954a50dda6a25710634f6 Mon Sep 17 00:00:00 2001 From: "yolanda.robla@canonical.com" <> Date: Thu, 20 Mar 2014 11:33:09 +0100 Subject: [PATCH 03/19] adding 2 interfaces to discriminate --- metadata.yaml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/metadata.yaml b/metadata.yaml index 5907e5d7..d8e523e0 100644 --- a/metadata.yaml +++ b/metadata.yaml @@ -9,6 +9,10 @@ categories: provides: cloud-controller: interface: nova + db: + interface: nova + neutrondb: + interface: nova requires: shared-db: interface: mysql-shared From a381d773f1a83dbfa0daa2b9e9d31bf33f24780d Mon Sep 17 00:00:00 2001 From: "yolanda.robla@canonical.com" <> Date: Thu, 20 Mar 2014 11:51:13 +0100 Subject: [PATCH 04/19] updating metadata --- metadata.yaml | 2 -- 1 file changed, 2 deletions(-) diff --git a/metadata.yaml b/metadata.yaml index d8e523e0..28003ae4 100644 --- a/metadata.yaml +++ b/metadata.yaml @@ -11,8 +11,6 @@ provides: interface: nova db: interface: nova - neutrondb: - interface: nova requires: shared-db: interface: mysql-shared From 890b904f99c8cb984b250abfc290ff57cb61b13b Mon Sep 17 00:00:00 2001 From: "yolanda.robla@canonical.com" <> Date: Thu, 20 Mar 2014 12:07:52 +0100 Subject: [PATCH 05/19] adding psycopg2 package --- hooks/nova_cc_utils.py | 1 + 1 file changed, 1 insertion(+) diff --git a/hooks/nova_cc_utils.py b/hooks/nova_cc_utils.py index f5190ed1..8b439201 100644 --- a/hooks/nova_cc_utils.py +++ b/hooks/nova_cc_utils.py @@ -49,6 +49,7 @@ BASE_PACKAGES = [ 'haproxy', 'python-keystoneclient', 'python-mysqldb', + 'python-psycopg2', 'uuid', ] From f8a29467b1eb12b83d2a92065120a49ba9379ba7 Mon Sep 17 00:00:00 2001 From: "yolanda.robla@canonical.com" <> Date: Thu, 20 Mar 2014 14:54:08 +0100 Subject: [PATCH 06/19] adding postgresql to neutron --- hooks/charmhelpers/contrib/openstack/context.py | 1 + hooks/nova_cc_context.py | 9 --------- hooks/nova_cc_utils.py | 8 +++++++- 3 files changed, 8 insertions(+), 10 deletions(-) diff --git a/hooks/charmhelpers/contrib/openstack/context.py b/hooks/charmhelpers/contrib/openstack/context.py index 6cc3c971..b6069259 100644 --- a/hooks/charmhelpers/contrib/openstack/context.py +++ b/hooks/charmhelpers/contrib/openstack/context.py @@ -146,6 +146,7 @@ class SharedDBContext(OSContextGenerator): 'database': self.database, 'database_user': self.user, 'database_password': passwd, + 'database_type': 'mysql' } if context_complete(ctxt): return ctxt diff --git a/hooks/nova_cc_context.py b/hooks/nova_cc_context.py index 382a5f81..3ce72f3f 100644 --- a/hooks/nova_cc_context.py +++ b/hooks/nova_cc_context.py @@ -164,15 +164,6 @@ class IdentityServiceContext(context.IdentityServiceContext): ctxt['keystone_ec2_url'] = ec2_tokens return ctxt -class SharedDBContext(context.SharedDBContext): - - interfaces = ['shared-db'] - - def __call__(self): - # add database type - ctxt = super(SharedDBContext, self).__call__() - ctxt['database_type'] = 'mysql' - return ctxt class NovaPostgresqlDBContext(context.PostgresqlDBContext): interfaces = ['pgsql-nova-db'] diff --git a/hooks/nova_cc_utils.py b/hooks/nova_cc_utils.py index 8b439201..0e06e028 100644 --- a/hooks/nova_cc_utils.py +++ b/hooks/nova_cc_utils.py @@ -29,6 +29,7 @@ from charmhelpers.fetch import ( from charmhelpers.core.hookenv import ( config, log, + is_relation_made, relation_get, relation_ids, remote_unit, @@ -85,7 +86,7 @@ BASE_RESOURCE_MAP = OrderedDict([ (NOVA_CONF, { 'services': BASE_SERVICES, 'contexts': [context.AMQPContext(), - nova_cc_context.SharedDBContext(relation_prefix='nova'), + context.SharedDBContext(relation_prefix='nova'), nova_cc_context.NovaPostgresqlDBContext(), nova_cc_context.NeutronPostgresqlDBContext(), context.ImageServiceContext(), @@ -195,6 +196,11 @@ def resource_map(): resource_map[conf]['contexts'].append( nova_cc_context.NeutronCCContext()) + # update for postgres + if is_relation_made('pgsql-neutron-db'): + resource_map[conf]['contexts'].append( + context.PostgresqlDBContext()) + # nova-conductor for releases >= G. if os_release('nova-common') not in ['essex', 'folsom']: resource_map['/etc/nova/nova.conf']['services'] += ['nova-conductor'] From 72b2dbf352dd5c4fd8d8e57efb9fc5238410b220 Mon Sep 17 00:00:00 2001 From: "yolanda.robla@canonical.com" <> Date: Thu, 20 Mar 2014 18:58:12 +0100 Subject: [PATCH 07/19] only adding nova postgres db --- hooks/nova_cc_utils.py | 1 - 1 file changed, 1 deletion(-) diff --git a/hooks/nova_cc_utils.py b/hooks/nova_cc_utils.py index 0e06e028..2450ee2c 100644 --- a/hooks/nova_cc_utils.py +++ b/hooks/nova_cc_utils.py @@ -88,7 +88,6 @@ BASE_RESOURCE_MAP = OrderedDict([ 'contexts': [context.AMQPContext(), context.SharedDBContext(relation_prefix='nova'), nova_cc_context.NovaPostgresqlDBContext(), - nova_cc_context.NeutronPostgresqlDBContext(), context.ImageServiceContext(), context.OSConfigFlagContext(), context.SubordinateConfigContext( From 6cb64f567da782a3549b721ff2cda6cd95d1a8bb Mon Sep 17 00:00:00 2001 From: "yolanda.robla@canonical.com" <> Date: Thu, 20 Mar 2014 19:17:35 +0100 Subject: [PATCH 08/19] fixes in postgres for neutron --- hooks/nova_cc_utils.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/hooks/nova_cc_utils.py b/hooks/nova_cc_utils.py index 2450ee2c..31ffa2aa 100644 --- a/hooks/nova_cc_utils.py +++ b/hooks/nova_cc_utils.py @@ -196,9 +196,9 @@ def resource_map(): nova_cc_context.NeutronCCContext()) # update for postgres - if is_relation_made('pgsql-neutron-db'): - resource_map[conf]['contexts'].append( - context.PostgresqlDBContext()) + resource_map[conf]['contexts'].append( + nova_cc_context.NeutronPostgresqlDBContext()) + # nova-conductor for releases >= G. if os_release('nova-common') not in ['essex', 'folsom']: From b7b3a6a29dc09d7759e32e2d845f32a00cc3ae6a Mon Sep 17 00:00:00 2001 From: "yolanda.robla@canonical.com" <> Date: Mon, 24 Mar 2014 12:36:04 +0100 Subject: [PATCH 09/19] fixes in oscontext and in db relations --- hooks/nova_cc_hooks.py | 29 +++++++++++++++++------------ 1 file changed, 17 insertions(+), 12 deletions(-) diff --git a/hooks/nova_cc_hooks.py b/hooks/nova_cc_hooks.py index f4e2c40f..546300ab 100755 --- a/hooks/nova_cc_hooks.py +++ b/hooks/nova_cc_hooks.py @@ -131,7 +131,8 @@ def db_joined(): # error, postgresql is used e = ('Attempting to associate a mysql database when there is already ' 'associated a postgresql one') - raise context.OSContextError(e) + log(e, level=ERROR) + raise relation_set(nova_database=config('database'), nova_username=config('database-user'), @@ -149,7 +150,8 @@ def pgsql_nova_db_joined(): # raise error e = ('Attempting to associate a postgresql database when there is already ' 'associated a mysql one') - raise context.OSContextError(e) + log(e, level=ERROR) + raise relation_set(database=config('database')), @@ -160,7 +162,8 @@ def pgsql_neutron_db_joined(): # raise error e = ('Attempting to associate a postgresql database when there is already ' 'associated a mysql one') - raise context.OSContextError(e) + log(e, level=ERROR) + raise relation_set(database=config('neutron-database')), @@ -186,20 +189,13 @@ def db_changed(): @hooks.hook('pgsql-nova-db-relation-changed') -@hooks.hook('pgsql-neutron-db-relation-changed') @restart_on_change(restart_map()) -def db_changed(): - if 'pgsql-nova-db' not in CONFIGS.complete_contexts() or \ - 'pgsql-neutron-db' not in CONFIGS.complete_contexts(): +def postgresql_nova_db_changed(): + if 'pgsql-nova-db' not in CONFIGS.complete_contexts(): log('pgsql-*-db relation incomplete. Peer not ready?') return CONFIGS.write(NOVA_CONF) - if network_manager() in ['neutron', 'quantum']: - plugin = neutron_plugin() - # DB config might have been moved to main neutron.conf in H? - CONFIGS.write(neutron_plugin_attribute(plugin, 'config')) - if eligible_leader(CLUSTER_RES): migrate_database() log('Triggering remote cloud-compute restarts.') @@ -207,6 +203,15 @@ def db_changed(): for rid in relation_ids('cloud-compute')] +@hooks.hook('pgsql-neutron-db-relation-changed') +@restart_on_change(restart_map()) +def postgresql_neutron_db_changed(): + if network_manager() in ['neutron', 'quantum']: + plugin = neutron_plugin() + # DB config might have been moved to main neutron.conf in H? + CONFIGS.write(neutron_plugin_attribute(plugin, 'config')) + + @hooks.hook('image-service-relation-changed') @restart_on_change(restart_map()) def image_service_changed(): From ef3fabf1c863503f3c5521c9db0a02843ffa1162 Mon Sep 17 00:00:00 2001 From: "yolanda.robla@canonical.com" <> Date: Mon, 24 Mar 2014 13:01:12 +0100 Subject: [PATCH 10/19] raising generic exception --- hooks/nova_cc_hooks.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/hooks/nova_cc_hooks.py b/hooks/nova_cc_hooks.py index 546300ab..59719c80 100755 --- a/hooks/nova_cc_hooks.py +++ b/hooks/nova_cc_hooks.py @@ -15,6 +15,7 @@ from charmhelpers.core.hookenv import ( charm_dir, is_relation_made, log, + ERROR, relation_get, relation_ids, relation_set, @@ -132,7 +133,7 @@ def db_joined(): e = ('Attempting to associate a mysql database when there is already ' 'associated a postgresql one') log(e, level=ERROR) - raise + raise Exception(e) relation_set(nova_database=config('database'), nova_username=config('database-user'), @@ -151,7 +152,7 @@ def pgsql_nova_db_joined(): e = ('Attempting to associate a postgresql database when there is already ' 'associated a mysql one') log(e, level=ERROR) - raise + raise Exception(e) relation_set(database=config('database')), @@ -163,7 +164,7 @@ def pgsql_neutron_db_joined(): e = ('Attempting to associate a postgresql database when there is already ' 'associated a mysql one') log(e, level=ERROR) - raise + raise Exception(e) relation_set(database=config('neutron-database')), From 129bcd6e457ba2b7b8635d3943e455bb1d0e1446 Mon Sep 17 00:00:00 2001 From: "yolanda.robla@canonical.com" <> Date: Mon, 24 Mar 2014 15:31:59 +0100 Subject: [PATCH 11/19] fixed lint bugs resync charmhelpers --- hooks/charmhelpers/contrib/openstack/context.py | 4 ++-- .../contrib/openstack/templates/ceph.conf | 9 +++++---- hooks/charmhelpers/fetch/archiveurl.py | 15 +++++++++++++++ hooks/nova_cc_hooks.py | 5 +---- 4 files changed, 23 insertions(+), 10 deletions(-) diff --git a/hooks/charmhelpers/contrib/openstack/context.py b/hooks/charmhelpers/contrib/openstack/context.py index b6069259..d547a4fb 100644 --- a/hooks/charmhelpers/contrib/openstack/context.py +++ b/hooks/charmhelpers/contrib/openstack/context.py @@ -146,7 +146,7 @@ class SharedDBContext(OSContextGenerator): 'database': self.database, 'database_user': self.user, 'database_password': passwd, - 'database_type': 'mysql' + 'database_type': 'mysql', } if context_complete(ctxt): return ctxt @@ -162,7 +162,7 @@ class PostgresqlDBContext(OSContextGenerator): def __call__(self): self.database = self.database or config('database') if self.database is None: - log('Could not generate shared_db context. ' + log('Could not generate postgresql_db context. ' 'Missing required charm config options. ' '(database name)') raise OSContextError diff --git a/hooks/charmhelpers/contrib/openstack/templates/ceph.conf b/hooks/charmhelpers/contrib/openstack/templates/ceph.conf index 874ad2bd..81a9719f 100644 --- a/hooks/charmhelpers/contrib/openstack/templates/ceph.conf +++ b/hooks/charmhelpers/contrib/openstack/templates/ceph.conf @@ -3,12 +3,13 @@ # cinder configuration file maintained by Juju # local changes may be overwritten. ############################################################################### -{% if auth -%} [global] +{% if auth -%} auth_supported = {{ auth }} keyring = /etc/ceph/$cluster.$name.keyring mon host = {{ mon_hosts }} {% endif -%} -log to syslog = {{ use_syslog }} -err to syslog = {{ use_syslog }} -clog to syslog = {{ use_syslog }} + log to syslog = {{ use_syslog }} + err to syslog = {{ use_syslog }} + clog to syslog = {{ use_syslog }} + diff --git a/hooks/charmhelpers/fetch/archiveurl.py b/hooks/charmhelpers/fetch/archiveurl.py index e35b8f15..87e7071a 100644 --- a/hooks/charmhelpers/fetch/archiveurl.py +++ b/hooks/charmhelpers/fetch/archiveurl.py @@ -1,5 +1,7 @@ import os import urllib2 +import urlparse + from charmhelpers.fetch import ( BaseFetchHandler, UnhandledSource @@ -24,6 +26,19 @@ class ArchiveUrlFetchHandler(BaseFetchHandler): def download(self, source, dest): # propogate all exceptions # URLError, OSError, etc + proto, netloc, path, params, query, fragment = urlparse.urlparse(source) + if proto in ('http', 'https'): + auth, barehost = urllib2.splituser(netloc) + if auth is not None: + source = urlparse.urlunparse((proto, barehost, path, params, query, fragment)) + username, password = urllib2.splitpasswd(auth) + passman = urllib2.HTTPPasswordMgrWithDefaultRealm() + # Realm is set to None in add_password to force the username and password + # to be used whatever the realm + passman.add_password(None, source, username, password) + authhandler = urllib2.HTTPBasicAuthHandler(passman) + opener = urllib2.build_opener(authhandler) + urllib2.install_opener(opener) response = urllib2.urlopen(source) try: with open(dest, 'w') as dest_file: diff --git a/hooks/nova_cc_hooks.py b/hooks/nova_cc_hooks.py index 59719c80..cc2c8de1 100755 --- a/hooks/nova_cc_hooks.py +++ b/hooks/nova_cc_hooks.py @@ -31,8 +31,6 @@ from charmhelpers.fetch import ( apt_install, apt_update ) -from charmhelpers.contrib.openstack import context - from charmhelpers.contrib.openstack.utils import ( configure_installation_source, openstack_upgrade_available, @@ -134,7 +132,7 @@ def db_joined(): 'associated a postgresql one') log(e, level=ERROR) raise Exception(e) - + relation_set(nova_database=config('database'), nova_username=config('database-user'), nova_hostname=unit_get('private-address')) @@ -203,7 +201,6 @@ def postgresql_nova_db_changed(): [compute_joined(rid=rid, remote_restart=True) for rid in relation_ids('cloud-compute')] - @hooks.hook('pgsql-neutron-db-relation-changed') @restart_on_change(restart_map()) def postgresql_neutron_db_changed(): From ceda7d6a7048900429b9ecb719597d8ce9c83744 Mon Sep 17 00:00:00 2001 From: "yolanda.robla@canonical.com" <> Date: Tue, 25 Mar 2014 09:50:53 +0100 Subject: [PATCH 12/19] changes in templates to support database type --- hooks/nova_cc_utils.py | 8 +++----- templates/grizzly/nova.conf | 2 +- templates/havana/nova.conf | 2 +- templates/icehouse/neutron.conf | 2 +- 4 files changed, 6 insertions(+), 8 deletions(-) diff --git a/hooks/nova_cc_utils.py b/hooks/nova_cc_utils.py index 77b8ef73..dba3a586 100644 --- a/hooks/nova_cc_utils.py +++ b/hooks/nova_cc_utils.py @@ -30,7 +30,6 @@ from charmhelpers.fetch import ( from charmhelpers.core.hookenv import ( config, log, - is_relation_made, relation_get, relation_ids, remote_unit, @@ -129,9 +128,9 @@ BASE_RESOURCE_MAP = OrderedDict([ 'services': ['neutron-server'], 'contexts': [context.AMQPContext(), context.SharedDBContext( - user=config('neutron-database-user'), - database=config('neutron-database'), - relation_prefix='neutron'), + user=config('neutron-database-user'), + database=config('neutron-database'), + relation_prefix='neutron'), nova_cc_context.IdentityServiceContext(), nova_cc_context.NeutronCCContext(), nova_cc_context.HAProxyContext()], @@ -209,7 +208,6 @@ def resource_map(): resource_map[conf]['contexts'].append( nova_cc_context.NeutronPostgresqlDBContext()) - # nova-conductor for releases >= G. if os_release('nova-common') not in ['essex', 'folsom']: resource_map['/etc/nova/nova.conf']['services'] += ['nova-conductor'] diff --git a/templates/grizzly/nova.conf b/templates/grizzly/nova.conf index 007e4f0e..7bf5c9bb 100644 --- a/templates/grizzly/nova.conf +++ b/templates/grizzly/nova.conf @@ -25,7 +25,7 @@ keystone_ec2_url = {{ keystone_ec2_url }} {% endif -%} {% if database_host -%} -sql_connection = mysql://{{ database_user }}:{{ database_password }}@{{ database_host }}/{{ database }} +sql_connection = {{database_type}}://{{ database_user }}:{{ database_password }}@{{ database_host }}/{{ database }} {% endif -%} {% if rabbitmq_host or rabbitmq_hosts -%} diff --git a/templates/havana/nova.conf b/templates/havana/nova.conf index 5b2f22cd..d252516c 100644 --- a/templates/havana/nova.conf +++ b/templates/havana/nova.conf @@ -25,7 +25,7 @@ keystone_ec2_url = {{ keystone_ec2_url }} {% endif -%} {% if database_host -%} -sql_connection = mysql://{{ database_user }}:{{ database_password }}@{{ database_host }}/{{ database }} +sql_connection = {{database_type}}://{{ database_user }}:{{ database_password }}@{{ database_host }}/{{ database }} {% endif -%} {% if rabbitmq_host or rabbitmq_hosts -%} diff --git a/templates/icehouse/neutron.conf b/templates/icehouse/neutron.conf index 5b3e1088..ff60656f 100644 --- a/templates/icehouse/neutron.conf +++ b/templates/icehouse/neutron.conf @@ -67,7 +67,7 @@ admin_password = {{ admin_password }} [database] {% if database_host -%} -connection = mysql://{{ database_user }}:{{ database_password }}@{{ database_host }}/{{ database }} +connection = {{database_type}}://{{ database_user }}:{{ database_password }}@{{ database_host }}/{{ database }} {% else -%} connection = sqlite:////var/lib/neutron/neutron.sqlite {% endif -%} From abbe9589ceff3aa792fbe416ea116bba567d22ad Mon Sep 17 00:00:00 2001 From: "yolanda.robla@canonical.com" <> Date: Tue, 25 Mar 2014 10:00:50 +0100 Subject: [PATCH 13/19] typo --- hooks/nova_cc_hooks.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/hooks/nova_cc_hooks.py b/hooks/nova_cc_hooks.py index 745f3087..24c1a72e 100755 --- a/hooks/nova_cc_hooks.py +++ b/hooks/nova_cc_hooks.py @@ -153,7 +153,7 @@ def pgsql_nova_db_joined(): log(e, level=ERROR) raise Exception(e) - relation_set(database=config('database')), + relation_set(database=config('database')) @hooks.hook('pgsql-neutron-db-relation-joined') @@ -165,7 +165,7 @@ def pgsql_neutron_db_joined(): log(e, level=ERROR) raise Exception(e) - relation_set(database=config('neutron-database')), + relation_set(database=config('neutron-database')) @hooks.hook('shared-db-relation-changed') From 5f48dbbc1f53249c60c84d4c6cd4696a9bfbaf74 Mon Sep 17 00:00:00 2001 From: "yolanda.robla@canonical.com" <> Date: Tue, 25 Mar 2014 10:11:04 +0100 Subject: [PATCH 14/19] pushed initial readme --- README.txt | 15 +++++++++++++++ 1 file changed, 15 insertions(+) create mode 100644 README.txt diff --git a/README.txt b/README.txt new file mode 100644 index 00000000..984859ad --- /dev/null +++ b/README.txt @@ -0,0 +1,15 @@ +===================== +nova-cloud-controller +===================== + +Cloud controller node for Openstack nova. Contains nova-schedule, nova-api, nova-network and nova-objectstore. + +****************************************************** +Special considerations to be deployed using Postgresql +****************************************************** + +juju deploy nova-cloud-controller +juju deploy postgresql + +juju add-relation "nova-cloud-controller:pgsql-nova-db" "postgresql:db" +juju add-relation "nova-cloud-controller:pgsql-neutron-db" "postgresql:db" From 16364f3eb231d1fbd1d0fc3e900d984bf9420bc2 Mon Sep 17 00:00:00 2001 From: "yolanda.robla@canonical.com" <> Date: Thu, 27 Mar 2014 12:46:44 +0100 Subject: [PATCH 15/19] added db testing --- hooks/nova_cc_hooks.py | 2 +- unit_tests/test_nova_cc_hooks.py | 103 +++++++++++++++++++++++++++++-- 2 files changed, 100 insertions(+), 5 deletions(-) diff --git a/hooks/nova_cc_hooks.py b/hooks/nova_cc_hooks.py index 24c1a72e..6b9ad522 100755 --- a/hooks/nova_cc_hooks.py +++ b/hooks/nova_cc_hooks.py @@ -192,7 +192,7 @@ def db_changed(): @restart_on_change(restart_map()) def postgresql_nova_db_changed(): if 'pgsql-nova-db' not in CONFIGS.complete_contexts(): - log('pgsql-*-db relation incomplete. Peer not ready?') + log('pgsql-nova-db relation incomplete. Peer not ready?') return CONFIGS.write(NOVA_CONF) diff --git a/unit_tests/test_nova_cc_hooks.py b/unit_tests/test_nova_cc_hooks.py index e0d93e76..a0e5a941 100644 --- a/unit_tests/test_nova_cc_hooks.py +++ b/unit_tests/test_nova_cc_hooks.py @@ -1,8 +1,9 @@ -from mock import MagicMock, patch - +from mock import call, MagicMock, patch from test_utils import CharmTestCase -import nova_cc_utils as utils +with patch('charmhelpers.core.hookenv.config') as config: + config.return_value = 'neutron' + import nova_cc_utils as utils _reg = utils.register_configs _map = utils.restart_map @@ -15,7 +16,6 @@ import nova_cc_hooks as hooks utils.register_configs = _reg utils.restart_map = _map - TO_PATCH = [ 'api_port', 'apt_update', @@ -29,8 +29,11 @@ TO_PATCH = [ 'determine_packages', 'determine_ports', 'open_port', + 'is_relation_made', + 'log', 'relation_get', 'relation_set', + 'relation_ids', 'ssh_compute_add', 'ssh_known_hosts_b64', 'ssh_authorized_keys_b64', @@ -42,6 +45,7 @@ TO_PATCH = [ 'eligible_leader', 'keystone_ca_cert_b64', 'neutron_plugin', + 'migrate_database', ] @@ -63,6 +67,7 @@ class NovaCCHooksTests(CharmTestCase): def setUp(self): super(NovaCCHooksTests, self).setUp(hooks, TO_PATCH) + self.config.side_effect = self.test_config.get self.relation_get.side_effect = self.test_relation.get self.charm_dir.return_value = '/var/lib/juju/charms/nova/charm' @@ -138,3 +143,93 @@ class NovaCCHooksTests(CharmTestCase): quantum_url='http://nova-cc-host1:9696', quantum_plugin='nvp', relation_id=None, **FAKE_KS_AUTH_CFG) + + def test_db_joined(self): + self.unit_get.return_value = 'nova.foohost.com' + self.is_relation_made.return_value = False + hooks.db_joined() + self.relation_set.assert_called_with(nova_database='nova', + nova_username='nova', + nova_hostname='nova.foohost.com') + self.unit_get.assert_called_with('private-address') + + def test_postgresql_nova_db_joined(self): + self.is_relation_made.return_value = False + hooks.pgsql_nova_db_joined() + self.relation_set.assert_called_with(database='nova') + + def test_postgresql_neutron_db_joined(self): + self.is_relation_made.return_value = False + hooks.pgsql_neutron_db_joined() + self.relation_set.assert_called_with(database='neutron') + + def test_db_joined_with_postgresql(self): + self.is_relation_made.return_value = True + + with self.assertRaises(Exception) as context: + hooks.db_joined() + self.assertEqual(context.exception.message, + 'Attempting to associate a mysql database when there is already ' + 'associated a postgresql one') + + def test_postgresql_nova_joined_with_db(self): + self.is_relation_made.return_value = True + + with self.assertRaises(Exception) as context: + hooks.pgsql_nova_db_joined() + self.assertEqual(context.exception.message, + 'Attempting to associate a postgresql database when there is already ' + 'associated a mysql one') + + def test_postgresql_neutron_joined_with_db(self): + self.is_relation_made.return_value = True + + with self.assertRaises(Exception) as context: + hooks.pgsql_neutron_db_joined() + self.assertEqual(context.exception.message, + 'Attempting to associate a postgresql database when there is already ' + 'associated a mysql one') + + @patch.object(hooks, 'CONFIGS') + def test_db_changed_missing_relation_data(self, configs): + configs.complete_contexts = MagicMock() + configs.complete_contexts.return_value = [] + hooks.db_changed() + self.log.assert_called_with( + 'shared-db relation incomplete. Peer not ready?' + ) + + @patch.object(hooks, 'CONFIGS') + def test_postgresql_nova_db_changed_missing_relation_data(self, configs): + configs.complete_contexts = MagicMock() + configs.complete_contexts.return_value = [] + hooks.postgresql_nova_db_changed() + self.log.assert_called_with( + 'pgsql-nova-db relation incomplete. Peer not ready?' + ) + + def _shared_db_test(self, configs): + configs.complete_contexts = MagicMock() + configs.complete_contexts.return_value = ['shared-db'] + configs.write = MagicMock() + hooks.db_changed() + + def _postgresql_db_test(self, configs): + configs.complete_contexts = MagicMock() + configs.complete_contexts.return_value = ['pgsql-nova-db'] + configs.write = MagicMock() + hooks.postgresql_nova_db_changed() + + @patch.object(hooks, 'CONFIGS') + def test_db_changed(self, configs): + self._shared_db_test(configs) + self.assertEquals([call('/etc/nova/nova.conf')], + configs.write.call_args_list) + self.migrate_database.assert_called_with() + + @patch.object(hooks, 'CONFIGS') + def test_postgresql_db_changed(self, configs): + self._postgresql_db_test(configs) + self.assertEquals([call('/etc/nova/nova.conf')], + configs.write.call_args_list) + self.migrate_database.assert_called_with() From 36a37b420c64653178641b27800ab3ee96ad10f7 Mon Sep 17 00:00:00 2001 From: "yolanda.robla@canonical.com" <> Date: Fri, 28 Mar 2014 14:13:18 +0100 Subject: [PATCH 16/19] not ssl in postgres --- hooks/charmhelpers/contrib/openstack/context.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/hooks/charmhelpers/contrib/openstack/context.py b/hooks/charmhelpers/contrib/openstack/context.py index ba909522..d3ea5ff6 100644 --- a/hooks/charmhelpers/contrib/openstack/context.py +++ b/hooks/charmhelpers/contrib/openstack/context.py @@ -179,9 +179,7 @@ class PostgresqlDBContext(OSContextGenerator): 'database_password': relation_get('password', rid=rid, unit=unit), 'database_type': 'postgresql', } - if context_complete(ctxt): - db_ssl(rdata, ctxt, self.ssl_dir) - return ctxt + return ctxt return {} From 76b768e79fec624e9ad3b9d36267debc3b6a773e Mon Sep 17 00:00:00 2001 From: "yolanda.robla@canonical.com" <> Date: Fri, 28 Mar 2014 14:14:32 +0100 Subject: [PATCH 17/19] not using ssl --- hooks/charmhelpers/contrib/openstack/context.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/hooks/charmhelpers/contrib/openstack/context.py b/hooks/charmhelpers/contrib/openstack/context.py index d3ea5ff6..b564a01a 100644 --- a/hooks/charmhelpers/contrib/openstack/context.py +++ b/hooks/charmhelpers/contrib/openstack/context.py @@ -179,7 +179,8 @@ class PostgresqlDBContext(OSContextGenerator): 'database_password': relation_get('password', rid=rid, unit=unit), 'database_type': 'postgresql', } - return ctxt + if context_complete(ctxt): + return ctxt return {} From 96a03a567c3c8ecc06abd3c3a91746a212db7773 Mon Sep 17 00:00:00 2001 From: "yolanda.robla@canonical.com" <> Date: Fri, 28 Mar 2014 15:17:48 +0100 Subject: [PATCH 18/19] typo in template --- templates/icehouse/neutron.conf | 1 + 1 file changed, 1 insertion(+) diff --git a/templates/icehouse/neutron.conf b/templates/icehouse/neutron.conf index 2f44eaa7..36464be4 100644 --- a/templates/icehouse/neutron.conf +++ b/templates/icehouse/neutron.conf @@ -73,6 +73,7 @@ admin_password = {{ admin_password }} {% endif -%} [database] +{% if database_host -%} connection = {{ database_type }}://{{ database_user }}:{{ database_password }}@{{ database_host }}/{{ database }}{% if database_ssl_ca %}?ssl_ca={{ database_ssl_ca }}{% if database_ssl_cert %}&ssl_cert={{ database_ssl_cert }}&ssl_key={{ database_ssl_key }}{% endif %}{% endif %} {% else -%} connection = sqlite:////var/lib/neutron/neutron.sqlite From f967dc60179ac9fc973958575dee6a17b5d73776 Mon Sep 17 00:00:00 2001 From: "yolanda.robla@canonical.com" <> Date: Mon, 31 Mar 2014 12:56:51 +0200 Subject: [PATCH 19/19] resynced context --- hooks/charmhelpers/contrib/openstack/context.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/hooks/charmhelpers/contrib/openstack/context.py b/hooks/charmhelpers/contrib/openstack/context.py index b564a01a..8429d832 100644 --- a/hooks/charmhelpers/contrib/openstack/context.py +++ b/hooks/charmhelpers/contrib/openstack/context.py @@ -147,10 +147,11 @@ class SharedDBContext(OSContextGenerator): 'database_host': rdata.get('db_host'), 'database': self.database, 'database_user': self.user, - 'database_password': passwd, - 'database_type': 'mysql', + 'database_password': rdata.get(password_setting), + 'database_type': 'mysql' } if context_complete(ctxt): + db_ssl(rdata, ctxt, self.ssl_dir) return ctxt return {}