From 22388aefe6885e45f91099ab2a169f7f4014d327 Mon Sep 17 00:00:00 2001 From: Louis Bouchard Date: Wed, 30 Apr 2014 14:13:22 +0200 Subject: [PATCH 01/48] Changes on the nova-cloud-controller side --- hooks/nova_cc_hooks.py | 17 +++++++++++++---- hooks/nova_cc_utils.py | 18 +++++++++++++----- 2 files changed, 26 insertions(+), 9 deletions(-) diff --git a/hooks/nova_cc_hooks.py b/hooks/nova_cc_hooks.py index ce9d7438..7bf6e8fe 100755 --- a/hooks/nova_cc_hooks.py +++ b/hooks/nova_cc_hooks.py @@ -54,8 +54,8 @@ from nova_cc_utils import ( save_script_rc, ssh_compute_add, ssh_compute_remove, - ssh_known_hosts_b64, - ssh_authorized_keys_b64, + ssh_known_hosts_lines_b64, + ssh_authorized_keys_lines_b64, register_configs, restart_map, volume_service, @@ -347,8 +347,17 @@ def compute_changed(): log('SSH migration set but peer did not publish key.') return ssh_compute_add(key) - relation_set(known_hosts=ssh_known_hosts_b64(), - authorized_keys=ssh_authorized_keys_b64()) + for line in ssh_known_hosts_lines_b64(): + relation_name = 'known_hosts_{}'.format(index) + relation_set(relation_name=line) + index+=1 + index = 0 + relation_set(known_hosts_max_index=index) + for line in ssh_authorized_keys_lines_b64(): + relation_name = 'authorized_keys_{}'.format(index) + relation_set(relation_name=line) + index+=1 + relation_set(authorized_keys_max_index=index) if relation_get('nova_ssh_public_key'): key = relation_get('nova_ssh_public_key') ssh_compute_add(key, user='nova') diff --git a/hooks/nova_cc_utils.py b/hooks/nova_cc_utils.py index eb61f1ef..49c54410 100644 --- a/hooks/nova_cc_utils.py +++ b/hooks/nova_cc_utils.py @@ -603,14 +603,22 @@ def ssh_compute_add(public_key, user=None): add_authorized_key(public_key, user) -def ssh_known_hosts_b64(user=None): +def ssh_known_hosts_lines_b64(user=None): + known_hosts_list = [] + with open(known_hosts(user)) as hosts: - return b64encode(hosts.read()) + for hosts_line in hosts: + known_hosts_list.append(b64encode(hosts_line.rstrip())) + return(known_hosts_list) -def ssh_authorized_keys_b64(user=None): - with open(authorized_keys(user)) as keys: - return b64encode(keys.read()) +def ssh_authorized_keys_lines_b64(user=None): + authorized_keys_list = [] + + with open(authorized_keys(user)) as hosts: + for authkey_line in hosts: + authorized_keys_list.append(b64encode(authkey_line.rstrip())) + return(authorized_keys_list) def ssh_compute_remove(public_key, user=None): From 100383bdb2bc75eac5c0db6b033424904387f1b6 Mon Sep 17 00:00:00 2001 From: Louis Bouchard Date: Fri, 2 May 2014 18:07:52 +0200 Subject: [PATCH 02/48] Functional line by line modification, except for the 'nova' user. Need more work on the 'nova' specific case --- hooks/nova_cc_hooks.py | 21 +++++++++++---------- hooks/nova_cc_utils.py | 14 ++++++++------ 2 files changed, 19 insertions(+), 16 deletions(-) diff --git a/hooks/nova_cc_hooks.py b/hooks/nova_cc_hooks.py index 7bf6e8fe..a485f936 100755 --- a/hooks/nova_cc_hooks.py +++ b/hooks/nova_cc_hooks.py @@ -54,8 +54,8 @@ from nova_cc_utils import ( save_script_rc, ssh_compute_add, ssh_compute_remove, - ssh_known_hosts_lines_b64, - ssh_authorized_keys_lines_b64, + ssh_known_hosts_lines, + ssh_authorized_keys_lines, register_configs, restart_map, volume_service, @@ -347,16 +347,17 @@ def compute_changed(): log('SSH migration set but peer did not publish key.') return ssh_compute_add(key) - for line in ssh_known_hosts_lines_b64(): - relation_name = 'known_hosts_{}'.format(index) - relation_set(relation_name=line) - index+=1 index = 0 + for line in ssh_known_hosts_lines(): + relation_set(relation_settings= + {'known_hosts_{}'.format(index): line}) + index += 1 relation_set(known_hosts_max_index=index) - for line in ssh_authorized_keys_lines_b64(): - relation_name = 'authorized_keys_{}'.format(index) - relation_set(relation_name=line) - index+=1 + index = 0 + for line in ssh_authorized_keys_lines(): + relation_set(relation_settings= + {'authorized_keys_{}'.format(index): line}) + index += 1 relation_set(authorized_keys_max_index=index) if relation_get('nova_ssh_public_key'): key = relation_get('nova_ssh_public_key') diff --git a/hooks/nova_cc_utils.py b/hooks/nova_cc_utils.py index 49c54410..7cc3255b 100644 --- a/hooks/nova_cc_utils.py +++ b/hooks/nova_cc_utils.py @@ -603,21 +603,23 @@ def ssh_compute_add(public_key, user=None): add_authorized_key(public_key, user) -def ssh_known_hosts_lines_b64(user=None): +def ssh_known_hosts_lines(user=None): known_hosts_list = [] with open(known_hosts(user)) as hosts: for hosts_line in hosts: - known_hosts_list.append(b64encode(hosts_line.rstrip())) + if hosts_line.rstrip(): + known_hosts_list.append(hosts_line.rstrip()) return(known_hosts_list) -def ssh_authorized_keys_lines_b64(user=None): +def ssh_authorized_keys_lines(user=None): authorized_keys_list = [] - with open(authorized_keys(user)) as hosts: - for authkey_line in hosts: - authorized_keys_list.append(b64encode(authkey_line.rstrip())) + with open(authorized_keys(user)) as keys: + for authkey_line in keys: + if authkey_line.rstrip(): + authorized_keys_list.append(authkey_line.rstrip()) return(authorized_keys_list) From 222862916873a0e5b78d2c26890c740c8a445ac5 Mon Sep 17 00:00:00 2001 From: Louis Bouchard Date: Tue, 6 May 2014 15:43:57 +0200 Subject: [PATCH 03/48] Support for line by line for 'nova' user --- hooks/nova_cc_hooks.py | 18 ++++++++++++++++-- 1 file changed, 16 insertions(+), 2 deletions(-) diff --git a/hooks/nova_cc_hooks.py b/hooks/nova_cc_hooks.py index a485f936..5056414a 100755 --- a/hooks/nova_cc_hooks.py +++ b/hooks/nova_cc_hooks.py @@ -362,8 +362,22 @@ def compute_changed(): if relation_get('nova_ssh_public_key'): key = relation_get('nova_ssh_public_key') ssh_compute_add(key, user='nova') - relation_set(nova_known_hosts=ssh_known_hosts_b64(user='nova'), - nova_authorized_keys=ssh_authorized_keys_b64(user='nova')) + index = 0 + for line in ssh_known_hosts_lines(user='nova'): + relation_set(relation_settings= + {'{}_known_hosts_{}'.format('nova',index): line}) + index += 1 + relation_set(relation_settings= + {'{}_known_hosts_max_index'.format('nova'): index}) + index = 0 + for line in ssh_authorized_keys_lines(user='nova'): + relation_set(relation_settings= + {'{}_authorized_keys_{}'.format('nova', index): line}) + index += 1 + relation_set(relation_settings= + {'{}_authorized_keys_max_index'.format('nova'): index}) +# relation_set(nova_known_hosts=ssh_known_hosts_b64(user='nova'), +# nova_authorized_keys=ssh_authorized_keys_b64(user='nova')) @hooks.hook('cloud-compute-relation-departed') From 4867b9821e7ee06f581bfa44b93fadfd1c3bc84c Mon Sep 17 00:00:00 2001 From: Louis Bouchard Date: Tue, 6 May 2014 16:00:46 +0200 Subject: [PATCH 04/48] Removed unwanted comments --- hooks/nova_cc_hooks.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/hooks/nova_cc_hooks.py b/hooks/nova_cc_hooks.py index 5056414a..469347c2 100755 --- a/hooks/nova_cc_hooks.py +++ b/hooks/nova_cc_hooks.py @@ -376,8 +376,6 @@ def compute_changed(): index += 1 relation_set(relation_settings= {'{}_authorized_keys_max_index'.format('nova'): index}) -# relation_set(nova_known_hosts=ssh_known_hosts_b64(user='nova'), -# nova_authorized_keys=ssh_authorized_keys_b64(user='nova')) @hooks.hook('cloud-compute-relation-departed') From 9ac5f05d27083085fac0628633f5378e498a898a Mon Sep 17 00:00:00 2001 From: Louis Bouchard Date: Tue, 6 May 2014 16:09:06 +0200 Subject: [PATCH 05/48] PEP8 cleanup --- hooks/nova_cc_hooks.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/hooks/nova_cc_hooks.py b/hooks/nova_cc_hooks.py index 469347c2..5e403918 100755 --- a/hooks/nova_cc_hooks.py +++ b/hooks/nova_cc_hooks.py @@ -365,17 +365,17 @@ def compute_changed(): index = 0 for line in ssh_known_hosts_lines(user='nova'): relation_set(relation_settings= - {'{}_known_hosts_{}'.format('nova',index): line}) + {'{}_known_hosts_{}'.format('nova', index): line}) index += 1 relation_set(relation_settings= - {'{}_known_hosts_max_index'.format('nova'): index}) + {'{}_known_hosts_max_index'.format('nova'): index}) index = 0 for line in ssh_authorized_keys_lines(user='nova'): relation_set(relation_settings= {'{}_authorized_keys_{}'.format('nova', index): line}) index += 1 relation_set(relation_settings= - {'{}_authorized_keys_max_index'.format('nova'): index}) + {'{}_authorized_keys_max_index'.format('nova'): index}) @hooks.hook('cloud-compute-relation-departed') From 23e94aa88ea4bbbe098023fce2761290dd053223 Mon Sep 17 00:00:00 2001 From: Louis Bouchard Date: Thu, 22 May 2014 10:47:41 +0200 Subject: [PATCH 06/48] Adapt unit tests to changes made by previous commit Tests now cover multi-line known_hosts & authorized_keys --- unit_tests/test_nova_cc_hooks.py | 23 +++++++++++++++++------ 1 file changed, 17 insertions(+), 6 deletions(-) diff --git a/unit_tests/test_nova_cc_hooks.py b/unit_tests/test_nova_cc_hooks.py index 126a59c1..8dc49923 100644 --- a/unit_tests/test_nova_cc_hooks.py +++ b/unit_tests/test_nova_cc_hooks.py @@ -35,8 +35,8 @@ TO_PATCH = [ 'relation_set', 'relation_ids', 'ssh_compute_add', - 'ssh_known_hosts_b64', - 'ssh_authorized_keys_b64', + 'ssh_known_hosts_lines', + 'ssh_authorized_keys_lines', 'save_script_rc', 'execd_preinstall', 'network_manager', @@ -98,12 +98,23 @@ class NovaCCHooksTests(CharmTestCase): self.test_relation.set({ 'migration_auth_type': 'ssh', 'ssh_public_key': 'fookey', 'private-address': '10.0.0.1'}) - self.ssh_known_hosts_b64.return_value = 'hosts' - self.ssh_authorized_keys_b64.return_value = 'keys' + self.ssh_known_hosts_lines.return_value = [ + 'k_h_0', 'k_h_1', 'k_h_2'] + self.ssh_authorized_keys_lines.return_value = [ + 'auth_0', 'auth_1', 'auth_2'] hooks.compute_changed() self.ssh_compute_add.assert_called_with('fookey') - self.relation_set.assert_called_with(known_hosts='hosts', - authorized_keys='keys') + expected_relations = [ + call(relation_settings={'authorized_keys_0': 'auth_0'}), + call(relation_settings={'authorized_keys_1': 'auth_1'}), + call(relation_settings={'authorized_keys_2': 'auth_2'}), + call(relation_settings={'known_hosts_0': 'k_h_0'}), + call(relation_settings={'known_hosts_1': 'k_h_1'}), + call(relation_settings={'known_hosts_2': 'k_h_2'}), + call(authorized_keys_max_index=3), + call(known_hosts_max_index=3)] + self.assertEquals(sorted(self.relation_set.call_args_list), + sorted(expected_relations)) @patch.object(hooks, '_auth_config') def test_compute_joined_neutron(self, auth_config): From ef618cd10592d061aee7f7b949924a21ee9485f4 Mon Sep 17 00:00:00 2001 From: Louis Bouchard Date: Thu, 22 May 2014 14:23:16 +0200 Subject: [PATCH 07/48] Add new test_compute_changed_nova_public_key unit test Test the compute_changed hook when 'enable-resize' is used by nova-compute --- unit_tests/test_nova_cc_hooks.py | 22 ++++++++++++++++++++++ 1 file changed, 22 insertions(+) diff --git a/unit_tests/test_nova_cc_hooks.py b/unit_tests/test_nova_cc_hooks.py index 8dc49923..ef052823 100644 --- a/unit_tests/test_nova_cc_hooks.py +++ b/unit_tests/test_nova_cc_hooks.py @@ -116,6 +116,28 @@ class NovaCCHooksTests(CharmTestCase): self.assertEquals(sorted(self.relation_set.call_args_list), sorted(expected_relations)) + def test_compute_changed_nova_public_key(self): + self.test_relation.set({ + 'nova_ssh_public_key': 'fookey', + 'private-address': '10.0.0.1'}) + self.ssh_known_hosts_lines.return_value = [ + 'k_h_0', 'k_h_1', 'k_h_2'] + self.ssh_authorized_keys_lines.return_value = [ + 'auth_0', 'auth_1', 'auth_2'] + hooks.compute_changed() + self.ssh_compute_add.assert_called_with('fookey', user='nova') + expected_relations = [ + call(relation_settings={'nova_authorized_keys_0': 'auth_0'}), + call(relation_settings={'nova_authorized_keys_1': 'auth_1'}), + call(relation_settings={'nova_authorized_keys_2': 'auth_2'}), + call(relation_settings={'nova_known_hosts_0': 'k_h_0'}), + call(relation_settings={'nova_known_hosts_1': 'k_h_1'}), + call(relation_settings={'nova_known_hosts_2': 'k_h_2'}), + call(relation_settings={'nova_known_hosts_max_index': 3}), + call(relation_settings={'nova_authorized_keys_max_index': 3})] + self.assertEquals(sorted(self.relation_set.call_args_list), + sorted(expected_relations)) + @patch.object(hooks, '_auth_config') def test_compute_joined_neutron(self, auth_config): self.network_manager.return_value = 'neutron' From c2a29b7eb7bcdde5c37e22815e9288eba80d3a0f Mon Sep 17 00:00:00 2001 From: Louis Bouchard Date: Fri, 23 May 2014 17:24:52 +0200 Subject: [PATCH 08/48] Modify hook funcions & util to work in non-relation hooks Some hook functions did not work when invoked from non-relation hooks (compute_changed in particular). Modified all utils hook functions to honour relation-id & unit-id when required --- hooks/nova_cc_hooks.py | 44 +++++++++++++------------- hooks/nova_cc_utils.py | 70 ++++++++++++++++++++++-------------------- 2 files changed, 60 insertions(+), 54 deletions(-) diff --git a/hooks/nova_cc_hooks.py b/hooks/nova_cc_hooks.py index 5e403918..fee92839 100755 --- a/hooks/nova_cc_hooks.py +++ b/hooks/nova_cc_hooks.py @@ -19,6 +19,7 @@ from charmhelpers.core.hookenv import ( relation_get, relation_ids, relation_set, + related_units, open_port, unit_get, ) @@ -339,48 +340,49 @@ def compute_joined(rid=None, remote_restart=False): @hooks.hook('cloud-compute-relation-changed') -def compute_changed(): - migration_auth = relation_get('migration_auth_type') +def compute_changed(rid=None, uid=None): + migration_auth = relation_get(rid=rid, + unit=uid, attribute='migration_auth_type') if migration_auth == 'ssh': - key = relation_get('ssh_public_key') + key = relation_get(rid=rid, unit=uid, attribute='ssh_public_key') if not key: log('SSH migration set but peer did not publish key.') return - ssh_compute_add(key) + ssh_compute_add(key, rid=rid, uid=uid) index = 0 - for line in ssh_known_hosts_lines(): - relation_set(relation_settings= + for line in ssh_known_hosts_lines(uid=uid): + relation_set(relation_id=rid, relation_settings= {'known_hosts_{}'.format(index): line}) index += 1 - relation_set(known_hosts_max_index=index) + relation_set(relation_id=rid, known_hosts_max_index=index) index = 0 - for line in ssh_authorized_keys_lines(): - relation_set(relation_settings= + for line in ssh_authorized_keys_lines(uid=uid): + relation_set(relation_id=rid, relation_settings= {'authorized_keys_{}'.format(index): line}) index += 1 - relation_set(authorized_keys_max_index=index) - if relation_get('nova_ssh_public_key'): - key = relation_get('nova_ssh_public_key') - ssh_compute_add(key, user='nova') + relation_set(relation_id=rid, authorized_keys_max_index=index) + if relation_get(rid=rid, unit=uid, attribute='nova_ssh_public_key'): + key = relation_get(rid=rid, unit=uid, attribute='nova_ssh_public_key') + ssh_compute_add(key, rid=rid, uid=uid, user='nova') index = 0 - for line in ssh_known_hosts_lines(user='nova'): - relation_set(relation_settings= + for line in ssh_known_hosts_lines(uid=uid, user='nova'): + relation_set(relation_id=rid, relation_settings= {'{}_known_hosts_{}'.format('nova', index): line}) index += 1 - relation_set(relation_settings= + relation_set(relation_id=rid, relation_settings= {'{}_known_hosts_max_index'.format('nova'): index}) index = 0 - for line in ssh_authorized_keys_lines(user='nova'): - relation_set(relation_settings= + for line in ssh_authorized_keys_lines(uid=uid, user='nova'): + relation_set(relation_id=rid, relation_settings= {'{}_authorized_keys_{}'.format('nova', index): line}) index += 1 - relation_set(relation_settings= + relation_set(relation_id=rid, relation_settings= {'{}_authorized_keys_max_index'.format('nova'): index}) @hooks.hook('cloud-compute-relation-departed') -def compute_departed(): - ssh_compute_remove(public_key=relation_get('ssh_public_key')) +def compute_departed(uid=None): + ssh_compute_remove(uid=uid, public_key=relation_get('ssh_public_key')) @hooks.hook('neutron-network-service-relation-joined', diff --git a/hooks/nova_cc_utils.py b/hooks/nova_cc_utils.py index db0bfa62..07758146 100644 --- a/hooks/nova_cc_utils.py +++ b/hooks/nova_cc_utils.py @@ -510,8 +510,11 @@ def keystone_ca_cert_b64(): return b64encode(_in.read()) -def ssh_directory_for_unit(user=None): - remote_service = remote_unit().split('/')[0] +def ssh_directory_for_unit(remunit=None, user=None): + if remunit: + remote_service = remunit.split('/')[0] + else: + remote_service = remote_unit().split('/')[0] if user: remote_service = "{}_{}".format(remote_service, user) _dir = os.path.join(NOVA_SSH_DIR, remote_service) @@ -525,29 +528,29 @@ def ssh_directory_for_unit(user=None): return _dir -def known_hosts(user=None): - return os.path.join(ssh_directory_for_unit(user), 'known_hosts') +def known_hosts(unit=None, user=None): + return os.path.join(ssh_directory_for_unit(unit, user), 'known_hosts') -def authorized_keys(user=None): - return os.path.join(ssh_directory_for_unit(user), 'authorized_keys') +def authorized_keys(unit=None, user=None): + return os.path.join(ssh_directory_for_unit(unit, user), 'authorized_keys') -def ssh_known_host_key(host, user=None): - cmd = ['ssh-keygen', '-f', known_hosts(user), '-H', '-F', host] +def ssh_known_host_key(host, uid=None, user=None): + cmd = ['ssh-keygen', '-f', known_hosts(uid, user), '-H', '-F', host] try: return subprocess.check_output(cmd).strip() except subprocess.CalledProcessError: return None -def remove_known_host(host, user=None): +def remove_known_host(host, uid=None, user=None): log('Removing SSH known host entry for compute host at %s' % host) - cmd = ['ssh-keygen', '-f', known_hosts(user), '-R', host] + cmd = ['ssh-keygen', '-f', known_hosts(uid, user), '-R', host] subprocess.check_call(cmd) -def add_known_host(host, user=None): +def add_known_host(host, uid=None, user=None): '''Add variations of host to a known hosts file.''' cmd = ['ssh-keyscan', '-H', '-t', 'rsa', host] try: @@ -556,33 +559,34 @@ def add_known_host(host, user=None): log('Could not obtain SSH host key from %s' % host, level=ERROR) raise e - current_key = ssh_known_host_key(host, user) + current_key = ssh_known_host_key(host, uid, user) if current_key: if remote_key == current_key: log('Known host key for compute host %s up to date.' % host) return else: - remove_known_host(host, user) + remove_known_host(host, uid, user) log('Adding SSH host key to known hosts for compute node at %s.' % host) - with open(known_hosts(user), 'a') as out: + with open(known_hosts(uid, user), 'a') as out: out.write(remote_key + '\n') -def ssh_authorized_key_exists(public_key, user=None): - with open(authorized_keys(user)) as keys: +def ssh_authorized_key_exists(public_key, uid=None, user=None): + with open(authorized_keys(uid, user)) as keys: return (' %s ' % public_key) in keys.read() -def add_authorized_key(public_key, user=None): - with open(authorized_keys(user), 'a') as keys: +def add_authorized_key(public_key, uid=None, user=None): + with open(authorized_keys(uid, user), 'a') as keys: keys.write(public_key + '\n') -def ssh_compute_add(public_key, user=None): +def ssh_compute_add(public_key, rid=None, uid=None, user=None): # If remote compute node hands us a hostname, ensure we have a # known hosts entry for its IP, hostname and FQDN. - private_address = relation_get('private-address') + private_address = relation_get(rid=rid, unit=uid, + attribute='private-address') hosts = [private_address] if not is_ip(private_address): @@ -594,41 +598,41 @@ def ssh_compute_add(public_key, user=None): hosts.append(hn.split('.')[0]) for host in list(set(hosts)): - if not ssh_known_host_key(host, user): - add_known_host(host, user) + if not ssh_known_host_key(host, uid, user): + add_known_host(host, uid, user) - if not ssh_authorized_key_exists(public_key, user): + if not ssh_authorized_key_exists(public_key, uid, user): log('Saving SSH authorized key for compute host at %s.' % private_address) - add_authorized_key(public_key, user) + add_authorized_key(public_key, uid, user) -def ssh_known_hosts_lines(user=None): +def ssh_known_hosts_lines(uid=None, user=None): known_hosts_list = [] - with open(known_hosts(user)) as hosts: + with open(known_hosts(uid, user)) as hosts: for hosts_line in hosts: if hosts_line.rstrip(): known_hosts_list.append(hosts_line.rstrip()) return(known_hosts_list) -def ssh_authorized_keys_lines(user=None): +def ssh_authorized_keys_lines(uid=None, user=None): authorized_keys_list = [] - with open(authorized_keys(user)) as keys: + with open(authorized_keys(uid, user)) as keys: for authkey_line in keys: if authkey_line.rstrip(): authorized_keys_list.append(authkey_line.rstrip()) return(authorized_keys_list) -def ssh_compute_remove(public_key, user=None): - if not (os.path.isfile(authorized_keys(user)) or - os.path.isfile(known_hosts(user))): +def ssh_compute_remove(public_key, uid=None, user=None): + if not (os.path.isfile(authorized_keys(uid, user)) or + os.path.isfile(known_hosts(uid, user))): return - with open(authorized_keys(user)) as _keys: + with open(authorized_keys(uid, user)) as _keys: keys = [k.strip() for k in _keys.readlines()] if public_key not in keys: @@ -636,7 +640,7 @@ def ssh_compute_remove(public_key, user=None): [keys.remove(key) for key in keys if key == public_key] - with open(authorized_keys(user), 'w') as _keys: + with open(authorized_keys(uid, user), 'w') as _keys: keys = '\n'.join(keys) if not keys.endswith('\n'): keys += '\n' From 3bad8a474619280583390b85891a1c3e1591faa2 Mon Sep 17 00:00:00 2001 From: Louis Bouchard Date: Tue, 27 May 2014 14:15:42 +0200 Subject: [PATCH 09/48] Modified relation_get mechanism in compute_changed Get all relations in one call in use dictionary --- hooks/nova_cc_hooks.py | 18 +++++++++++------- 1 file changed, 11 insertions(+), 7 deletions(-) diff --git a/hooks/nova_cc_hooks.py b/hooks/nova_cc_hooks.py index fee92839..fd809fe1 100755 --- a/hooks/nova_cc_hooks.py +++ b/hooks/nova_cc_hooks.py @@ -341,10 +341,11 @@ def compute_joined(rid=None, remote_restart=False): @hooks.hook('cloud-compute-relation-changed') def compute_changed(rid=None, uid=None): - migration_auth = relation_get(rid=rid, - unit=uid, attribute='migration_auth_type') - if migration_auth == 'ssh': - key = relation_get(rid=rid, unit=uid, attribute='ssh_public_key') + rel_settings = relation_get(rid=rid, unit=uid) + if 'migration_auth_type' not in rel_settings: + return + if rel_settings['migration_auth_type'] == 'ssh': + key = rel_settings['ssh_public_key'] if not key: log('SSH migration set but peer did not publish key.') return @@ -361,9 +362,9 @@ def compute_changed(rid=None, uid=None): {'authorized_keys_{}'.format(index): line}) index += 1 relation_set(relation_id=rid, authorized_keys_max_index=index) - if relation_get(rid=rid, unit=uid, attribute='nova_ssh_public_key'): - key = relation_get(rid=rid, unit=uid, attribute='nova_ssh_public_key') - ssh_compute_add(key, rid=rid, uid=uid, user='nova') + if rel_settings['nova_ssh_public_key']: + ssh_compute_add(rel_settings['nova_ssh_public_key'], + rid=rid, uid=uid, user='nova') index = 0 for line in ssh_known_hosts_lines(uid=uid, user='nova'): relation_set(relation_id=rid, relation_settings= @@ -522,6 +523,9 @@ def upgrade_charm(): amqp_joined(relation_id=r_id) for r_id in relation_ids('identity-service'): identity_joined(rid=r_id) + for r_id in relation_ids('cloud-compute'): + for unit in related_units(r_id): + compute_changed(r_id, unit) def main(): From 10c618f1ff4c4c3e34d590a11115e5a035efae60 Mon Sep 17 00:00:00 2001 From: Louis Bouchard Date: Tue, 27 May 2014 15:43:49 +0200 Subject: [PATCH 10/48] Add missing rel_settings test --- hooks/nova_cc_hooks.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/hooks/nova_cc_hooks.py b/hooks/nova_cc_hooks.py index fd809fe1..7d74e8b1 100755 --- a/hooks/nova_cc_hooks.py +++ b/hooks/nova_cc_hooks.py @@ -362,6 +362,8 @@ def compute_changed(rid=None, uid=None): {'authorized_keys_{}'.format(index): line}) index += 1 relation_set(relation_id=rid, authorized_keys_max_index=index) + if 'nova_ssh_public_key' not in rel_settings: + return if rel_settings['nova_ssh_public_key']: ssh_compute_add(rel_settings['nova_ssh_public_key'], rid=rid, uid=uid, user='nova') From f16dca68e2c8a5a0025f5962ca81cab4d8948629 Mon Sep 17 00:00:00 2001 From: Louis Bouchard Date: Tue, 27 May 2014 15:45:33 +0200 Subject: [PATCH 11/48] Fix unit_test to honour calls with unit & relation ids --- unit_tests/test_nova_cc_hooks.py | 38 ++++++++++++++++---------------- unit_tests/test_nova_cc_utils.py | 20 ++++++++--------- 2 files changed, 29 insertions(+), 29 deletions(-) diff --git a/unit_tests/test_nova_cc_hooks.py b/unit_tests/test_nova_cc_hooks.py index ef052823..1388a5d1 100644 --- a/unit_tests/test_nova_cc_hooks.py +++ b/unit_tests/test_nova_cc_hooks.py @@ -103,38 +103,38 @@ class NovaCCHooksTests(CharmTestCase): self.ssh_authorized_keys_lines.return_value = [ 'auth_0', 'auth_1', 'auth_2'] hooks.compute_changed() - self.ssh_compute_add.assert_called_with('fookey') + self.ssh_compute_add.assert_called_with('fookey', rid=None, uid=None) expected_relations = [ - call(relation_settings={'authorized_keys_0': 'auth_0'}), - call(relation_settings={'authorized_keys_1': 'auth_1'}), - call(relation_settings={'authorized_keys_2': 'auth_2'}), - call(relation_settings={'known_hosts_0': 'k_h_0'}), - call(relation_settings={'known_hosts_1': 'k_h_1'}), - call(relation_settings={'known_hosts_2': 'k_h_2'}), - call(authorized_keys_max_index=3), - call(known_hosts_max_index=3)] + call(relation_settings={'authorized_keys_0': 'auth_0'}, relation_id=None), + call(relation_settings={'authorized_keys_1': 'auth_1'}, relation_id=None), + call(relation_settings={'authorized_keys_2': 'auth_2'}, relation_id=None), + call(relation_settings={'known_hosts_0': 'k_h_0'}, relation_id=None), + call(relation_settings={'known_hosts_1': 'k_h_1'}, relation_id=None), + call(relation_settings={'known_hosts_2': 'k_h_2'}, relation_id=None), + call(authorized_keys_max_index=3, relation_id=None), + call(known_hosts_max_index=3, relation_id=None)] self.assertEquals(sorted(self.relation_set.call_args_list), sorted(expected_relations)) def test_compute_changed_nova_public_key(self): self.test_relation.set({ - 'nova_ssh_public_key': 'fookey', + 'migration_auth_type': 'sasl', 'nova_ssh_public_key': 'fookey', 'private-address': '10.0.0.1'}) self.ssh_known_hosts_lines.return_value = [ 'k_h_0', 'k_h_1', 'k_h_2'] self.ssh_authorized_keys_lines.return_value = [ 'auth_0', 'auth_1', 'auth_2'] hooks.compute_changed() - self.ssh_compute_add.assert_called_with('fookey', user='nova') + self.ssh_compute_add.assert_called_with('fookey', user='nova', rid=None, uid=None) expected_relations = [ - call(relation_settings={'nova_authorized_keys_0': 'auth_0'}), - call(relation_settings={'nova_authorized_keys_1': 'auth_1'}), - call(relation_settings={'nova_authorized_keys_2': 'auth_2'}), - call(relation_settings={'nova_known_hosts_0': 'k_h_0'}), - call(relation_settings={'nova_known_hosts_1': 'k_h_1'}), - call(relation_settings={'nova_known_hosts_2': 'k_h_2'}), - call(relation_settings={'nova_known_hosts_max_index': 3}), - call(relation_settings={'nova_authorized_keys_max_index': 3})] + call(relation_settings={'nova_authorized_keys_0': 'auth_0'}, relation_id=None), + call(relation_settings={'nova_authorized_keys_1': 'auth_1'}, relation_id=None), + call(relation_settings={'nova_authorized_keys_2': 'auth_2'}, relation_id=None), + call(relation_settings={'nova_known_hosts_0': 'k_h_0'}, relation_id=None), + call(relation_settings={'nova_known_hosts_1': 'k_h_1'}, relation_id=None), + call(relation_settings={'nova_known_hosts_2': 'k_h_2'}, relation_id=None), + call(relation_settings={'nova_known_hosts_max_index': 3}, relation_id=None), + call(relation_settings={'nova_authorized_keys_max_index': 3}, relation_id=None)] self.assertEquals(sorted(self.relation_set.call_args_list), sorted(expected_relations)) diff --git a/unit_tests/test_nova_cc_utils.py b/unit_tests/test_nova_cc_utils.py index b148f448..9ddeb3a7 100644 --- a/unit_tests/test_nova_cc_utils.py +++ b/unit_tests/test_nova_cc_utils.py @@ -321,8 +321,8 @@ class NovaCCUtilsTests(CharmTestCase): check_output.return_value = 'fookey' host_key.return_value = 'fookey_old' with patch_open() as (_open, _file): - utils.add_known_host('foohost') - rm.assert_called_with('foohost', None) + utils.add_known_host('foohost', None, None) + rm.assert_called_with('foohost', None, None) @patch.object(utils, 'known_hosts') @patch.object(utils, 'remove_known_host') @@ -355,19 +355,19 @@ class NovaCCUtilsTests(CharmTestCase): def test_known_hosts(self, ssh_dir): ssh_dir.return_value = '/tmp/foo' self.assertEquals(utils.known_hosts(), '/tmp/foo/known_hosts') - ssh_dir.assert_called_with(None) + ssh_dir.assert_called_with(None, None) self.assertEquals(utils.known_hosts('bar'), '/tmp/foo/known_hosts') - ssh_dir.assert_called_with('bar') + ssh_dir.assert_called_with('bar', None) @patch.object(utils, 'ssh_directory_for_unit') def test_authorized_keys(self, ssh_dir): ssh_dir.return_value = '/tmp/foo' self.assertEquals(utils.authorized_keys(), '/tmp/foo/authorized_keys') - ssh_dir.assert_called_with(None) + ssh_dir.assert_called_with(None, None) self.assertEquals( utils.authorized_keys('bar'), '/tmp/foo/authorized_keys') - ssh_dir.assert_called_with('bar') + ssh_dir.assert_called_with('bar', None) @patch.object(utils, 'known_hosts') @patch('subprocess.check_call') @@ -461,9 +461,9 @@ class NovaCCUtilsTests(CharmTestCase): _check_output.assert_called_with( ['ssh-keygen', '-f', '/foo/known_hosts', '-H', '-F', 'test']) - _known_hosts.assert_called_with(None) + _known_hosts.assert_called_with(None, None) utils.ssh_known_host_key('test', 'bar') - _known_hosts.assert_called_with('bar') + _known_hosts.assert_called_with('bar', None) @patch.object(utils, 'known_hosts') @patch('subprocess.check_call') @@ -473,9 +473,9 @@ class NovaCCUtilsTests(CharmTestCase): _check_call.assert_called_with( ['ssh-keygen', '-f', '/foo/known_hosts', '-R', 'test']) - _known_hosts.assert_called_with(None) + _known_hosts.assert_called_with(None, None) utils.remove_known_host('test', 'bar') - _known_hosts.assert_called_with('bar') + _known_hosts.assert_called_with('bar', None) @patch('subprocess.check_output') def test_migrate_database(self, check_output): From 909871b19ee08ecc63eb3e39f64db22e117548fa Mon Sep 17 00:00:00 2001 From: Louis Bouchard Date: Tue, 27 May 2014 16:00:56 +0200 Subject: [PATCH 12/48] More PEP8 cleanup --- unit_tests/test_nova_cc_hooks.py | 45 +++++++++++++++++++++----------- 1 file changed, 30 insertions(+), 15 deletions(-) diff --git a/unit_tests/test_nova_cc_hooks.py b/unit_tests/test_nova_cc_hooks.py index 1388a5d1..894c66fb 100644 --- a/unit_tests/test_nova_cc_hooks.py +++ b/unit_tests/test_nova_cc_hooks.py @@ -105,12 +105,18 @@ class NovaCCHooksTests(CharmTestCase): hooks.compute_changed() self.ssh_compute_add.assert_called_with('fookey', rid=None, uid=None) expected_relations = [ - call(relation_settings={'authorized_keys_0': 'auth_0'}, relation_id=None), - call(relation_settings={'authorized_keys_1': 'auth_1'}, relation_id=None), - call(relation_settings={'authorized_keys_2': 'auth_2'}, relation_id=None), - call(relation_settings={'known_hosts_0': 'k_h_0'}, relation_id=None), - call(relation_settings={'known_hosts_1': 'k_h_1'}, relation_id=None), - call(relation_settings={'known_hosts_2': 'k_h_2'}, relation_id=None), + call(relation_settings={'authorized_keys_0': 'auth_0'}, + relation_id=None), + call(relation_settings={'authorized_keys_1': 'auth_1'}, + relation_id=None), + call(relation_settings={'authorized_keys_2': 'auth_2'}, + relation_id=None), + call(relation_settings={'known_hosts_0': 'k_h_0'}, + relation_id=None), + call(relation_settings={'known_hosts_1': 'k_h_1'}, + relation_id=None), + call(relation_settings={'known_hosts_2': 'k_h_2'}, + relation_id=None), call(authorized_keys_max_index=3, relation_id=None), call(known_hosts_max_index=3, relation_id=None)] self.assertEquals(sorted(self.relation_set.call_args_list), @@ -125,16 +131,25 @@ class NovaCCHooksTests(CharmTestCase): self.ssh_authorized_keys_lines.return_value = [ 'auth_0', 'auth_1', 'auth_2'] hooks.compute_changed() - self.ssh_compute_add.assert_called_with('fookey', user='nova', rid=None, uid=None) + self.ssh_compute_add.assert_called_with('fookey', user='nova', + rid=None, uid=None) expected_relations = [ - call(relation_settings={'nova_authorized_keys_0': 'auth_0'}, relation_id=None), - call(relation_settings={'nova_authorized_keys_1': 'auth_1'}, relation_id=None), - call(relation_settings={'nova_authorized_keys_2': 'auth_2'}, relation_id=None), - call(relation_settings={'nova_known_hosts_0': 'k_h_0'}, relation_id=None), - call(relation_settings={'nova_known_hosts_1': 'k_h_1'}, relation_id=None), - call(relation_settings={'nova_known_hosts_2': 'k_h_2'}, relation_id=None), - call(relation_settings={'nova_known_hosts_max_index': 3}, relation_id=None), - call(relation_settings={'nova_authorized_keys_max_index': 3}, relation_id=None)] + call(relation_settings={'nova_authorized_keys_0': 'auth_0'}, + relation_id=None), + call(relation_settings={'nova_authorized_keys_1': 'auth_1'}, + relation_id=None), + call(relation_settings={'nova_authorized_keys_2': 'auth_2'}, + relation_id=None), + call(relation_settings={'nova_known_hosts_0': 'k_h_0'}, + relation_id=None), + call(relation_settings={'nova_known_hosts_1': 'k_h_1'}, + relation_id=None), + call(relation_settings={'nova_known_hosts_2': 'k_h_2'}, + relation_id=None), + call(relation_settings={'nova_known_hosts_max_index': 3}, + relation_id=None), + call(relation_settings={'nova_authorized_keys_max_index': 3}, + relation_id=None)] self.assertEquals(sorted(self.relation_set.call_args_list), sorted(expected_relations)) From 10c27c367072b13bc2cd1ecae5af627b63351f3d Mon Sep 17 00:00:00 2001 From: Louis Bouchard Date: Tue, 24 Jun 2014 12:53:08 +0200 Subject: [PATCH 13/48] Implement changes following Merge Proposal - Replaced uid by unit - Rolled back change to compute_departed - Use rel_settings.get() --- hooks/nova_cc_hooks.py | 22 ++++++++-------- hooks/nova_cc_utils.py | 60 +++++++++++++++++++++--------------------- 2 files changed, 41 insertions(+), 41 deletions(-) diff --git a/hooks/nova_cc_hooks.py b/hooks/nova_cc_hooks.py index 7d74e8b1..771918fa 100755 --- a/hooks/nova_cc_hooks.py +++ b/hooks/nova_cc_hooks.py @@ -340,24 +340,24 @@ def compute_joined(rid=None, remote_restart=False): @hooks.hook('cloud-compute-relation-changed') -def compute_changed(rid=None, uid=None): - rel_settings = relation_get(rid=rid, unit=uid) +def compute_changed(rid=None, unit=None): + rel_settings = relation_get(rid=rid, unit=unit) if 'migration_auth_type' not in rel_settings: return if rel_settings['migration_auth_type'] == 'ssh': - key = rel_settings['ssh_public_key'] + key = rel_settings.get('ssh_public_key') if not key: log('SSH migration set but peer did not publish key.') return - ssh_compute_add(key, rid=rid, uid=uid) + ssh_compute_add(key, rid=rid, unit=unit) index = 0 - for line in ssh_known_hosts_lines(uid=uid): + for line in ssh_known_hosts_lines(unit=unit): relation_set(relation_id=rid, relation_settings= {'known_hosts_{}'.format(index): line}) index += 1 relation_set(relation_id=rid, known_hosts_max_index=index) index = 0 - for line in ssh_authorized_keys_lines(uid=uid): + for line in ssh_authorized_keys_lines(unit=unit): relation_set(relation_id=rid, relation_settings= {'authorized_keys_{}'.format(index): line}) index += 1 @@ -366,16 +366,16 @@ def compute_changed(rid=None, uid=None): return if rel_settings['nova_ssh_public_key']: ssh_compute_add(rel_settings['nova_ssh_public_key'], - rid=rid, uid=uid, user='nova') + rid=rid, unit=unit, user='nova') index = 0 - for line in ssh_known_hosts_lines(uid=uid, user='nova'): + for line in ssh_known_hosts_lines(unit=unit, user='nova'): relation_set(relation_id=rid, relation_settings= {'{}_known_hosts_{}'.format('nova', index): line}) index += 1 relation_set(relation_id=rid, relation_settings= {'{}_known_hosts_max_index'.format('nova'): index}) index = 0 - for line in ssh_authorized_keys_lines(uid=uid, user='nova'): + for line in ssh_authorized_keys_lines(unit=unit, user='nova'): relation_set(relation_id=rid, relation_settings= {'{}_authorized_keys_{}'.format('nova', index): line}) index += 1 @@ -384,8 +384,8 @@ def compute_changed(rid=None, uid=None): @hooks.hook('cloud-compute-relation-departed') -def compute_departed(uid=None): - ssh_compute_remove(uid=uid, public_key=relation_get('ssh_public_key')) +def compute_departed(): + ssh_compute_remove(public_key=relation_get('ssh_public_key')) @hooks.hook('neutron-network-service-relation-joined', diff --git a/hooks/nova_cc_utils.py b/hooks/nova_cc_utils.py index 07758146..9bc0da77 100644 --- a/hooks/nova_cc_utils.py +++ b/hooks/nova_cc_utils.py @@ -510,9 +510,9 @@ def keystone_ca_cert_b64(): return b64encode(_in.read()) -def ssh_directory_for_unit(remunit=None, user=None): - if remunit: - remote_service = remunit.split('/')[0] +def ssh_directory_for_unit(unit=None, user=None): + if unit: + remote_service = unit.split('/')[0] else: remote_service = remote_unit().split('/')[0] if user: @@ -536,21 +536,21 @@ def authorized_keys(unit=None, user=None): return os.path.join(ssh_directory_for_unit(unit, user), 'authorized_keys') -def ssh_known_host_key(host, uid=None, user=None): - cmd = ['ssh-keygen', '-f', known_hosts(uid, user), '-H', '-F', host] +def ssh_known_host_key(host, unit=None, user=None): + cmd = ['ssh-keygen', '-f', known_hosts(unit, user), '-H', '-F', host] try: return subprocess.check_output(cmd).strip() except subprocess.CalledProcessError: return None -def remove_known_host(host, uid=None, user=None): +def remove_known_host(host, unit=None, user=None): log('Removing SSH known host entry for compute host at %s' % host) - cmd = ['ssh-keygen', '-f', known_hosts(uid, user), '-R', host] + cmd = ['ssh-keygen', '-f', known_hosts(unit, user), '-R', host] subprocess.check_call(cmd) -def add_known_host(host, uid=None, user=None): +def add_known_host(host, unit=None, user=None): '''Add variations of host to a known hosts file.''' cmd = ['ssh-keyscan', '-H', '-t', 'rsa', host] try: @@ -559,33 +559,33 @@ def add_known_host(host, uid=None, user=None): log('Could not obtain SSH host key from %s' % host, level=ERROR) raise e - current_key = ssh_known_host_key(host, uid, user) + current_key = ssh_known_host_key(host, unit, user) if current_key: if remote_key == current_key: log('Known host key for compute host %s up to date.' % host) return else: - remove_known_host(host, uid, user) + remove_known_host(host, unit, user) log('Adding SSH host key to known hosts for compute node at %s.' % host) - with open(known_hosts(uid, user), 'a') as out: + with open(known_hosts(unit, user), 'a') as out: out.write(remote_key + '\n') -def ssh_authorized_key_exists(public_key, uid=None, user=None): - with open(authorized_keys(uid, user)) as keys: +def ssh_authorized_key_exists(public_key, unit=None, user=None): + with open(authorized_keys(unit, user)) as keys: return (' %s ' % public_key) in keys.read() -def add_authorized_key(public_key, uid=None, user=None): - with open(authorized_keys(uid, user), 'a') as keys: +def add_authorized_key(public_key, unit=None, user=None): + with open(authorized_keys(unit, user), 'a') as keys: keys.write(public_key + '\n') -def ssh_compute_add(public_key, rid=None, uid=None, user=None): +def ssh_compute_add(public_key, rid=None, unit=None, user=None): # If remote compute node hands us a hostname, ensure we have a # known hosts entry for its IP, hostname and FQDN. - private_address = relation_get(rid=rid, unit=uid, + private_address = relation_get(rid=rid, unit=unit, attribute='private-address') hosts = [private_address] @@ -598,41 +598,41 @@ def ssh_compute_add(public_key, rid=None, uid=None, user=None): hosts.append(hn.split('.')[0]) for host in list(set(hosts)): - if not ssh_known_host_key(host, uid, user): - add_known_host(host, uid, user) + if not ssh_known_host_key(host, unit, user): + add_known_host(host, unit, user) - if not ssh_authorized_key_exists(public_key, uid, user): + if not ssh_authorized_key_exists(public_key, unit, user): log('Saving SSH authorized key for compute host at %s.' % private_address) - add_authorized_key(public_key, uid, user) + add_authorized_key(public_key, unit, user) -def ssh_known_hosts_lines(uid=None, user=None): +def ssh_known_hosts_lines(unit=None, user=None): known_hosts_list = [] - with open(known_hosts(uid, user)) as hosts: + with open(known_hosts(unit, user)) as hosts: for hosts_line in hosts: if hosts_line.rstrip(): known_hosts_list.append(hosts_line.rstrip()) return(known_hosts_list) -def ssh_authorized_keys_lines(uid=None, user=None): +def ssh_authorized_keys_lines(unit=None, user=None): authorized_keys_list = [] - with open(authorized_keys(uid, user)) as keys: + with open(authorized_keys(unit, user)) as keys: for authkey_line in keys: if authkey_line.rstrip(): authorized_keys_list.append(authkey_line.rstrip()) return(authorized_keys_list) -def ssh_compute_remove(public_key, uid=None, user=None): - if not (os.path.isfile(authorized_keys(uid, user)) or - os.path.isfile(known_hosts(uid, user))): +def ssh_compute_remove(public_key, unit=None, user=None): + if not (os.path.isfile(authorized_keys(unit, user)) or + os.path.isfile(known_hosts(unit, user))): return - with open(authorized_keys(uid, user)) as _keys: + with open(authorized_keys(unit, user)) as _keys: keys = [k.strip() for k in _keys.readlines()] if public_key not in keys: @@ -640,7 +640,7 @@ def ssh_compute_remove(public_key, uid=None, user=None): [keys.remove(key) for key in keys if key == public_key] - with open(authorized_keys(uid, user), 'w') as _keys: + with open(authorized_keys(unit, user), 'w') as _keys: keys = '\n'.join(keys) if not keys.endswith('\n'): keys += '\n' From f9e489c86cd89bfe69783037944e285b3bcb51a6 Mon Sep 17 00:00:00 2001 From: Corey Bryant Date: Tue, 24 Jun 2014 17:19:15 +0000 Subject: [PATCH 14/48] Move charm-helpers.yaml to charm-helpers-hooks.yaml and add charm-helpers-tests.yaml. --- Makefile | 3 ++- charm-helpers.yaml => charm-helpers-hooks.yaml | 0 charm-helpers-tests.yaml | 5 +++++ 3 files changed, 7 insertions(+), 1 deletion(-) rename charm-helpers.yaml => charm-helpers-hooks.yaml (100%) create mode 100644 charm-helpers-tests.yaml diff --git a/Makefile b/Makefile index 2ec849dc..7d92e615 100644 --- a/Makefile +++ b/Makefile @@ -10,7 +10,8 @@ test: @$(PYTHON) /usr/bin/nosetests --nologcapture --with-coverage unit_tests sync: - @charm-helper-sync -c charm-helpers.yaml + @charm-helper-sync -c charm-helpers-hooks.yaml + @charm-helper-sync -c charm-helpers-tests.yaml publish: lint test bzr push lp:charms/nova-cloud-controller diff --git a/charm-helpers.yaml b/charm-helpers-hooks.yaml similarity index 100% rename from charm-helpers.yaml rename to charm-helpers-hooks.yaml diff --git a/charm-helpers-tests.yaml b/charm-helpers-tests.yaml new file mode 100644 index 00000000..48b12f6f --- /dev/null +++ b/charm-helpers-tests.yaml @@ -0,0 +1,5 @@ +branch: lp:charm-helpers +destination: tests/charmhelpers +include: + - contrib.amulet + - contrib.openstack.amulet From 021c07bdd367797f86f5af40af005441f321422a Mon Sep 17 00:00:00 2001 From: James Page Date: Wed, 25 Jun 2014 13:04:44 +0100 Subject: [PATCH 15/48] Sync network-splits charm-helper --- charm-helpers.yaml | 3 +- .../charmhelpers/contrib/network/__init__.py | 0 hooks/charmhelpers/contrib/network/ip.py | 69 +++++++++++ .../charmhelpers/contrib/openstack/context.py | 6 +- hooks/charmhelpers/core/fstab.py | 114 ++++++++++++++++++ hooks/charmhelpers/core/host.py | 26 +++- hooks/charmhelpers/fetch/bzrurl.py | 3 +- 7 files changed, 211 insertions(+), 10 deletions(-) create mode 100644 hooks/charmhelpers/contrib/network/__init__.py create mode 100644 hooks/charmhelpers/contrib/network/ip.py create mode 100644 hooks/charmhelpers/core/fstab.py diff --git a/charm-helpers.yaml b/charm-helpers.yaml index 35e5834f..9925e2b8 100644 --- a/charm-helpers.yaml +++ b/charm-helpers.yaml @@ -1,4 +1,4 @@ -branch: lp:charm-helpers +branch: lp:~james-page/charm-helpers/network-splits destination: hooks/charmhelpers include: - core @@ -8,3 +8,4 @@ include: - contrib.hahelpers: - apache - payload.execd + - contrib.network.ip diff --git a/hooks/charmhelpers/contrib/network/__init__.py b/hooks/charmhelpers/contrib/network/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/hooks/charmhelpers/contrib/network/ip.py b/hooks/charmhelpers/contrib/network/ip.py new file mode 100644 index 00000000..44c7c975 --- /dev/null +++ b/hooks/charmhelpers/contrib/network/ip.py @@ -0,0 +1,69 @@ +import sys + +from charmhelpers.fetch import apt_install +from charmhelpers.core.hookenv import ( + ERROR, log, +) + +try: + import netifaces +except ImportError: + apt_install('python-netifaces') + import netifaces + +try: + import netaddr +except ImportError: + apt_install('python-netaddr') + import netaddr + + +def _validate_cidr(network): + try: + netaddr.IPNetwork(network) + except (netaddr.core.AddrFormatError, ValueError): + raise ValueError("Network (%s) is not in CIDR presentation format" % + network) + + +def get_address_in_network(network, fallback=None, fatal=False): + """ + Get an IPv4 address within the network from the host. + + Args: + network (str): CIDR presentation format. For example, + '192.168.1.0/24'. + fallback (str): If no address is found, return fallback. + fatal (boolean): If no address is found, fallback is not + set and fatal is True then exit(1). + """ + + def not_found_error_out(): + log("No IP address found in network: %s" % network, + level=ERROR) + sys.exit(1) + + if network is None: + if fallback is not None: + return fallback + else: + if fatal: + not_found_error_out() + + _validate_cidr(network) + for iface in netifaces.interfaces(): + addresses = netifaces.ifaddresses(iface) + if netifaces.AF_INET in addresses: + addr = addresses[netifaces.AF_INET][0]['addr'] + netmask = addresses[netifaces.AF_INET][0]['netmask'] + cidr = netaddr.IPNetwork("%s/%s" % (addr, netmask)) + if cidr in netaddr.IPNetwork(network): + return str(cidr.ip) + + if fallback is not None: + return fallback + + if fatal: + not_found_error_out() + + return None diff --git a/hooks/charmhelpers/contrib/openstack/context.py b/hooks/charmhelpers/contrib/openstack/context.py index 474d51ea..eaeb7eb2 100644 --- a/hooks/charmhelpers/contrib/openstack/context.py +++ b/hooks/charmhelpers/contrib/openstack/context.py @@ -332,10 +332,12 @@ class CephContext(OSContextGenerator): use_syslog = str(config('use-syslog')).lower() for rid in relation_ids('ceph'): for unit in related_units(rid): - mon_hosts.append(relation_get('private-address', rid=rid, - unit=unit)) auth = relation_get('auth', rid=rid, unit=unit) key = relation_get('key', rid=rid, unit=unit) + ceph_addr = \ + relation_get('ceph-public-address', rid=rid, unit=unit) or \ + relation_get('private-address', rid=rid, unit=unit) + mon_hosts.append(ceph_addr) ctxt = { 'mon_hosts': ' '.join(mon_hosts), diff --git a/hooks/charmhelpers/core/fstab.py b/hooks/charmhelpers/core/fstab.py new file mode 100644 index 00000000..cdd72616 --- /dev/null +++ b/hooks/charmhelpers/core/fstab.py @@ -0,0 +1,114 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- + +__author__ = 'Jorge Niedbalski R. ' + +import os + + +class Fstab(file): + """This class extends file in order to implement a file reader/writer + for file `/etc/fstab` + """ + + class Entry(object): + """Entry class represents a non-comment line on the `/etc/fstab` file + """ + def __init__(self, device, mountpoint, filesystem, + options, d=0, p=0): + self.device = device + self.mountpoint = mountpoint + self.filesystem = filesystem + + if not options: + options = "defaults" + + self.options = options + self.d = d + self.p = p + + def __eq__(self, o): + return str(self) == str(o) + + def __str__(self): + return "{} {} {} {} {} {}".format(self.device, + self.mountpoint, + self.filesystem, + self.options, + self.d, + self.p) + + DEFAULT_PATH = os.path.join(os.path.sep, 'etc', 'fstab') + + def __init__(self, path=None): + if path: + self._path = path + else: + self._path = self.DEFAULT_PATH + file.__init__(self, self._path, 'r+') + + def _hydrate_entry(self, line): + return Fstab.Entry(*filter( + lambda x: x not in ('', None), + line.strip("\n").split(" "))) + + @property + def entries(self): + self.seek(0) + for line in self.readlines(): + try: + if not line.startswith("#"): + yield self._hydrate_entry(line) + except ValueError: + pass + + def get_entry_by_attr(self, attr, value): + for entry in self.entries: + e_attr = getattr(entry, attr) + if e_attr == value: + return entry + return None + + def add_entry(self, entry): + if self.get_entry_by_attr('device', entry.device): + return False + + self.write(str(entry) + '\n') + self.truncate() + return entry + + def remove_entry(self, entry): + self.seek(0) + + lines = self.readlines() + + found = False + for index, line in enumerate(lines): + if not line.startswith("#"): + if self._hydrate_entry(line) == entry: + found = True + break + + if not found: + return False + + lines.remove(line) + + self.seek(0) + self.write(''.join(lines)) + self.truncate() + return True + + @classmethod + def remove_by_mountpoint(cls, mountpoint, path=None): + fstab = cls(path=path) + entry = fstab.get_entry_by_attr('mountpoint', mountpoint) + if entry: + return fstab.remove_entry(entry) + return False + + @classmethod + def add(cls, device, mountpoint, filesystem, options=None, path=None): + return cls(path=path).add_entry(Fstab.Entry(device, + mountpoint, filesystem, + options=options)) diff --git a/hooks/charmhelpers/core/host.py b/hooks/charmhelpers/core/host.py index 186147f6..46bfd36a 100644 --- a/hooks/charmhelpers/core/host.py +++ b/hooks/charmhelpers/core/host.py @@ -17,6 +17,7 @@ import apt_pkg from collections import OrderedDict from hookenv import log +from fstab import Fstab def service_start(service_name): @@ -35,7 +36,8 @@ def service_restart(service_name): def service_reload(service_name, restart_on_failure=False): - """Reload a system service, optionally falling back to restart if reload fails""" + """Reload a system service, optionally falling back to restart if + reload fails""" service_result = service('reload', service_name) if not service_result and restart_on_failure: service_result = service('restart', service_name) @@ -144,7 +146,19 @@ def write_file(path, content, owner='root', group='root', perms=0444): target.write(content) -def mount(device, mountpoint, options=None, persist=False): +def fstab_remove(mp): + """Remove the given mountpoint entry from /etc/fstab + """ + return Fstab.remove_by_mountpoint(mp) + + +def fstab_add(dev, mp, fs, options=None): + """Adds the given device entry to the /etc/fstab file + """ + return Fstab.add(dev, mp, fs, options=options) + + +def mount(device, mountpoint, options=None, persist=False, filesystem="ext3"): """Mount a filesystem at a particular mountpoint""" cmd_args = ['mount'] if options is not None: @@ -155,9 +169,9 @@ def mount(device, mountpoint, options=None, persist=False): except subprocess.CalledProcessError, e: log('Error mounting {} at {}\n{}'.format(device, mountpoint, e.output)) return False + if persist: - # TODO: update fstab - pass + return fstab_add(device, mountpoint, filesystem, options=options) return True @@ -169,9 +183,9 @@ def umount(mountpoint, persist=False): except subprocess.CalledProcessError, e: log('Error unmounting {}\n{}'.format(mountpoint, e.output)) return False + if persist: - # TODO: update fstab - pass + return fstab_remove(mountpoint) return True diff --git a/hooks/charmhelpers/fetch/bzrurl.py b/hooks/charmhelpers/fetch/bzrurl.py index db5dd9a3..0e580e47 100644 --- a/hooks/charmhelpers/fetch/bzrurl.py +++ b/hooks/charmhelpers/fetch/bzrurl.py @@ -39,7 +39,8 @@ class BzrUrlFetchHandler(BaseFetchHandler): def install(self, source): url_parts = self.parse_url(source) branch_name = url_parts.path.strip("/").split("/")[-1] - dest_dir = os.path.join(os.environ.get('CHARM_DIR'), "fetched", branch_name) + dest_dir = os.path.join(os.environ.get('CHARM_DIR'), "fetched", + branch_name) if not os.path.exists(dest_dir): mkdir(dest_dir, perms=0755) try: From 96ca29d0f2fa9806a11d904e69185b5fab05b989 Mon Sep 17 00:00:00 2001 From: James Page Date: Wed, 25 Jun 2014 16:24:48 +0100 Subject: [PATCH 16/48] Add support for internal and public networks --- config.yaml | 17 ++++++++++++ hooks/nova_cc_hooks.py | 25 +++++++++++++---- hooks/nova_cc_utils.py | 63 +++++++++++++++++++++++++----------------- 3 files changed, 74 insertions(+), 31 deletions(-) diff --git a/config.yaml b/config.yaml index 4758fd46..6c81a8a9 100644 --- a/config.yaml +++ b/config.yaml @@ -165,3 +165,20 @@ options: description: | This is uuid of the default NVP/NSX L3 Gateway Service. # end of NVP/NSX configuration + # Network configuration options + # by default all access is over 'private-address' + os-internal-network: + type: string + description: | + The IP address and netmask of the OpenStack Internal network (e.g., + 192.168.0.0/24) + . + This network will be used for admin and internal endpoints. + os-public-network: + type: string + description: | + The IP address and netmask of the OpenStack Public network (e.g., + 192.168.0.0/24) + . + This network will be used for public endpoints. + diff --git a/hooks/nova_cc_hooks.py b/hooks/nova_cc_hooks.py index c668a1d6..823fa4d7 100755 --- a/hooks/nova_cc_hooks.py +++ b/hooks/nova_cc_hooks.py @@ -80,6 +80,7 @@ from charmhelpers.contrib.hahelpers.cluster import ( ) from charmhelpers.payload.execd import execd_preinstall +from charmhelpers.contrib.network.ip import get_address_in_network hooks = Hooks() CONFIGS = register_configs() @@ -110,6 +111,8 @@ def config_changed(): save_script_rc() configure_https() CONFIGS.write_all() + for r_id in relation_ids('identity-service'): + identity_joined(rid=r_id) @hooks.hook('amqp-relation-joined') @@ -230,8 +233,14 @@ def image_service_changed(): def identity_joined(rid=None): if not eligible_leader(CLUSTER_RES): return - base_url = canonical_url(CONFIGS) - relation_set(relation_id=rid, **determine_endpoints(base_url)) + public_url = canonical_url(CONFIGS, + address=get_address_in_network(config('os-public-network'), + unit_get('public-address'))) + internal_url = canonical_url(CONFIGS, + address=get_address_in_network(config('os-internal-network'), + unit_get('private-address'))) + relation_set(relation_id=rid, **determine_endpoints(public_url, + internal_url)) @hooks.hook('identity-service-relation-changed') @@ -319,8 +328,10 @@ def neutron_settings(): 'quantum_plugin': neutron_plugin(), 'region': config('region'), 'quantum_security_groups': config('quantum-security-groups'), - 'quantum_url': (canonical_url(CONFIGS) + ':' + - str(api_port('neutron-server'))), + 'quantum_url': "{}:{}".format(canonical_url(CONFIGS, + get_address_in_network(config('os-internal-network'), + unit_get('private-address'))), + str(api_port('neutron-server'))), }) neutron_url = urlparse(neutron_settings['quantum_url']) neutron_settings['quantum_host'] = neutron_url.hostname @@ -495,8 +506,10 @@ def nova_vmware_relation_joined(rid=None): rel_settings.update({ 'quantum_plugin': neutron_plugin(), 'quantum_security_groups': config('quantum-security-groups'), - 'quantum_url': (canonical_url(CONFIGS) + ':' + - str(api_port('neutron-server')))}) + 'quantum_url': "{}:{}".format(canonical_url(CONFIGS, + get_address_in_network(config('os-internal-network'), + unit_get('private-address'))), + str(api_port('neutron-server')))}) relation_set(relation_id=rid, **rel_settings) diff --git a/hooks/nova_cc_utils.py b/hooks/nova_cc_utils.py index bb07ddf3..b3e8da36 100644 --- a/hooks/nova_cc_utils.py +++ b/hooks/nova_cc_utils.py @@ -643,50 +643,63 @@ def ssh_compute_remove(public_key, user=None): _keys.write(keys) -def determine_endpoints(url): +def determine_endpoints(public_url, internal_url): '''Generates a dictionary containing all relevant endpoints to be passed to keystone as relation settings.''' region = config('region') os_rel = os_release('nova-common') if os_rel >= 'grizzly': - nova_url = ('%s:%s/v2/$(tenant_id)s' % - (url, api_port('nova-api-os-compute'))) + nova_public_url = ('%s:%s/v2/$(tenant_id)s' % + (public_url, api_port('nova-api-os-compute'))) + nova_internal_url = ('%s:%s/v2/$(tenant_id)s' % + (internal_url, api_port('nova-api-os-compute'))) else: - nova_url = ('%s:%s/v1.1/$(tenant_id)s' % - (url, api_port('nova-api-os-compute'))) - ec2_url = '%s:%s/services/Cloud' % (url, api_port('nova-api-ec2')) - nova_volume_url = ('%s:%s/v1/$(tenant_id)s' % - (url, api_port('nova-api-os-compute'))) - neutron_url = '%s:%s' % (url, api_port('neutron-server')) - s3_url = '%s:%s' % (url, api_port('nova-objectstore')) + nova_public_url = ('%s:%s/v1.1/$(tenant_id)s' % + (public_url, api_port('nova-api-os-compute'))) + nova_internal_url = ('%s:%s/v1.1/$(tenant_id)s' % + (internal_url, api_port('nova-api-os-compute'))) + + ec2_public_url = '%s:%s/services/Cloud' % (public_url, api_port('nova-api-ec2')) + ec2_internal_url = '%s:%s/services/Cloud' % (internal_url, api_port('nova-api-ec2')) + + nova_volume_public_url = ('%s:%s/v1/$(tenant_id)s' % + (public_url, api_port('nova-api-os-compute'))) + nova_volume_internal_url = ('%s:%s/v1/$(tenant_id)s' % + (internal_url, api_port('nova-api-os-compute'))) + + neutron_public_url = '%s:%s' % (public_url, api_port('neutron-server')) + neutron_internal_url = '%s:%s' % (internal_url, api_port('neutron-server')) + + s3_public_url = '%s:%s' % (public_url, api_port('nova-objectstore')) + s3_internal_url = '%s:%s' % (internal_url, api_port('nova-objectstore')) # the base endpoints endpoints = { 'nova_service': 'nova', 'nova_region': region, - 'nova_public_url': nova_url, - 'nova_admin_url': nova_url, - 'nova_internal_url': nova_url, + 'nova_public_url': nova_public_url, + 'nova_admin_url': nova_internal_url, + 'nova_internal_url': nova_internal_url, 'ec2_service': 'ec2', 'ec2_region': region, - 'ec2_public_url': ec2_url, - 'ec2_admin_url': ec2_url, - 'ec2_internal_url': ec2_url, + 'ec2_public_url': ec2_public_url, + 'ec2_admin_url': ec2_internal_url, + 'ec2_internal_url': ec2_internal_url, 's3_service': 's3', 's3_region': region, - 's3_public_url': s3_url, - 's3_admin_url': s3_url, - 's3_internal_url': s3_url, + 's3_public_url': s3_public_url, + 's3_admin_url': s3_internal_url, + 's3_internal_url': s3_internal_url, } if relation_ids('nova-volume-service'): endpoints.update({ 'nova-volume_service': 'nova-volume', 'nova-volume_region': region, - 'nova-volume_public_url': nova_volume_url, - 'nova-volume_admin_url': nova_volume_url, - 'nova-volume_internal_url': nova_volume_url, + 'nova-volume_public_url': nova_volume_public_url, + 'nova-volume_admin_url': nova_volume_internal_url, + 'nova-volume_internal_url': nova_volume_internal_url, }) # XXX: Keep these relations named quantum_*?? @@ -702,9 +715,9 @@ def determine_endpoints(url): endpoints.update({ 'quantum_service': 'quantum', 'quantum_region': region, - 'quantum_public_url': neutron_url, - 'quantum_admin_url': neutron_url, - 'quantum_internal_url': neutron_url, + 'quantum_public_url': neutron_public_url, + 'quantum_admin_url': neutron_internal_url, + 'quantum_internal_url': neutron_internal_url, }) return endpoints From 18489f4850a4a496704082d4d9ec5d0a953a5ac8 Mon Sep 17 00:00:00 2001 From: James Page Date: Wed, 25 Jun 2014 16:27:46 +0100 Subject: [PATCH 17/48] Fixup tests --- unit_tests/test_nova_cc_utils.py | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/unit_tests/test_nova_cc_utils.py b/unit_tests/test_nova_cc_utils.py index 8cc17c2b..df183b8b 100644 --- a/unit_tests/test_nova_cc_utils.py +++ b/unit_tests/test_nova_cc_utils.py @@ -440,7 +440,8 @@ class NovaCCUtilsTests(CharmTestCase): self.is_relation_made.return_value = False self.relation_ids.return_value = [] self.assertEquals( - BASE_ENDPOINTS, utils.determine_endpoints('http://foohost.com')) + BASE_ENDPOINTS, utils.determine_endpoints('http://foohost.com', + 'http://foohost.com')) def test_determine_endpoints_nova_volume(self): self.is_relation_made.return_value = False @@ -456,7 +457,8 @@ class NovaCCUtilsTests(CharmTestCase): 'nova-volume_region': 'RegionOne', 'nova-volume_service': 'nova-volume'}) self.assertEquals( - endpoints, utils.determine_endpoints('http://foohost.com')) + endpoints, utils.determine_endpoints('http://foohost.com', + 'http://foohost.com')) def test_determine_endpoints_quantum_neutron(self): self.is_relation_made.return_value = False @@ -470,7 +472,8 @@ class NovaCCUtilsTests(CharmTestCase): 'quantum_region': 'RegionOne', 'quantum_service': 'quantum'}) self.assertEquals( - endpoints, utils.determine_endpoints('http://foohost.com')) + endpoints, utils.determine_endpoints('http://foohost.com', + 'http://foohost.com')) def test_determine_endpoints_neutron_api_rel(self): self.is_relation_made.return_value = True @@ -484,7 +487,8 @@ class NovaCCUtilsTests(CharmTestCase): 'quantum_region': None, 'quantum_service': None}) self.assertEquals( - endpoints, utils.determine_endpoints('http://foohost.com')) + endpoints, utils.determine_endpoints('http://foohost.com', + 'http://foohost.com')) @patch.object(utils, 'known_hosts') @patch('subprocess.check_output') From c25bd9dedf73aa623a45210d4b7ab7bc538e544a Mon Sep 17 00:00:00 2001 From: James Page Date: Wed, 25 Jun 2014 16:30:08 +0100 Subject: [PATCH 18/48] Add cluster helper to sync --- charm-helpers.yaml | 1 + hooks/charmhelpers/contrib/hahelpers/cluster.py | 4 ++-- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/charm-helpers.yaml b/charm-helpers.yaml index 9925e2b8..44c7a8b9 100644 --- a/charm-helpers.yaml +++ b/charm-helpers.yaml @@ -7,5 +7,6 @@ include: - contrib.storage - contrib.hahelpers: - apache + - cluster - payload.execd - contrib.network.ip diff --git a/hooks/charmhelpers/contrib/hahelpers/cluster.py b/hooks/charmhelpers/contrib/hahelpers/cluster.py index bf832f7d..cc0d1095 100644 --- a/hooks/charmhelpers/contrib/hahelpers/cluster.py +++ b/hooks/charmhelpers/contrib/hahelpers/cluster.py @@ -163,7 +163,7 @@ def get_hacluster_config(): return conf -def canonical_url(configs, vip_setting='vip'): +def canonical_url(configs, vip_setting='vip', address=None): ''' Returns the correct HTTP URL to this host given the state of HTTPS configuration and hacluster. @@ -179,5 +179,5 @@ def canonical_url(configs, vip_setting='vip'): if is_clustered(): addr = config_get(vip_setting) else: - addr = unit_get('private-address') + addr = address or unit_get('private-address') return '%s://%s' % (scheme, addr) From 6dac953777a2536aa6173d1a91f001bef4d736c7 Mon Sep 17 00:00:00 2001 From: James Page Date: Fri, 27 Jun 2014 11:37:18 +0100 Subject: [PATCH 19/48] Add extra admin-network configuration --- config.yaml | 9 ++++++++- hooks/nova_cc_hooks.py | 6 +++++- hooks/nova_cc_utils.py | 21 +++++++++++++++------ 3 files changed, 28 insertions(+), 8 deletions(-) diff --git a/config.yaml b/config.yaml index 6c81a8a9..40e2bb96 100644 --- a/config.yaml +++ b/config.yaml @@ -167,13 +167,20 @@ options: # end of NVP/NSX configuration # Network configuration options # by default all access is over 'private-address' + os-admin-network: + type: string + description: | + The IP address and netmask of the OpenStack Admin network (e.g., + 192.168.0.0/24) + . + This network will be used for admin endpoints. os-internal-network: type: string description: | The IP address and netmask of the OpenStack Internal network (e.g., 192.168.0.0/24) . - This network will be used for admin and internal endpoints. + This network will be used for internal endpoints. os-public-network: type: string description: | diff --git a/hooks/nova_cc_hooks.py b/hooks/nova_cc_hooks.py index 823fa4d7..5635aea6 100755 --- a/hooks/nova_cc_hooks.py +++ b/hooks/nova_cc_hooks.py @@ -239,8 +239,12 @@ def identity_joined(rid=None): internal_url = canonical_url(CONFIGS, address=get_address_in_network(config('os-internal-network'), unit_get('private-address'))) + admin_url = canonical_url(CONFIGS, + address=get_address_in_network(config('os-admin-network'), + unit_get('private-address'))) relation_set(relation_id=rid, **determine_endpoints(public_url, - internal_url)) + internal_url, + admin_url)) @hooks.hook('identity-service-relation-changed') diff --git a/hooks/nova_cc_utils.py b/hooks/nova_cc_utils.py index b3e8da36..ec82d0ed 100644 --- a/hooks/nova_cc_utils.py +++ b/hooks/nova_cc_utils.py @@ -643,7 +643,7 @@ def ssh_compute_remove(public_key, user=None): _keys.write(keys) -def determine_endpoints(public_url, internal_url): +def determine_endpoints(public_url, internal_url, admin_url): '''Generates a dictionary containing all relevant endpoints to be passed to keystone as relation settings.''' region = config('region') @@ -654,42 +654,51 @@ def determine_endpoints(public_url, internal_url): (public_url, api_port('nova-api-os-compute'))) nova_internal_url = ('%s:%s/v2/$(tenant_id)s' % (internal_url, api_port('nova-api-os-compute'))) + nova_admin_url = ('%s:%s/v2/$(tenant_id)s' % + (admin_url, api_port('nova-api-os-compute'))) else: nova_public_url = ('%s:%s/v1.1/$(tenant_id)s' % (public_url, api_port('nova-api-os-compute'))) nova_internal_url = ('%s:%s/v1.1/$(tenant_id)s' % (internal_url, api_port('nova-api-os-compute'))) + nova_admin_url = ('%s:%s/v1.1/$(tenant_id)s' % + (admin_url, api_port('nova-api-os-compute'))) ec2_public_url = '%s:%s/services/Cloud' % (public_url, api_port('nova-api-ec2')) ec2_internal_url = '%s:%s/services/Cloud' % (internal_url, api_port('nova-api-ec2')) + ec2_admin_url = '%s:%s/services/Cloud' % (admin_url, api_port('nova-api-ec2')) nova_volume_public_url = ('%s:%s/v1/$(tenant_id)s' % (public_url, api_port('nova-api-os-compute'))) nova_volume_internal_url = ('%s:%s/v1/$(tenant_id)s' % (internal_url, api_port('nova-api-os-compute'))) + nova_volume_admin_url = ('%s:%s/v1/$(tenant_id)s' % + (admin_url, api_port('nova-api-os-compute'))) neutron_public_url = '%s:%s' % (public_url, api_port('neutron-server')) neutron_internal_url = '%s:%s' % (internal_url, api_port('neutron-server')) + neutron_admin_url = '%s:%s' % (admin_url, api_port('neutron-server')) s3_public_url = '%s:%s' % (public_url, api_port('nova-objectstore')) s3_internal_url = '%s:%s' % (internal_url, api_port('nova-objectstore')) + s3_admin_url = '%s:%s' % (admin_url, api_port('nova-objectstore')) # the base endpoints endpoints = { 'nova_service': 'nova', 'nova_region': region, 'nova_public_url': nova_public_url, - 'nova_admin_url': nova_internal_url, + 'nova_admin_url': nova_admin_url, 'nova_internal_url': nova_internal_url, 'ec2_service': 'ec2', 'ec2_region': region, 'ec2_public_url': ec2_public_url, - 'ec2_admin_url': ec2_internal_url, + 'ec2_admin_url': ec2_admin_url, 'ec2_internal_url': ec2_internal_url, 's3_service': 's3', 's3_region': region, 's3_public_url': s3_public_url, - 's3_admin_url': s3_internal_url, + 's3_admin_url': s3_admin_url, 's3_internal_url': s3_internal_url, } @@ -698,7 +707,7 @@ def determine_endpoints(public_url, internal_url): 'nova-volume_service': 'nova-volume', 'nova-volume_region': region, 'nova-volume_public_url': nova_volume_public_url, - 'nova-volume_admin_url': nova_volume_internal_url, + 'nova-volume_admin_url': nova_volume_admin_url, 'nova-volume_internal_url': nova_volume_internal_url, }) @@ -716,7 +725,7 @@ def determine_endpoints(public_url, internal_url): 'quantum_service': 'quantum', 'quantum_region': region, 'quantum_public_url': neutron_public_url, - 'quantum_admin_url': neutron_internal_url, + 'quantum_admin_url': neutron_admin_url, 'quantum_internal_url': neutron_internal_url, }) From 58a9e396e800e6d63f09a6bdbae88d1de31eebe7 Mon Sep 17 00:00:00 2001 From: James Page Date: Fri, 27 Jun 2014 14:52:06 +0100 Subject: [PATCH 20/48] Resync helpers --- .../charmhelpers/contrib/hahelpers/cluster.py | 1 + hooks/charmhelpers/contrib/network/ip.py | 12 +- .../contrib/openstack/amulet/__init__.py | 0 .../contrib/openstack/amulet/deployment.py | 38 ++++ .../contrib/openstack/amulet/utils.py | 209 ++++++++++++++++++ .../charmhelpers/contrib/openstack/context.py | 58 +++-- .../charmhelpers/contrib/openstack/neutron.py | 14 ++ .../contrib/openstack/templating.py | 37 ++-- hooks/charmhelpers/contrib/openstack/utils.py | 7 +- .../contrib/storage/linux/ceph.py | 2 +- .../contrib/storage/linux/utils.py | 1 + hooks/charmhelpers/core/fstab.py | 4 +- hooks/charmhelpers/core/hookenv.py | 9 +- hooks/charmhelpers/core/host.py | 14 +- hooks/charmhelpers/fetch/__init__.py | 40 ++-- 15 files changed, 378 insertions(+), 68 deletions(-) create mode 100644 hooks/charmhelpers/contrib/openstack/amulet/__init__.py create mode 100644 hooks/charmhelpers/contrib/openstack/amulet/deployment.py create mode 100644 hooks/charmhelpers/contrib/openstack/amulet/utils.py diff --git a/hooks/charmhelpers/contrib/hahelpers/cluster.py b/hooks/charmhelpers/contrib/hahelpers/cluster.py index cc0d1095..dd89f347 100644 --- a/hooks/charmhelpers/contrib/hahelpers/cluster.py +++ b/hooks/charmhelpers/contrib/hahelpers/cluster.py @@ -170,6 +170,7 @@ def canonical_url(configs, vip_setting='vip', address=None): :configs : OSTemplateRenderer: A config tempating object to inspect for a complete https context. + :vip_setting: str: Setting in charm config that specifies VIP address. ''' diff --git a/hooks/charmhelpers/contrib/network/ip.py b/hooks/charmhelpers/contrib/network/ip.py index 44c7c975..15a6731c 100644 --- a/hooks/charmhelpers/contrib/network/ip.py +++ b/hooks/charmhelpers/contrib/network/ip.py @@ -30,12 +30,12 @@ def get_address_in_network(network, fallback=None, fatal=False): """ Get an IPv4 address within the network from the host. - Args: - network (str): CIDR presentation format. For example, - '192.168.1.0/24'. - fallback (str): If no address is found, return fallback. - fatal (boolean): If no address is found, fallback is not - set and fatal is True then exit(1). + :param network (str): CIDR presentation format. For example, + '192.168.1.0/24'. + :param fallback (str): If no address is found, return fallback. + :param fatal (boolean): If no address is found, fallback is not + set and fatal is True then exit(1). + """ def not_found_error_out(): diff --git a/hooks/charmhelpers/contrib/openstack/amulet/__init__.py b/hooks/charmhelpers/contrib/openstack/amulet/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/hooks/charmhelpers/contrib/openstack/amulet/deployment.py b/hooks/charmhelpers/contrib/openstack/amulet/deployment.py new file mode 100644 index 00000000..9e164821 --- /dev/null +++ b/hooks/charmhelpers/contrib/openstack/amulet/deployment.py @@ -0,0 +1,38 @@ +from charmhelpers.contrib.amulet.deployment import ( + AmuletDeployment +) + + +class OpenStackAmuletDeployment(AmuletDeployment): + """This class inherits from AmuletDeployment and has additional support + that is specifically for use by OpenStack charms.""" + + def __init__(self, series=None, openstack=None): + """Initialize the deployment environment.""" + self.openstack = None + super(OpenStackAmuletDeployment, self).__init__(series) + + if openstack: + self.openstack = openstack + + def _configure_services(self, configs): + """Configure all of the services.""" + for service, config in configs.iteritems(): + if service == self.this_service: + config['openstack-origin'] = self.openstack + self.d.configure(service, config) + + def _get_openstack_release(self): + """Return an integer representing the enum value of the openstack + release.""" + self.precise_essex, self.precise_folsom, self.precise_grizzly, \ + self.precise_havana, self.precise_icehouse, \ + self.trusty_icehouse = range(6) + releases = { + ('precise', None): self.precise_essex, + ('precise', 'cloud:precise-folsom'): self.precise_folsom, + ('precise', 'cloud:precise-grizzly'): self.precise_grizzly, + ('precise', 'cloud:precise-havana'): self.precise_havana, + ('precise', 'cloud:precise-icehouse'): self.precise_icehouse, + ('trusty', None): self.trusty_icehouse} + return releases[(self.series, self.openstack)] diff --git a/hooks/charmhelpers/contrib/openstack/amulet/utils.py b/hooks/charmhelpers/contrib/openstack/amulet/utils.py new file mode 100644 index 00000000..6515f907 --- /dev/null +++ b/hooks/charmhelpers/contrib/openstack/amulet/utils.py @@ -0,0 +1,209 @@ +import logging +import os +import time +import urllib + +import glanceclient.v1.client as glance_client +import keystoneclient.v2_0 as keystone_client +import novaclient.v1_1.client as nova_client + +from charmhelpers.contrib.amulet.utils import ( + AmuletUtils +) + +DEBUG = logging.DEBUG +ERROR = logging.ERROR + + +class OpenStackAmuletUtils(AmuletUtils): + """This class inherits from AmuletUtils and has additional support + that is specifically for use by OpenStack charms.""" + + def __init__(self, log_level=ERROR): + """Initialize the deployment environment.""" + super(OpenStackAmuletUtils, self).__init__(log_level) + + def validate_endpoint_data(self, endpoints, admin_port, internal_port, + public_port, expected): + """Validate actual endpoint data vs expected endpoint data. The ports + are used to find the matching endpoint.""" + found = False + for ep in endpoints: + self.log.debug('endpoint: {}'.format(repr(ep))) + if admin_port in ep.adminurl and internal_port in ep.internalurl \ + and public_port in ep.publicurl: + found = True + actual = {'id': ep.id, + 'region': ep.region, + 'adminurl': ep.adminurl, + 'internalurl': ep.internalurl, + 'publicurl': ep.publicurl, + 'service_id': ep.service_id} + ret = self._validate_dict_data(expected, actual) + if ret: + return 'unexpected endpoint data - {}'.format(ret) + + if not found: + return 'endpoint not found' + + def validate_svc_catalog_endpoint_data(self, expected, actual): + """Validate a list of actual service catalog endpoints vs a list of + expected service catalog endpoints.""" + self.log.debug('actual: {}'.format(repr(actual))) + for k, v in expected.iteritems(): + if k in actual: + ret = self._validate_dict_data(expected[k][0], actual[k][0]) + if ret: + return self.endpoint_error(k, ret) + else: + return "endpoint {} does not exist".format(k) + return ret + + def validate_tenant_data(self, expected, actual): + """Validate a list of actual tenant data vs list of expected tenant + data.""" + self.log.debug('actual: {}'.format(repr(actual))) + for e in expected: + found = False + for act in actual: + a = {'enabled': act.enabled, 'description': act.description, + 'name': act.name, 'id': act.id} + if e['name'] == a['name']: + found = True + ret = self._validate_dict_data(e, a) + if ret: + return "unexpected tenant data - {}".format(ret) + if not found: + return "tenant {} does not exist".format(e.name) + return ret + + def validate_role_data(self, expected, actual): + """Validate a list of actual role data vs a list of expected role + data.""" + self.log.debug('actual: {}'.format(repr(actual))) + for e in expected: + found = False + for act in actual: + a = {'name': act.name, 'id': act.id} + if e['name'] == a['name']: + found = True + ret = self._validate_dict_data(e, a) + if ret: + return "unexpected role data - {}".format(ret) + if not found: + return "role {} does not exist".format(e.name) + return ret + + def validate_user_data(self, expected, actual): + """Validate a list of actual user data vs a list of expected user + data.""" + self.log.debug('actual: {}'.format(repr(actual))) + for e in expected: + found = False + for act in actual: + a = {'enabled': act.enabled, 'name': act.name, + 'email': act.email, 'tenantId': act.tenantId, + 'id': act.id} + if e['name'] == a['name']: + found = True + ret = self._validate_dict_data(e, a) + if ret: + return "unexpected user data - {}".format(ret) + if not found: + return "user {} does not exist".format(e.name) + return ret + + def validate_flavor_data(self, expected, actual): + """Validate a list of actual flavors vs a list of expected flavors.""" + self.log.debug('actual: {}'.format(repr(actual))) + act = [a.name for a in actual] + return self._validate_list_data(expected, act) + + def tenant_exists(self, keystone, tenant): + """Return True if tenant exists""" + return tenant in [t.name for t in keystone.tenants.list()] + + def authenticate_keystone_admin(self, keystone_sentry, user, password, + tenant): + """Authenticates admin user with the keystone admin endpoint.""" + service_ip = \ + keystone_sentry.relation('shared-db', + 'mysql:shared-db')['private-address'] + ep = "http://{}:35357/v2.0".format(service_ip.strip().decode('utf-8')) + return keystone_client.Client(username=user, password=password, + tenant_name=tenant, auth_url=ep) + + def authenticate_keystone_user(self, keystone, user, password, tenant): + """Authenticates a regular user with the keystone public endpoint.""" + ep = keystone.service_catalog.url_for(service_type='identity', + endpoint_type='publicURL') + return keystone_client.Client(username=user, password=password, + tenant_name=tenant, auth_url=ep) + + def authenticate_glance_admin(self, keystone): + """Authenticates admin user with glance.""" + ep = keystone.service_catalog.url_for(service_type='image', + endpoint_type='adminURL') + return glance_client.Client(ep, token=keystone.auth_token) + + def authenticate_nova_user(self, keystone, user, password, tenant): + """Authenticates a regular user with nova-api.""" + ep = keystone.service_catalog.url_for(service_type='identity', + endpoint_type='publicURL') + return nova_client.Client(username=user, api_key=password, + project_id=tenant, auth_url=ep) + + def create_cirros_image(self, glance, image_name): + """Download the latest cirros image and upload it to glance.""" + http_proxy = os.getenv('AMULET_HTTP_PROXY') + self.log.debug('AMULET_HTTP_PROXY: {}'.format(http_proxy)) + if http_proxy: + proxies = {'http': http_proxy} + opener = urllib.FancyURLopener(proxies) + else: + opener = urllib.FancyURLopener() + + f = opener.open("http://download.cirros-cloud.net/version/released") + version = f.read().strip() + cirros_img = "tests/cirros-{}-x86_64-disk.img".format(version) + + if not os.path.exists(cirros_img): + cirros_url = "http://{}/{}/{}".format("download.cirros-cloud.net", + version, cirros_img) + opener.retrieve(cirros_url, cirros_img) + f.close() + + with open(cirros_img) as f: + image = glance.images.create(name=image_name, is_public=True, + disk_format='qcow2', + container_format='bare', data=f) + return image + + def delete_image(self, glance, image): + """Delete the specified image.""" + glance.images.delete(image) + + def create_instance(self, nova, image_name, instance_name, flavor): + """Create the specified instance.""" + image = nova.images.find(name=image_name) + flavor = nova.flavors.find(name=flavor) + instance = nova.servers.create(name=instance_name, image=image, + flavor=flavor) + + count = 1 + status = instance.status + while status == 'BUILD' and count < 10: + time.sleep(5) + instance = nova.servers.get(instance.id) + status = instance.status + self.log.debug('instance status: {}'.format(status)) + count += 1 + + if status == 'BUILD': + return None + + return instance + + def delete_instance(self, nova, instance): + """Delete the specified instance.""" + nova.servers.delete(instance) diff --git a/hooks/charmhelpers/contrib/openstack/context.py b/hooks/charmhelpers/contrib/openstack/context.py index eaeb7eb2..aea11b34 100644 --- a/hooks/charmhelpers/contrib/openstack/context.py +++ b/hooks/charmhelpers/contrib/openstack/context.py @@ -243,23 +243,31 @@ class IdentityServiceContext(OSContextGenerator): class AMQPContext(OSContextGenerator): - interfaces = ['amqp'] - def __init__(self, ssl_dir=None): + def __init__(self, ssl_dir=None, rel_name='amqp', relation_prefix=None): self.ssl_dir = ssl_dir + self.rel_name = rel_name + self.relation_prefix = relation_prefix + self.interfaces = [rel_name] def __call__(self): log('Generating template context for amqp') conf = config() + user_setting = 'rabbit-user' + vhost_setting = 'rabbit-vhost' + if self.relation_prefix: + user_setting = self.relation_prefix + '-rabbit-user' + vhost_setting = self.relation_prefix + '-rabbit-vhost' + try: - username = conf['rabbit-user'] - vhost = conf['rabbit-vhost'] + username = conf[user_setting] + vhost = conf[vhost_setting] except KeyError as e: log('Could not generate shared_db context. ' 'Missing required charm config options: %s.' % e) raise OSContextError ctxt = {} - for rid in relation_ids('amqp'): + for rid in relation_ids(self.rel_name): ha_vip_only = False for unit in related_units(rid): if relation_get('clustered', rid=rid, unit=unit): @@ -420,12 +428,13 @@ class ApacheSSLContext(OSContextGenerator): """ Generates a context for an apache vhost configuration that configures HTTPS reverse proxying for one or many endpoints. Generated context - looks something like: - { - 'namespace': 'cinder', - 'private_address': 'iscsi.mycinderhost.com', - 'endpoints': [(8776, 8766), (8777, 8767)] - } + looks something like:: + + { + 'namespace': 'cinder', + 'private_address': 'iscsi.mycinderhost.com', + 'endpoints': [(8776, 8766), (8777, 8767)] + } The endpoints list consists of a tuples mapping external ports to internal ports. @@ -543,6 +552,26 @@ class NeutronContext(OSContextGenerator): return nvp_ctxt + def n1kv_ctxt(self): + driver = neutron_plugin_attribute(self.plugin, 'driver', + self.network_manager) + n1kv_config = neutron_plugin_attribute(self.plugin, 'config', + self.network_manager) + n1kv_ctxt = { + 'core_plugin': driver, + 'neutron_plugin': 'n1kv', + 'neutron_security_groups': self.neutron_security_groups, + 'local_ip': unit_private_ip(), + 'config': n1kv_config, + 'vsm_ip': config('n1kv-vsm-ip'), + 'vsm_username': config('n1kv-vsm-username'), + 'vsm_password': config('n1kv-vsm-password'), + 'restrict_policy_profiles': config( + 'n1kv_restrict_policy_profiles'), + } + + return n1kv_ctxt + def neutron_ctxt(self): if https(): proto = 'https' @@ -574,6 +603,8 @@ class NeutronContext(OSContextGenerator): ctxt.update(self.ovs_ctxt()) elif self.plugin in ['nvp', 'nsx']: ctxt.update(self.nvp_ctxt()) + elif self.plugin == 'n1kv': + ctxt.update(self.n1kv_ctxt()) alchemy_flags = config('neutron-alchemy-flags') if alchemy_flags: @@ -613,7 +644,7 @@ class SubordinateConfigContext(OSContextGenerator): The subordinate interface allows subordinates to export their configuration requirements to the principle for multiple config files and multiple serivces. Ie, a subordinate that has interfaces - to both glance and nova may export to following yaml blob as json: + to both glance and nova may export to following yaml blob as json:: glance: /etc/glance/glance-api.conf: @@ -632,7 +663,8 @@ class SubordinateConfigContext(OSContextGenerator): It is then up to the principle charms to subscribe this context to the service+config file it is interestd in. Configuration data will - be available in the template context, in glance's case, as: + be available in the template context, in glance's case, as:: + ctxt = { ... other context ... 'subordinate_config': { diff --git a/hooks/charmhelpers/contrib/openstack/neutron.py b/hooks/charmhelpers/contrib/openstack/neutron.py index ba97622c..84d97bca 100644 --- a/hooks/charmhelpers/contrib/openstack/neutron.py +++ b/hooks/charmhelpers/contrib/openstack/neutron.py @@ -128,6 +128,20 @@ def neutron_plugins(): 'server_packages': ['neutron-server', 'neutron-plugin-vmware'], 'server_services': ['neutron-server'] + }, + 'n1kv': { + 'config': '/etc/neutron/plugins/cisco/cisco_plugins.ini', + 'driver': 'neutron.plugins.cisco.network_plugin.PluginV2', + 'contexts': [ + context.SharedDBContext(user=config('neutron-database-user'), + database=config('neutron-database'), + relation_prefix='neutron', + ssl_dir=NEUTRON_CONF_DIR)], + 'services': [], + 'packages': [['neutron-plugin-cisco']], + 'server_packages': ['neutron-server', + 'neutron-plugin-cisco'], + 'server_services': ['neutron-server'] } } if release >= 'icehouse': diff --git a/hooks/charmhelpers/contrib/openstack/templating.py b/hooks/charmhelpers/contrib/openstack/templating.py index 4595778c..f5442712 100644 --- a/hooks/charmhelpers/contrib/openstack/templating.py +++ b/hooks/charmhelpers/contrib/openstack/templating.py @@ -30,17 +30,17 @@ def get_loader(templates_dir, os_release): loading dir. A charm may also ship a templates dir with this module - and it will be appended to the bottom of the search list, eg: - hooks/charmhelpers/contrib/openstack/templates. + and it will be appended to the bottom of the search list, eg:: - :param templates_dir: str: Base template directory containing release - sub-directories. - :param os_release : str: OpenStack release codename to construct template - loader. + hooks/charmhelpers/contrib/openstack/templates - :returns : jinja2.ChoiceLoader constructed with a list of - jinja2.FilesystemLoaders, ordered in descending - order by OpenStack release. + :param templates_dir (str): Base template directory containing release + sub-directories. + :param os_release (str): OpenStack release codename to construct template + loader. + :returns: jinja2.ChoiceLoader constructed with a list of + jinja2.FilesystemLoaders, ordered in descending + order by OpenStack release. """ tmpl_dirs = [(rel, os.path.join(templates_dir, rel)) for rel in OPENSTACK_CODENAMES.itervalues()] @@ -111,7 +111,8 @@ class OSConfigRenderer(object): and ease the burden of managing config templates across multiple OpenStack releases. - Basic usage: + Basic usage:: + # import some common context generates from charmhelpers from charmhelpers.contrib.openstack import context @@ -131,21 +132,19 @@ class OSConfigRenderer(object): # write out all registered configs configs.write_all() - Details: + **OpenStack Releases and template loading** - OpenStack Releases and template loading - --------------------------------------- When the object is instantiated, it is associated with a specific OS release. This dictates how the template loader will be constructed. The constructed loader attempts to load the template from several places in the following order: - - from the most recent OS release-specific template dir (if one exists) - - the base templates_dir - - a template directory shipped in the charm with this helper file. + - from the most recent OS release-specific template dir (if one exists) + - the base templates_dir + - a template directory shipped in the charm with this helper file. + For the example above, '/tmp/templates' contains the following structure:: - For the example above, '/tmp/templates' contains the following structure: /tmp/templates/nova.conf /tmp/templates/api-paste.ini /tmp/templates/grizzly/api-paste.ini @@ -169,8 +168,8 @@ class OSConfigRenderer(object): $CHARM/hooks/charmhelpers/contrib/openstack/templates. This allows us to ship common templates (haproxy, apache) with the helpers. - Context generators - --------------------------------------- + **Context generators** + Context generators are used to generate template contexts during hook execution. Doing so may require inspecting service relations, charm config, etc. When registered, a config file is associated with a list diff --git a/hooks/charmhelpers/contrib/openstack/utils.py b/hooks/charmhelpers/contrib/openstack/utils.py index 1a44ab1f..127b03fe 100644 --- a/hooks/charmhelpers/contrib/openstack/utils.py +++ b/hooks/charmhelpers/contrib/openstack/utils.py @@ -3,7 +3,6 @@ # Common python helper functions used for OpenStack charms. from collections import OrderedDict -import apt_pkg as apt import subprocess import os import socket @@ -85,6 +84,8 @@ def get_os_codename_install_source(src): '''Derive OpenStack release codename from a given installation source.''' ubuntu_rel = lsb_release()['DISTRIB_CODENAME'] rel = '' + if src is None: + return rel if src in ['distro', 'distro-proposed']: try: rel = UBUNTU_OPENSTACK_RELEASE[ubuntu_rel] @@ -132,6 +133,7 @@ def get_os_version_codename(codename): def get_os_codename_package(package, fatal=True): '''Derive OpenStack release codename from an installed package.''' + import apt_pkg as apt apt.init() # Tell apt to build an in-memory cache to prevent race conditions (if @@ -189,7 +191,7 @@ def get_os_version_package(pkg, fatal=True): for version, cname in vers_map.iteritems(): if cname == codename: return version - #e = "Could not determine OpenStack version for package: %s" % pkg + # e = "Could not determine OpenStack version for package: %s" % pkg # error_out(e) @@ -325,6 +327,7 @@ def openstack_upgrade_available(package): """ + import apt_pkg as apt src = config('openstack-origin') cur_vers = get_os_version_package(package) available_vers = get_os_version_install_source(src) diff --git a/hooks/charmhelpers/contrib/storage/linux/ceph.py b/hooks/charmhelpers/contrib/storage/linux/ceph.py index 12417410..768438a4 100644 --- a/hooks/charmhelpers/contrib/storage/linux/ceph.py +++ b/hooks/charmhelpers/contrib/storage/linux/ceph.py @@ -303,7 +303,7 @@ def ensure_ceph_storage(service, pool, rbd_img, sizemb, mount_point, blk_device, fstype, system_services=[]): """ NOTE: This function must only be called from a single service unit for - the same rbd_img otherwise data loss will occur. + the same rbd_img otherwise data loss will occur. Ensures given pool and RBD image exists, is mapped to a block device, and the device is formatted and mounted at the given mount_point. diff --git a/hooks/charmhelpers/contrib/storage/linux/utils.py b/hooks/charmhelpers/contrib/storage/linux/utils.py index b87ef26d..8d0f6116 100644 --- a/hooks/charmhelpers/contrib/storage/linux/utils.py +++ b/hooks/charmhelpers/contrib/storage/linux/utils.py @@ -37,6 +37,7 @@ def zap_disk(block_device): check_call(['dd', 'if=/dev/zero', 'of=%s' % (block_device), 'bs=512', 'count=100', 'seek=%s' % (gpt_end)]) + def is_device_mounted(device): '''Given a device path, return True if that device is mounted, and False if it isn't. diff --git a/hooks/charmhelpers/core/fstab.py b/hooks/charmhelpers/core/fstab.py index cdd72616..cfaf0a65 100644 --- a/hooks/charmhelpers/core/fstab.py +++ b/hooks/charmhelpers/core/fstab.py @@ -48,9 +48,11 @@ class Fstab(file): file.__init__(self, self._path, 'r+') def _hydrate_entry(self, line): + # NOTE: use split with no arguments to split on any + # whitespace including tabs return Fstab.Entry(*filter( lambda x: x not in ('', None), - line.strip("\n").split(" "))) + line.strip("\n").split())) @property def entries(self): diff --git a/hooks/charmhelpers/core/hookenv.py b/hooks/charmhelpers/core/hookenv.py index c2e66f66..c9530433 100644 --- a/hooks/charmhelpers/core/hookenv.py +++ b/hooks/charmhelpers/core/hookenv.py @@ -25,7 +25,7 @@ cache = {} def cached(func): """Cache return values for multiple executions of func + args - For example: + For example:: @cached def unit_get(attribute): @@ -445,18 +445,19 @@ class UnregisteredHookError(Exception): class Hooks(object): """A convenient handler for hook functions. - Example: + Example:: + hooks = Hooks() # register a hook, taking its name from the function name @hooks.hook() def install(): - ... + pass # your code here # register a hook, providing a custom hook name @hooks.hook("config-changed") def config_changed(): - ... + pass # your code here if __name__ == "__main__": # execute a hook based on the name the program is called by diff --git a/hooks/charmhelpers/core/host.py b/hooks/charmhelpers/core/host.py index 46bfd36a..8b617a42 100644 --- a/hooks/charmhelpers/core/host.py +++ b/hooks/charmhelpers/core/host.py @@ -12,7 +12,6 @@ import random import string import subprocess import hashlib -import apt_pkg from collections import OrderedDict @@ -212,13 +211,13 @@ def file_hash(path): def restart_on_change(restart_map, stopstart=False): """Restart services based on configuration files changing - This function is used a decorator, for example + This function is used a decorator, for example:: @restart_on_change({ '/etc/ceph/ceph.conf': [ 'cinder-api', 'cinder-volume' ] }) def ceph_client_changed(): - ... + pass # your code here In this example, the cinder-api and cinder-volume services would be restarted if /etc/ceph/ceph.conf is changed by the @@ -314,10 +313,13 @@ def get_nic_hwaddr(nic): def cmp_pkgrevno(package, revno, pkgcache=None): '''Compare supplied revno with the revno of the installed package - 1 => Installed revno is greater than supplied arg - 0 => Installed revno is the same as supplied arg - -1 => Installed revno is less than supplied arg + + * 1 => Installed revno is greater than supplied arg + * 0 => Installed revno is the same as supplied arg + * -1 => Installed revno is less than supplied arg + ''' + import apt_pkg if not pkgcache: apt_pkg.init() pkgcache = apt_pkg.Cache() diff --git a/hooks/charmhelpers/fetch/__init__.py b/hooks/charmhelpers/fetch/__init__.py index e8e837a5..5be512ce 100644 --- a/hooks/charmhelpers/fetch/__init__.py +++ b/hooks/charmhelpers/fetch/__init__.py @@ -13,7 +13,6 @@ from charmhelpers.core.hookenv import ( config, log, ) -import apt_pkg import os @@ -117,6 +116,7 @@ class BaseFetchHandler(object): def filter_installed_packages(packages): """Returns a list of packages that require installation""" + import apt_pkg apt_pkg.init() # Tell apt to build an in-memory cache to prevent race conditions (if @@ -235,31 +235,39 @@ def configure_sources(update=False, sources_var='install_sources', keys_var='install_keys'): """ - Configure multiple sources from charm configuration + Configure multiple sources from charm configuration. + + The lists are encoded as yaml fragments in the configuration. + The frament needs to be included as a string. Example config: - install_sources: + install_sources: | - "ppa:foo" - "http://example.com/repo precise main" - install_keys: + install_keys: | - null - "a1b2c3d4" Note that 'null' (a.k.a. None) should not be quoted. """ - sources = safe_load(config(sources_var)) - keys = config(keys_var) - if keys is not None: - keys = safe_load(keys) - if isinstance(sources, basestring) and ( - keys is None or isinstance(keys, basestring)): - add_source(sources, keys) + sources = safe_load((config(sources_var) or '').strip()) or [] + keys = safe_load((config(keys_var) or '').strip()) or None + + if isinstance(sources, basestring): + sources = [sources] + + if keys is None: + for source in sources: + add_source(source, None) else: - if not len(sources) == len(keys): - msg = 'Install sources and keys lists are different lengths' - raise SourceConfigError(msg) - for src_num in range(len(sources)): - add_source(sources[src_num], keys[src_num]) + if isinstance(keys, basestring): + keys = [keys] + + if len(sources) != len(keys): + raise SourceConfigError( + 'Install sources and keys lists are different lengths') + for source, key in zip(sources, keys): + add_source(source, key) if update: apt_update(fatal=True) From 2792a4dc5952b8570f44be51f133fc627a5fea9f Mon Sep 17 00:00:00 2001 From: James Page Date: Wed, 2 Jul 2014 09:22:25 +0100 Subject: [PATCH 21/48] Resync helpers --- .bzrignore | 2 ++ Makefile | 12 +++++--- hooks/charmhelpers/contrib/network/ip.py | 26 +++++++++++++++++ .../contrib/openstack/amulet/deployment.py | 29 +++++++++++++++---- .../contrib/openstack/amulet/utils.py | 10 +++---- .../charmhelpers/contrib/openstack/context.py | 17 +++++++++++ 6 files changed, 81 insertions(+), 15 deletions(-) create mode 100644 .bzrignore diff --git a/.bzrignore b/.bzrignore new file mode 100644 index 00000000..a2c7a097 --- /dev/null +++ b/.bzrignore @@ -0,0 +1,2 @@ +bin +.coverage diff --git a/Makefile b/Makefile index 2ec849dc..3a508187 100644 --- a/Makefile +++ b/Makefile @@ -6,11 +6,15 @@ lint: @charm proof test: - @echo Starting tests... - @$(PYTHON) /usr/bin/nosetests --nologcapture --with-coverage unit_tests + @$(PYTHON) /usr/bin/nosetests --nologcapture --with-coverage unit_tests -sync: - @charm-helper-sync -c charm-helpers.yaml +bin/charm_helpers_sync.py: + @mkdir -p bin + @bzr cat lp:charm-helpers/tools/charm_helpers_sync/charm_helpers_sync.py \ + > bin/charm_helpers_sync.py + +sync: bin/charm_helpers_sync.py + @$(PYTHON) bin/charm_helpers_sync.py -c charm-helpers.yaml publish: lint test bzr push lp:charms/nova-cloud-controller diff --git a/hooks/charmhelpers/contrib/network/ip.py b/hooks/charmhelpers/contrib/network/ip.py index 15a6731c..f2fa263f 100644 --- a/hooks/charmhelpers/contrib/network/ip.py +++ b/hooks/charmhelpers/contrib/network/ip.py @@ -67,3 +67,29 @@ def get_address_in_network(network, fallback=None, fatal=False): not_found_error_out() return None + + +def is_address_in_network(network, address): + """ + Determine whether the provided address is within a network range. + + :param network (str): CIDR presentation format. For example, + '192.168.1.0/24'. + :param address: An individual IPv4 or IPv6 address without a net + mask or subnet prefix. For example, '192.168.1.1'. + :returns boolean: Flag indicating whether address is in network. + """ + try: + network = netaddr.IPNetwork(network) + except (netaddr.core.AddrFormatError, ValueError): + raise ValueError("Network (%s) is not in CIDR presentation format" % + network) + try: + address = netaddr.IPAddress(address) + except (netaddr.core.AddrFormatError, ValueError): + raise ValueError("Address (%s) is not in correct presentation format" % + address) + if address in network: + return True + else: + return False diff --git a/hooks/charmhelpers/contrib/openstack/amulet/deployment.py b/hooks/charmhelpers/contrib/openstack/amulet/deployment.py index 9e164821..e476b6f2 100644 --- a/hooks/charmhelpers/contrib/openstack/amulet/deployment.py +++ b/hooks/charmhelpers/contrib/openstack/amulet/deployment.py @@ -7,19 +7,36 @@ class OpenStackAmuletDeployment(AmuletDeployment): """This class inherits from AmuletDeployment and has additional support that is specifically for use by OpenStack charms.""" - def __init__(self, series=None, openstack=None): + def __init__(self, series=None, openstack=None, source=None): """Initialize the deployment environment.""" - self.openstack = None super(OpenStackAmuletDeployment, self).__init__(series) + self.openstack = openstack + self.source = source - if openstack: - self.openstack = openstack + def _add_services(self, this_service, other_services): + """Add services to the deployment and set openstack-origin.""" + super(OpenStackAmuletDeployment, self)._add_services(this_service, + other_services) + name = 0 + services = other_services + services.append(this_service) + use_source = ['mysql', 'mongodb', 'rabbitmq-server', 'ceph'] + + if self.openstack: + for svc in services: + if svc[name] not in use_source: + config = {'openstack-origin': self.openstack} + self.d.configure(svc[name], config) + + if self.source: + for svc in services: + if svc[name] in use_source: + config = {'source': self.source} + self.d.configure(svc[name], config) def _configure_services(self, configs): """Configure all of the services.""" for service, config in configs.iteritems(): - if service == self.this_service: - config['openstack-origin'] = self.openstack self.d.configure(service, config) def _get_openstack_release(self): diff --git a/hooks/charmhelpers/contrib/openstack/amulet/utils.py b/hooks/charmhelpers/contrib/openstack/amulet/utils.py index 6515f907..222281e3 100644 --- a/hooks/charmhelpers/contrib/openstack/amulet/utils.py +++ b/hooks/charmhelpers/contrib/openstack/amulet/utils.py @@ -74,7 +74,7 @@ class OpenStackAmuletUtils(AmuletUtils): if ret: return "unexpected tenant data - {}".format(ret) if not found: - return "tenant {} does not exist".format(e.name) + return "tenant {} does not exist".format(e['name']) return ret def validate_role_data(self, expected, actual): @@ -91,7 +91,7 @@ class OpenStackAmuletUtils(AmuletUtils): if ret: return "unexpected role data - {}".format(ret) if not found: - return "role {} does not exist".format(e.name) + return "role {} does not exist".format(e['name']) return ret def validate_user_data(self, expected, actual): @@ -110,7 +110,7 @@ class OpenStackAmuletUtils(AmuletUtils): if ret: return "unexpected user data - {}".format(ret) if not found: - return "user {} does not exist".format(e.name) + return "user {} does not exist".format(e['name']) return ret def validate_flavor_data(self, expected, actual): @@ -192,8 +192,8 @@ class OpenStackAmuletUtils(AmuletUtils): count = 1 status = instance.status - while status == 'BUILD' and count < 10: - time.sleep(5) + while status != 'ACTIVE' and count < 60: + time.sleep(3) instance = nova.servers.get(instance.id) status = instance.status self.log.debug('instance status: {}'.format(status)) diff --git a/hooks/charmhelpers/contrib/openstack/context.py b/hooks/charmhelpers/contrib/openstack/context.py index aea11b34..b21fca60 100644 --- a/hooks/charmhelpers/contrib/openstack/context.py +++ b/hooks/charmhelpers/contrib/openstack/context.py @@ -21,6 +21,7 @@ from charmhelpers.core.hookenv import ( relation_get, relation_ids, related_units, + relation_set, unit_get, unit_private_ip, ERROR, @@ -42,6 +43,8 @@ from charmhelpers.contrib.openstack.neutron import ( neutron_plugin_attribute, ) +from charmhelpers.contrib.network.ip import get_address_in_network + CA_CERT_PATH = '/usr/local/share/ca-certificates/keystone_juju_ca_cert.crt' @@ -134,8 +137,22 @@ class SharedDBContext(OSContextGenerator): 'Missing required charm config options. ' '(database name and user)') raise OSContextError + ctxt = {} + # NOTE(jamespage) if mysql charm provides a network upon which + # access to the database should be made, reconfigure relation + # with the service units local address and defer execution + access_network = relation_get('access-network') + if access_network is not None: + access_hostname = get_address_in_network(access_network, + unit_get('private-address')) + set_hostname = relation_get(attribute='hostname', + unit=local_unit()) + if set_hostname != access_hostname: + relation_set(hostname=access_hostname) + return ctxt # Defer any further hook execution for now.... + password_setting = 'password' if self.relation_prefix: password_setting = self.relation_prefix + '_password' From 888b520dba1dba127dbbbc0d86c0baf499f97412 Mon Sep 17 00:00:00 2001 From: James Page Date: Wed, 2 Jul 2014 10:21:24 +0100 Subject: [PATCH 22/48] Resync helpers --- hooks/charmhelpers/contrib/openstack/context.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/hooks/charmhelpers/contrib/openstack/context.py b/hooks/charmhelpers/contrib/openstack/context.py index b21fca60..8d7f4945 100644 --- a/hooks/charmhelpers/contrib/openstack/context.py +++ b/hooks/charmhelpers/contrib/openstack/context.py @@ -150,7 +150,11 @@ class SharedDBContext(OSContextGenerator): set_hostname = relation_get(attribute='hostname', unit=local_unit()) if set_hostname != access_hostname: - relation_set(hostname=access_hostname) + if self.relation_prefix is not None: + hostname_key = "{}_hostname".format(self.relation_prefix) + else: + hostname_key = "hostname" + relation_set(relation_settings={hostname_key: access_hostname}) return ctxt # Defer any further hook execution for now.... password_setting = 'password' From 73be2a0fc9534a993eb9d434904038e3cc1b6af9 Mon Sep 17 00:00:00 2001 From: James Page Date: Wed, 2 Jul 2014 10:26:17 +0100 Subject: [PATCH 23/48] Resync helpers --- hooks/charmhelpers/contrib/openstack/context.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/hooks/charmhelpers/contrib/openstack/context.py b/hooks/charmhelpers/contrib/openstack/context.py index 8d7f4945..9da9c1ff 100644 --- a/hooks/charmhelpers/contrib/openstack/context.py +++ b/hooks/charmhelpers/contrib/openstack/context.py @@ -145,15 +145,15 @@ class SharedDBContext(OSContextGenerator): # with the service units local address and defer execution access_network = relation_get('access-network') if access_network is not None: + if self.relation_prefix is not None: + hostname_key = "{}_hostname".format(self.relation_prefix) + else: + hostname_key = "hostname" access_hostname = get_address_in_network(access_network, unit_get('private-address')) - set_hostname = relation_get(attribute='hostname', + set_hostname = relation_get(attribute=hostname_key, unit=local_unit()) if set_hostname != access_hostname: - if self.relation_prefix is not None: - hostname_key = "{}_hostname".format(self.relation_prefix) - else: - hostname_key = "hostname" relation_set(relation_settings={hostname_key: access_hostname}) return ctxt # Defer any further hook execution for now.... From a3fe5c2d136b9d34164717e4221ba58aecc7fbb4 Mon Sep 17 00:00:00 2001 From: James Page Date: Wed, 2 Jul 2014 10:40:27 +0100 Subject: [PATCH 24/48] Resync helpers --- hooks/charmhelpers/contrib/openstack/vip.py | 25 +++++++++++++++++++++ 1 file changed, 25 insertions(+) create mode 100644 hooks/charmhelpers/contrib/openstack/vip.py diff --git a/hooks/charmhelpers/contrib/openstack/vip.py b/hooks/charmhelpers/contrib/openstack/vip.py new file mode 100644 index 00000000..d8c42f90 --- /dev/null +++ b/hooks/charmhelpers/contrib/openstack/vip.py @@ -0,0 +1,25 @@ + +from netaddr import IPAddress, IPNetwork + +class VIPConfiguration(): + + def __init__(self, configuration): + self.vip = [] + for vip in configuration.split(): + self.vips.append(IPAddress(vip)) + + def getVIP(self, network): + ''' Determine the VIP for the provided network + :network str: CIDR presented network, e.g. 192.168.1.1/24 + :returns str: IP address of VIP in provided network or None + ''' + network = IPNetwork(network) + for vip in self.vips: + if vip in network: + return str(vip) + return None + + def getNIC(self, network): + ''' Determine the physical network interface in use + for the specified network''' + From c201dcf3430ba3c0c13ab132820f1d45885b4ee6 Mon Sep 17 00:00:00 2001 From: James Page Date: Fri, 4 Jul 2014 12:49:08 +0100 Subject: [PATCH 25/48] Switch to new canonical_url helper --- hooks/charmhelpers/contrib/network/ip.py | 83 ++++++++++++++++++- .../charmhelpers/contrib/openstack/context.py | 6 +- hooks/charmhelpers/contrib/openstack/ip.py | 75 +++++++++++++++++ .../contrib/openstack/templates/haproxy.cfg | 7 +- hooks/charmhelpers/contrib/openstack/vip.py | 25 ------ hooks/nova_cc_hooks.py | 29 +++---- hooks/nova_cc_utils.py | 1 - unit_tests/test_nova_cc_utils.py | 4 + 8 files changed, 180 insertions(+), 50 deletions(-) create mode 100644 hooks/charmhelpers/contrib/openstack/ip.py delete mode 100644 hooks/charmhelpers/contrib/openstack/vip.py diff --git a/hooks/charmhelpers/contrib/network/ip.py b/hooks/charmhelpers/contrib/network/ip.py index f2fa263f..e0f9eb66 100644 --- a/hooks/charmhelpers/contrib/network/ip.py +++ b/hooks/charmhelpers/contrib/network/ip.py @@ -28,7 +28,7 @@ def _validate_cidr(network): def get_address_in_network(network, fallback=None, fatal=False): """ - Get an IPv4 address within the network from the host. + Get an IPv4 or IPv6 address within the network from the host. :param network (str): CIDR presentation format. For example, '192.168.1.0/24'. @@ -51,14 +51,23 @@ def get_address_in_network(network, fallback=None, fatal=False): not_found_error_out() _validate_cidr(network) + network = netaddr.IPNetwork(network) for iface in netifaces.interfaces(): addresses = netifaces.ifaddresses(iface) - if netifaces.AF_INET in addresses: + if network.version == 4 and netifaces.AF_INET in addresses: addr = addresses[netifaces.AF_INET][0]['addr'] netmask = addresses[netifaces.AF_INET][0]['netmask'] cidr = netaddr.IPNetwork("%s/%s" % (addr, netmask)) - if cidr in netaddr.IPNetwork(network): + if cidr in network: return str(cidr.ip) + if network.version == 6 and netifaces.AF_INET6 in addresses: + for addr in addresses[netifaces.AF_INET6]: + if 'fe80' not in addr['addr']: + netmask = addr['netmask'] + cidr = netaddr.IPNetwork("%s/%s" % (addr['addr'], + netmask)) + if cidr in network: + return str(cidr.ip) if fallback is not None: return fallback @@ -69,6 +78,17 @@ def get_address_in_network(network, fallback=None, fatal=False): return None +def is_ipv6(address): + '''Determine whether provided address is IPv6 or not''' + try: + address = netaddr.IPAddress(address) + except netaddr.AddrFormatError: + # probably a hostname - so not an address at all! + return False + else: + return address.version == 6 + + def is_address_in_network(network, address): """ Determine whether the provided address is within a network range. @@ -93,3 +113,60 @@ def is_address_in_network(network, address): return True else: return False + + +def _get_for_address(address, key): + """Retrieve an attribute of or the physical interface that + the IP address provided could be bound to. + + :param address (str): An individual IPv4 or IPv6 address without a net + mask or subnet prefix. For example, '192.168.1.1'. + :param key: 'iface' for the physical interface name or an attribute + of the configured interface, for example 'netmask'. + :returns str: Requested attribute or None if address is not bindable. + """ + address = netaddr.IPAddress(address) + for iface in netifaces.interfaces(): + addresses = netifaces.ifaddresses(iface) + if address.version == 4 and netifaces.AF_INET in addresses: + addr = addresses[netifaces.AF_INET][0]['addr'] + netmask = addresses[netifaces.AF_INET][0]['netmask'] + cidr = netaddr.IPNetwork("%s/%s" % (addr, netmask)) + if address in cidr: + if key == 'iface': + return iface + else: + return addresses[netifaces.AF_INET][0][key] + if address.version == 6 and netifaces.AF_INET6 in addresses: + for addr in addresses[netifaces.AF_INET6]: + if 'fe80' not in addr['addr']: + cidr = netaddr.IPNetwork("%s/%s" % (addr['addr'], + addr['netmask'])) + if address in cidr: + if key == 'iface': + return iface + else: + return addr[key] + return None + + +def get_iface_for_address(address): + """Determine the physical interface to which an IP address could be bound + + :param address (str): An individual IPv4 or IPv6 address without a net + mask or subnet prefix. For example, '192.168.1.1'. + :returns str: Interface name or None if address is not bindable. + """ + return _get_for_address(address, 'iface') + + +def get_netmask_for_address(address): + """Determine the netmask of the physical interface to which and IP address + could be bound + + :param address (str): An individual IPv4 or IPv6 address without a net + mask or subnet prefix. For example, '192.168.1.1'. + :returns str: Netmask of configured interface or None if address is + not bindable. + """ + return _get_for_address(address, 'netmask') diff --git a/hooks/charmhelpers/contrib/openstack/context.py b/hooks/charmhelpers/contrib/openstack/context.py index 9da9c1ff..1e8cfc72 100644 --- a/hooks/charmhelpers/contrib/openstack/context.py +++ b/hooks/charmhelpers/contrib/openstack/context.py @@ -148,7 +148,7 @@ class SharedDBContext(OSContextGenerator): if self.relation_prefix is not None: hostname_key = "{}_hostname".format(self.relation_prefix) else: - hostname_key = "hostname" + hostname_key = "hostname" access_hostname = get_address_in_network(access_network, unit_get('private-address')) set_hostname = relation_get(attribute=hostname_key, @@ -400,7 +400,9 @@ class HAProxyContext(OSContextGenerator): cluster_hosts = {} l_unit = local_unit().replace('/', '-') - cluster_hosts[l_unit] = unit_get('private-address') + cluster_hosts[l_unit] = \ + get_address_in_network(config('os-internal-network'), + unit_get('private-address')) for rid in relation_ids('cluster'): for unit in related_units(rid): diff --git a/hooks/charmhelpers/contrib/openstack/ip.py b/hooks/charmhelpers/contrib/openstack/ip.py new file mode 100644 index 00000000..7e7a536f --- /dev/null +++ b/hooks/charmhelpers/contrib/openstack/ip.py @@ -0,0 +1,75 @@ +from charmhelpers.core.hookenv import ( + config, + unit_get, +) + +from charmhelpers.contrib.network.ip import ( + get_address_in_network, + is_address_in_network, + is_ipv6, +) + +from charmhelpers.contrib.hahelpers.cluster import is_clustered + +PUBLIC = 'public' +INTERNAL = 'int' +ADMIN = 'admin' + +_address_map = { + PUBLIC: { + 'config': 'os-public-network', + 'fallback': 'public-address' + }, + INTERNAL: { + 'config': 'os-internal-network', + 'fallback': 'private-address' + }, + ADMIN: { + 'config': 'os-admin-network', + 'fallback': 'private-address' + } +} + + +def canonical_url(configs, endpoint_type=PUBLIC): + ''' + Returns the correct HTTP URL to this host given the state of HTTPS + configuration, hacluster and charm configuration. + + :configs OSTemplateRenderer: A config tempating object to inspect for + a complete https context. + :endpoint_type str: The endpoint type to resolve. + + :returns str: Base URL for services on the current service unit. + ''' + scheme = 'http' + if 'https' in configs.complete_contexts(): + scheme = 'https' + address = resolve_address(endpoint_type) + if is_ipv6(address): + address = "[{}]".format(address) + return '%s://%s' % (scheme, address) + + +def resolve_address(endpoint_type=PUBLIC): + resolved_address = None + if is_clustered(): + if config(_address_map[endpoint_type]['config']) is None: + # Assume vip is simple and pass back directly + resolved_address = config('vip') + else: + for vip in config('vip').split(): + if is_address_in_network( + config(_address_map[endpoint_type]['config']), + vip): + resolved_address = vip + else: + resolved_address = get_address_in_network( + config(_address_map[endpoint_type]['config']), + unit_get(_address_map[endpoint_type]['fallback']) + ) + if resolved_address is None: + raise ValueError('Unable to resolve a suitable IP address' + ' based on charm state and configuration') + else: + return resolved_address diff --git a/hooks/charmhelpers/contrib/openstack/templates/haproxy.cfg b/hooks/charmhelpers/contrib/openstack/templates/haproxy.cfg index 56ed913e..a95eddd1 100644 --- a/hooks/charmhelpers/contrib/openstack/templates/haproxy.cfg +++ b/hooks/charmhelpers/contrib/openstack/templates/haproxy.cfg @@ -27,7 +27,12 @@ listen stats :8888 {% if units -%} {% for service, ports in service_ports.iteritems() -%} -listen {{ service }} 0.0.0.0:{{ ports[0] }} +listen {{ service }}_ipv4 0.0.0.0:{{ ports[0] }} + balance roundrobin + {% for unit, address in units.iteritems() -%} + server {{ unit }} {{ address }}:{{ ports[1] }} check + {% endfor %} +listen {{ service }}_ipv6 :::{{ ports[0] }} balance roundrobin {% for unit, address in units.iteritems() -%} server {{ unit }} {{ address }}:{{ ports[1] }} check diff --git a/hooks/charmhelpers/contrib/openstack/vip.py b/hooks/charmhelpers/contrib/openstack/vip.py deleted file mode 100644 index d8c42f90..00000000 --- a/hooks/charmhelpers/contrib/openstack/vip.py +++ /dev/null @@ -1,25 +0,0 @@ - -from netaddr import IPAddress, IPNetwork - -class VIPConfiguration(): - - def __init__(self, configuration): - self.vip = [] - for vip in configuration.split(): - self.vips.append(IPAddress(vip)) - - def getVIP(self, network): - ''' Determine the VIP for the provided network - :network str: CIDR presented network, e.g. 192.168.1.1/24 - :returns str: IP address of VIP in provided network or None - ''' - network = IPNetwork(network) - for vip in self.vips: - if vip in network: - return str(vip) - return None - - def getNIC(self, network): - ''' Determine the physical network interface in use - for the specified network''' - diff --git a/hooks/nova_cc_hooks.py b/hooks/nova_cc_hooks.py index 5635aea6..1f0cea37 100755 --- a/hooks/nova_cc_hooks.py +++ b/hooks/nova_cc_hooks.py @@ -73,14 +73,17 @@ from nova_cc_utils import ( ) from charmhelpers.contrib.hahelpers.cluster import ( - canonical_url, eligible_leader, get_hacluster_config, is_leader, ) from charmhelpers.payload.execd import execd_preinstall -from charmhelpers.contrib.network.ip import get_address_in_network + +from charmhelpers.contrib.openstack.ip import ( + canonical_url, + PUBLIC, INTERNAL, ADMIN +) hooks = Hooks() CONFIGS = register_configs() @@ -233,15 +236,9 @@ def image_service_changed(): def identity_joined(rid=None): if not eligible_leader(CLUSTER_RES): return - public_url = canonical_url(CONFIGS, - address=get_address_in_network(config('os-public-network'), - unit_get('public-address'))) - internal_url = canonical_url(CONFIGS, - address=get_address_in_network(config('os-internal-network'), - unit_get('private-address'))) - admin_url = canonical_url(CONFIGS, - address=get_address_in_network(config('os-admin-network'), - unit_get('private-address'))) + public_url = canonical_url(CONFIGS, PUBLIC) + internal_url = canonical_url(CONFIGS, INTERNAL) + admin_url = canonical_url(CONFIGS, ADMIN) relation_set(relation_id=rid, **determine_endpoints(public_url, internal_url, admin_url)) @@ -332,9 +329,7 @@ def neutron_settings(): 'quantum_plugin': neutron_plugin(), 'region': config('region'), 'quantum_security_groups': config('quantum-security-groups'), - 'quantum_url': "{}:{}".format(canonical_url(CONFIGS, - get_address_in_network(config('os-internal-network'), - unit_get('private-address'))), + 'quantum_url': "{}:{}".format(canonical_url(CONFIGS, INTERNAL), str(api_port('neutron-server'))), }) neutron_url = urlparse(neutron_settings['quantum_url']) @@ -510,9 +505,7 @@ def nova_vmware_relation_joined(rid=None): rel_settings.update({ 'quantum_plugin': neutron_plugin(), 'quantum_security_groups': config('quantum-security-groups'), - 'quantum_url': "{}:{}".format(canonical_url(CONFIGS, - get_address_in_network(config('os-internal-network'), - unit_get('private-address'))), + 'quantum_url': "{}:{}".format(canonical_url(CONFIGS, INTERNAL), str(api_port('neutron-server')))}) relation_set(relation_id=rid, **rel_settings) @@ -542,7 +535,7 @@ def neutron_api_relation_joined(rid=None): service_stop('neutron-server') for id_rid in relation_ids('identity-service'): identity_joined(rid=id_rid) - nova_url = canonical_url(CONFIGS) + ":8774/v2" + nova_url = canonical_url(CONFIGS, INTERNAL) + ":8774/v2" relation_set(relation_id=rid, nova_url=nova_url) diff --git a/hooks/nova_cc_utils.py b/hooks/nova_cc_utils.py index ec82d0ed..eb70f59e 100644 --- a/hooks/nova_cc_utils.py +++ b/hooks/nova_cc_utils.py @@ -42,7 +42,6 @@ from charmhelpers.core.host import ( service_start, ) - import nova_cc_context TEMPLATES = 'templates/' diff --git a/unit_tests/test_nova_cc_utils.py b/unit_tests/test_nova_cc_utils.py index df183b8b..7c5e56cb 100644 --- a/unit_tests/test_nova_cc_utils.py +++ b/unit_tests/test_nova_cc_utils.py @@ -441,6 +441,7 @@ class NovaCCUtilsTests(CharmTestCase): self.relation_ids.return_value = [] self.assertEquals( BASE_ENDPOINTS, utils.determine_endpoints('http://foohost.com', + 'http://foohost.com', 'http://foohost.com')) def test_determine_endpoints_nova_volume(self): @@ -458,6 +459,7 @@ class NovaCCUtilsTests(CharmTestCase): 'nova-volume_service': 'nova-volume'}) self.assertEquals( endpoints, utils.determine_endpoints('http://foohost.com', + 'http://foohost.com', 'http://foohost.com')) def test_determine_endpoints_quantum_neutron(self): @@ -473,6 +475,7 @@ class NovaCCUtilsTests(CharmTestCase): 'quantum_service': 'quantum'}) self.assertEquals( endpoints, utils.determine_endpoints('http://foohost.com', + 'http://foohost.com', 'http://foohost.com')) def test_determine_endpoints_neutron_api_rel(self): @@ -488,6 +491,7 @@ class NovaCCUtilsTests(CharmTestCase): 'quantum_service': None}) self.assertEquals( endpoints, utils.determine_endpoints('http://foohost.com', + 'http://foohost.com', 'http://foohost.com')) @patch.object(utils, 'known_hosts') From e2e7e4798d0e8e862a8a97e29d6ee3fbcef27f6f Mon Sep 17 00:00:00 2001 From: Corey Bryant Date: Fri, 11 Jul 2014 02:43:50 +0000 Subject: [PATCH 26/48] Sync with charm-helpers --- .../contrib/openstack/amulet/__init__.py | 0 .../contrib/openstack/amulet/deployment.py | 57 ++++ .../contrib/openstack/amulet/utils.py | 253 ++++++++++++++++++ .../charmhelpers/contrib/openstack/context.py | 58 +++- .../charmhelpers/contrib/openstack/neutron.py | 14 + .../contrib/openstack/templating.py | 37 ++- hooks/charmhelpers/contrib/openstack/utils.py | 7 +- .../contrib/storage/linux/ceph.py | 2 +- .../contrib/storage/linux/utils.py | 1 + hooks/charmhelpers/core/fstab.py | 116 ++++++++ hooks/charmhelpers/core/hookenv.py | 9 +- hooks/charmhelpers/core/host.py | 40 ++- hooks/charmhelpers/fetch/__init__.py | 40 +-- hooks/charmhelpers/fetch/bzrurl.py | 3 +- tests/charmhelpers/__init__.py | 0 tests/charmhelpers/contrib/__init__.py | 0 tests/charmhelpers/contrib/amulet/__init__.py | 0 .../charmhelpers/contrib/amulet/deployment.py | 63 +++++ tests/charmhelpers/contrib/amulet/utils.py | 157 +++++++++++ .../contrib/openstack/__init__.py | 0 .../contrib/openstack/amulet/__init__.py | 0 .../contrib/openstack/amulet/deployment.py | 57 ++++ .../contrib/openstack/amulet/utils.py | 253 ++++++++++++++++++ 23 files changed, 1099 insertions(+), 68 deletions(-) create mode 100644 hooks/charmhelpers/contrib/openstack/amulet/__init__.py create mode 100644 hooks/charmhelpers/contrib/openstack/amulet/deployment.py create mode 100644 hooks/charmhelpers/contrib/openstack/amulet/utils.py create mode 100644 hooks/charmhelpers/core/fstab.py create mode 100644 tests/charmhelpers/__init__.py create mode 100644 tests/charmhelpers/contrib/__init__.py create mode 100644 tests/charmhelpers/contrib/amulet/__init__.py create mode 100644 tests/charmhelpers/contrib/amulet/deployment.py create mode 100644 tests/charmhelpers/contrib/amulet/utils.py create mode 100644 tests/charmhelpers/contrib/openstack/__init__.py create mode 100644 tests/charmhelpers/contrib/openstack/amulet/__init__.py create mode 100644 tests/charmhelpers/contrib/openstack/amulet/deployment.py create mode 100644 tests/charmhelpers/contrib/openstack/amulet/utils.py diff --git a/hooks/charmhelpers/contrib/openstack/amulet/__init__.py b/hooks/charmhelpers/contrib/openstack/amulet/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/hooks/charmhelpers/contrib/openstack/amulet/deployment.py b/hooks/charmhelpers/contrib/openstack/amulet/deployment.py new file mode 100644 index 00000000..de2a6bfe --- /dev/null +++ b/hooks/charmhelpers/contrib/openstack/amulet/deployment.py @@ -0,0 +1,57 @@ +from charmhelpers.contrib.amulet.deployment import ( + AmuletDeployment +) + + +class OpenStackAmuletDeployment(AmuletDeployment): + """This class inherits from AmuletDeployment and has additional support + that is specifically for use by OpenStack charms.""" + + def __init__(self, series, openstack=None, source=None): + """Initialize the deployment environment.""" + super(OpenStackAmuletDeployment, self).__init__(series) + self.openstack = openstack + self.source = source + + def _add_services(self, this_service, other_services): + """Add services to the deployment and set openstack-origin.""" + super(OpenStackAmuletDeployment, self)._add_services(this_service, + other_services) + name = 0 + services = other_services + services.append(this_service) + use_source = ['mysql', 'mongodb', 'rabbitmq-server', 'ceph'] + + if self.openstack: + for svc in services: + charm_name = self._get_charm_name(svc[name]) + if charm_name not in use_source: + config = {'openstack-origin': self.openstack} + self.d.configure(svc[name], config) + + if self.source: + for svc in services: + charm_name = self._get_charm_name(svc[name]) + if charm_name in use_source: + config = {'source': self.source} + self.d.configure(svc[name], config) + + def _configure_services(self, configs): + """Configure all of the services.""" + for service, config in configs.iteritems(): + self.d.configure(service, config) + + def _get_openstack_release(self): + """Return an integer representing the enum value of the openstack + release.""" + self.precise_essex, self.precise_folsom, self.precise_grizzly, \ + self.precise_havana, self.precise_icehouse, \ + self.trusty_icehouse = range(6) + releases = { + ('precise', None): self.precise_essex, + ('precise', 'cloud:precise-folsom'): self.precise_folsom, + ('precise', 'cloud:precise-grizzly'): self.precise_grizzly, + ('precise', 'cloud:precise-havana'): self.precise_havana, + ('precise', 'cloud:precise-icehouse'): self.precise_icehouse, + ('trusty', None): self.trusty_icehouse} + return releases[(self.series, self.openstack)] diff --git a/hooks/charmhelpers/contrib/openstack/amulet/utils.py b/hooks/charmhelpers/contrib/openstack/amulet/utils.py new file mode 100644 index 00000000..806bd212 --- /dev/null +++ b/hooks/charmhelpers/contrib/openstack/amulet/utils.py @@ -0,0 +1,253 @@ +import logging +import os +import time +import urllib + +import glanceclient.v1.client as glance_client +import keystoneclient.v2_0 as keystone_client +import novaclient.v1_1.client as nova_client + +from charmhelpers.contrib.amulet.utils import ( + AmuletUtils +) + +DEBUG = logging.DEBUG +ERROR = logging.ERROR + + +class OpenStackAmuletUtils(AmuletUtils): + """This class inherits from AmuletUtils and has additional support + that is specifically for use by OpenStack charms.""" + + def __init__(self, log_level=ERROR): + """Initialize the deployment environment.""" + super(OpenStackAmuletUtils, self).__init__(log_level) + + def validate_endpoint_data(self, endpoints, admin_port, internal_port, + public_port, expected): + """Validate actual endpoint data vs expected endpoint data. The ports + are used to find the matching endpoint.""" + found = False + for ep in endpoints: + self.log.debug('endpoint: {}'.format(repr(ep))) + if admin_port in ep.adminurl and internal_port in ep.internalurl \ + and public_port in ep.publicurl: + found = True + actual = {'id': ep.id, + 'region': ep.region, + 'adminurl': ep.adminurl, + 'internalurl': ep.internalurl, + 'publicurl': ep.publicurl, + 'service_id': ep.service_id} + ret = self._validate_dict_data(expected, actual) + if ret: + return 'unexpected endpoint data - {}'.format(ret) + + if not found: + return 'endpoint not found' + + def validate_svc_catalog_endpoint_data(self, expected, actual): + """Validate a list of actual service catalog endpoints vs a list of + expected service catalog endpoints.""" + self.log.debug('actual: {}'.format(repr(actual))) + for k, v in expected.iteritems(): + if k in actual: + ret = self._validate_dict_data(expected[k][0], actual[k][0]) + if ret: + return self.endpoint_error(k, ret) + else: + return "endpoint {} does not exist".format(k) + return ret + + def validate_tenant_data(self, expected, actual): + """Validate a list of actual tenant data vs list of expected tenant + data.""" + self.log.debug('actual: {}'.format(repr(actual))) + for e in expected: + found = False + for act in actual: + a = {'enabled': act.enabled, 'description': act.description, + 'name': act.name, 'id': act.id} + if e['name'] == a['name']: + found = True + ret = self._validate_dict_data(e, a) + if ret: + return "unexpected tenant data - {}".format(ret) + if not found: + return "tenant {} does not exist".format(e['name']) + return ret + + def validate_role_data(self, expected, actual): + """Validate a list of actual role data vs a list of expected role + data.""" + self.log.debug('actual: {}'.format(repr(actual))) + for e in expected: + found = False + for act in actual: + a = {'name': act.name, 'id': act.id} + if e['name'] == a['name']: + found = True + ret = self._validate_dict_data(e, a) + if ret: + return "unexpected role data - {}".format(ret) + if not found: + return "role {} does not exist".format(e['name']) + return ret + + def validate_user_data(self, expected, actual): + """Validate a list of actual user data vs a list of expected user + data.""" + self.log.debug('actual: {}'.format(repr(actual))) + for e in expected: + found = False + for act in actual: + a = {'enabled': act.enabled, 'name': act.name, + 'email': act.email, 'tenantId': act.tenantId, + 'id': act.id} + if e['name'] == a['name']: + found = True + ret = self._validate_dict_data(e, a) + if ret: + return "unexpected user data - {}".format(ret) + if not found: + return "user {} does not exist".format(e['name']) + return ret + + def validate_flavor_data(self, expected, actual): + """Validate a list of actual flavors vs a list of expected flavors.""" + self.log.debug('actual: {}'.format(repr(actual))) + act = [a.name for a in actual] + return self._validate_list_data(expected, act) + + def tenant_exists(self, keystone, tenant): + """Return True if tenant exists""" + return tenant in [t.name for t in keystone.tenants.list()] + + def authenticate_keystone_admin(self, keystone_sentry, user, password, + tenant): + """Authenticates admin user with the keystone admin endpoint.""" + service_ip = \ + keystone_sentry.relation('shared-db', + 'mysql:shared-db')['private-address'] + ep = "http://{}:35357/v2.0".format(service_ip.strip().decode('utf-8')) + return keystone_client.Client(username=user, password=password, + tenant_name=tenant, auth_url=ep) + + def authenticate_keystone_user(self, keystone, user, password, tenant): + """Authenticates a regular user with the keystone public endpoint.""" + ep = keystone.service_catalog.url_for(service_type='identity', + endpoint_type='publicURL') + return keystone_client.Client(username=user, password=password, + tenant_name=tenant, auth_url=ep) + + def authenticate_glance_admin(self, keystone): + """Authenticates admin user with glance.""" + ep = keystone.service_catalog.url_for(service_type='image', + endpoint_type='adminURL') + return glance_client.Client(ep, token=keystone.auth_token) + + def authenticate_nova_user(self, keystone, user, password, tenant): + """Authenticates a regular user with nova-api.""" + ep = keystone.service_catalog.url_for(service_type='identity', + endpoint_type='publicURL') + return nova_client.Client(username=user, api_key=password, + project_id=tenant, auth_url=ep) + + def create_cirros_image(self, glance, image_name): + """Download the latest cirros image and upload it to glance.""" + http_proxy = os.getenv('AMULET_HTTP_PROXY') + self.log.debug('AMULET_HTTP_PROXY: {}'.format(http_proxy)) + if http_proxy: + proxies = {'http': http_proxy} + opener = urllib.FancyURLopener(proxies) + else: + opener = urllib.FancyURLopener() + + f = opener.open("http://download.cirros-cloud.net/version/released") + version = f.read().strip() + cirros_img = "tests/cirros-{}-x86_64-disk.img".format(version) + + if not os.path.exists(cirros_img): + cirros_url = "http://{}/{}/{}".format("download.cirros-cloud.net", + version, cirros_img) + opener.retrieve(cirros_url, cirros_img) + f.close() + + with open(cirros_img) as f: + image = glance.images.create(name=image_name, is_public=True, + disk_format='qcow2', + container_format='bare', data=f) + count = 1 + status = image.status + while status != 'active' and count < 10: + time.sleep(3) + image = glance.images.get(image.id) + status = image.status + self.log.debug('image status: {}'.format(status)) + count += 1 + + if status != 'active': + self.log.error('image creation timed out') + return None + + return image + + def delete_image(self, glance, image): + """Delete the specified image.""" + num_before = len(list(glance.images.list())) + glance.images.delete(image) + + count = 1 + num_after = len(list(glance.images.list())) + while num_after != (num_before - 1) and count < 10: + time.sleep(3) + num_after = len(list(glance.images.list())) + self.log.debug('number of images: {}'.format(num_after)) + count += 1 + + if num_after != (num_before - 1): + self.log.error('image deletion timed out') + return False + + return True + + def create_instance(self, nova, image_name, instance_name, flavor): + """Create the specified instance.""" + image = nova.images.find(name=image_name) + flavor = nova.flavors.find(name=flavor) + instance = nova.servers.create(name=instance_name, image=image, + flavor=flavor) + + count = 1 + status = instance.status + while status != 'ACTIVE' and count < 60: + time.sleep(3) + instance = nova.servers.get(instance.id) + status = instance.status + self.log.debug('instance status: {}'.format(status)) + count += 1 + + if status != 'ACTIVE': + self.log.error('instance creation timed out') + return None + + return instance + + def delete_instance(self, nova, instance): + """Delete the specified instance.""" + num_before = len(list(nova.servers.list())) + nova.servers.delete(instance) + + count = 1 + num_after = len(list(nova.servers.list())) + while num_after != (num_before - 1) and count < 10: + time.sleep(3) + num_after = len(list(nova.servers.list())) + self.log.debug('number of instances: {}'.format(num_after)) + count += 1 + + if num_after != (num_before - 1): + self.log.error('instance deletion timed out') + return False + + return True diff --git a/hooks/charmhelpers/contrib/openstack/context.py b/hooks/charmhelpers/contrib/openstack/context.py index 474d51ea..eff2bd3c 100644 --- a/hooks/charmhelpers/contrib/openstack/context.py +++ b/hooks/charmhelpers/contrib/openstack/context.py @@ -243,23 +243,31 @@ class IdentityServiceContext(OSContextGenerator): class AMQPContext(OSContextGenerator): - interfaces = ['amqp'] - def __init__(self, ssl_dir=None): + def __init__(self, ssl_dir=None, rel_name='amqp', relation_prefix=None): self.ssl_dir = ssl_dir + self.rel_name = rel_name + self.relation_prefix = relation_prefix + self.interfaces = [rel_name] def __call__(self): log('Generating template context for amqp') conf = config() + user_setting = 'rabbit-user' + vhost_setting = 'rabbit-vhost' + if self.relation_prefix: + user_setting = self.relation_prefix + '-rabbit-user' + vhost_setting = self.relation_prefix + '-rabbit-vhost' + try: - username = conf['rabbit-user'] - vhost = conf['rabbit-vhost'] + username = conf[user_setting] + vhost = conf[vhost_setting] except KeyError as e: log('Could not generate shared_db context. ' 'Missing required charm config options: %s.' % e) raise OSContextError ctxt = {} - for rid in relation_ids('amqp'): + for rid in relation_ids(self.rel_name): ha_vip_only = False for unit in related_units(rid): if relation_get('clustered', rid=rid, unit=unit): @@ -418,12 +426,13 @@ class ApacheSSLContext(OSContextGenerator): """ Generates a context for an apache vhost configuration that configures HTTPS reverse proxying for one or many endpoints. Generated context - looks something like: - { - 'namespace': 'cinder', - 'private_address': 'iscsi.mycinderhost.com', - 'endpoints': [(8776, 8766), (8777, 8767)] - } + looks something like:: + + { + 'namespace': 'cinder', + 'private_address': 'iscsi.mycinderhost.com', + 'endpoints': [(8776, 8766), (8777, 8767)] + } The endpoints list consists of a tuples mapping external ports to internal ports. @@ -541,6 +550,26 @@ class NeutronContext(OSContextGenerator): return nvp_ctxt + def n1kv_ctxt(self): + driver = neutron_plugin_attribute(self.plugin, 'driver', + self.network_manager) + n1kv_config = neutron_plugin_attribute(self.plugin, 'config', + self.network_manager) + n1kv_ctxt = { + 'core_plugin': driver, + 'neutron_plugin': 'n1kv', + 'neutron_security_groups': self.neutron_security_groups, + 'local_ip': unit_private_ip(), + 'config': n1kv_config, + 'vsm_ip': config('n1kv-vsm-ip'), + 'vsm_username': config('n1kv-vsm-username'), + 'vsm_password': config('n1kv-vsm-password'), + 'restrict_policy_profiles': config( + 'n1kv_restrict_policy_profiles'), + } + + return n1kv_ctxt + def neutron_ctxt(self): if https(): proto = 'https' @@ -572,6 +601,8 @@ class NeutronContext(OSContextGenerator): ctxt.update(self.ovs_ctxt()) elif self.plugin in ['nvp', 'nsx']: ctxt.update(self.nvp_ctxt()) + elif self.plugin == 'n1kv': + ctxt.update(self.n1kv_ctxt()) alchemy_flags = config('neutron-alchemy-flags') if alchemy_flags: @@ -611,7 +642,7 @@ class SubordinateConfigContext(OSContextGenerator): The subordinate interface allows subordinates to export their configuration requirements to the principle for multiple config files and multiple serivces. Ie, a subordinate that has interfaces - to both glance and nova may export to following yaml blob as json: + to both glance and nova may export to following yaml blob as json:: glance: /etc/glance/glance-api.conf: @@ -630,7 +661,8 @@ class SubordinateConfigContext(OSContextGenerator): It is then up to the principle charms to subscribe this context to the service+config file it is interestd in. Configuration data will - be available in the template context, in glance's case, as: + be available in the template context, in glance's case, as:: + ctxt = { ... other context ... 'subordinate_config': { diff --git a/hooks/charmhelpers/contrib/openstack/neutron.py b/hooks/charmhelpers/contrib/openstack/neutron.py index ba97622c..84d97bca 100644 --- a/hooks/charmhelpers/contrib/openstack/neutron.py +++ b/hooks/charmhelpers/contrib/openstack/neutron.py @@ -128,6 +128,20 @@ def neutron_plugins(): 'server_packages': ['neutron-server', 'neutron-plugin-vmware'], 'server_services': ['neutron-server'] + }, + 'n1kv': { + 'config': '/etc/neutron/plugins/cisco/cisco_plugins.ini', + 'driver': 'neutron.plugins.cisco.network_plugin.PluginV2', + 'contexts': [ + context.SharedDBContext(user=config('neutron-database-user'), + database=config('neutron-database'), + relation_prefix='neutron', + ssl_dir=NEUTRON_CONF_DIR)], + 'services': [], + 'packages': [['neutron-plugin-cisco']], + 'server_packages': ['neutron-server', + 'neutron-plugin-cisco'], + 'server_services': ['neutron-server'] } } if release >= 'icehouse': diff --git a/hooks/charmhelpers/contrib/openstack/templating.py b/hooks/charmhelpers/contrib/openstack/templating.py index 4595778c..f5442712 100644 --- a/hooks/charmhelpers/contrib/openstack/templating.py +++ b/hooks/charmhelpers/contrib/openstack/templating.py @@ -30,17 +30,17 @@ def get_loader(templates_dir, os_release): loading dir. A charm may also ship a templates dir with this module - and it will be appended to the bottom of the search list, eg: - hooks/charmhelpers/contrib/openstack/templates. + and it will be appended to the bottom of the search list, eg:: - :param templates_dir: str: Base template directory containing release - sub-directories. - :param os_release : str: OpenStack release codename to construct template - loader. + hooks/charmhelpers/contrib/openstack/templates - :returns : jinja2.ChoiceLoader constructed with a list of - jinja2.FilesystemLoaders, ordered in descending - order by OpenStack release. + :param templates_dir (str): Base template directory containing release + sub-directories. + :param os_release (str): OpenStack release codename to construct template + loader. + :returns: jinja2.ChoiceLoader constructed with a list of + jinja2.FilesystemLoaders, ordered in descending + order by OpenStack release. """ tmpl_dirs = [(rel, os.path.join(templates_dir, rel)) for rel in OPENSTACK_CODENAMES.itervalues()] @@ -111,7 +111,8 @@ class OSConfigRenderer(object): and ease the burden of managing config templates across multiple OpenStack releases. - Basic usage: + Basic usage:: + # import some common context generates from charmhelpers from charmhelpers.contrib.openstack import context @@ -131,21 +132,19 @@ class OSConfigRenderer(object): # write out all registered configs configs.write_all() - Details: + **OpenStack Releases and template loading** - OpenStack Releases and template loading - --------------------------------------- When the object is instantiated, it is associated with a specific OS release. This dictates how the template loader will be constructed. The constructed loader attempts to load the template from several places in the following order: - - from the most recent OS release-specific template dir (if one exists) - - the base templates_dir - - a template directory shipped in the charm with this helper file. + - from the most recent OS release-specific template dir (if one exists) + - the base templates_dir + - a template directory shipped in the charm with this helper file. + For the example above, '/tmp/templates' contains the following structure:: - For the example above, '/tmp/templates' contains the following structure: /tmp/templates/nova.conf /tmp/templates/api-paste.ini /tmp/templates/grizzly/api-paste.ini @@ -169,8 +168,8 @@ class OSConfigRenderer(object): $CHARM/hooks/charmhelpers/contrib/openstack/templates. This allows us to ship common templates (haproxy, apache) with the helpers. - Context generators - --------------------------------------- + **Context generators** + Context generators are used to generate template contexts during hook execution. Doing so may require inspecting service relations, charm config, etc. When registered, a config file is associated with a list diff --git a/hooks/charmhelpers/contrib/openstack/utils.py b/hooks/charmhelpers/contrib/openstack/utils.py index 1a44ab1f..127b03fe 100644 --- a/hooks/charmhelpers/contrib/openstack/utils.py +++ b/hooks/charmhelpers/contrib/openstack/utils.py @@ -3,7 +3,6 @@ # Common python helper functions used for OpenStack charms. from collections import OrderedDict -import apt_pkg as apt import subprocess import os import socket @@ -85,6 +84,8 @@ def get_os_codename_install_source(src): '''Derive OpenStack release codename from a given installation source.''' ubuntu_rel = lsb_release()['DISTRIB_CODENAME'] rel = '' + if src is None: + return rel if src in ['distro', 'distro-proposed']: try: rel = UBUNTU_OPENSTACK_RELEASE[ubuntu_rel] @@ -132,6 +133,7 @@ def get_os_version_codename(codename): def get_os_codename_package(package, fatal=True): '''Derive OpenStack release codename from an installed package.''' + import apt_pkg as apt apt.init() # Tell apt to build an in-memory cache to prevent race conditions (if @@ -189,7 +191,7 @@ def get_os_version_package(pkg, fatal=True): for version, cname in vers_map.iteritems(): if cname == codename: return version - #e = "Could not determine OpenStack version for package: %s" % pkg + # e = "Could not determine OpenStack version for package: %s" % pkg # error_out(e) @@ -325,6 +327,7 @@ def openstack_upgrade_available(package): """ + import apt_pkg as apt src = config('openstack-origin') cur_vers = get_os_version_package(package) available_vers = get_os_version_install_source(src) diff --git a/hooks/charmhelpers/contrib/storage/linux/ceph.py b/hooks/charmhelpers/contrib/storage/linux/ceph.py index 12417410..768438a4 100644 --- a/hooks/charmhelpers/contrib/storage/linux/ceph.py +++ b/hooks/charmhelpers/contrib/storage/linux/ceph.py @@ -303,7 +303,7 @@ def ensure_ceph_storage(service, pool, rbd_img, sizemb, mount_point, blk_device, fstype, system_services=[]): """ NOTE: This function must only be called from a single service unit for - the same rbd_img otherwise data loss will occur. + the same rbd_img otherwise data loss will occur. Ensures given pool and RBD image exists, is mapped to a block device, and the device is formatted and mounted at the given mount_point. diff --git a/hooks/charmhelpers/contrib/storage/linux/utils.py b/hooks/charmhelpers/contrib/storage/linux/utils.py index b87ef26d..8d0f6116 100644 --- a/hooks/charmhelpers/contrib/storage/linux/utils.py +++ b/hooks/charmhelpers/contrib/storage/linux/utils.py @@ -37,6 +37,7 @@ def zap_disk(block_device): check_call(['dd', 'if=/dev/zero', 'of=%s' % (block_device), 'bs=512', 'count=100', 'seek=%s' % (gpt_end)]) + def is_device_mounted(device): '''Given a device path, return True if that device is mounted, and False if it isn't. diff --git a/hooks/charmhelpers/core/fstab.py b/hooks/charmhelpers/core/fstab.py new file mode 100644 index 00000000..cfaf0a65 --- /dev/null +++ b/hooks/charmhelpers/core/fstab.py @@ -0,0 +1,116 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- + +__author__ = 'Jorge Niedbalski R. ' + +import os + + +class Fstab(file): + """This class extends file in order to implement a file reader/writer + for file `/etc/fstab` + """ + + class Entry(object): + """Entry class represents a non-comment line on the `/etc/fstab` file + """ + def __init__(self, device, mountpoint, filesystem, + options, d=0, p=0): + self.device = device + self.mountpoint = mountpoint + self.filesystem = filesystem + + if not options: + options = "defaults" + + self.options = options + self.d = d + self.p = p + + def __eq__(self, o): + return str(self) == str(o) + + def __str__(self): + return "{} {} {} {} {} {}".format(self.device, + self.mountpoint, + self.filesystem, + self.options, + self.d, + self.p) + + DEFAULT_PATH = os.path.join(os.path.sep, 'etc', 'fstab') + + def __init__(self, path=None): + if path: + self._path = path + else: + self._path = self.DEFAULT_PATH + file.__init__(self, self._path, 'r+') + + def _hydrate_entry(self, line): + # NOTE: use split with no arguments to split on any + # whitespace including tabs + return Fstab.Entry(*filter( + lambda x: x not in ('', None), + line.strip("\n").split())) + + @property + def entries(self): + self.seek(0) + for line in self.readlines(): + try: + if not line.startswith("#"): + yield self._hydrate_entry(line) + except ValueError: + pass + + def get_entry_by_attr(self, attr, value): + for entry in self.entries: + e_attr = getattr(entry, attr) + if e_attr == value: + return entry + return None + + def add_entry(self, entry): + if self.get_entry_by_attr('device', entry.device): + return False + + self.write(str(entry) + '\n') + self.truncate() + return entry + + def remove_entry(self, entry): + self.seek(0) + + lines = self.readlines() + + found = False + for index, line in enumerate(lines): + if not line.startswith("#"): + if self._hydrate_entry(line) == entry: + found = True + break + + if not found: + return False + + lines.remove(line) + + self.seek(0) + self.write(''.join(lines)) + self.truncate() + return True + + @classmethod + def remove_by_mountpoint(cls, mountpoint, path=None): + fstab = cls(path=path) + entry = fstab.get_entry_by_attr('mountpoint', mountpoint) + if entry: + return fstab.remove_entry(entry) + return False + + @classmethod + def add(cls, device, mountpoint, filesystem, options=None, path=None): + return cls(path=path).add_entry(Fstab.Entry(device, + mountpoint, filesystem, + options=options)) diff --git a/hooks/charmhelpers/core/hookenv.py b/hooks/charmhelpers/core/hookenv.py index c2e66f66..c9530433 100644 --- a/hooks/charmhelpers/core/hookenv.py +++ b/hooks/charmhelpers/core/hookenv.py @@ -25,7 +25,7 @@ cache = {} def cached(func): """Cache return values for multiple executions of func + args - For example: + For example:: @cached def unit_get(attribute): @@ -445,18 +445,19 @@ class UnregisteredHookError(Exception): class Hooks(object): """A convenient handler for hook functions. - Example: + Example:: + hooks = Hooks() # register a hook, taking its name from the function name @hooks.hook() def install(): - ... + pass # your code here # register a hook, providing a custom hook name @hooks.hook("config-changed") def config_changed(): - ... + pass # your code here if __name__ == "__main__": # execute a hook based on the name the program is called by diff --git a/hooks/charmhelpers/core/host.py b/hooks/charmhelpers/core/host.py index 186147f6..8b617a42 100644 --- a/hooks/charmhelpers/core/host.py +++ b/hooks/charmhelpers/core/host.py @@ -12,11 +12,11 @@ import random import string import subprocess import hashlib -import apt_pkg from collections import OrderedDict from hookenv import log +from fstab import Fstab def service_start(service_name): @@ -35,7 +35,8 @@ def service_restart(service_name): def service_reload(service_name, restart_on_failure=False): - """Reload a system service, optionally falling back to restart if reload fails""" + """Reload a system service, optionally falling back to restart if + reload fails""" service_result = service('reload', service_name) if not service_result and restart_on_failure: service_result = service('restart', service_name) @@ -144,7 +145,19 @@ def write_file(path, content, owner='root', group='root', perms=0444): target.write(content) -def mount(device, mountpoint, options=None, persist=False): +def fstab_remove(mp): + """Remove the given mountpoint entry from /etc/fstab + """ + return Fstab.remove_by_mountpoint(mp) + + +def fstab_add(dev, mp, fs, options=None): + """Adds the given device entry to the /etc/fstab file + """ + return Fstab.add(dev, mp, fs, options=options) + + +def mount(device, mountpoint, options=None, persist=False, filesystem="ext3"): """Mount a filesystem at a particular mountpoint""" cmd_args = ['mount'] if options is not None: @@ -155,9 +168,9 @@ def mount(device, mountpoint, options=None, persist=False): except subprocess.CalledProcessError, e: log('Error mounting {} at {}\n{}'.format(device, mountpoint, e.output)) return False + if persist: - # TODO: update fstab - pass + return fstab_add(device, mountpoint, filesystem, options=options) return True @@ -169,9 +182,9 @@ def umount(mountpoint, persist=False): except subprocess.CalledProcessError, e: log('Error unmounting {}\n{}'.format(mountpoint, e.output)) return False + if persist: - # TODO: update fstab - pass + return fstab_remove(mountpoint) return True @@ -198,13 +211,13 @@ def file_hash(path): def restart_on_change(restart_map, stopstart=False): """Restart services based on configuration files changing - This function is used a decorator, for example + This function is used a decorator, for example:: @restart_on_change({ '/etc/ceph/ceph.conf': [ 'cinder-api', 'cinder-volume' ] }) def ceph_client_changed(): - ... + pass # your code here In this example, the cinder-api and cinder-volume services would be restarted if /etc/ceph/ceph.conf is changed by the @@ -300,10 +313,13 @@ def get_nic_hwaddr(nic): def cmp_pkgrevno(package, revno, pkgcache=None): '''Compare supplied revno with the revno of the installed package - 1 => Installed revno is greater than supplied arg - 0 => Installed revno is the same as supplied arg - -1 => Installed revno is less than supplied arg + + * 1 => Installed revno is greater than supplied arg + * 0 => Installed revno is the same as supplied arg + * -1 => Installed revno is less than supplied arg + ''' + import apt_pkg if not pkgcache: apt_pkg.init() pkgcache = apt_pkg.Cache() diff --git a/hooks/charmhelpers/fetch/__init__.py b/hooks/charmhelpers/fetch/__init__.py index e8e837a5..5be512ce 100644 --- a/hooks/charmhelpers/fetch/__init__.py +++ b/hooks/charmhelpers/fetch/__init__.py @@ -13,7 +13,6 @@ from charmhelpers.core.hookenv import ( config, log, ) -import apt_pkg import os @@ -117,6 +116,7 @@ class BaseFetchHandler(object): def filter_installed_packages(packages): """Returns a list of packages that require installation""" + import apt_pkg apt_pkg.init() # Tell apt to build an in-memory cache to prevent race conditions (if @@ -235,31 +235,39 @@ def configure_sources(update=False, sources_var='install_sources', keys_var='install_keys'): """ - Configure multiple sources from charm configuration + Configure multiple sources from charm configuration. + + The lists are encoded as yaml fragments in the configuration. + The frament needs to be included as a string. Example config: - install_sources: + install_sources: | - "ppa:foo" - "http://example.com/repo precise main" - install_keys: + install_keys: | - null - "a1b2c3d4" Note that 'null' (a.k.a. None) should not be quoted. """ - sources = safe_load(config(sources_var)) - keys = config(keys_var) - if keys is not None: - keys = safe_load(keys) - if isinstance(sources, basestring) and ( - keys is None or isinstance(keys, basestring)): - add_source(sources, keys) + sources = safe_load((config(sources_var) or '').strip()) or [] + keys = safe_load((config(keys_var) or '').strip()) or None + + if isinstance(sources, basestring): + sources = [sources] + + if keys is None: + for source in sources: + add_source(source, None) else: - if not len(sources) == len(keys): - msg = 'Install sources and keys lists are different lengths' - raise SourceConfigError(msg) - for src_num in range(len(sources)): - add_source(sources[src_num], keys[src_num]) + if isinstance(keys, basestring): + keys = [keys] + + if len(sources) != len(keys): + raise SourceConfigError( + 'Install sources and keys lists are different lengths') + for source, key in zip(sources, keys): + add_source(source, key) if update: apt_update(fatal=True) diff --git a/hooks/charmhelpers/fetch/bzrurl.py b/hooks/charmhelpers/fetch/bzrurl.py index db5dd9a3..0e580e47 100644 --- a/hooks/charmhelpers/fetch/bzrurl.py +++ b/hooks/charmhelpers/fetch/bzrurl.py @@ -39,7 +39,8 @@ class BzrUrlFetchHandler(BaseFetchHandler): def install(self, source): url_parts = self.parse_url(source) branch_name = url_parts.path.strip("/").split("/")[-1] - dest_dir = os.path.join(os.environ.get('CHARM_DIR'), "fetched", branch_name) + dest_dir = os.path.join(os.environ.get('CHARM_DIR'), "fetched", + branch_name) if not os.path.exists(dest_dir): mkdir(dest_dir, perms=0755) try: diff --git a/tests/charmhelpers/__init__.py b/tests/charmhelpers/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/tests/charmhelpers/contrib/__init__.py b/tests/charmhelpers/contrib/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/tests/charmhelpers/contrib/amulet/__init__.py b/tests/charmhelpers/contrib/amulet/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/tests/charmhelpers/contrib/amulet/deployment.py b/tests/charmhelpers/contrib/amulet/deployment.py new file mode 100644 index 00000000..9e194320 --- /dev/null +++ b/tests/charmhelpers/contrib/amulet/deployment.py @@ -0,0 +1,63 @@ +import amulet +import re + + +class AmuletDeployment(object): + """This class provides generic Amulet deployment and test runner + methods.""" + + def __init__(self, series): + """Initialize the deployment environment.""" + self.series = series + self.d = amulet.Deployment(series=self.series) + + def _get_charm_name(self, service_name): + """Gets the charm name from the service name. Unique service names can + be specified with a '-service#' suffix (e.g. mysql-service1).""" + if re.match(r"^.*-service\d{1,3}$", service_name): + charm_name = re.sub('\-service\d{1,3}$', '', service_name) + else: + charm_name = service_name + return charm_name + + def _add_services(self, this_service, other_services): + """Add services to the deployment where this_service is the local charm + that we're focused on testing and other_services are the other + charms that come from the charm store.""" + name, units = range(2) + + charm_name = self._get_charm_name(this_service[name]) + self.d.add(this_service[name], + units=this_service[units]) + + for svc in other_services: + charm_name = self._get_charm_name(svc[name]) + self.d.add(svc[name], + charm='cs:{}/{}'.format(self.series, charm_name), + units=svc[units]) + + def _add_relations(self, relations): + """Add all of the relations for the services.""" + for k, v in relations.iteritems(): + self.d.relate(k, v) + + def _configure_services(self, configs): + """Configure all of the services.""" + for service, config in configs.iteritems(): + self.d.configure(service, config) + + def _deploy(self): + """Deploy environment and wait for all hooks to finish executing.""" + try: + self.d.setup() + self.d.sentry.wait() + except amulet.helpers.TimeoutError: + amulet.raise_status(amulet.FAIL, msg="Deployment timed out") + except: + raise + + def run_tests(self): + """Run all of the methods that are prefixed with 'test_'.""" + for test in dir(self): + if test.startswith('test_'): + getattr(self, test)() diff --git a/tests/charmhelpers/contrib/amulet/utils.py b/tests/charmhelpers/contrib/amulet/utils.py new file mode 100644 index 00000000..03e4a818 --- /dev/null +++ b/tests/charmhelpers/contrib/amulet/utils.py @@ -0,0 +1,157 @@ +import ConfigParser +import io +import logging +import re +import sys +from time import sleep + + +class AmuletUtils(object): + """This class provides common utility functions that are used by Amulet + tests.""" + + def __init__(self, log_level=logging.ERROR): + self.log = self.get_logger(level=log_level) + + def get_logger(self, name="amulet-logger", level=logging.DEBUG): + """Get a logger object that will log to stdout.""" + log = logging + logger = log.getLogger(name) + fmt = \ + log.Formatter("%(asctime)s %(funcName)s %(levelname)s: %(message)s") + + handler = log.StreamHandler(stream=sys.stdout) + handler.setLevel(level) + handler.setFormatter(fmt) + + logger.addHandler(handler) + logger.setLevel(level) + + return logger + + def valid_ip(self, ip): + if re.match(r"^\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3}$", ip): + return True + else: + return False + + def valid_url(self, url): + p = re.compile( + r'^(?:http|ftp)s?://' + r'(?:(?:[A-Z0-9](?:[A-Z0-9-]{0,61}[A-Z0-9])?\.)+(?:[A-Z]{2,6}\.?|[A-Z0-9-]{2,}\.?)|' # flake8: noqa + r'localhost|' + r'\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3})' + r'(?::\d+)?' + r'(?:/?|[/?]\S+)$', + re.IGNORECASE) + if p.match(url): + return True + else: + return False + + def validate_services(self, commands): + """Verify the specified services are running on the corresponding + service units.""" + for k, v in commands.iteritems(): + for cmd in v: + output, code = k.run(cmd) + if code != 0: + return "command `{}` returned {}".format(cmd, str(code)) + return None + + def _get_config(self, unit, filename): + """Get a ConfigParser object for parsing a unit's config file.""" + file_contents = unit.file_contents(filename) + config = ConfigParser.ConfigParser() + config.readfp(io.StringIO(file_contents)) + return config + + def validate_config_data(self, sentry_unit, config_file, section, expected): + """Verify that the specified section of the config file contains + the expected option key:value pairs.""" + config = self._get_config(sentry_unit, config_file) + + if section != 'DEFAULT' and not config.has_section(section): + return "section [{}] does not exist".format(section) + + for k in expected.keys(): + if not config.has_option(section, k): + return "section [{}] is missing option {}".format(section, k) + if config.get(section, k) != expected[k]: + return "section [{}] {}:{} != expected {}:{}".format(section, + k, config.get(section, k), k, expected[k]) + return None + + def _validate_dict_data(self, expected, actual): + """Compare expected dictionary data vs actual dictionary data. + The values in the 'expected' dictionary can be strings, bools, ints, + longs, or can be a function that evaluate a variable and returns a + bool.""" + for k, v in expected.iteritems(): + if k in actual: + if isinstance(v, basestring) or \ + isinstance(v, bool) or \ + isinstance(v, (int, long)): + if v != actual[k]: + return "{}:{}".format(k, actual[k]) + elif not v(actual[k]): + return "{}:{}".format(k, actual[k]) + else: + return "key '{}' does not exist".format(k) + return None + + def validate_relation_data(self, sentry_unit, relation, expected): + """Validate actual relation data based on expected relation data.""" + actual = sentry_unit.relation(relation[0], relation[1]) + self.log.debug('actual: {}'.format(repr(actual))) + return self._validate_dict_data(expected, actual) + + def _validate_list_data(self, expected, actual): + """Compare expected list vs actual list data.""" + for e in expected: + if e not in actual: + return "expected item {} not found in actual list".format(e) + return None + + def not_null(self, string): + if string != None: + return True + else: + return False + + def _get_file_mtime(self, sentry_unit, filename): + """Get last modification time of file.""" + return sentry_unit.file_stat(filename)['mtime'] + + def _get_dir_mtime(self, sentry_unit, directory): + """Get last modification time of directory.""" + return sentry_unit.directory_stat(directory)['mtime'] + + def _get_proc_start_time(self, sentry_unit, service, pgrep_full=False): + """Determine start time of the process based on the last modification + time of the /proc/pid directory. If pgrep_full is True, the process + name is matched against the full command line.""" + if pgrep_full: + cmd = 'pgrep -o -f {}'.format(service) + else: + cmd = 'pgrep -o {}'.format(service) + proc_dir = '/proc/{}'.format(sentry_unit.run(cmd)[0].strip()) + return self._get_dir_mtime(sentry_unit, proc_dir) + + def service_restarted(self, sentry_unit, service, filename, + pgrep_full=False, sleep_time=20): + """Compare a service's start time vs a file's last modification time + (such as a config file for that service) to determine if the service + has been restarted.""" + sleep(sleep_time) + if self._get_proc_start_time(sentry_unit, service, pgrep_full) >= \ + self._get_file_mtime(sentry_unit, filename): + return True + else: + return False + + def relation_error(self, name, data): + return 'unexpected relation data in {} - {}'.format(name, data) + + def endpoint_error(self, name, data): + return 'unexpected endpoint data in {} - {}'.format(name, data) diff --git a/tests/charmhelpers/contrib/openstack/__init__.py b/tests/charmhelpers/contrib/openstack/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/tests/charmhelpers/contrib/openstack/amulet/__init__.py b/tests/charmhelpers/contrib/openstack/amulet/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/tests/charmhelpers/contrib/openstack/amulet/deployment.py b/tests/charmhelpers/contrib/openstack/amulet/deployment.py new file mode 100644 index 00000000..de2a6bfe --- /dev/null +++ b/tests/charmhelpers/contrib/openstack/amulet/deployment.py @@ -0,0 +1,57 @@ +from charmhelpers.contrib.amulet.deployment import ( + AmuletDeployment +) + + +class OpenStackAmuletDeployment(AmuletDeployment): + """This class inherits from AmuletDeployment and has additional support + that is specifically for use by OpenStack charms.""" + + def __init__(self, series, openstack=None, source=None): + """Initialize the deployment environment.""" + super(OpenStackAmuletDeployment, self).__init__(series) + self.openstack = openstack + self.source = source + + def _add_services(self, this_service, other_services): + """Add services to the deployment and set openstack-origin.""" + super(OpenStackAmuletDeployment, self)._add_services(this_service, + other_services) + name = 0 + services = other_services + services.append(this_service) + use_source = ['mysql', 'mongodb', 'rabbitmq-server', 'ceph'] + + if self.openstack: + for svc in services: + charm_name = self._get_charm_name(svc[name]) + if charm_name not in use_source: + config = {'openstack-origin': self.openstack} + self.d.configure(svc[name], config) + + if self.source: + for svc in services: + charm_name = self._get_charm_name(svc[name]) + if charm_name in use_source: + config = {'source': self.source} + self.d.configure(svc[name], config) + + def _configure_services(self, configs): + """Configure all of the services.""" + for service, config in configs.iteritems(): + self.d.configure(service, config) + + def _get_openstack_release(self): + """Return an integer representing the enum value of the openstack + release.""" + self.precise_essex, self.precise_folsom, self.precise_grizzly, \ + self.precise_havana, self.precise_icehouse, \ + self.trusty_icehouse = range(6) + releases = { + ('precise', None): self.precise_essex, + ('precise', 'cloud:precise-folsom'): self.precise_folsom, + ('precise', 'cloud:precise-grizzly'): self.precise_grizzly, + ('precise', 'cloud:precise-havana'): self.precise_havana, + ('precise', 'cloud:precise-icehouse'): self.precise_icehouse, + ('trusty', None): self.trusty_icehouse} + return releases[(self.series, self.openstack)] diff --git a/tests/charmhelpers/contrib/openstack/amulet/utils.py b/tests/charmhelpers/contrib/openstack/amulet/utils.py new file mode 100644 index 00000000..806bd212 --- /dev/null +++ b/tests/charmhelpers/contrib/openstack/amulet/utils.py @@ -0,0 +1,253 @@ +import logging +import os +import time +import urllib + +import glanceclient.v1.client as glance_client +import keystoneclient.v2_0 as keystone_client +import novaclient.v1_1.client as nova_client + +from charmhelpers.contrib.amulet.utils import ( + AmuletUtils +) + +DEBUG = logging.DEBUG +ERROR = logging.ERROR + + +class OpenStackAmuletUtils(AmuletUtils): + """This class inherits from AmuletUtils and has additional support + that is specifically for use by OpenStack charms.""" + + def __init__(self, log_level=ERROR): + """Initialize the deployment environment.""" + super(OpenStackAmuletUtils, self).__init__(log_level) + + def validate_endpoint_data(self, endpoints, admin_port, internal_port, + public_port, expected): + """Validate actual endpoint data vs expected endpoint data. The ports + are used to find the matching endpoint.""" + found = False + for ep in endpoints: + self.log.debug('endpoint: {}'.format(repr(ep))) + if admin_port in ep.adminurl and internal_port in ep.internalurl \ + and public_port in ep.publicurl: + found = True + actual = {'id': ep.id, + 'region': ep.region, + 'adminurl': ep.adminurl, + 'internalurl': ep.internalurl, + 'publicurl': ep.publicurl, + 'service_id': ep.service_id} + ret = self._validate_dict_data(expected, actual) + if ret: + return 'unexpected endpoint data - {}'.format(ret) + + if not found: + return 'endpoint not found' + + def validate_svc_catalog_endpoint_data(self, expected, actual): + """Validate a list of actual service catalog endpoints vs a list of + expected service catalog endpoints.""" + self.log.debug('actual: {}'.format(repr(actual))) + for k, v in expected.iteritems(): + if k in actual: + ret = self._validate_dict_data(expected[k][0], actual[k][0]) + if ret: + return self.endpoint_error(k, ret) + else: + return "endpoint {} does not exist".format(k) + return ret + + def validate_tenant_data(self, expected, actual): + """Validate a list of actual tenant data vs list of expected tenant + data.""" + self.log.debug('actual: {}'.format(repr(actual))) + for e in expected: + found = False + for act in actual: + a = {'enabled': act.enabled, 'description': act.description, + 'name': act.name, 'id': act.id} + if e['name'] == a['name']: + found = True + ret = self._validate_dict_data(e, a) + if ret: + return "unexpected tenant data - {}".format(ret) + if not found: + return "tenant {} does not exist".format(e['name']) + return ret + + def validate_role_data(self, expected, actual): + """Validate a list of actual role data vs a list of expected role + data.""" + self.log.debug('actual: {}'.format(repr(actual))) + for e in expected: + found = False + for act in actual: + a = {'name': act.name, 'id': act.id} + if e['name'] == a['name']: + found = True + ret = self._validate_dict_data(e, a) + if ret: + return "unexpected role data - {}".format(ret) + if not found: + return "role {} does not exist".format(e['name']) + return ret + + def validate_user_data(self, expected, actual): + """Validate a list of actual user data vs a list of expected user + data.""" + self.log.debug('actual: {}'.format(repr(actual))) + for e in expected: + found = False + for act in actual: + a = {'enabled': act.enabled, 'name': act.name, + 'email': act.email, 'tenantId': act.tenantId, + 'id': act.id} + if e['name'] == a['name']: + found = True + ret = self._validate_dict_data(e, a) + if ret: + return "unexpected user data - {}".format(ret) + if not found: + return "user {} does not exist".format(e['name']) + return ret + + def validate_flavor_data(self, expected, actual): + """Validate a list of actual flavors vs a list of expected flavors.""" + self.log.debug('actual: {}'.format(repr(actual))) + act = [a.name for a in actual] + return self._validate_list_data(expected, act) + + def tenant_exists(self, keystone, tenant): + """Return True if tenant exists""" + return tenant in [t.name for t in keystone.tenants.list()] + + def authenticate_keystone_admin(self, keystone_sentry, user, password, + tenant): + """Authenticates admin user with the keystone admin endpoint.""" + service_ip = \ + keystone_sentry.relation('shared-db', + 'mysql:shared-db')['private-address'] + ep = "http://{}:35357/v2.0".format(service_ip.strip().decode('utf-8')) + return keystone_client.Client(username=user, password=password, + tenant_name=tenant, auth_url=ep) + + def authenticate_keystone_user(self, keystone, user, password, tenant): + """Authenticates a regular user with the keystone public endpoint.""" + ep = keystone.service_catalog.url_for(service_type='identity', + endpoint_type='publicURL') + return keystone_client.Client(username=user, password=password, + tenant_name=tenant, auth_url=ep) + + def authenticate_glance_admin(self, keystone): + """Authenticates admin user with glance.""" + ep = keystone.service_catalog.url_for(service_type='image', + endpoint_type='adminURL') + return glance_client.Client(ep, token=keystone.auth_token) + + def authenticate_nova_user(self, keystone, user, password, tenant): + """Authenticates a regular user with nova-api.""" + ep = keystone.service_catalog.url_for(service_type='identity', + endpoint_type='publicURL') + return nova_client.Client(username=user, api_key=password, + project_id=tenant, auth_url=ep) + + def create_cirros_image(self, glance, image_name): + """Download the latest cirros image and upload it to glance.""" + http_proxy = os.getenv('AMULET_HTTP_PROXY') + self.log.debug('AMULET_HTTP_PROXY: {}'.format(http_proxy)) + if http_proxy: + proxies = {'http': http_proxy} + opener = urllib.FancyURLopener(proxies) + else: + opener = urllib.FancyURLopener() + + f = opener.open("http://download.cirros-cloud.net/version/released") + version = f.read().strip() + cirros_img = "tests/cirros-{}-x86_64-disk.img".format(version) + + if not os.path.exists(cirros_img): + cirros_url = "http://{}/{}/{}".format("download.cirros-cloud.net", + version, cirros_img) + opener.retrieve(cirros_url, cirros_img) + f.close() + + with open(cirros_img) as f: + image = glance.images.create(name=image_name, is_public=True, + disk_format='qcow2', + container_format='bare', data=f) + count = 1 + status = image.status + while status != 'active' and count < 10: + time.sleep(3) + image = glance.images.get(image.id) + status = image.status + self.log.debug('image status: {}'.format(status)) + count += 1 + + if status != 'active': + self.log.error('image creation timed out') + return None + + return image + + def delete_image(self, glance, image): + """Delete the specified image.""" + num_before = len(list(glance.images.list())) + glance.images.delete(image) + + count = 1 + num_after = len(list(glance.images.list())) + while num_after != (num_before - 1) and count < 10: + time.sleep(3) + num_after = len(list(glance.images.list())) + self.log.debug('number of images: {}'.format(num_after)) + count += 1 + + if num_after != (num_before - 1): + self.log.error('image deletion timed out') + return False + + return True + + def create_instance(self, nova, image_name, instance_name, flavor): + """Create the specified instance.""" + image = nova.images.find(name=image_name) + flavor = nova.flavors.find(name=flavor) + instance = nova.servers.create(name=instance_name, image=image, + flavor=flavor) + + count = 1 + status = instance.status + while status != 'ACTIVE' and count < 60: + time.sleep(3) + instance = nova.servers.get(instance.id) + status = instance.status + self.log.debug('instance status: {}'.format(status)) + count += 1 + + if status != 'ACTIVE': + self.log.error('instance creation timed out') + return None + + return instance + + def delete_instance(self, nova, instance): + """Delete the specified instance.""" + num_before = len(list(nova.servers.list())) + nova.servers.delete(instance) + + count = 1 + num_after = len(list(nova.servers.list())) + while num_after != (num_before - 1) and count < 10: + time.sleep(3) + num_after = len(list(nova.servers.list())) + self.log.debug('number of instances: {}'.format(num_after)) + count += 1 + + if num_after != (num_before - 1): + self.log.error('instance deletion timed out') + return False + + return True From 34577fafae584515d07dd921b31b902a5e31e82d Mon Sep 17 00:00:00 2001 From: Corey Bryant Date: Fri, 11 Jul 2014 17:34:39 +0000 Subject: [PATCH 27/48] Add Amulet basic tests --- Makefile | 13 +- tests/00-setup | 10 + tests/10-basic-precise-essex | 10 + tests/11-basic-precise-folsom | 18 ++ tests/12-basic-precise-grizzly | 12 + tests/13-basic-precise-havana | 12 + tests/14-basic-precise-icehouse | 12 + tests/15-basic-trusty-icehouse | 10 + tests/README | 47 +++ tests/basic_deployment.py | 520 ++++++++++++++++++++++++++++++++ 10 files changed, 661 insertions(+), 3 deletions(-) create mode 100755 tests/00-setup create mode 100755 tests/10-basic-precise-essex create mode 100755 tests/11-basic-precise-folsom create mode 100755 tests/12-basic-precise-grizzly create mode 100755 tests/13-basic-precise-havana create mode 100755 tests/14-basic-precise-icehouse create mode 100755 tests/15-basic-trusty-icehouse create mode 100644 tests/README create mode 100644 tests/basic_deployment.py diff --git a/Makefile b/Makefile index 7d92e615..b7ccbd73 100644 --- a/Makefile +++ b/Makefile @@ -2,13 +2,20 @@ PYTHON := /usr/bin/env python lint: - @flake8 --exclude hooks/charmhelpers hooks unit_tests + @flake8 --exclude hooks/charmhelpers hooks unit_tests tests @charm proof -test: - @echo Starting tests... +unit_test: + @echo Starting unit tests... @$(PYTHON) /usr/bin/nosetests --nologcapture --with-coverage unit_tests +test: + @echo Starting Amulet tests... + # coreycb note: The -v should only be temporary until Amulet sends + # raise_status() messages to stderr: + # https://bugs.launchpad.net/amulet/+bug/1320357 + @juju test -v -p AMULET_HTTP_PROXY + sync: @charm-helper-sync -c charm-helpers-hooks.yaml @charm-helper-sync -c charm-helpers-tests.yaml diff --git a/tests/00-setup b/tests/00-setup new file mode 100755 index 00000000..a877fa00 --- /dev/null +++ b/tests/00-setup @@ -0,0 +1,10 @@ +#!/bin/bash + +set -ex + +sudo add-apt-repository --yes ppa:juju/stable +sudo apt-get update --yes +sudo apt-get install --yes python-amulet +sudo apt-get install --yes python-glanceclient +sudo apt-get install --yes python-keystoneclient +sudo apt-get install --yes python-novaclient diff --git a/tests/10-basic-precise-essex b/tests/10-basic-precise-essex new file mode 100755 index 00000000..d5fd239a --- /dev/null +++ b/tests/10-basic-precise-essex @@ -0,0 +1,10 @@ +#!/usr/bin/python + +"""Amulet tests on a basic nova cloud controller deployment on + precise-essex.""" + +from basic_deployment import NovaCCBasicDeployment + +if __name__ == '__main__': + deployment = NovaCCBasicDeployment(series='precise') + deployment.run_tests() diff --git a/tests/11-basic-precise-folsom b/tests/11-basic-precise-folsom new file mode 100755 index 00000000..4be178c3 --- /dev/null +++ b/tests/11-basic-precise-folsom @@ -0,0 +1,18 @@ +#!/usr/bin/python + +"""Amulet tests on a basic nova cloud controller deployment on + precise-folsom.""" + +import amulet +from basic_deployment import NovaCCBasicDeployment + +if __name__ == '__main__': + # NOTE(coreycb): Skipping failing test until resolved. 'nova-manage db sync' + # fails in shared-db-relation-changed (only fails on folsom) + message = "Skipping failing test until resolved" + amulet.raise_status(amulet.SKIP, msg=message) + + deployment = NovaCCBasicDeployment(series='precise', + openstack='cloud:precise-folsom', + source='cloud:precise-updates/folsom') + deployment.run_tests() diff --git a/tests/12-basic-precise-grizzly b/tests/12-basic-precise-grizzly new file mode 100755 index 00000000..29fe56c5 --- /dev/null +++ b/tests/12-basic-precise-grizzly @@ -0,0 +1,12 @@ +#!/usr/bin/python + +"""Amulet tests on a basic nova cloud controller deployment on + precise-grizzly.""" + +from basic_deployment import NovaCCBasicDeployment + +if __name__ == '__main__': + deployment = NovaCCBasicDeployment(series='precise', + openstack='cloud:precise-grizzly', + source='cloud:precise-updates/grizzly') + deployment.run_tests() diff --git a/tests/13-basic-precise-havana b/tests/13-basic-precise-havana new file mode 100755 index 00000000..0841c113 --- /dev/null +++ b/tests/13-basic-precise-havana @@ -0,0 +1,12 @@ +#!/usr/bin/python + +"""Amulet tests on a basic nova cloud controller deployment on + precise-havana.""" + +from basic_deployment import NovaCCBasicDeployment + +if __name__ == '__main__': + deployment = NovaCCBasicDeployment(series='precise', + openstack='cloud:precise-havana', + source='cloud:precise-updates/havana') + deployment.run_tests() diff --git a/tests/14-basic-precise-icehouse b/tests/14-basic-precise-icehouse new file mode 100755 index 00000000..6ec6330b --- /dev/null +++ b/tests/14-basic-precise-icehouse @@ -0,0 +1,12 @@ +#!/usr/bin/python + +"""Amulet tests on a basic nova cloud controller deployment on + precise-icehouse.""" + +from basic_deployment import NovaCCBasicDeployment + +if __name__ == '__main__': + deployment = NovaCCBasicDeployment(series='precise', + openstack='cloud:precise-icehouse', + source='cloud:precise-updates/icehouse') + deployment.run_tests() diff --git a/tests/15-basic-trusty-icehouse b/tests/15-basic-trusty-icehouse new file mode 100755 index 00000000..14a3cbb0 --- /dev/null +++ b/tests/15-basic-trusty-icehouse @@ -0,0 +1,10 @@ +#!/usr/bin/python + +"""Amulet tests on a basic nova cloud controller deployment on + trusty-icehouse.""" + +from basic_deployment import NovaCCBasicDeployment + +if __name__ == '__main__': + deployment = NovaCCBasicDeployment(series='trusty') + deployment.run_tests() diff --git a/tests/README b/tests/README new file mode 100644 index 00000000..51539469 --- /dev/null +++ b/tests/README @@ -0,0 +1,47 @@ +This directory provides Amulet tests that focus on verification of Nova Cloud +Controller deployments. + +If you use a web proxy server to access the web, you'll need to set the +AMULET_HTTP_PROXY environment variable to the http URL of the proxy server. + +The following examples demonstrate different ways that tests can be executed. +All examples are run from the charm's root directory. + + * To run all tests (starting with 00-setup): + + make test + + * To run a specific test module (or modules): + + juju test -v -p AMULET_HTTP_PROXY 15-basic-trusty-icehouse + + * To run a specific test module (or modules), and keep the environment + deployed after a failure: + + juju test --set-e -v -p AMULET_HTTP_PROXY 15-basic-trusty-icehouse + + * To re-run a test module against an already deployed environment (one + that was deployed by a previous call to 'juju test --set-e'): + + ./tests/15-basic-trusty-icehouse + +For debugging and test development purposes, all code should be idempotent. +In other words, the code should have the ability to be re-run without changing +the results beyond the initial run. This enables editing and re-running of a +test module against an already deployed environment, as described above. + +Manual debugging tips: + + * Set the following env vars before using the OpenStack CLI as admin: + export OS_AUTH_URL=http://`juju-deployer -f keystone 2>&1 | tail -n 1`:5000/v2.0 + export OS_TENANT_NAME=admin + export OS_USERNAME=admin + export OS_PASSWORD=openstack + export OS_REGION_NAME=RegionOne + + * Set the following env vars before using the OpenStack CLI as demoUser: + export OS_AUTH_URL=http://`juju-deployer -f keystone 2>&1 | tail -n 1`:5000/v2.0 + export OS_TENANT_NAME=demoTenant + export OS_USERNAME=demoUser + export OS_PASSWORD=password + export OS_REGION_NAME=RegionOne diff --git a/tests/basic_deployment.py b/tests/basic_deployment.py new file mode 100644 index 00000000..c680e686 --- /dev/null +++ b/tests/basic_deployment.py @@ -0,0 +1,520 @@ +#!/usr/bin/python + +import amulet + +from charmhelpers.contrib.openstack.amulet.deployment import ( + OpenStackAmuletDeployment +) + +from charmhelpers.contrib.openstack.amulet.utils import ( + OpenStackAmuletUtils, + DEBUG, # flake8: noqa + ERROR +) + +# Use DEBUG to turn on debug logging +u = OpenStackAmuletUtils(ERROR) + + +class NovaCCBasicDeployment(OpenStackAmuletDeployment): + """Amulet tests on a basic nova cloud controller deployment.""" + + def __init__(self, series=None, openstack=None, source=None): + """Deploy the entire test environment.""" + super(NovaCCBasicDeployment, self).__init__(series, openstack, source) + self._add_services() + self._add_relations() + self._configure_services() + self._deploy() + self._initialize_tests() + + def _add_services(self): + """Add the service that we're testing, including the number of units, + where nova-cloud-controller is local, and the other charms are from + the charm store.""" + this_service = ('nova-cloud-controller', 1) + other_services = [('mysql', 1), ('rabbitmq-server', 1), + ('nova-compute', 2), ('keystone', 1), ('glance', 1)] + super(NovaCCBasicDeployment, self)._add_services(this_service, + other_services) + + def _add_relations(self): + """Add all of the relations for the services.""" + relations = { + 'nova-cloud-controller:shared-db': 'mysql:shared-db', + 'nova-cloud-controller:identity-service': 'keystone:identity-service', + 'nova-cloud-controller:amqp': 'rabbitmq-server:amqp', + 'nova-cloud-controller:cloud-compute': 'nova-compute:cloud-compute', + 'nova-cloud-controller:image-service': 'glance:image-service', + 'nova-compute:image-service': 'glance:image-service', + 'nova-compute:shared-db': 'mysql:shared-db', + 'nova-compute:amqp': 'rabbitmq-server:amqp', + 'keystone:shared-db': 'mysql:shared-db', + 'glance:identity-service': 'keystone:identity-service', + 'glance:shared-db': 'mysql:shared-db', + 'glance:amqp': 'rabbitmq-server:amqp' + } + super(NovaCCBasicDeployment, self)._add_relations(relations) + + def _configure_services(self): + """Configure all of the services.""" + keystone_config = {'admin-password': 'openstack', + 'admin-token': 'ubuntutesting'} + configs = {'keystone': keystone_config} + super(NovaCCBasicDeployment, self)._configure_services(configs) + + def _initialize_tests(self): + """Perform final initialization before tests get run.""" + # Access the sentries for inspecting service units + self.mysql_sentry = self.d.sentry.unit['mysql/0'] + self.keystone_sentry = self.d.sentry.unit['keystone/0'] + self.rabbitmq_sentry = self.d.sentry.unit['rabbitmq-server/0'] + self.nova_cc_sentry = self.d.sentry.unit['nova-cloud-controller/0'] + self.nova_compute_sentry = self.d.sentry.unit['nova-compute/0'] + self.glance_sentry = self.d.sentry.unit['glance/0'] + + # Authenticate admin with keystone + self.keystone = u.authenticate_keystone_admin(self.keystone_sentry, + user='admin', + password='openstack', + tenant='admin') + + # Authenticate admin with glance endpoint + self.glance = u.authenticate_glance_admin(self.keystone) + + # Create a demo tenant/role/user + self.demo_tenant = 'demoTenant' + self.demo_role = 'demoRole' + self.demo_user = 'demoUser' + if not u.tenant_exists(self.keystone, self.demo_tenant): + tenant = self.keystone.tenants.create(tenant_name=self.demo_tenant, + description='demo tenant', + enabled=True) + self.keystone.roles.create(name=self.demo_role) + self.keystone.users.create(name=self.demo_user, + password='password', + tenant_id=tenant.id, + email='demo@demo.com') + + # Authenticate demo user with keystone + self.keystone_demo = \ + u.authenticate_keystone_user(self.keystone, user=self.demo_user, + password='password', + tenant=self.demo_tenant) + + # Authenticate demo user with nova-api + self.nova_demo = u.authenticate_nova_user(self.keystone, + user=self.demo_user, + password='password', + tenant=self.demo_tenant) + + def test_services(self): + """Verify the expected services are running on the corresponding + service units.""" + commands = { + self.mysql_sentry: ['status mysql'], + self.rabbitmq_sentry: ['sudo service rabbitmq-server status'], + self.nova_cc_sentry: ['status nova-api-ec2', + 'status nova-api-os-compute', + 'status nova-objectstore', + 'status nova-cert', + 'status nova-scheduler'], + self.nova_compute_sentry: ['status nova-compute', + 'status nova-network', + 'status nova-api'], + self.keystone_sentry: ['status keystone'], + self.glance_sentry: ['status glance-registry', 'status glance-api'] + } + if self._get_openstack_release() >= self.precise_grizzly: + commands[self.nova_cc_sentry] = ['status nova-conductor'] + + ret = u.validate_services(commands) + if ret: + amulet.raise_status(amulet.FAIL, msg=ret) + + def test_service_catalog(self): + """Verify that the service catalog endpoint data is valid.""" + endpoint_vol = {'adminURL': u.valid_url, + 'region': 'RegionOne', + 'publicURL': u.valid_url, + 'internalURL': u.valid_url} + endpoint_id = {'adminURL': u.valid_url, + 'region': 'RegionOne', + 'publicURL': u.valid_url, + 'internalURL': u.valid_url} + if self._get_openstack_release() >= self.precise_folsom: + endpoint_vol['id'] = u.not_null + endpoint_id['id'] = u.not_null + expected = {'s3': [endpoint_vol], 'compute': [endpoint_vol], + 'ec2': [endpoint_vol], 'identity': [endpoint_id]} + actual = self.keystone_demo.service_catalog.get_endpoints() + + ret = u.validate_svc_catalog_endpoint_data(expected, actual) + if ret: + amulet.raise_status(amulet.FAIL, msg=ret) + + def test_openstack_compute_api_endpoint(self): + """Verify the openstack compute api (osapi) endpoint data.""" + endpoints = self.keystone.endpoints.list() + admin_port = internal_port = public_port = '8774' + expected = {'id': u.not_null, + 'region': 'RegionOne', + 'adminurl': u.valid_url, + 'internalurl': u.valid_url, + 'publicurl': u.valid_url, + 'service_id': u.not_null} + + ret = u.validate_endpoint_data(endpoints, admin_port, internal_port, + public_port, expected) + if ret: + message = 'osapi endpoint: {}'.format(ret) + amulet.raise_status(amulet.FAIL, msg=message) + + def test_ec2_api_endpoint(self): + """Verify the EC2 api endpoint data.""" + endpoints = self.keystone.endpoints.list() + admin_port = internal_port = public_port = '8773' + expected = {'id': u.not_null, + 'region': 'RegionOne', + 'adminurl': u.valid_url, + 'internalurl': u.valid_url, + 'publicurl': u.valid_url, + 'service_id': u.not_null} + + ret = u.validate_endpoint_data(endpoints, admin_port, internal_port, + public_port, expected) + if ret: + message = 'EC2 endpoint: {}'.format(ret) + amulet.raise_status(amulet.FAIL, msg=message) + + def test_s3_api_endpoint(self): + """Verify the S3 api endpoint data.""" + endpoints = self.keystone.endpoints.list() + admin_port = internal_port = public_port = '3333' + expected = {'id': u.not_null, + 'region': 'RegionOne', + 'adminurl': u.valid_url, + 'internalurl': u.valid_url, + 'publicurl': u.valid_url, + 'service_id': u.not_null} + + ret = u.validate_endpoint_data(endpoints, admin_port, internal_port, + public_port, expected) + if ret: + message = 'S3 endpoint: {}'.format(ret) + amulet.raise_status(amulet.FAIL, msg=message) + + def test_nova_cc_shared_db_relation(self): + """Verify the nova-cc to mysql shared-db relation data""" + unit = self.nova_cc_sentry + relation = ['shared-db', 'mysql:shared-db'] + expected = { + 'private-address': u.valid_ip, + 'nova_database': 'nova', + 'nova_username': 'nova', + 'nova_hostname': u.valid_ip + } + + ret = u.validate_relation_data(unit, relation, expected) + if ret: + message = u.relation_error('nova-cc shared-db', ret) + amulet.raise_status(amulet.FAIL, msg=message) + + def test_mysql_shared_db_relation(self): + """Verify the mysql to nova-cc shared-db relation data""" + unit = self.mysql_sentry + relation = ['shared-db', 'nova-cloud-controller:shared-db'] + expected = { + 'private-address': u.valid_ip, + 'nova_password': u.not_null, + 'db_host': u.valid_ip + } + + ret = u.validate_relation_data(unit, relation, expected) + if ret: + message = u.relation_error('mysql shared-db', ret) + amulet.raise_status(amulet.FAIL, msg=message) + + def test_nova_cc_identity_service_relation(self): + """Verify the nova-cc to keystone identity-service relation data""" + unit = self.nova_cc_sentry + relation = ['identity-service', 'keystone:identity-service'] + expected = { + 'nova_internal_url': u.valid_url, + 'nova_public_url': u.valid_url, + 's3_public_url': u.valid_url, + 's3_service': 's3', + 'ec2_admin_url': u.valid_url, + 'ec2_internal_url': u.valid_url, + 'nova_service': 'nova', + 's3_region': 'RegionOne', + 'private-address': u.valid_ip, + 'nova_region': 'RegionOne', + 'ec2_public_url': u.valid_url, + 'ec2_region': 'RegionOne', + 's3_internal_url': u.valid_url, + 's3_admin_url': u.valid_url, + 'nova_admin_url': u.valid_url, + 'ec2_service': 'ec2' + } + + ret = u.validate_relation_data(unit, relation, expected) + if ret: + message = u.relation_error('nova-cc identity-service', ret) + amulet.raise_status(amulet.FAIL, msg=message) + + def test_keystone_identity_service_relation(self): + """Verify the keystone to nova-cc identity-service relation data""" + unit = self.keystone_sentry + relation = ['identity-service', + 'nova-cloud-controller:identity-service'] + expected = { + 'service_protocol': 'http', + 'service_tenant': 'services', + 'admin_token': 'ubuntutesting', + 'service_password': u.not_null, + 'service_port': '5000', + 'auth_port': '35357', + 'auth_protocol': 'http', + 'private-address': u.valid_ip, + 'https_keystone': 'False', + 'auth_host': u.valid_ip, + 'service_username': 's3_ec2_nova', + 'service_tenant_id': u.not_null, + 'service_host': u.valid_ip + } + + ret = u.validate_relation_data(unit, relation, expected) + if ret: + message = u.relation_error('keystone identity-service', ret) + amulet.raise_status(amulet.FAIL, msg=message) + + def test_nova_cc_amqp_relation(self): + """Verify the nova-cc to rabbitmq-server amqp relation data""" + unit = self.nova_cc_sentry + relation = ['amqp', 'rabbitmq-server:amqp'] + expected = { + 'username': 'nova', + 'private-address': u.valid_ip, + 'vhost': 'openstack' + } + + ret = u.validate_relation_data(unit, relation, expected) + if ret: + message = u.relation_error('nova-cc amqp', ret) + amulet.raise_status(amulet.FAIL, msg=message) + + def test_rabbitmq_amqp_relation(self): + """Verify the rabbitmq-server to nova-cc amqp relation data""" + unit = self.rabbitmq_sentry + relation = ['amqp', 'nova-cloud-controller:amqp'] + expected = { + 'private-address': u.valid_ip, + 'password': u.not_null, + 'hostname': u.valid_ip + } + + ret = u.validate_relation_data(unit, relation, expected) + if ret: + message = u.relation_error('rabbitmq amqp', ret) + amulet.raise_status(amulet.FAIL, msg=message) + + def test_nova_cc_cloud_compute_relation(self): + """Verify the nova-cc to nova-compute cloud-compute relation data""" + unit = self.nova_cc_sentry + relation = ['cloud-compute', 'nova-compute:cloud-compute'] + expected = { + 'volume_service': 'cinder', + 'network_manager': 'flatdhcpmanager', + 'ec2_host': u.valid_ip, + 'private-address': u.valid_ip, + 'restart_trigger': u.not_null + } + if self._get_openstack_release() == self.precise_essex: + expected['volume_service'] = 'nova-volume' + + ret = u.validate_relation_data(unit, relation, expected) + if ret: + message = u.relation_error('nova-cc cloud-compute', ret) + amulet.raise_status(amulet.FAIL, msg=message) + + def test_nova_cloud_compute_relation(self): + """Verify the nova-compute to nova-cc cloud-compute relation data""" + unit = self.nova_compute_sentry + relation = ['cloud-compute', 'nova-cloud-controller:cloud-compute'] + expected = { + 'private-address': u.valid_ip, + } + + ret = u.validate_relation_data(unit, relation, expected) + if ret: + message = u.relation_error('nova-compute cloud-compute', ret) + amulet.raise_status(amulet.FAIL, msg=message) + + def test_nova_cc_image_service_relation(self): + """Verify the nova-cc to glance image-service relation data""" + unit = self.nova_cc_sentry + relation = ['image-service', 'glance:image-service'] + expected = { + 'private-address': u.valid_ip, + } + + ret = u.validate_relation_data(unit, relation, expected) + if ret: + message = u.relation_error('nova-cc image-service', ret) + amulet.raise_status(amulet.FAIL, msg=message) + + def test_glance_image_service_relation(self): + """Verify the glance to nova-cc image-service relation data""" + unit = self.glance_sentry + relation = ['image-service', 'nova-cloud-controller:image-service'] + expected = { + 'private-address': u.valid_ip, + 'glance-api-server': u.valid_url + } + + ret = u.validate_relation_data(unit, relation, expected) + if ret: + message = u.relation_error('glance image-service', ret) + amulet.raise_status(amulet.FAIL, msg=message) + + def test_restart_on_config_change(self): + """Verify that the specified services are restarted when the config + is changed.""" + # NOTE(coreycb): Skipping failing test on essex until resolved. + # config-flags don't take effect on essex. + if self._get_openstack_release() == self.precise_essex: + u.log.error("Skipping failing test until resolved") + return + + services = ['nova-api-ec2', 'nova-api-os-compute', 'nova-objectstore', + 'nova-cert', 'nova-scheduler', 'nova-conductor'] + self.d.configure('nova-cloud-controller', + {'config-flags': 'quota_cores=20,quota_instances=40,quota_ram=102400'}) + pgrep_full = True + + time = 20 + conf = '/etc/nova/nova.conf' + for s in services: + if not u.service_restarted(self.nova_cc_sentry, s, conf, + pgrep_full=True, sleep_time=time): + msg = "service {} didn't restart after config change".format(s) + amulet.raise_status(amulet.FAIL, msg=msg) + time = 0 + + def test_nova_default_config(self): + """Verify the data in the nova config file's default section.""" + # NOTE(coreycb): Currently no way to test on essex because config file + # has no section headers. + if self._get_openstack_release() == self.precise_essex: + return + + unit = self.nova_cc_sentry + conf = '/etc/nova/nova.conf' + rabbitmq_relation = self.rabbitmq_sentry.relation('amqp', + 'nova-cloud-controller:amqp') + glance_relation = self.glance_sentry.relation('image-service', + 'nova-cloud-controller:image-service') + mysql_relation = self.mysql_sentry.relation('shared-db', + 'nova-cloud-controller:shared-db') + db_uri = "mysql://{}:{}@{}/{}".format('nova', + mysql_relation['nova_password'], + mysql_relation['db_host'], + 'nova') + keystone_ep = self.keystone_demo.service_catalog.url_for(\ + service_type='identity', + endpoint_type='publicURL') + keystone_ec2 = "{}/ec2tokens".format(keystone_ep) + + expected = {'dhcpbridge_flagfile': '/etc/nova/nova.conf', + 'dhcpbridge': '/usr/bin/nova-dhcpbridge', + 'logdir': '/var/log/nova', + 'state_path': '/var/lib/nova', + 'lock_path': '/var/lock/nova', + 'force_dhcp_release': 'True', + 'iscsi_helper': 'tgtadm', + 'libvirt_use_virtio_for_bridges': 'True', + 'connection_type': 'libvirt', + 'root_helper': 'sudo nova-rootwrap /etc/nova/rootwrap.conf', + 'verbose': 'True', + 'ec2_private_dns_show_ip': 'True', + 'api_paste_config': '/etc/nova/api-paste.ini', + 'volumes_path': '/var/lib/nova/volumes', + 'enabled_apis': 'ec2,osapi_compute,metadata', + 'auth_strategy': 'keystone', + 'compute_driver': 'libvirt.LibvirtDriver', + 'keystone_ec2_url': keystone_ec2, + 'sql_connection': db_uri, + 'rabbit_userid': 'nova', + 'rabbit_virtual_host': 'openstack', + 'rabbit_password': rabbitmq_relation['password'], + 'rabbit_host': rabbitmq_relation['hostname'], + 'glance_api_servers': glance_relation['glance-api-server'], + 'network_manager': 'nova.network.manager.FlatDHCPManager', + 's3_listen_port': '3333', + 'osapi_compute_listen_port': '8774', + 'ec2_listen_port': '8773'} + + ret = u.validate_config_data(unit, conf, 'DEFAULT', expected) + if ret: + message = "nova config error: {}".format(ret) + amulet.raise_status(amulet.FAIL, msg=message) + + + def test_nova_keystone_authtoken_config(self): + """Verify the data in the nova config file's keystone_authtoken + section. This data only exists since icehouse.""" + if self._get_openstack_release() < self.precise_icehouse: + return + + unit = self.nova_cc_sentry + conf = '/etc/nova/nova.conf' + keystone_relation = self.keystone_sentry.relation('identity-service', + 'nova-cloud-controller:identity-service') + keystone_uri = "http://{}:{}/".format(keystone_relation['service_host'], + keystone_relation['service_port']) + expected = {'auth_uri': keystone_uri, + 'auth_host': keystone_relation['service_host'], + 'auth_port': keystone_relation['auth_port'], + 'auth_protocol': keystone_relation['auth_protocol'], + 'admin_tenant_name': keystone_relation['service_tenant'], + 'admin_user': keystone_relation['service_username'], + 'admin_password': keystone_relation['service_password']} + + ret = u.validate_config_data(unit, conf, 'keystone_authtoken', expected) + if ret: + message = "nova config error: {}".format(ret) + amulet.raise_status(amulet.FAIL, msg=message) + + def test_image_instance_create(self): + """Create an image/instance, verify they exist, and delete them.""" + # NOTE(coreycb): Skipping failing test on essex until resolved. essex + # nova API calls are getting "Malformed request url (HTTP + # 400)". + if self._get_openstack_release() == self.precise_essex: + u.log.error("Skipping failing test until resolved") + return + + image = u.create_cirros_image(self.glance, "cirros-image") + if not image: + amulet.raise_status(amulet.FAIL, msg="Image create failed") + + instance = u.create_instance(self.nova_demo, "cirros-image", "cirros", + "m1.tiny") + if not instance: + amulet.raise_status(amulet.FAIL, msg="Instance create failed") + + found = False + for instance in self.nova_demo.servers.list(): + if instance.name == 'cirros': + found = True + if instance.status != 'ACTIVE': + msg = "cirros instance is not active" + amulet.raise_status(amulet.FAIL, msg=message) + + if not found: + message = "nova cirros instance does not exist" + amulet.raise_status(amulet.FAIL, msg=message) + + u.delete_image(self.glance, image) + u.delete_instance(self.nova_demo, instance) From 43d879d4fee380b08be9e202eed393b66b60e6d3 Mon Sep 17 00:00:00 2001 From: Louis Bouchard Date: Wed, 16 Jul 2014 11:40:00 +0200 Subject: [PATCH 28/48] Fix unit test following uid->unit change in hook --- unit_tests/test_nova_cc_hooks.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/unit_tests/test_nova_cc_hooks.py b/unit_tests/test_nova_cc_hooks.py index 894c66fb..eaa3b978 100644 --- a/unit_tests/test_nova_cc_hooks.py +++ b/unit_tests/test_nova_cc_hooks.py @@ -103,7 +103,7 @@ class NovaCCHooksTests(CharmTestCase): self.ssh_authorized_keys_lines.return_value = [ 'auth_0', 'auth_1', 'auth_2'] hooks.compute_changed() - self.ssh_compute_add.assert_called_with('fookey', rid=None, uid=None) + self.ssh_compute_add.assert_called_with('fookey', rid=None, unit=None) expected_relations = [ call(relation_settings={'authorized_keys_0': 'auth_0'}, relation_id=None), @@ -132,7 +132,7 @@ class NovaCCHooksTests(CharmTestCase): 'auth_0', 'auth_1', 'auth_2'] hooks.compute_changed() self.ssh_compute_add.assert_called_with('fookey', user='nova', - rid=None, uid=None) + rid=None, unit=None) expected_relations = [ call(relation_settings={'nova_authorized_keys_0': 'auth_0'}, relation_id=None), From 67d87c5fcf2bb643b73301ce4f940a188bfb2dcd Mon Sep 17 00:00:00 2001 From: James Page Date: Wed, 16 Jul 2014 14:15:48 +0100 Subject: [PATCH 29/48] Support multi-network vip configuration --- hooks/nova_cc_hooks.py | 26 ++++++++++++++++++++++---- hooks/nova_cc_utils.py | 2 +- 2 files changed, 23 insertions(+), 5 deletions(-) diff --git a/hooks/nova_cc_hooks.py b/hooks/nova_cc_hooks.py index 1f0cea37..3e05e2d9 100755 --- a/hooks/nova_cc_hooks.py +++ b/hooks/nova_cc_hooks.py @@ -85,6 +85,11 @@ from charmhelpers.contrib.openstack.ip import ( PUBLIC, INTERNAL, ADMIN ) +from charmhelpers.contrib.network.ip import ( + get_iface_for_address, + get_netmask_for_address +) + hooks = Hooks() CONFIGS = register_configs() @@ -425,15 +430,28 @@ def cluster_changed(): def ha_joined(): config = get_hacluster_config() resources = { - 'res_nova_vip': 'ocf:heartbeat:IPaddr2', 'res_nova_haproxy': 'lsb:haproxy', } - vip_params = 'params ip="%s" cidr_netmask="%s" nic="%s"' % \ - (config['vip'], config['vip_cidr'], config['vip_iface']) resource_params = { - 'res_nova_vip': vip_params, 'res_nova_haproxy': 'op monitor interval="5s"' } + vip_group = [] + for vip in config['vip'].split(): + iface = get_iface_for_address(vip) + if iface is not None: + vip_key = 'res_nova_{}_vip'.format(iface) + resources[vip_key] = 'ocf:heartbeat:IPaddr2' + resource_params[vip_key] = ( + 'params ip="{vip}" cidr_netmask="{netmask}"' + ' nic="{iface}"'.format(vip=vip, + iface=iface, + netmask=get_netmask_for_address(vip)) + ) + vip_group.append(vip_key) + + if len(vip_group) > 1: + relation_set(groups={'grp_nova_vips': ' '.join(vip_group)}) + init_services = { 'res_nova_haproxy': 'haproxy' } diff --git a/hooks/nova_cc_utils.py b/hooks/nova_cc_utils.py index eb70f59e..db3aa52c 100644 --- a/hooks/nova_cc_utils.py +++ b/hooks/nova_cc_utils.py @@ -46,7 +46,7 @@ import nova_cc_context TEMPLATES = 'templates/' -CLUSTER_RES = 'res_nova_vip' +CLUSTER_RES = 'grp_nova_vips' # removed from original: charm-helper-sh BASE_PACKAGES = [ From f20d0e8709494e9a6edb34f92dbf313531daeee4 Mon Sep 17 00:00:00 2001 From: James Page Date: Wed, 16 Jul 2014 14:19:17 +0100 Subject: [PATCH 30/48] Tidy lint --- hooks/nova_cc_context.py | 1 + hooks/nova_cc_hooks.py | 6 +++--- hooks/nova_cc_utils.py | 24 ++++++++++++++---------- 3 files changed, 18 insertions(+), 13 deletions(-) diff --git a/hooks/nova_cc_context.py b/hooks/nova_cc_context.py index 3e75a012..8f7fccca 100644 --- a/hooks/nova_cc_context.py +++ b/hooks/nova_cc_context.py @@ -39,6 +39,7 @@ class ApacheSSLContext(context.ApacheSSLContext): class NeutronAPIContext(context.OSContextGenerator): + def __call__(self): log('Generating template context from neutron api relation') ctxt = {} diff --git a/hooks/nova_cc_hooks.py b/hooks/nova_cc_hooks.py index 3e05e2d9..61b77922 100755 --- a/hooks/nova_cc_hooks.py +++ b/hooks/nova_cc_hooks.py @@ -45,7 +45,7 @@ from charmhelpers.contrib.openstack.neutron import ( from nova_cc_context import ( NeutronAPIContext - ) +) from nova_cc_utils import ( api_port, @@ -120,7 +120,7 @@ def config_changed(): configure_https() CONFIGS.write_all() for r_id in relation_ids('identity-service'): - identity_joined(rid=r_id) + identity_joined(rid=r_id) @hooks.hook('amqp-relation-joined') @@ -451,7 +451,7 @@ def ha_joined(): if len(vip_group) > 1: relation_set(groups={'grp_nova_vips': ' '.join(vip_group)}) - + init_services = { 'res_nova_haproxy': 'haproxy' } diff --git a/hooks/nova_cc_utils.py b/hooks/nova_cc_utils.py index db3aa52c..85d5de78 100644 --- a/hooks/nova_cc_utils.py +++ b/hooks/nova_cc_utils.py @@ -654,30 +654,34 @@ def determine_endpoints(public_url, internal_url, admin_url): nova_internal_url = ('%s:%s/v2/$(tenant_id)s' % (internal_url, api_port('nova-api-os-compute'))) nova_admin_url = ('%s:%s/v2/$(tenant_id)s' % - (admin_url, api_port('nova-api-os-compute'))) + (admin_url, api_port('nova-api-os-compute'))) else: nova_public_url = ('%s:%s/v1.1/$(tenant_id)s' % (public_url, api_port('nova-api-os-compute'))) nova_internal_url = ('%s:%s/v1.1/$(tenant_id)s' % (internal_url, api_port('nova-api-os-compute'))) nova_admin_url = ('%s:%s/v1.1/$(tenant_id)s' % - (admin_url, api_port('nova-api-os-compute'))) - - ec2_public_url = '%s:%s/services/Cloud' % (public_url, api_port('nova-api-ec2')) - ec2_internal_url = '%s:%s/services/Cloud' % (internal_url, api_port('nova-api-ec2')) - ec2_admin_url = '%s:%s/services/Cloud' % (admin_url, api_port('nova-api-ec2')) - + (admin_url, api_port('nova-api-os-compute'))) + + ec2_public_url = '%s:%s/services/Cloud' % ( + public_url, api_port('nova-api-ec2')) + ec2_internal_url = '%s:%s/services/Cloud' % ( + internal_url, api_port('nova-api-ec2')) + ec2_admin_url = '%s:%s/services/Cloud' % (admin_url, + api_port('nova-api-ec2')) + nova_volume_public_url = ('%s:%s/v1/$(tenant_id)s' % (public_url, api_port('nova-api-os-compute'))) nova_volume_internal_url = ('%s:%s/v1/$(tenant_id)s' % - (internal_url, api_port('nova-api-os-compute'))) + (internal_url, + api_port('nova-api-os-compute'))) nova_volume_admin_url = ('%s:%s/v1/$(tenant_id)s' % - (admin_url, api_port('nova-api-os-compute'))) + (admin_url, api_port('nova-api-os-compute'))) neutron_public_url = '%s:%s' % (public_url, api_port('neutron-server')) neutron_internal_url = '%s:%s' % (internal_url, api_port('neutron-server')) neutron_admin_url = '%s:%s' % (admin_url, api_port('neutron-server')) - + s3_public_url = '%s:%s' % (public_url, api_port('nova-objectstore')) s3_internal_url = '%s:%s' % (internal_url, api_port('nova-objectstore')) s3_admin_url = '%s:%s' % (admin_url, api_port('nova-objectstore')) From dfb1b3425d1b1e37a328e69e1bc424127815bc93 Mon Sep 17 00:00:00 2001 From: James Page Date: Wed, 16 Jul 2014 14:42:38 +0100 Subject: [PATCH 31/48] Resync helpers, drop surplus vip config --- config.yaml | 14 +++++--------- hooks/charmhelpers/contrib/hahelpers/cluster.py | 4 ++-- hooks/charmhelpers/contrib/openstack/context.py | 13 +++++++++---- 3 files changed, 16 insertions(+), 15 deletions(-) diff --git a/config.yaml b/config.yaml index 40e2bb96..907e6369 100644 --- a/config.yaml +++ b/config.yaml @@ -97,15 +97,11 @@ options: # HA configuration settings vip: type: string - description: "Virtual IP to use to front API services in ha configuration" - vip_iface: - type: string - default: eth0 - description: "Network Interface where to place the Virtual IP" - vip_cidr: - type: int - default: 24 - description: "Netmask that will be used for the Virtual IP" + description: | + Virtual IP(s) to use to front API services in HA configuration. + . + If multiple networks are being used, a VIP should be provided for each + network, separated by spaces. ha-bindiface: type: string default: eth0 diff --git a/hooks/charmhelpers/contrib/hahelpers/cluster.py b/hooks/charmhelpers/contrib/hahelpers/cluster.py index dd89f347..5e9ff01e 100644 --- a/hooks/charmhelpers/contrib/hahelpers/cluster.py +++ b/hooks/charmhelpers/contrib/hahelpers/cluster.py @@ -146,12 +146,12 @@ def get_hacluster_config(): Obtains all relevant configuration from charm configuration required for initiating a relation to hacluster: - ha-bindiface, ha-mcastport, vip, vip_iface, vip_cidr + ha-bindiface, ha-mcastport, vip returns: dict: A dict containing settings keyed by setting name. raises: HAIncompleteConfig if settings are missing. ''' - settings = ['ha-bindiface', 'ha-mcastport', 'vip', 'vip_iface', 'vip_cidr'] + settings = ['ha-bindiface', 'ha-mcastport', 'vip'] conf = {} for setting in settings: conf[setting] = config_get(setting) diff --git a/hooks/charmhelpers/contrib/openstack/context.py b/hooks/charmhelpers/contrib/openstack/context.py index 1e8cfc72..92c41b23 100644 --- a/hooks/charmhelpers/contrib/openstack/context.py +++ b/hooks/charmhelpers/contrib/openstack/context.py @@ -25,6 +25,7 @@ from charmhelpers.core.hookenv import ( unit_get, unit_private_ip, ERROR, + INFO ) from charmhelpers.contrib.hahelpers.cluster import ( @@ -714,7 +715,7 @@ class SubordinateConfigContext(OSContextGenerator): self.interface = interface def __call__(self): - ctxt = {} + ctxt = {'sections': {}} for rid in relation_ids(self.interface): for unit in related_units(rid): sub_config = relation_get('subordinate_configuration', @@ -740,10 +741,14 @@ class SubordinateConfigContext(OSContextGenerator): sub_config = sub_config[self.config_file] for k, v in sub_config.iteritems(): - ctxt[k] = v + if k == 'sections': + for section, config_dict in v.iteritems(): + log("adding section '%s'" % (section)) + ctxt[k][section] = config_dict + else: + ctxt[k] = v - if not ctxt: - ctxt['sections'] = {} + log("%d section(s) found" % (len(ctxt['sections'])), level=INFO) return ctxt From e74e0f74a1ff9932afd64a90a1679391e4b63186 Mon Sep 17 00:00:00 2001 From: James Page Date: Thu, 24 Jul 2014 11:31:19 +0100 Subject: [PATCH 32/48] Rebase on trunk helpers --- charm-helpers.yaml | 2 +- .../charmhelpers/contrib/hahelpers/cluster.py | 4 +-- hooks/charmhelpers/contrib/network/ip.py | 30 +++++-------------- hooks/charmhelpers/core/host.py | 4 +++ 4 files changed, 14 insertions(+), 26 deletions(-) diff --git a/charm-helpers.yaml b/charm-helpers.yaml index 44c7a8b9..d6c3fe59 100644 --- a/charm-helpers.yaml +++ b/charm-helpers.yaml @@ -1,4 +1,4 @@ -branch: lp:~james-page/charm-helpers/network-splits +branch: lp:charm-helpers destination: hooks/charmhelpers include: - core diff --git a/hooks/charmhelpers/contrib/hahelpers/cluster.py b/hooks/charmhelpers/contrib/hahelpers/cluster.py index 5e9ff01e..505de6b2 100644 --- a/hooks/charmhelpers/contrib/hahelpers/cluster.py +++ b/hooks/charmhelpers/contrib/hahelpers/cluster.py @@ -163,7 +163,7 @@ def get_hacluster_config(): return conf -def canonical_url(configs, vip_setting='vip', address=None): +def canonical_url(configs, vip_setting='vip'): ''' Returns the correct HTTP URL to this host given the state of HTTPS configuration and hacluster. @@ -180,5 +180,5 @@ def canonical_url(configs, vip_setting='vip', address=None): if is_clustered(): addr = config_get(vip_setting) else: - addr = address or unit_get('private-address') + addr = unit_get('private-address') return '%s://%s' % (scheme, addr) diff --git a/hooks/charmhelpers/contrib/network/ip.py b/hooks/charmhelpers/contrib/network/ip.py index e0f9eb66..0972e91a 100644 --- a/hooks/charmhelpers/contrib/network/ip.py +++ b/hooks/charmhelpers/contrib/network/ip.py @@ -1,5 +1,7 @@ import sys +from functools import partial + from charmhelpers.fetch import apt_install from charmhelpers.core.hookenv import ( ERROR, log, @@ -62,10 +64,9 @@ def get_address_in_network(network, fallback=None, fatal=False): return str(cidr.ip) if network.version == 6 and netifaces.AF_INET6 in addresses: for addr in addresses[netifaces.AF_INET6]: - if 'fe80' not in addr['addr']: - netmask = addr['netmask'] + if not addr['addr'].startswith('fe80'): cidr = netaddr.IPNetwork("%s/%s" % (addr['addr'], - netmask)) + addr['netmask'])) if cidr in network: return str(cidr.ip) @@ -139,7 +140,7 @@ def _get_for_address(address, key): return addresses[netifaces.AF_INET][0][key] if address.version == 6 and netifaces.AF_INET6 in addresses: for addr in addresses[netifaces.AF_INET6]: - if 'fe80' not in addr['addr']: + if not addr['addr'].startswith('fe80'): cidr = netaddr.IPNetwork("%s/%s" % (addr['addr'], addr['netmask'])) if address in cidr: @@ -150,23 +151,6 @@ def _get_for_address(address, key): return None -def get_iface_for_address(address): - """Determine the physical interface to which an IP address could be bound +get_iface_for_address = partial(_get_for_address, key='iface') - :param address (str): An individual IPv4 or IPv6 address without a net - mask or subnet prefix. For example, '192.168.1.1'. - :returns str: Interface name or None if address is not bindable. - """ - return _get_for_address(address, 'iface') - - -def get_netmask_for_address(address): - """Determine the netmask of the physical interface to which and IP address - could be bound - - :param address (str): An individual IPv4 or IPv6 address without a net - mask or subnet prefix. For example, '192.168.1.1'. - :returns str: Netmask of configured interface or None if address is - not bindable. - """ - return _get_for_address(address, 'netmask') +get_netmask_for_address = partial(_get_for_address, key='netmask') diff --git a/hooks/charmhelpers/core/host.py b/hooks/charmhelpers/core/host.py index 8b617a42..d934f940 100644 --- a/hooks/charmhelpers/core/host.py +++ b/hooks/charmhelpers/core/host.py @@ -322,6 +322,10 @@ def cmp_pkgrevno(package, revno, pkgcache=None): import apt_pkg if not pkgcache: apt_pkg.init() + # Force Apt to build its cache in memory. That way we avoid race + # conditions with other applications building the cache in the same + # place. + apt_pkg.config.set("Dir::Cache::pkgcache", "") pkgcache = apt_pkg.Cache() pkg = pkgcache[package] return apt_pkg.version_compare(pkg.current_ver.ver_str, revno) From ad51df5cc55fda135a9c62dde91140378ae4c5d5 Mon Sep 17 00:00:00 2001 From: James Page Date: Fri, 25 Jul 2014 15:39:57 +0100 Subject: [PATCH 33/48] Add basic service guard support to inhibit services from runnig until minimum relation requirements have been fulfilled --- hooks/nova_cc_context.py | 1 + hooks/nova_cc_hooks.py | 16 +++++++++++++++- hooks/nova_cc_utils.py | 34 ++++++++++++++++++++++++++++++++++ 3 files changed, 50 insertions(+), 1 deletion(-) diff --git a/hooks/nova_cc_context.py b/hooks/nova_cc_context.py index 3e75a012..8f7fccca 100644 --- a/hooks/nova_cc_context.py +++ b/hooks/nova_cc_context.py @@ -39,6 +39,7 @@ class ApacheSSLContext(context.ApacheSSLContext): class NeutronAPIContext(context.OSContextGenerator): + def __call__(self): log('Generating template context from neutron api relation') ctxt = {} diff --git a/hooks/nova_cc_hooks.py b/hooks/nova_cc_hooks.py index 56d03693..659f22eb 100755 --- a/hooks/nova_cc_hooks.py +++ b/hooks/nova_cc_hooks.py @@ -70,7 +70,9 @@ from nova_cc_utils import ( NOVA_CONF, QUANTUM_CONF, NEUTRON_CONF, - QUANTUM_API_PASTE + QUANTUM_API_PASTE, + service_guard, + guard_map, ) from charmhelpers.contrib.hahelpers.cluster import ( @@ -103,6 +105,7 @@ def install(): @hooks.hook('config-changed') +@service_guard(guard_map(), CONFIGS) @restart_on_change(restart_map(), stopstart=True) def config_changed(): global CONFIGS @@ -121,6 +124,7 @@ def amqp_joined(relation_id=None): @hooks.hook('amqp-relation-changed') @hooks.hook('amqp-relation-departed') +@service_guard(guard_map(), CONFIGS) @restart_on_change(restart_map()) def amqp_changed(): if 'amqp' not in CONFIGS.complete_contexts(): @@ -179,6 +183,7 @@ def pgsql_neutron_db_joined(): @hooks.hook('shared-db-relation-changed') +@service_guard(guard_map(), CONFIGS) @restart_on_change(restart_map()) def db_changed(): if 'shared-db' not in CONFIGS.complete_contexts(): @@ -194,6 +199,7 @@ def db_changed(): @hooks.hook('pgsql-nova-db-relation-changed') +@service_guard(guard_map(), CONFIGS) @restart_on_change(restart_map()) def postgresql_nova_db_changed(): if 'pgsql-nova-db' not in CONFIGS.complete_contexts(): @@ -209,6 +215,7 @@ def postgresql_nova_db_changed(): @hooks.hook('pgsql-neutron-db-relation-changed') +@service_guard(guard_map(), CONFIGS) @restart_on_change(restart_map()) def postgresql_neutron_db_changed(): if network_manager() in ['neutron', 'quantum']: @@ -218,6 +225,7 @@ def postgresql_neutron_db_changed(): @hooks.hook('image-service-relation-changed') +@service_guard(guard_map(), CONFIGS) @restart_on_change(restart_map()) def image_service_changed(): if 'image-service' not in CONFIGS.complete_contexts(): @@ -236,6 +244,7 @@ def identity_joined(rid=None): @hooks.hook('identity-service-relation-changed') +@service_guard(guard_map(), CONFIGS) @restart_on_change(restart_map()) def identity_changed(): if 'identity-service' not in CONFIGS.complete_contexts(): @@ -259,6 +268,7 @@ def identity_changed(): @hooks.hook('nova-volume-service-relation-joined', 'cinder-volume-service-relation-joined') +@service_guard(guard_map(), CONFIGS) @restart_on_change(restart_map()) def volume_joined(): CONFIGS.write(NOVA_CONF) @@ -450,6 +460,7 @@ def quantum_joined(rid=None): @hooks.hook('cluster-relation-changed', 'cluster-relation-departed') +@service_guard(guard_map(), CONFIGS) @restart_on_change(restart_map(), stopstart=True) def cluster_changed(): CONFIGS.write_all() @@ -546,6 +557,7 @@ def nova_vmware_relation_joined(rid=None): @hooks.hook('nova-vmware-relation-changed') +@service_guard(guard_map(), CONFIGS) @restart_on_change(restart_map()) def nova_vmware_relation_changed(): CONFIGS.write('/etc/nova/nova.conf') @@ -577,6 +589,7 @@ def neutron_api_relation_joined(rid=None): @hooks.hook('neutron-api-relation-changed') +@service_guard(guard_map(), CONFIGS) @restart_on_change(restart_map()) def neutron_api_relation_changed(): CONFIGS.write(NOVA_CONF) @@ -587,6 +600,7 @@ def neutron_api_relation_changed(): @hooks.hook('neutron-api-relation-broken') +@service_guard(guard_map(), CONFIGS) @restart_on_change(restart_map()) def neutron_api_relation_broken(): if os.path.isfile('/etc/init/neutron-server.override'): diff --git a/hooks/nova_cc_utils.py b/hooks/nova_cc_utils.py index e2d0702d..bccd5e01 100644 --- a/hooks/nova_cc_utils.py +++ b/hooks/nova_cc_utils.py @@ -728,3 +728,37 @@ def neutron_plugin(): # quantum-plugin config setting can be safely overriden # as we only supported OVS in G/neutron return config('neutron-plugin') or config('quantum-plugin') + + +def guard_map(): + gmap = {} + rinterfaces = ['identity-service', 'amqp'] + if relation_ids('pgsql-nova-db'): + rinterfaces.append('pgsql-nova-db') + elif relation_ids('pgsql-neutron-db'): + rinterfaces.append('pgsql-nova-db') + rinterfaces.append('pgsql-neutron-db') + else: + rinterfaces.append('shared-db') + + for svc in BASE_SERVICES + ['neutron-server', 'quantum-server']: + gmap[svc] = rinterfaces + + return gmap + + +def service_guard(guard_map, contexts): + ''' Inhibit services in guard_map from running unless + required interfaces are found complete on contexts ''' + def wrap(f): + def wrapped_f(*args): + incomplete_services = {} + for service in guard_map: + for interface in guard_map[service]: + if interface not in contexts.complete_contexts(): + incomplete_services.append(service) + f(*args) + for service in incomplete_services: + service('stop', service) + return wrapped_f + return wrap From 88f576352dbde27f988f0c89217c4eff97bbe854 Mon Sep 17 00:00:00 2001 From: James Page Date: Fri, 25 Jul 2014 15:46:40 +0100 Subject: [PATCH 34/48] Fixup --- hooks/nova_cc_utils.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/hooks/nova_cc_utils.py b/hooks/nova_cc_utils.py index bccd5e01..0f8a9114 100644 --- a/hooks/nova_cc_utils.py +++ b/hooks/nova_cc_utils.py @@ -752,7 +752,7 @@ def service_guard(guard_map, contexts): required interfaces are found complete on contexts ''' def wrap(f): def wrapped_f(*args): - incomplete_services = {} + incomplete_services = [] for service in guard_map: for interface in guard_map[service]: if interface not in contexts.complete_contexts(): From f2293d312ef1281f5c52b385d8afbbd54c966b62 Mon Sep 17 00:00:00 2001 From: James Page Date: Fri, 25 Jul 2014 15:49:32 +0100 Subject: [PATCH 35/48] Sorry - its friday --- hooks/nova_cc_utils.py | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/hooks/nova_cc_utils.py b/hooks/nova_cc_utils.py index 0f8a9114..4ed11b8a 100644 --- a/hooks/nova_cc_utils.py +++ b/hooks/nova_cc_utils.py @@ -40,6 +40,7 @@ from charmhelpers.core.hookenv import ( from charmhelpers.core.host import ( service_start, + service ) @@ -753,12 +754,12 @@ def service_guard(guard_map, contexts): def wrap(f): def wrapped_f(*args): incomplete_services = [] - for service in guard_map: - for interface in guard_map[service]: + for svc in guard_map: + for interface in guard_map[svc]: if interface not in contexts.complete_contexts(): - incomplete_services.append(service) + incomplete_services.append(svc) f(*args) - for service in incomplete_services: - service('stop', service) + for svc in incomplete_services: + service('stop', svc) return wrapped_f return wrap From 6829a12f1285beb0fa24e5736697b2ff8ffeba34 Mon Sep 17 00:00:00 2001 From: James Page Date: Fri, 25 Jul 2014 15:51:36 +0100 Subject: [PATCH 36/48] Add conductor to list of services to stop --- hooks/nova_cc_utils.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/hooks/nova_cc_utils.py b/hooks/nova_cc_utils.py index 4ed11b8a..cc331646 100644 --- a/hooks/nova_cc_utils.py +++ b/hooks/nova_cc_utils.py @@ -742,7 +742,7 @@ def guard_map(): else: rinterfaces.append('shared-db') - for svc in BASE_SERVICES + ['neutron-server', 'quantum-server']: + for svc in BASE_SERVICES + ['neutron-server', 'quantum-server', 'nova-conductor']: gmap[svc] = rinterfaces return gmap From c1a8cd2622f44cd86d3772328e9fcb1a917e3e1e Mon Sep 17 00:00:00 2001 From: James Page Date: Fri, 25 Jul 2014 15:52:24 +0100 Subject: [PATCH 37/48] Add logging --- hooks/nova_cc_utils.py | 1 + 1 file changed, 1 insertion(+) diff --git a/hooks/nova_cc_utils.py b/hooks/nova_cc_utils.py index cc331646..10626be5 100644 --- a/hooks/nova_cc_utils.py +++ b/hooks/nova_cc_utils.py @@ -760,6 +760,7 @@ def service_guard(guard_map, contexts): incomplete_services.append(svc) f(*args) for svc in incomplete_services: + log('Service {} has unfulfilled interface requirements, stopping.'.format(svc)) service('stop', svc) return wrapped_f return wrap From afd08026c9553be69822e0b1420454ecd6baa291 Mon Sep 17 00:00:00 2001 From: James Page Date: Fri, 25 Jul 2014 15:57:48 +0100 Subject: [PATCH 38/48] Make termination of services conditional on them actually running --- hooks/nova_cc_utils.py | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/hooks/nova_cc_utils.py b/hooks/nova_cc_utils.py index 10626be5..d24680d9 100644 --- a/hooks/nova_cc_utils.py +++ b/hooks/nova_cc_utils.py @@ -40,7 +40,8 @@ from charmhelpers.core.hookenv import ( from charmhelpers.core.host import ( service_start, - service + service_stop, + service_running ) @@ -760,7 +761,8 @@ def service_guard(guard_map, contexts): incomplete_services.append(svc) f(*args) for svc in incomplete_services: - log('Service {} has unfulfilled interface requirements, stopping.'.format(svc)) - service('stop', svc) + if service_running(svc): + log('Service {} has unfulfilled interface requirements, stopping.'.format(svc)) + service_stop(svc) return wrapped_f return wrap From 95520a436e32eb1a2d233264ae243aa4970a701f Mon Sep 17 00:00:00 2001 From: James Page Date: Mon, 28 Jul 2014 12:39:11 +0100 Subject: [PATCH 39/48] Add missing helper --- hooks/charmhelpers/contrib/openstack/ip.py | 75 ++++++++++++++++++++++ 1 file changed, 75 insertions(+) create mode 100644 hooks/charmhelpers/contrib/openstack/ip.py diff --git a/hooks/charmhelpers/contrib/openstack/ip.py b/hooks/charmhelpers/contrib/openstack/ip.py new file mode 100644 index 00000000..7e7a536f --- /dev/null +++ b/hooks/charmhelpers/contrib/openstack/ip.py @@ -0,0 +1,75 @@ +from charmhelpers.core.hookenv import ( + config, + unit_get, +) + +from charmhelpers.contrib.network.ip import ( + get_address_in_network, + is_address_in_network, + is_ipv6, +) + +from charmhelpers.contrib.hahelpers.cluster import is_clustered + +PUBLIC = 'public' +INTERNAL = 'int' +ADMIN = 'admin' + +_address_map = { + PUBLIC: { + 'config': 'os-public-network', + 'fallback': 'public-address' + }, + INTERNAL: { + 'config': 'os-internal-network', + 'fallback': 'private-address' + }, + ADMIN: { + 'config': 'os-admin-network', + 'fallback': 'private-address' + } +} + + +def canonical_url(configs, endpoint_type=PUBLIC): + ''' + Returns the correct HTTP URL to this host given the state of HTTPS + configuration, hacluster and charm configuration. + + :configs OSTemplateRenderer: A config tempating object to inspect for + a complete https context. + :endpoint_type str: The endpoint type to resolve. + + :returns str: Base URL for services on the current service unit. + ''' + scheme = 'http' + if 'https' in configs.complete_contexts(): + scheme = 'https' + address = resolve_address(endpoint_type) + if is_ipv6(address): + address = "[{}]".format(address) + return '%s://%s' % (scheme, address) + + +def resolve_address(endpoint_type=PUBLIC): + resolved_address = None + if is_clustered(): + if config(_address_map[endpoint_type]['config']) is None: + # Assume vip is simple and pass back directly + resolved_address = config('vip') + else: + for vip in config('vip').split(): + if is_address_in_network( + config(_address_map[endpoint_type]['config']), + vip): + resolved_address = vip + else: + resolved_address = get_address_in_network( + config(_address_map[endpoint_type]['config']), + unit_get(_address_map[endpoint_type]['fallback']) + ) + if resolved_address is None: + raise ValueError('Unable to resolve a suitable IP address' + ' based on charm state and configuration') + else: + return resolved_address From f5ed6b80af0a683000cefc6e5c318a6c384d6667 Mon Sep 17 00:00:00 2001 From: James Page Date: Mon, 28 Jul 2014 13:05:42 +0100 Subject: [PATCH 40/48] [trivia] Add missing helper --- .coverage | 8 + charm-helpers-hooks.yaml | 1 + .../charmhelpers/contrib/network/__init__.py | 0 hooks/charmhelpers/contrib/network/ip.py | 156 ++++++++++++++++++ 4 files changed, 165 insertions(+) create mode 100644 .coverage create mode 100644 hooks/charmhelpers/contrib/network/__init__.py create mode 100644 hooks/charmhelpers/contrib/network/ip.py diff --git a/.coverage b/.coverage new file mode 100644 index 00000000..3a6594a6 --- /dev/null +++ b/.coverage @@ -0,0 +1,8 @@ +}q(U collectorqUcoverage v3.7.1qUlinesq}q(Um/home/jamespage/src/charms/landing/nova-cloud-controller/hooks/charmhelpers/contrib/openstack/alternatives.pyq]Uh/home/jamespage/src/charms/landing/nova-cloud-controller/hooks/charmhelpers/contrib/storage/linux/lvm.pyq]q(KK KK(K3KEKOeUg/home/jamespage/src/charms/landing/nova-cloud-controller/hooks/charmhelpers/contrib/storage/__init__.pyq ]q +KaU\/home/jamespage/src/charms/landing/nova-cloud-controller/hooks/charmhelpers/core/__init__.pyq ]q KaUr/home/jamespage/src/charms/landing/nova-cloud-controller/hooks/charmhelpers/contrib/openstack/amulet/deployment.pyq ]UO/home/jamespage/src/charms/landing/nova-cloud-controller/hooks/nova_cc_hooks.pyq]q(KKKKKK K KK!K%K*K/K3KLKSKUKVKYK[K\K]K^K`KaKfKiKjKmKnKoKpKqKtKuKzK{K|KKKKKKKKKKKKKKKKKKKKKKKKKKKKKKKKKKKKKKKKKKKKKKKKKKKKMMMM M%M2M3M4M5M6M8M9M;M<M?MAMBMCMDMEMGMHMIMJMMMNMOMQMRMSMTMUMVMWMXM[M\M]M_M`MaMdMhMkMlMoMpMqMrMtMuMvMyMzM{M|M}M~MMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMM!M$M%M*M5M6M7M8M9M:M;M<M=M>M?M@MCMDMFMGMHMIMJMMMNMPMQMRMSMTMUMVMYM`eUm/home/jamespage/src/charms/landing/nova-cloud-controller/hooks/charmhelpers/contrib/storage/linux/__init__.pyq]qKaUi/home/jamespage/src/charms/landing/nova-cloud-controller/hooks/charmhelpers/contrib/storage/linux/ceph.pyq]UO/home/jamespage/src/charms/landing/nova-cloud-controller/hooks/nova_cc_utils.pyq]q(KKKKKKK K +K KKKK)K.K0K2K6K7K8K9K:K;K?K@KAKBKCKFKGKHKIKJKKKLKOKPKQKSKTKUKVKWKXKYKZK[K\K^K_K`KaKbKcKdKeKfKgKhKiKjKkKlKmKnKpKqKrKtKuKvKwKxKyKzK{K|K}K~KKKKKKKKKKKKKKKKKKKKKKKKKKKKKKKKKKKKKKKKKKKKKKKKKKKKKKKKKKKKKKKKKKKKKKKKKKKKKKKKKKKKKMMMMM MMMMMMMMMM!M#M$M%M&M'M(M)M*M-M.M/M0M1M2M3M4M5M7M8M9M;M=M@MEMHMJMMMNMOMPMQMRMSMUMdMgMmMqMrMuMwMxM{MMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMM M +M MMMMMMMMMMMMMM M#M$M%M&M+M,M-M.M1M3M4M5M:M;M<M=M>M@MBMCMDMGMHMIMLMQMlMvMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMeUf/home/jamespage/src/charms/landing/nova-cloud-controller/hooks/charmhelpers/contrib/openstack/utils.pyq]q(KKKKK K KKKKKKKK!K%K&K'K(K)K*K+K,K0K1K2K3K4K5K6K7K;KK?K@KAKBKCKDKEKFKGKHKKKNKSKnKsK|KKKKKKM,M>MRMsMMMMeUi/home/jamespage/src/charms/landing/nova-cloud-controller/hooks/charmhelpers/contrib/hahelpers/__init__.pyq]qKaUk/home/jamespage/src/charms/landing/nova-cloud-controller/hooks/charmhelpers/contrib/openstack/templating.pyq]q(KKKK K KKKKKFKJKKKUKaKkKKKKKKKMMMeUm/home/jamespage/src/charms/landing/nova-cloud-controller/hooks/charmhelpers/contrib/openstack/amulet/utils.pyq]UX/home/jamespage/src/charms/landing/nova-cloud-controller/hooks/charmhelpers/core/host.pyq]q(KKK K +K K K KKKKKKK K%K.K4KAKWKbKmKyKKKKKKKKKKKKKKKKKKKKKKKMMMM%M0M:eUh/home/jamespage/src/charms/landing/nova-cloud-controller/hooks/charmhelpers/contrib/openstack/context.pyq]q(KKKKKK KKK&K+K/K1K4K5K8K?KJKoKpKrKvKwKzKKKKKKKKKKKKKKM M MMMMMM]M^M`MMMMMMMMMMMMMMMMM M MMMMM%M4MCMWMgMMMMMMMMMMMMMMeUg/home/jamespage/src/charms/landing/nova-cloud-controller/hooks/charmhelpers/contrib/hahelpers/apache.pyq ]q!(K K KK)K6eUm/home/jamespage/src/charms/landing/nova-cloud-controller/hooks/charmhelpers/contrib/storage/linux/loopback.pyq"]q#(KKKKK K-eUQ/home/jamespage/src/charms/landing/nova-cloud-controller/hooks/nova_cc_context.pyq$]q%(KKKK KKKKK K"K)K*KK?K@KAKBKIKLKMKNKQKRKUKVKYKZK]K_KaKfKkKnKuKKKKKKKKKMM*M0MCeUi/home/jamespage/src/charms/landing/nova-cloud-controller/hooks/charmhelpers/contrib/openstack/__init__.pyq7]q8KaUj/home/jamespage/src/charms/landing/nova-cloud-controller/hooks/charmhelpers/contrib/storage/linux/utils.pyq9]q:(KKKKK KK)eU\/home/jamespage/src/charms/landing/nova-cloud-controller/hooks/charmhelpers/payload/execd.pyq;]q<(KKKKK K KK$K0eU_/home/jamespage/src/charms/landing/nova-cloud-controller/hooks/charmhelpers/payload/__init__.pyq=]q>KaUa/home/jamespage/src/charms/landing/nova-cloud-controller/hooks/charmhelpers/contrib/network/ip.pyq?]q@(KKKKK +K KKKKKRK]KwKKeU_/home/jamespage/src/charms/landing/nova-cloud-controller/hooks/charmhelpers/contrib/__init__.pyqA]qBKaUh/home/jamespage/src/charms/landing/nova-cloud-controller/hooks/charmhelpers/contrib/hahelpers/cluster.pyqC]qD(K K +K KKKKK)K9KAKJKWKpKKKeUY/home/jamespage/src/charms/landing/nova-cloud-controller/hooks/charmhelpers/core/fstab.pyqE]qF(KKK K KKKKK!K)K+K2K9KCKJKRKhKiKpKqeuu. \ No newline at end of file diff --git a/charm-helpers-hooks.yaml b/charm-helpers-hooks.yaml index 35e5834f..e8d393c9 100644 --- a/charm-helpers-hooks.yaml +++ b/charm-helpers-hooks.yaml @@ -8,3 +8,4 @@ include: - contrib.hahelpers: - apache - payload.execd + - contrib.network.ip diff --git a/hooks/charmhelpers/contrib/network/__init__.py b/hooks/charmhelpers/contrib/network/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/hooks/charmhelpers/contrib/network/ip.py b/hooks/charmhelpers/contrib/network/ip.py new file mode 100644 index 00000000..0972e91a --- /dev/null +++ b/hooks/charmhelpers/contrib/network/ip.py @@ -0,0 +1,156 @@ +import sys + +from functools import partial + +from charmhelpers.fetch import apt_install +from charmhelpers.core.hookenv import ( + ERROR, log, +) + +try: + import netifaces +except ImportError: + apt_install('python-netifaces') + import netifaces + +try: + import netaddr +except ImportError: + apt_install('python-netaddr') + import netaddr + + +def _validate_cidr(network): + try: + netaddr.IPNetwork(network) + except (netaddr.core.AddrFormatError, ValueError): + raise ValueError("Network (%s) is not in CIDR presentation format" % + network) + + +def get_address_in_network(network, fallback=None, fatal=False): + """ + Get an IPv4 or IPv6 address within the network from the host. + + :param network (str): CIDR presentation format. For example, + '192.168.1.0/24'. + :param fallback (str): If no address is found, return fallback. + :param fatal (boolean): If no address is found, fallback is not + set and fatal is True then exit(1). + + """ + + def not_found_error_out(): + log("No IP address found in network: %s" % network, + level=ERROR) + sys.exit(1) + + if network is None: + if fallback is not None: + return fallback + else: + if fatal: + not_found_error_out() + + _validate_cidr(network) + network = netaddr.IPNetwork(network) + for iface in netifaces.interfaces(): + addresses = netifaces.ifaddresses(iface) + if network.version == 4 and netifaces.AF_INET in addresses: + addr = addresses[netifaces.AF_INET][0]['addr'] + netmask = addresses[netifaces.AF_INET][0]['netmask'] + cidr = netaddr.IPNetwork("%s/%s" % (addr, netmask)) + if cidr in network: + return str(cidr.ip) + if network.version == 6 and netifaces.AF_INET6 in addresses: + for addr in addresses[netifaces.AF_INET6]: + if not addr['addr'].startswith('fe80'): + cidr = netaddr.IPNetwork("%s/%s" % (addr['addr'], + addr['netmask'])) + if cidr in network: + return str(cidr.ip) + + if fallback is not None: + return fallback + + if fatal: + not_found_error_out() + + return None + + +def is_ipv6(address): + '''Determine whether provided address is IPv6 or not''' + try: + address = netaddr.IPAddress(address) + except netaddr.AddrFormatError: + # probably a hostname - so not an address at all! + return False + else: + return address.version == 6 + + +def is_address_in_network(network, address): + """ + Determine whether the provided address is within a network range. + + :param network (str): CIDR presentation format. For example, + '192.168.1.0/24'. + :param address: An individual IPv4 or IPv6 address without a net + mask or subnet prefix. For example, '192.168.1.1'. + :returns boolean: Flag indicating whether address is in network. + """ + try: + network = netaddr.IPNetwork(network) + except (netaddr.core.AddrFormatError, ValueError): + raise ValueError("Network (%s) is not in CIDR presentation format" % + network) + try: + address = netaddr.IPAddress(address) + except (netaddr.core.AddrFormatError, ValueError): + raise ValueError("Address (%s) is not in correct presentation format" % + address) + if address in network: + return True + else: + return False + + +def _get_for_address(address, key): + """Retrieve an attribute of or the physical interface that + the IP address provided could be bound to. + + :param address (str): An individual IPv4 or IPv6 address without a net + mask or subnet prefix. For example, '192.168.1.1'. + :param key: 'iface' for the physical interface name or an attribute + of the configured interface, for example 'netmask'. + :returns str: Requested attribute or None if address is not bindable. + """ + address = netaddr.IPAddress(address) + for iface in netifaces.interfaces(): + addresses = netifaces.ifaddresses(iface) + if address.version == 4 and netifaces.AF_INET in addresses: + addr = addresses[netifaces.AF_INET][0]['addr'] + netmask = addresses[netifaces.AF_INET][0]['netmask'] + cidr = netaddr.IPNetwork("%s/%s" % (addr, netmask)) + if address in cidr: + if key == 'iface': + return iface + else: + return addresses[netifaces.AF_INET][0][key] + if address.version == 6 and netifaces.AF_INET6 in addresses: + for addr in addresses[netifaces.AF_INET6]: + if not addr['addr'].startswith('fe80'): + cidr = netaddr.IPNetwork("%s/%s" % (addr['addr'], + addr['netmask'])) + if address in cidr: + if key == 'iface': + return iface + else: + return addr[key] + return None + + +get_iface_for_address = partial(_get_for_address, key='iface') + +get_netmask_for_address = partial(_get_for_address, key='netmask') From cb04542baa1031448b13ad1c17cb875809689e3c Mon Sep 17 00:00:00 2001 From: James Page Date: Mon, 28 Jul 2014 13:05:58 +0100 Subject: [PATCH 41/48] Remove .coverage --- .coverage | 8 -------- 1 file changed, 8 deletions(-) delete mode 100644 .coverage diff --git a/.coverage b/.coverage deleted file mode 100644 index 3a6594a6..00000000 --- a/.coverage +++ /dev/null @@ -1,8 +0,0 @@ -}q(U collectorqUcoverage v3.7.1qUlinesq}q(Um/home/jamespage/src/charms/landing/nova-cloud-controller/hooks/charmhelpers/contrib/openstack/alternatives.pyq]Uh/home/jamespage/src/charms/landing/nova-cloud-controller/hooks/charmhelpers/contrib/storage/linux/lvm.pyq]q(KK KK(K3KEKOeUg/home/jamespage/src/charms/landing/nova-cloud-controller/hooks/charmhelpers/contrib/storage/__init__.pyq ]q -KaU\/home/jamespage/src/charms/landing/nova-cloud-controller/hooks/charmhelpers/core/__init__.pyq ]q KaUr/home/jamespage/src/charms/landing/nova-cloud-controller/hooks/charmhelpers/contrib/openstack/amulet/deployment.pyq ]UO/home/jamespage/src/charms/landing/nova-cloud-controller/hooks/nova_cc_hooks.pyq]q(KKKKKK K KK!K%K*K/K3KLKSKUKVKYK[K\K]K^K`KaKfKiKjKmKnKoKpKqKtKuKzK{K|KKKKKKKKKKKKKKKKKKKKKKKKKKKKKKKKKKKKKKKKKKKKKKKKKKKKMMMM M%M2M3M4M5M6M8M9M;M<M?MAMBMCMDMEMGMHMIMJMMMNMOMQMRMSMTMUMVMWMXM[M\M]M_M`MaMdMhMkMlMoMpMqMrMtMuMvMyMzM{M|M}M~MMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMM!M$M%M*M5M6M7M8M9M:M;M<M=M>M?M@MCMDMFMGMHMIMJMMMNMPMQMRMSMTMUMVMYM`eUm/home/jamespage/src/charms/landing/nova-cloud-controller/hooks/charmhelpers/contrib/storage/linux/__init__.pyq]qKaUi/home/jamespage/src/charms/landing/nova-cloud-controller/hooks/charmhelpers/contrib/storage/linux/ceph.pyq]UO/home/jamespage/src/charms/landing/nova-cloud-controller/hooks/nova_cc_utils.pyq]q(KKKKKKK K -K KKKK)K.K0K2K6K7K8K9K:K;K?K@KAKBKCKFKGKHKIKJKKKLKOKPKQKSKTKUKVKWKXKYKZK[K\K^K_K`KaKbKcKdKeKfKgKhKiKjKkKlKmKnKpKqKrKtKuKvKwKxKyKzK{K|K}K~KKKKKKKKKKKKKKKKKKKKKKKKKKKKKKKKKKKKKKKKKKKKKKKKKKKKKKKKKKKKKKKKKKKKKKKKKKKKKKKKKKKKKMMMMM MMMMMMMMMM!M#M$M%M&M'M(M)M*M-M.M/M0M1M2M3M4M5M7M8M9M;M=M@MEMHMJMMMNMOMPMQMRMSMUMdMgMmMqMrMuMwMxM{MMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMM M -M MMMMMMMMMMMMMM M#M$M%M&M+M,M-M.M1M3M4M5M:M;M<M=M>M@MBMCMDMGMHMIMLMQMlMvMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMeUf/home/jamespage/src/charms/landing/nova-cloud-controller/hooks/charmhelpers/contrib/openstack/utils.pyq]q(KKKKK K KKKKKKKK!K%K&K'K(K)K*K+K,K0K1K2K3K4K5K6K7K;KK?K@KAKBKCKDKEKFKGKHKKKNKSKnKsK|KKKKKKM,M>MRMsMMMMeUi/home/jamespage/src/charms/landing/nova-cloud-controller/hooks/charmhelpers/contrib/hahelpers/__init__.pyq]qKaUk/home/jamespage/src/charms/landing/nova-cloud-controller/hooks/charmhelpers/contrib/openstack/templating.pyq]q(KKKK K KKKKKFKJKKKUKaKkKKKKKKKMMMeUm/home/jamespage/src/charms/landing/nova-cloud-controller/hooks/charmhelpers/contrib/openstack/amulet/utils.pyq]UX/home/jamespage/src/charms/landing/nova-cloud-controller/hooks/charmhelpers/core/host.pyq]q(KKK K -K K K KKKKKKK K%K.K4KAKWKbKmKyKKKKKKKKKKKKKKKKKKKKKKKMMMM%M0M:eUh/home/jamespage/src/charms/landing/nova-cloud-controller/hooks/charmhelpers/contrib/openstack/context.pyq]q(KKKKKK KKK&K+K/K1K4K5K8K?KJKoKpKrKvKwKzKKKKKKKKKKKKKKM M MMMMMM]M^M`MMMMMMMMMMMMMMMMM M MMMMM%M4MCMWMgMMMMMMMMMMMMMMeUg/home/jamespage/src/charms/landing/nova-cloud-controller/hooks/charmhelpers/contrib/hahelpers/apache.pyq ]q!(K K KK)K6eUm/home/jamespage/src/charms/landing/nova-cloud-controller/hooks/charmhelpers/contrib/storage/linux/loopback.pyq"]q#(KKKKK K-eUQ/home/jamespage/src/charms/landing/nova-cloud-controller/hooks/nova_cc_context.pyq$]q%(KKKK KKKKK K"K)K*KK?K@KAKBKIKLKMKNKQKRKUKVKYKZK]K_KaKfKkKnKuKKKKKKKKKMM*M0MCeUi/home/jamespage/src/charms/landing/nova-cloud-controller/hooks/charmhelpers/contrib/openstack/__init__.pyq7]q8KaUj/home/jamespage/src/charms/landing/nova-cloud-controller/hooks/charmhelpers/contrib/storage/linux/utils.pyq9]q:(KKKKK KK)eU\/home/jamespage/src/charms/landing/nova-cloud-controller/hooks/charmhelpers/payload/execd.pyq;]q<(KKKKK K KK$K0eU_/home/jamespage/src/charms/landing/nova-cloud-controller/hooks/charmhelpers/payload/__init__.pyq=]q>KaUa/home/jamespage/src/charms/landing/nova-cloud-controller/hooks/charmhelpers/contrib/network/ip.pyq?]q@(KKKKK -K KKKKKRK]KwKKeU_/home/jamespage/src/charms/landing/nova-cloud-controller/hooks/charmhelpers/contrib/__init__.pyqA]qBKaUh/home/jamespage/src/charms/landing/nova-cloud-controller/hooks/charmhelpers/contrib/hahelpers/cluster.pyqC]qD(K K -K KKKKK)K9KAKJKWKpKKKeUY/home/jamespage/src/charms/landing/nova-cloud-controller/hooks/charmhelpers/core/fstab.pyqE]qF(KKK K KKKKK!K)K+K2K9KCKJKRKhKiKpKqeuu. \ No newline at end of file From 34b3afafaba9ac8d124bc8b35cbdd0f3b0df0f37 Mon Sep 17 00:00:00 2001 From: James Page Date: Tue, 29 Jul 2014 12:43:41 +0100 Subject: [PATCH 42/48] Updates --- hooks/nova_cc_utils.py | 64 +++++++++++++++++++++++++++--------------- 1 file changed, 41 insertions(+), 23 deletions(-) diff --git a/hooks/nova_cc_utils.py b/hooks/nova_cc_utils.py index d24680d9..9b3dd2f6 100644 --- a/hooks/nova_cc_utils.py +++ b/hooks/nova_cc_utils.py @@ -733,36 +733,54 @@ def neutron_plugin(): def guard_map(): + '''Map of services and required interfaces that must be present before + the service should be allowed to start''' gmap = {} - rinterfaces = ['identity-service', 'amqp'] - if relation_ids('pgsql-nova-db'): - rinterfaces.append('pgsql-nova-db') - elif relation_ids('pgsql-neutron-db'): - rinterfaces.append('pgsql-nova-db') - rinterfaces.append('pgsql-neutron-db') - else: - rinterfaces.append('shared-db') + nova_services = deepcopy(BASE_SERVICES) + if os_release('nova-common') not in ['essex', 'folsom']: + nova_services.append('nova-conductor') - for svc in BASE_SERVICES + ['neutron-server', 'quantum-server', 'nova-conductor']: - gmap[svc] = rinterfaces + nova_interfaces = ['identity-service', 'amqp'] + if relation_ids('pgsql-nova-db'): + nova_interfaces.append('pgsql-nova-db') + else: + nova_interfaces.append('shared-db') + + for svc in nova_services: + gmap[svc] = nova_interfaces + + net_manager = network_manager() + if net_manager in ['neutron', 'quantum']: + neutron_interfaces = ['identity-service', 'amqp'] + if relation_ids('pgsql-neutron-db'): + neutron_interfaces.append('pgsql-neutron-db') + else: + neutron_interfaces.append('shared-db') + if network_manager() == 'quantum': + gmap['quantum-server'] = neutron_interfaces + else: + gmap['neutron-server'] = neutron_interfaces return gmap -def service_guard(guard_map, contexts): - ''' Inhibit services in guard_map from running unless - required interfaces are found complete on contexts ''' +def service_guard(guard_map, contexts, active=False): + '''Inhibit services in guard_map from running unless + required interfaces are found complete in contexts.''' def wrap(f): def wrapped_f(*args): - incomplete_services = [] - for svc in guard_map: - for interface in guard_map[svc]: - if interface not in contexts.complete_contexts(): - incomplete_services.append(svc) - f(*args) - for svc in incomplete_services: - if service_running(svc): - log('Service {} has unfulfilled interface requirements, stopping.'.format(svc)) - service_stop(svc) + if active is True: + incomplete_services = [] + for svc in guard_map: + for interface in guard_map[svc]: + if interface not in contexts.complete_contexts(): + incomplete_services.append(svc) + f(*args) + for svc in incomplete_services: + if service_running(svc): + log('Service {} has unfulfilled interface requirements, stopping.'.format(svc)) + service_stop(svc) + else: + f(*args) return wrapped_f return wrap From 7355fd88b413ceecb35ba7f1cf688e53453bc1cb Mon Sep 17 00:00:00 2001 From: James Page Date: Tue, 29 Jul 2014 12:50:07 +0100 Subject: [PATCH 43/48] Disable service-guard by default --- config.yaml | 20 +++++++++++++++++++- hooks/nova_cc_hooks.py | 36 ++++++++++++++++++++++++------------ 2 files changed, 43 insertions(+), 13 deletions(-) diff --git a/config.yaml b/config.yaml index 907e6369..355458ac 100644 --- a/config.yaml +++ b/config.yaml @@ -184,4 +184,22 @@ options: 192.168.0.0/24) . This network will be used for public endpoints. - + service-guard: + type: boolean + default: false + description: | + Ensure required relations are made and complete before allowing services + to be started + . + By default, services may be up and accepting API request from install + onwards. + . + Enabling this flag ensures that services will not be started until the + minimum 'core relations' have been made between this charm and other + charms. + . + For this charm the following relations must be made: + . + * shared-db or (pgsql-nova-db, pgsql-neutron-db) + * amqp + * identity-service diff --git a/hooks/nova_cc_hooks.py b/hooks/nova_cc_hooks.py index f5bc7824..300aa1d2 100755 --- a/hooks/nova_cc_hooks.py +++ b/hooks/nova_cc_hooks.py @@ -114,7 +114,8 @@ def install(): @hooks.hook('config-changed') -@service_guard(guard_map(), CONFIGS) +@service_guard(guard_map(), CONFIGS, + active=config('service-guard')) @restart_on_change(restart_map(), stopstart=True) def config_changed(): global CONFIGS @@ -135,7 +136,8 @@ def amqp_joined(relation_id=None): @hooks.hook('amqp-relation-changed') @hooks.hook('amqp-relation-departed') -@service_guard(guard_map(), CONFIGS) +@service_guard(guard_map(), CONFIGS, + active=config('service-guard')) @restart_on_change(restart_map()) def amqp_changed(): if 'amqp' not in CONFIGS.complete_contexts(): @@ -194,7 +196,8 @@ def pgsql_neutron_db_joined(): @hooks.hook('shared-db-relation-changed') -@service_guard(guard_map(), CONFIGS) +@service_guard(guard_map(), CONFIGS, + active=config('service-guard')) @restart_on_change(restart_map()) def db_changed(): if 'shared-db' not in CONFIGS.complete_contexts(): @@ -210,7 +213,8 @@ def db_changed(): @hooks.hook('pgsql-nova-db-relation-changed') -@service_guard(guard_map(), CONFIGS) +@service_guard(guard_map(), CONFIGS, + active=config('service-guard')) @restart_on_change(restart_map()) def postgresql_nova_db_changed(): if 'pgsql-nova-db' not in CONFIGS.complete_contexts(): @@ -226,7 +230,8 @@ def postgresql_nova_db_changed(): @hooks.hook('pgsql-neutron-db-relation-changed') -@service_guard(guard_map(), CONFIGS) +@service_guard(guard_map(), CONFIGS, + active=config('service-guard')) @restart_on_change(restart_map()) def postgresql_neutron_db_changed(): if network_manager() in ['neutron', 'quantum']: @@ -236,7 +241,8 @@ def postgresql_neutron_db_changed(): @hooks.hook('image-service-relation-changed') -@service_guard(guard_map(), CONFIGS) +@service_guard(guard_map(), CONFIGS, + active=config('service-guard')) @restart_on_change(restart_map()) def image_service_changed(): if 'image-service' not in CONFIGS.complete_contexts(): @@ -259,7 +265,8 @@ def identity_joined(rid=None): @hooks.hook('identity-service-relation-changed') -@service_guard(guard_map(), CONFIGS) +@service_guard(guard_map(), CONFIGS, + active=config('service-guard')) @restart_on_change(restart_map()) def identity_changed(): if 'identity-service' not in CONFIGS.complete_contexts(): @@ -283,7 +290,8 @@ def identity_changed(): @hooks.hook('nova-volume-service-relation-joined', 'cinder-volume-service-relation-joined') -@service_guard(guard_map(), CONFIGS) +@service_guard(guard_map(), CONFIGS, + active=config('service-guard')) @restart_on_change(restart_map()) def volume_joined(): CONFIGS.write(NOVA_CONF) @@ -475,7 +483,8 @@ def quantum_joined(rid=None): @hooks.hook('cluster-relation-changed', 'cluster-relation-departed') -@service_guard(guard_map(), CONFIGS) +@service_guard(guard_map(), CONFIGS, + active=config('service-guard')) @restart_on_change(restart_map(), stopstart=True) def cluster_changed(): CONFIGS.write_all() @@ -585,7 +594,8 @@ def nova_vmware_relation_joined(rid=None): @hooks.hook('nova-vmware-relation-changed') -@service_guard(guard_map(), CONFIGS) +@service_guard(guard_map(), CONFIGS, + active=config('service-guard')) @restart_on_change(restart_map()) def nova_vmware_relation_changed(): CONFIGS.write('/etc/nova/nova.conf') @@ -617,7 +627,8 @@ def neutron_api_relation_joined(rid=None): @hooks.hook('neutron-api-relation-changed') -@service_guard(guard_map(), CONFIGS) +@service_guard(guard_map(), CONFIGS, + active=config('service-guard')) @restart_on_change(restart_map()) def neutron_api_relation_changed(): CONFIGS.write(NOVA_CONF) @@ -628,7 +639,8 @@ def neutron_api_relation_changed(): @hooks.hook('neutron-api-relation-broken') -@service_guard(guard_map(), CONFIGS) +@service_guard(guard_map(), CONFIGS, + active=config('service-guard')) @restart_on_change(restart_map()) def neutron_api_relation_broken(): if os.path.isfile('/etc/init/neutron-server.override'): From 9c71d510bb9e53184279f1ea10d04512199996ff Mon Sep 17 00:00:00 2001 From: James Page Date: Tue, 29 Jul 2014 13:23:46 +0100 Subject: [PATCH 44/48] Refine service guard function, disable by default --- hooks/nova_cc_utils.py | 13 ++-- unit_tests/test_nova_cc_hooks.py | 6 +- unit_tests/test_nova_cc_utils.py | 114 ++++++++++++++++++++++++++++++- unit_tests/test_utils.py | 6 +- 4 files changed, 128 insertions(+), 11 deletions(-) diff --git a/hooks/nova_cc_utils.py b/hooks/nova_cc_utils.py index e8662b0f..d356d59d 100644 --- a/hooks/nova_cc_utils.py +++ b/hooks/nova_cc_utils.py @@ -780,11 +780,11 @@ def guard_map(): if relation_ids('pgsql-neutron-db'): neutron_interfaces.append('pgsql-neutron-db') else: - neutron_interfaces.append('shared-db') + neutron_interfaces.append('shared-db') if network_manager() == 'quantum': - gmap['quantum-server'] = neutron_interfaces + gmap['quantum-server'] = neutron_interfaces else: - gmap['neutron-server'] = neutron_interfaces + gmap['neutron-server'] = neutron_interfaces return gmap @@ -802,9 +802,10 @@ def service_guard(guard_map, contexts, active=False): incomplete_services.append(svc) f(*args) for svc in incomplete_services: - if service_running(svc): - log('Service {} has unfulfilled interface requirements, stopping.'.format(svc)) - service_stop(svc) + if service_running(svc): + log('Service {} has unfulfilled ' + 'interface requirements, stopping.'.format(svc)) + service_stop(svc) else: f(*args) return wrapped_f diff --git a/unit_tests/test_nova_cc_hooks.py b/unit_tests/test_nova_cc_hooks.py index f4c0b7df..d5696e36 100644 --- a/unit_tests/test_nova_cc_hooks.py +++ b/unit_tests/test_nova_cc_hooks.py @@ -11,7 +11,11 @@ _map = utils.restart_map utils.register_configs = MagicMock() utils.restart_map = MagicMock() -import nova_cc_hooks as hooks +with patch('nova_cc_utils.guard_map') as gmap: + with patch('charmhelpers.core.hookenv.config') as config: + config.return_value = False + gmap.return_value = {} + import nova_cc_hooks as hooks utils.register_configs = _reg utils.restart_map = _map diff --git a/unit_tests/test_nova_cc_utils.py b/unit_tests/test_nova_cc_utils.py index 45e764f7..4573f5be 100644 --- a/unit_tests/test_nova_cc_utils.py +++ b/unit_tests/test_nova_cc_utils.py @@ -35,7 +35,9 @@ TO_PATCH = [ 'remote_unit', '_save_script_rc', 'service_start', - 'services' + 'services', + 'service_running', + 'service_stop' ] SCRIPTRC_ENV_VARS = { @@ -596,3 +598,113 @@ class NovaCCUtilsTests(CharmTestCase): utils.do_openstack_upgrade() expected = [call('cloud:precise-icehouse')] self.assertEquals(_do_openstack_upgrade.call_args_list, expected) + + def test_guard_map_nova(self): + self.relation_ids.return_value = [] + self.os_release.return_value = 'havana' + self.assertEqual( + {'nova-api-ec2': ['identity-service', 'amqp', 'shared-db'], + 'nova-api-os-compute': ['identity-service', 'amqp', 'shared-db'], + 'nova-cert': ['identity-service', 'amqp', 'shared-db'], + 'nova-conductor': ['identity-service', 'amqp', 'shared-db'], + 'nova-objectstore': ['identity-service', 'amqp', 'shared-db'], + 'nova-scheduler': ['identity-service', 'amqp', 'shared-db']}, + utils.guard_map() + ) + self.os_release.return_value = 'essex' + self.assertEqual( + {'nova-api-ec2': ['identity-service', 'amqp', 'shared-db'], + 'nova-api-os-compute': ['identity-service', 'amqp', 'shared-db'], + 'nova-cert': ['identity-service', 'amqp', 'shared-db'], + 'nova-objectstore': ['identity-service', 'amqp', 'shared-db'], + 'nova-scheduler': ['identity-service', 'amqp', 'shared-db']}, + utils.guard_map() + ) + + def test_guard_map_neutron(self): + self.relation_ids.return_value = [] + self.network_manager.return_value = 'neutron' + self.os_release.return_value = 'icehouse' + self.assertEqual( + {'neutron-server': ['identity-service', 'amqp', 'shared-db'], + 'nova-api-ec2': ['identity-service', 'amqp', 'shared-db'], + 'nova-api-os-compute': ['identity-service', 'amqp', 'shared-db'], + 'nova-cert': ['identity-service', 'amqp', 'shared-db'], + 'nova-conductor': ['identity-service', 'amqp', 'shared-db'], + 'nova-objectstore': ['identity-service', 'amqp', 'shared-db'], + 'nova-scheduler': ['identity-service', 'amqp', 'shared-db'], }, + utils.guard_map() + ) + self.network_manager.return_value = 'quantum' + self.os_release.return_value = 'grizzly' + self.assertEqual( + {'quantum-server': ['identity-service', 'amqp', 'shared-db'], + 'nova-api-ec2': ['identity-service', 'amqp', 'shared-db'], + 'nova-api-os-compute': ['identity-service', 'amqp', 'shared-db'], + 'nova-cert': ['identity-service', 'amqp', 'shared-db'], + 'nova-conductor': ['identity-service', 'amqp', 'shared-db'], + 'nova-objectstore': ['identity-service', 'amqp', 'shared-db'], + 'nova-scheduler': ['identity-service', 'amqp', 'shared-db'], }, + utils.guard_map() + ) + + def test_guard_map_pgsql(self): + self.relation_ids.return_value = ['pgsql:1'] + self.network_manager.return_value = 'neutron' + self.os_release.return_value = 'icehouse' + self.assertEqual( + {'neutron-server': ['identity-service', 'amqp', + 'pgsql-neutron-db'], + 'nova-api-ec2': ['identity-service', 'amqp', 'pgsql-nova-db'], + 'nova-api-os-compute': ['identity-service', 'amqp', + 'pgsql-nova-db'], + 'nova-cert': ['identity-service', 'amqp', 'pgsql-nova-db'], + 'nova-conductor': ['identity-service', 'amqp', 'pgsql-nova-db'], + 'nova-objectstore': ['identity-service', 'amqp', + 'pgsql-nova-db'], + 'nova-scheduler': ['identity-service', 'amqp', + 'pgsql-nova-db'], }, + utils.guard_map() + ) + + def test_service_guard_inactive(self): + '''Ensure that if disabled, service guards nothing''' + contexts = MagicMock() + + @utils.service_guard({'test': ['interfacea', 'interfaceb']}, + contexts, False) + def dummy_func(): + pass + dummy_func() + self.assertFalse(self.service_running.called) + self.assertFalse(contexts.complete_contexts.called) + + def test_service_guard_active_guard(self): + '''Ensure services with incomplete interfaces are stopped''' + contexts = MagicMock() + contexts.complete_contexts.return_value = ['interfacea'] + self.service_running.return_value = True + + @utils.service_guard({'test': ['interfacea', 'interfaceb']}, + contexts, True) + def dummy_func(): + pass + dummy_func() + self.service_running.assert_called_with('test') + self.service_stop.assert_called_with('test') + self.assertTrue(contexts.complete_contexts.called) + + def test_service_guard_active_release(self): + '''Ensure services with complete interfaces are not stopped''' + contexts = MagicMock() + contexts.complete_contexts.return_value = ['interfacea', + 'interfaceb'] + + @utils.service_guard({'test': ['interfacea', 'interfaceb']}, + contexts, True) + def dummy_func(): + pass + dummy_func() + self.assertFalse(self.service_running.called) + self.assertFalse(self.service_stop.called) + self.assertTrue(contexts.complete_contexts.called) diff --git a/unit_tests/test_utils.py b/unit_tests/test_utils.py index c9c7bace..a59f8970 100644 --- a/unit_tests/test_utils.py +++ b/unit_tests/test_utils.py @@ -82,9 +82,9 @@ class TestConfig(object): return self.config def set(self, attr, value): - if attr not in self.config: - raise KeyError - self.config[attr] = value + if attr not in self.config: + raise KeyError + self.config[attr] = value class TestRelation(object): From 804671d5321242e08d1a4ce498add2f1e448f826 Mon Sep 17 00:00:00 2001 From: James Page Date: Tue, 29 Jul 2014 13:25:27 +0100 Subject: [PATCH 45/48] Ensure services are disabled if required relations are broken --- hooks/nova_cc_hooks.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/hooks/nova_cc_hooks.py b/hooks/nova_cc_hooks.py index 300aa1d2..c388b10f 100755 --- a/hooks/nova_cc_hooks.py +++ b/hooks/nova_cc_hooks.py @@ -554,6 +554,8 @@ def ha_changed(): 'pgsql-nova-db-relation-broken', 'pgsql-neutron-db-relation-broken', 'quantum-network-service-relation-broken') +@service_guard(guard_map(), CONFIGS, + active=config('service-guard')) def relation_broken(): CONFIGS.write_all() From a57261cd963a0345812342ae4bedc47735b6e4b0 Mon Sep 17 00:00:00 2001 From: James Page Date: Tue, 29 Jul 2014 14:32:55 +0100 Subject: [PATCH 46/48] Don't add neutron stuff if related to neutron-api charm --- hooks/nova_cc_utils.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/hooks/nova_cc_utils.py b/hooks/nova_cc_utils.py index d356d59d..d55efdc5 100644 --- a/hooks/nova_cc_utils.py +++ b/hooks/nova_cc_utils.py @@ -775,7 +775,8 @@ def guard_map(): gmap[svc] = nova_interfaces net_manager = network_manager() - if net_manager in ['neutron', 'quantum']: + if net_manager in ['neutron', 'quantum'] and \ + not is_relation_made('neutron-api'): neutron_interfaces = ['identity-service', 'amqp'] if relation_ids('pgsql-neutron-db'): neutron_interfaces.append('pgsql-neutron-db') From 8ed106f6ed47f4bb85b89bbfcbdfcd58bd31e552 Mon Sep 17 00:00:00 2001 From: James Page Date: Tue, 29 Jul 2014 14:37:27 +0100 Subject: [PATCH 47/48] Fixup unit tests --- unit_tests/test_nova_cc_utils.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/unit_tests/test_nova_cc_utils.py b/unit_tests/test_nova_cc_utils.py index 4573f5be..1174087d 100644 --- a/unit_tests/test_nova_cc_utils.py +++ b/unit_tests/test_nova_cc_utils.py @@ -625,6 +625,7 @@ class NovaCCUtilsTests(CharmTestCase): self.relation_ids.return_value = [] self.network_manager.return_value = 'neutron' self.os_release.return_value = 'icehouse' + self.is_relation_made.return_value = False self.assertEqual( {'neutron-server': ['identity-service', 'amqp', 'shared-db'], 'nova-api-ec2': ['identity-service', 'amqp', 'shared-db'], @@ -651,6 +652,7 @@ class NovaCCUtilsTests(CharmTestCase): def test_guard_map_pgsql(self): self.relation_ids.return_value = ['pgsql:1'] self.network_manager.return_value = 'neutron' + self.is_relation_made.return_value = False self.os_release.return_value = 'icehouse' self.assertEqual( {'neutron-server': ['identity-service', 'amqp', From 50d5cf5baeae508b59a5698fa038801107e16d14 Mon Sep 17 00:00:00 2001 From: James Page Date: Tue, 29 Jul 2014 14:38:28 +0100 Subject: [PATCH 48/48] Tidy lint --- unit_tests/test_nova_cc_utils.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/unit_tests/test_nova_cc_utils.py b/unit_tests/test_nova_cc_utils.py index 1174087d..6b03dca4 100644 --- a/unit_tests/test_nova_cc_utils.py +++ b/unit_tests/test_nova_cc_utils.py @@ -652,7 +652,7 @@ class NovaCCUtilsTests(CharmTestCase): def test_guard_map_pgsql(self): self.relation_ids.return_value = ['pgsql:1'] self.network_manager.return_value = 'neutron' - self.is_relation_made.return_value = False + self.is_relation_made.return_value = False self.os_release.return_value = 'icehouse' self.assertEqual( {'neutron-server': ['identity-service', 'amqp',