diff --git a/config.yaml b/config.yaml index 907e6369..355458ac 100644 --- a/config.yaml +++ b/config.yaml @@ -184,4 +184,22 @@ options: 192.168.0.0/24) . This network will be used for public endpoints. - + service-guard: + type: boolean + default: false + description: | + Ensure required relations are made and complete before allowing services + to be started + . + By default, services may be up and accepting API request from install + onwards. + . + Enabling this flag ensures that services will not be started until the + minimum 'core relations' have been made between this charm and other + charms. + . + For this charm the following relations must be made: + . + * shared-db or (pgsql-nova-db, pgsql-neutron-db) + * amqp + * identity-service diff --git a/hooks/nova_cc_hooks.py b/hooks/nova_cc_hooks.py index eae54877..c388b10f 100755 --- a/hooks/nova_cc_hooks.py +++ b/hooks/nova_cc_hooks.py @@ -70,7 +70,9 @@ from nova_cc_utils import ( NOVA_CONF, QUANTUM_CONF, NEUTRON_CONF, - QUANTUM_API_PASTE + QUANTUM_API_PASTE, + service_guard, + guard_map, ) from charmhelpers.contrib.hahelpers.cluster import ( @@ -112,6 +114,8 @@ def install(): @hooks.hook('config-changed') +@service_guard(guard_map(), CONFIGS, + active=config('service-guard')) @restart_on_change(restart_map(), stopstart=True) def config_changed(): global CONFIGS @@ -132,6 +136,8 @@ def amqp_joined(relation_id=None): @hooks.hook('amqp-relation-changed') @hooks.hook('amqp-relation-departed') +@service_guard(guard_map(), CONFIGS, + active=config('service-guard')) @restart_on_change(restart_map()) def amqp_changed(): if 'amqp' not in CONFIGS.complete_contexts(): @@ -190,6 +196,8 @@ def pgsql_neutron_db_joined(): @hooks.hook('shared-db-relation-changed') +@service_guard(guard_map(), CONFIGS, + active=config('service-guard')) @restart_on_change(restart_map()) def db_changed(): if 'shared-db' not in CONFIGS.complete_contexts(): @@ -205,6 +213,8 @@ def db_changed(): @hooks.hook('pgsql-nova-db-relation-changed') +@service_guard(guard_map(), CONFIGS, + active=config('service-guard')) @restart_on_change(restart_map()) def postgresql_nova_db_changed(): if 'pgsql-nova-db' not in CONFIGS.complete_contexts(): @@ -220,6 +230,8 @@ def postgresql_nova_db_changed(): @hooks.hook('pgsql-neutron-db-relation-changed') +@service_guard(guard_map(), CONFIGS, + active=config('service-guard')) @restart_on_change(restart_map()) def postgresql_neutron_db_changed(): if network_manager() in ['neutron', 'quantum']: @@ -229,6 +241,8 @@ def postgresql_neutron_db_changed(): @hooks.hook('image-service-relation-changed') +@service_guard(guard_map(), CONFIGS, + active=config('service-guard')) @restart_on_change(restart_map()) def image_service_changed(): if 'image-service' not in CONFIGS.complete_contexts(): @@ -251,6 +265,8 @@ def identity_joined(rid=None): @hooks.hook('identity-service-relation-changed') +@service_guard(guard_map(), CONFIGS, + active=config('service-guard')) @restart_on_change(restart_map()) def identity_changed(): if 'identity-service' not in CONFIGS.complete_contexts(): @@ -274,6 +290,8 @@ def identity_changed(): @hooks.hook('nova-volume-service-relation-joined', 'cinder-volume-service-relation-joined') +@service_guard(guard_map(), CONFIGS, + active=config('service-guard')) @restart_on_change(restart_map()) def volume_joined(): CONFIGS.write(NOVA_CONF) @@ -465,6 +483,8 @@ def quantum_joined(rid=None): @hooks.hook('cluster-relation-changed', 'cluster-relation-departed') +@service_guard(guard_map(), CONFIGS, + active=config('service-guard')) @restart_on_change(restart_map(), stopstart=True) def cluster_changed(): CONFIGS.write_all() @@ -534,6 +554,8 @@ def ha_changed(): 'pgsql-nova-db-relation-broken', 'pgsql-neutron-db-relation-broken', 'quantum-network-service-relation-broken') +@service_guard(guard_map(), CONFIGS, + active=config('service-guard')) def relation_broken(): CONFIGS.write_all() @@ -574,6 +596,8 @@ def nova_vmware_relation_joined(rid=None): @hooks.hook('nova-vmware-relation-changed') +@service_guard(guard_map(), CONFIGS, + active=config('service-guard')) @restart_on_change(restart_map()) def nova_vmware_relation_changed(): CONFIGS.write('/etc/nova/nova.conf') @@ -605,6 +629,8 @@ def neutron_api_relation_joined(rid=None): @hooks.hook('neutron-api-relation-changed') +@service_guard(guard_map(), CONFIGS, + active=config('service-guard')) @restart_on_change(restart_map()) def neutron_api_relation_changed(): CONFIGS.write(NOVA_CONF) @@ -615,6 +641,8 @@ def neutron_api_relation_changed(): @hooks.hook('neutron-api-relation-broken') +@service_guard(guard_map(), CONFIGS, + active=config('service-guard')) @restart_on_change(restart_map()) def neutron_api_relation_broken(): if os.path.isfile('/etc/init/neutron-server.override'): diff --git a/hooks/nova_cc_utils.py b/hooks/nova_cc_utils.py index 1bbf3dc1..d55efdc5 100644 --- a/hooks/nova_cc_utils.py +++ b/hooks/nova_cc_utils.py @@ -40,6 +40,8 @@ from charmhelpers.core.hookenv import ( from charmhelpers.core.host import ( service_start, + service_stop, + service_running ) import nova_cc_context @@ -753,3 +755,59 @@ def neutron_plugin(): # quantum-plugin config setting can be safely overriden # as we only supported OVS in G/neutron return config('neutron-plugin') or config('quantum-plugin') + + +def guard_map(): + '''Map of services and required interfaces that must be present before + the service should be allowed to start''' + gmap = {} + nova_services = deepcopy(BASE_SERVICES) + if os_release('nova-common') not in ['essex', 'folsom']: + nova_services.append('nova-conductor') + + nova_interfaces = ['identity-service', 'amqp'] + if relation_ids('pgsql-nova-db'): + nova_interfaces.append('pgsql-nova-db') + else: + nova_interfaces.append('shared-db') + + for svc in nova_services: + gmap[svc] = nova_interfaces + + net_manager = network_manager() + if net_manager in ['neutron', 'quantum'] and \ + not is_relation_made('neutron-api'): + neutron_interfaces = ['identity-service', 'amqp'] + if relation_ids('pgsql-neutron-db'): + neutron_interfaces.append('pgsql-neutron-db') + else: + neutron_interfaces.append('shared-db') + if network_manager() == 'quantum': + gmap['quantum-server'] = neutron_interfaces + else: + gmap['neutron-server'] = neutron_interfaces + + return gmap + + +def service_guard(guard_map, contexts, active=False): + '''Inhibit services in guard_map from running unless + required interfaces are found complete in contexts.''' + def wrap(f): + def wrapped_f(*args): + if active is True: + incomplete_services = [] + for svc in guard_map: + for interface in guard_map[svc]: + if interface not in contexts.complete_contexts(): + incomplete_services.append(svc) + f(*args) + for svc in incomplete_services: + if service_running(svc): + log('Service {} has unfulfilled ' + 'interface requirements, stopping.'.format(svc)) + service_stop(svc) + else: + f(*args) + return wrapped_f + return wrap diff --git a/unit_tests/test_nova_cc_hooks.py b/unit_tests/test_nova_cc_hooks.py index f4c0b7df..d5696e36 100644 --- a/unit_tests/test_nova_cc_hooks.py +++ b/unit_tests/test_nova_cc_hooks.py @@ -11,7 +11,11 @@ _map = utils.restart_map utils.register_configs = MagicMock() utils.restart_map = MagicMock() -import nova_cc_hooks as hooks +with patch('nova_cc_utils.guard_map') as gmap: + with patch('charmhelpers.core.hookenv.config') as config: + config.return_value = False + gmap.return_value = {} + import nova_cc_hooks as hooks utils.register_configs = _reg utils.restart_map = _map diff --git a/unit_tests/test_nova_cc_utils.py b/unit_tests/test_nova_cc_utils.py index 45e764f7..a4909c81 100644 --- a/unit_tests/test_nova_cc_utils.py +++ b/unit_tests/test_nova_cc_utils.py @@ -35,7 +35,9 @@ TO_PATCH = [ 'remote_unit', '_save_script_rc', 'service_start', - 'services' + 'services', + 'service_running', + 'service_stop' ] SCRIPTRC_ENV_VARS = { @@ -596,3 +598,115 @@ class NovaCCUtilsTests(CharmTestCase): utils.do_openstack_upgrade() expected = [call('cloud:precise-icehouse')] self.assertEquals(_do_openstack_upgrade.call_args_list, expected) + + def test_guard_map_nova(self): + self.relation_ids.return_value = [] + self.os_release.return_value = 'havana' + self.assertEqual( + {'nova-api-ec2': ['identity-service', 'amqp', 'shared-db'], + 'nova-api-os-compute': ['identity-service', 'amqp', 'shared-db'], + 'nova-cert': ['identity-service', 'amqp', 'shared-db'], + 'nova-conductor': ['identity-service', 'amqp', 'shared-db'], + 'nova-objectstore': ['identity-service', 'amqp', 'shared-db'], + 'nova-scheduler': ['identity-service', 'amqp', 'shared-db']}, + utils.guard_map() + ) + self.os_release.return_value = 'essex' + self.assertEqual( + {'nova-api-ec2': ['identity-service', 'amqp', 'shared-db'], + 'nova-api-os-compute': ['identity-service', 'amqp', 'shared-db'], + 'nova-cert': ['identity-service', 'amqp', 'shared-db'], + 'nova-objectstore': ['identity-service', 'amqp', 'shared-db'], + 'nova-scheduler': ['identity-service', 'amqp', 'shared-db']}, + utils.guard_map() + ) + + def test_guard_map_neutron(self): + self.relation_ids.return_value = [] + self.network_manager.return_value = 'neutron' + self.os_release.return_value = 'icehouse' + self.is_relation_made.return_value = False + self.assertEqual( + {'neutron-server': ['identity-service', 'amqp', 'shared-db'], + 'nova-api-ec2': ['identity-service', 'amqp', 'shared-db'], + 'nova-api-os-compute': ['identity-service', 'amqp', 'shared-db'], + 'nova-cert': ['identity-service', 'amqp', 'shared-db'], + 'nova-conductor': ['identity-service', 'amqp', 'shared-db'], + 'nova-objectstore': ['identity-service', 'amqp', 'shared-db'], + 'nova-scheduler': ['identity-service', 'amqp', 'shared-db'], }, + utils.guard_map() + ) + self.network_manager.return_value = 'quantum' + self.os_release.return_value = 'grizzly' + self.assertEqual( + {'quantum-server': ['identity-service', 'amqp', 'shared-db'], + 'nova-api-ec2': ['identity-service', 'amqp', 'shared-db'], + 'nova-api-os-compute': ['identity-service', 'amqp', 'shared-db'], + 'nova-cert': ['identity-service', 'amqp', 'shared-db'], + 'nova-conductor': ['identity-service', 'amqp', 'shared-db'], + 'nova-objectstore': ['identity-service', 'amqp', 'shared-db'], + 'nova-scheduler': ['identity-service', 'amqp', 'shared-db'], }, + utils.guard_map() + ) + + def test_guard_map_pgsql(self): + self.relation_ids.return_value = ['pgsql:1'] + self.network_manager.return_value = 'neutron' + self.is_relation_made.return_value = False + self.os_release.return_value = 'icehouse' + self.assertEqual( + {'neutron-server': ['identity-service', 'amqp', + 'pgsql-neutron-db'], + 'nova-api-ec2': ['identity-service', 'amqp', 'pgsql-nova-db'], + 'nova-api-os-compute': ['identity-service', 'amqp', + 'pgsql-nova-db'], + 'nova-cert': ['identity-service', 'amqp', 'pgsql-nova-db'], + 'nova-conductor': ['identity-service', 'amqp', 'pgsql-nova-db'], + 'nova-objectstore': ['identity-service', 'amqp', + 'pgsql-nova-db'], + 'nova-scheduler': ['identity-service', 'amqp', + 'pgsql-nova-db'], }, + utils.guard_map() + ) + + def test_service_guard_inactive(self): + '''Ensure that if disabled, service guards nothing''' + contexts = MagicMock() + + @utils.service_guard({'test': ['interfacea', 'interfaceb']}, + contexts, False) + def dummy_func(): + pass + dummy_func() + self.assertFalse(self.service_running.called) + self.assertFalse(contexts.complete_contexts.called) + + def test_service_guard_active_guard(self): + '''Ensure services with incomplete interfaces are stopped''' + contexts = MagicMock() + contexts.complete_contexts.return_value = ['interfacea'] + self.service_running.return_value = True + + @utils.service_guard({'test': ['interfacea', 'interfaceb']}, + contexts, True) + def dummy_func(): + pass + dummy_func() + self.service_running.assert_called_with('test') + self.service_stop.assert_called_with('test') + self.assertTrue(contexts.complete_contexts.called) + + def test_service_guard_active_release(self): + '''Ensure services with complete interfaces are not stopped''' + contexts = MagicMock() + contexts.complete_contexts.return_value = ['interfacea', + 'interfaceb'] + + @utils.service_guard({'test': ['interfacea', 'interfaceb']}, + contexts, True) + def dummy_func(): + pass + dummy_func() + self.assertFalse(self.service_running.called) + self.assertFalse(self.service_stop.called) + self.assertTrue(contexts.complete_contexts.called) diff --git a/unit_tests/test_utils.py b/unit_tests/test_utils.py index c9c7bace..a59f8970 100644 --- a/unit_tests/test_utils.py +++ b/unit_tests/test_utils.py @@ -82,9 +82,9 @@ class TestConfig(object): return self.config def set(self, attr, value): - if attr not in self.config: - raise KeyError - self.config[attr] = value + if attr not in self.config: + raise KeyError + self.config[attr] = value class TestRelation(object):