From 739f1f8dcc5f42f55ccdba12b4e28631947975bd Mon Sep 17 00:00:00 2001 From: Ryan Beisner Date: Sat, 22 Aug 2015 04:58:53 +0000 Subject: [PATCH 1/7] update makefile, update amulet test targets, update readme --- Makefile | 2 +- tests/00-setup | 10 +++++++--- tests/019-basic-vivid-kilo | 0 tests/020-basic-trusty-liberty | 11 +++++++++++ tests/021-basic-wily-liberty | 9 +++++++++ tests/README | 10 ++++++++++ tests/tests.yaml | 20 ++++++++++++++++++++ 7 files changed, 58 insertions(+), 4 deletions(-) mode change 100644 => 100755 tests/019-basic-vivid-kilo create mode 100644 tests/020-basic-trusty-liberty create mode 100644 tests/021-basic-wily-liberty create mode 100644 tests/tests.yaml diff --git a/Makefile b/Makefile index 748f136c..052bd3ba 100644 --- a/Makefile +++ b/Makefile @@ -24,6 +24,6 @@ sync: bin/charm_helpers_sync.py @$(PYTHON) bin/charm_helpers_sync.py -c charm-helpers-hooks.yaml @$(PYTHON) bin/charm_helpers_sync.py -c charm-helpers-tests.yaml -publish: lint unit_test +publish: lint test bzr push lp:charms/neutron-gateway bzr push lp:charms/trusty/neutron-gateway diff --git a/tests/00-setup b/tests/00-setup index 27476744..95dffc44 100755 --- a/tests/00-setup +++ b/tests/00-setup @@ -4,9 +4,13 @@ set -ex sudo add-apt-repository --yes ppa:juju/stable sudo apt-get update --yes -sudo apt-get install --yes python-amulet \ +sudo apt-get install --yes amulet \ + python-cinderclient \ python-distro-info \ - python-neutronclient \ + python-glanceclient \ + python-heatclient \ python-keystoneclient \ + python-neutronclient \ python-novaclient \ - python-glanceclient + python-pika \ + python-swiftclient diff --git a/tests/019-basic-vivid-kilo b/tests/019-basic-vivid-kilo old mode 100644 new mode 100755 diff --git a/tests/020-basic-trusty-liberty b/tests/020-basic-trusty-liberty new file mode 100644 index 00000000..a2d0c9e1 --- /dev/null +++ b/tests/020-basic-trusty-liberty @@ -0,0 +1,11 @@ +#!/usr/bin/python + +"""Amulet tests on a basic quantum-gateway deployment on trusty-liberty.""" + +from basic_deployment import NeutronGatewayBasicDeployment + +if __name__ == '__main__': + deployment = NeutronGatewayBasicDeployment(series='trusty', + openstack='cloud:trusty-liberty', + source='cloud:trusty-updates/liberty') + deployment.run_tests() diff --git a/tests/021-basic-wily-liberty b/tests/021-basic-wily-liberty new file mode 100644 index 00000000..4dc44baa --- /dev/null +++ b/tests/021-basic-wily-liberty @@ -0,0 +1,9 @@ +#!/usr/bin/python + +"""Amulet tests on a basic quantum-gateway deployment on wily-liberty.""" + +from basic_deployment import NeutronGatewayBasicDeployment + +if __name__ == '__main__': + deployment = NeutronGatewayBasicDeployment(series='wily') + deployment.run_tests() diff --git a/tests/README b/tests/README index 80a2cc02..c52d967d 100644 --- a/tests/README +++ b/tests/README @@ -1,6 +1,16 @@ This directory provides Amulet tests that focus on verification of quantum-gateway deployments. +test_* methods are called in lexical sort order, although each individual test +should be idempotent, and expected to pass regardless of run order. + +Test name convention to ensure desired test order: + 1xx service and endpoint checks + 2xx relation checks + 3xx config checks + 4xx functional checks + 9xx restarts and other final checks + In order to run tests, you'll need charm-tools installed (in addition to juju, of course): sudo add-apt-repository ppa:juju/stable diff --git a/tests/tests.yaml b/tests/tests.yaml new file mode 100644 index 00000000..b288149b --- /dev/null +++ b/tests/tests.yaml @@ -0,0 +1,20 @@ +bootstrap: true +reset: true +virtualenv: true +makefile: + - lint + - test +sources: + - ppa:juju/stable +packages: + - amulet + - python-amulet + - python-cinderclient + - python-distro-info + - python-glanceclient + - python-heatclient + - python-keystoneclient + - python-neutronclient + - python-novaclient + - python-pika + - python-swiftclient From 64756f5937753ad6e60c6212018b49722420c8c8 Mon Sep 17 00:00:00 2001 From: Ryan Beisner Date: Sat, 22 Aug 2015 05:23:15 +0000 Subject: [PATCH 2/7] old lint cleanup --- tests/basic_deployment.py | 175 ++++++++++++++++++++------------------ 1 file changed, 94 insertions(+), 81 deletions(-) diff --git a/tests/basic_deployment.py b/tests/basic_deployment.py index 9f9f59ce..32a8f3e3 100644 --- a/tests/basic_deployment.py +++ b/tests/basic_deployment.py @@ -13,8 +13,8 @@ from charmhelpers.contrib.openstack.amulet.deployment import ( from charmhelpers.contrib.openstack.amulet.utils import ( OpenStackAmuletUtils, - DEBUG, # flake8: noqa - ERROR + DEBUG, + #ERROR ) # Use DEBUG to turn on debug logging @@ -45,30 +45,36 @@ class NeutronGatewayBasicDeployment(OpenStackAmuletDeployment): """ this_service = {'name': 'neutron-gateway'} other_services = [{'name': 'mysql'}, - {'name': 'rabbitmq-server'}, {'name': 'keystone'}, + {'name': 'rabbitmq-server'}, + {'name': 'keystone'}, {'name': 'nova-cloud-controller'}] if self._get_openstack_release() >= self.trusty_kilo: other_services.append({'name': 'neutron-api'}) - super(NeutronGatewayBasicDeployment, self)._add_services(this_service, - other_services) + + super(NeutronGatewayBasicDeployment, self)._add_services( + this_service, other_services) def _add_relations(self): """Add all of the relations for the services.""" relations = { - 'keystone:shared-db': 'mysql:shared-db', - 'neutron-gateway:shared-db': 'mysql:shared-db', - 'neutron-gateway:amqp': 'rabbitmq-server:amqp', - 'nova-cloud-controller:quantum-network-service': \ - 'neutron-gateway:quantum-network-service', - 'nova-cloud-controller:shared-db': 'mysql:shared-db', - 'nova-cloud-controller:identity-service': 'keystone:identity-service', - 'nova-cloud-controller:amqp': 'rabbitmq-server:amqp' + 'keystone:shared-db': 'mysql:shared-db', + 'neutron-gateway:shared-db': 'mysql:shared-db', + 'neutron-gateway:amqp': 'rabbitmq-server:amqp', + 'nova-cloud-controller:quantum-network-service': + 'neutron-gateway:quantum-network-service', + 'nova-cloud-controller:shared-db': 'mysql:shared-db', + 'nova-cloud-controller:identity-service': + 'keystone:identity-service', + 'nova-cloud-controller:amqp': 'rabbitmq-server:amqp' } if self._get_openstack_release() >= self.trusty_kilo: - relations['neutron-api:shared-db'] = 'mysql:shared-db' - relations['neutron-api:amqp'] = 'rabbitmq-server:amqp' - relations['neutron-api:neutron-api'] = 'nova-cloud-controller:neutron-api' - relations['neutron-api:identity-service'] = 'keystone:identity-service' + relations.update({ + 'neutron-api:shared-db': 'mysql:shared-db', + 'neutron-api:amqp': 'rabbitmq-server:amqp', + 'neutron-api:neutron-api': + 'nova-cloud-controller:neutron-api', + 'neutron-api:identity-service': 'keystone:identity-service' + }) super(NeutronGatewayBasicDeployment, self)._add_relations(relations) def _configure_services(self): @@ -83,16 +89,16 @@ class NeutronGatewayBasicDeployment(OpenStackAmuletDeployment): openstack_origin_git = { 'repositories': [ {'name': 'requirements', - 'repository': 'git://github.com/openstack/requirements', + 'repository': 'git://github.com/openstack/requirements', # noqa 'branch': branch}, {'name': 'neutron-fwaas', - 'repository': 'git://github.com/openstack/neutron-fwaas', + 'repository': 'git://github.com/openstack/neutron-fwaas', # noqa 'branch': branch}, {'name': 'neutron-lbaas', - 'repository': 'git://github.com/openstack/neutron-lbaas', + 'repository': 'git://github.com/openstack/neutron-lbaas', # noqa 'branch': branch}, {'name': 'neutron-vpnaas', - 'repository': 'git://github.com/openstack/neutron-vpnaas', + 'repository': 'git://github.com/openstack/neutron-vpnaas', # noqa 'branch': branch}, {'name': 'neutron', 'repository': 'git://github.com/openstack/neutron', @@ -122,7 +128,10 @@ class NeutronGatewayBasicDeployment(OpenStackAmuletDeployment): 'http_proxy': amulet_http_proxy, 'https_proxy': amulet_http_proxy, } - neutron_gateway_config['openstack-origin-git'] = yaml.dump(openstack_origin_git) + + neutron_gateway_config['openstack-origin-git'] = \ + yaml.dump(openstack_origin_git) + keystone_config = {'admin-password': 'openstack', 'admin-token': 'ubuntutesting'} nova_cc_config = {'network-manager': 'Quantum', @@ -137,7 +146,7 @@ class NeutronGatewayBasicDeployment(OpenStackAmuletDeployment): # Access the sentries for inspecting service units self.mysql_sentry = self.d.sentry.unit['mysql/0'] self.keystone_sentry = self.d.sentry.unit['keystone/0'] - self.rabbitmq_sentry = self.d.sentry.unit['rabbitmq-server/0'] + self.rmq_sentry = self.d.sentry.unit['rabbitmq-server/0'] self.nova_cc_sentry = self.d.sentry.unit['nova-cloud-controller/0'] self.neutron_gateway_sentry = self.d.sentry.unit['neutron-gateway/0'] @@ -150,7 +159,6 @@ class NeutronGatewayBasicDeployment(OpenStackAmuletDeployment): password='openstack', tenant='admin') - # Authenticate admin with neutron ep = self.keystone.service_catalog.url_for(service_type='identity', endpoint_type='publicURL') @@ -240,7 +248,7 @@ class NeutronGatewayBasicDeployment(OpenStackAmuletDeployment): def test_rabbitmq_amqp_relation(self): """Verify the rabbitmq-server to neutron-gateway amqp relation data""" - unit = self.rabbitmq_sentry + unit = self.rmq_sentry relation = ['amqp', 'neutron-gateway:amqp'] expected = { 'private-address': u.valid_ip, @@ -337,8 +345,8 @@ class NeutronGatewayBasicDeployment(OpenStackAmuletDeployment): def test_neutron_config(self): """Verify the data in the neutron config file.""" unit = self.neutron_gateway_sentry - rabbitmq_relation = self.rabbitmq_sentry.relation('amqp', - 'neutron-gateway:amqp') + rmq_ng_rel = self.rmq_sentry.relation('amqp', + 'neutron-gateway:amqp') conf = '/etc/neutron/neutron.conf' expected = { @@ -360,25 +368,27 @@ class NeutronGatewayBasicDeployment(OpenStackAmuletDeployment): if self._get_openstack_release() >= self.trusty_kilo: oslo_concurrency = { 'oslo_concurrency': { - 'lock_path':'/var/lock/neutron' + 'lock_path': '/var/lock/neutron' } } oslo_messaging_rabbit = { 'oslo_messaging_rabbit': { 'rabbit_userid': 'neutron', 'rabbit_virtual_host': 'openstack', - 'rabbit_password': rabbitmq_relation['password'], - 'rabbit_host': rabbitmq_relation['hostname'], + 'rabbit_password': rmq_ng_rel['password'], + 'rabbit_host': rmq_ng_rel['hostname'], } } expected.update(oslo_concurrency) expected.update(oslo_messaging_rabbit) else: - expected['DEFAULT']['lock_path'] = '/var/lock/neutron' - expected['DEFAULT']['rabbit_userid'] = 'neutron' - expected['DEFAULT']['rabbit_virtual_host'] = 'openstack' - expected['DEFAULT']['rabbit_password'] = rabbitmq_relation['password'] - expected['DEFAULT']['rabbit_host'] = rabbitmq_relation['hostname'] + expected['DEFAULT'].update({ + 'lock_path': '/var/lock/neutron', + 'rabbit_userid': 'neutron', + 'rabbit_virtual_host': 'openstack', + 'rabbit_password': rmq_ng_rel['password'], + 'rabbit_host': rmq_ng_rel['hostname'] + }) for section, pairs in expected.iteritems(): ret = u.validate_config_data(unit, conf, section, pairs) @@ -394,7 +404,7 @@ class NeutronGatewayBasicDeployment(OpenStackAmuletDeployment): unit = self.neutron_gateway_sentry conf = '/etc/neutron/plugins/ml2/ml2_conf.ini' - neutron_gateway_relation = unit.relation('shared-db', 'mysql:shared-db') + ng_db_rel = unit.relation('shared-db', 'mysql:shared-db') expected = { 'ml2': { 'type_drivers': 'gre,vxlan,vlan,flat', @@ -409,7 +419,7 @@ class NeutronGatewayBasicDeployment(OpenStackAmuletDeployment): }, 'ovs': { 'enable_tunneling': 'True', - 'local_ip': neutron_gateway_relation['private-address'] + 'local_ip': ng_db_rel['private-address'] }, 'agent': { 'tunnel_types': 'gre', @@ -475,9 +485,11 @@ class NeutronGatewayBasicDeployment(OpenStackAmuletDeployment): def test_l3_agent_config(self): """Verify the data in the l3 agent config file.""" unit = self.neutron_gateway_sentry - nova_cc_relation = self.nova_cc_sentry.relation(\ - 'quantum-network-service', - 'neutron-gateway:quantum-network-service') + + nova_cc_relation = self.nova_cc_sentry.relation( + 'quantum-network-service', + 'neutron-gateway:quantum-network-service') + ep = self.keystone.service_catalog.url_for(service_type='identity', endpoint_type='publicURL') @@ -526,8 +538,9 @@ class NeutronGatewayBasicDeployment(OpenStackAmuletDeployment): } } if self._get_openstack_release() >= self.trusty_kilo: - expected['DEFAULT']['device_driver'] = ('neutron_lbaas.services.' + - 'loadbalancer.drivers.haproxy.namespace_driver.HaproxyNSDriver') + expected['DEFAULT']['device_driver'] = \ + ('neutron_lbaas.services.loadbalancer.drivers.haproxy.' + 'namespace_driver.HaproxyNSDriver') for section, pairs in expected.iteritems(): ret = u.validate_config_data(unit, conf, section, pairs) @@ -540,10 +553,11 @@ class NeutronGatewayBasicDeployment(OpenStackAmuletDeployment): unit = self.neutron_gateway_sentry ep = self.keystone.service_catalog.url_for(service_type='identity', endpoint_type='publicURL') - neutron_gateway_relation = unit.relation('shared-db', 'mysql:shared-db') - nova_cc_relation = self.nova_cc_sentry.relation(\ - 'quantum-network-service', - 'neutron-gateway:quantum-network-service') + ng_db_rel = unit.relation('shared-db', 'mysql:shared-db') + + nova_cc_relation = self.nova_cc_sentry.relation( + 'quantum-network-service', + 'neutron-gateway:quantum-network-service') conf = '/etc/neutron/metadata_agent.ini' expected = { @@ -553,9 +567,9 @@ class NeutronGatewayBasicDeployment(OpenStackAmuletDeployment): 'admin_user': 'quantum_s3_ec2_nova', 'admin_password': nova_cc_relation['service_password'], 'root_helper': 'sudo neutron-rootwrap ' - '/etc/neutron/rootwrap.conf', + '/etc/neutron/rootwrap.conf', 'state_path': '/var/lib/neutron', - 'nova_metadata_ip': neutron_gateway_relation['private-address'], + 'nova_metadata_ip': ng_db_rel['private-address'], 'nova_metadata_port': '8775' } if self._get_openstack_release() >= self.trusty_kilo: @@ -590,22 +604,20 @@ class NeutronGatewayBasicDeployment(OpenStackAmuletDeployment): ret = u.validate_config_data(unit, conf, 'DEFAULT', expected) if ret: message = "metering agent config error: {}".format(ret) + amulet.raise_status(amulet.FAIL, msg=message) def test_nova_config(self): """Verify the data in the nova config file.""" unit = self.neutron_gateway_sentry conf = '/etc/nova/nova.conf' - mysql_relation = self.mysql_sentry.relation('shared-db', - 'neutron-gateway:shared-db') - db_uri = "mysql://{}:{}@{}/{}".format('nova', - mysql_relation['password'], - mysql_relation['db_host'], - 'nova') - rabbitmq_relation = self.rabbitmq_sentry.relation('amqp', - 'neutron-gateway:amqp') - nova_cc_relation = self.nova_cc_sentry.relation(\ - 'quantum-network-service', - 'neutron-gateway:quantum-network-service') + + rmq_ng_rel = self.rmq_sentry.relation('amqp', + 'neutron-gateway:amqp') + + nova_cc_relation = self.nova_cc_sentry.relation( + 'quantum-network-service', + 'neutron-gateway:quantum-network-service') + ep = self.keystone.service_catalog.url_for(service_type='identity', endpoint_type='publicURL') @@ -636,35 +648,35 @@ class NeutronGatewayBasicDeployment(OpenStackAmuletDeployment): } oslo_concurrency = { 'oslo_concurrency': { - 'lock_path':'/var/lock/nova' + 'lock_path': '/var/lock/nova' } } oslo_messaging_rabbit = { 'oslo_messaging_rabbit': { 'rabbit_userid': 'neutron', 'rabbit_virtual_host': 'openstack', - 'rabbit_password': rabbitmq_relation['password'], - 'rabbit_host': rabbitmq_relation['hostname'], + 'rabbit_password': rmq_ng_rel['password'], + 'rabbit_host': rmq_ng_rel['hostname'], } } expected.update(neutron) expected.update(oslo_concurrency) expected.update(oslo_messaging_rabbit) else: - d = 'DEFAULT' - expected[d]['lock_path'] = '/var/lock/nova' - expected[d]['rabbit_userid'] = 'neutron' - expected[d]['rabbit_virtual_host'] = 'openstack' - expected[d]['rabbit_password'] = rabbitmq_relation['password'] - expected[d]['rabbit_host'] = rabbitmq_relation['hostname'] - expected[d]['service_neutron_metadata_proxy'] = 'True' - expected[d]['neutron_auth_strategy'] = 'keystone' - expected[d]['neutron_url'] = nova_cc_relation['quantum_url'] - expected[d]['neutron_admin_tenant_name'] = 'services' - expected[d]['neutron_admin_username'] = 'quantum_s3_ec2_nova' - expected[d]['neutron_admin_password'] = \ - nova_cc_relation['service_password'] - expected[d]['neutron_admin_auth_url'] = ep + expected['DEFAULT'].update({ + 'lock_path': '/var/lock/nova', + 'rabbit_userid': 'neutron', + 'rabbit_virtual_host': 'openstack', + 'rabbit_password': rmq_ng_rel['password'], + 'rabbit_host': rmq_ng_rel['hostname'], + 'service_neutron_metadata_proxy': 'True', + 'neutron_auth_strategy': 'keystone', + 'neutron_url': nova_cc_relation['quantum_url'], + 'neutron_admin_tenant_name': 'services', + 'neutron_admin_username': 'quantum_s3_ec2_nova', + 'neutron_admin_password': nova_cc_relation['service_password'], + 'neutron_admin_auth_url': ep + }) for section, pairs in expected.iteritems(): ret = u.validate_config_data(unit, conf, section, pairs) @@ -679,12 +691,12 @@ class NeutronGatewayBasicDeployment(OpenStackAmuletDeployment): return unit = self.neutron_gateway_sentry - neutron_gateway_relation = unit.relation('shared-db', 'mysql:shared-db') + ng_db_rel = unit.relation('shared-db', 'mysql:shared-db') conf = '/etc/neutron/plugins/openvswitch/ovs_neutron_plugin.ini' expected = { 'ovs': { - 'local_ip': neutron_gateway_relation['private-address'], + 'local_ip': ng_db_rel['private-address'], 'tenant_network_type': 'gre', 'enable_tunneling': 'True', 'tunnel_id_ranges': '1:1000' @@ -720,8 +732,9 @@ class NeutronGatewayBasicDeployment(OpenStackAmuletDeployment): } } if self._get_openstack_release() >= self.trusty_kilo: - expected['vpnagent']['vpn_device_driver'] = ('neutron_vpnaas.' + - 'services.vpn.device_drivers.ipsec.OpenSwanDriver') + expected['vpnagent']['vpn_device_driver'] = \ + ('neutron_vpnaas.services.vpn.device_drivers.' + 'ipsec.OpenSwanDriver') for section, pairs in expected.iteritems(): ret = u.validate_config_data(unit, conf, section, pairs) @@ -743,7 +756,7 @@ class NeutronGatewayBasicDeployment(OpenStackAmuletDeployment): # Create a network and verify that it exists network = {'name': net_name} - self.neutron.create_network({'network':network}) + self.neutron.create_network({'network': network}) networks = self.neutron.list_networks(name=net_name) net_len = len(networks['networks']) From 3af6124604987bfafb401ff3bb2f45a411a6d218 Mon Sep 17 00:00:00 2001 From: Ryan Beisner Date: Sat, 22 Aug 2015 06:07:09 +0000 Subject: [PATCH 3/7] update tests for vivid --- tests/basic_deployment.py | 499 ++++++++++++++++++++++---------------- 1 file changed, 286 insertions(+), 213 deletions(-) diff --git a/tests/basic_deployment.py b/tests/basic_deployment.py index 32a8f3e3..20cc4180 100644 --- a/tests/basic_deployment.py +++ b/tests/basic_deployment.py @@ -48,6 +48,7 @@ class NeutronGatewayBasicDeployment(OpenStackAmuletDeployment): {'name': 'rabbitmq-server'}, {'name': 'keystone'}, {'name': 'nova-cloud-controller'}] +# why neutron-api only in > K? if self._get_openstack_release() >= self.trusty_kilo: other_services.append({'name': 'neutron-api'}) @@ -67,6 +68,7 @@ class NeutronGatewayBasicDeployment(OpenStackAmuletDeployment): 'keystone:identity-service', 'nova-cloud-controller:amqp': 'rabbitmq-server:amqp' } +# why neutron-api only in > K? if self._get_openstack_release() >= self.trusty_kilo: relations.update({ 'neutron-api:shared-db': 'mysql:shared-db', @@ -168,40 +170,101 @@ class NeutronGatewayBasicDeployment(OpenStackAmuletDeployment): tenant_name='admin', region_name='RegionOne') - def test_services(self): + def test_100_services(self): """Verify the expected services are running on the corresponding service units.""" - neutron_services = ['status neutron-dhcp-agent', - 'status neutron-lbaas-agent', - 'status neutron-metadata-agent', - 'status neutron-metering-agent', - 'status neutron-ovs-cleanup', - 'status neutron-plugin-openvswitch-agent'] + neutron_services = ['neutron-dhcp-agent', + 'neutron-lbaas-agent', + 'neutron-metadata-agent', + 'neutron-metering-agent', + 'neutron-ovs-cleanup', + 'neutron-plugin-openvswitch-agent'] if self._get_openstack_release() <= self.trusty_juno: - neutron_services.append('status neutron-vpn-agent') + neutron_services.append('neutron-vpn-agent') - nova_cc_services = ['status nova-api-ec2', - 'status nova-api-os-compute', - 'status nova-objectstore', - 'status nova-cert', - 'status nova-scheduler'] - if self._get_openstack_release() >= self.precise_grizzly: - nova_cc_services.append('status nova-conductor') + nova_cc_services = ['nova-api-ec2', + 'nova-api-os-compute', + 'nova-objectstore', + 'nova-cert', + 'nova-scheduler', + 'nova-conductor'] commands = { - self.mysql_sentry: ['status mysql'], - self.keystone_sentry: ['status keystone'], + self.mysql_sentry: ['mysql'], + self.keystone_sentry: ['keystone'], self.nova_cc_sentry: nova_cc_services, self.neutron_gateway_sentry: neutron_services } - ret = u.validate_services(commands) + ret = u.validate_services_by_name(commands) if ret: amulet.raise_status(amulet.FAIL, msg=ret) - def test_neutron_gateway_shared_db_relation(self): + def test_102_service_catalog(self): + """Verify that the service catalog endpoint data is valid.""" + u.log.debug('Checking keystone service catalog...') + endpoint_check = { + 'adminURL': u.valid_url, + 'id': u.not_null, + 'region': 'RegionOne', + 'publicURL': u.valid_url, + 'internalURL': u.valid_url + } + expected = { + 'network': [endpoint_check], + 'compute': [endpoint_check], + 'identity': [endpoint_check] + } + actual = self.keystone.service_catalog.get_endpoints() + + ret = u.validate_svc_catalog_endpoint_data(expected, actual) + if ret: + amulet.raise_status(amulet.FAIL, msg=ret) + + def test_104_network_endpoint(self): + """Verify the neutron network endpoint data.""" + u.log.debug('Checking neutron network api endpoint data...') + endpoints = self.keystone.endpoints.list() + admin_port = internal_port = public_port = '9696' + expected = { + 'id': u.not_null, + 'region': 'RegionOne', + 'adminurl': u.valid_url, + 'internalurl': u.valid_url, + 'publicurl': u.valid_url, + 'service_id': u.not_null + } + ret = u.validate_endpoint_data(endpoints, admin_port, internal_port, + public_port, expected) + + if ret: + amulet.raise_status(amulet.FAIL, + msg='glance endpoint: {}'.format(ret)) + + def test_110_users(self): + """Verify expected users.""" + u.log.debug('Checking keystone users...') + expected = [ + {'name': 'quantum_nova', + 'enabled': True, + 'tenantId': u.not_null, + 'id': u.not_null, + 'email': 'juju@localhost'}, + {'name': 'admin', + 'enabled': True, + 'tenantId': u.not_null, + 'id': u.not_null, + 'email': 'juju@localhost'} + ] + actual = self.keystone.users.list() + ret = u.validate_user_data(expected, actual) + if ret: + amulet.raise_status(amulet.FAIL, msg=ret) + + def test_200_neutron_gateway_mysql_shared_db_relation(self): """Verify the neutron-gateway to mysql shared-db relation data""" + u.log.debug('Checking neutron-gateway:mysql db relation data...') unit = self.neutron_gateway_sentry relation = ['shared-db', 'mysql:shared-db'] expected = { @@ -216,8 +279,9 @@ class NeutronGatewayBasicDeployment(OpenStackAmuletDeployment): message = u.relation_error('neutron-gateway shared-db', ret) amulet.raise_status(amulet.FAIL, msg=message) - def test_mysql_shared_db_relation(self): + def test_201_mysql_neutron_gateway_shared_db_relation(self): """Verify the mysql to neutron-gateway shared-db relation data""" + u.log.debug('Checking mysql:neutron-gateway db relation data...') unit = self.mysql_sentry relation = ['shared-db', 'neutron-gateway:shared-db'] expected = { @@ -231,8 +295,9 @@ class NeutronGatewayBasicDeployment(OpenStackAmuletDeployment): message = u.relation_error('mysql shared-db', ret) amulet.raise_status(amulet.FAIL, msg=message) - def test_neutron_gateway_amqp_relation(self): + def test_202_neutron_gateway_rabbitmq_amqp_relation(self): """Verify the neutron-gateway to rabbitmq-server amqp relation data""" + u.log.debug('Checking neutron-gateway:rmq amqp relation data...') unit = self.neutron_gateway_sentry relation = ['amqp', 'rabbitmq-server:amqp'] expected = { @@ -246,9 +311,10 @@ class NeutronGatewayBasicDeployment(OpenStackAmuletDeployment): message = u.relation_error('neutron-gateway amqp', ret) amulet.raise_status(amulet.FAIL, msg=message) - def test_rabbitmq_amqp_relation(self): + def test_203_rabbitmq_neutron_gateway_amqp_relation(self): """Verify the rabbitmq-server to neutron-gateway amqp relation data""" - unit = self.rmq_sentry + u.log.debug('Checking rmq:neutron-gateway amqp relation data...') + unit = self.rabbitmq_sentry relation = ['amqp', 'neutron-gateway:amqp'] expected = { 'private-address': u.valid_ip, @@ -261,9 +327,11 @@ class NeutronGatewayBasicDeployment(OpenStackAmuletDeployment): message = u.relation_error('rabbitmq amqp', ret) amulet.raise_status(amulet.FAIL, msg=message) - def test_neutron_gateway_network_service_relation(self): + def test_204_neutron_gateway_network_service_relation(self): """Verify the neutron-gateway to nova-cc quantum-network-service relation data""" + u.log.debug('Checking neutron-gateway:nova-cc net svc ' + 'relation data...') unit = self.neutron_gateway_sentry relation = ['quantum-network-service', 'nova-cloud-controller:quantum-network-service'] @@ -276,9 +344,11 @@ class NeutronGatewayBasicDeployment(OpenStackAmuletDeployment): message = u.relation_error('neutron-gateway network-service', ret) amulet.raise_status(amulet.FAIL, msg=message) - def test_nova_cc_network_service_relation(self): + def test_205_nova_cc_network_service_relation(self): """Verify the nova-cc to neutron-gateway quantum-network-service relation data""" + u.log.debug('Checking nova-cc:neutron-gateway net svc ' + 'relation data...') unit = self.nova_cc_sentry relation = ['quantum-network-service', 'neutron-gateway:quantum-network-service'] @@ -297,56 +367,27 @@ class NeutronGatewayBasicDeployment(OpenStackAmuletDeployment): 'keystone_host': u.valid_ip, 'quantum_plugin': 'ovs', 'auth_host': u.valid_ip, - 'service_username': 'quantum_s3_ec2_nova', 'service_tenant_name': 'services' } + if self._get_openstack_release() >= self.trusty_kilo: - expected['service_username'] = 'nova' + # Kilo or later + expected['service_username'] = 'quantum_nova' + else: + # Juno or earlier + expected['service_username'] = 'quantum_s3_ec2_nova' ret = u.validate_relation_data(unit, relation, expected) if ret: message = u.relation_error('nova-cc network-service', ret) amulet.raise_status(amulet.FAIL, msg=message) - def test_z_restart_on_config_change(self): - """Verify that the specified services are restarted when the config - is changed. - - Note(coreycb): The method name with the _z_ is a little odd - but it forces the test to run last. It just makes things - easier because restarting services requires re-authorization. - """ - conf = '/etc/neutron/neutron.conf' - - services = ['neutron-dhcp-agent', - 'neutron-lbaas-agent', - 'neutron-metadata-agent', - 'neutron-metering-agent', - 'neutron-openvswitch-agent'] - - if self._get_openstack_release() <= self.trusty_juno: - services.append('neutron-vpn-agent') - - u.log.debug("Making config change on neutron-gateway...") - self.d.configure('neutron-gateway', {'debug': 'True'}) - - time = 60 - for s in services: - u.log.debug("Checking that service restarted: {}".format(s)) - if not u.service_restarted(self.neutron_gateway_sentry, s, conf, - pgrep_full=True, sleep_time=time): - self.d.configure('neutron-gateway', {'debug': 'False'}) - msg = "service {} didn't restart after config change".format(s) - amulet.raise_status(amulet.FAIL, msg=msg) - time = 0 - - self.d.configure('neutron-gateway', {'debug': 'False'}) - - def test_neutron_config(self): + def test_300_neutron_config(self): """Verify the data in the neutron config file.""" + u.log.debug('Checking neutron gateway config file data...') unit = self.neutron_gateway_sentry - rmq_ng_rel = self.rmq_sentry.relation('amqp', - 'neutron-gateway:amqp') + rmq_ng_rel = self.rmq_sentry.relation( + 'amqp', 'neutron-gateway:amqp') conf = '/etc/neutron/neutron.conf' expected = { @@ -358,36 +399,34 @@ class NeutronGatewayBasicDeployment(OpenStackAmuletDeployment): 'notification_driver': 'neutron.openstack.common.notifier.' 'list_notifier', 'list_notifier_drivers': 'neutron.openstack.common.' - 'notifier.rabbit_notifier' + 'notifier.rabbit_notifier', + 'core_plugin': 'neutron.plugins.ml2.plugin.Ml2Plugin' }, 'agent': { 'root_helper': 'sudo /usr/bin/neutron-rootwrap ' '/etc/neutron/rootwrap.conf' } } + if self._get_openstack_release() >= self.trusty_kilo: - oslo_concurrency = { - 'oslo_concurrency': { - 'lock_path': '/var/lock/neutron' - } - } - oslo_messaging_rabbit = { - 'oslo_messaging_rabbit': { - 'rabbit_userid': 'neutron', - 'rabbit_virtual_host': 'openstack', - 'rabbit_password': rmq_ng_rel['password'], - 'rabbit_host': rmq_ng_rel['hostname'], - } - } - expected.update(oslo_concurrency) - expected.update(oslo_messaging_rabbit) - else: - expected['DEFAULT'].update({ - 'lock_path': '/var/lock/neutron', + # Kilo or later + expected['oslo_messaging_rabbit'] = { 'rabbit_userid': 'neutron', 'rabbit_virtual_host': 'openstack', 'rabbit_password': rmq_ng_rel['password'], - 'rabbit_host': rmq_ng_rel['hostname'] + 'rabbit_host': rmq_ng_rel['hostname'], + } + expected['oslo_concurrency'] = { + 'lock_path': '/var/lock/neutron' + } + else: + # Juno or earlier + expected['DEFAULT'].update({ + 'rabbit_userid': 'neutron', + 'rabbit_virtual_host': 'openstack', + 'rabbit_password': rmq_ng_rel['password'], + 'rabbit_host': rmq_ng_rel['hostname'], + 'lock_path': '/var/lock/neutron', }) for section, pairs in expected.iteritems(): @@ -396,15 +435,17 @@ class NeutronGatewayBasicDeployment(OpenStackAmuletDeployment): message = "neutron config error: {}".format(ret) amulet.raise_status(amulet.FAIL, msg=message) - def test_ml2_config(self): + def test_301_neutron_ml2_config(self): """Verify the data in the ml2 config file. This is only available since icehouse.""" + u.log.debug('Checking neutron gateway ml2 config file data...') if self._get_openstack_release() < self.precise_icehouse: return unit = self.neutron_gateway_sentry conf = '/etc/neutron/plugins/ml2/ml2_conf.ini' ng_db_rel = unit.relation('shared-db', 'mysql:shared-db') + expected = { 'ml2': { 'type_drivers': 'gre,vxlan,vlan,flat', @@ -437,8 +478,9 @@ class NeutronGatewayBasicDeployment(OpenStackAmuletDeployment): message = "ml2 config error: {}".format(ret) amulet.raise_status(amulet.FAIL, msg=message) - def test_dhcp_agent_config(self): + def test_302_neutron_dhcp_agent_config(self): """Verify the data in the dhcp agent config file.""" + u.log.debug('Checking neutron gateway dhcp agent config file data...') unit = self.neutron_gateway_sentry conf = '/etc/neutron/dhcp_agent.ini' expected = { @@ -450,46 +492,45 @@ class NeutronGatewayBasicDeployment(OpenStackAmuletDeployment): '/etc/neutron/rootwrap.conf', 'ovs_use_veth': 'True' } + section = 'DEFAULT' - ret = u.validate_config_data(unit, conf, 'DEFAULT', expected) + ret = u.validate_config_data(unit, conf, section, expected) if ret: message = "dhcp agent config error: {}".format(ret) amulet.raise_status(amulet.FAIL, msg=message) - def test_fwaas_driver_config(self): + def test_303_neutron_fwaas_driver_config(self): """Verify the data in the fwaas driver config file. This is only available since havana.""" - if self._get_openstack_release() < self.precise_havana: - return - + u.log.debug('Checking neutron gateway fwaas config file data...') unit = self.neutron_gateway_sentry conf = '/etc/neutron/fwaas_driver.ini' - if self._get_openstack_release() >= self.trusty_kilo: - expected = { - 'driver': 'neutron_fwaas.services.firewall.drivers.' - 'linux.iptables_fwaas.IptablesFwaasDriver', - 'enabled': 'True' - } - else: - expected = { - 'driver': 'neutron.services.firewall.drivers.' - 'linux.iptables_fwaas.IptablesFwaasDriver', - 'enabled': 'True' - } + expected = { + 'enabled': 'True' + } + section = 'fwaas' - ret = u.validate_config_data(unit, conf, 'fwaas', expected) + if self._get_openstack_release() >= self.trusty_kilo: + # Kilo or later + expected['driver'] = ('neutron_fwaas.services.firewall.drivers.' + 'linux.iptables_fwaas.IptablesFwaasDriver') + else: + # Juno or earlier + expected['driver'] = ('neutron.services.firewall.drivers.linux.' + 'iptables_fwaas.IptablesFwaasDriver') + + ret = u.validate_config_data(unit, conf, section, expected) if ret: message = "fwaas driver config error: {}".format(ret) amulet.raise_status(amulet.FAIL, msg=message) - def test_l3_agent_config(self): + def test_304_neutron_l3_agent_config(self): """Verify the data in the l3 agent config file.""" + u.log.debug('Checking neutron gateway l3 agent config file data...') unit = self.neutron_gateway_sentry - - nova_cc_relation = self.nova_cc_sentry.relation( + ncc_ng_rel = self.nova_cc_sentry.relation( 'quantum-network-service', 'neutron-gateway:quantum-network-service') - ep = self.keystone.service_catalog.url_for(service_type='identity', endpoint_type='publicURL') @@ -500,24 +541,31 @@ class NeutronGatewayBasicDeployment(OpenStackAmuletDeployment): 'auth_url': ep, 'auth_region': 'RegionOne', 'admin_tenant_name': 'services', - 'admin_user': 'quantum_s3_ec2_nova', - 'admin_password': nova_cc_relation['service_password'], + 'admin_password': ncc_ng_rel['service_password'], 'root_helper': 'sudo /usr/bin/neutron-rootwrap ' '/etc/neutron/rootwrap.conf', 'ovs_use_veth': 'True', 'handle_internal_only_routers': 'True' } - if self._get_openstack_release() >= self.trusty_kilo: - expected['admin_user'] = 'nova' + section = 'DEFAULT' - ret = u.validate_config_data(unit, conf, 'DEFAULT', expected) + if self._get_openstack_release() >= self.trusty_kilo: + # Kilo or later + expected['admin_user'] = 'quantum_nova' +#? expected['admin_user'] = 'nova' + else: + # Juno or earlier + expected['admin_user'] = 'quantum_s3_ec2_nova' + + ret = u.validate_config_data(unit, conf, section, expected) if ret: message = "l3 agent config error: {}".format(ret) amulet.raise_status(amulet.FAIL, msg=message) - def test_lbaas_agent_config(self): + def test_305_neutron_lbaas_agent_config(self): """Verify the data in the lbaas agent config file. This is only available since havana.""" + u.log.debug('Checking neutron gateway lbaas config file data...') if self._get_openstack_release() < self.precise_havana: return @@ -525,22 +573,27 @@ class NeutronGatewayBasicDeployment(OpenStackAmuletDeployment): conf = '/etc/neutron/lbaas_agent.ini' expected = { 'DEFAULT': { - 'periodic_interval': '10', 'interface_driver': 'neutron.agent.linux.interface.' 'OVSInterfaceDriver', + 'periodic_interval': '10', 'ovs_use_veth': 'False', - 'device_driver': 'neutron.services.loadbalancer.drivers.' - 'haproxy.namespace_driver.HaproxyNSDriver' }, 'haproxy': { 'loadbalancer_state_path': '$state_path/lbaas', 'user_group': 'nogroup' } } + if self._get_openstack_release() >= self.trusty_kilo: + # Kilo or later expected['DEFAULT']['device_driver'] = \ ('neutron_lbaas.services.loadbalancer.drivers.haproxy.' 'namespace_driver.HaproxyNSDriver') + else: + # Juno or earlier + expected['DEFAULT']['device_driver'] = \ + ('neutron.services.loadbalancer.drivers.haproxy.' + 'namespace_driver.HaproxyNSDriver') for section, pairs in expected.iteritems(): ret = u.validate_config_data(unit, conf, section, pairs) @@ -548,13 +601,15 @@ class NeutronGatewayBasicDeployment(OpenStackAmuletDeployment): message = "lbaas agent config error: {}".format(ret) amulet.raise_status(amulet.FAIL, msg=message) - def test_metadata_agent_config(self): + def test_306_neutron_metadata_agent_config(self): """Verify the data in the metadata agent config file.""" + u.log.debug('Checking neutron gateway metadata agent ' + 'config file data...') unit = self.neutron_gateway_sentry ep = self.keystone.service_catalog.url_for(service_type='identity', endpoint_type='publicURL') - ng_db_rel = unit.relation('shared-db', 'mysql:shared-db') - + ng_db_rel = unit.relation('shared-db', + 'mysql:shared-db') nova_cc_relation = self.nova_cc_sentry.relation( 'quantum-network-service', 'neutron-gateway:quantum-network-service') @@ -564,31 +619,34 @@ class NeutronGatewayBasicDeployment(OpenStackAmuletDeployment): 'auth_url': ep, 'auth_region': 'RegionOne', 'admin_tenant_name': 'services', - 'admin_user': 'quantum_s3_ec2_nova', 'admin_password': nova_cc_relation['service_password'], 'root_helper': 'sudo neutron-rootwrap ' '/etc/neutron/rootwrap.conf', 'state_path': '/var/lib/neutron', 'nova_metadata_ip': ng_db_rel['private-address'], - 'nova_metadata_port': '8775' + 'nova_metadata_port': '8775', + 'cache_url': 'memory://?default_ttl=5' } + section = 'DEFAULT' + if self._get_openstack_release() >= self.trusty_kilo: - expected['admin_user'] = 'nova' + # Kilo or later + expected['admin_user'] = 'quantum_nova' +#? expected['admin_user'] = 'nova' + else: + # Juno or earlier + expected['admin_user'] = 'quantum_s3_ec2_nova' - if self._get_openstack_release() >= self.precise_icehouse: - expected['cache_url'] = 'memory://?default_ttl=5' - - ret = u.validate_config_data(unit, conf, 'DEFAULT', expected) + ret = u.validate_config_data(unit, conf, section, expected) if ret: message = "metadata agent config error: {}".format(ret) amulet.raise_status(amulet.FAIL, msg=message) - def test_metering_agent_config(self): + def test_307_neutron_metering_agent_config(self): """Verify the data in the metering agent config file. This is only available since havana.""" - if self._get_openstack_release() < self.precise_havana: - return - + u.log.debug('Checking neutron gateway metering agent ' + 'config file data...') unit = self.neutron_gateway_sentry conf = '/etc/neutron/metering_agent.ini' expected = { @@ -600,24 +658,24 @@ class NeutronGatewayBasicDeployment(OpenStackAmuletDeployment): 'OVSInterfaceDriver', 'use_namespaces': 'True' } + section = 'DEFAULT' - ret = u.validate_config_data(unit, conf, 'DEFAULT', expected) + ret = u.validate_config_data(unit, conf, section, expected) if ret: message = "metering agent config error: {}".format(ret) amulet.raise_status(amulet.FAIL, msg=message) - def test_nova_config(self): + def test_308_neutron_nova_config(self): """Verify the data in the nova config file.""" + u.log.debug('Checking neutron gateway nova config file data...') unit = self.neutron_gateway_sentry conf = '/etc/nova/nova.conf' - rmq_ng_rel = self.rmq_sentry.relation('amqp', - 'neutron-gateway:amqp') - + rabbitmq_relation = self.rabbitmq_sentry.relation( + 'amqp', 'neutron-gateway:amqp') nova_cc_relation = self.nova_cc_sentry.relation( 'quantum-network-service', 'neutron-gateway:quantum-network-service') - ep = self.keystone.service_catalog.url_for(service_type='identity', endpoint_type='publicURL') @@ -634,48 +692,43 @@ class NeutronGatewayBasicDeployment(OpenStackAmuletDeployment): 'network_api_class': 'nova.network.neutronv2.api.API', } } + if self._get_openstack_release() >= self.trusty_kilo: - neutron = { - 'neutron': { - 'auth_strategy': 'keystone', - 'url': nova_cc_relation['quantum_url'], - 'admin_tenant_name': 'services', - 'admin_username': 'nova', - 'admin_password': nova_cc_relation['service_password'], - 'admin_auth_url': ep, - 'service_metadata_proxy': 'True', - } - } - oslo_concurrency = { - 'oslo_concurrency': { - 'lock_path': '/var/lock/nova' - } - } - oslo_messaging_rabbit = { - 'oslo_messaging_rabbit': { - 'rabbit_userid': 'neutron', - 'rabbit_virtual_host': 'openstack', - 'rabbit_password': rmq_ng_rel['password'], - 'rabbit_host': rmq_ng_rel['hostname'], - } - } - expected.update(neutron) - expected.update(oslo_concurrency) - expected.update(oslo_messaging_rabbit) - else: - expected['DEFAULT'].update({ - 'lock_path': '/var/lock/nova', + # Kilo or later + expected['oslo_messaging_rabbit'] = { 'rabbit_userid': 'neutron', 'rabbit_virtual_host': 'openstack', - 'rabbit_password': rmq_ng_rel['password'], - 'rabbit_host': rmq_ng_rel['hostname'], - 'service_neutron_metadata_proxy': 'True', + 'rabbit_password': rabbitmq_relation['password'], + 'rabbit_host': rabbitmq_relation['hostname'], + } + expected['oslo_concurrency'] = { + 'lock_path': '/var/lock/nova' + } + expected['neutron'] = { + 'auth_strategy': 'keystone', + 'url': nova_cc_relation['quantum_url'], + 'admin_tenant_name': 'services', + 'admin_username': 'quantum_nova', + 'admin_password': nova_cc_relation['service_password'], + 'admin_auth_url': ep, + 'service_metadata_proxy': 'True', + 'metadata_proxy_shared_secret': u.not_null + } + else: + # Juno or earlier + expected['DEFAULT'].update({ + 'rabbit_userid': 'neutron', + 'rabbit_virtual_host': 'openstack', + 'rabbit_password': rabbitmq_relation['password'], + 'rabbit_host': rabbitmq_relation['hostname'], + 'lock_path': '/var/lock/nova', 'neutron_auth_strategy': 'keystone', 'neutron_url': nova_cc_relation['quantum_url'], 'neutron_admin_tenant_name': 'services', 'neutron_admin_username': 'quantum_s3_ec2_nova', 'neutron_admin_password': nova_cc_relation['service_password'], - 'neutron_admin_auth_url': ep + 'neutron_admin_auth_url': ep, + 'service_neutron_metadata_proxy': 'True', }) for section, pairs in expected.iteritems(): @@ -684,57 +737,30 @@ class NeutronGatewayBasicDeployment(OpenStackAmuletDeployment): message = "nova config error: {}".format(ret) amulet.raise_status(amulet.FAIL, msg=message) - def test_ovs_neutron_plugin_config(self): - """Verify the data in the ovs neutron plugin config file. The ovs - plugin is not used by default since icehouse.""" - if self._get_openstack_release() >= self.precise_icehouse: - return - - unit = self.neutron_gateway_sentry - ng_db_rel = unit.relation('shared-db', 'mysql:shared-db') - - conf = '/etc/neutron/plugins/openvswitch/ovs_neutron_plugin.ini' - expected = { - 'ovs': { - 'local_ip': ng_db_rel['private-address'], - 'tenant_network_type': 'gre', - 'enable_tunneling': 'True', - 'tunnel_id_ranges': '1:1000' - }, - 'agent': { - 'polling_interval': '10', - 'root_helper': 'sudo /usr/bin/neutron-rootwrap ' - '/etc/neutron/rootwrap.conf' - } - } - - for section, pairs in expected.iteritems(): - ret = u.validate_config_data(unit, conf, section, pairs) - if ret: - message = "ovs neutron plugin config error: {}".format(ret) - amulet.raise_status(amulet.FAIL, msg=message) - - def test_vpn_agent_config(self): + def test_309_neutron_vpn_agent_config(self): """Verify the data in the vpn agent config file. This isn't available prior to havana.""" - if self._get_openstack_release() < self.precise_havana: - return - + u.log.debug('Checking neutron gateway vpn agent config file data...') unit = self.neutron_gateway_sentry conf = '/etc/neutron/vpn_agent.ini' expected = { - 'vpnagent': { - 'vpn_device_driver': 'neutron.services.vpn.device_drivers.' - 'ipsec.OpenSwanDriver' - }, 'ipsec': { 'ipsec_status_check_interval': '60' } } + if self._get_openstack_release() >= self.trusty_kilo: - expected['vpnagent']['vpn_device_driver'] = \ - ('neutron_vpnaas.services.vpn.device_drivers.' - 'ipsec.OpenSwanDriver') + # Kilo or later + expected['vpnagent'] = { + 'vpn_device_driver': 'neutron_vpnaas.services.vpn.' + 'device_drivers.ipsec.OpenSwanDriver' + } + else: + # Juno or earlier + expected['vpnagent'] = { + 'vpn_device_driver': 'neutron.services.vpn.device_drivers.' + 'ipsec.OpenSwanDriver' + } for section, pairs in expected.iteritems(): ret = u.validate_config_data(unit, conf, section, pairs) @@ -742,8 +768,9 @@ class NeutronGatewayBasicDeployment(OpenStackAmuletDeployment): message = "vpn agent config error: {}".format(ret) amulet.raise_status(amulet.FAIL, msg=message) - def test_create_network(self): + def test_400_create_network(self): """Create a network, verify that it exists, and then delete it.""" + u.log.debug('Creating neutron network...') self.neutron.format = 'json' net_name = 'ext_net' @@ -764,9 +791,55 @@ class NeutronGatewayBasicDeployment(OpenStackAmuletDeployment): msg = "Expected 1 network, found {}".format(net_len) amulet.raise_status(amulet.FAIL, msg=msg) + u.log.debug('Confirming new neutron network...') network = networks['networks'][0] if network['name'] != net_name: amulet.raise_status(amulet.FAIL, msg="network ext_net not found") #Cleanup + u.log.debug('Deleting neutron network...') self.neutron.delete_network(network['id']) + + def test_900_restart_on_config_change(self): + """Verify that the specified services are restarted when the + config is changed.""" + + sentry = self.neutron_gateway_sentry + juju_service = 'neutron-gateway' + + # Expected default and alternate values + set_default = {'debug': 'False'} + set_alternate = {'debug': 'True'} + + # Services which are expected to restart upon config change, + # and corresponding config files affected by the change + conf_file = '/etc/neutron/neutron.conf' + services = { + 'neutron-dhcp-agent': conf_file, + 'neutron-lbaas-agent': conf_file, + 'neutron-metadata-agent': conf_file, + 'neutron-metering-agent': conf_file, + 'neutron-openvswitch-agent': conf_file + } + + if self._get_openstack_release() <= self.trusty_juno: + services.update({'neutron-vpn-agent': conf_file}) + + # Make config change, check for service restarts + u.log.debug('Making config change on {}...'.format(juju_service)) + mtime = u.get_sentry_time(sentry) + self.d.configure(juju_service, set_alternate) + + sleep_time = 90 + for s, conf_file in services.iteritems(): + u.log.debug("Checking that service restarted: {}".format(s)) + if not u.validate_service_config_changed(sentry, mtime, s, + conf_file, + pgrep_full=True, + sleep_time=sleep_time): + self.d.configure(juju_service, set_default) + msg = "service {} didn't restart after config change".format(s) + amulet.raise_status(amulet.FAIL, msg=msg) + sleep_time = 0 + + self.d.configure(juju_service, set_default) From 69018ba66fa1d3919050a82a786c15074589bec9 Mon Sep 17 00:00:00 2001 From: Ryan Beisner Date: Tue, 15 Sep 2015 20:43:41 +0000 Subject: [PATCH 4/7] update tests for liberty, misc test updates for kilo --- tests/020-basic-trusty-liberty | 0 tests/021-basic-wily-liberty | 0 tests/basic_deployment.py | 226 ++++++++++++++++++++++++++++----- 3 files changed, 194 insertions(+), 32 deletions(-) mode change 100644 => 100755 tests/020-basic-trusty-liberty mode change 100644 => 100755 tests/021-basic-wily-liberty diff --git a/tests/020-basic-trusty-liberty b/tests/020-basic-trusty-liberty old mode 100644 new mode 100755 diff --git a/tests/021-basic-wily-liberty b/tests/021-basic-wily-liberty old mode 100644 new mode 100755 diff --git a/tests/basic_deployment.py b/tests/basic_deployment.py index 20cc4180..8f1eec89 100644 --- a/tests/basic_deployment.py +++ b/tests/basic_deployment.py @@ -47,10 +47,8 @@ class NeutronGatewayBasicDeployment(OpenStackAmuletDeployment): other_services = [{'name': 'mysql'}, {'name': 'rabbitmq-server'}, {'name': 'keystone'}, - {'name': 'nova-cloud-controller'}] -# why neutron-api only in > K? - if self._get_openstack_release() >= self.trusty_kilo: - other_services.append({'name': 'neutron-api'}) + {'name': 'nova-cloud-controller'}, + {'name': 'neutron-api'}] super(NeutronGatewayBasicDeployment, self)._add_services( this_service, other_services) @@ -64,19 +62,14 @@ class NeutronGatewayBasicDeployment(OpenStackAmuletDeployment): 'nova-cloud-controller:quantum-network-service': 'neutron-gateway:quantum-network-service', 'nova-cloud-controller:shared-db': 'mysql:shared-db', - 'nova-cloud-controller:identity-service': - 'keystone:identity-service', - 'nova-cloud-controller:amqp': 'rabbitmq-server:amqp' + 'nova-cloud-controller:identity-service': 'keystone:' + 'identity-service', + 'nova-cloud-controller:amqp': 'rabbitmq-server:amqp', + 'neutron-api:shared-db': 'mysql:shared-db', + 'neutron-api:amqp': 'rabbitmq-server:amqp', + 'neutron-api:neutron-api': 'nova-cloud-controller:neutron-api', + 'neutron-api:identity-service': 'keystone:identity-service' } -# why neutron-api only in > K? - if self._get_openstack_release() >= self.trusty_kilo: - relations.update({ - 'neutron-api:shared-db': 'mysql:shared-db', - 'neutron-api:amqp': 'rabbitmq-server:amqp', - 'neutron-api:neutron-api': - 'nova-cloud-controller:neutron-api', - 'neutron-api:identity-service': 'keystone:identity-service' - }) super(NeutronGatewayBasicDeployment, self)._add_relations(relations) def _configure_services(self): @@ -151,6 +144,7 @@ class NeutronGatewayBasicDeployment(OpenStackAmuletDeployment): self.rmq_sentry = self.d.sentry.unit['rabbitmq-server/0'] self.nova_cc_sentry = self.d.sentry.unit['nova-cloud-controller/0'] self.neutron_gateway_sentry = self.d.sentry.unit['neutron-gateway/0'] + self.neutron_api_sentry = self.d.sentry.unit['neutron-api/0'] # Let things settle a bit before moving forward time.sleep(30) @@ -246,17 +240,37 @@ class NeutronGatewayBasicDeployment(OpenStackAmuletDeployment): """Verify expected users.""" u.log.debug('Checking keystone users...') expected = [ - {'name': 'quantum_nova', + {'name': 'admin', 'enabled': True, 'tenantId': u.not_null, 'id': u.not_null, 'email': 'juju@localhost'}, - {'name': 'admin', + {'name': 'quantum', 'enabled': True, 'tenantId': u.not_null, 'id': u.not_null, 'email': 'juju@localhost'} ] + + if self._get_openstack_release() >= self.trusty_kilo: + # Kilo or later + expected.append({ + 'name': 'nova', + 'enabled': True, + 'tenantId': u.not_null, + 'id': u.not_null, + 'email': 'juju@localhost' + }) + else: + # Juno and earlier + expected.append({ + 'name': 's3_ec2_nova', + 'enabled': True, + 'tenantId': u.not_null, + 'id': u.not_null, + 'email': 'juju@localhost' + }) + actual = self.keystone.users.list() ret = u.validate_user_data(expected, actual) if ret: @@ -314,7 +328,7 @@ class NeutronGatewayBasicDeployment(OpenStackAmuletDeployment): def test_203_rabbitmq_neutron_gateway_amqp_relation(self): """Verify the rabbitmq-server to neutron-gateway amqp relation data""" u.log.debug('Checking rmq:neutron-gateway amqp relation data...') - unit = self.rabbitmq_sentry + unit = self.rmq_sentry relation = ['amqp', 'neutron-gateway:amqp'] expected = { 'private-address': u.valid_ip, @@ -372,16 +386,167 @@ class NeutronGatewayBasicDeployment(OpenStackAmuletDeployment): if self._get_openstack_release() >= self.trusty_kilo: # Kilo or later - expected['service_username'] = 'quantum_nova' + expected['service_username'] = 'nova' else: # Juno or earlier - expected['service_username'] = 'quantum_s3_ec2_nova' + expected['service_username'] = 's3_ec2_nova' ret = u.validate_relation_data(unit, relation, expected) if ret: message = u.relation_error('nova-cc network-service', ret) amulet.raise_status(amulet.FAIL, msg=message) + def test_206_neutron_api_shared_db_relation(self): + """Verify the neutron-api to mysql shared-db relation data""" + u.log.debug('Checking neutron-api:mysql db relation data...') + unit = self.neutron_api_sentry + relation = ['shared-db', 'mysql:shared-db'] + expected = { + 'private-address': u.valid_ip, + 'database': 'neutron', + 'username': 'neutron', + 'hostname': u.valid_ip + } + + ret = u.validate_relation_data(unit, relation, expected) + if ret: + message = u.relation_error('neutron-api shared-db', ret) + amulet.raise_status(amulet.FAIL, msg=message) + + def test_207_shared_db_neutron_api_relation(self): + """Verify the mysql to neutron-api shared-db relation data""" + u.log.debug('Checking mysql:neutron-api db relation data...') + unit = self.mysql_sentry + relation = ['shared-db', 'neutron-api:shared-db'] + expected = { + 'db_host': u.valid_ip, + 'private-address': u.valid_ip, + 'password': u.not_null + } + + if self._get_openstack_release() == self.precise_icehouse: + # Precise + expected['allowed_units'] = 'nova-cloud-controller/0 neutron-api/0' + else: + # Not Precise + expected['allowed_units'] = 'neutron-api/0' + + ret = u.validate_relation_data(unit, relation, expected) + if ret: + message = u.relation_error('mysql shared-db', ret) + amulet.raise_status(amulet.FAIL, msg=message) + + def test_208_neutron_api_amqp_relation(self): + """Verify the neutron-api to rabbitmq-server amqp relation data""" + u.log.debug('Checking neutron-api:amqp relation data...') + unit = self.neutron_api_sentry + relation = ['amqp', 'rabbitmq-server:amqp'] + expected = { + 'username': 'neutron', + 'private-address': u.valid_ip, + 'vhost': 'openstack' + } + + ret = u.validate_relation_data(unit, relation, expected) + if ret: + message = u.relation_error('neutron-api amqp', ret) + amulet.raise_status(amulet.FAIL, msg=message) + + def test_209_amqp_neutron_api_relation(self): + """Verify the rabbitmq-server to neutron-api amqp relation data""" + u.log.debug('Checking amqp:neutron-api relation data...') + unit = self.rmq_sentry + relation = ['amqp', 'neutron-api:amqp'] + expected = { + 'hostname': u.valid_ip, + 'private-address': u.valid_ip, + 'password': u.not_null + } + + ret = u.validate_relation_data(unit, relation, expected) + if ret: + message = u.relation_error('rabbitmq amqp', ret) + amulet.raise_status(amulet.FAIL, msg=message) + + def test_210_neutron_api_keystone_identity_relation(self): + """Verify the neutron-api to keystone identity-service relation data""" + u.log.debug('Checking neutron-api:keystone id relation data...') + unit = self.neutron_api_sentry + relation = ['identity-service', 'keystone:identity-service'] + api_ip = unit.relation('identity-service', + 'keystone:identity-service')['private-address'] + api_endpoint = 'http://{}:9696'.format(api_ip) + expected = { + 'private-address': u.valid_ip, + 'quantum_region': 'RegionOne', + 'quantum_service': 'quantum', + 'quantum_admin_url': api_endpoint, + 'quantum_internal_url': api_endpoint, + 'quantum_public_url': api_endpoint, + } + + ret = u.validate_relation_data(unit, relation, expected) + if ret: + message = u.relation_error('neutron-api identity-service', ret) + amulet.raise_status(amulet.FAIL, msg=message) + + def test_211_keystone_neutron_api_identity_relation(self): + """Verify the keystone to neutron-api identity-service relation data""" + u.log.debug('Checking keystone:neutron-api id relation data...') + unit = self.keystone_sentry + relation = ['identity-service', 'neutron-api:identity-service'] + rel_ks_id = unit.relation('identity-service', + 'neutron-api:identity-service') + id_ip = rel_ks_id['private-address'] + expected = { + 'admin_token': 'ubuntutesting', + 'auth_host': id_ip, + 'auth_port': "35357", + 'auth_protocol': 'http', + 'private-address': id_ip, + 'service_host': id_ip, + } + ret = u.validate_relation_data(unit, relation, expected) + if ret: + message = u.relation_error('neutron-api identity-service', ret) + amulet.raise_status(amulet.FAIL, msg=message) + + def test_212_neutron_api_novacc_relation(self): + """Verify the neutron-api to nova-cloud-controller relation data""" + u.log.debug('Checking neutron-api:novacc relation data...') + unit = self.neutron_api_sentry + relation = ['neutron-api', 'nova-cloud-controller:neutron-api'] + api_ip = unit.relation('identity-service', + 'keystone:identity-service')['private-address'] + api_endpoint = 'http://{}:9696'.format(api_ip) + expected = { + 'private-address': api_ip, + 'neutron-plugin': 'ovs', + 'neutron-security-groups': "no", + 'neutron-url': api_endpoint, + } + ret = u.validate_relation_data(unit, relation, expected) + if ret: + message = u.relation_error('neutron-api neutron-api', ret) + amulet.raise_status(amulet.FAIL, msg=message) + + def test_213_novacc_neutron_api_relation(self): + """Verify the nova-cloud-controller to neutron-api relation data""" + u.log.debug('Checking novacc:neutron-api relation data...') + unit = self.nova_cc_sentry + relation = ['neutron-api', 'neutron-api:neutron-api'] + cc_ip = unit.relation('neutron-api', + 'neutron-api:neutron-api')['private-address'] + cc_endpoint = 'http://{}:8774/v2'.format(cc_ip) + expected = { + 'private-address': cc_ip, + 'nova_url': cc_endpoint, + } + ret = u.validate_relation_data(unit, relation, expected) + if ret: + message = u.relation_error('nova-cc neutron-api', ret) + amulet.raise_status(amulet.FAIL, msg=message) + def test_300_neutron_config(self): """Verify the data in the neutron config file.""" u.log.debug('Checking neutron gateway config file data...') @@ -400,7 +565,6 @@ class NeutronGatewayBasicDeployment(OpenStackAmuletDeployment): 'list_notifier', 'list_notifier_drivers': 'neutron.openstack.common.' 'notifier.rabbit_notifier', - 'core_plugin': 'neutron.plugins.ml2.plugin.Ml2Plugin' }, 'agent': { 'root_helper': 'sudo /usr/bin/neutron-rootwrap ' @@ -551,11 +715,10 @@ class NeutronGatewayBasicDeployment(OpenStackAmuletDeployment): if self._get_openstack_release() >= self.trusty_kilo: # Kilo or later - expected['admin_user'] = 'quantum_nova' -#? expected['admin_user'] = 'nova' + expected['admin_user'] = 'nova' else: # Juno or earlier - expected['admin_user'] = 'quantum_s3_ec2_nova' + expected['admin_user'] = 's3_ec2_nova' ret = u.validate_config_data(unit, conf, section, expected) if ret: @@ -609,7 +772,7 @@ class NeutronGatewayBasicDeployment(OpenStackAmuletDeployment): ep = self.keystone.service_catalog.url_for(service_type='identity', endpoint_type='publicURL') ng_db_rel = unit.relation('shared-db', - 'mysql:shared-db') + 'mysql:shared-db') nova_cc_relation = self.nova_cc_sentry.relation( 'quantum-network-service', 'neutron-gateway:quantum-network-service') @@ -631,11 +794,10 @@ class NeutronGatewayBasicDeployment(OpenStackAmuletDeployment): if self._get_openstack_release() >= self.trusty_kilo: # Kilo or later - expected['admin_user'] = 'quantum_nova' -#? expected['admin_user'] = 'nova' + expected['admin_user'] = 'nova' else: # Juno or earlier - expected['admin_user'] = 'quantum_s3_ec2_nova' + expected['admin_user'] = 's3_ec2_nova' ret = u.validate_config_data(unit, conf, section, expected) if ret: @@ -671,7 +833,7 @@ class NeutronGatewayBasicDeployment(OpenStackAmuletDeployment): unit = self.neutron_gateway_sentry conf = '/etc/nova/nova.conf' - rabbitmq_relation = self.rabbitmq_sentry.relation( + rabbitmq_relation = self.rmq_sentry.relation( 'amqp', 'neutron-gateway:amqp') nova_cc_relation = self.nova_cc_sentry.relation( 'quantum-network-service', @@ -708,7 +870,7 @@ class NeutronGatewayBasicDeployment(OpenStackAmuletDeployment): 'auth_strategy': 'keystone', 'url': nova_cc_relation['quantum_url'], 'admin_tenant_name': 'services', - 'admin_username': 'quantum_nova', + 'admin_username': 'nova', 'admin_password': nova_cc_relation['service_password'], 'admin_auth_url': ep, 'service_metadata_proxy': 'True', @@ -725,7 +887,7 @@ class NeutronGatewayBasicDeployment(OpenStackAmuletDeployment): 'neutron_auth_strategy': 'keystone', 'neutron_url': nova_cc_relation['quantum_url'], 'neutron_admin_tenant_name': 'services', - 'neutron_admin_username': 'quantum_s3_ec2_nova', + 'neutron_admin_username': 's3_ec2_nova', 'neutron_admin_password': nova_cc_relation['service_password'], 'neutron_admin_auth_url': ep, 'service_neutron_metadata_proxy': 'True', From f57afce1ac3bfacc9e1a9ece68854980e0f91ed9 Mon Sep 17 00:00:00 2001 From: Ryan Beisner Date: Tue, 15 Sep 2015 20:46:20 +0000 Subject: [PATCH 5/7] resync tests/charmhelpers --- .../charmhelpers/contrib/amulet/deployment.py | 6 +- tests/charmhelpers/contrib/amulet/utils.py | 293 +++++++++++--- .../contrib/openstack/amulet/deployment.py | 32 +- .../contrib/openstack/amulet/utils.py | 359 ++++++++++++++++++ 4 files changed, 628 insertions(+), 62 deletions(-) diff --git a/tests/charmhelpers/contrib/amulet/deployment.py b/tests/charmhelpers/contrib/amulet/deployment.py index 367d6b47..d451698d 100644 --- a/tests/charmhelpers/contrib/amulet/deployment.py +++ b/tests/charmhelpers/contrib/amulet/deployment.py @@ -51,7 +51,8 @@ class AmuletDeployment(object): if 'units' not in this_service: this_service['units'] = 1 - self.d.add(this_service['name'], units=this_service['units']) + self.d.add(this_service['name'], units=this_service['units'], + constraints=this_service.get('constraints')) for svc in other_services: if 'location' in svc: @@ -64,7 +65,8 @@ class AmuletDeployment(object): if 'units' not in svc: svc['units'] = 1 - self.d.add(svc['name'], charm=branch_location, units=svc['units']) + self.d.add(svc['name'], charm=branch_location, units=svc['units'], + constraints=svc.get('constraints')) def _add_relations(self, relations): """Add all of the relations for the services.""" diff --git a/tests/charmhelpers/contrib/amulet/utils.py b/tests/charmhelpers/contrib/amulet/utils.py index 7816c934..55c86347 100644 --- a/tests/charmhelpers/contrib/amulet/utils.py +++ b/tests/charmhelpers/contrib/amulet/utils.py @@ -19,9 +19,11 @@ import json import logging import os import re +import socket import subprocess import sys import time +import uuid import amulet import distro_info @@ -114,7 +116,7 @@ class AmuletUtils(object): # /!\ DEPRECATION WARNING (beisner): # New and existing tests should be rewritten to use # validate_services_by_name() as it is aware of init systems. - self.log.warn('/!\\ DEPRECATION WARNING: use ' + self.log.warn('DEPRECATION WARNING: use ' 'validate_services_by_name instead of validate_services ' 'due to init system differences.') @@ -269,33 +271,52 @@ class AmuletUtils(object): """Get last modification time of directory.""" return sentry_unit.directory_stat(directory)['mtime'] - def _get_proc_start_time(self, sentry_unit, service, pgrep_full=False): - """Get process' start time. + def _get_proc_start_time(self, sentry_unit, service, pgrep_full=None): + """Get start time of a process based on the last modification time + of the /proc/pid directory. - Determine start time of the process based on the last modification - time of the /proc/pid directory. If pgrep_full is True, the process - name is matched against the full command line. - """ - if pgrep_full: - cmd = 'pgrep -o -f {}'.format(service) - else: - cmd = 'pgrep -o {}'.format(service) - cmd = cmd + ' | grep -v pgrep || exit 0' - cmd_out = sentry_unit.run(cmd) - self.log.debug('CMDout: ' + str(cmd_out)) - if cmd_out[0]: - self.log.debug('Pid for %s %s' % (service, str(cmd_out[0]))) - proc_dir = '/proc/{}'.format(cmd_out[0].strip()) - return self._get_dir_mtime(sentry_unit, proc_dir) + :sentry_unit: The sentry unit to check for the service on + :service: service name to look for in process table + :pgrep_full: [Deprecated] Use full command line search mode with pgrep + :returns: epoch time of service process start + :param commands: list of bash commands + :param sentry_units: list of sentry unit pointers + :returns: None if successful; Failure message otherwise + """ + if pgrep_full is not None: + # /!\ DEPRECATION WARNING (beisner): + # No longer implemented, as pidof is now used instead of pgrep. + # https://bugs.launchpad.net/charm-helpers/+bug/1474030 + self.log.warn('DEPRECATION WARNING: pgrep_full bool is no ' + 'longer implemented re: lp 1474030.') + + pid_list = self.get_process_id_list(sentry_unit, service) + pid = pid_list[0] + proc_dir = '/proc/{}'.format(pid) + self.log.debug('Pid for {} on {}: {}'.format( + service, sentry_unit.info['unit_name'], pid)) + + return self._get_dir_mtime(sentry_unit, proc_dir) def service_restarted(self, sentry_unit, service, filename, - pgrep_full=False, sleep_time=20): + pgrep_full=None, sleep_time=20): """Check if service was restarted. Compare a service's start time vs a file's last modification time (such as a config file for that service) to determine if the service has been restarted. """ + # /!\ DEPRECATION WARNING (beisner): + # This method is prone to races in that no before-time is known. + # Use validate_service_config_changed instead. + + # NOTE(beisner) pgrep_full is no longer implemented, as pidof is now + # used instead of pgrep. pgrep_full is still passed through to ensure + # deprecation WARNS. lp1474030 + self.log.warn('DEPRECATION WARNING: use ' + 'validate_service_config_changed instead of ' + 'service_restarted due to known races.') + time.sleep(sleep_time) if (self._get_proc_start_time(sentry_unit, service, pgrep_full) >= self._get_file_mtime(sentry_unit, filename)): @@ -304,15 +325,15 @@ class AmuletUtils(object): return False def service_restarted_since(self, sentry_unit, mtime, service, - pgrep_full=False, sleep_time=20, - retry_count=2): + pgrep_full=None, sleep_time=20, + retry_count=2, retry_sleep_time=30): """Check if service was been started after a given time. Args: sentry_unit (sentry): The sentry unit to check for the service on mtime (float): The epoch time to check against service (string): service name to look for in process table - pgrep_full (boolean): Use full command line search mode with pgrep + pgrep_full: [Deprecated] Use full command line search mode with pgrep sleep_time (int): Seconds to sleep before looking for process retry_count (int): If service is not found, how many times to retry @@ -321,30 +342,44 @@ class AmuletUtils(object): False if service is older than mtime or if service was not found. """ - self.log.debug('Checking %s restarted since %s' % (service, mtime)) + # NOTE(beisner) pgrep_full is no longer implemented, as pidof is now + # used instead of pgrep. pgrep_full is still passed through to ensure + # deprecation WARNS. lp1474030 + + unit_name = sentry_unit.info['unit_name'] + self.log.debug('Checking that %s service restarted since %s on ' + '%s' % (service, mtime, unit_name)) time.sleep(sleep_time) - proc_start_time = self._get_proc_start_time(sentry_unit, service, - pgrep_full) - while retry_count > 0 and not proc_start_time: - self.log.debug('No pid file found for service %s, will retry %i ' - 'more times' % (service, retry_count)) - time.sleep(30) - proc_start_time = self._get_proc_start_time(sentry_unit, service, - pgrep_full) - retry_count = retry_count - 1 + proc_start_time = None + tries = 0 + while tries <= retry_count and not proc_start_time: + try: + proc_start_time = self._get_proc_start_time(sentry_unit, + service, + pgrep_full) + self.log.debug('Attempt {} to get {} proc start time on {} ' + 'OK'.format(tries, service, unit_name)) + except IOError: + # NOTE(beisner) - race avoidance, proc may not exist yet. + # https://bugs.launchpad.net/charm-helpers/+bug/1474030 + self.log.debug('Attempt {} to get {} proc start time on {} ' + 'failed'.format(tries, service, unit_name)) + time.sleep(retry_sleep_time) + tries += 1 if not proc_start_time: self.log.warn('No proc start time found, assuming service did ' 'not start') return False if proc_start_time >= mtime: - self.log.debug('proc start time is newer than provided mtime' - '(%s >= %s)' % (proc_start_time, mtime)) + self.log.debug('Proc start time is newer than provided mtime' + '(%s >= %s) on %s (OK)' % (proc_start_time, + mtime, unit_name)) return True else: - self.log.warn('proc start time (%s) is older than provided mtime ' - '(%s), service did not restart' % (proc_start_time, - mtime)) + self.log.warn('Proc start time (%s) is older than provided mtime ' + '(%s) on %s, service did not ' + 'restart' % (proc_start_time, mtime, unit_name)) return False def config_updated_since(self, sentry_unit, filename, mtime, @@ -374,8 +409,9 @@ class AmuletUtils(object): return False def validate_service_config_changed(self, sentry_unit, mtime, service, - filename, pgrep_full=False, - sleep_time=20, retry_count=2): + filename, pgrep_full=None, + sleep_time=20, retry_count=2, + retry_sleep_time=30): """Check service and file were updated after mtime Args: @@ -383,9 +419,10 @@ class AmuletUtils(object): mtime (float): The epoch time to check against service (string): service name to look for in process table filename (string): The file to check mtime of - pgrep_full (boolean): Use full command line search mode with pgrep - sleep_time (int): Seconds to sleep before looking for process + pgrep_full: [Deprecated] Use full command line search mode with pgrep + sleep_time (int): Initial sleep in seconds to pass to test helpers retry_count (int): If service is not found, how many times to retry + retry_sleep_time (int): Time in seconds to wait between retries Typical Usage: u = OpenStackAmuletUtils(ERROR) @@ -402,15 +439,25 @@ class AmuletUtils(object): mtime, False if service is older than mtime or if service was not found or if filename was modified before mtime. """ - self.log.debug('Checking %s restarted since %s' % (service, mtime)) - time.sleep(sleep_time) - service_restart = self.service_restarted_since(sentry_unit, mtime, - service, - pgrep_full=pgrep_full, - sleep_time=0, - retry_count=retry_count) - config_update = self.config_updated_since(sentry_unit, filename, mtime, - sleep_time=0) + + # NOTE(beisner) pgrep_full is no longer implemented, as pidof is now + # used instead of pgrep. pgrep_full is still passed through to ensure + # deprecation WARNS. lp1474030 + + service_restart = self.service_restarted_since( + sentry_unit, mtime, + service, + pgrep_full=pgrep_full, + sleep_time=sleep_time, + retry_count=retry_count, + retry_sleep_time=retry_sleep_time) + + config_update = self.config_updated_since( + sentry_unit, + filename, + mtime, + sleep_time=0) + return service_restart and config_update def get_sentry_time(self, sentry_unit): @@ -428,7 +475,6 @@ class AmuletUtils(object): """Return a list of all Ubuntu releases in order of release.""" _d = distro_info.UbuntuDistroInfo() _release_list = _d.all - self.log.debug('Ubuntu release list: {}'.format(_release_list)) return _release_list def file_to_url(self, file_rel_path): @@ -568,6 +614,142 @@ class AmuletUtils(object): return None + def validate_sectionless_conf(self, file_contents, expected): + """A crude conf parser. Useful to inspect configuration files which + do not have section headers (as would be necessary in order to use + the configparser). Such as openstack-dashboard or rabbitmq confs.""" + for line in file_contents.split('\n'): + if '=' in line: + args = line.split('=') + if len(args) <= 1: + continue + key = args[0].strip() + value = args[1].strip() + if key in expected.keys(): + if expected[key] != value: + msg = ('Config mismatch. Expected, actual: {}, ' + '{}'.format(expected[key], value)) + amulet.raise_status(amulet.FAIL, msg=msg) + + def get_unit_hostnames(self, units): + """Return a dict of juju unit names to hostnames.""" + host_names = {} + for unit in units: + host_names[unit.info['unit_name']] = \ + str(unit.file_contents('/etc/hostname').strip()) + self.log.debug('Unit host names: {}'.format(host_names)) + return host_names + + def run_cmd_unit(self, sentry_unit, cmd): + """Run a command on a unit, return the output and exit code.""" + output, code = sentry_unit.run(cmd) + if code == 0: + self.log.debug('{} `{}` command returned {} ' + '(OK)'.format(sentry_unit.info['unit_name'], + cmd, code)) + else: + msg = ('{} `{}` command returned {} ' + '{}'.format(sentry_unit.info['unit_name'], + cmd, code, output)) + amulet.raise_status(amulet.FAIL, msg=msg) + return str(output), code + + def file_exists_on_unit(self, sentry_unit, file_name): + """Check if a file exists on a unit.""" + try: + sentry_unit.file_stat(file_name) + return True + except IOError: + return False + except Exception as e: + msg = 'Error checking file {}: {}'.format(file_name, e) + amulet.raise_status(amulet.FAIL, msg=msg) + + def file_contents_safe(self, sentry_unit, file_name, + max_wait=60, fatal=False): + """Get file contents from a sentry unit. Wrap amulet file_contents + with retry logic to address races where a file checks as existing, + but no longer exists by the time file_contents is called. + Return None if file not found. Optionally raise if fatal is True.""" + unit_name = sentry_unit.info['unit_name'] + file_contents = False + tries = 0 + while not file_contents and tries < (max_wait / 4): + try: + file_contents = sentry_unit.file_contents(file_name) + except IOError: + self.log.debug('Attempt {} to open file {} from {} ' + 'failed'.format(tries, file_name, + unit_name)) + time.sleep(4) + tries += 1 + + if file_contents: + return file_contents + elif not fatal: + return None + elif fatal: + msg = 'Failed to get file contents from unit.' + amulet.raise_status(amulet.FAIL, msg) + + def port_knock_tcp(self, host="localhost", port=22, timeout=15): + """Open a TCP socket to check for a listening sevice on a host. + + :param host: host name or IP address, default to localhost + :param port: TCP port number, default to 22 + :param timeout: Connect timeout, default to 15 seconds + :returns: True if successful, False if connect failed + """ + + # Resolve host name if possible + try: + connect_host = socket.gethostbyname(host) + host_human = "{} ({})".format(connect_host, host) + except socket.error as e: + self.log.warn('Unable to resolve address: ' + '{} ({}) Trying anyway!'.format(host, e)) + connect_host = host + host_human = connect_host + + # Attempt socket connection + try: + knock = socket.socket(socket.AF_INET, socket.SOCK_STREAM) + knock.settimeout(timeout) + knock.connect((connect_host, port)) + knock.close() + self.log.debug('Socket connect OK for host ' + '{} on port {}.'.format(host_human, port)) + return True + except socket.error as e: + self.log.debug('Socket connect FAIL for' + ' {} port {} ({})'.format(host_human, port, e)) + return False + + def port_knock_units(self, sentry_units, port=22, + timeout=15, expect_success=True): + """Open a TCP socket to check for a listening sevice on each + listed juju unit. + + :param sentry_units: list of sentry unit pointers + :param port: TCP port number, default to 22 + :param timeout: Connect timeout, default to 15 seconds + :expect_success: True by default, set False to invert logic + :returns: None if successful, Failure message otherwise + """ + for unit in sentry_units: + host = unit.info['public-address'] + connected = self.port_knock_tcp(host, port, timeout) + if not connected and expect_success: + return 'Socket connect failed.' + elif connected and not expect_success: + return 'Socket connected unexpectedly.' + + def get_uuid_epoch_stamp(self): + """Returns a stamp string based on uuid4 and epoch time. Useful in + generating test messages which need to be unique-ish.""" + return '[{}-{}]'.format(uuid.uuid4(), time.time()) + +# amulet juju action helpers: def run_action(self, unit_sentry, action, _check_output=subprocess.check_output): """Run the named action on a given unit sentry. @@ -594,3 +776,12 @@ class AmuletUtils(object): output = _check_output(command, universal_newlines=True) data = json.loads(output) return data.get(u"status") == "completed" + + def status_get(self, unit): + """Return the current service status of this unit.""" + raw_status, return_code = unit.run( + "status-get --format=json --include-data") + if return_code != 0: + return ("unknown", "") + status = json.loads(raw_status) + return (status["status"], status["message"]) diff --git a/tests/charmhelpers/contrib/openstack/amulet/deployment.py b/tests/charmhelpers/contrib/openstack/amulet/deployment.py index 07ee2ef1..722bc645 100644 --- a/tests/charmhelpers/contrib/openstack/amulet/deployment.py +++ b/tests/charmhelpers/contrib/openstack/amulet/deployment.py @@ -44,20 +44,31 @@ class OpenStackAmuletDeployment(AmuletDeployment): Determine if the local branch being tested is derived from its stable or next (dev) branch, and based on this, use the corresonding stable or next branches for the other_services.""" + + # Charms outside the lp:~openstack-charmers namespace base_charms = ['mysql', 'mongodb', 'nrpe'] + # Force these charms to current series even when using an older series. + # ie. Use trusty/nrpe even when series is precise, as the P charm + # does not possess the necessary external master config and hooks. + force_series_current = ['nrpe'] + if self.series in ['precise', 'trusty']: base_series = self.series else: base_series = self.current_next - if self.stable: - for svc in other_services: + for svc in other_services: + if svc['name'] in force_series_current: + base_series = self.current_next + # If a location has been explicitly set, use it + if svc.get('location'): + continue + if self.stable: temp = 'lp:charms/{}/{}' svc['location'] = temp.format(base_series, svc['name']) - else: - for svc in other_services: + else: if svc['name'] in base_charms: temp = 'lp:charms/{}/{}' svc['location'] = temp.format(base_series, @@ -66,6 +77,7 @@ class OpenStackAmuletDeployment(AmuletDeployment): temp = 'lp:~openstack-charmers/charms/{}/{}/next' svc['location'] = temp.format(self.current_next, svc['name']) + return other_services def _add_services(self, this_service, other_services): @@ -77,21 +89,23 @@ class OpenStackAmuletDeployment(AmuletDeployment): services = other_services services.append(this_service) + + # Charms which should use the source config option use_source = ['mysql', 'mongodb', 'rabbitmq-server', 'ceph', 'ceph-osd', 'ceph-radosgw'] - # Most OpenStack subordinate charms do not expose an origin option - # as that is controlled by the principle. - ignore = ['cinder-ceph', 'hacluster', 'neutron-openvswitch', 'nrpe'] + + # Charms which can not use openstack-origin, ie. many subordinates + no_origin = ['cinder-ceph', 'hacluster', 'neutron-openvswitch', 'nrpe'] if self.openstack: for svc in services: - if svc['name'] not in use_source + ignore: + if svc['name'] not in use_source + no_origin: config = {'openstack-origin': self.openstack} self.d.configure(svc['name'], config) if self.source: for svc in services: - if svc['name'] in use_source and svc['name'] not in ignore: + if svc['name'] in use_source and svc['name'] not in no_origin: config = {'source': self.source} self.d.configure(svc['name'], config) diff --git a/tests/charmhelpers/contrib/openstack/amulet/utils.py b/tests/charmhelpers/contrib/openstack/amulet/utils.py index 03f79277..b1397419 100644 --- a/tests/charmhelpers/contrib/openstack/amulet/utils.py +++ b/tests/charmhelpers/contrib/openstack/amulet/utils.py @@ -27,6 +27,7 @@ import glanceclient.v1.client as glance_client import heatclient.v1.client as heat_client import keystoneclient.v2_0 as keystone_client import novaclient.v1_1.client as nova_client +import pika import swiftclient from charmhelpers.contrib.amulet.utils import ( @@ -602,3 +603,361 @@ class OpenStackAmuletUtils(AmuletUtils): self.log.debug('Ceph {} samples (OK): ' '{}'.format(sample_type, samples)) return None + +# rabbitmq/amqp specific helpers: + def add_rmq_test_user(self, sentry_units, + username="testuser1", password="changeme"): + """Add a test user via the first rmq juju unit, check connection as + the new user against all sentry units. + + :param sentry_units: list of sentry unit pointers + :param username: amqp user name, default to testuser1 + :param password: amqp user password + :returns: None if successful. Raise on error. + """ + self.log.debug('Adding rmq user ({})...'.format(username)) + + # Check that user does not already exist + cmd_user_list = 'rabbitmqctl list_users' + output, _ = self.run_cmd_unit(sentry_units[0], cmd_user_list) + if username in output: + self.log.warning('User ({}) already exists, returning ' + 'gracefully.'.format(username)) + return + + perms = '".*" ".*" ".*"' + cmds = ['rabbitmqctl add_user {} {}'.format(username, password), + 'rabbitmqctl set_permissions {} {}'.format(username, perms)] + + # Add user via first unit + for cmd in cmds: + output, _ = self.run_cmd_unit(sentry_units[0], cmd) + + # Check connection against the other sentry_units + self.log.debug('Checking user connect against units...') + for sentry_unit in sentry_units: + connection = self.connect_amqp_by_unit(sentry_unit, ssl=False, + username=username, + password=password) + connection.close() + + def delete_rmq_test_user(self, sentry_units, username="testuser1"): + """Delete a rabbitmq user via the first rmq juju unit. + + :param sentry_units: list of sentry unit pointers + :param username: amqp user name, default to testuser1 + :param password: amqp user password + :returns: None if successful or no such user. + """ + self.log.debug('Deleting rmq user ({})...'.format(username)) + + # Check that the user exists + cmd_user_list = 'rabbitmqctl list_users' + output, _ = self.run_cmd_unit(sentry_units[0], cmd_user_list) + + if username not in output: + self.log.warning('User ({}) does not exist, returning ' + 'gracefully.'.format(username)) + return + + # Delete the user + cmd_user_del = 'rabbitmqctl delete_user {}'.format(username) + output, _ = self.run_cmd_unit(sentry_units[0], cmd_user_del) + + def get_rmq_cluster_status(self, sentry_unit): + """Execute rabbitmq cluster status command on a unit and return + the full output. + + :param unit: sentry unit + :returns: String containing console output of cluster status command + """ + cmd = 'rabbitmqctl cluster_status' + output, _ = self.run_cmd_unit(sentry_unit, cmd) + self.log.debug('{} cluster_status:\n{}'.format( + sentry_unit.info['unit_name'], output)) + return str(output) + + def get_rmq_cluster_running_nodes(self, sentry_unit): + """Parse rabbitmqctl cluster_status output string, return list of + running rabbitmq cluster nodes. + + :param unit: sentry unit + :returns: List containing node names of running nodes + """ + # NOTE(beisner): rabbitmqctl cluster_status output is not + # json-parsable, do string chop foo, then json.loads that. + str_stat = self.get_rmq_cluster_status(sentry_unit) + if 'running_nodes' in str_stat: + pos_start = str_stat.find("{running_nodes,") + 15 + pos_end = str_stat.find("]},", pos_start) + 1 + str_run_nodes = str_stat[pos_start:pos_end].replace("'", '"') + run_nodes = json.loads(str_run_nodes) + return run_nodes + else: + return [] + + def validate_rmq_cluster_running_nodes(self, sentry_units): + """Check that all rmq unit hostnames are represented in the + cluster_status output of all units. + + :param host_names: dict of juju unit names to host names + :param units: list of sentry unit pointers (all rmq units) + :returns: None if successful, otherwise return error message + """ + host_names = self.get_unit_hostnames(sentry_units) + errors = [] + + # Query every unit for cluster_status running nodes + for query_unit in sentry_units: + query_unit_name = query_unit.info['unit_name'] + running_nodes = self.get_rmq_cluster_running_nodes(query_unit) + + # Confirm that every unit is represented in the queried unit's + # cluster_status running nodes output. + for validate_unit in sentry_units: + val_host_name = host_names[validate_unit.info['unit_name']] + val_node_name = 'rabbit@{}'.format(val_host_name) + + if val_node_name not in running_nodes: + errors.append('Cluster member check failed on {}: {} not ' + 'in {}\n'.format(query_unit_name, + val_node_name, + running_nodes)) + if errors: + return ''.join(errors) + + def rmq_ssl_is_enabled_on_unit(self, sentry_unit, port=None): + """Check a single juju rmq unit for ssl and port in the config file.""" + host = sentry_unit.info['public-address'] + unit_name = sentry_unit.info['unit_name'] + + conf_file = '/etc/rabbitmq/rabbitmq.config' + conf_contents = str(self.file_contents_safe(sentry_unit, + conf_file, max_wait=16)) + # Checks + conf_ssl = 'ssl' in conf_contents + conf_port = str(port) in conf_contents + + # Port explicitly checked in config + if port and conf_port and conf_ssl: + self.log.debug('SSL is enabled @{}:{} ' + '({})'.format(host, port, unit_name)) + return True + elif port and not conf_port and conf_ssl: + self.log.debug('SSL is enabled @{} but not on port {} ' + '({})'.format(host, port, unit_name)) + return False + # Port not checked (useful when checking that ssl is disabled) + elif not port and conf_ssl: + self.log.debug('SSL is enabled @{}:{} ' + '({})'.format(host, port, unit_name)) + return True + elif not port and not conf_ssl: + self.log.debug('SSL not enabled @{}:{} ' + '({})'.format(host, port, unit_name)) + return False + else: + msg = ('Unknown condition when checking SSL status @{}:{} ' + '({})'.format(host, port, unit_name)) + amulet.raise_status(amulet.FAIL, msg) + + def validate_rmq_ssl_enabled_units(self, sentry_units, port=None): + """Check that ssl is enabled on rmq juju sentry units. + + :param sentry_units: list of all rmq sentry units + :param port: optional ssl port override to validate + :returns: None if successful, otherwise return error message + """ + for sentry_unit in sentry_units: + if not self.rmq_ssl_is_enabled_on_unit(sentry_unit, port=port): + return ('Unexpected condition: ssl is disabled on unit ' + '({})'.format(sentry_unit.info['unit_name'])) + return None + + def validate_rmq_ssl_disabled_units(self, sentry_units): + """Check that ssl is enabled on listed rmq juju sentry units. + + :param sentry_units: list of all rmq sentry units + :returns: True if successful. Raise on error. + """ + for sentry_unit in sentry_units: + if self.rmq_ssl_is_enabled_on_unit(sentry_unit): + return ('Unexpected condition: ssl is enabled on unit ' + '({})'.format(sentry_unit.info['unit_name'])) + return None + + def configure_rmq_ssl_on(self, sentry_units, deployment, + port=None, max_wait=60): + """Turn ssl charm config option on, with optional non-default + ssl port specification. Confirm that it is enabled on every + unit. + + :param sentry_units: list of sentry units + :param deployment: amulet deployment object pointer + :param port: amqp port, use defaults if None + :param max_wait: maximum time to wait in seconds to confirm + :returns: None if successful. Raise on error. + """ + self.log.debug('Setting ssl charm config option: on') + + # Enable RMQ SSL + config = {'ssl': 'on'} + if port: + config['ssl_port'] = port + + deployment.configure('rabbitmq-server', config) + + # Confirm + tries = 0 + ret = self.validate_rmq_ssl_enabled_units(sentry_units, port=port) + while ret and tries < (max_wait / 4): + time.sleep(4) + self.log.debug('Attempt {}: {}'.format(tries, ret)) + ret = self.validate_rmq_ssl_enabled_units(sentry_units, port=port) + tries += 1 + + if ret: + amulet.raise_status(amulet.FAIL, ret) + + def configure_rmq_ssl_off(self, sentry_units, deployment, max_wait=60): + """Turn ssl charm config option off, confirm that it is disabled + on every unit. + + :param sentry_units: list of sentry units + :param deployment: amulet deployment object pointer + :param max_wait: maximum time to wait in seconds to confirm + :returns: None if successful. Raise on error. + """ + self.log.debug('Setting ssl charm config option: off') + + # Disable RMQ SSL + config = {'ssl': 'off'} + deployment.configure('rabbitmq-server', config) + + # Confirm + tries = 0 + ret = self.validate_rmq_ssl_disabled_units(sentry_units) + while ret and tries < (max_wait / 4): + time.sleep(4) + self.log.debug('Attempt {}: {}'.format(tries, ret)) + ret = self.validate_rmq_ssl_disabled_units(sentry_units) + tries += 1 + + if ret: + amulet.raise_status(amulet.FAIL, ret) + + def connect_amqp_by_unit(self, sentry_unit, ssl=False, + port=None, fatal=True, + username="testuser1", password="changeme"): + """Establish and return a pika amqp connection to the rabbitmq service + running on a rmq juju unit. + + :param sentry_unit: sentry unit pointer + :param ssl: boolean, default to False + :param port: amqp port, use defaults if None + :param fatal: boolean, default to True (raises on connect error) + :param username: amqp user name, default to testuser1 + :param password: amqp user password + :returns: pika amqp connection pointer or None if failed and non-fatal + """ + host = sentry_unit.info['public-address'] + unit_name = sentry_unit.info['unit_name'] + + # Default port logic if port is not specified + if ssl and not port: + port = 5671 + elif not ssl and not port: + port = 5672 + + self.log.debug('Connecting to amqp on {}:{} ({}) as ' + '{}...'.format(host, port, unit_name, username)) + + try: + credentials = pika.PlainCredentials(username, password) + parameters = pika.ConnectionParameters(host=host, port=port, + credentials=credentials, + ssl=ssl, + connection_attempts=3, + retry_delay=5, + socket_timeout=1) + connection = pika.BlockingConnection(parameters) + assert connection.server_properties['product'] == 'RabbitMQ' + self.log.debug('Connect OK') + return connection + except Exception as e: + msg = ('amqp connection failed to {}:{} as ' + '{} ({})'.format(host, port, username, str(e))) + if fatal: + amulet.raise_status(amulet.FAIL, msg) + else: + self.log.warn(msg) + return None + + def publish_amqp_message_by_unit(self, sentry_unit, message, + queue="test", ssl=False, + username="testuser1", + password="changeme", + port=None): + """Publish an amqp message to a rmq juju unit. + + :param sentry_unit: sentry unit pointer + :param message: amqp message string + :param queue: message queue, default to test + :param username: amqp user name, default to testuser1 + :param password: amqp user password + :param ssl: boolean, default to False + :param port: amqp port, use defaults if None + :returns: None. Raises exception if publish failed. + """ + self.log.debug('Publishing message to {} queue:\n{}'.format(queue, + message)) + connection = self.connect_amqp_by_unit(sentry_unit, ssl=ssl, + port=port, + username=username, + password=password) + + # NOTE(beisner): extra debug here re: pika hang potential: + # https://github.com/pika/pika/issues/297 + # https://groups.google.com/forum/#!topic/rabbitmq-users/Ja0iyfF0Szw + self.log.debug('Defining channel...') + channel = connection.channel() + self.log.debug('Declaring queue...') + channel.queue_declare(queue=queue, auto_delete=False, durable=True) + self.log.debug('Publishing message...') + channel.basic_publish(exchange='', routing_key=queue, body=message) + self.log.debug('Closing channel...') + channel.close() + self.log.debug('Closing connection...') + connection.close() + + def get_amqp_message_by_unit(self, sentry_unit, queue="test", + username="testuser1", + password="changeme", + ssl=False, port=None): + """Get an amqp message from a rmq juju unit. + + :param sentry_unit: sentry unit pointer + :param queue: message queue, default to test + :param username: amqp user name, default to testuser1 + :param password: amqp user password + :param ssl: boolean, default to False + :param port: amqp port, use defaults if None + :returns: amqp message body as string. Raise if get fails. + """ + connection = self.connect_amqp_by_unit(sentry_unit, ssl=ssl, + port=port, + username=username, + password=password) + channel = connection.channel() + method_frame, _, body = channel.basic_get(queue) + + if method_frame: + self.log.debug('Retreived message from {} queue:\n{}'.format(queue, + body)) + channel.basic_ack(method_frame.delivery_tag) + channel.close() + connection.close() + return body + else: + msg = 'No message retrieved.' + amulet.raise_status(amulet.FAIL, msg) From 9164b2c37f804411f07e3d96daeec5ba0438c1c4 Mon Sep 17 00:00:00 2001 From: Ryan Beisner Date: Mon, 21 Sep 2015 20:38:18 +0000 Subject: [PATCH 6/7] update file check retry; resync tests/charmhelpers for amulet file check race fix --- tests/basic_deployment.py | 16 +++--- tests/charmhelpers/contrib/amulet/utils.py | 63 ++++++++++++++++------ 2 files changed, 56 insertions(+), 23 deletions(-) diff --git a/tests/basic_deployment.py b/tests/basic_deployment.py index 8f1eec89..64edd6dc 100644 --- a/tests/basic_deployment.py +++ b/tests/basic_deployment.py @@ -981,27 +981,29 @@ class NeutronGatewayBasicDeployment(OpenStackAmuletDeployment): 'neutron-lbaas-agent': conf_file, 'neutron-metadata-agent': conf_file, 'neutron-metering-agent': conf_file, - 'neutron-openvswitch-agent': conf_file + 'neutron-openvswitch-agent': conf_file, } if self._get_openstack_release() <= self.trusty_juno: services.update({'neutron-vpn-agent': conf_file}) - # Make config change, check for service restarts + # Make config change, check for svc restart, conf file mod time change u.log.debug('Making config change on {}...'.format(juju_service)) mtime = u.get_sentry_time(sentry) self.d.configure(juju_service, set_alternate) - sleep_time = 90 +# sleep_time = 90 for s, conf_file in services.iteritems(): u.log.debug("Checking that service restarted: {}".format(s)) if not u.validate_service_config_changed(sentry, mtime, s, - conf_file, - pgrep_full=True, - sleep_time=sleep_time): + conf_file): +# conf_file, +# sleep_time=sleep_time): self.d.configure(juju_service, set_default) msg = "service {} didn't restart after config change".format(s) amulet.raise_status(amulet.FAIL, msg=msg) - sleep_time = 0 + + # Only do initial sleep on first service check +# sleep_time = 0 self.d.configure(juju_service, set_default) diff --git a/tests/charmhelpers/contrib/amulet/utils.py b/tests/charmhelpers/contrib/amulet/utils.py index 55c86347..2591a9b1 100644 --- a/tests/charmhelpers/contrib/amulet/utils.py +++ b/tests/charmhelpers/contrib/amulet/utils.py @@ -326,7 +326,7 @@ class AmuletUtils(object): def service_restarted_since(self, sentry_unit, mtime, service, pgrep_full=None, sleep_time=20, - retry_count=2, retry_sleep_time=30): + retry_count=30, retry_sleep_time=10): """Check if service was been started after a given time. Args: @@ -334,8 +334,9 @@ class AmuletUtils(object): mtime (float): The epoch time to check against service (string): service name to look for in process table pgrep_full: [Deprecated] Use full command line search mode with pgrep - sleep_time (int): Seconds to sleep before looking for process - retry_count (int): If service is not found, how many times to retry + sleep_time (int): Initial sleep time (s) before looking for file + retry_sleep_time (int): Time (s) to sleep between retries + retry_count (int): If file is not found, how many times to retry Returns: bool: True if service found and its start time it newer than mtime, @@ -359,11 +360,12 @@ class AmuletUtils(object): pgrep_full) self.log.debug('Attempt {} to get {} proc start time on {} ' 'OK'.format(tries, service, unit_name)) - except IOError: + except IOError as e: # NOTE(beisner) - race avoidance, proc may not exist yet. # https://bugs.launchpad.net/charm-helpers/+bug/1474030 self.log.debug('Attempt {} to get {} proc start time on {} ' - 'failed'.format(tries, service, unit_name)) + 'failed\n{}'.format(tries, service, + unit_name, e)) time.sleep(retry_sleep_time) tries += 1 @@ -383,35 +385,62 @@ class AmuletUtils(object): return False def config_updated_since(self, sentry_unit, filename, mtime, - sleep_time=20): + sleep_time=20, retry_count=30, + retry_sleep_time=10): """Check if file was modified after a given time. Args: sentry_unit (sentry): The sentry unit to check the file mtime on filename (string): The file to check mtime of mtime (float): The epoch time to check against - sleep_time (int): Seconds to sleep before looking for process + sleep_time (int): Initial sleep time (s) before looking for file + retry_sleep_time (int): Time (s) to sleep between retries + retry_count (int): If file is not found, how many times to retry Returns: bool: True if file was modified more recently than mtime, False if - file was modified before mtime, + file was modified before mtime, or if file not found. """ - self.log.debug('Checking %s updated since %s' % (filename, mtime)) + unit_name = sentry_unit.info['unit_name'] + self.log.debug('Checking that %s updated since %s on ' + '%s' % (filename, mtime, unit_name)) time.sleep(sleep_time) - file_mtime = self._get_file_mtime(sentry_unit, filename) + file_mtime = None + tries = 0 + while tries <= retry_count and not file_mtime: + try: + file_mtime = self._get_file_mtime(sentry_unit, filename) + self.log.debug('Attempt {} to get {} file mtime on {} ' + 'OK'.format(tries, filename, unit_name)) + except IOError as e: + # NOTE(beisner) - race avoidance, file may not exist yet. + # https://bugs.launchpad.net/charm-helpers/+bug/1474030 + self.log.debug('Attempt {} to get {} file mtime on {} ' + 'failed\n{}'.format(tries, filename, + unit_name, e)) + time.sleep(retry_sleep_time) + tries += 1 + + if not file_mtime: + self.log.warn('Could not determine file mtime, assuming ' + 'file does not exist') + return False + if file_mtime >= mtime: self.log.debug('File mtime is newer than provided mtime ' - '(%s >= %s)' % (file_mtime, mtime)) + '(%s >= %s) on %s (OK)' % (file_mtime, + mtime, unit_name)) return True else: - self.log.warn('File mtime %s is older than provided mtime %s' - % (file_mtime, mtime)) + self.log.warn('File mtime is older than provided mtime' + '(%s < on %s) on %s' % (file_mtime, + mtime, unit_name)) return False def validate_service_config_changed(self, sentry_unit, mtime, service, filename, pgrep_full=None, - sleep_time=20, retry_count=2, - retry_sleep_time=30): + sleep_time=20, retry_count=30, + retry_sleep_time=10): """Check service and file were updated after mtime Args: @@ -456,7 +485,9 @@ class AmuletUtils(object): sentry_unit, filename, mtime, - sleep_time=0) + sleep_time=sleep_time, + retry_count=retry_count, + retry_sleep_time=retry_sleep_time) return service_restart and config_update From 1111c59d709c7e2cbf08da48355cffceccd7a895 Mon Sep 17 00:00:00 2001 From: Ryan Beisner Date: Tue, 22 Sep 2015 15:22:25 +0000 Subject: [PATCH 7/7] disable wily test target, to be re-enabled separately after validation --- tests/021-basic-wily-liberty | 0 1 file changed, 0 insertions(+), 0 deletions(-) mode change 100755 => 100644 tests/021-basic-wily-liberty diff --git a/tests/021-basic-wily-liberty b/tests/021-basic-wily-liberty old mode 100755 new mode 100644