From 35beb586dadd54c1b7a1917b2ae73f4cb517eecf Mon Sep 17 00:00:00 2001 From: Joe Guo Date: Sun, 1 Sep 2019 21:08:13 +1200 Subject: [PATCH] Migrate functional tests from Amulet to Zaza Migrate glance tests from Amulet to Zaza, which is Python3 only. func-test-pr: https://github.com/openstack-charmers/zaza-openstack-tests/pull/50 Partial-Bug: 1828424 Change-Id: I053a056d8e2a79ed22ea8a976a8f378994b1e8eb Signed-off-by: Joe Guo --- Makefile | 6 +- test-requirements.txt | 31 +- tests/README.md | 9 - tests/basic_deployment.py | 553 ------ tests/bundles/bionic-queens.yaml | 33 + tests/bundles/bionic-rocky.yaml | 36 + tests/bundles/bionic-stein.yaml | 36 + tests/bundles/bionic-train.yaml | 36 + tests/bundles/disco-stein.yaml | 35 + tests/bundles/trusty-mitaka.yaml | 39 + tests/bundles/xenial-mitaka.yaml | 32 + tests/bundles/xenial-ocata.yaml | 36 + tests/bundles/xenial-pike.yaml | 36 + tests/bundles/xenial-queens.yaml | 36 + .../contrib/openstack/amulet/utils.py | 1587 ----------------- tests/charmhelpers/core/host.py | 1066 ----------- tests/dev-basic-cosmic-rocky | 23 - tests/gate-basic-bionic-queens | 23 - tests/gate-basic-bionic-rocky | 25 - tests/gate-basic-bionic-stein | 25 - tests/gate-basic-disco-stein | 23 - tests/gate-basic-trusty-mitaka | 25 - tests/gate-basic-xenial-mitaka | 23 - tests/gate-basic-xenial-ocata | 25 - tests/gate-basic-xenial-pike | 25 - tests/gate-basic-xenial-queens | 25 - tests/tests.yaml | 43 +- tox.ini | 66 +- 28 files changed, 412 insertions(+), 3546 deletions(-) delete mode 100644 tests/README.md delete mode 100644 tests/basic_deployment.py create mode 100644 tests/bundles/bionic-queens.yaml create mode 100644 tests/bundles/bionic-rocky.yaml create mode 100644 tests/bundles/bionic-stein.yaml create mode 100644 tests/bundles/bionic-train.yaml create mode 100644 tests/bundles/disco-stein.yaml create mode 100644 tests/bundles/trusty-mitaka.yaml create mode 100644 tests/bundles/xenial-mitaka.yaml create mode 100644 tests/bundles/xenial-ocata.yaml create mode 100644 tests/bundles/xenial-pike.yaml create mode 100644 tests/bundles/xenial-queens.yaml delete mode 100644 tests/charmhelpers/contrib/openstack/amulet/utils.py delete mode 100644 tests/charmhelpers/core/host.py delete mode 100755 tests/dev-basic-cosmic-rocky delete mode 100755 tests/gate-basic-bionic-queens delete mode 100755 tests/gate-basic-bionic-rocky delete mode 100755 tests/gate-basic-bionic-stein delete mode 100755 tests/gate-basic-disco-stein delete mode 100755 tests/gate-basic-trusty-mitaka delete mode 100755 tests/gate-basic-xenial-mitaka delete mode 100755 tests/gate-basic-xenial-ocata delete mode 100755 tests/gate-basic-xenial-pike delete mode 100755 tests/gate-basic-xenial-queens diff --git a/Makefile b/Makefile index dcfd21f6..abeee96b 100644 --- a/Makefile +++ b/Makefile @@ -1,5 +1,5 @@ #!/usr/bin/make -PYTHON := /usr/bin/env python +PYTHON := /usr/bin/env python3 lint: @tox -e pep8 @@ -9,8 +9,8 @@ test: @tox -e py27 functional_test: - @echo Starting functional tests... - @tox -e func27 + @echo Starting Zaza functional tests... + @tox -e func bin/charm_helpers_sync.py: @mkdir -p bin diff --git a/test-requirements.txt b/test-requirements.txt index 272ce1da..f4b5cc68 100644 --- a/test-requirements.txt +++ b/test-requirements.txt @@ -1,29 +1,10 @@ -# The order of packages is significant, because pip processes them in the order -# of appearance. Changing the order has an impact on the overall integration -# process, which may cause wedges in the gate later. -charm-tools>=2.4.4 -coverage>=3.6 +charm-tools>=2.6.1 +requests>=2.18.4 mock>=1.2 flake8>=2.2.4,<=2.4.1 stestr>=2.2.0 -requests>=2.18.4 -# BEGIN: Amulet OpenStack Charm Helper Requirements -# Liberty client lower constraints -amulet>=1.14.3,<2.0;python_version=='2.7' -bundletester>=0.6.1,<1.0;python_version=='2.7' -python-ceilometerclient>=1.5.0 -python-cinderclient>=1.4.0 +git+https://github.com/openstack-charmers/zaza.git#egg=zaza;python_version>='3.0' +git+https://github.com/openstack-charmers/zaza-openstack-tests.git#egg=zaza.openstack +coverage>=4.5.2 + python-glanceclient>=1.1.0 -python-heatclient>=0.8.0 -python-keystoneclient>=1.7.1 -python-neutronclient>=3.1.0 -python-novaclient>=2.30.1 -python-openstackclient>=1.7.0 -python-swiftclient>=2.6.0 -pika>=0.10.0,<1.0 -distro-info -git+https://github.com/juju/charm-helpers.git#egg=charmhelpers -# END: Amulet OpenStack Charm Helper Requirements -# NOTE: workaround for 14.04 pip/tox -pytz -pyudev # for ceph-* charm unit tests (not mocked?) diff --git a/tests/README.md b/tests/README.md deleted file mode 100644 index 046be7fb..00000000 --- a/tests/README.md +++ /dev/null @@ -1,9 +0,0 @@ -# Overview - -This directory provides Amulet tests to verify basic deployment functionality -from the perspective of this charm, its requirements and its features, as -exercised in a subset of the full OpenStack deployment test bundle topology. - -For full details on functional testing of OpenStack charms please refer to -the [functional testing](http://docs.openstack.org/developer/charm-guide/testing.html#functional-testing) -section of the OpenStack Charm Guide. diff --git a/tests/basic_deployment.py b/tests/basic_deployment.py deleted file mode 100644 index d57febad..00000000 --- a/tests/basic_deployment.py +++ /dev/null @@ -1,553 +0,0 @@ -#!/usr/bin/env python -# -# Copyright 2016 Canonical Ltd -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -""" -Basic glance amulet functional tests. -""" - -import amulet -import time - -from charmhelpers.contrib.openstack.amulet.deployment import ( - OpenStackAmuletDeployment -) - -from charmhelpers.contrib.openstack.amulet.utils import ( - OpenStackAmuletUtils, - DEBUG, - # ERROR -) - -# Use DEBUG to turn on debug logging -u = OpenStackAmuletUtils(DEBUG) - - -class GlanceBasicDeployment(OpenStackAmuletDeployment): - """Amulet tests on a basic file-backed glance deployment. Verify - relations, service status, endpoint service catalog, create and - delete new image.""" - - def __init__(self, series=None, openstack=None, source=None, - stable=False): - """Deploy the entire test environment.""" - super(GlanceBasicDeployment, self).__init__(series, openstack, - source, stable) - self._add_services() - self._add_relations() - self._configure_services() - self._deploy() - - u.log.info('Waiting on extended status checks...') - exclude_services = [] - self._auto_wait_for_status(exclude_services=exclude_services) - - self.d.sentry.wait() - self._initialize_tests() - - def _assert_services(self, should_run): - if self._get_openstack_release() >= self.bionic_stein: - services = ('apache2', 'haproxy', 'glance-api') - else: - services = ('apache2', 'haproxy', 'glance-api', 'glance-registry') - - u.get_unit_process_ids( - {self.glance_sentry: services}, - expect_success=should_run) - - def _add_services(self): - """Add services - - Add the services that we're testing, where glance is local, - and the rest of the service are from lp branches that are - compatible with the local charm (e.g. stable or next). - """ - this_service = {'name': 'glance'} - other_services = [ - self.get_percona_service_entry(), - {'name': 'rabbitmq-server'}, - {'name': 'keystone'}, - ] - super(GlanceBasicDeployment, self)._add_services(this_service, - other_services) - - def _add_relations(self): - """Add relations for the services.""" - relations = {'glance:identity-service': 'keystone:identity-service', - 'glance:shared-db': 'percona-cluster:shared-db', - 'keystone:shared-db': 'percona-cluster:shared-db', - 'glance:amqp': 'rabbitmq-server:amqp'} - super(GlanceBasicDeployment, self)._add_relations(relations) - - def _configure_services(self): - """Configure all of the services.""" - glance_config = {} - keystone_config = { - 'admin-password': 'openstack', - 'admin-token': 'ubuntutesting', - } - pxc_config = { - 'dataset-size': '25%', - 'max-connections': 1000, - 'root-password': 'ChangeMe123', - 'sst-password': 'ChangeMe123', - } - configs = { - 'glance': glance_config, - 'keystone': keystone_config, - 'percona-cluster': pxc_config, - } - super(GlanceBasicDeployment, self)._configure_services(configs) - - def _initialize_tests(self): - """Perform final initialization before tests get run.""" - # Access the sentries for inspecting service units - self.pxc_sentry = self.d.sentry['percona-cluster'][0] - self.glance_sentry = self.d.sentry['glance'][0] - self.keystone_sentry = self.d.sentry['keystone'][0] - self.rabbitmq_sentry = self.d.sentry['rabbitmq-server'][0] - u.log.debug('openstack release val: {}'.format( - self._get_openstack_release())) - u.log.debug('openstack release str: {}'.format( - self._get_openstack_release_string())) - - # Authenticate admin with keystone - self.keystone_session, self.keystone = u.get_default_keystone_session( - self.keystone_sentry, - openstack_release=self._get_openstack_release()) - - force_v1_client = False - if self._get_openstack_release() == self.trusty_icehouse: - # Updating image properties (such as arch or hypervisor) using the - # v2 api in icehouse results in: - # https://bugs.launchpad.net/python-glanceclient/+bug/1371559 - u.log.debug('Forcing glance to use v1 api') - force_v1_client = True - - # Authenticate admin with glance endpoint - self.glance = u.authenticate_glance_admin( - self.keystone, - force_v1_client=force_v1_client) - - def test_100_services(self): - """Verify that the expected services are running on the - corresponding service units.""" - services = { - self.keystone_sentry: ['keystone'], - self.rabbitmq_sentry: ['rabbitmq-server'], - } - if self._get_openstack_release() >= self.bionic_stein: - services.update( - {self.glance_sentry: ['glance-api']}) - else: - services.update( - {self.glance_sentry: ['glance-api', 'glance-registry']}) - if self._get_openstack_release() >= self.trusty_liberty: - services[self.keystone_sentry] = ['apache2'] - ret = u.validate_services_by_name(services) - if ret: - amulet.raise_status(amulet.FAIL, msg=ret) - - def test_102_service_catalog(self): - """Verify that the service catalog endpoint data is valid.""" - u.log.debug('Checking keystone service catalog...') - endpoint_check = { - 'adminURL': u.valid_url, - 'id': u.not_null, - 'region': 'RegionOne', - 'publicURL': u.valid_url, - 'internalURL': u.valid_url - } - expected = { - 'image': [endpoint_check], - 'identity': [endpoint_check] - } - actual = self.keystone.service_catalog.get_endpoints() - - ret = u.validate_svc_catalog_endpoint_data( - expected, - actual, - openstack_release=self._get_openstack_release()) - if ret: - amulet.raise_status(amulet.FAIL, msg=ret) - - def test_104_glance_endpoint(self): - """Verify the glance endpoint data.""" - u.log.debug('Checking glance api endpoint data...') - endpoints = self.keystone.endpoints.list() - admin_port = internal_port = public_port = '9292' - expected = { - 'id': u.not_null, - 'region': 'RegionOne', - 'adminurl': u.valid_url, - 'internalurl': u.valid_url, - 'publicurl': u.valid_url, - 'service_id': u.not_null - } - ret = u.validate_endpoint_data( - endpoints, - admin_port, - internal_port, - public_port, - expected, - openstack_release=self._get_openstack_release()) - if ret: - amulet.raise_status(amulet.FAIL, - msg='glance endpoint: {}'.format(ret)) - - def test_106_keystone_endpoint(self): - """Verify the keystone endpoint data.""" - u.log.debug('Checking keystone api endpoint data...') - endpoints = self.keystone.endpoints.list() - admin_port = '35357' - internal_port = public_port = '5000' - expected = { - 'id': u.not_null, - 'region': 'RegionOne', - 'adminurl': u.valid_url, - 'internalurl': u.valid_url, - 'publicurl': u.valid_url, - 'service_id': u.not_null - } - ret = u.validate_endpoint_data( - endpoints, - admin_port, - internal_port, - public_port, - expected, - openstack_release=self._get_openstack_release()) - if ret: - amulet.raise_status(amulet.FAIL, - msg='keystone endpoint: {}'.format(ret)) - - def test_110_users(self): - """Verify expected users.""" - u.log.debug('Checking keystone users...') - if self._get_openstack_release() >= self.xenial_queens: - expected = [ - {'name': 'glance', - 'enabled': True, - 'default_project_id': u.not_null, - 'id': u.not_null, - 'email': 'juju@localhost'} - ] - domain = self.keystone.domains.find(name='service_domain') - actual = self.keystone.users.list(domain=domain) - api_version = 3 - else: - expected = [ - {'name': 'glance', - 'enabled': True, - 'tenantId': u.not_null, - 'id': u.not_null, - 'email': 'juju@localhost'}, - {'name': 'admin', - 'enabled': True, - 'tenantId': u.not_null, - 'id': u.not_null, - 'email': 'juju@localhost'} - ] - actual = self.keystone.users.list() - api_version = 2 - ret = u.validate_user_data(expected, actual, api_version) - if ret: - amulet.raise_status(amulet.FAIL, msg=ret) - - def test_115_memcache(self): - u.validate_memcache(self.glance_sentry, - '/etc/glance/glance-api.conf', - self._get_openstack_release(), - earliest_release=self.trusty_mitaka) - if self._get_openstack_release() < self.bionic_stein: - u.validate_memcache(self.glance_sentry, - '/etc/glance/glance-registry.conf', - self._get_openstack_release(), - earliest_release=self.trusty_mitaka) - - def test_200_mysql_glance_db_relation(self): - """Verify the mysql:glance shared-db relation data""" - u.log.debug('Checking mysql to glance shared-db relation data...') - unit = self.pxc_sentry - relation = ['shared-db', 'glance:shared-db'] - expected = { - 'private-address': u.valid_ip, - 'db_host': u.valid_ip - } - ret = u.validate_relation_data(unit, relation, expected) - if ret: - message = u.relation_error('mysql shared-db', ret) - amulet.raise_status(amulet.FAIL, msg=message) - - def test_201_glance_mysql_db_relation(self): - """Verify the glance:mysql shared-db relation data""" - u.log.debug('Checking glance to mysql shared-db relation data...') - unit = self.glance_sentry - relation = ['shared-db', 'percona-cluster:shared-db'] - expected = { - 'private-address': u.valid_ip, - 'hostname': u.valid_ip, - 'username': 'glance', - 'database': 'glance' - } - ret = u.validate_relation_data(unit, relation, expected) - if ret: - message = u.relation_error('glance shared-db', ret) - amulet.raise_status(amulet.FAIL, msg=message) - - def test_202_keystone_glance_id_relation(self): - """Verify the keystone:glance identity-service relation data""" - u.log.debug('Checking keystone to glance id relation data...') - unit = self.keystone_sentry - relation = ['identity-service', - 'glance:identity-service'] - expected = { - 'service_protocol': 'http', - 'service_tenant': 'services', - 'admin_token': 'ubuntutesting', - 'service_password': u.not_null, - 'service_port': '5000', - 'auth_port': '35357', - 'auth_protocol': 'http', - 'private-address': u.valid_ip, - 'auth_host': u.valid_ip, - 'service_username': 'glance', - 'service_tenant_id': u.not_null, - 'service_host': u.valid_ip - } - ret = u.validate_relation_data(unit, relation, expected) - if ret: - message = u.relation_error('keystone identity-service', ret) - amulet.raise_status(amulet.FAIL, msg=message) - - def test_203_glance_keystone_id_relation(self): - """Verify the glance:keystone identity-service relation data""" - u.log.debug('Checking glance to keystone relation data...') - unit = self.glance_sentry - relation = ['identity-service', - 'keystone:identity-service'] - expected = { - 'service': 'glance', - 'region': 'RegionOne', - 'public_url': u.valid_url, - 'internal_url': u.valid_url, - 'admin_url': u.valid_url, - 'private-address': u.valid_ip - } - ret = u.validate_relation_data(unit, relation, expected) - if ret: - message = u.relation_error('glance identity-service', ret) - amulet.raise_status(amulet.FAIL, msg=message) - - def test_204_rabbitmq_glance_amqp_relation(self): - """Verify the rabbitmq-server:glance amqp relation data""" - u.log.debug('Checking rmq to glance amqp relation data...') - unit = self.rabbitmq_sentry - relation = ['amqp', 'glance:amqp'] - expected = { - 'private-address': u.valid_ip, - 'password': u.not_null, - 'hostname': u.valid_ip - } - ret = u.validate_relation_data(unit, relation, expected) - if ret: - message = u.relation_error('rabbitmq amqp', ret) - amulet.raise_status(amulet.FAIL, msg=message) - - def test_205_glance_rabbitmq_amqp_relation(self): - """Verify the glance:rabbitmq-server amqp relation data""" - u.log.debug('Checking glance to rmq amqp relation data...') - unit = self.glance_sentry - relation = ['amqp', 'rabbitmq-server:amqp'] - expected = { - 'private-address': u.valid_ip, - 'vhost': 'openstack', - 'username': u.not_null - } - ret = u.validate_relation_data(unit, relation, expected) - if ret: - message = u.relation_error('glance amqp', ret) - amulet.raise_status(amulet.FAIL, msg=message) - - def test_302_glance_registry_default_config(self): - """Verify configs in glance-registry.conf""" - if self._get_openstack_release() >= self.bionic_stein: - u.log.debug('Skipping check of glance registry config file for ' - '>= bionic-stein') - return - u.log.debug('Checking glance registry config file...') - unit = self.glance_sentry - rel_my_gl = self.pxc_sentry.relation('shared-db', 'glance:shared-db') - if self._get_openstack_release() < self.xenial_queens: - dialect = 'mysql' - else: - dialect = 'mysql+pymysql' - db_uri = "{}://{}:{}@{}/{}".format(dialect, - 'glance', - rel_my_gl['password'], - rel_my_gl['db_host'], - 'glance') - conf = '/etc/glance/glance-registry.conf' - - expected = { - 'DEFAULT': { - 'use_syslog': 'False', - 'log_file': '/var/log/glance/registry.log', - 'debug': 'False', - 'verbose': 'False', - 'bind_host': '0.0.0.0', - 'bind_port': '9191' - }, - } - - if self._get_openstack_release() >= self.trusty_kilo: - # Kilo or later - expected['database'] = { - 'idle_timeout': '3600', - 'connection': db_uri - } - else: - # Juno or earlier - expected['database'] = { - 'idle_timeout': '3600', - 'connection': db_uri - } - - for section, pairs in expected.iteritems(): - ret = u.validate_config_data(unit, conf, section, pairs) - if ret: - message = "glance registry paste config error: {}".format(ret) - amulet.raise_status(amulet.FAIL, msg=message) - - def test_410_glance_image_create_delete(self): - """Create new cirros image in glance, verify, then delete it.""" - u.log.debug('Creating, checking and deleting glance image...') - img_new = u.create_cirros_image(self.glance, "cirros-image-1") - img_id = img_new.id - u.delete_resource(self.glance.images, img_id, msg="glance image") - - def test_411_set_disk_format(self): - sleep_time = 30 - if self._get_openstack_release() >= self.trusty_kilo: - section = 'image_format' - elif self._get_openstack_release() > self.trusty_icehouse: - section = 'DEFAULT' - else: - u.log.debug('Test not supported before juno') - return - sentry = self.glance_sentry - juju_service = 'glance' - - # Expected default and alternate values - set_default = { - 'disk-formats': 'ami,ari,aki,vhd,vmdk,raw,qcow2,vdi,iso,root-tar'} - set_alternate = {'disk-formats': 'qcow2'} - - # Config file affected by juju set config change - conf_file = '/etc/glance/glance-api.conf' - - # Make config change, check for service restarts - u.log.debug('Setting disk format {}...'.format(juju_service)) - self.d.configure(juju_service, set_alternate) - - u.log.debug('Sleeping to let hooks fire') - time.sleep(sleep_time) - u.log.debug("Checking disk format option has updated") - ret = u.validate_config_data( - sentry, - conf_file, - section, - {'disk_formats': 'qcow2'}) - if ret: - msg = "disk_formats was not updated in section {} in {}".format( - section, - conf_file) - amulet.raise_status(amulet.FAIL, msg=msg) - - self.d.configure(juju_service, set_default) - - def test_500_security_checklist_action(self): - """Verify expected result on a default install""" - u.log.debug("Testing security-checklist") - sentry_unit = self.glance_sentry - - action_id = u.run_action(sentry_unit, "security-checklist") - u.wait_on_action(action_id) - data = amulet.actions.get_action_output(action_id, full_output=True) - assert data.get(u"status") == "failed", \ - "Security check is expected to not pass by default" - - def test_900_glance_restart_on_config_change(self): - """Verify that the specified services are restarted when the config - is changed.""" - sentry = self.glance_sentry - juju_service = 'glance' - - # Expected default and alternate values - set_default = {'use-syslog': 'False'} - set_alternate = {'use-syslog': 'True'} - - # Config file affected by juju set config change - conf_file = '/etc/glance/glance-api.conf' - - # Services which are expected to restart upon config change - services = { - 'glance-api': conf_file, - } - if self._get_openstack_release() < self.bionic_stein: - services.update({'glance-registry': conf_file}) - - # Make config change, check for service restarts - u.log.debug('Making config change on {}...'.format(juju_service)) - mtime = u.get_sentry_time(sentry) - self.d.configure(juju_service, set_alternate) - - sleep_time = 30 - for s, conf_file in services.iteritems(): - u.log.debug("Checking that service restarted: {}".format(s)) - if not u.validate_service_config_changed(sentry, mtime, s, - conf_file, - retry_count=4, - retry_sleep_time=20, - sleep_time=sleep_time): - self.d.configure(juju_service, set_default) - msg = "service {} didn't restart after config change".format(s) - amulet.raise_status(amulet.FAIL, msg=msg) - sleep_time = 0 - - self.d.configure(juju_service, set_default) - - def test_901_pause_resume(self): - """Test pause and resume actions.""" - u.log.debug('Checking pause and resume actions...') - - unit = self.d.sentry['glance'][0] - unit_name = unit.info['unit_name'] - u.log.debug("Unit name: {}".format(unit_name)) - - u.log.debug('Checking for active status on {}'.format(unit_name)) - assert u.status_get(unit)[0] == "active" - - u.log.debug('Running pause action on {}'.format(unit_name)) - self._assert_services(should_run=True) - action_id = u.run_action(unit, "pause") - u.log.debug('Waiting on action {}'.format(action_id)) - assert u.wait_on_action(action_id), "Pause action failed." - self._assert_services(should_run=False) - - u.log.debug('Running resume action on {}'.format(unit_name)) - action_id = u.run_action(unit, "resume") - u.log.debug('Waiting on action {}'.format(action_id)) - assert u.wait_on_action(action_id), "Resume action failed" - self._assert_services(should_run=True) diff --git a/tests/bundles/bionic-queens.yaml b/tests/bundles/bionic-queens.yaml new file mode 100644 index 00000000..3cc8ace1 --- /dev/null +++ b/tests/bundles/bionic-queens.yaml @@ -0,0 +1,33 @@ +series: bionic + +comment: + - 'machines section to decide order of deployment. database sooner = faster' + +machines: + '0': + constraints: mem=3072M + '1': + '2': + +relations: + - ["keystone:shared-db", "mysql:shared-db"] + - ["glance:shared-db", "mysql:shared-db"] + - ["glance:identity-service", "keystone:identity-service"] + +applications: + mysql: + charm: cs:~openstack-charmers-next/percona-cluster + num_units: 1 + to: + - '0' + keystone: + series: bionic + charm: cs:~openstack-charmers-next/keystone + num_units: 1 + to: + - '1' + glance: + charm: ../../../glance + num_units: 1 + to: + - '2' diff --git a/tests/bundles/bionic-rocky.yaml b/tests/bundles/bionic-rocky.yaml new file mode 100644 index 00000000..3f833ebf --- /dev/null +++ b/tests/bundles/bionic-rocky.yaml @@ -0,0 +1,36 @@ +series: bionic + +comment: + - 'machines section to decide order of deployment. database sooner = faster' + +machines: + '0': + constraints: mem=3072M + '1': + '2': + +relations: + - ["keystone:shared-db", "mysql:shared-db"] + - ["glance:shared-db", "mysql:shared-db"] + - ["glance:identity-service", "keystone:identity-service"] + +applications: + mysql: + charm: cs:~openstack-charmers-next/percona-cluster + num_units: 1 + to: + - '0' + keystone: + charm: cs:~openstack-charmers-next/keystone + num_units: 1 + options: + openstack-origin: cloud:bionic-rocky + to: + - '1' + glance: + charm: ../../../glance + num_units: 1 + options: + openstack-origin: cloud:bionic-rocky + to: + - '2' diff --git a/tests/bundles/bionic-stein.yaml b/tests/bundles/bionic-stein.yaml new file mode 100644 index 00000000..c16db07c --- /dev/null +++ b/tests/bundles/bionic-stein.yaml @@ -0,0 +1,36 @@ +series: bionic + +comment: + - 'machines section to decide order of deployment. database sooner = faster' + +machines: + '0': + constraints: mem=3072M + '1': + '2': + +relations: + - ["keystone:shared-db", "mysql:shared-db"] + - ["glance:shared-db", "mysql:shared-db"] + - ["glance:identity-service", "keystone:identity-service"] + +applications: + mysql: + charm: cs:~openstack-charmers-next/percona-cluster + num_units: 1 + to: + - '0' + keystone: + charm: cs:~openstack-charmers-next/keystone + num_units: 1 + options: + openstack-origin: cloud:bionic-stein + to: + - '1' + glance: + charm: ../../../glance + num_units: 1 + options: + openstack-origin: cloud:bionic-stein + to: + - '2' diff --git a/tests/bundles/bionic-train.yaml b/tests/bundles/bionic-train.yaml new file mode 100644 index 00000000..e33ad39a --- /dev/null +++ b/tests/bundles/bionic-train.yaml @@ -0,0 +1,36 @@ +series: bionic + +comment: + - 'machines section to decide order of deployment. database sooner = faster' + +machines: + '0': + constraints: mem=3072M + '1': + '2': + +relations: + - ["keystone:shared-db", "mysql:shared-db"] + - ["glance:shared-db", "mysql:shared-db"] + - ["glance:identity-service", "keystone:identity-service"] + +applications: + mysql: + charm: cs:~openstack-charmers-next/percona-cluster + num_units: 1 + to: + - '0' + keystone: + charm: cs:~openstack-charmers-next/keystone + num_units: 1 + options: + openstack-origin: cloud:bionic-train + to: + - '1' + glance: + charm: ../../../glance + num_units: 1 + options: + openstack-origin: cloud:bionic-train + to: + - '2' diff --git a/tests/bundles/disco-stein.yaml b/tests/bundles/disco-stein.yaml new file mode 100644 index 00000000..dbaa945c --- /dev/null +++ b/tests/bundles/disco-stein.yaml @@ -0,0 +1,35 @@ +series: disco + +comment: + - 'machines section to decide order of deployment. database sooner = faster' + +machines: + '0': + constraints: mem=3072M + '1': + '2': + +relations: + - - keystone:shared-db + - mysql:shared-db + - - glance:shared-db + - mysql:shared-db + - - glance:identity-service + - keystone:identity-service + +applications: + mysql: + charm: cs:~openstack-charmers-next/percona-cluster + num_units: 1 + to: + - '0' + keystone: + charm: cs:~openstack-charmers-next/keystone + num_units: 1 + to: + - '1' + glance: + charm: ../../../glance + num_units: 1 + to: + - '2' diff --git a/tests/bundles/trusty-mitaka.yaml b/tests/bundles/trusty-mitaka.yaml new file mode 100644 index 00000000..c60b85b6 --- /dev/null +++ b/tests/bundles/trusty-mitaka.yaml @@ -0,0 +1,39 @@ +series: trusty + +comment: + - 'machines section to decide order of deployment. database sooner = faster' + - 'virt-type=kvm is workaround while awaiting new release of python-libjuju' + +machines: + '0': + constraints: virt-type=kvm mem=3072M + '1': + constraints: virt-type=kvm + '2': + constraints: virt-type=kvm + +relations: + - ["keystone:shared-db", "mysql:shared-db"] + - ["glance:shared-db", "mysql:shared-db"] + - ["glance:identity-service", "keystone:identity-service"] + +applications: + mysql: + charm: cs:trusty/percona-cluster + num_units: 1 + to: + - '0' + keystone: + charm: cs:~openstack-charmers-next/keystone + num_units: 1 + options: + openstack-origin: cloud:trusty-mitaka + to: + - '1' + glance: + charm: ../../../glance + num_units: 1 + options: + openstack-origin: cloud:trusty-mitaka + to: + - '2' diff --git a/tests/bundles/xenial-mitaka.yaml b/tests/bundles/xenial-mitaka.yaml new file mode 100644 index 00000000..2e7527d2 --- /dev/null +++ b/tests/bundles/xenial-mitaka.yaml @@ -0,0 +1,32 @@ +series: xenial + +comment: + - 'machines section to decide order of deployment. database sooner = faster' + +machines: + '0': + constraints: mem=3072M + '1': + '2': + +relations: + - ["keystone:shared-db", "mysql:shared-db"] + - ["glance:shared-db", "mysql:shared-db"] + - ["glance:identity-service", "keystone:identity-service"] + +applications: + mysql: + charm: cs:~openstack-charmers-next/percona-cluster + num_units: 1 + to: + - '0' + keystone: + charm: cs:~openstack-charmers-next/keystone + num_units: 1 + to: + - '1' + glance: + charm: ../../../glance + num_units: 1 + to: + - '2' diff --git a/tests/bundles/xenial-ocata.yaml b/tests/bundles/xenial-ocata.yaml new file mode 100644 index 00000000..9f823b61 --- /dev/null +++ b/tests/bundles/xenial-ocata.yaml @@ -0,0 +1,36 @@ +series: xenial + +comment: + - 'machines section to decide order of deployment. database sooner = faster' + +machines: + '0': + constraints: mem=3072M + '1': + '2': + +relations: + - ["keystone:shared-db", "mysql:shared-db"] + - ["glance:shared-db", "mysql:shared-db"] + - ["glance:identity-service", "keystone:identity-service"] + +applications: + mysql: + charm: cs:~openstack-charmers-next/percona-cluster + num_units: 1 + to: + - '0' + keystone: + charm: cs:~openstack-charmers-next/keystone + num_units: 1 + options: + openstack-origin: cloud:xenial-ocata + to: + - '1' + glance: + charm: ../../../glance + num_units: 1 + options: + openstack-origin: cloud:xenial-ocata + to: + - '2' diff --git a/tests/bundles/xenial-pike.yaml b/tests/bundles/xenial-pike.yaml new file mode 100644 index 00000000..1bbe5be7 --- /dev/null +++ b/tests/bundles/xenial-pike.yaml @@ -0,0 +1,36 @@ +series: xenial + +comment: + - 'machines section to decide order of deployment. database sooner = faster' + +machines: + '0': + constraints: mem=3072M + '1': + '2': + +relations: + - ["keystone:shared-db", "mysql:shared-db"] + - ["glance:shared-db", "mysql:shared-db"] + - ["glance:identity-service", "keystone:identity-service"] + +applications: + mysql: + charm: cs:~openstack-charmers-next/percona-cluster + num_units: 1 + to: + - '0' + keystone: + charm: cs:~openstack-charmers-next/keystone + num_units: 1 + options: + openstack-origin: cloud:xenial-pike + to: + - '1' + glance: + charm: ../../../glance + num_units: 1 + options: + openstack-origin: cloud:xenial-pike + to: + - '2' diff --git a/tests/bundles/xenial-queens.yaml b/tests/bundles/xenial-queens.yaml new file mode 100644 index 00000000..c8b93fa0 --- /dev/null +++ b/tests/bundles/xenial-queens.yaml @@ -0,0 +1,36 @@ +series: xenial + +comment: + - 'machines section to decide order of deployment. database sooner = faster' + +machines: + '0': + constraints: mem=3072M + '1': + '2': + +relations: + - ["keystone:shared-db", "mysql:shared-db"] + - ["glance:shared-db", "mysql:shared-db"] + - ["glance:identity-service", "keystone:identity-service"] + +applications: + mysql: + charm: cs:~openstack-charmers-next/percona-cluster + num_units: 1 + to: + - '0' + keystone: + charm: cs:~openstack-charmers-next/keystone + num_units: 1 + options: + openstack-origin: cloud:xenial-queens + to: + - '1' + glance: + charm: ../../../glance + num_units: 1 + options: + openstack-origin: cloud:xenial-queens + to: + - '2' diff --git a/tests/charmhelpers/contrib/openstack/amulet/utils.py b/tests/charmhelpers/contrib/openstack/amulet/utils.py deleted file mode 100644 index 9133e9b3..00000000 --- a/tests/charmhelpers/contrib/openstack/amulet/utils.py +++ /dev/null @@ -1,1587 +0,0 @@ -# Copyright 2014-2015 Canonical Limited. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -import amulet -import json -import logging -import os -import re -import six -import time -import urllib -import urlparse - -import cinderclient.v1.client as cinder_client -import cinderclient.v2.client as cinder_clientv2 -import glanceclient.v1 as glance_client -import glanceclient.v2 as glance_clientv2 -import heatclient.v1.client as heat_client -from keystoneclient.v2_0 import client as keystone_client -from keystoneauth1.identity import ( - v3, - v2, -) -from keystoneauth1 import session as keystone_session -from keystoneclient.v3 import client as keystone_client_v3 -from novaclient import exceptions - -import novaclient.client as nova_client -import novaclient -import pika -import swiftclient - -from charmhelpers.core.decorators import retry_on_exception -from charmhelpers.contrib.amulet.utils import ( - AmuletUtils -) -from charmhelpers.core.host import CompareHostReleases - -DEBUG = logging.DEBUG -ERROR = logging.ERROR - -NOVA_CLIENT_VERSION = "2" - -OPENSTACK_RELEASES_PAIRS = [ - 'trusty_icehouse', 'trusty_kilo', 'trusty_liberty', - 'trusty_mitaka', 'xenial_mitaka', 'xenial_newton', - 'yakkety_newton', 'xenial_ocata', 'zesty_ocata', - 'xenial_pike', 'artful_pike', 'xenial_queens', - 'bionic_queens', 'bionic_rocky', 'cosmic_rocky'] - - -class OpenStackAmuletUtils(AmuletUtils): - """OpenStack amulet utilities. - - This class inherits from AmuletUtils and has additional support - that is specifically for use by OpenStack charm tests. - """ - - def __init__(self, log_level=ERROR): - """Initialize the deployment environment.""" - super(OpenStackAmuletUtils, self).__init__(log_level) - - def validate_endpoint_data(self, endpoints, admin_port, internal_port, - public_port, expected, openstack_release=None): - """Validate endpoint data. Pick the correct validator based on - OpenStack release. Expected data should be in the v2 format: - { - 'id': id, - 'region': region, - 'adminurl': adminurl, - 'internalurl': internalurl, - 'publicurl': publicurl, - 'service_id': service_id} - - """ - validation_function = self.validate_v2_endpoint_data - xenial_queens = OPENSTACK_RELEASES_PAIRS.index('xenial_queens') - if openstack_release and openstack_release >= xenial_queens: - validation_function = self.validate_v3_endpoint_data - expected = { - 'id': expected['id'], - 'region': expected['region'], - 'region_id': 'RegionOne', - 'url': self.valid_url, - 'interface': self.not_null, - 'service_id': expected['service_id']} - return validation_function(endpoints, admin_port, internal_port, - public_port, expected) - - def validate_v2_endpoint_data(self, endpoints, admin_port, internal_port, - public_port, expected): - """Validate endpoint data. - - Validate actual endpoint data vs expected endpoint data. The ports - are used to find the matching endpoint. - """ - self.log.debug('Validating endpoint data...') - self.log.debug('actual: {}'.format(repr(endpoints))) - found = False - for ep in endpoints: - self.log.debug('endpoint: {}'.format(repr(ep))) - if (admin_port in ep.adminurl and - internal_port in ep.internalurl and - public_port in ep.publicurl): - found = True - actual = {'id': ep.id, - 'region': ep.region, - 'adminurl': ep.adminurl, - 'internalurl': ep.internalurl, - 'publicurl': ep.publicurl, - 'service_id': ep.service_id} - ret = self._validate_dict_data(expected, actual) - if ret: - return 'unexpected endpoint data - {}'.format(ret) - - if not found: - return 'endpoint not found' - - def validate_v3_endpoint_data(self, endpoints, admin_port, internal_port, - public_port, expected, expected_num_eps=3): - """Validate keystone v3 endpoint data. - - Validate the v3 endpoint data which has changed from v2. The - ports are used to find the matching endpoint. - - The new v3 endpoint data looks like: - - ['}, - region=RegionOne, - region_id=RegionOne, - service_id=17f842a0dc084b928e476fafe67e4095, - url=http://10.5.6.5:9312>, - '}, - region=RegionOne, - region_id=RegionOne, - service_id=72fc8736fb41435e8b3584205bb2cfa3, - url=http://10.5.6.6:35357/v3>, - ... ] - """ - self.log.debug('Validating v3 endpoint data...') - self.log.debug('actual: {}'.format(repr(endpoints))) - found = [] - for ep in endpoints: - self.log.debug('endpoint: {}'.format(repr(ep))) - if ((admin_port in ep.url and ep.interface == 'admin') or - (internal_port in ep.url and ep.interface == 'internal') or - (public_port in ep.url and ep.interface == 'public')): - found.append(ep.interface) - # note we ignore the links member. - actual = {'id': ep.id, - 'region': ep.region, - 'region_id': ep.region_id, - 'interface': self.not_null, - 'url': ep.url, - 'service_id': ep.service_id, } - ret = self._validate_dict_data(expected, actual) - if ret: - return 'unexpected endpoint data - {}'.format(ret) - - if len(found) != expected_num_eps: - return 'Unexpected number of endpoints found' - - def convert_svc_catalog_endpoint_data_to_v3(self, ep_data): - """Convert v2 endpoint data into v3. - - { - 'service_name1': [ - { - 'adminURL': adminURL, - 'id': id, - 'region': region. - 'publicURL': publicURL, - 'internalURL': internalURL - }], - 'service_name2': [ - { - 'adminURL': adminURL, - 'id': id, - 'region': region. - 'publicURL': publicURL, - 'internalURL': internalURL - }], - } - """ - self.log.warn("Endpoint ID and Region ID validation is limited to not " - "null checks after v2 to v3 conversion") - for svc in ep_data.keys(): - assert len(ep_data[svc]) == 1, "Unknown data format" - svc_ep_data = ep_data[svc][0] - ep_data[svc] = [ - { - 'url': svc_ep_data['adminURL'], - 'interface': 'admin', - 'region': svc_ep_data['region'], - 'region_id': self.not_null, - 'id': self.not_null}, - { - 'url': svc_ep_data['publicURL'], - 'interface': 'public', - 'region': svc_ep_data['region'], - 'region_id': self.not_null, - 'id': self.not_null}, - { - 'url': svc_ep_data['internalURL'], - 'interface': 'internal', - 'region': svc_ep_data['region'], - 'region_id': self.not_null, - 'id': self.not_null}] - return ep_data - - def validate_svc_catalog_endpoint_data(self, expected, actual, - openstack_release=None): - """Validate service catalog endpoint data. Pick the correct validator - for the OpenStack version. Expected data should be in the v2 format: - { - 'service_name1': [ - { - 'adminURL': adminURL, - 'id': id, - 'region': region. - 'publicURL': publicURL, - 'internalURL': internalURL - }], - 'service_name2': [ - { - 'adminURL': adminURL, - 'id': id, - 'region': region. - 'publicURL': publicURL, - 'internalURL': internalURL - }], - } - - """ - validation_function = self.validate_v2_svc_catalog_endpoint_data - xenial_queens = OPENSTACK_RELEASES_PAIRS.index('xenial_queens') - if openstack_release and openstack_release >= xenial_queens: - validation_function = self.validate_v3_svc_catalog_endpoint_data - expected = self.convert_svc_catalog_endpoint_data_to_v3(expected) - return validation_function(expected, actual) - - def validate_v2_svc_catalog_endpoint_data(self, expected, actual): - """Validate service catalog endpoint data. - - Validate a list of actual service catalog endpoints vs a list of - expected service catalog endpoints. - """ - self.log.debug('Validating service catalog endpoint data...') - self.log.debug('actual: {}'.format(repr(actual))) - for k, v in six.iteritems(expected): - if k in actual: - ret = self._validate_dict_data(expected[k][0], actual[k][0]) - if ret: - return self.endpoint_error(k, ret) - else: - return "endpoint {} does not exist".format(k) - return ret - - def validate_v3_svc_catalog_endpoint_data(self, expected, actual): - """Validate the keystone v3 catalog endpoint data. - - Validate a list of dictinaries that make up the keystone v3 service - catalogue. - - It is in the form of: - - - {u'identity': [{u'id': u'48346b01c6804b298cdd7349aadb732e', - u'interface': u'admin', - u'region': u'RegionOne', - u'region_id': u'RegionOne', - u'url': u'http://10.5.5.224:35357/v3'}, - {u'id': u'8414f7352a4b47a69fddd9dbd2aef5cf', - u'interface': u'public', - u'region': u'RegionOne', - u'region_id': u'RegionOne', - u'url': u'http://10.5.5.224:5000/v3'}, - {u'id': u'd5ca31440cc24ee1bf625e2996fb6a5b', - u'interface': u'internal', - u'region': u'RegionOne', - u'region_id': u'RegionOne', - u'url': u'http://10.5.5.224:5000/v3'}], - u'key-manager': [{u'id': u'68ebc17df0b045fcb8a8a433ebea9e62', - u'interface': u'public', - u'region': u'RegionOne', - u'region_id': u'RegionOne', - u'url': u'http://10.5.5.223:9311'}, - {u'id': u'9cdfe2a893c34afd8f504eb218cd2f9d', - u'interface': u'internal', - u'region': u'RegionOne', - u'region_id': u'RegionOne', - u'url': u'http://10.5.5.223:9311'}, - {u'id': u'f629388955bc407f8b11d8b7ca168086', - u'interface': u'admin', - u'region': u'RegionOne', - u'region_id': u'RegionOne', - u'url': u'http://10.5.5.223:9312'}]} - - Note, that an added complication is that the order of admin, public, - internal against 'interface' in each region. - - Thus, the function sorts the expected and actual lists using the - interface key as a sort key, prior to the comparison. - """ - self.log.debug('Validating v3 service catalog endpoint data...') - self.log.debug('actual: {}'.format(repr(actual))) - for k, v in six.iteritems(expected): - if k in actual: - l_expected = sorted(v, key=lambda x: x['interface']) - l_actual = sorted(actual[k], key=lambda x: x['interface']) - if len(l_actual) != len(l_expected): - return ("endpoint {} has differing number of interfaces " - " - expected({}), actual({})" - .format(k, len(l_expected), len(l_actual))) - for i_expected, i_actual in zip(l_expected, l_actual): - self.log.debug("checking interface {}" - .format(i_expected['interface'])) - ret = self._validate_dict_data(i_expected, i_actual) - if ret: - return self.endpoint_error(k, ret) - else: - return "endpoint {} does not exist".format(k) - return ret - - def validate_tenant_data(self, expected, actual): - """Validate tenant data. - - Validate a list of actual tenant data vs list of expected tenant - data. - """ - self.log.debug('Validating tenant data...') - self.log.debug('actual: {}'.format(repr(actual))) - for e in expected: - found = False - for act in actual: - a = {'enabled': act.enabled, 'description': act.description, - 'name': act.name, 'id': act.id} - if e['name'] == a['name']: - found = True - ret = self._validate_dict_data(e, a) - if ret: - return "unexpected tenant data - {}".format(ret) - if not found: - return "tenant {} does not exist".format(e['name']) - return ret - - def validate_role_data(self, expected, actual): - """Validate role data. - - Validate a list of actual role data vs a list of expected role - data. - """ - self.log.debug('Validating role data...') - self.log.debug('actual: {}'.format(repr(actual))) - for e in expected: - found = False - for act in actual: - a = {'name': act.name, 'id': act.id} - if e['name'] == a['name']: - found = True - ret = self._validate_dict_data(e, a) - if ret: - return "unexpected role data - {}".format(ret) - if not found: - return "role {} does not exist".format(e['name']) - return ret - - def validate_user_data(self, expected, actual, api_version=None): - """Validate user data. - - Validate a list of actual user data vs a list of expected user - data. - """ - self.log.debug('Validating user data...') - self.log.debug('actual: {}'.format(repr(actual))) - for e in expected: - found = False - for act in actual: - if e['name'] == act.name: - a = {'enabled': act.enabled, 'name': act.name, - 'email': act.email, 'id': act.id} - if api_version == 3: - a['default_project_id'] = getattr(act, - 'default_project_id', - 'none') - else: - a['tenantId'] = act.tenantId - found = True - ret = self._validate_dict_data(e, a) - if ret: - return "unexpected user data - {}".format(ret) - if not found: - return "user {} does not exist".format(e['name']) - return ret - - def validate_flavor_data(self, expected, actual): - """Validate flavor data. - - Validate a list of actual flavors vs a list of expected flavors. - """ - self.log.debug('Validating flavor data...') - self.log.debug('actual: {}'.format(repr(actual))) - act = [a.name for a in actual] - return self._validate_list_data(expected, act) - - def tenant_exists(self, keystone, tenant): - """Return True if tenant exists.""" - self.log.debug('Checking if tenant exists ({})...'.format(tenant)) - return tenant in [t.name for t in keystone.tenants.list()] - - @retry_on_exception(num_retries=5, base_delay=1) - def keystone_wait_for_propagation(self, sentry_relation_pairs, - api_version): - """Iterate over list of sentry and relation tuples and verify that - api_version has the expected value. - - :param sentry_relation_pairs: list of sentry, relation name tuples used - for monitoring propagation of relation - data - :param api_version: api_version to expect in relation data - :returns: None if successful. Raise on error. - """ - for (sentry, relation_name) in sentry_relation_pairs: - rel = sentry.relation('identity-service', - relation_name) - self.log.debug('keystone relation data: {}'.format(rel)) - if rel.get('api_version') != str(api_version): - raise Exception("api_version not propagated through relation" - " data yet ('{}' != '{}')." - "".format(rel.get('api_version'), api_version)) - - def keystone_configure_api_version(self, sentry_relation_pairs, deployment, - api_version): - """Configure preferred-api-version of keystone in deployment and - monitor provided list of relation objects for propagation - before returning to caller. - - :param sentry_relation_pairs: list of sentry, relation tuples used for - monitoring propagation of relation data - :param deployment: deployment to configure - :param api_version: value preferred-api-version will be set to - :returns: None if successful. Raise on error. - """ - self.log.debug("Setting keystone preferred-api-version: '{}'" - "".format(api_version)) - - config = {'preferred-api-version': api_version} - deployment.d.configure('keystone', config) - deployment._auto_wait_for_status() - self.keystone_wait_for_propagation(sentry_relation_pairs, api_version) - - def authenticate_cinder_admin(self, keystone, api_version=2): - """Authenticates admin user with cinder.""" - self.log.debug('Authenticating cinder admin...') - _clients = { - 1: cinder_client.Client, - 2: cinder_clientv2.Client} - return _clients[api_version](session=keystone.session) - - def authenticate_keystone(self, keystone_ip, username, password, - api_version=False, admin_port=False, - user_domain_name=None, domain_name=None, - project_domain_name=None, project_name=None): - """Authenticate with Keystone""" - self.log.debug('Authenticating with keystone...') - if not api_version: - api_version = 2 - sess, auth = self.get_keystone_session( - keystone_ip=keystone_ip, - username=username, - password=password, - api_version=api_version, - admin_port=admin_port, - user_domain_name=user_domain_name, - domain_name=domain_name, - project_domain_name=project_domain_name, - project_name=project_name - ) - if api_version == 2: - client = keystone_client.Client(session=sess) - else: - client = keystone_client_v3.Client(session=sess) - # This populates the client.service_catalog - client.auth_ref = auth.get_access(sess) - return client - - def get_keystone_session(self, keystone_ip, username, password, - api_version=False, admin_port=False, - user_domain_name=None, domain_name=None, - project_domain_name=None, project_name=None): - """Return a keystone session object""" - ep = self.get_keystone_endpoint(keystone_ip, - api_version=api_version, - admin_port=admin_port) - if api_version == 2: - auth = v2.Password( - username=username, - password=password, - tenant_name=project_name, - auth_url=ep - ) - sess = keystone_session.Session(auth=auth) - else: - auth = v3.Password( - user_domain_name=user_domain_name, - username=username, - password=password, - domain_name=domain_name, - project_domain_name=project_domain_name, - project_name=project_name, - auth_url=ep - ) - sess = keystone_session.Session(auth=auth) - return (sess, auth) - - def get_keystone_endpoint(self, keystone_ip, api_version=None, - admin_port=False): - """Return keystone endpoint""" - port = 5000 - if admin_port: - port = 35357 - base_ep = "http://{}:{}".format(keystone_ip.strip().decode('utf-8'), - port) - if api_version == 2: - ep = base_ep + "/v2.0" - else: - ep = base_ep + "/v3" - return ep - - def get_default_keystone_session(self, keystone_sentry, - openstack_release=None, api_version=2): - """Return a keystone session object and client object assuming standard - default settings - - Example call in amulet tests: - self.keystone_session, self.keystone = u.get_default_keystone_session( - self.keystone_sentry, - openstack_release=self._get_openstack_release()) - - The session can then be used to auth other clients: - neutronclient.Client(session=session) - aodh_client.Client(session=session) - eyc - """ - self.log.debug('Authenticating keystone admin...') - # 11 => xenial_queens - if api_version == 3 or (openstack_release and openstack_release >= 11): - client_class = keystone_client_v3.Client - api_version = 3 - else: - client_class = keystone_client.Client - keystone_ip = keystone_sentry.info['public-address'] - session, auth = self.get_keystone_session( - keystone_ip, - api_version=api_version, - username='admin', - password='openstack', - project_name='admin', - user_domain_name='admin_domain', - project_domain_name='admin_domain') - client = client_class(session=session) - # This populates the client.service_catalog - client.auth_ref = auth.get_access(session) - return session, client - - def authenticate_keystone_admin(self, keystone_sentry, user, password, - tenant=None, api_version=None, - keystone_ip=None, user_domain_name=None, - project_domain_name=None, - project_name=None): - """Authenticates admin user with the keystone admin endpoint.""" - self.log.debug('Authenticating keystone admin...') - if not keystone_ip: - keystone_ip = keystone_sentry.info['public-address'] - - # To support backward compatibility usage of this function - if not project_name: - project_name = tenant - if api_version == 3 and not user_domain_name: - user_domain_name = 'admin_domain' - if api_version == 3 and not project_domain_name: - project_domain_name = 'admin_domain' - if api_version == 3 and not project_name: - project_name = 'admin' - - return self.authenticate_keystone( - keystone_ip, user, password, - api_version=api_version, - user_domain_name=user_domain_name, - project_domain_name=project_domain_name, - project_name=project_name, - admin_port=True) - - def authenticate_keystone_user(self, keystone, user, password, tenant): - """Authenticates a regular user with the keystone public endpoint.""" - self.log.debug('Authenticating keystone user ({})...'.format(user)) - ep = keystone.service_catalog.url_for(service_type='identity', - interface='publicURL') - keystone_ip = urlparse.urlparse(ep).hostname - - return self.authenticate_keystone(keystone_ip, user, password, - project_name=tenant) - - def authenticate_glance_admin(self, keystone, force_v1_client=False): - """Authenticates admin user with glance.""" - self.log.debug('Authenticating glance admin...') - ep = keystone.service_catalog.url_for(service_type='image', - interface='adminURL') - if not force_v1_client and keystone.session: - return glance_clientv2.Client("2", session=keystone.session) - else: - return glance_client.Client(ep, token=keystone.auth_token) - - def authenticate_heat_admin(self, keystone): - """Authenticates the admin user with heat.""" - self.log.debug('Authenticating heat admin...') - ep = keystone.service_catalog.url_for(service_type='orchestration', - interface='publicURL') - if keystone.session: - return heat_client.Client(endpoint=ep, session=keystone.session) - else: - return heat_client.Client(endpoint=ep, token=keystone.auth_token) - - def authenticate_nova_user(self, keystone, user, password, tenant): - """Authenticates a regular user with nova-api.""" - self.log.debug('Authenticating nova user ({})...'.format(user)) - ep = keystone.service_catalog.url_for(service_type='identity', - interface='publicURL') - if keystone.session: - return nova_client.Client(NOVA_CLIENT_VERSION, - session=keystone.session, - auth_url=ep) - elif novaclient.__version__[0] >= "7": - return nova_client.Client(NOVA_CLIENT_VERSION, - username=user, password=password, - project_name=tenant, auth_url=ep) - else: - return nova_client.Client(NOVA_CLIENT_VERSION, - username=user, api_key=password, - project_id=tenant, auth_url=ep) - - def authenticate_swift_user(self, keystone, user, password, tenant): - """Authenticates a regular user with swift api.""" - self.log.debug('Authenticating swift user ({})...'.format(user)) - ep = keystone.service_catalog.url_for(service_type='identity', - interface='publicURL') - if keystone.session: - return swiftclient.Connection(session=keystone.session) - else: - return swiftclient.Connection(authurl=ep, - user=user, - key=password, - tenant_name=tenant, - auth_version='2.0') - - def create_flavor(self, nova, name, ram, vcpus, disk, flavorid="auto", - ephemeral=0, swap=0, rxtx_factor=1.0, is_public=True): - """Create the specified flavor.""" - try: - nova.flavors.find(name=name) - except (exceptions.NotFound, exceptions.NoUniqueMatch): - self.log.debug('Creating flavor ({})'.format(name)) - nova.flavors.create(name, ram, vcpus, disk, flavorid, - ephemeral, swap, rxtx_factor, is_public) - - def glance_create_image(self, glance, image_name, image_url, - download_dir='tests', - hypervisor_type=None, - disk_format='qcow2', - architecture='x86_64', - container_format='bare'): - """Download an image and upload it to glance, validate its status - and return an image object pointer. KVM defaults, can override for - LXD. - - :param glance: pointer to authenticated glance api connection - :param image_name: display name for new image - :param image_url: url to retrieve - :param download_dir: directory to store downloaded image file - :param hypervisor_type: glance image hypervisor property - :param disk_format: glance image disk format - :param architecture: glance image architecture property - :param container_format: glance image container format - :returns: glance image pointer - """ - self.log.debug('Creating glance image ({}) from ' - '{}...'.format(image_name, image_url)) - - # Download image - http_proxy = os.getenv('AMULET_HTTP_PROXY') - self.log.debug('AMULET_HTTP_PROXY: {}'.format(http_proxy)) - if http_proxy: - proxies = {'http': http_proxy} - opener = urllib.FancyURLopener(proxies) - else: - opener = urllib.FancyURLopener() - - abs_file_name = os.path.join(download_dir, image_name) - if not os.path.exists(abs_file_name): - opener.retrieve(image_url, abs_file_name) - - # Create glance image - glance_properties = { - 'architecture': architecture, - } - if hypervisor_type: - glance_properties['hypervisor_type'] = hypervisor_type - # Create glance image - if float(glance.version) < 2.0: - with open(abs_file_name) as f: - image = glance.images.create( - name=image_name, - is_public=True, - disk_format=disk_format, - container_format=container_format, - properties=glance_properties, - data=f) - else: - image = glance.images.create( - name=image_name, - visibility="public", - disk_format=disk_format, - container_format=container_format) - glance.images.upload(image.id, open(abs_file_name, 'rb')) - glance.images.update(image.id, **glance_properties) - - # Wait for image to reach active status - img_id = image.id - ret = self.resource_reaches_status(glance.images, img_id, - expected_stat='active', - msg='Image status wait') - if not ret: - msg = 'Glance image failed to reach expected state.' - amulet.raise_status(amulet.FAIL, msg=msg) - - # Re-validate new image - self.log.debug('Validating image attributes...') - val_img_name = glance.images.get(img_id).name - val_img_stat = glance.images.get(img_id).status - val_img_cfmt = glance.images.get(img_id).container_format - val_img_dfmt = glance.images.get(img_id).disk_format - - if float(glance.version) < 2.0: - val_img_pub = glance.images.get(img_id).is_public - else: - val_img_pub = glance.images.get(img_id).visibility == "public" - - msg_attr = ('Image attributes - name:{} public:{} id:{} stat:{} ' - 'container fmt:{} disk fmt:{}'.format( - val_img_name, val_img_pub, img_id, - val_img_stat, val_img_cfmt, val_img_dfmt)) - - if val_img_name == image_name and val_img_stat == 'active' \ - and val_img_pub is True and val_img_cfmt == container_format \ - and val_img_dfmt == disk_format: - self.log.debug(msg_attr) - else: - msg = ('Image validation failed, {}'.format(msg_attr)) - amulet.raise_status(amulet.FAIL, msg=msg) - - return image - - def create_cirros_image(self, glance, image_name, hypervisor_type=None): - """Download the latest cirros image and upload it to glance, - validate and return a resource pointer. - - :param glance: pointer to authenticated glance connection - :param image_name: display name for new image - :param hypervisor_type: glance image hypervisor property - :returns: glance image pointer - """ - # /!\ DEPRECATION WARNING - self.log.warn('/!\\ DEPRECATION WARNING: use ' - 'glance_create_image instead of ' - 'create_cirros_image.') - - self.log.debug('Creating glance cirros image ' - '({})...'.format(image_name)) - - # Get cirros image URL - http_proxy = os.getenv('AMULET_HTTP_PROXY') - self.log.debug('AMULET_HTTP_PROXY: {}'.format(http_proxy)) - if http_proxy: - proxies = {'http': http_proxy} - opener = urllib.FancyURLopener(proxies) - else: - opener = urllib.FancyURLopener() - - f = opener.open('http://download.cirros-cloud.net/version/released') - version = f.read().strip() - cirros_img = 'cirros-{}-x86_64-disk.img'.format(version) - cirros_url = 'http://{}/{}/{}'.format('download.cirros-cloud.net', - version, cirros_img) - f.close() - - return self.glance_create_image( - glance, - image_name, - cirros_url, - hypervisor_type=hypervisor_type) - - def delete_image(self, glance, image): - """Delete the specified image.""" - - # /!\ DEPRECATION WARNING - self.log.warn('/!\\ DEPRECATION WARNING: use ' - 'delete_resource instead of delete_image.') - self.log.debug('Deleting glance image ({})...'.format(image)) - return self.delete_resource(glance.images, image, msg='glance image') - - def create_instance(self, nova, image_name, instance_name, flavor): - """Create the specified instance.""" - self.log.debug('Creating instance ' - '({}|{}|{})'.format(instance_name, image_name, flavor)) - image = nova.glance.find_image(image_name) - flavor = nova.flavors.find(name=flavor) - instance = nova.servers.create(name=instance_name, image=image, - flavor=flavor) - - count = 1 - status = instance.status - while status != 'ACTIVE' and count < 60: - time.sleep(3) - instance = nova.servers.get(instance.id) - status = instance.status - self.log.debug('instance status: {}'.format(status)) - count += 1 - - if status != 'ACTIVE': - self.log.error('instance creation timed out') - return None - - return instance - - def delete_instance(self, nova, instance): - """Delete the specified instance.""" - - # /!\ DEPRECATION WARNING - self.log.warn('/!\\ DEPRECATION WARNING: use ' - 'delete_resource instead of delete_instance.') - self.log.debug('Deleting instance ({})...'.format(instance)) - return self.delete_resource(nova.servers, instance, - msg='nova instance') - - def create_or_get_keypair(self, nova, keypair_name="testkey"): - """Create a new keypair, or return pointer if it already exists.""" - try: - _keypair = nova.keypairs.get(keypair_name) - self.log.debug('Keypair ({}) already exists, ' - 'using it.'.format(keypair_name)) - return _keypair - except Exception: - self.log.debug('Keypair ({}) does not exist, ' - 'creating it.'.format(keypair_name)) - - _keypair = nova.keypairs.create(name=keypair_name) - return _keypair - - def _get_cinder_obj_name(self, cinder_object): - """Retrieve name of cinder object. - - :param cinder_object: cinder snapshot or volume object - :returns: str cinder object name - """ - # v1 objects store name in 'display_name' attr but v2+ use 'name' - try: - return cinder_object.display_name - except AttributeError: - return cinder_object.name - - def create_cinder_volume(self, cinder, vol_name="demo-vol", vol_size=1, - img_id=None, src_vol_id=None, snap_id=None): - """Create cinder volume, optionally from a glance image, OR - optionally as a clone of an existing volume, OR optionally - from a snapshot. Wait for the new volume status to reach - the expected status, validate and return a resource pointer. - - :param vol_name: cinder volume display name - :param vol_size: size in gigabytes - :param img_id: optional glance image id - :param src_vol_id: optional source volume id to clone - :param snap_id: optional snapshot id to use - :returns: cinder volume pointer - """ - # Handle parameter input and avoid impossible combinations - if img_id and not src_vol_id and not snap_id: - # Create volume from image - self.log.debug('Creating cinder volume from glance image...') - bootable = 'true' - elif src_vol_id and not img_id and not snap_id: - # Clone an existing volume - self.log.debug('Cloning cinder volume...') - bootable = cinder.volumes.get(src_vol_id).bootable - elif snap_id and not src_vol_id and not img_id: - # Create volume from snapshot - self.log.debug('Creating cinder volume from snapshot...') - snap = cinder.volume_snapshots.find(id=snap_id) - vol_size = snap.size - snap_vol_id = cinder.volume_snapshots.get(snap_id).volume_id - bootable = cinder.volumes.get(snap_vol_id).bootable - elif not img_id and not src_vol_id and not snap_id: - # Create volume - self.log.debug('Creating cinder volume...') - bootable = 'false' - else: - # Impossible combination of parameters - msg = ('Invalid method use - name:{} size:{} img_id:{} ' - 'src_vol_id:{} snap_id:{}'.format(vol_name, vol_size, - img_id, src_vol_id, - snap_id)) - amulet.raise_status(amulet.FAIL, msg=msg) - - # Create new volume - try: - vol_new = cinder.volumes.create(display_name=vol_name, - imageRef=img_id, - size=vol_size, - source_volid=src_vol_id, - snapshot_id=snap_id) - vol_id = vol_new.id - except TypeError: - vol_new = cinder.volumes.create(name=vol_name, - imageRef=img_id, - size=vol_size, - source_volid=src_vol_id, - snapshot_id=snap_id) - vol_id = vol_new.id - except Exception as e: - msg = 'Failed to create volume: {}'.format(e) - amulet.raise_status(amulet.FAIL, msg=msg) - - # Wait for volume to reach available status - ret = self.resource_reaches_status(cinder.volumes, vol_id, - expected_stat="available", - msg="Volume status wait") - if not ret: - msg = 'Cinder volume failed to reach expected state.' - amulet.raise_status(amulet.FAIL, msg=msg) - - # Re-validate new volume - self.log.debug('Validating volume attributes...') - val_vol_name = self._get_cinder_obj_name(cinder.volumes.get(vol_id)) - val_vol_boot = cinder.volumes.get(vol_id).bootable - val_vol_stat = cinder.volumes.get(vol_id).status - val_vol_size = cinder.volumes.get(vol_id).size - msg_attr = ('Volume attributes - name:{} id:{} stat:{} boot:' - '{} size:{}'.format(val_vol_name, vol_id, - val_vol_stat, val_vol_boot, - val_vol_size)) - - if val_vol_boot == bootable and val_vol_stat == 'available' \ - and val_vol_name == vol_name and val_vol_size == vol_size: - self.log.debug(msg_attr) - else: - msg = ('Volume validation failed, {}'.format(msg_attr)) - amulet.raise_status(amulet.FAIL, msg=msg) - - return vol_new - - def delete_resource(self, resource, resource_id, - msg="resource", max_wait=120): - """Delete one openstack resource, such as one instance, keypair, - image, volume, stack, etc., and confirm deletion within max wait time. - - :param resource: pointer to os resource type, ex:glance_client.images - :param resource_id: unique name or id for the openstack resource - :param msg: text to identify purpose in logging - :param max_wait: maximum wait time in seconds - :returns: True if successful, otherwise False - """ - self.log.debug('Deleting OpenStack resource ' - '{} ({})'.format(resource_id, msg)) - num_before = len(list(resource.list())) - resource.delete(resource_id) - - tries = 0 - num_after = len(list(resource.list())) - while num_after != (num_before - 1) and tries < (max_wait / 4): - self.log.debug('{} delete check: ' - '{} [{}:{}] {}'.format(msg, tries, - num_before, - num_after, - resource_id)) - time.sleep(4) - num_after = len(list(resource.list())) - tries += 1 - - self.log.debug('{}: expected, actual count = {}, ' - '{}'.format(msg, num_before - 1, num_after)) - - if num_after == (num_before - 1): - return True - else: - self.log.error('{} delete timed out'.format(msg)) - return False - - def resource_reaches_status(self, resource, resource_id, - expected_stat='available', - msg='resource', max_wait=120): - """Wait for an openstack resources status to reach an - expected status within a specified time. Useful to confirm that - nova instances, cinder vols, snapshots, glance images, heat stacks - and other resources eventually reach the expected status. - - :param resource: pointer to os resource type, ex: heat_client.stacks - :param resource_id: unique id for the openstack resource - :param expected_stat: status to expect resource to reach - :param msg: text to identify purpose in logging - :param max_wait: maximum wait time in seconds - :returns: True if successful, False if status is not reached - """ - - tries = 0 - resource_stat = resource.get(resource_id).status - while resource_stat != expected_stat and tries < (max_wait / 4): - self.log.debug('{} status check: ' - '{} [{}:{}] {}'.format(msg, tries, - resource_stat, - expected_stat, - resource_id)) - time.sleep(4) - resource_stat = resource.get(resource_id).status - tries += 1 - - self.log.debug('{}: expected, actual status = {}, ' - '{}'.format(msg, resource_stat, expected_stat)) - - if resource_stat == expected_stat: - return True - else: - self.log.debug('{} never reached expected status: ' - '{}'.format(resource_id, expected_stat)) - return False - - def get_ceph_osd_id_cmd(self, index): - """Produce a shell command that will return a ceph-osd id.""" - return ("`initctl list | grep 'ceph-osd ' | " - "awk 'NR=={} {{ print $2 }}' | " - "grep -o '[0-9]*'`".format(index + 1)) - - def get_ceph_pools(self, sentry_unit): - """Return a dict of ceph pools from a single ceph unit, with - pool name as keys, pool id as vals.""" - pools = {} - cmd = 'sudo ceph osd lspools' - output, code = sentry_unit.run(cmd) - if code != 0: - msg = ('{} `{}` returned {} ' - '{}'.format(sentry_unit.info['unit_name'], - cmd, code, output)) - amulet.raise_status(amulet.FAIL, msg=msg) - - # For mimic ceph osd lspools output - output = output.replace("\n", ",") - - # Example output: 0 data,1 metadata,2 rbd,3 cinder,4 glance, - for pool in str(output).split(','): - pool_id_name = pool.split(' ') - if len(pool_id_name) == 2: - pool_id = pool_id_name[0] - pool_name = pool_id_name[1] - pools[pool_name] = int(pool_id) - - self.log.debug('Pools on {}: {}'.format(sentry_unit.info['unit_name'], - pools)) - return pools - - def get_ceph_df(self, sentry_unit): - """Return dict of ceph df json output, including ceph pool state. - - :param sentry_unit: Pointer to amulet sentry instance (juju unit) - :returns: Dict of ceph df output - """ - cmd = 'sudo ceph df --format=json' - output, code = sentry_unit.run(cmd) - if code != 0: - msg = ('{} `{}` returned {} ' - '{}'.format(sentry_unit.info['unit_name'], - cmd, code, output)) - amulet.raise_status(amulet.FAIL, msg=msg) - return json.loads(output) - - def get_ceph_pool_sample(self, sentry_unit, pool_id=0): - """Take a sample of attributes of a ceph pool, returning ceph - pool name, object count and disk space used for the specified - pool ID number. - - :param sentry_unit: Pointer to amulet sentry instance (juju unit) - :param pool_id: Ceph pool ID - :returns: List of pool name, object count, kb disk space used - """ - df = self.get_ceph_df(sentry_unit) - for pool in df['pools']: - if pool['id'] == pool_id: - pool_name = pool['name'] - obj_count = pool['stats']['objects'] - kb_used = pool['stats']['kb_used'] - - self.log.debug('Ceph {} pool (ID {}): {} objects, ' - '{} kb used'.format(pool_name, pool_id, - obj_count, kb_used)) - return pool_name, obj_count, kb_used - - def validate_ceph_pool_samples(self, samples, sample_type="resource pool"): - """Validate ceph pool samples taken over time, such as pool - object counts or pool kb used, before adding, after adding, and - after deleting items which affect those pool attributes. The - 2nd element is expected to be greater than the 1st; 3rd is expected - to be less than the 2nd. - - :param samples: List containing 3 data samples - :param sample_type: String for logging and usage context - :returns: None if successful, Failure message otherwise - """ - original, created, deleted = range(3) - if samples[created] <= samples[original] or \ - samples[deleted] >= samples[created]: - return ('Ceph {} samples ({}) ' - 'unexpected.'.format(sample_type, samples)) - else: - self.log.debug('Ceph {} samples (OK): ' - '{}'.format(sample_type, samples)) - return None - - # rabbitmq/amqp specific helpers: - - def rmq_wait_for_cluster(self, deployment, init_sleep=15, timeout=1200): - """Wait for rmq units extended status to show cluster readiness, - after an optional initial sleep period. Initial sleep is likely - necessary to be effective following a config change, as status - message may not instantly update to non-ready.""" - - if init_sleep: - time.sleep(init_sleep) - - message = re.compile('^Unit is ready and clustered$') - deployment._auto_wait_for_status(message=message, - timeout=timeout, - include_only=['rabbitmq-server']) - - def add_rmq_test_user(self, sentry_units, - username="testuser1", password="changeme"): - """Add a test user via the first rmq juju unit, check connection as - the new user against all sentry units. - - :param sentry_units: list of sentry unit pointers - :param username: amqp user name, default to testuser1 - :param password: amqp user password - :returns: None if successful. Raise on error. - """ - self.log.debug('Adding rmq user ({})...'.format(username)) - - # Check that user does not already exist - cmd_user_list = 'rabbitmqctl list_users' - output, _ = self.run_cmd_unit(sentry_units[0], cmd_user_list) - if username in output: - self.log.warning('User ({}) already exists, returning ' - 'gracefully.'.format(username)) - return - - perms = '".*" ".*" ".*"' - cmds = ['rabbitmqctl add_user {} {}'.format(username, password), - 'rabbitmqctl set_permissions {} {}'.format(username, perms)] - - # Add user via first unit - for cmd in cmds: - output, _ = self.run_cmd_unit(sentry_units[0], cmd) - - # Check connection against the other sentry_units - self.log.debug('Checking user connect against units...') - for sentry_unit in sentry_units: - connection = self.connect_amqp_by_unit(sentry_unit, ssl=False, - username=username, - password=password) - connection.close() - - def delete_rmq_test_user(self, sentry_units, username="testuser1"): - """Delete a rabbitmq user via the first rmq juju unit. - - :param sentry_units: list of sentry unit pointers - :param username: amqp user name, default to testuser1 - :param password: amqp user password - :returns: None if successful or no such user. - """ - self.log.debug('Deleting rmq user ({})...'.format(username)) - - # Check that the user exists - cmd_user_list = 'rabbitmqctl list_users' - output, _ = self.run_cmd_unit(sentry_units[0], cmd_user_list) - - if username not in output: - self.log.warning('User ({}) does not exist, returning ' - 'gracefully.'.format(username)) - return - - # Delete the user - cmd_user_del = 'rabbitmqctl delete_user {}'.format(username) - output, _ = self.run_cmd_unit(sentry_units[0], cmd_user_del) - - def get_rmq_cluster_status(self, sentry_unit): - """Execute rabbitmq cluster status command on a unit and return - the full output. - - :param unit: sentry unit - :returns: String containing console output of cluster status command - """ - cmd = 'rabbitmqctl cluster_status' - output, _ = self.run_cmd_unit(sentry_unit, cmd) - self.log.debug('{} cluster_status:\n{}'.format( - sentry_unit.info['unit_name'], output)) - return str(output) - - def get_rmq_cluster_running_nodes(self, sentry_unit): - """Parse rabbitmqctl cluster_status output string, return list of - running rabbitmq cluster nodes. - - :param unit: sentry unit - :returns: List containing node names of running nodes - """ - # NOTE(beisner): rabbitmqctl cluster_status output is not - # json-parsable, do string chop foo, then json.loads that. - str_stat = self.get_rmq_cluster_status(sentry_unit) - if 'running_nodes' in str_stat: - pos_start = str_stat.find("{running_nodes,") + 15 - pos_end = str_stat.find("]},", pos_start) + 1 - str_run_nodes = str_stat[pos_start:pos_end].replace("'", '"') - run_nodes = json.loads(str_run_nodes) - return run_nodes - else: - return [] - - def validate_rmq_cluster_running_nodes(self, sentry_units): - """Check that all rmq unit hostnames are represented in the - cluster_status output of all units. - - :param host_names: dict of juju unit names to host names - :param units: list of sentry unit pointers (all rmq units) - :returns: None if successful, otherwise return error message - """ - host_names = self.get_unit_hostnames(sentry_units) - errors = [] - - # Query every unit for cluster_status running nodes - for query_unit in sentry_units: - query_unit_name = query_unit.info['unit_name'] - running_nodes = self.get_rmq_cluster_running_nodes(query_unit) - - # Confirm that every unit is represented in the queried unit's - # cluster_status running nodes output. - for validate_unit in sentry_units: - val_host_name = host_names[validate_unit.info['unit_name']] - val_node_name = 'rabbit@{}'.format(val_host_name) - - if val_node_name not in running_nodes: - errors.append('Cluster member check failed on {}: {} not ' - 'in {}\n'.format(query_unit_name, - val_node_name, - running_nodes)) - if errors: - return ''.join(errors) - - def rmq_ssl_is_enabled_on_unit(self, sentry_unit, port=None): - """Check a single juju rmq unit for ssl and port in the config file.""" - host = sentry_unit.info['public-address'] - unit_name = sentry_unit.info['unit_name'] - - conf_file = '/etc/rabbitmq/rabbitmq.config' - conf_contents = str(self.file_contents_safe(sentry_unit, - conf_file, max_wait=16)) - # Checks - conf_ssl = 'ssl' in conf_contents - conf_port = str(port) in conf_contents - - # Port explicitly checked in config - if port and conf_port and conf_ssl: - self.log.debug('SSL is enabled @{}:{} ' - '({})'.format(host, port, unit_name)) - return True - elif port and not conf_port and conf_ssl: - self.log.debug('SSL is enabled @{} but not on port {} ' - '({})'.format(host, port, unit_name)) - return False - # Port not checked (useful when checking that ssl is disabled) - elif not port and conf_ssl: - self.log.debug('SSL is enabled @{}:{} ' - '({})'.format(host, port, unit_name)) - return True - elif not conf_ssl: - self.log.debug('SSL not enabled @{}:{} ' - '({})'.format(host, port, unit_name)) - return False - else: - msg = ('Unknown condition when checking SSL status @{}:{} ' - '({})'.format(host, port, unit_name)) - amulet.raise_status(amulet.FAIL, msg) - - def validate_rmq_ssl_enabled_units(self, sentry_units, port=None): - """Check that ssl is enabled on rmq juju sentry units. - - :param sentry_units: list of all rmq sentry units - :param port: optional ssl port override to validate - :returns: None if successful, otherwise return error message - """ - for sentry_unit in sentry_units: - if not self.rmq_ssl_is_enabled_on_unit(sentry_unit, port=port): - return ('Unexpected condition: ssl is disabled on unit ' - '({})'.format(sentry_unit.info['unit_name'])) - return None - - def validate_rmq_ssl_disabled_units(self, sentry_units): - """Check that ssl is enabled on listed rmq juju sentry units. - - :param sentry_units: list of all rmq sentry units - :returns: True if successful. Raise on error. - """ - for sentry_unit in sentry_units: - if self.rmq_ssl_is_enabled_on_unit(sentry_unit): - return ('Unexpected condition: ssl is enabled on unit ' - '({})'.format(sentry_unit.info['unit_name'])) - return None - - def configure_rmq_ssl_on(self, sentry_units, deployment, - port=None, max_wait=60): - """Turn ssl charm config option on, with optional non-default - ssl port specification. Confirm that it is enabled on every - unit. - - :param sentry_units: list of sentry units - :param deployment: amulet deployment object pointer - :param port: amqp port, use defaults if None - :param max_wait: maximum time to wait in seconds to confirm - :returns: None if successful. Raise on error. - """ - self.log.debug('Setting ssl charm config option: on') - - # Enable RMQ SSL - config = {'ssl': 'on'} - if port: - config['ssl_port'] = port - - deployment.d.configure('rabbitmq-server', config) - - # Wait for unit status - self.rmq_wait_for_cluster(deployment) - - # Confirm - tries = 0 - ret = self.validate_rmq_ssl_enabled_units(sentry_units, port=port) - while ret and tries < (max_wait / 4): - time.sleep(4) - self.log.debug('Attempt {}: {}'.format(tries, ret)) - ret = self.validate_rmq_ssl_enabled_units(sentry_units, port=port) - tries += 1 - - if ret: - amulet.raise_status(amulet.FAIL, ret) - - def configure_rmq_ssl_off(self, sentry_units, deployment, max_wait=60): - """Turn ssl charm config option off, confirm that it is disabled - on every unit. - - :param sentry_units: list of sentry units - :param deployment: amulet deployment object pointer - :param max_wait: maximum time to wait in seconds to confirm - :returns: None if successful. Raise on error. - """ - self.log.debug('Setting ssl charm config option: off') - - # Disable RMQ SSL - config = {'ssl': 'off'} - deployment.d.configure('rabbitmq-server', config) - - # Wait for unit status - self.rmq_wait_for_cluster(deployment) - - # Confirm - tries = 0 - ret = self.validate_rmq_ssl_disabled_units(sentry_units) - while ret and tries < (max_wait / 4): - time.sleep(4) - self.log.debug('Attempt {}: {}'.format(tries, ret)) - ret = self.validate_rmq_ssl_disabled_units(sentry_units) - tries += 1 - - if ret: - amulet.raise_status(amulet.FAIL, ret) - - def connect_amqp_by_unit(self, sentry_unit, ssl=False, - port=None, fatal=True, - username="testuser1", password="changeme"): - """Establish and return a pika amqp connection to the rabbitmq service - running on a rmq juju unit. - - :param sentry_unit: sentry unit pointer - :param ssl: boolean, default to False - :param port: amqp port, use defaults if None - :param fatal: boolean, default to True (raises on connect error) - :param username: amqp user name, default to testuser1 - :param password: amqp user password - :returns: pika amqp connection pointer or None if failed and non-fatal - """ - host = sentry_unit.info['public-address'] - unit_name = sentry_unit.info['unit_name'] - - # Default port logic if port is not specified - if ssl and not port: - port = 5671 - elif not ssl and not port: - port = 5672 - - self.log.debug('Connecting to amqp on {}:{} ({}) as ' - '{}...'.format(host, port, unit_name, username)) - - try: - credentials = pika.PlainCredentials(username, password) - parameters = pika.ConnectionParameters(host=host, port=port, - credentials=credentials, - ssl=ssl, - connection_attempts=3, - retry_delay=5, - socket_timeout=1) - connection = pika.BlockingConnection(parameters) - assert connection.is_open is True - assert connection.is_closing is False - self.log.debug('Connect OK') - return connection - except Exception as e: - msg = ('amqp connection failed to {}:{} as ' - '{} ({})'.format(host, port, username, str(e))) - if fatal: - amulet.raise_status(amulet.FAIL, msg) - else: - self.log.warn(msg) - return None - - def publish_amqp_message_by_unit(self, sentry_unit, message, - queue="test", ssl=False, - username="testuser1", - password="changeme", - port=None): - """Publish an amqp message to a rmq juju unit. - - :param sentry_unit: sentry unit pointer - :param message: amqp message string - :param queue: message queue, default to test - :param username: amqp user name, default to testuser1 - :param password: amqp user password - :param ssl: boolean, default to False - :param port: amqp port, use defaults if None - :returns: None. Raises exception if publish failed. - """ - self.log.debug('Publishing message to {} queue:\n{}'.format(queue, - message)) - connection = self.connect_amqp_by_unit(sentry_unit, ssl=ssl, - port=port, - username=username, - password=password) - - # NOTE(beisner): extra debug here re: pika hang potential: - # https://github.com/pika/pika/issues/297 - # https://groups.google.com/forum/#!topic/rabbitmq-users/Ja0iyfF0Szw - self.log.debug('Defining channel...') - channel = connection.channel() - self.log.debug('Declaring queue...') - channel.queue_declare(queue=queue, auto_delete=False, durable=True) - self.log.debug('Publishing message...') - channel.basic_publish(exchange='', routing_key=queue, body=message) - self.log.debug('Closing channel...') - channel.close() - self.log.debug('Closing connection...') - connection.close() - - def get_amqp_message_by_unit(self, sentry_unit, queue="test", - username="testuser1", - password="changeme", - ssl=False, port=None): - """Get an amqp message from a rmq juju unit. - - :param sentry_unit: sentry unit pointer - :param queue: message queue, default to test - :param username: amqp user name, default to testuser1 - :param password: amqp user password - :param ssl: boolean, default to False - :param port: amqp port, use defaults if None - :returns: amqp message body as string. Raise if get fails. - """ - connection = self.connect_amqp_by_unit(sentry_unit, ssl=ssl, - port=port, - username=username, - password=password) - channel = connection.channel() - method_frame, _, body = channel.basic_get(queue) - - if method_frame: - self.log.debug('Retreived message from {} queue:\n{}'.format(queue, - body)) - channel.basic_ack(method_frame.delivery_tag) - channel.close() - connection.close() - return body - else: - msg = 'No message retrieved.' - amulet.raise_status(amulet.FAIL, msg) - - def validate_memcache(self, sentry_unit, conf, os_release, - earliest_release=5, section='keystone_authtoken', - check_kvs=None): - """Check Memcache is running and is configured to be used - - Example call from Amulet test: - - def test_110_memcache(self): - u.validate_memcache(self.neutron_api_sentry, - '/etc/neutron/neutron.conf', - self._get_openstack_release()) - - :param sentry_unit: sentry unit - :param conf: OpenStack config file to check memcache settings - :param os_release: Current OpenStack release int code - :param earliest_release: Earliest Openstack release to check int code - :param section: OpenStack config file section to check - :param check_kvs: Dict of settings to check in config file - :returns: None - """ - if os_release < earliest_release: - self.log.debug('Skipping memcache checks for deployment. {} <' - 'mitaka'.format(os_release)) - return - _kvs = check_kvs or {'memcached_servers': 'inet6:[::1]:11211'} - self.log.debug('Checking memcached is running') - ret = self.validate_services_by_name({sentry_unit: ['memcached']}) - if ret: - amulet.raise_status(amulet.FAIL, msg='Memcache running check' - 'failed {}'.format(ret)) - else: - self.log.debug('OK') - self.log.debug('Checking memcache url is configured in {}'.format( - conf)) - if self.validate_config_data(sentry_unit, conf, section, _kvs): - message = "Memcache config error in: {}".format(conf) - amulet.raise_status(amulet.FAIL, msg=message) - else: - self.log.debug('OK') - self.log.debug('Checking memcache configuration in ' - '/etc/memcached.conf') - contents = self.file_contents_safe(sentry_unit, '/etc/memcached.conf', - fatal=True) - ubuntu_release, _ = self.run_cmd_unit(sentry_unit, 'lsb_release -cs') - if CompareHostReleases(ubuntu_release) <= 'trusty': - memcache_listen_addr = 'ip6-localhost' - else: - memcache_listen_addr = '::1' - expected = { - '-p': '11211', - '-l': memcache_listen_addr} - found = [] - for key, value in expected.items(): - for line in contents.split('\n'): - if line.startswith(key): - self.log.debug('Checking {} is set to {}'.format( - key, - value)) - assert value == line.split()[-1] - self.log.debug(line.split()[-1]) - found.append(key) - if sorted(found) == sorted(expected.keys()): - self.log.debug('OK') - else: - message = "Memcache config error in: /etc/memcached.conf" - amulet.raise_status(amulet.FAIL, msg=message) diff --git a/tests/charmhelpers/core/host.py b/tests/charmhelpers/core/host.py deleted file mode 100644 index 0ebfdbd1..00000000 --- a/tests/charmhelpers/core/host.py +++ /dev/null @@ -1,1066 +0,0 @@ -# Copyright 2014-2015 Canonical Limited. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -"""Tools for working with the host system""" -# Copyright 2012 Canonical Ltd. -# -# Authors: -# Nick Moffitt -# Matthew Wedgwood - -import os -import re -import pwd -import glob -import grp -import random -import string -import subprocess -import hashlib -import functools -import itertools -import six - -from contextlib import contextmanager -from collections import OrderedDict -from .hookenv import log, INFO, DEBUG, local_unit, charm_name -from .fstab import Fstab -from charmhelpers.osplatform import get_platform - -__platform__ = get_platform() -if __platform__ == "ubuntu": - from charmhelpers.core.host_factory.ubuntu import ( - service_available, - add_new_group, - lsb_release, - cmp_pkgrevno, - CompareHostReleases, - ) # flake8: noqa -- ignore F401 for this import -elif __platform__ == "centos": - from charmhelpers.core.host_factory.centos import ( - service_available, - add_new_group, - lsb_release, - cmp_pkgrevno, - CompareHostReleases, - ) # flake8: noqa -- ignore F401 for this import - -UPDATEDB_PATH = '/etc/updatedb.conf' - -def service_start(service_name, **kwargs): - """Start a system service. - - The specified service name is managed via the system level init system. - Some init systems (e.g. upstart) require that additional arguments be - provided in order to directly control service instances whereas other init - systems allow for addressing instances of a service directly by name (e.g. - systemd). - - The kwargs allow for the additional parameters to be passed to underlying - init systems for those systems which require/allow for them. For example, - the ceph-osd upstart script requires the id parameter to be passed along - in order to identify which running daemon should be reloaded. The follow- - ing example stops the ceph-osd service for instance id=4: - - service_stop('ceph-osd', id=4) - - :param service_name: the name of the service to stop - :param **kwargs: additional parameters to pass to the init system when - managing services. These will be passed as key=value - parameters to the init system's commandline. kwargs - are ignored for systemd enabled systems. - """ - return service('start', service_name, **kwargs) - - -def service_stop(service_name, **kwargs): - """Stop a system service. - - The specified service name is managed via the system level init system. - Some init systems (e.g. upstart) require that additional arguments be - provided in order to directly control service instances whereas other init - systems allow for addressing instances of a service directly by name (e.g. - systemd). - - The kwargs allow for the additional parameters to be passed to underlying - init systems for those systems which require/allow for them. For example, - the ceph-osd upstart script requires the id parameter to be passed along - in order to identify which running daemon should be reloaded. The follow- - ing example stops the ceph-osd service for instance id=4: - - service_stop('ceph-osd', id=4) - - :param service_name: the name of the service to stop - :param **kwargs: additional parameters to pass to the init system when - managing services. These will be passed as key=value - parameters to the init system's commandline. kwargs - are ignored for systemd enabled systems. - """ - return service('stop', service_name, **kwargs) - - -def service_restart(service_name, **kwargs): - """Restart a system service. - - The specified service name is managed via the system level init system. - Some init systems (e.g. upstart) require that additional arguments be - provided in order to directly control service instances whereas other init - systems allow for addressing instances of a service directly by name (e.g. - systemd). - - The kwargs allow for the additional parameters to be passed to underlying - init systems for those systems which require/allow for them. For example, - the ceph-osd upstart script requires the id parameter to be passed along - in order to identify which running daemon should be restarted. The follow- - ing example restarts the ceph-osd service for instance id=4: - - service_restart('ceph-osd', id=4) - - :param service_name: the name of the service to restart - :param **kwargs: additional parameters to pass to the init system when - managing services. These will be passed as key=value - parameters to the init system's commandline. kwargs - are ignored for init systems not allowing additional - parameters via the commandline (systemd). - """ - return service('restart', service_name) - - -def service_reload(service_name, restart_on_failure=False, **kwargs): - """Reload a system service, optionally falling back to restart if - reload fails. - - The specified service name is managed via the system level init system. - Some init systems (e.g. upstart) require that additional arguments be - provided in order to directly control service instances whereas other init - systems allow for addressing instances of a service directly by name (e.g. - systemd). - - The kwargs allow for the additional parameters to be passed to underlying - init systems for those systems which require/allow for them. For example, - the ceph-osd upstart script requires the id parameter to be passed along - in order to identify which running daemon should be reloaded. The follow- - ing example restarts the ceph-osd service for instance id=4: - - service_reload('ceph-osd', id=4) - - :param service_name: the name of the service to reload - :param restart_on_failure: boolean indicating whether to fallback to a - restart if the reload fails. - :param **kwargs: additional parameters to pass to the init system when - managing services. These will be passed as key=value - parameters to the init system's commandline. kwargs - are ignored for init systems not allowing additional - parameters via the commandline (systemd). - """ - service_result = service('reload', service_name, **kwargs) - if not service_result and restart_on_failure: - service_result = service('restart', service_name, **kwargs) - return service_result - - -def service_pause(service_name, init_dir="/etc/init", initd_dir="/etc/init.d", - **kwargs): - """Pause a system service. - - Stop it, and prevent it from starting again at boot. - - :param service_name: the name of the service to pause - :param init_dir: path to the upstart init directory - :param initd_dir: path to the sysv init directory - :param **kwargs: additional parameters to pass to the init system when - managing services. These will be passed as key=value - parameters to the init system's commandline. kwargs - are ignored for init systems which do not support - key=value arguments via the commandline. - """ - stopped = True - if service_running(service_name, **kwargs): - stopped = service_stop(service_name, **kwargs) - upstart_file = os.path.join(init_dir, "{}.conf".format(service_name)) - sysv_file = os.path.join(initd_dir, service_name) - if init_is_systemd(): - service('disable', service_name) - service('mask', service_name) - elif os.path.exists(upstart_file): - override_path = os.path.join( - init_dir, '{}.override'.format(service_name)) - with open(override_path, 'w') as fh: - fh.write("manual\n") - elif os.path.exists(sysv_file): - subprocess.check_call(["update-rc.d", service_name, "disable"]) - else: - raise ValueError( - "Unable to detect {0} as SystemD, Upstart {1} or" - " SysV {2}".format( - service_name, upstart_file, sysv_file)) - return stopped - - -def service_resume(service_name, init_dir="/etc/init", - initd_dir="/etc/init.d", **kwargs): - """Resume a system service. - - Reenable starting again at boot. Start the service. - - :param service_name: the name of the service to resume - :param init_dir: the path to the init dir - :param initd dir: the path to the initd dir - :param **kwargs: additional parameters to pass to the init system when - managing services. These will be passed as key=value - parameters to the init system's commandline. kwargs - are ignored for systemd enabled systems. - """ - upstart_file = os.path.join(init_dir, "{}.conf".format(service_name)) - sysv_file = os.path.join(initd_dir, service_name) - if init_is_systemd(): - service('unmask', service_name) - service('enable', service_name) - elif os.path.exists(upstart_file): - override_path = os.path.join( - init_dir, '{}.override'.format(service_name)) - if os.path.exists(override_path): - os.unlink(override_path) - elif os.path.exists(sysv_file): - subprocess.check_call(["update-rc.d", service_name, "enable"]) - else: - raise ValueError( - "Unable to detect {0} as SystemD, Upstart {1} or" - " SysV {2}".format( - service_name, upstart_file, sysv_file)) - started = service_running(service_name, **kwargs) - - if not started: - started = service_start(service_name, **kwargs) - return started - - -def service(action, service_name, **kwargs): - """Control a system service. - - :param action: the action to take on the service - :param service_name: the name of the service to perform th action on - :param **kwargs: additional params to be passed to the service command in - the form of key=value. - """ - if init_is_systemd(): - cmd = ['systemctl', action, service_name] - else: - cmd = ['service', service_name, action] - for key, value in six.iteritems(kwargs): - parameter = '%s=%s' % (key, value) - cmd.append(parameter) - return subprocess.call(cmd) == 0 - - -_UPSTART_CONF = "/etc/init/{}.conf" -_INIT_D_CONF = "/etc/init.d/{}" - - -def service_running(service_name, **kwargs): - """Determine whether a system service is running. - - :param service_name: the name of the service - :param **kwargs: additional args to pass to the service command. This is - used to pass additional key=value arguments to the - service command line for managing specific instance - units (e.g. service ceph-osd status id=2). The kwargs - are ignored in systemd services. - """ - if init_is_systemd(): - return service('is-active', service_name) - else: - if os.path.exists(_UPSTART_CONF.format(service_name)): - try: - cmd = ['status', service_name] - for key, value in six.iteritems(kwargs): - parameter = '%s=%s' % (key, value) - cmd.append(parameter) - output = subprocess.check_output(cmd, - stderr=subprocess.STDOUT).decode('UTF-8') - except subprocess.CalledProcessError: - return False - else: - # This works for upstart scripts where the 'service' command - # returns a consistent string to represent running - # 'start/running' - if ("start/running" in output or - "is running" in output or - "up and running" in output): - return True - elif os.path.exists(_INIT_D_CONF.format(service_name)): - # Check System V scripts init script return codes - return service('status', service_name) - return False - - -SYSTEMD_SYSTEM = '/run/systemd/system' - - -def init_is_systemd(): - """Return True if the host system uses systemd, False otherwise.""" - if lsb_release()['DISTRIB_CODENAME'] == 'trusty': - return False - return os.path.isdir(SYSTEMD_SYSTEM) - - -def adduser(username, password=None, shell='/bin/bash', - system_user=False, primary_group=None, - secondary_groups=None, uid=None, home_dir=None): - """Add a user to the system. - - Will log but otherwise succeed if the user already exists. - - :param str username: Username to create - :param str password: Password for user; if ``None``, create a system user - :param str shell: The default shell for the user - :param bool system_user: Whether to create a login or system user - :param str primary_group: Primary group for user; defaults to username - :param list secondary_groups: Optional list of additional groups - :param int uid: UID for user being created - :param str home_dir: Home directory for user - - :returns: The password database entry struct, as returned by `pwd.getpwnam` - """ - try: - user_info = pwd.getpwnam(username) - log('user {0} already exists!'.format(username)) - if uid: - user_info = pwd.getpwuid(int(uid)) - log('user with uid {0} already exists!'.format(uid)) - except KeyError: - log('creating user {0}'.format(username)) - cmd = ['useradd'] - if uid: - cmd.extend(['--uid', str(uid)]) - if home_dir: - cmd.extend(['--home', str(home_dir)]) - if system_user or password is None: - cmd.append('--system') - else: - cmd.extend([ - '--create-home', - '--shell', shell, - '--password', password, - ]) - if not primary_group: - try: - grp.getgrnam(username) - primary_group = username # avoid "group exists" error - except KeyError: - pass - if primary_group: - cmd.extend(['-g', primary_group]) - if secondary_groups: - cmd.extend(['-G', ','.join(secondary_groups)]) - cmd.append(username) - subprocess.check_call(cmd) - user_info = pwd.getpwnam(username) - return user_info - - -def user_exists(username): - """Check if a user exists""" - try: - pwd.getpwnam(username) - user_exists = True - except KeyError: - user_exists = False - return user_exists - - -def uid_exists(uid): - """Check if a uid exists""" - try: - pwd.getpwuid(uid) - uid_exists = True - except KeyError: - uid_exists = False - return uid_exists - - -def group_exists(groupname): - """Check if a group exists""" - try: - grp.getgrnam(groupname) - group_exists = True - except KeyError: - group_exists = False - return group_exists - - -def gid_exists(gid): - """Check if a gid exists""" - try: - grp.getgrgid(gid) - gid_exists = True - except KeyError: - gid_exists = False - return gid_exists - - -def add_group(group_name, system_group=False, gid=None): - """Add a group to the system - - Will log but otherwise succeed if the group already exists. - - :param str group_name: group to create - :param bool system_group: Create system group - :param int gid: GID for user being created - - :returns: The password database entry struct, as returned by `grp.getgrnam` - """ - try: - group_info = grp.getgrnam(group_name) - log('group {0} already exists!'.format(group_name)) - if gid: - group_info = grp.getgrgid(gid) - log('group with gid {0} already exists!'.format(gid)) - except KeyError: - log('creating group {0}'.format(group_name)) - add_new_group(group_name, system_group, gid) - group_info = grp.getgrnam(group_name) - return group_info - - -def add_user_to_group(username, group): - """Add a user to a group""" - cmd = ['gpasswd', '-a', username, group] - log("Adding user {} to group {}".format(username, group)) - subprocess.check_call(cmd) - - -def chage(username, lastday=None, expiredate=None, inactive=None, - mindays=None, maxdays=None, root=None, warndays=None): - """Change user password expiry information - - :param str username: User to update - :param str lastday: Set when password was changed in YYYY-MM-DD format - :param str expiredate: Set when user's account will no longer be - accessible in YYYY-MM-DD format. - -1 will remove an account expiration date. - :param str inactive: Set the number of days of inactivity after a password - has expired before the account is locked. - -1 will remove an account's inactivity. - :param str mindays: Set the minimum number of days between password - changes to MIN_DAYS. - 0 indicates the password can be changed anytime. - :param str maxdays: Set the maximum number of days during which a - password is valid. - -1 as MAX_DAYS will remove checking maxdays - :param str root: Apply changes in the CHROOT_DIR directory - :param str warndays: Set the number of days of warning before a password - change is required - :raises subprocess.CalledProcessError: if call to chage fails - """ - cmd = ['chage'] - if root: - cmd.extend(['--root', root]) - if lastday: - cmd.extend(['--lastday', lastday]) - if expiredate: - cmd.extend(['--expiredate', expiredate]) - if inactive: - cmd.extend(['--inactive', inactive]) - if mindays: - cmd.extend(['--mindays', mindays]) - if maxdays: - cmd.extend(['--maxdays', maxdays]) - if warndays: - cmd.extend(['--warndays', warndays]) - cmd.append(username) - subprocess.check_call(cmd) - -remove_password_expiry = functools.partial(chage, expiredate='-1', inactive='-1', mindays='0', maxdays='-1') - -def rsync(from_path, to_path, flags='-r', options=None, timeout=None): - """Replicate the contents of a path""" - options = options or ['--delete', '--executability'] - cmd = ['/usr/bin/rsync', flags] - if timeout: - cmd = ['timeout', str(timeout)] + cmd - cmd.extend(options) - cmd.append(from_path) - cmd.append(to_path) - log(" ".join(cmd)) - return subprocess.check_output(cmd, stderr=subprocess.STDOUT).decode('UTF-8').strip() - - -def symlink(source, destination): - """Create a symbolic link""" - log("Symlinking {} as {}".format(source, destination)) - cmd = [ - 'ln', - '-sf', - source, - destination, - ] - subprocess.check_call(cmd) - - -def mkdir(path, owner='root', group='root', perms=0o555, force=False): - """Create a directory""" - log("Making dir {} {}:{} {:o}".format(path, owner, group, - perms)) - uid = pwd.getpwnam(owner).pw_uid - gid = grp.getgrnam(group).gr_gid - realpath = os.path.abspath(path) - path_exists = os.path.exists(realpath) - if path_exists and force: - if not os.path.isdir(realpath): - log("Removing non-directory file {} prior to mkdir()".format(path)) - os.unlink(realpath) - os.makedirs(realpath, perms) - elif not path_exists: - os.makedirs(realpath, perms) - os.chown(realpath, uid, gid) - os.chmod(realpath, perms) - - -def write_file(path, content, owner='root', group='root', perms=0o444): - """Create or overwrite a file with the contents of a byte string.""" - uid = pwd.getpwnam(owner).pw_uid - gid = grp.getgrnam(group).gr_gid - # lets see if we can grab the file and compare the context, to avoid doing - # a write. - existing_content = None - existing_uid, existing_gid = None, None - try: - with open(path, 'rb') as target: - existing_content = target.read() - stat = os.stat(path) - existing_uid, existing_gid = stat.st_uid, stat.st_gid - except: - pass - if content != existing_content: - log("Writing file {} {}:{} {:o}".format(path, owner, group, perms), - level=DEBUG) - with open(path, 'wb') as target: - os.fchown(target.fileno(), uid, gid) - os.fchmod(target.fileno(), perms) - if six.PY3 and isinstance(content, six.string_types): - content = content.encode('UTF-8') - target.write(content) - return - # the contents were the same, but we might still need to change the - # ownership. - if existing_uid != uid: - log("Changing uid on already existing content: {} -> {}" - .format(existing_uid, uid), level=DEBUG) - os.chown(path, uid, -1) - if existing_gid != gid: - log("Changing gid on already existing content: {} -> {}" - .format(existing_gid, gid), level=DEBUG) - os.chown(path, -1, gid) - - -def fstab_remove(mp): - """Remove the given mountpoint entry from /etc/fstab""" - return Fstab.remove_by_mountpoint(mp) - - -def fstab_add(dev, mp, fs, options=None): - """Adds the given device entry to the /etc/fstab file""" - return Fstab.add(dev, mp, fs, options=options) - - -def mount(device, mountpoint, options=None, persist=False, filesystem="ext3"): - """Mount a filesystem at a particular mountpoint""" - cmd_args = ['mount'] - if options is not None: - cmd_args.extend(['-o', options]) - cmd_args.extend([device, mountpoint]) - try: - subprocess.check_output(cmd_args) - except subprocess.CalledProcessError as e: - log('Error mounting {} at {}\n{}'.format(device, mountpoint, e.output)) - return False - - if persist: - return fstab_add(device, mountpoint, filesystem, options=options) - return True - - -def umount(mountpoint, persist=False): - """Unmount a filesystem""" - cmd_args = ['umount', mountpoint] - try: - subprocess.check_output(cmd_args) - except subprocess.CalledProcessError as e: - log('Error unmounting {}\n{}'.format(mountpoint, e.output)) - return False - - if persist: - return fstab_remove(mountpoint) - return True - - -def mounts(): - """Get a list of all mounted volumes as [[mountpoint,device],[...]]""" - with open('/proc/mounts') as f: - # [['/mount/point','/dev/path'],[...]] - system_mounts = [m[1::-1] for m in [l.strip().split() - for l in f.readlines()]] - return system_mounts - - -def fstab_mount(mountpoint): - """Mount filesystem using fstab""" - cmd_args = ['mount', mountpoint] - try: - subprocess.check_output(cmd_args) - except subprocess.CalledProcessError as e: - log('Error unmounting {}\n{}'.format(mountpoint, e.output)) - return False - return True - - -def file_hash(path, hash_type='md5'): - """Generate a hash checksum of the contents of 'path' or None if not found. - - :param str hash_type: Any hash alrgorithm supported by :mod:`hashlib`, - such as md5, sha1, sha256, sha512, etc. - """ - if os.path.exists(path): - h = getattr(hashlib, hash_type)() - with open(path, 'rb') as source: - h.update(source.read()) - return h.hexdigest() - else: - return None - - -def path_hash(path): - """Generate a hash checksum of all files matching 'path'. Standard - wildcards like '*' and '?' are supported, see documentation for the 'glob' - module for more information. - - :return: dict: A { filename: hash } dictionary for all matched files. - Empty if none found. - """ - return { - filename: file_hash(filename) - for filename in glob.iglob(path) - } - - -def check_hash(path, checksum, hash_type='md5'): - """Validate a file using a cryptographic checksum. - - :param str checksum: Value of the checksum used to validate the file. - :param str hash_type: Hash algorithm used to generate `checksum`. - Can be any hash alrgorithm supported by :mod:`hashlib`, - such as md5, sha1, sha256, sha512, etc. - :raises ChecksumError: If the file fails the checksum - - """ - actual_checksum = file_hash(path, hash_type) - if checksum != actual_checksum: - raise ChecksumError("'%s' != '%s'" % (checksum, actual_checksum)) - - -class ChecksumError(ValueError): - """A class derived from Value error to indicate the checksum failed.""" - pass - - -def restart_on_change(restart_map, stopstart=False, restart_functions=None): - """Restart services based on configuration files changing - - This function is used a decorator, for example:: - - @restart_on_change({ - '/etc/ceph/ceph.conf': [ 'cinder-api', 'cinder-volume' ] - '/etc/apache/sites-enabled/*': [ 'apache2' ] - }) - def config_changed(): - pass # your code here - - In this example, the cinder-api and cinder-volume services - would be restarted if /etc/ceph/ceph.conf is changed by the - ceph_client_changed function. The apache2 service would be - restarted if any file matching the pattern got changed, created - or removed. Standard wildcards are supported, see documentation - for the 'glob' module for more information. - - @param restart_map: {path_file_name: [service_name, ...] - @param stopstart: DEFAULT false; whether to stop, start OR restart - @param restart_functions: nonstandard functions to use to restart services - {svc: func, ...} - @returns result from decorated function - """ - def wrap(f): - @functools.wraps(f) - def wrapped_f(*args, **kwargs): - return restart_on_change_helper( - (lambda: f(*args, **kwargs)), restart_map, stopstart, - restart_functions) - return wrapped_f - return wrap - - -def restart_on_change_helper(lambda_f, restart_map, stopstart=False, - restart_functions=None): - """Helper function to perform the restart_on_change function. - - This is provided for decorators to restart services if files described - in the restart_map have changed after an invocation of lambda_f(). - - @param lambda_f: function to call. - @param restart_map: {file: [service, ...]} - @param stopstart: whether to stop, start or restart a service - @param restart_functions: nonstandard functions to use to restart services - {svc: func, ...} - @returns result of lambda_f() - """ - if restart_functions is None: - restart_functions = {} - checksums = {path: path_hash(path) for path in restart_map} - r = lambda_f() - # create a list of lists of the services to restart - restarts = [restart_map[path] - for path in restart_map - if path_hash(path) != checksums[path]] - # create a flat list of ordered services without duplicates from lists - services_list = list(OrderedDict.fromkeys(itertools.chain(*restarts))) - if services_list: - actions = ('stop', 'start') if stopstart else ('restart',) - for service_name in services_list: - if service_name in restart_functions: - restart_functions[service_name](service_name) - else: - for action in actions: - service(action, service_name) - return r - - -def pwgen(length=None): - """Generate a random pasword.""" - if length is None: - # A random length is ok to use a weak PRNG - length = random.choice(range(35, 45)) - alphanumeric_chars = [ - l for l in (string.ascii_letters + string.digits) - if l not in 'l0QD1vAEIOUaeiou'] - # Use a crypto-friendly PRNG (e.g. /dev/urandom) for making the - # actual password - random_generator = random.SystemRandom() - random_chars = [ - random_generator.choice(alphanumeric_chars) for _ in range(length)] - return(''.join(random_chars)) - - -def is_phy_iface(interface): - """Returns True if interface is not virtual, otherwise False.""" - if interface: - sys_net = '/sys/class/net' - if os.path.isdir(sys_net): - for iface in glob.glob(os.path.join(sys_net, '*')): - if '/virtual/' in os.path.realpath(iface): - continue - - if interface == os.path.basename(iface): - return True - - return False - - -def get_bond_master(interface): - """Returns bond master if interface is bond slave otherwise None. - - NOTE: the provided interface is expected to be physical - """ - if interface: - iface_path = '/sys/class/net/%s' % (interface) - if os.path.exists(iface_path): - if '/virtual/' in os.path.realpath(iface_path): - return None - - master = os.path.join(iface_path, 'master') - if os.path.exists(master): - master = os.path.realpath(master) - # make sure it is a bond master - if os.path.exists(os.path.join(master, 'bonding')): - return os.path.basename(master) - - return None - - -def list_nics(nic_type=None): - """Return a list of nics of given type(s)""" - if isinstance(nic_type, six.string_types): - int_types = [nic_type] - else: - int_types = nic_type - - interfaces = [] - if nic_type: - for int_type in int_types: - cmd = ['ip', 'addr', 'show', 'label', int_type + '*'] - ip_output = subprocess.check_output(cmd).decode('UTF-8') - ip_output = ip_output.split('\n') - ip_output = (line for line in ip_output if line) - for line in ip_output: - if line.split()[1].startswith(int_type): - matched = re.search('.*: (' + int_type + - r'[0-9]+\.[0-9]+)@.*', line) - if matched: - iface = matched.groups()[0] - else: - iface = line.split()[1].replace(":", "") - - if iface not in interfaces: - interfaces.append(iface) - else: - cmd = ['ip', 'a'] - ip_output = subprocess.check_output(cmd).decode('UTF-8').split('\n') - ip_output = (line.strip() for line in ip_output if line) - - key = re.compile('^[0-9]+:\s+(.+):') - for line in ip_output: - matched = re.search(key, line) - if matched: - iface = matched.group(1) - iface = iface.partition("@")[0] - if iface not in interfaces: - interfaces.append(iface) - - return interfaces - - -def set_nic_mtu(nic, mtu): - """Set the Maximum Transmission Unit (MTU) on a network interface.""" - cmd = ['ip', 'link', 'set', nic, 'mtu', mtu] - subprocess.check_call(cmd) - - -def get_nic_mtu(nic): - """Return the Maximum Transmission Unit (MTU) for a network interface.""" - cmd = ['ip', 'addr', 'show', nic] - ip_output = subprocess.check_output(cmd).decode('UTF-8').split('\n') - mtu = "" - for line in ip_output: - words = line.split() - if 'mtu' in words: - mtu = words[words.index("mtu") + 1] - return mtu - - -def get_nic_hwaddr(nic): - """Return the Media Access Control (MAC) for a network interface.""" - cmd = ['ip', '-o', '-0', 'addr', 'show', nic] - ip_output = subprocess.check_output(cmd).decode('UTF-8') - hwaddr = "" - words = ip_output.split() - if 'link/ether' in words: - hwaddr = words[words.index('link/ether') + 1] - return hwaddr - - -@contextmanager -def chdir(directory): - """Change the current working directory to a different directory for a code - block and return the previous directory after the block exits. Useful to - run commands from a specificed directory. - - :param str directory: The directory path to change to for this context. - """ - cur = os.getcwd() - try: - yield os.chdir(directory) - finally: - os.chdir(cur) - - -def chownr(path, owner, group, follow_links=True, chowntopdir=False): - """Recursively change user and group ownership of files and directories - in given path. Doesn't chown path itself by default, only its children. - - :param str path: The string path to start changing ownership. - :param str owner: The owner string to use when looking up the uid. - :param str group: The group string to use when looking up the gid. - :param bool follow_links: Also follow and chown links if True - :param bool chowntopdir: Also chown path itself if True - """ - uid = pwd.getpwnam(owner).pw_uid - gid = grp.getgrnam(group).gr_gid - if follow_links: - chown = os.chown - else: - chown = os.lchown - - if chowntopdir: - broken_symlink = os.path.lexists(path) and not os.path.exists(path) - if not broken_symlink: - chown(path, uid, gid) - for root, dirs, files in os.walk(path, followlinks=follow_links): - for name in dirs + files: - full = os.path.join(root, name) - broken_symlink = os.path.lexists(full) and not os.path.exists(full) - if not broken_symlink: - chown(full, uid, gid) - - -def lchownr(path, owner, group): - """Recursively change user and group ownership of files and directories - in a given path, not following symbolic links. See the documentation for - 'os.lchown' for more information. - - :param str path: The string path to start changing ownership. - :param str owner: The owner string to use when looking up the uid. - :param str group: The group string to use when looking up the gid. - """ - chownr(path, owner, group, follow_links=False) - - -def owner(path): - """Returns a tuple containing the username & groupname owning the path. - - :param str path: the string path to retrieve the ownership - :return tuple(str, str): A (username, groupname) tuple containing the - name of the user and group owning the path. - :raises OSError: if the specified path does not exist - """ - stat = os.stat(path) - username = pwd.getpwuid(stat.st_uid)[0] - groupname = grp.getgrgid(stat.st_gid)[0] - return username, groupname - - -def get_total_ram(): - """The total amount of system RAM in bytes. - - This is what is reported by the OS, and may be overcommitted when - there are multiple containers hosted on the same machine. - """ - with open('/proc/meminfo', 'r') as f: - for line in f.readlines(): - if line: - key, value, unit = line.split() - if key == 'MemTotal:': - assert unit == 'kB', 'Unknown unit' - return int(value) * 1024 # Classic, not KiB. - raise NotImplementedError() - - -UPSTART_CONTAINER_TYPE = '/run/container_type' - - -def is_container(): - """Determine whether unit is running in a container - - @return: boolean indicating if unit is in a container - """ - if init_is_systemd(): - # Detect using systemd-detect-virt - return subprocess.call(['systemd-detect-virt', - '--container']) == 0 - else: - # Detect using upstart container file marker - return os.path.exists(UPSTART_CONTAINER_TYPE) - - -def add_to_updatedb_prunepath(path, updatedb_path=UPDATEDB_PATH): - """Adds the specified path to the mlocate's udpatedb.conf PRUNEPATH list. - - This method has no effect if the path specified by updatedb_path does not - exist or is not a file. - - @param path: string the path to add to the updatedb.conf PRUNEPATHS value - @param updatedb_path: the path the updatedb.conf file - """ - if not os.path.exists(updatedb_path) or os.path.isdir(updatedb_path): - # If the updatedb.conf file doesn't exist then don't attempt to update - # the file as the package providing mlocate may not be installed on - # the local system - return - - with open(updatedb_path, 'r+') as f_id: - updatedb_text = f_id.read() - output = updatedb(updatedb_text, path) - f_id.seek(0) - f_id.write(output) - f_id.truncate() - - -def updatedb(updatedb_text, new_path): - lines = [line for line in updatedb_text.split("\n")] - for i, line in enumerate(lines): - if line.startswith("PRUNEPATHS="): - paths_line = line.split("=")[1].replace('"', '') - paths = paths_line.split(" ") - if new_path not in paths: - paths.append(new_path) - lines[i] = 'PRUNEPATHS="{}"'.format(' '.join(paths)) - output = "\n".join(lines) - return output - - -def modulo_distribution(modulo=3, wait=30, non_zero_wait=False): - """ Modulo distribution - - This helper uses the unit number, a modulo value and a constant wait time - to produce a calculated wait time distribution. This is useful in large - scale deployments to distribute load during an expensive operation such as - service restarts. - - If you have 1000 nodes that need to restart 100 at a time 1 minute at a - time: - - time.wait(modulo_distribution(modulo=100, wait=60)) - restart() - - If you need restarts to happen serially set modulo to the exact number of - nodes and set a high constant wait time: - - time.wait(modulo_distribution(modulo=10, wait=120)) - restart() - - @param modulo: int The modulo number creates the group distribution - @param wait: int The constant time wait value - @param non_zero_wait: boolean Override unit % modulo == 0, - return modulo * wait. Used to avoid collisions with - leader nodes which are often given priority. - @return: int Calculated time to wait for unit operation - """ - unit_number = int(local_unit().split('/')[1]) - calculated_wait_time = (unit_number % modulo) * wait - if non_zero_wait and calculated_wait_time == 0: - return modulo * wait - else: - return calculated_wait_time - - -def install_ca_cert(ca_cert, name=None): - """ - Install the given cert as a trusted CA. - - The ``name`` is the stem of the filename where the cert is written, and if - not provided, it will default to ``juju-{charm_name}``. - - If the cert is empty or None, or is unchanged, nothing is done. - """ - if not ca_cert: - return - if not isinstance(ca_cert, bytes): - ca_cert = ca_cert.encode('utf8') - if not name: - name = 'juju-{}'.format(charm_name()) - cert_file = '/usr/local/share/ca-certificates/{}.crt'.format(name) - new_hash = hashlib.md5(ca_cert).hexdigest() - if file_hash(cert_file) == new_hash: - return - log("Installing new CA cert at: {}".format(cert_file), level=INFO) - write_file(cert_file, ca_cert) - subprocess.check_call(['update-ca-certificates', '--fresh']) diff --git a/tests/dev-basic-cosmic-rocky b/tests/dev-basic-cosmic-rocky deleted file mode 100755 index b9910b2d..00000000 --- a/tests/dev-basic-cosmic-rocky +++ /dev/null @@ -1,23 +0,0 @@ -#!/usr/bin/env python -# -# Copyright 2016 Canonical Ltd -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -"""Amulet tests on a basic Glance deployment on cosmic-rocky.""" - -from basic_deployment import GlanceBasicDeployment - -if __name__ == '__main__': - deployment = GlanceBasicDeployment(series='cosmic') - deployment.run_tests() diff --git a/tests/gate-basic-bionic-queens b/tests/gate-basic-bionic-queens deleted file mode 100755 index 41763eae..00000000 --- a/tests/gate-basic-bionic-queens +++ /dev/null @@ -1,23 +0,0 @@ -#!/usr/bin/env python -# -# Copyright 2016 Canonical Ltd -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -"""Amulet tests on a basic Glance deployment on bionic-queens.""" - -from basic_deployment import GlanceBasicDeployment - -if __name__ == '__main__': - deployment = GlanceBasicDeployment(series='bionic') - deployment.run_tests() diff --git a/tests/gate-basic-bionic-rocky b/tests/gate-basic-bionic-rocky deleted file mode 100755 index 6a121557..00000000 --- a/tests/gate-basic-bionic-rocky +++ /dev/null @@ -1,25 +0,0 @@ -#!/usr/bin/env python -# -# Copyright 2016 Canonical Ltd -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -"""Amulet tests on a basic glance deployment on bionic-rocky.""" - -from basic_deployment import GlanceBasicDeployment - -if __name__ == '__main__': - deployment = GlanceBasicDeployment(series='bionic', - openstack='cloud:bionic-rocky', - source='cloud:bionic-updates/rocky') - deployment.run_tests() diff --git a/tests/gate-basic-bionic-stein b/tests/gate-basic-bionic-stein deleted file mode 100755 index f364db29..00000000 --- a/tests/gate-basic-bionic-stein +++ /dev/null @@ -1,25 +0,0 @@ -#!/usr/bin/env python -# -# Copyright 2016 Canonical Ltd -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -"""Amulet tests on a basic glance deployment on bionic-stein.""" - -from basic_deployment import GlanceBasicDeployment - -if __name__ == '__main__': - deployment = GlanceBasicDeployment(series='bionic', - openstack='cloud:bionic-stein', - source='cloud:bionic-stein') - deployment.run_tests() diff --git a/tests/gate-basic-disco-stein b/tests/gate-basic-disco-stein deleted file mode 100755 index 3a7b6d50..00000000 --- a/tests/gate-basic-disco-stein +++ /dev/null @@ -1,23 +0,0 @@ -#!/usr/bin/env python -# -# Copyright 2016 Canonical Ltd -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -"""Amulet tests on a basic Glance deployment on disco-stein.""" - -from basic_deployment import GlanceBasicDeployment - -if __name__ == '__main__': - deployment = GlanceBasicDeployment(series='disco') - deployment.run_tests() diff --git a/tests/gate-basic-trusty-mitaka b/tests/gate-basic-trusty-mitaka deleted file mode 100755 index 6eeaa127..00000000 --- a/tests/gate-basic-trusty-mitaka +++ /dev/null @@ -1,25 +0,0 @@ -#!/usr/bin/env python -# -# Copyright 2016 Canonical Ltd -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -"""Amulet tests on a basic glance deployment on trusty-mitaka.""" - -from basic_deployment import GlanceBasicDeployment - -if __name__ == '__main__': - deployment = GlanceBasicDeployment(series='trusty', - openstack='cloud:trusty-mitaka', - source='cloud:trusty-updates/mitaka') - deployment.run_tests() diff --git a/tests/gate-basic-xenial-mitaka b/tests/gate-basic-xenial-mitaka deleted file mode 100755 index 98b343e5..00000000 --- a/tests/gate-basic-xenial-mitaka +++ /dev/null @@ -1,23 +0,0 @@ -#!/usr/bin/env python -# -# Copyright 2016 Canonical Ltd -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -"""Amulet tests on a basic Glance deployment on xenial-mitaka.""" - -from basic_deployment import GlanceBasicDeployment - -if __name__ == '__main__': - deployment = GlanceBasicDeployment(series='xenial') - deployment.run_tests() diff --git a/tests/gate-basic-xenial-ocata b/tests/gate-basic-xenial-ocata deleted file mode 100755 index ce43ab0c..00000000 --- a/tests/gate-basic-xenial-ocata +++ /dev/null @@ -1,25 +0,0 @@ -#!/usr/bin/env python -# -# Copyright 2016 Canonical Ltd -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -"""Amulet tests on a basic glance deployment on xenial-ocata.""" - -from basic_deployment import GlanceBasicDeployment - -if __name__ == '__main__': - deployment = GlanceBasicDeployment(series='xenial', - openstack='cloud:xenial-ocata', - source='cloud:xenial-updates/ocata') - deployment.run_tests() diff --git a/tests/gate-basic-xenial-pike b/tests/gate-basic-xenial-pike deleted file mode 100755 index 2f913048..00000000 --- a/tests/gate-basic-xenial-pike +++ /dev/null @@ -1,25 +0,0 @@ -#!/usr/bin/env python -# -# Copyright 2016 Canonical Ltd -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -"""Amulet tests on a basic glance deployment on xenial-pike.""" - -from basic_deployment import GlanceBasicDeployment - -if __name__ == '__main__': - deployment = GlanceBasicDeployment(series='xenial', - openstack='cloud:xenial-pike', - source='cloud:xenial-updates/pike') - deployment.run_tests() diff --git a/tests/gate-basic-xenial-queens b/tests/gate-basic-xenial-queens deleted file mode 100755 index 44f06e6f..00000000 --- a/tests/gate-basic-xenial-queens +++ /dev/null @@ -1,25 +0,0 @@ -#!/usr/bin/env python -# -# Copyright 2016 Canonical Ltd -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -"""Amulet tests on a basic glance deployment on xenial-queens.""" - -from basic_deployment import GlanceBasicDeployment - -if __name__ == '__main__': - deployment = GlanceBasicDeployment(series='xenial', - openstack='cloud:xenial-queens', - source='cloud:xenial-updates/queens') - deployment.run_tests() diff --git a/tests/tests.yaml b/tests/tests.yaml index a03e7bad..77b8661a 100644 --- a/tests/tests.yaml +++ b/tests/tests.yaml @@ -1,18 +1,25 @@ -# Bootstrap the model if necessary. -bootstrap: True -# Re-use bootstrap node. -reset: True -# Use tox/requirements to drive the venv instead of bundletester's venv feature. -virtualenv: False -# Leave makefile empty, otherwise unit/lint tests will rerun ahead of amulet. -makefile: [] -# Do not specify juju PPA sources. Juju is presumed to be pre-installed -# and configured in all test runner environments. -#sources: -# Do not specify or rely on system packages. -#packages: -# Do not specify python packages here. Use test-requirements.txt -# and tox instead. ie. The venv is constructed before bundletester -# is invoked. -#python-packages: -reset_timeout: 600 +charm_name: glance + +comment: + - 'the glance configure job validates operation of identity-service relation' + +configure: + - zaza.openstack.charm_tests.glance.setup.add_lts_image + +smoke_bundles: + - bionic-stein +gate_bundles: + - trusty-mitaka + - xenial-mitaka + - xenial-ocata + - xenial-pike + - xenial-queens + - bionic-queens + - bionic-rocky + - bionic-stein + - disco-stein +dev_bundles: + - bionic-train + +tests: + - zaza.openstack.charm_tests.glance.tests.GlanceTest diff --git a/tox.ini b/tox.ini index e37ffd69..7c1d62e3 100644 --- a/tox.ini +++ b/tox.ini @@ -2,7 +2,7 @@ # This file is managed centrally by release-tools and should not be modified # within individual charm repos. [tox] -envlist = pep8,py27,py37 +envlist = pep8,py37 skipsdist = True # NOTE(beisner): Avoid build/test env pollution by not enabling sitepackages. sitepackages = False @@ -13,18 +13,12 @@ skip_missing_interpreters = False setenv = VIRTUAL_ENV={envdir} PYTHONHASHSEED=0 CHARM_DIR={envdir} - AMULET_SETUP_TIMEOUT=5400 install_command = pip install {opts} {packages} commands = stestr run {posargs} whitelist_externals = juju -passenv = HOME TERM AMULET_* CS_API_* OS_* - -[testenv:py27] -basepython = python2.7 -deps = -r{toxinidir}/requirements.txt - -r{toxinidir}/test-requirements.txt -commands = /bin/true +passenv = HOME TERM CS_API_* OS_* +deps = -r{toxinidir}/test-requirements.txt [testenv:py35] basepython = python3.5 @@ -41,6 +35,11 @@ basepython = python3.7 deps = -r{toxinidir}/requirements.txt -r{toxinidir}/test-requirements.txt +[testenv:py3] +basepython = python3 +deps = -r{toxinidir}/requirements.txt + -r{toxinidir}/test-requirements.txt + [testenv:pep8] basepython = python3 deps = -r{toxinidir}/requirements.txt @@ -80,49 +79,30 @@ omit = basepython = python3 commands = {posargs} -[testenv:func27-noop] -# DRY RUN - For Debug -basepython = python2.7 -deps = -r{toxinidir}/requirements.txt - -r{toxinidir}/test-requirements.txt +[testenv:func-noop] +basepython = python3 commands = - bundletester -vl DEBUG -r json -o func-results.json --test-pattern "gate-*" -n --no-destroy + /bin/true -[testenv:func27] -# Charm Functional Test -# Run all gate tests which are +x (expected to always pass) -basepython = python2.7 -deps = -r{toxinidir}/requirements.txt - -r{toxinidir}/test-requirements.txt +[testenv:func] +basepython = python3 commands = - bundletester -vl DEBUG -r json -o func-results.json --test-pattern "gate-*" --no-destroy + functest-run-suite --keep-model -[testenv:func27-smoke] -# Charm Functional Test -# Run a specific test as an Amulet smoke test (expected to always pass) -basepython = python2.7 -deps = -r{toxinidir}/requirements.txt - -r{toxinidir}/test-requirements.txt +[testenv:func-smoke] +basepython = python3 commands = - bundletester -vl DEBUG -r json -o func-results.json gate-basic-bionic-stein --no-destroy + functest-run-suite --keep-model --smoke -[testenv:func27-dfs] -# Charm Functional Test -# Run all deploy-from-source tests which are +x (may not always pass!) -basepython = python2.7 -deps = -r{toxinidir}/requirements.txt - -r{toxinidir}/test-requirements.txt +[testenv:func-dev] +basepython = python3 commands = - bundletester -vl DEBUG -r json -o func-results.json --test-pattern "dfs-*" --no-destroy + functest-run-suite --keep-model --dev -[testenv:func27-dev] -# Charm Functional Test -# Run all development test targets which are +x (may not always pass!) -basepython = python2.7 -deps = -r{toxinidir}/requirements.txt - -r{toxinidir}/test-requirements.txt +[testenv:func-target] +basepython = python3 commands = - bundletester -vl DEBUG -r json -o func-results.json --test-pattern "dev-*" --no-destroy + functest-run-suite --keep-model --bundle {posargs} [flake8] ignore = E402,E226