diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..e91ebf4 --- /dev/null +++ b/.gitignore @@ -0,0 +1,10 @@ +tests/*.py[cod] + +# Mr Developer +.idea + +# Linux swap file +*.swp + +# Tests results +.tox diff --git a/test-requirements.txt b/test-requirements.txt new file mode 100644 index 0000000..9beacb5 --- /dev/null +++ b/test-requirements.txt @@ -0,0 +1,6 @@ +testtools +requests +paramiko +python-muranoclient +python-heatclient +python-novaclient \ No newline at end of file diff --git a/tests/__init__.py b/tests/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/tests/base.py b/tests/base.py new file mode 100644 index 0000000..04fc182 --- /dev/null +++ b/tests/base.py @@ -0,0 +1,570 @@ +# Copyright (c) 2016 Mirantis Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +# implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import json +import logging +import os +import socket +import shutil +import time +import uuid + +import paramiko +import requests +import testtools +import yaml +import muranoclient.common.exceptions as exceptions + +import clients + +ARTIFACTS_DIR = os.environ.get('ARTIFACTS_DIR', 'artifacts') + +LOG = logging.getLogger(__name__) +LOG.setLevel(logging.DEBUG) +if not os.path.exists(ARTIFACTS_DIR): + os.makedirs(ARTIFACTS_DIR) +fh = logging.FileHandler(os.path.join(ARTIFACTS_DIR, 'runner.log')) +formatter = logging.Formatter( + '%(asctime)s - %(name)s - %(levelname)s - %(message)s' +) +fh.setFormatter(formatter) +LOG.addHandler(fh) + +ch = logging.StreamHandler() +ch.setLevel(logging.DEBUG) +ch.setFormatter(formatter) +LOG.addHandler(ch) + +# Sometimes need to pass some boolean from bash env. Since each bash +# variable is string, we need such simply hack +_boolean_states = { + '1': True, 'yes': True, 'true': True, 'on': True, + '0': False, 'no': False, 'false': False, 'off': False +} + +def str2bool(name, default): + value = os.environ.get(name, '') + return _boolean_states.get(value.lower(), default) + +TIMEOUT_DELAY = 30 + + +class MuranoTestsBase(testtools.TestCase, clients.ClientsBase): + + def setUp(self): + super(MuranoTestsBase, self).setUp() + self.os_username = os.environ.get('OS_USERNAME') + self.os_password = os.environ.get('OS_PASSWORD') + self.os_tenant_name = os.environ.get('OS_TENANT_NAME') + self.os_auth_uri = os.environ.get('OS_AUTH_URL') + + self.keystone = self.initialize_keystone_client() + self.heat = self.initialize_heat_client(self.keystone) + self.murano = self.initialize_murano_client(self.keystone) + self.nova = self.initialize_nova_client(self.keystone) + + # Since its really useful to debug deployment after it fails lets + # add such possibility + self.os_cleanup_before = str2bool('OS_CLEANUP_BEFORE', True) + self.os_cleanup_after = str2bool('OS_CLEANUP_AFTER', False) + + if self.os_cleanup_before: + self.cleanup_up_tenant() + + # Counter for murano deployment logger + self.latest_report = 0 + + # Application instance parameters + self.flavor = os.environ.get('OS_FLAVOR', 'm1.medium') + self.k8s_image = os.environ.get('OS_KUBERNETES_IMAGE') + self.k8s_image_user = os.environ.get( + 'OS_KUBERNETES_IMAGE_USER', 'debian' + ) + self.files = [] + self.keyname, self.pr_key, self.pub_key = self._create_keypair() + self.availability_zone = os.environ.get('OS_ZONE', 'nova') + + self.envs = [] + + LOG.info('Running test: {0}'.format(self._testMethodName)) + + def tearDown(self): + for env in self.envs: + self._collect_murano_agent_logs(env) + if self.os_cleanup_after: + for env in self.envs: + try: + self.delete_env(env) + except Exception: + self.delete_stack(env) + self.nova.keypairs.delete(self.keyname) + for file in self.files: + if os.path.isfile(file): + os.remove(file) + elif os.path.isdir(file): + shutil.rmtree(file) + + super(MuranoTestsBase, self).tearDown() + + @staticmethod + def rand_name(name='murano_ci_test_'): + return name + str(time.strftime("%Y_%m_%d_%H_%M_%S")) + + @staticmethod + def generate_id(): + return uuid.uuid4() + + def create_file(self, name, context): + with open(name, 'w') as f: + f.write(context) + path_to_file = os.path.join(os.getcwd(), name) + self.files.append(path_to_file) + return path_to_file + + def cleanup_up_tenant(self): + LOG.debug('Removing EVERYTHING in tenant: {0}'.format( + self.keystone.tenant_name)) + for env in self.murano.environments.list(): + self.delete_env(env) + self.delete_stack(env) + for key in self.nova.keypairs.list(): + if key.name.startswith("murano_ci_keypair"): + self.nova.keypairs.delete(key) + return + + def get_deployment_report(self, environment, deployment): + history = '' + report = self.murano.deployments.reports(environment.id, deployment.id) + for status in report: + history += '\t{0} - {1}\n'.format(status.created, status.text) + return history + + def _log_report(self, environment): + deployment = self.murano.deployments.list(environment.id)[0] + details = deployment.result['result']['details'] + LOG.error('Exception found:\n {0}'.format(details)) + report = self.get_deployment_report(environment, deployment) + LOG.debug('Report:\n {0}\n'.format(report)) + + def _log_latest(self, environment): + deployment = self.murano.deployments.list(environment.id)[0] + history = self.get_deployment_report(environment, deployment) + if self.latest_report != len(history) or self.latest_report == 0: + tmp = len(history) + history = history[self.latest_report:] + LOG.debug("Last report from murano engine:\n{}".format((history))) + self.latest_report = tmp + return history + + def _collect_murano_agent_logs(self, environment): + fips = self.get_services_fips(environment) + logs_dir = "{0}/{1}".format(ARTIFACTS_DIR, environment.name) + os.makedirs(logs_dir) + for service, fip in fips.iteritems(): + try: + ssh = paramiko.SSHClient() + ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy()) + ssh.connect( + fip, + username=self.k8s_image_user, + key_filename=self.pr_key + ) + ftp = ssh.open_sftp() + ftp.get( + '/var/log/murano-agent.log', + os.path.join(logs_dir, '{0}-agent.log'.format(service)) + ) + ftp.close() + except Exception as e: + LOG.warning( + "Couldn't collect murano-agent " + "logs of {0} (IP: {1}): {2}".format(service, fip, e) + ) + + def _create_keypair(self): + kp_name = self.rand_name('murano_ci_keypair_') + keypair = self.nova.keypairs.create(kp_name) + pr_key_file = self.create_file( + 'id_{}'.format(kp_name), keypair.private_key + ) + # Note: by default, permissions of created file with + # private keypair is too open + os.chmod(pr_key_file, 0600) + + pub_key_file = self.create_file( + 'id_{}.pub'.format(kp_name), keypair.public_key + ) + return kp_name, pr_key_file, pub_key_file + + def _get_stack(self, environment_id): + for stack in self.heat.stacks.list(): + if environment_id in stack.description: + return stack + + def delete_stack(self, environment): + stack = self._get_stack(environment.id) + if not stack: + return + else: + try: + self.heat.stacks.delete(stack.id) + except Exception as e: + LOG.warning("Unable delete stack:{}".format(stack)) + LOG.exception(e) + pass + + def create_env(self): + name = self.rand_name() + environment = self.murano.environments.create({'name': name}) + self.envs.append(environment) + if self.os_cleanup_after: + self.addCleanup(self.delete_env, environment) + LOG.debug('Created Environment:\n {0}'.format(environment)) + + return environment + + def delete_env(self, environment, timeout=360): + try: + self.murano.environments.delete(environment.id) + start_time = time.time() + while time.time() - start_time < timeout: + try: + self.murano.environments.get(environment.id) + time.sleep(1) + except exceptions.HTTPNotFound: + return + raise exceptions.HTTPOverLimit( + 'Environment "{0}" was not deleted in {1} seconds'.format( + environment.id, timeout) + ) + except (exceptions.HTTPForbidden, exceptions.HTTPOverLimit, + exceptions.HTTPNotFound): + try: + self.murano.environments.delete(environment.id, abandon=True) + LOG.warning( + 'Environment "{0}" from test {1} abandoned'.format( + environment.id, self._testMethodName)) + except exceptions.HTTPNotFound: + return + + start_time = time.time() + while time.time() - start_time < timeout: + try: + self.murano.environments.get(environment.id) + time.sleep(1) + except exceptions.HTTPNotFound: + return + raise Exception( + 'Environment "{0}" was not deleted in {1} seconds'.format( + environment.id, timeout) + ) + + def get_env(self, environment): + return self.murano.environments.get(environment.id) + + def deploy_env(self, environment, session): + self.murano.sessions.deploy(environment.id, session.id) + return self.wait_for_environment_deploy(environment) + + def wait_for_environment_deploy(self, env, timeout=7200): + start_time = time.time() + status = self.get_env(env).manager.get(env.id).status + + while status != 'ready': + status = self.get_env(env).manager.get(env.id).status + LOG.debug('Deployment status:{}...nothing new..'.format(status)) + self._log_latest(env) + + if time.time() - start_time > timeout: + time.sleep(60) + self.fail( + 'Environment deployment wasn\'t' + 'finished in {} seconds'.format(self.timeout) + ) + elif status == 'deploy failure': + self._log_report(env) + self.fail( + 'Environment has incorrect status "{0}"'.format(status) + ) + + time.sleep(TIMEOUT_DELAY) + LOG.debug('Environment "{0}" is ready'.format(self.get_env(env).name)) + return self.get_env(env).manager.get(env.id) + + def create_session(self, environment): + return self.murano.sessions.configure(environment.id) + + def create_service(self, environment, session, json_data, to_json=True): + LOG.debug('Adding service:\n {0}'.format(json_data)) + service = self.murano.services.post( + environment.id, + path='/', + data=json_data, + session_id=session.id + ) + if to_json: + service = service.to_dict() + service = json.dumps(service) + LOG.debug('Create Service json: {0}'.format(yaml.load(service))) + return yaml.load(service) + else: + LOG.debug('Create Service: {0}'.format(service)) + return service + + @staticmethod + def guess_fip(env_obj_model): + + result = {} + + def _finditem(obj, result): + if 'floatingIpAddress' in obj.get('instance', []): + result[obj['?']['package']] = obj['instance'][ + 'floatingIpAddress'] + for k, v in obj.items(): + if isinstance(v, dict): + _finditem(v, result) + _finditem(env_obj_model, result) + + return result + + def get_services_fips(self, environment): + fips = {} + for service in environment.services: + fips.update(self.guess_fip(service)) + + return fips + + def check_ports_open(self, ip, ports): + for port in ports: + result = 1 + start_time = time.time() + while time.time() - start_time < 60: + sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM) + result = sock.connect_ex((str(ip), port)) + sock.close() + + if result == 0: + LOG.debug('{} port is opened on instance'.format(port)) + break + time.sleep(5) + + if result != 0: + self.fail('{} port is not opened on instance'.format(port)) + + def check_url_access(self, ip, path, port): + proto = 'http' if port not in (443, 8443) else 'https' + url = '{proto}://{ip}:{port}/{path}'.format( + proto=proto, + ip=ip, + port=port, + path=path + ) + + resp = requests.get(url, timeout=60) + + return resp.status_code + + def deployment_success_check(self, environment, services_map): + deployment = self.murano.deployments.list(environment.id)[-1] + + self.assertEqual( + 'success', deployment.state, + 'Deployment status is "{0}"'.format(deployment.state) + ) + + fips = self.get_services_fips(environment) + + for service in services_map: + LOG.debug( + 'Checking ports availability on "{}" app instance'.format( + service) + ) + self.check_ports_open( + fips[service], services_map[service]['ports'] + ) + if services_map[service]['url']: + LOG.debug( + 'Checking {0} app url "{1}" availability'.format( + service, services_map[service]['url'] + ) + ) + self.check_url_access( + fips[service], + services_map[service]['url'], + services_map[service]['url_port'] + ) + + def wait_for(self, func, expected, debug_msg, fail_msg, timeout, **kwargs): + def check(exp, cur): + if isinstance(cur, list) or isinstance(cur, str): + return exp not in cur + else: + return exp != cur + + LOG.debug(debug_msg) + start_time = time.time() + + current = func(**kwargs) + + while check(expected, current): + current = func(**kwargs) + + if time.time() - start_time > timeout: + self.fail("Time is out. {0}".format(fail_msg)) + time.sleep(TIMEOUT_DELAY) + LOG.debug('Expected result has been achieved.') + + def create_k8s_cluster(self, params): + gateways = [] + + for gateway_num in range(params['max_gateways']): + gateways.append( + { + "instance": { + "name": "gateway-{0}".format(gateway_num), + "assignFloatingIp": True, + "keyname": params['keypair_name'], + "flavor": params['flavor'], + "image": params['kubernetes_image'], + "availabilityZone": 'nova', + "?": { + "type": "io.murano.resources.LinuxMuranoInstance", + "id": str(uuid.uuid4()) + } + }, + "?": { + "type": "com.mirantis.docker.kubernetes." + "KubernetesGatewayNode", + "id": str(uuid.uuid4()) + } + }) + + minions = [] + + for minion_num in range(params['max_nodes']): + minions.append( + { + "instance": { + "name": "minion-{0}".format(minion_num), + "assignFloatingIp": True, + "keyname": params['keypair_name'], + "flavor": params['flavor'], + "image": params['kubernetes_image'], + "availabilityZone": 'nova', + "?": { + "type": "io.murano.resources.LinuxMuranoInstance", + "id": str(uuid.uuid4()) + } + }, + "?": { + "type": "com.mirantis.docker.kubernetes." + "KubernetesMinionNode", + "id": str(uuid.uuid4()) + }, + "exposeCAdvisor": params['cadvisor'] + }) + + k8s_cluster_json = { + "gatewayCount": params['initial_gateways'], + "gatewayNodes": gateways, + "?": { + "_{id}".format(id=uuid.uuid4().hex): { + "name": "Kubernetes Cluster" + }, + "type": "com.mirantis.docker.kubernetes.KubernetesCluster", + "id": str(uuid.uuid4()) + }, + "nodeCount": params['initial_nodes'], + "dockerRegistry": "", + "gcloudKey": "", + "dockerMirror": "", + "masterNode": { + "instance": { + "name": "master-1", + "assignFloatingIp": True, + "keyname": params["keypair_name"], + "flavor": params["flavor"], + "image": params["kubernetes_image"], + "availabilityZone": 'nova', + "?": { + "type": "io.murano.resources.LinuxMuranoInstance", + "id": str(uuid.uuid4()) + } + }, + "?": { + "type": "com.mirantis.docker.kubernetes." + "KubernetesMasterNode", + "id": str(uuid.uuid4()) + } + }, + "minionNodes": minions, + "name": "KubeClusterTest" + } + + print k8s_cluster_json + + return k8s_cluster_json + + def create_k8s_pod(self, k8s_cluster, params): + k8s_pod_json = { + "kubernetesCluster": k8s_cluster, + "labels": params['labels'], + "name": "testpod", + "replicas": params['replicas'], + "?": { + "_{id}".format(id=uuid.uuid4().hex): { + "name": "Kubernetes Pod" + }, + "type": "com.mirantis.docker.kubernetes.KubernetesPod", + "id": str(uuid.uuid4()) + } + } + + return k8s_pod_json + + def get_k8s_instances(self, env): + def _get_instance_fip(server_id): + for _, addr in self.nova.servers.get(server_id).addresses.values(): + if addr["OS-EXT-IPS:type"] == "floating": + return addr["addr"] + + k8s_instances = {'gateways': [], 'minions': []} + + stack = self._get_stack(env.id) + + for res in self.heat.resources.list(stack.id): + if res.resource_type == "OS::Nova::Server": + if "gateway" in res.resource_name: + k8s_instances['gateways'].append( + _get_instance_fip(res.physical_resource_id) + ) + if "minion" in res.resource_name: + k8s_instances['minions'].append( + _get_instance_fip(res.physical_resource_id) + ) + + return k8s_instances + + + def run_k8s_action(self, environment, action): + def _get_action_id(environment, name): + env_data = environment.to_dict() + a_dict = env_data['services'][0]['?']['_actions'] + for action_id, action in a_dict.items(): + if action['name'] == name: + return action_id + + action_id = _get_action_id(environment, action) + self.murano.actions.call(environment.id, action_id) \ No newline at end of file diff --git a/tests/clients.py b/tests/clients.py new file mode 100644 index 0000000..5e2cc95 --- /dev/null +++ b/tests/clients.py @@ -0,0 +1,101 @@ +# Copyright (c) 2016 Mirantis Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +# implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import os + +from heatclient import client as heatclient +from keystoneclient.v2_0 import client as keystoneclient +from muranoclient import client as muranoclient +from novaclient import client as novaclient + + +class ClientsBase(object): + + @staticmethod + def initialize_keystone_client(): + username = os.environ.get('OS_USERNAME') + password = os.environ.get('OS_PASSWORD') + tenant_name = os.environ.get('OS_TENANT_NAME') + auth_url = os.environ.get('OS_AUTH_URL') + + keystone = keystoneclient.Client( + username=username, + password=password, + tenant_name=tenant_name, + auth_url=auth_url + ) + return keystone + + @classmethod + def get_endpoint(cls, service_type, endpoint_type): + ks_client = cls.initialize_keystone_client() + + return ks_client.service_catalog.url_for( + service_type=service_type, + endpoint_type=endpoint_type + ) + + @classmethod + def initialize_murano_client(cls, auth_client=None): + ks_client = (auth_client if auth_client + else cls.initialize_keystone_client()) + + murano_endpoint = cls.get_endpoint( + service_type='application-catalog', + endpoint_type='publicURL' + ) + + murano = muranoclient.Client( + '1', + endpoint=murano_endpoint, + token=ks_client.auth_token + ) + + return murano + + @classmethod + def initialize_heat_client(cls, auth_client=None): + ks_client = (auth_client if auth_client + else cls.initialize_keystone_client()) + + heat_endpoint = cls.get_endpoint( + service_type='orchestration', + endpoint_type='publicURL' + ) + + heat = heatclient.Client( + '1', + endpoint=heat_endpoint, + token=ks_client.auth_token + ) + + return heat + + @classmethod + def initialize_nova_client(cls, auth_client=None): + ks_client = (auth_client if auth_client + else cls.initialize_keystone_client()) + + nova = novaclient.Client( + '2', + username=None, + service_type='compute', + endpoint_type='publicURL', + auth_token=ks_client.auth_token, + auth_url=ks_client.auth_url + ) + nova.client.management_url = cls.get_endpoint('compute', 'publicURL') + + return nova \ No newline at end of file diff --git a/tests/test_k8s_app.py b/tests/test_k8s_app.py new file mode 100644 index 0000000..2db54d3 --- /dev/null +++ b/tests/test_k8s_app.py @@ -0,0 +1,131 @@ +# Copyright (c) 2016 Mirantis Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +# implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import base + + +class MuranoK8sTest(base.MuranoTestsBase): + + def test_deploy_scale_k8s(self): + """Check that it is possible to deploy K8s application and scale it + + Scenario: + 1. Create murano environment + 2. Create session for create environment. + 3. Initialize k8s cluster parameters and add app to env. + 4. Initialize k8s pod parameters and add app to env. + 5. Deploy session. + 7. Check env status and port availability on k8s master node. + 8. Get k8s minions' ips and check that correct initial number + of them was created and k8s api port is available on them. + 9. Run 'scaleNodesUp' action for k8s minions. + 10. Check that number of minions was increased and + k8s api port is available on all of them + 11. Run 'scaleNodesDown' action for k8s minions. + 12. Check that number of minions was decreased and + k8s api port is available on all of them + """ + + # Create murano environment + environment = self.create_env() + + # Create session for create environment. + session = self.create_session(environment) + + # Initialize k8s cluster parameters and add app to env + k8s_cluster_json = self.create_k8s_cluster( + { + 'initial_nodes': 1, + 'max_nodes': 2, + 'initial_gateways': 1, + 'max_gateways': 1, + 'cadvisor': True, + 'keypair_name': self.keyname, + 'flavor': self.flavor, + 'kubernetes_image': self.k8s_image + } + ) + k8s_cluster = self.create_service( + environment, + session, + k8s_cluster_json + ) + + # Initialize k8s pod parameters and add app to env. + k8s_pod_json = self.create_k8s_pod( + k8s_cluster, + { + 'labels': 'testkey=testvalue', + 'replicas': 2, + } + ) + self.create_service(environment, session, k8s_pod_json) + + # Deploy session. + self.deploy_env(environment, session) + + # Check env status and port availability on k8s master node. + environment = self.get_env(environment) + + check_services = { + 'com.mirantis.docker.kubernetes.KubernetesCluster': { + 'ports': [8080, 22], + 'url': 'api/', + 'url_port': 8080 + } + } + self.deployment_success_check(environment, check_services) + + # Get k8s minions' ips and check that correct initial number + # of them was created and k8s api port is available on them. + minions_ips = self.get_k8s_instances(environment)['minions'] + self.assertEqual(1, len(minions_ips)) + + for ip in minions_ips: + self.check_ports_open(ip, [4194]) + + # Run 'scaleNodesUp' action for k8s minions. + self.run_k8s_action( + environment=environment, + action='scaleNodesUp' + ) + self.wait_for_environment_deploy(environment) + + # Check that number of minions was increased and + # k8s api port is available on all of them + environment = self.get_env(environment) + + minions_ips = self.get_k8s_instances(environment)['minions'] + self.assertEqual(2, len(minions_ips)) + + for ip in minions_ips: + self.check_ports_open(ip, [4194]) + + # Run 'scaleNodesDown' action for k8s minions. + self.run_k8s_action( + environment=environment, + action='scaleNodesDown' + ) + self.wait_for_environment_deploy(environment) + + # Check that number of minions was increased and + # k8s api port is available on all of them + environment = self.get_env(environment) + + minions_ips = self.get_k8s_instances(environment)['minions'] + self.assertEqual(1, len(minions_ips)) + + for ip in minions_ips: + self.check_ports_open(ip, [4194]) diff --git a/tox.ini b/tox.ini new file mode 100644 index 0000000..cc3a64b --- /dev/null +++ b/tox.ini @@ -0,0 +1,20 @@ +[tox] +minversion = 1.6 +skipsdist = True +skip_missing_interpreters = True + +[testenv] +setenv = VIRTUAL_ENV={envdir} + LANG=en_US.UTF-8 + LANGUAGE=en_US:en + LC_ALL=C +passenv = OS_* MURANO* *ENDPOINT* +deps= + -r{toxinidir}/test-requirements.txt +distribute = false + +[testenv:venv] +commands = {posargs:} + +[testenv:deploy_scale_k8s] +commands = python -m unittest tests.test_k8s_app.MuranoK8sTest.test_deploy_scale_k8s