x
/
k8s-docker-suite-app-murano
Code Issues Proposed changes

Add k8s app test for per-commit job 

Scenario:
            1. Create murano environment
            2. Create session for create environment.
            3. Initialize k8s cluster parameters and add app to env.
            4. Initialize k8s pod parameters and add app to env.
            5. Deploy session.
            7. Check env status and port availability on k8s master node.
            8. Get k8s minions' ips and check that correct initial number
            of them was created and k8s api port is available on them.
            9. Run 'scaleNodesUp' action for k8s minions.
            10. Check that number of minions was increased and
            k8s api port is available on all of them
            11. Run 'scaleNodesDown' action for k8s minions.
            12. Check that number of minions was decreased and
            k8s api port is available on all of them

Change-Id: I63bf61b6f9e5fd45b8e6bde0fab87da5826bedc6
This commit is contained in:
Anastasia Kuznetsova 2016-09-06 19:04:27 +03:00
parent 6547024fa3
commit a143e7c707
7 changed files with 838 additions and 0 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

10
.gitignore vendored Normal file
View File

@ -0,0 +1,10 @@
tests/*.py[cod]
# Mr Developer
.idea
# Linux swap file
*.swp
# Tests results
.tox

6
test-requirements.txt Normal file
View File

@ -0,0 +1,6 @@
testtools
requests
paramiko
python-muranoclient
python-heatclient
python-novaclient

0
tests/__init__.py Normal file
View File

570
tests/base.py Normal file
View File

@ -0,0 +1,570 @@
# Copyright (c) 2016 Mirantis Inc.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
# implied.
# See the License for the specific language governing permissions and
# limitations under the License.
import json
import logging
import os
import socket
import shutil
import time
import uuid
import paramiko
import requests
import testtools
import yaml
import muranoclient.common.exceptions as exceptions
import clients
ARTIFACTS_DIR = os.environ.get('ARTIFACTS_DIR', 'artifacts')
LOG = logging.getLogger(__name__)
LOG.setLevel(logging.DEBUG)
if not os.path.exists(ARTIFACTS_DIR):
os.makedirs(ARTIFACTS_DIR)
fh = logging.FileHandler(os.path.join(ARTIFACTS_DIR, 'runner.log'))
formatter = logging.Formatter(
'%(asctime)s - %(name)s - %(levelname)s - %(message)s'
)
fh.setFormatter(formatter)
LOG.addHandler(fh)
ch = logging.StreamHandler()
ch.setLevel(logging.DEBUG)
ch.setFormatter(formatter)
LOG.addHandler(ch)
# Sometimes need to pass some boolean from bash env. Since each bash
# variable is string, we need such simply hack
_boolean_states = {
'1': True, 'yes': True, 'true': True, 'on': True,
'0': False, 'no': False, 'false': False, 'off': False
}
def str2bool(name, default):
value = os.environ.get(name, '')
return _boolean_states.get(value.lower(), default)
TIMEOUT_DELAY = 30
class MuranoTestsBase(testtools.TestCase, clients.ClientsBase):
def setUp(self):
super(MuranoTestsBase, self).setUp()
self.os_username = os.environ.get('OS_USERNAME')
self.os_password = os.environ.get('OS_PASSWORD')
self.os_tenant_name = os.environ.get('OS_TENANT_NAME')
self.os_auth_uri = os.environ.get('OS_AUTH_URL')
self.keystone = self.initialize_keystone_client()
self.heat = self.initialize_heat_client(self.keystone)
self.murano = self.initialize_murano_client(self.keystone)
self.nova = self.initialize_nova_client(self.keystone)
# Since its really useful to debug deployment after it fails lets
# add such possibility
self.os_cleanup_before = str2bool('OS_CLEANUP_BEFORE', True)
self.os_cleanup_after = str2bool('OS_CLEANUP_AFTER', False)
if self.os_cleanup_before:
self.cleanup_up_tenant()
# Counter for murano deployment logger
self.latest_report = 0
# Application instance parameters
self.flavor = os.environ.get('OS_FLAVOR', 'm1.medium')
self.k8s_image = os.environ.get('OS_KUBERNETES_IMAGE')
self.k8s_image_user = os.environ.get(
'OS_KUBERNETES_IMAGE_USER', 'debian'
)
self.files = []
self.keyname, self.pr_key, self.pub_key = self._create_keypair()
self.availability_zone = os.environ.get('OS_ZONE', 'nova')
self.envs = []
LOG.info('Running test: {0}'.format(self._testMethodName))
def tearDown(self):
for env in self.envs:
self._collect_murano_agent_logs(env)
if self.os_cleanup_after:
for env in self.envs:
try:
self.delete_env(env)
except Exception:
self.delete_stack(env)
self.nova.keypairs.delete(self.keyname)
for file in self.files:
if os.path.isfile(file):
os.remove(file)
elif os.path.isdir(file):
shutil.rmtree(file)
super(MuranoTestsBase, self).tearDown()
@staticmethod
def rand_name(name='murano_ci_test_'):
return name + str(time.strftime("%Y_%m_%d_%H_%M_%S"))
@staticmethod
def generate_id():
return uuid.uuid4()
def create_file(self, name, context):
with open(name, 'w') as f:
f.write(context)
path_to_file = os.path.join(os.getcwd(), name)
self.files.append(path_to_file)
return path_to_file
def cleanup_up_tenant(self):
LOG.debug('Removing EVERYTHING in tenant: {0}'.format(
self.keystone.tenant_name))
for env in self.murano.environments.list():
self.delete_env(env)
self.delete_stack(env)
for key in self.nova.keypairs.list():
if key.name.startswith("murano_ci_keypair"):
self.nova.keypairs.delete(key)
return
def get_deployment_report(self, environment, deployment):
history = ''
report = self.murano.deployments.reports(environment.id, deployment.id)
for status in report:
history += '\t{0} - {1}\n'.format(status.created, status.text)
return history
def _log_report(self, environment):
deployment = self.murano.deployments.list(environment.id)[0]
details = deployment.result['result']['details']
LOG.error('Exception found:\n {0}'.format(details))
report = self.get_deployment_report(environment, deployment)
LOG.debug('Report:\n {0}\n'.format(report))
def _log_latest(self, environment):
deployment = self.murano.deployments.list(environment.id)[0]
history = self.get_deployment_report(environment, deployment)
if self.latest_report != len(history) or self.latest_report == 0:
tmp = len(history)
history = history[self.latest_report:]
LOG.debug("Last report from murano engine:\n{}".format((history)))
self.latest_report = tmp
return history
def _collect_murano_agent_logs(self, environment):
fips = self.get_services_fips(environment)
logs_dir = "{0}/{1}".format(ARTIFACTS_DIR, environment.name)
os.makedirs(logs_dir)
for service, fip in fips.iteritems():
try:
ssh = paramiko.SSHClient()
ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy())
ssh.connect(
fip,
username=self.k8s_image_user,
key_filename=self.pr_key
)
ftp = ssh.open_sftp()
ftp.get(
'/var/log/murano-agent.log',
os.path.join(logs_dir, '{0}-agent.log'.format(service))
)
ftp.close()
except Exception as e:
LOG.warning(
"Couldn't collect murano-agent "
"logs of {0} (IP: {1}): {2}".format(service, fip, e)
)
def _create_keypair(self):
kp_name = self.rand_name('murano_ci_keypair_')
keypair = self.nova.keypairs.create(kp_name)
pr_key_file = self.create_file(
'id_{}'.format(kp_name), keypair.private_key
)
# Note: by default, permissions of created file with
# private keypair is too open
os.chmod(pr_key_file, 0600)
pub_key_file = self.create_file(
'id_{}.pub'.format(kp_name), keypair.public_key
)
return kp_name, pr_key_file, pub_key_file
def _get_stack(self, environment_id):
for stack in self.heat.stacks.list():
if environment_id in stack.description:
return stack
def delete_stack(self, environment):
stack = self._get_stack(environment.id)
if not stack:
return
else:
try:
self.heat.stacks.delete(stack.id)
except Exception as e:
LOG.warning("Unable delete stack:{}".format(stack))
LOG.exception(e)
pass
def create_env(self):
name = self.rand_name()
environment = self.murano.environments.create({'name': name})
self.envs.append(environment)
if self.os_cleanup_after:
self.addCleanup(self.delete_env, environment)
LOG.debug('Created Environment:\n {0}'.format(environment))
return environment
def delete_env(self, environment, timeout=360):
try:
self.murano.environments.delete(environment.id)
start_time = time.time()
while time.time() - start_time < timeout:
try:
self.murano.environments.get(environment.id)
time.sleep(1)
except exceptions.HTTPNotFound:
return
raise exceptions.HTTPOverLimit(
'Environment "{0}" was not deleted in {1} seconds'.format(
environment.id, timeout)
)
except (exceptions.HTTPForbidden, exceptions.HTTPOverLimit,
exceptions.HTTPNotFound):
try:
self.murano.environments.delete(environment.id, abandon=True)
LOG.warning(
'Environment "{0}" from test {1} abandoned'.format(
environment.id, self._testMethodName))
except exceptions.HTTPNotFound:
return
start_time = time.time()
while time.time() - start_time < timeout:
try:
self.murano.environments.get(environment.id)
time.sleep(1)
except exceptions.HTTPNotFound:
return
raise Exception(
'Environment "{0}" was not deleted in {1} seconds'.format(
environment.id, timeout)
)
def get_env(self, environment):
return self.murano.environments.get(environment.id)
def deploy_env(self, environment, session):
self.murano.sessions.deploy(environment.id, session.id)
return self.wait_for_environment_deploy(environment)
def wait_for_environment_deploy(self, env, timeout=7200):
start_time = time.time()
status = self.get_env(env).manager.get(env.id).status
while status != 'ready':
status = self.get_env(env).manager.get(env.id).status
LOG.debug('Deployment status:{}...nothing new..'.format(status))
self._log_latest(env)
if time.time() - start_time > timeout:
time.sleep(60)
self.fail(
'Environment deployment wasn\'t'
'finished in {} seconds'.format(self.timeout)
)
elif status == 'deploy failure':
self._log_report(env)
self.fail(
'Environment has incorrect status "{0}"'.format(status)
)
time.sleep(TIMEOUT_DELAY)
LOG.debug('Environment "{0}" is ready'.format(self.get_env(env).name))
return self.get_env(env).manager.get(env.id)
def create_session(self, environment):
return self.murano.sessions.configure(environment.id)
def create_service(self, environment, session, json_data, to_json=True):
LOG.debug('Adding service:\n {0}'.format(json_data))
service = self.murano.services.post(
environment.id,
path='/',
data=json_data,
session_id=session.id
)
if to_json:
service = service.to_dict()
service = json.dumps(service)
LOG.debug('Create Service json: {0}'.format(yaml.load(service)))
return yaml.load(service)
else:
LOG.debug('Create Service: {0}'.format(service))
return service
@staticmethod
def guess_fip(env_obj_model):
result = {}
def _finditem(obj, result):
if 'floatingIpAddress' in obj.get('instance', []):
result[obj['?']['package']] = obj['instance'][
'floatingIpAddress']
for k, v in obj.items():
if isinstance(v, dict):
_finditem(v, result)
_finditem(env_obj_model, result)
return result
def get_services_fips(self, environment):
fips = {}
for service in environment.services:
fips.update(self.guess_fip(service))
return fips
def check_ports_open(self, ip, ports):
for port in ports:
result = 1
start_time = time.time()
while time.time() - start_time < 60:
sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
result = sock.connect_ex((str(ip), port))
sock.close()
if result == 0:
LOG.debug('{} port is opened on instance'.format(port))
break
time.sleep(5)
if result != 0:
self.fail('{} port is not opened on instance'.format(port))
def check_url_access(self, ip, path, port):
proto = 'http' if port not in (443, 8443) else 'https'
url = '{proto}://{ip}:{port}/{path}'.format(
proto=proto,
ip=ip,
port=port,
path=path
)
resp = requests.get(url, timeout=60)
return resp.status_code
def deployment_success_check(self, environment, services_map):
deployment = self.murano.deployments.list(environment.id)[-1]
self.assertEqual(
'success', deployment.state,
'Deployment status is "{0}"'.format(deployment.state)
)
fips = self.get_services_fips(environment)
for service in services_map:
LOG.debug(
'Checking ports availability on "{}" app instance'.format(
service)
)
self.check_ports_open(
fips[service], services_map[service]['ports']
)
if services_map[service]['url']:
LOG.debug(
'Checking {0} app url "{1}" availability'.format(
service, services_map[service]['url']
)
)
self.check_url_access(
fips[service],
services_map[service]['url'],
services_map[service]['url_port']
)
def wait_for(self, func, expected, debug_msg, fail_msg, timeout, **kwargs):
def check(exp, cur):
if isinstance(cur, list) or isinstance(cur, str):
return exp not in cur
else:
return exp != cur
LOG.debug(debug_msg)
start_time = time.time()
current = func(**kwargs)
while check(expected, current):
current = func(**kwargs)
if time.time() - start_time > timeout:
self.fail("Time is out. {0}".format(fail_msg))
time.sleep(TIMEOUT_DELAY)
LOG.debug('Expected result has been achieved.')
def create_k8s_cluster(self, params):
gateways = []
for gateway_num in range(params['max_gateways']):
gateways.append(
{
"instance": {
"name": "gateway-{0}".format(gateway_num),
"assignFloatingIp": True,
"keyname": params['keypair_name'],
"flavor": params['flavor'],
"image": params['kubernetes_image'],
"availabilityZone": 'nova',
"?": {
"type": "io.murano.resources.LinuxMuranoInstance",
"id": str(uuid.uuid4())
}
},
"?": {
"type": "com.mirantis.docker.kubernetes."
"KubernetesGatewayNode",
"id": str(uuid.uuid4())
}
})
minions = []
for minion_num in range(params['max_nodes']):
minions.append(
{
"instance": {
"name": "minion-{0}".format(minion_num),
"assignFloatingIp": True,
"keyname": params['keypair_name'],
"flavor": params['flavor'],
"image": params['kubernetes_image'],
"availabilityZone": 'nova',
"?": {
"type": "io.murano.resources.LinuxMuranoInstance",
"id": str(uuid.uuid4())
}
},
"?": {
"type": "com.mirantis.docker.kubernetes."
"KubernetesMinionNode",
"id": str(uuid.uuid4())
},
"exposeCAdvisor": params['cadvisor']
})
k8s_cluster_json = {
"gatewayCount": params['initial_gateways'],
"gatewayNodes": gateways,
"?": {
"_{id}".format(id=uuid.uuid4().hex): {
"name": "Kubernetes Cluster"
},
"type": "com.mirantis.docker.kubernetes.KubernetesCluster",
"id": str(uuid.uuid4())
},
"nodeCount": params['initial_nodes'],
"dockerRegistry": "",
"gcloudKey": "",
"dockerMirror": "",
"masterNode": {
"instance": {
"name": "master-1",
"assignFloatingIp": True,
"keyname": params["keypair_name"],
"flavor": params["flavor"],
"image": params["kubernetes_image"],
"availabilityZone": 'nova',
"?": {
"type": "io.murano.resources.LinuxMuranoInstance",
"id": str(uuid.uuid4())
}
},
"?": {
"type": "com.mirantis.docker.kubernetes."
"KubernetesMasterNode",
"id": str(uuid.uuid4())
}
},
"minionNodes": minions,
"name": "KubeClusterTest"
}
print k8s_cluster_json
return k8s_cluster_json
def create_k8s_pod(self, k8s_cluster, params):
k8s_pod_json = {
"kubernetesCluster": k8s_cluster,
"labels": params['labels'],
"name": "testpod",
"replicas": params['replicas'],
"?": {
"_{id}".format(id=uuid.uuid4().hex): {
"name": "Kubernetes Pod"
},
"type": "com.mirantis.docker.kubernetes.KubernetesPod",
"id": str(uuid.uuid4())
}
}
return k8s_pod_json
def get_k8s_instances(self, env):
def _get_instance_fip(server_id):
for _, addr in self.nova.servers.get(server_id).addresses.values():
if addr["OS-EXT-IPS:type"] == "floating":
return addr["addr"]
k8s_instances = {'gateways': [], 'minions': []}
stack = self._get_stack(env.id)
for res in self.heat.resources.list(stack.id):
if res.resource_type == "OS::Nova::Server":
if "gateway" in res.resource_name:
k8s_instances['gateways'].append(
_get_instance_fip(res.physical_resource_id)
)
if "minion" in res.resource_name:
k8s_instances['minions'].append(
_get_instance_fip(res.physical_resource_id)
)
return k8s_instances
def run_k8s_action(self, environment, action):
def _get_action_id(environment, name):
env_data = environment.to_dict()
a_dict = env_data['services'][0]['?']['_actions']
for action_id, action in a_dict.items():
if action['name'] == name:
return action_id
action_id = _get_action_id(environment, action)
self.murano.actions.call(environment.id, action_id)

101
tests/clients.py Normal file
View File

@ -0,0 +1,101 @@
# Copyright (c) 2016 Mirantis Inc.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
# implied.
# See the License for the specific language governing permissions and
# limitations under the License.
import os
from heatclient import client as heatclient
from keystoneclient.v2_0 import client as keystoneclient
from muranoclient import client as muranoclient
from novaclient import client as novaclient
class ClientsBase(object):
@staticmethod
def initialize_keystone_client():
username = os.environ.get('OS_USERNAME')
password = os.environ.get('OS_PASSWORD')
tenant_name = os.environ.get('OS_TENANT_NAME')
auth_url = os.environ.get('OS_AUTH_URL')
keystone = keystoneclient.Client(
username=username,
password=password,
tenant_name=tenant_name,
auth_url=auth_url
)
return keystone
@classmethod
def get_endpoint(cls, service_type, endpoint_type):
ks_client = cls.initialize_keystone_client()
return ks_client.service_catalog.url_for(
service_type=service_type,
endpoint_type=endpoint_type
)
@classmethod
def initialize_murano_client(cls, auth_client=None):
ks_client = (auth_client if auth_client
else cls.initialize_keystone_client())
murano_endpoint = cls.get_endpoint(
service_type='application-catalog',
endpoint_type='publicURL'
)
murano = muranoclient.Client(
'1',
endpoint=murano_endpoint,
token=ks_client.auth_token
)
return murano
@classmethod
def initialize_heat_client(cls, auth_client=None):
ks_client = (auth_client if auth_client
else cls.initialize_keystone_client())
heat_endpoint = cls.get_endpoint(
service_type='orchestration',
endpoint_type='publicURL'
)
heat = heatclient.Client(
'1',
endpoint=heat_endpoint,
token=ks_client.auth_token
)
return heat
@classmethod
def initialize_nova_client(cls, auth_client=None):
ks_client = (auth_client if auth_client
else cls.initialize_keystone_client())
nova = novaclient.Client(
'2',
username=None,
service_type='compute',
endpoint_type='publicURL',
auth_token=ks_client.auth_token,
auth_url=ks_client.auth_url
)
nova.client.management_url = cls.get_endpoint('compute', 'publicURL')
return nova

131
tests/test_k8s_app.py Normal file
View File

@ -0,0 +1,131 @@
# Copyright (c) 2016 Mirantis Inc.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
# implied.
# See the License for the specific language governing permissions and
# limitations under the License.
import base
class MuranoK8sTest(base.MuranoTestsBase):
def test_deploy_scale_k8s(self):
"""Check that it is possible to deploy K8s application and scale it
Scenario:
1. Create murano environment
2. Create session for create environment.
3. Initialize k8s cluster parameters and add app to env.
4. Initialize k8s pod parameters and add app to env.
5. Deploy session.
7. Check env status and port availability on k8s master node.
8. Get k8s minions' ips and check that correct initial number
of them was created and k8s api port is available on them.
9. Run 'scaleNodesUp' action for k8s minions.
10. Check that number of minions was increased and
k8s api port is available on all of them
11. Run 'scaleNodesDown' action for k8s minions.
12. Check that number of minions was decreased and
k8s api port is available on all of them
"""
# Create murano environment
environment = self.create_env()
# Create session for create environment.
session = self.create_session(environment)
# Initialize k8s cluster parameters and add app to env
k8s_cluster_json = self.create_k8s_cluster(
{
'initial_nodes': 1,
'max_nodes': 2,
'initial_gateways': 1,
'max_gateways': 1,
'cadvisor': True,
'keypair_name': self.keyname,
'flavor': self.flavor,
'kubernetes_image': self.k8s_image
}
)
k8s_cluster = self.create_service(
environment,
session,
k8s_cluster_json
)
# Initialize k8s pod parameters and add app to env.
k8s_pod_json = self.create_k8s_pod(
k8s_cluster,
{
'labels': 'testkey=testvalue',
'replicas': 2,
}
)
self.create_service(environment, session, k8s_pod_json)
# Deploy session.
self.deploy_env(environment, session)
# Check env status and port availability on k8s master node.
environment = self.get_env(environment)
check_services = {
'com.mirantis.docker.kubernetes.KubernetesCluster': {
'ports': [8080, 22],
'url': 'api/',
'url_port': 8080
}
}
self.deployment_success_check(environment, check_services)
# Get k8s minions' ips and check that correct initial number
# of them was created and k8s api port is available on them.
minions_ips = self.get_k8s_instances(environment)['minions']
self.assertEqual(1, len(minions_ips))
for ip in minions_ips:
self.check_ports_open(ip, [4194])
# Run 'scaleNodesUp' action for k8s minions.
self.run_k8s_action(
environment=environment,
action='scaleNodesUp'
)
self.wait_for_environment_deploy(environment)
# Check that number of minions was increased and
# k8s api port is available on all of them
environment = self.get_env(environment)
minions_ips = self.get_k8s_instances(environment)['minions']
self.assertEqual(2, len(minions_ips))
for ip in minions_ips:
self.check_ports_open(ip, [4194])
# Run 'scaleNodesDown' action for k8s minions.
self.run_k8s_action(
environment=environment,
action='scaleNodesDown'
)
self.wait_for_environment_deploy(environment)
# Check that number of minions was increased and
# k8s api port is available on all of them
environment = self.get_env(environment)
minions_ips = self.get_k8s_instances(environment)['minions']
self.assertEqual(1, len(minions_ips))
for ip in minions_ips:
self.check_ports_open(ip, [4194])

20
tox.ini Normal file
View File

@ -0,0 +1,20 @@
[tox]
minversion = 1.6
skipsdist = True
skip_missing_interpreters = True
[testenv]
setenv = VIRTUAL_ENV={envdir}
LANG=en_US.UTF-8
LANGUAGE=en_US:en
LC_ALL=C
passenv = OS_* MURANO* *ENDPOINT*
deps=
-r{toxinidir}/test-requirements.txt
distribute = false
[testenv:venv]
commands = {posargs:}
[testenv:deploy_scale_k8s]
commands = python -m unittest tests.test_k8s_app.MuranoK8sTest.test_deploy_scale_k8s