diff --git a/fuel_ccp_tests/fixtures/ccp_fixtures.py b/fuel_ccp_tests/fixtures/ccp_fixtures.py index 5457e90..a314626 100644 --- a/fuel_ccp_tests/fixtures/ccp_fixtures.py +++ b/fuel_ccp_tests/fixtures/ccp_fixtures.py @@ -12,11 +12,13 @@ # License for the specific language governing permissions and limitations # under the License. import pytest - +from fuel_ccp_tests import logger from fuel_ccp_tests.helpers import ext from fuel_ccp_tests.managers import ccpmanager from fuel_ccp_tests import settings +LOG = logger.logger + @pytest.fixture(scope='function') def ccp_actions(config, underlay): @@ -86,7 +88,8 @@ def ccpcluster(revert_snapshot, config, hardware, ccp_actions.init_default_config(include_files=[ settings.CCP_DEPLOY_CONFIG, settings.CCP_SOURCES_CONFIG, - settings.CCP_DEPLOY_TOPOLOGY]) + settings.CCP_DEPLOY_TOPOLOGY, + settings.CCP_FETCH_CONFIG]) config.ccp.os_host = config.k8s.kube_host hardware.create_snapshot(ext.SNAPSHOT.ccp_deployed) @@ -102,3 +105,77 @@ def ccpcluster(revert_snapshot, config, hardware, pass return ccp_actions + + +@pytest.yield_fixture(scope='function') +def git_server_mock(config, underlay): + ''' + Instantiate a mock ssh git server. Server host repositories in the /git + project. Git project mounted to /docker_data/openstack on host machine + Setup password-less access to repo for vagrant user from kube_host + Populate with default openstack repositories + ''' + cmd_git_setup = 'docker run -d -p 2222:22 -p 3333:80 ' \ + '-v /docker_data/openstack:/git unixtastic/git-ssh-server' + LOG.info("Instantiating ssh-git-server mock instance") + ssh_server_docker_container_id = underlay.sudo_check_call( + cmd_git_setup, + host=config.k8s.kube_host)['stdout'][-1].strip() + LOG.info("Started ssh-git-server in {} container".format( + ssh_server_docker_container_id)) + + cmd_config_permisions = [ + 'ssh-keygen -b 2048 -t rsa -f ~/.ssh/id_rsa_git -q -N "" ', + 'echo \"{}\" >> ~/.ssh/config'.format( + 'IdentityFile ~/.ssh/id_rsa_git'), + 'echo \"{}\" >> ~/.ssh/config'.format( + 'HashKnownHosts no'), + 'echo \"{}\" >> ~/.ssh/config'.format( + 'StrictHostKeyChecking no')] + + cmd_config_permisions_sudo = \ + ['cd /docker_data/openstack', + 'mkdir /docker_data/openstack/.ssh', + 'chmod -R 700 /docker_data/openstack/.ssh', + 'touch /docker_data/openstack/.ssh/authorized_keys', + 'chmod 600 /docker_data/openstack/.ssh/authorized_keys', + 'cat ~/.ssh/id_rsa_git.pub >> ' + '/docker_data/openstack/.ssh/authorized_keys', + 'touch /docker_data/openstack/.hushlogin', + 'docker exec {} sed -i \'$ a {}\' /etc/ssh/sshd_config'.format( + ssh_server_docker_container_id, + 'AuthorizedKeysFile /git/.ssh/authorized_keys') + ] + LOG.info("Configuring public keys and permissions...") + for cmd in cmd_config_permisions: + underlay.check_call(cmd, + host=config.k8s.kube_host, expected=[0]) + for cmd in cmd_config_permisions_sudo: + underlay.sudo_check_call(cmd, + host=config.k8s.kube_host, expected=[0]) + + LOG.info("Configuring public keys and permissions completed") + + LOG.info("Cloning all default repos from openstack public repo") + for project_repository in ext.DEFAULT_REPOS: + LOG.info('Cloning {}...'.format(project_repository)) + underlay.sudo_check_call( + 'git clone --mirror {}/{}' + ' /docker_data/openstack/{}'.format( + settings.FUEL_CCP_ORIGIN_URL, + project_repository, project_repository), + host=config.k8s.kube_host, expected=[0]) + for cmd in ['docker exec {} /etc/init.d/ssh restart'.format( + ssh_server_docker_container_id), + 'docker exec {} chown -R git:git /git'.format( + ssh_server_docker_container_id)]: + underlay.sudo_check_call( + cmd, + host=config.k8s.kube_host, + expected=[0]) + yield ssh_server_docker_container_id + cmd_git_teardown = 'docker stop {ssh_git_id} && ' \ + 'docker rm {ssh_git_id}'\ + .format(ssh_git_id=ssh_server_docker_container_id) + LOG.info("Stoping and removing ssh-git-server mock instance...") + underlay.sudo_check_call(cmd_git_teardown, host=config.k8s.kube_host) diff --git a/fuel_ccp_tests/helpers/ext.py b/fuel_ccp_tests/helpers/ext.py index ae28790..a32fde3 100644 --- a/fuel_ccp_tests/helpers/ext.py +++ b/fuel_ccp_tests/helpers/ext.py @@ -84,3 +84,24 @@ class HttpCodes(enumerate): class Namespace(enumerate): BASE_NAMESPACE = 'ccp' + +DEFAULT_REPOS = ['fuel-ccp-debian-base', + 'fuel-ccp-entrypoint', + 'fuel-ccp-etcd', + 'fuel-ccp-glance', + 'fuel-ccp-heat', + 'fuel-ccp-horizon', + 'fuel-ccp-keystone', + 'fuel-ccp-mariadb', + 'fuel-ccp-memcached', + 'fuel-ccp-neutron', + 'fuel-ccp-nova', + 'fuel-ccp-openstack-base', + 'fuel-ccp-rabbitmq', + 'fuel-ccp-stacklight', + 'fuel-ccp-murano', + 'fuel-ccp-ironic', + 'fuel-ccp-cinder', + 'fuel-ccp-searchlight', + 'fuel-ccp-sahara' + ] diff --git a/fuel_ccp_tests/managers/ccpmanager.py b/fuel_ccp_tests/managers/ccpmanager.py index ce27c98..04b8fa8 100644 --- a/fuel_ccp_tests/managers/ccpmanager.py +++ b/fuel_ccp_tests/managers/ccpmanager.py @@ -11,6 +11,7 @@ # WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the # License for the specific language governing permissions and limitations # under the License. + import yaml from devops.error import DevopsCalledProcessError @@ -63,8 +64,10 @@ class CCPManager(object): self._default_params = v.copy() def init_default_config(self, include_files=None): - self.put_yaml_config('~/.ccp.yaml', settings.CCP_CONF) - self.add_includes('~/.ccp.yaml', include_files) + self.put_yaml_config(settings.CCP_CLI_PARAMS["config-file"], + settings.CCP_CONF) + self.add_includes(settings.CCP_CLI_PARAMS["config-file"], + include_files) def add_includes(self, path, files): def clear_tilda(p): @@ -134,34 +137,54 @@ class CCPManager(object): return ' '.join(["--{}={}".format( k, v) if v else "--{}".format(k) for (k, v) in params.items()]) - def run(self, cmd, components=None, params=None, suppress_output=False): - params = self.__build_param_string(params) - params = params or '' - if components: - if isinstance(components, str): - components = [components] - components = '-c {}'.format(' '.join(components)) - else: - components = '' + def run(self, cmd, components=None, params=None, + use_cli_params=False, suppress_output=False, + raise_on_err=True, error_info=None, + expected=None): + if suppress_output: ccp_out_redirect = ("> >(tee ccp.out.log > /dev/null) " "2> >(tee ccp.err.log >/dev/null)") else: ccp_out_redirect = "" + if use_cli_params is True: + params = self.__build_param_string(params) + params = params or '' + if components: + if isinstance(components, str): + components = [components] + components = '-c {}'.format(' '.join(components)) + else: + components = '' - cmd = "ccp {params} {cmd} {components} {ccp_out_redirect}".format( - params=params, cmd=cmd, components=components, - ccp_out_redirect=ccp_out_redirect) + cmd = "ccp {params} {cmd} {components} {ccp_out_redirect}".format( + params=params, cmd=cmd, components=components, + ccp_out_redirect=ccp_out_redirect) + else: + cmd = "ccp {cmd} {ccp_out_redirect}".format( + cmd=cmd, ccp_out_redirect=ccp_out_redirect) LOG.info("Running {cmd}".format(cmd=cmd)) with self.__underlay.remote( host=self.__config.k8s.kube_host) as remote: - remote.check_call(cmd) + res = remote.check_call(cmd, + raise_on_err=raise_on_err, + error_info=error_info, + expected=expected, + verbose=True) + return res - def fetch(self, components=None, params=None): - self.run('fetch', - components=components, - params=params) + def fetch(self, + params=None, + raise_on_err=True, + error_info=None, + expected=None): + # build config file + self.put_yaml_config(settings.CCP_FETCH_CONFIG, params) + return self.run('fetch', + raise_on_err=raise_on_err, + error_info=error_info, + expected=expected) def build(self, components=None, params=None, suppress_output=True): try: diff --git a/fuel_ccp_tests/settings.py b/fuel_ccp_tests/settings.py index 2da483f..6e27e3c 100644 --- a/fuel_ccp_tests/settings.py +++ b/fuel_ccp_tests/settings.py @@ -122,6 +122,9 @@ TOPOLOGY_PATH = os.environ.get('TOPOLOGY_PATH', FUEL_CCP_KEYSTONE_LOCAL_REPO = os.environ.get('FUEL_CCP_KEYSTONE_LOCAL_REPO', None) +FUEL_CCP_ORIGIN_URL = os.environ.get( + 'FUEL_CCP_ORIGIN_URL', + 'https://git.openstack.org:443/openstack/') OS_RELEASE = os.environ.get('OS_RELEASE', 'stable/newton') OS_REPOS = { @@ -176,6 +179,9 @@ CCP_CONF = { } CCP_SOURCES_CONFIG = '~/.ccp.build-sources.yaml' + +CCP_FETCH_CONFIG = '~/.ccp.fetch.yaml' + CCP_BUILD_SOURCES = { 'sources': OS_REPOS } @@ -200,6 +206,10 @@ CCP_DEFAULT_GLOBALS = { } } +CCP_ENVIRONMENT_PARAMS = { + "microservices_home": "$HOME/ccp-repos" +} + NETCHECKER_SERVER_DIR = os.environ.get( 'NETCHECKER_SERVER_DIR', os.path.join(os.getcwd(), 'mcp-netchecker-server') ) diff --git a/fuel_ccp_tests/tests/component/ccp/test_ccp_fetch.py b/fuel_ccp_tests/tests/component/ccp/test_ccp_fetch.py new file mode 100644 index 0000000..f986f5c --- /dev/null +++ b/fuel_ccp_tests/tests/component/ccp/test_ccp_fetch.py @@ -0,0 +1,268 @@ +# Copyright 2016 Mirantis, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. +import re + +import pytest + +from fuel_ccp_tests.helpers import ext +from fuel_ccp_tests import settings + + +@pytest.mark.revert_snapshot(ext.SNAPSHOT.k8s_deployed) +class TestCppFetch(object): + """Check ccp fetch feature + """ + def test_ccp_fetch_default(self, config, underlay, ccpcluster, show_step): + """Test for ccp Fetch all openstack repositories for master by http + using fuel-ccp tool + + Scenario: + 1. exec ccp fetch with config repositories: protocol: http; port: 80 + 2. Verify that downloaded repos match to default repos list + """ + config_fetch = { + 'repositories': { + 'path': '{}'.format(settings.CCP_ENVIRONMENT_PARAMS[ + "microservices_home"]), + 'protocol': 'http', + 'port': 80 + } + } + + show_step(1) + ccpcluster.fetch(params=config_fetch) + + show_step(2) + cmd_fetched = \ + ('ls -d1 {}/* | xargs -n 1 basename' + .format(settings.CCP_ENVIRONMENT_PARAMS["microservices_home"])) + res = [s.strip() for s in underlay.check_call( + cmd_fetched, + verbose=True, + host=config.k8s.kube_host)['stdout']] + assert set(ext.DEFAULT_REPOS) == set(res), \ + 'Unexpected repo\'s downloaded' \ + ' Expected count {}, actual {}'.format( + len(ext.DEFAULT_REPOS), len(res)) + + def test_ccp_fetch_some_via_ssh(self, config, underlay, ccpcluster, + git_server_mock, show_step): + """Test for ccp Fetch selected openstack repositories for master by ssh + using fuel-ccp tool + Precondition: + ccp config: + protocol: 'ssh' + username: 'git' + port: 2222 + hostname: 'localhost' + project: git + names: + - fuel-ccp-debian-base + - fuel-ccp-entrypoint + + Scenario: + 1. exec ccp fetch with config + 2. Verify that downloaded repos match to fuel-ccp-debian-base + and fuel-ccp-entrypointrepos list + 3. Verify that downloaded images from specified repository + """ + expected = ['fuel-ccp-debian-base', 'fuel-ccp-entrypoint'] + + config_fetch = { + 'repositories': { + 'path': '{}'.format(settings.CCP_ENVIRONMENT_PARAMS[ + "microservices_home"]), + 'protocol': 'ssh', + 'username': 'git', + 'port': 2222, + 'hostname': 'localhost', + 'project': 'git', + 'names': [ + 'fuel-ccp-debian-base', + 'fuel-ccp-entrypoint'] + } + } + + show_step(1) + ccpcluster.fetch(params=config_fetch) + + show_step(2) + cmd_fetched = \ + ('ls -d1 {}/* | xargs -n 1 basename' + .format(settings.CCP_ENVIRONMENT_PARAMS["microservices_home"])) + res = [s.strip() for s in underlay.check_call( + cmd_fetched, + verbose=True, + host=config.k8s.kube_host)['stdout']] + assert set(expected) == set(res), 'Unexpected repo\'s downloaded' \ + ' Expected count {}, actual {}' \ + .format(len(expected), len(res)) + + show_step(3) + cmd_origin = ( + "cd {}/{} &&".format(settings.CCP_ENVIRONMENT_PARAMS[ + "microservices_home"], expected[0]) + + "git remote show origin | grep ssh://git@localhost:2222") + underlay.check_call(cmd_origin, verbose=True, + host=config.k8s.kube_host), \ + 'Downloaded images are not from specified repository' + + def test_ccp_fetch_nonexisting_repo(self, config, underlay, ccpcluster, + show_step): + """Fetch nonexistent repo using fuel-ccp tool + Precondition: + ccp config: + repositories: + path: $microservices-home + fuel_ccp_debian_base: fuel-ccp-not-exist + + Scenario: + 1. exec ccp fetch with config + 2. Verify fatal: repository 'fuel-ccp-not-exist' not exists" + 3. Verify no traceback printed for known negative case + and exit code not 0 + 4. Verify that no repositories fetched + """ + exit_code = 1 + check_code_message = \ + 'Unexpected exit code. Expected {}.'.format(exit_code)\ + + "Defect https://bugs.launchpad.net/fuel-ccp/+bug/1608981" + err_message = \ + 'fatal: repository \'fuel-ccp-not-exist\' does not exist' + + config_fetch = { + 'repositories': { + 'path': '{}'.format(settings.CCP_ENVIRONMENT_PARAMS[ + "microservices_home"]), + 'fuel_ccp_debian_base': 'fuel-ccp-not-exist' + } + } + + show_step(1) + res = ccpcluster.fetch(params=config_fetch, + expected=[exit_code], + error_info=check_code_message) + + show_step(2) + assert err_message in res['stderr_str'] + + show_step(3) + assert 'Traceback' not in res['stderr_str'], ( + 'traceback printed' + 'Defect https://bugs.launchpad.net/fuel-ccp/+bug/1620354') + + show_step(4) + cmd_fetched = \ + ('ls -d1 {}/' + .format(settings.CCP_ENVIRONMENT_PARAMS["microservices_home"])) + error_info = \ + '{} - directory exist, but should not'.format( + settings.CCP_ENVIRONMENT_PARAMS["microservices_home"]) + underlay.check_call(cmd_fetched, expected=[2], verbose=True, + host=config.k8s.kube_host, error_info=error_info) + + def test_ccp_fetch_incorrect_url(self, config, underlay, ccpcluster, + show_step): + """Fetch repo by incorrect url using ccp tool + Precondition: + ccp config: + repositories: + path: $microservices-home + hostname: example.org + + Scenario: + 1. exec ccp fetch with config + 2. Verify fatal: repository 'http://example.org/' not found" + 3. Verify no traceback printed for known negative case + and exit code not 0 + 4. Verify that no repositories fetched + """ + exit_code = 2 + err_message = 'fatal: repository \'.*\' not found' + config_fetch = { + 'repositories': { + 'path': '{}'.format(settings.CCP_ENVIRONMENT_PARAMS[ + "microservices_home"]), + 'hostname': 'example.org'}} + check_code_message = \ + 'Unexpected exit code. Expected {}.'.format(exit_code)\ + + "https://bugs.launchpad.net/fuel-ccp/+bug/1608981" + + show_step(1) + res = ccpcluster.fetch( + params=config_fetch, + expected=[exit_code], + error_info=check_code_message) + + show_step(2) + assert re.search( + err_message, + "".join(res['stderr_str']), flags=re.MULTILINE) is not None + + show_step(3) + assert 'Traceback' not in res['stderr_str'], ( + 'traceback printed' + 'Defect https://bugs.launchpad.net/fuel-ccp/+bug/1608981') + + show_step(4) + cmd_fetched = \ + ('ls -d1 {}/*'.format( + settings.CCP_ENVIRONMENT_PARAMS["microservices_home"])) + underlay.check_call( + cmd_fetched, expected=[2], + verbose=True, + host=config.k8s.kube_host) + + def test_ccp_fetch_incorrect_schema(self, config, underlay, ccpcluster, + show_step): + """Fetch repo by incorrect schema using ccp tool + Precondition: + ccp config: + repositories: + path: $microservices-home + fuel_ccp_debian_base: htt://example.org + + Scenario: + 1. exec ccp fetch with config + 2. Verify fatal: Unable to find remote helper for \'htt\' + 3. Verify no traceback printed for known negative case + 4. Verify that no repositories fetched + """ + err_message = 'fatal: Unable to find remote helper for \'htt\'' + + config_fetch = { + 'repositories': { + 'path': '{}'.format(settings.CCP_ENVIRONMENT_PARAMS[ + "microservices_home"]), + 'fuel_ccp_debian_base': 'htt://example.org'}} + + show_step(1) + res = ccpcluster.fetch(params=config_fetch) + + show_step(2) + assert err_message in res['stderr_str'] + + show_step(3) + assert 'Traceback' not in res['stderr_str'], ( + 'traceback printed ' + 'Defect https://bugs.launchpad.net/fuel-ccp/+bug/1620361') + + show_step(4) + cmd_fetched = \ + ('ls -d1 {}/fuel-ccp-debian-base' + .format(settings.CCP_ENVIRONMENT_PARAMS["microservices_home"])) + underlay.check_call( + cmd_fetched, expected=[2], + verbose=True, + host=config.k8s.kube_host)