Merge "Tests to cover create mirror"
This commit is contained in:
commit
16b011dd5c
|
@ -173,6 +173,86 @@ Test Ironic
|
||||||
:members:
|
:members:
|
||||||
|
|
||||||
|
|
||||||
|
Fuel mirror verification
|
||||||
|
========================
|
||||||
|
|
||||||
|
Tests to check that mirror is created in various scenarios
|
||||||
|
----------------------------------------------------------
|
||||||
|
Fuel create mirror is made to simplify process of mirror creation for our
|
||||||
|
customers who do not have internet access on-site. It is rewritten from bash
|
||||||
|
to python.
|
||||||
|
|
||||||
|
Fuel create mirror features:
|
||||||
|
|
||||||
|
1) Minimize size of packages in a mirror;
|
||||||
|
|
||||||
|
2) Download packages in parallel.
|
||||||
|
|
||||||
|
Such features can cause some problems:
|
||||||
|
|
||||||
|
1) During packages resolving to minimize mirror size we found such issues:
|
||||||
|
|
||||||
|
1.1) Incorrect versions. When we have multiple mirrors, some version can be
|
||||||
|
skipped due to name duplication. But it is still needed by bootstrap/deploy.
|
||||||
|
|
||||||
|
1.2) Mirror/version collisions. Sometimes package present in number of mirrors
|
||||||
|
and not always correct version corresponds to correct site.
|
||||||
|
|
||||||
|
1.3) There are special mirror on Fuel iso, which differs from
|
||||||
|
http://mirror.fuel-infra.org/ .
|
||||||
|
|
||||||
|
2) With concurrent packages fetching complications are:
|
||||||
|
|
||||||
|
2.1) Some mirrors are unable to support download in multiple threads and fail
|
||||||
|
or reject to support concurrency. In such cases we are abandoning concurrent
|
||||||
|
downloads on such mirrors.
|
||||||
|
|
||||||
|
2.2) Common concurrency pitfalls: race conditions for resources like lists to
|
||||||
|
process.
|
||||||
|
|
||||||
|
2.3) Problems with offset based downloads. Some packages were broken and it had
|
||||||
|
been found out only during package installation.
|
||||||
|
|
||||||
|
.. automodule:: fuelweb_test.tests.tests_mirrors.test_create_mirror
|
||||||
|
:members:
|
||||||
|
|
||||||
|
Tests to verify installation from packages mirrors
|
||||||
|
--------------------------------------------------
|
||||||
|
After mirror is created we should be able to deploy environment with it.
|
||||||
|
|
||||||
|
Fuel-mirror updates default repo urls for deployment and we do not have to
|
||||||
|
set them up for new environments.But be careful. If you want to deploy
|
||||||
|
environments with vanilla mirrors from iso, You should update settings in
|
||||||
|
environment. Currently there is no option to update default mirrors from
|
||||||
|
UI/cli.
|
||||||
|
|
||||||
|
Fuel-mirror updates repo list with internal structures:
|
||||||
|
https://github.com/bgaifullin/packetary/blob/packetary3/contrib/fuel_mirror/fuel_mirror/commands/create.py#L224-L243
|
||||||
|
|
||||||
|
Repository should be able to do two things:
|
||||||
|
|
||||||
|
1) Create bootstrap iso for provisioning;
|
||||||
|
|
||||||
|
2) Provide packages for deployment. Packages from dependencies in http://mirror.fuel-infra.org/ do not cover all the needed packages.
|
||||||
|
So we need to mix in list of required packages:
|
||||||
|
https://github.com/bgaifullin/packetary/blob/packetary3/contrib/fuel_mirror/etc/config.yaml#L46-L96
|
||||||
|
|
||||||
|
Problems:
|
||||||
|
|
||||||
|
1) We need to install not only 'depends', but also 'recommends' packages:
|
||||||
|
https://wiki.ubuntu.com/LucidLynx/ReleaseNotes/#Recommended_packages_installed_by_default
|
||||||
|
http://askubuntu.com/questions/18545/installing-suggested-recommended-packages
|
||||||
|
|
||||||
|
2) We have a problem with support of a custom packages list.
|
||||||
|
It is only tracked via system test failure without exact team assigned for a
|
||||||
|
job. Also debootstrap and other tools are not informative about package errors.
|
||||||
|
It may fail with 'unable to mount', '/proc not mounted', 'file not found' even
|
||||||
|
if a problem is a missing package.
|
||||||
|
|
||||||
|
.. automodule:: fuelweb_test.tests.tests_mirrors.test_use_mirror
|
||||||
|
:members:
|
||||||
|
|
||||||
|
|
||||||
GD based tests
|
GD based tests
|
||||||
==============
|
==============
|
||||||
|
|
||||||
|
|
|
@ -125,6 +125,8 @@ def import_tests():
|
||||||
from tests import test_node_reinstallation # noqa
|
from tests import test_node_reinstallation # noqa
|
||||||
from tests import test_ubuntu_bootstrap # noqa
|
from tests import test_ubuntu_bootstrap # noqa
|
||||||
from tests import test_net_templates # noqa
|
from tests import test_net_templates # noqa
|
||||||
|
from tests.tests_mirrors import test_create_mirror # noqa
|
||||||
|
from tests.tests_mirrors import test_use_mirror # noqa
|
||||||
from system_test.tests import test_create_deploy_ostf # noqa
|
from system_test.tests import test_create_deploy_ostf # noqa
|
||||||
from system_test.tests import test_deploy_check_rados # noqa
|
from system_test.tests import test_deploy_check_rados # noqa
|
||||||
from system_test.tests.strength import destroy_controllers # noqa
|
from system_test.tests.strength import destroy_controllers # noqa
|
||||||
|
|
|
@ -0,0 +1,78 @@
|
||||||
|
# Copyright 2015 Mirantis, Inc.
|
||||||
|
#
|
||||||
|
# Licensed under the Apache License, Version 2.0 (the "License"); you may
|
||||||
|
# not use this file except in compliance with the License. You may obtain
|
||||||
|
# a copy of the License at
|
||||||
|
#
|
||||||
|
# http://www.apache.org/licenses/LICENSE-2.0
|
||||||
|
#
|
||||||
|
# Unless required by applicable law or agreed to in writing, software
|
||||||
|
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
|
||||||
|
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
|
||||||
|
# License for the specific language governing permissions and limitations
|
||||||
|
# under the License.
|
||||||
|
|
||||||
|
from proboscis import test
|
||||||
|
from proboscis import SkipTest
|
||||||
|
|
||||||
|
from fuelweb_test import logger
|
||||||
|
from fuelweb_test.tests.base_test_case import SetupEnvironment
|
||||||
|
from fuelweb_test.tests.base_test_case import TestBasic
|
||||||
|
from fuelweb_test.helpers.utils import run_on_remote
|
||||||
|
|
||||||
|
|
||||||
|
@test(groups=['fuel-mirror'])
|
||||||
|
class TestCreateMirror(TestBasic):
|
||||||
|
"""Tests to check on CI that create mirror functionality is working.
|
||||||
|
|
||||||
|
That is a part of functional testing. Integration testing is done in
|
||||||
|
group 'use-mirror'.
|
||||||
|
Tests are run on subset of mirror to speed up tests.
|
||||||
|
Tests should be run on prepared OS snapshot as it only checks packaging
|
||||||
|
subsystem of distribution.
|
||||||
|
|
||||||
|
Tests are checking that user can install package with no dependencies to
|
||||||
|
ensure that most trivial case working.
|
||||||
|
Then user need to install package with dependency without dependencies.
|
||||||
|
At last we need to install package with multiple dependencies.
|
||||||
|
|
||||||
|
Seems that best way is not to hard code packages, but to fetch them with
|
||||||
|
python debian/rpm package and prepare indexes with it.
|
||||||
|
|
||||||
|
Also we need to check script download behaviour with connectivity issues.
|
||||||
|
|
||||||
|
Code should support rpms and debs in DRY manner.
|
||||||
|
Code should be maintainable for future versions (no hardcoded mirror paths)
|
||||||
|
"""
|
||||||
|
|
||||||
|
@test(groups=['fuel-mirror'],
|
||||||
|
depends_on=[SetupEnvironment.setup_master])
|
||||||
|
def prepare_mirrors_environment(self):
|
||||||
|
# TODO(akostrikov) Create the same Dockerfile for centos 6.5?
|
||||||
|
# TODO(akostrikov) Test yum.
|
||||||
|
snapshot_name = 'prepare_mirrors_environment'
|
||||||
|
self.check_run(snapshot_name)
|
||||||
|
self.env.revert_snapshot('empty')
|
||||||
|
logger.info('Prepare environment for mirror checks.')
|
||||||
|
with self.env.d_env.get_admin_remote() as remote:
|
||||||
|
run_on_remote(remote, 'docker pull ubuntu')
|
||||||
|
run_on_remote(remote, 'docker pull nginx')
|
||||||
|
# TODO(akostrikov) add check that images are present.
|
||||||
|
self.env.make_snapshot(snapshot_name, is_make=True)
|
||||||
|
|
||||||
|
@test(groups=['fuel-mirror', 'create-mirror'],
|
||||||
|
depends_on=[prepare_mirrors_environment])
|
||||||
|
def no_dependencies_package_install(self):
|
||||||
|
# TODO(akostrikov) Run in ubuntu docker image 'create mirror'
|
||||||
|
# and try to apt-get update
|
||||||
|
raise SkipTest('Not implemented yet')
|
||||||
|
|
||||||
|
@test(groups=['fuel-mirror', 'create-mirror'])
|
||||||
|
def check_download_with_network_issues(self):
|
||||||
|
# TODO(akostrikov) Wait for https://review.openstack.org/#/c/242533/
|
||||||
|
raise SkipTest('Not implemented yet')
|
||||||
|
|
||||||
|
@test(groups=['fuel-mirror', 'create-mirror'])
|
||||||
|
def check_download_with_proxy(self):
|
||||||
|
# TODO(akostrikov) Wait for https://review.openstack.org/#/c/242533/
|
||||||
|
raise SkipTest('Not implemented yet')
|
|
@ -0,0 +1,153 @@
|
||||||
|
# Copyright 2015 Mirantis, Inc.
|
||||||
|
#
|
||||||
|
# Licensed under the Apache License, Version 2.0 (the "License"); you may
|
||||||
|
# not use this file except in compliance with the License. You may obtain
|
||||||
|
# a copy of the License at
|
||||||
|
#
|
||||||
|
# http://www.apache.org/licenses/LICENSE-2.0
|
||||||
|
#
|
||||||
|
# Unless required by applicable law or agreed to in writing, software
|
||||||
|
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
|
||||||
|
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
|
||||||
|
# License for the specific language governing permissions and limitations
|
||||||
|
# under the License.
|
||||||
|
|
||||||
|
from proboscis import test
|
||||||
|
from proboscis import SkipTest
|
||||||
|
|
||||||
|
from fuelweb_test.settings import DEPLOYMENT_MODE
|
||||||
|
from fuelweb_test.settings import NEUTRON_SEGMENT
|
||||||
|
from fuelweb_test.tests.base_test_case import SetupEnvironment
|
||||||
|
from fuelweb_test.tests.base_test_case import TestBasic
|
||||||
|
from fuelweb_test.helpers.utils import run_on_remote
|
||||||
|
|
||||||
|
|
||||||
|
@test(groups=['fuel-mirror'])
|
||||||
|
class TestUseMirror(TestBasic):
|
||||||
|
"""Tests custom mirrors to deploy environment.
|
||||||
|
|
||||||
|
Full documentation is in fuel-qa-docs in /doc folder of fuel-qa. It is
|
||||||
|
autogenerated and can be found by keyword 'mirror'.
|
||||||
|
|
||||||
|
This test doesn't only checks create mirror utility but also state of our
|
||||||
|
mirrors. Most probable problem is absence of packet. It is possible that
|
||||||
|
OS now requires new package for bootstrap, or puppet has new dependency
|
||||||
|
that is not reflected in our mirror.
|
||||||
|
"""
|
||||||
|
|
||||||
|
@test(groups=['fuel-mirror', 'use-mirror'],
|
||||||
|
depends_on=[SetupEnvironment.prepare_slaves_5])
|
||||||
|
def deploy_with_custom_mirror(self):
|
||||||
|
"""Create mirror for deployment without internet dependencies.
|
||||||
|
|
||||||
|
TODO(akostrikov) wait for good way to package packetary/fuel-mirror.
|
||||||
|
Good package covers steps from 1 to 7.
|
||||||
|
|
||||||
|
Scenario:
|
||||||
|
1. Install packages to build fuel-mirror and packetary
|
||||||
|
2. Download packetary from github
|
||||||
|
3. Checkout to packetary3 version of packetary
|
||||||
|
4. Install packetary and fuel-mirror
|
||||||
|
5. Create configuration directory
|
||||||
|
6. Copy configuration file to configuration directory
|
||||||
|
7. Update config file with real master ip.
|
||||||
|
8. Run create mirror command
|
||||||
|
9. Create cluster with neutron networking
|
||||||
|
10. Add 3 nodes with controller role
|
||||||
|
11. Add 1 node with compute role and 1 node with cinder role
|
||||||
|
12. Run network verification
|
||||||
|
13. Deploy the cluster
|
||||||
|
14. Run OSTF
|
||||||
|
15. Create snapshot
|
||||||
|
|
||||||
|
Duration 90m
|
||||||
|
Snapshot deploy_with_custom_mirror
|
||||||
|
"""
|
||||||
|
self.env.revert_snapshot('ready_with_5_slaves')
|
||||||
|
|
||||||
|
with self.env.d_env.get_admin_remote() as remote:
|
||||||
|
# FIXME(akostrikov) This should be removed with correct install.
|
||||||
|
# All that above is a hack.
|
||||||
|
self.show_step(1)
|
||||||
|
run_on_remote(remote,
|
||||||
|
'yum install git python-lxml.x86_64 '
|
||||||
|
'python-eventlet -y')
|
||||||
|
self.show_step(2)
|
||||||
|
run_on_remote(remote,
|
||||||
|
'cd /opt && rm -rf packetary && '
|
||||||
|
'git clone https://github.com/bgaifullin/packetary')
|
||||||
|
self.show_step(3)
|
||||||
|
run_on_remote(remote,
|
||||||
|
'cd /opt/packetary && git checkout packetary3')
|
||||||
|
self.show_step(4)
|
||||||
|
run_on_remote(remote, 'cd /opt/packetary && pip install -e .')
|
||||||
|
run_on_remote(remote,
|
||||||
|
'cd /opt/packetary/contrib/fuel_mirror/ && '
|
||||||
|
'pip install -e .')
|
||||||
|
self.show_step(5)
|
||||||
|
run_on_remote(remote, 'mkdir -p /etc/fuel-mirror/')
|
||||||
|
self.show_step(6)
|
||||||
|
run_on_remote(remote,
|
||||||
|
'cp /opt/packetary/contrib/fuel_mirror/'
|
||||||
|
'etc/config.yaml /etc/fuel-mirror/config.yaml')
|
||||||
|
self.show_step(7)
|
||||||
|
admin_ip = str(
|
||||||
|
self.env.d_env.nodes().admin.get_ip_address_by_network_name(
|
||||||
|
'admin'))
|
||||||
|
cmd = "sed -r 's/{prev_ip}'/{admin_ip}/ -i'' {config_path}".format(
|
||||||
|
prev_ip='10.20.0.2',
|
||||||
|
admin_ip=admin_ip,
|
||||||
|
config_path='/etc/fuel-mirror/config.yaml'
|
||||||
|
)
|
||||||
|
run_on_remote(remote, cmd)
|
||||||
|
self.show_step(8)
|
||||||
|
run_on_remote(remote, 'fuel-mirror create --ubuntu')
|
||||||
|
|
||||||
|
self.show_step(9)
|
||||||
|
|
||||||
|
cluster_id = self.fuel_web.create_cluster(
|
||||||
|
name=self.__class__.__name__,
|
||||||
|
mode=DEPLOYMENT_MODE,
|
||||||
|
settings={
|
||||||
|
"net_provider": 'neutron',
|
||||||
|
"net_segment_type": NEUTRON_SEGMENT['tun'],
|
||||||
|
'tenant': 'packetary',
|
||||||
|
'user': 'packetary',
|
||||||
|
'password': 'packetary'
|
||||||
|
}
|
||||||
|
)
|
||||||
|
self.show_step(10)
|
||||||
|
self.show_step(11)
|
||||||
|
self.fuel_web.update_nodes(
|
||||||
|
cluster_id,
|
||||||
|
{
|
||||||
|
'slave-01': ['controller'],
|
||||||
|
'slave-02': ['controller'],
|
||||||
|
'slave-03': ['controller'],
|
||||||
|
'slave-04': ['compute'],
|
||||||
|
'slave-05': ['cinder']
|
||||||
|
}
|
||||||
|
)
|
||||||
|
self.show_step(12)
|
||||||
|
self.fuel_web.verify_network(cluster_id)
|
||||||
|
self.show_step(13)
|
||||||
|
self.fuel_web.deploy_cluster_wait(cluster_id)
|
||||||
|
|
||||||
|
self.show_step(14)
|
||||||
|
self.fuel_web.run_ostf(
|
||||||
|
cluster_id=cluster_id,
|
||||||
|
test_sets=['ha', 'smoke', 'sanity'])
|
||||||
|
|
||||||
|
self.env.make_snapshot('deploy_with_custom_mirror')
|
||||||
|
|
||||||
|
@test(groups=['fuel-mirror', 'use-mirror'])
|
||||||
|
def deploy_no_official_access(self):
|
||||||
|
# TODO(akostrikov) add firewall rules to verify that there is no
|
||||||
|
# connection to official mirrors during mirror creation and deployment.
|
||||||
|
raise SkipTest('Not implemented yet')
|
||||||
|
|
||||||
|
@test(groups=['fuel-mirror', 'use-mirror'])
|
||||||
|
def deploy_with_proxy(self):
|
||||||
|
# TODO(akostrikov) add tests to verify that fuel-mirror works with
|
||||||
|
# proxies too.
|
||||||
|
raise SkipTest('Not implemented yet')
|
Loading…
Reference in New Issue