Merge "Tests to cover create mirror"
This commit is contained in:
commit
16b011dd5c
|
@ -173,6 +173,86 @@ Test Ironic
|
|||
:members:
|
||||
|
||||
|
||||
Fuel mirror verification
|
||||
========================
|
||||
|
||||
Tests to check that mirror is created in various scenarios
|
||||
----------------------------------------------------------
|
||||
Fuel create mirror is made to simplify process of mirror creation for our
|
||||
customers who do not have internet access on-site. It is rewritten from bash
|
||||
to python.
|
||||
|
||||
Fuel create mirror features:
|
||||
|
||||
1) Minimize size of packages in a mirror;
|
||||
|
||||
2) Download packages in parallel.
|
||||
|
||||
Such features can cause some problems:
|
||||
|
||||
1) During packages resolving to minimize mirror size we found such issues:
|
||||
|
||||
1.1) Incorrect versions. When we have multiple mirrors, some version can be
|
||||
skipped due to name duplication. But it is still needed by bootstrap/deploy.
|
||||
|
||||
1.2) Mirror/version collisions. Sometimes package present in number of mirrors
|
||||
and not always correct version corresponds to correct site.
|
||||
|
||||
1.3) There are special mirror on Fuel iso, which differs from
|
||||
http://mirror.fuel-infra.org/ .
|
||||
|
||||
2) With concurrent packages fetching complications are:
|
||||
|
||||
2.1) Some mirrors are unable to support download in multiple threads and fail
|
||||
or reject to support concurrency. In such cases we are abandoning concurrent
|
||||
downloads on such mirrors.
|
||||
|
||||
2.2) Common concurrency pitfalls: race conditions for resources like lists to
|
||||
process.
|
||||
|
||||
2.3) Problems with offset based downloads. Some packages were broken and it had
|
||||
been found out only during package installation.
|
||||
|
||||
.. automodule:: fuelweb_test.tests.tests_mirrors.test_create_mirror
|
||||
:members:
|
||||
|
||||
Tests to verify installation from packages mirrors
|
||||
--------------------------------------------------
|
||||
After mirror is created we should be able to deploy environment with it.
|
||||
|
||||
Fuel-mirror updates default repo urls for deployment and we do not have to
|
||||
set them up for new environments.But be careful. If you want to deploy
|
||||
environments with vanilla mirrors from iso, You should update settings in
|
||||
environment. Currently there is no option to update default mirrors from
|
||||
UI/cli.
|
||||
|
||||
Fuel-mirror updates repo list with internal structures:
|
||||
https://github.com/bgaifullin/packetary/blob/packetary3/contrib/fuel_mirror/fuel_mirror/commands/create.py#L224-L243
|
||||
|
||||
Repository should be able to do two things:
|
||||
|
||||
1) Create bootstrap iso for provisioning;
|
||||
|
||||
2) Provide packages for deployment. Packages from dependencies in http://mirror.fuel-infra.org/ do not cover all the needed packages.
|
||||
So we need to mix in list of required packages:
|
||||
https://github.com/bgaifullin/packetary/blob/packetary3/contrib/fuel_mirror/etc/config.yaml#L46-L96
|
||||
|
||||
Problems:
|
||||
|
||||
1) We need to install not only 'depends', but also 'recommends' packages:
|
||||
https://wiki.ubuntu.com/LucidLynx/ReleaseNotes/#Recommended_packages_installed_by_default
|
||||
http://askubuntu.com/questions/18545/installing-suggested-recommended-packages
|
||||
|
||||
2) We have a problem with support of a custom packages list.
|
||||
It is only tracked via system test failure without exact team assigned for a
|
||||
job. Also debootstrap and other tools are not informative about package errors.
|
||||
It may fail with 'unable to mount', '/proc not mounted', 'file not found' even
|
||||
if a problem is a missing package.
|
||||
|
||||
.. automodule:: fuelweb_test.tests.tests_mirrors.test_use_mirror
|
||||
:members:
|
||||
|
||||
|
||||
GD based tests
|
||||
==============
|
||||
|
||||
|
|
|
@ -125,6 +125,8 @@ def import_tests():
|
|||
from tests import test_node_reinstallation # noqa
|
||||
from tests import test_ubuntu_bootstrap # noqa
|
||||
from tests import test_net_templates # noqa
|
||||
from tests.tests_mirrors import test_create_mirror # noqa
|
||||
from tests.tests_mirrors import test_use_mirror # noqa
|
||||
from system_test.tests import test_create_deploy_ostf # noqa
|
||||
from system_test.tests import test_deploy_check_rados # noqa
|
||||
from system_test.tests.strength import destroy_controllers # noqa
|
||||
|
|
|
@ -0,0 +1,78 @@
|
|||
# Copyright 2015 Mirantis, Inc.
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License"); you may
|
||||
# not use this file except in compliance with the License. You may obtain
|
||||
# a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
|
||||
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
|
||||
# License for the specific language governing permissions and limitations
|
||||
# under the License.
|
||||
|
||||
from proboscis import test
|
||||
from proboscis import SkipTest
|
||||
|
||||
from fuelweb_test import logger
|
||||
from fuelweb_test.tests.base_test_case import SetupEnvironment
|
||||
from fuelweb_test.tests.base_test_case import TestBasic
|
||||
from fuelweb_test.helpers.utils import run_on_remote
|
||||
|
||||
|
||||
@test(groups=['fuel-mirror'])
|
||||
class TestCreateMirror(TestBasic):
|
||||
"""Tests to check on CI that create mirror functionality is working.
|
||||
|
||||
That is a part of functional testing. Integration testing is done in
|
||||
group 'use-mirror'.
|
||||
Tests are run on subset of mirror to speed up tests.
|
||||
Tests should be run on prepared OS snapshot as it only checks packaging
|
||||
subsystem of distribution.
|
||||
|
||||
Tests are checking that user can install package with no dependencies to
|
||||
ensure that most trivial case working.
|
||||
Then user need to install package with dependency without dependencies.
|
||||
At last we need to install package with multiple dependencies.
|
||||
|
||||
Seems that best way is not to hard code packages, but to fetch them with
|
||||
python debian/rpm package and prepare indexes with it.
|
||||
|
||||
Also we need to check script download behaviour with connectivity issues.
|
||||
|
||||
Code should support rpms and debs in DRY manner.
|
||||
Code should be maintainable for future versions (no hardcoded mirror paths)
|
||||
"""
|
||||
|
||||
@test(groups=['fuel-mirror'],
|
||||
depends_on=[SetupEnvironment.setup_master])
|
||||
def prepare_mirrors_environment(self):
|
||||
# TODO(akostrikov) Create the same Dockerfile for centos 6.5?
|
||||
# TODO(akostrikov) Test yum.
|
||||
snapshot_name = 'prepare_mirrors_environment'
|
||||
self.check_run(snapshot_name)
|
||||
self.env.revert_snapshot('empty')
|
||||
logger.info('Prepare environment for mirror checks.')
|
||||
with self.env.d_env.get_admin_remote() as remote:
|
||||
run_on_remote(remote, 'docker pull ubuntu')
|
||||
run_on_remote(remote, 'docker pull nginx')
|
||||
# TODO(akostrikov) add check that images are present.
|
||||
self.env.make_snapshot(snapshot_name, is_make=True)
|
||||
|
||||
@test(groups=['fuel-mirror', 'create-mirror'],
|
||||
depends_on=[prepare_mirrors_environment])
|
||||
def no_dependencies_package_install(self):
|
||||
# TODO(akostrikov) Run in ubuntu docker image 'create mirror'
|
||||
# and try to apt-get update
|
||||
raise SkipTest('Not implemented yet')
|
||||
|
||||
@test(groups=['fuel-mirror', 'create-mirror'])
|
||||
def check_download_with_network_issues(self):
|
||||
# TODO(akostrikov) Wait for https://review.openstack.org/#/c/242533/
|
||||
raise SkipTest('Not implemented yet')
|
||||
|
||||
@test(groups=['fuel-mirror', 'create-mirror'])
|
||||
def check_download_with_proxy(self):
|
||||
# TODO(akostrikov) Wait for https://review.openstack.org/#/c/242533/
|
||||
raise SkipTest('Not implemented yet')
|
|
@ -0,0 +1,153 @@
|
|||
# Copyright 2015 Mirantis, Inc.
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License"); you may
|
||||
# not use this file except in compliance with the License. You may obtain
|
||||
# a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
|
||||
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
|
||||
# License for the specific language governing permissions and limitations
|
||||
# under the License.
|
||||
|
||||
from proboscis import test
|
||||
from proboscis import SkipTest
|
||||
|
||||
from fuelweb_test.settings import DEPLOYMENT_MODE
|
||||
from fuelweb_test.settings import NEUTRON_SEGMENT
|
||||
from fuelweb_test.tests.base_test_case import SetupEnvironment
|
||||
from fuelweb_test.tests.base_test_case import TestBasic
|
||||
from fuelweb_test.helpers.utils import run_on_remote
|
||||
|
||||
|
||||
@test(groups=['fuel-mirror'])
|
||||
class TestUseMirror(TestBasic):
|
||||
"""Tests custom mirrors to deploy environment.
|
||||
|
||||
Full documentation is in fuel-qa-docs in /doc folder of fuel-qa. It is
|
||||
autogenerated and can be found by keyword 'mirror'.
|
||||
|
||||
This test doesn't only checks create mirror utility but also state of our
|
||||
mirrors. Most probable problem is absence of packet. It is possible that
|
||||
OS now requires new package for bootstrap, or puppet has new dependency
|
||||
that is not reflected in our mirror.
|
||||
"""
|
||||
|
||||
@test(groups=['fuel-mirror', 'use-mirror'],
|
||||
depends_on=[SetupEnvironment.prepare_slaves_5])
|
||||
def deploy_with_custom_mirror(self):
|
||||
"""Create mirror for deployment without internet dependencies.
|
||||
|
||||
TODO(akostrikov) wait for good way to package packetary/fuel-mirror.
|
||||
Good package covers steps from 1 to 7.
|
||||
|
||||
Scenario:
|
||||
1. Install packages to build fuel-mirror and packetary
|
||||
2. Download packetary from github
|
||||
3. Checkout to packetary3 version of packetary
|
||||
4. Install packetary and fuel-mirror
|
||||
5. Create configuration directory
|
||||
6. Copy configuration file to configuration directory
|
||||
7. Update config file with real master ip.
|
||||
8. Run create mirror command
|
||||
9. Create cluster with neutron networking
|
||||
10. Add 3 nodes with controller role
|
||||
11. Add 1 node with compute role and 1 node with cinder role
|
||||
12. Run network verification
|
||||
13. Deploy the cluster
|
||||
14. Run OSTF
|
||||
15. Create snapshot
|
||||
|
||||
Duration 90m
|
||||
Snapshot deploy_with_custom_mirror
|
||||
"""
|
||||
self.env.revert_snapshot('ready_with_5_slaves')
|
||||
|
||||
with self.env.d_env.get_admin_remote() as remote:
|
||||
# FIXME(akostrikov) This should be removed with correct install.
|
||||
# All that above is a hack.
|
||||
self.show_step(1)
|
||||
run_on_remote(remote,
|
||||
'yum install git python-lxml.x86_64 '
|
||||
'python-eventlet -y')
|
||||
self.show_step(2)
|
||||
run_on_remote(remote,
|
||||
'cd /opt && rm -rf packetary && '
|
||||
'git clone https://github.com/bgaifullin/packetary')
|
||||
self.show_step(3)
|
||||
run_on_remote(remote,
|
||||
'cd /opt/packetary && git checkout packetary3')
|
||||
self.show_step(4)
|
||||
run_on_remote(remote, 'cd /opt/packetary && pip install -e .')
|
||||
run_on_remote(remote,
|
||||
'cd /opt/packetary/contrib/fuel_mirror/ && '
|
||||
'pip install -e .')
|
||||
self.show_step(5)
|
||||
run_on_remote(remote, 'mkdir -p /etc/fuel-mirror/')
|
||||
self.show_step(6)
|
||||
run_on_remote(remote,
|
||||
'cp /opt/packetary/contrib/fuel_mirror/'
|
||||
'etc/config.yaml /etc/fuel-mirror/config.yaml')
|
||||
self.show_step(7)
|
||||
admin_ip = str(
|
||||
self.env.d_env.nodes().admin.get_ip_address_by_network_name(
|
||||
'admin'))
|
||||
cmd = "sed -r 's/{prev_ip}'/{admin_ip}/ -i'' {config_path}".format(
|
||||
prev_ip='10.20.0.2',
|
||||
admin_ip=admin_ip,
|
||||
config_path='/etc/fuel-mirror/config.yaml'
|
||||
)
|
||||
run_on_remote(remote, cmd)
|
||||
self.show_step(8)
|
||||
run_on_remote(remote, 'fuel-mirror create --ubuntu')
|
||||
|
||||
self.show_step(9)
|
||||
|
||||
cluster_id = self.fuel_web.create_cluster(
|
||||
name=self.__class__.__name__,
|
||||
mode=DEPLOYMENT_MODE,
|
||||
settings={
|
||||
"net_provider": 'neutron',
|
||||
"net_segment_type": NEUTRON_SEGMENT['tun'],
|
||||
'tenant': 'packetary',
|
||||
'user': 'packetary',
|
||||
'password': 'packetary'
|
||||
}
|
||||
)
|
||||
self.show_step(10)
|
||||
self.show_step(11)
|
||||
self.fuel_web.update_nodes(
|
||||
cluster_id,
|
||||
{
|
||||
'slave-01': ['controller'],
|
||||
'slave-02': ['controller'],
|
||||
'slave-03': ['controller'],
|
||||
'slave-04': ['compute'],
|
||||
'slave-05': ['cinder']
|
||||
}
|
||||
)
|
||||
self.show_step(12)
|
||||
self.fuel_web.verify_network(cluster_id)
|
||||
self.show_step(13)
|
||||
self.fuel_web.deploy_cluster_wait(cluster_id)
|
||||
|
||||
self.show_step(14)
|
||||
self.fuel_web.run_ostf(
|
||||
cluster_id=cluster_id,
|
||||
test_sets=['ha', 'smoke', 'sanity'])
|
||||
|
||||
self.env.make_snapshot('deploy_with_custom_mirror')
|
||||
|
||||
@test(groups=['fuel-mirror', 'use-mirror'])
|
||||
def deploy_no_official_access(self):
|
||||
# TODO(akostrikov) add firewall rules to verify that there is no
|
||||
# connection to official mirrors during mirror creation and deployment.
|
||||
raise SkipTest('Not implemented yet')
|
||||
|
||||
@test(groups=['fuel-mirror', 'use-mirror'])
|
||||
def deploy_with_proxy(self):
|
||||
# TODO(akostrikov) add tests to verify that fuel-mirror works with
|
||||
# proxies too.
|
||||
raise SkipTest('Not implemented yet')
|
Loading…
Reference in New Issue