x/tobiko
Code Issues Proposed changes

Use Ubuntu minimal image to run Iperf3 server

Browse Source 
- install ipref3 using virt-customize
- enables QoS server test on DevStack based setups
- remove iperf3 installation from iperf3 shell APIs
- raise default QoS BW limit from 1Kbps to 1 Mbps
- increase QoS bandwith limit tollerance range (90%-110%)

Depends-On: https://review.opendev.org/c/x/devstack-plugin-tobiko/+/788899
Change-Id: Idc7d98b34ce59d14408b15edb6061a36f796a8fc
This commit is contained in:
Federico Ressi 2021-04-27 09:50:58 +02:00
parent 6f39740e53
commit ed84b3fcd6
12 changed files with 146 additions and 64 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
bindep.txt
View File

@ -8,6 +8,7 @@ git [platform:redhat]
iperf3 [platform:redhat]
iproute [platform:redhat]
libffi-devel [platform:redhat]
libguestfs-tools-c [platform:redhat]
make [platform:redhat]
openssl-devel [platform:redhat]
python-docutils [platform:rhel-7]
@ -27,6 +28,7 @@ gcc [platform:ubuntu]
git [platform:ubuntu]
iperf3 [platform:ubuntu]
libffi-dev [platform:ubuntu]
libguestfs-tools [platform:ubuntu]
libssl-dev [platform:ubuntu]
make [platform:ubuntu]
python-docutils [platform:ubuntu]

1
tobiko/openstack/glance/__init__.py
View File

@ -32,6 +32,7 @@ FileGlanceImageFixture = _image.FileGlanceImageFixture
GlanceImageFixture = _image.GlanceImageFixture
HasImageMixin = _image.HasImageMixin
URLGlanceImageFixture = _image.URLGlanceImageFixture
CustomizedGlanceImageFixture = _image.CustomizedGlanceImageFixture
open_image_file = _io.open_image_file

58
tobiko/openstack/glance/_image.py
View File

@ -28,6 +28,7 @@ from tobiko.config import get_bool_env
from tobiko.openstack.glance import _client
from tobiko.openstack.glance import _io
from tobiko.openstack import keystone
from tobiko.shell import sh
LOG = log.getLogger(__name__)
@ -326,7 +327,9 @@ class FileGlanceImageFixture(UploadGranceImageFixture):
return os.path.join(self.real_image_dir, self.image_file)
def get_image_data(self):
image_file = self.real_image_file
return self.get_image_file(image_file=self.real_image_file)
def get_image_file(self, image_file: str):
image_size = os.path.getsize(image_file)
LOG.debug('Uploading image %r data from file %r (%d bytes)',
self.image_name, image_file, image_size)
@ -338,20 +341,19 @@ class FileGlanceImageFixture(UploadGranceImageFixture):
class URLGlanceImageFixture(FileGlanceImageFixture):
image_url: typing.Optional[str] = None
image_url: str
def __init__(self, image_url=None, **kwargs):
def __init__(self, image_url: typing.Optional[str] = None, **kwargs):
super(URLGlanceImageFixture, self).__init__(**kwargs)
if image_url:
self.image_url = image_url
else:
if image_url is None:
image_url = self.image_url
else:
self.image_url = image_url
tobiko.check_valid_type(image_url, str)
def get_image_data(self):
def get_image_file(self, image_file: str):
http_request = requests.get(self.image_url, stream=True)
expected_size = int(http_request.headers.get('content-length', 0))
image_file = self.real_image_file
chunks = http_request.iter_content(chunk_size=io.DEFAULT_BUFFER_SIZE)
download_image = True
if expected_size:
@ -373,7 +375,9 @@ class URLGlanceImageFixture(FileGlanceImageFixture):
self._download_image_file(image_file=image_file,
chunks=chunks,
expected_size=expected_size)
return super(URLGlanceImageFixture, self).get_image_data()
image_file = self.customize_image_file(base_file=image_file)
return super(URLGlanceImageFixture, self).get_image_file(
image_file=image_file)
def _download_image_file(self, image_file, chunks, expected_size):
image_dir = os.path.dirname(image_file)
@ -396,6 +400,42 @@ class URLGlanceImageFixture(FileGlanceImageFixture):
raise RuntimeError(message)
os.rename(temp_file, image_file)
def customize_image_file(self, base_file: str) -> str:
return base_file
class CustomizedGlanceImageFixture(URLGlanceImageFixture):
install_packages: typing.Sequence[str] = tuple()
def customize_image_file(self, base_file: str) -> str:
customized_file = base_file + '.1'
if os.path.isfile(customized_file):
if (os.stat(base_file).st_mtime_ns <
os.stat(customized_file).st_mtime_ns):
LOG.debug(f"Image file is up to date '{customized_file}'")
return customized_file
else:
LOG.debug(f"Remove obsolete image file '{customized_file}'")
os.remove(customized_file)
work_file = sh.execute('mktemp').stdout.strip()
try:
LOG.debug(f"Copy base image file: '{base_file}' to '{work_file}'")
sh.put_file(base_file, work_file)
command = sh.shell_command(['virt-customize', '-a', work_file])
execute = False
for package in self.install_packages:
execute = True
command += ['--install', package]
if execute:
sh.execute(command, sudo=True)
sh.get_file(work_file, customized_file)
return customized_file
finally:
sh.execute(['rm', '-f', work_file])
class InvalidGlanceImageStatus(tobiko.TobikoException):
message = ("Invalid image {image_name!r} (id {image_id!r}) status: "

2
tobiko/openstack/neutron/config.py
View File

@ -57,7 +57,7 @@ OPTIONS = [
default=['/etc/resolv.conf'],
help="File to parse for getting default nameservers list"),
cfg.IntOpt('bwlimit_kbps',
default=100,
default=1000,
help="The BW limit value configured for the QoS Policy Rule"),
cfg.StrOpt('direction',
default='egress',

1
tobiko/openstack/stacks/__init__.py
View File

@ -81,6 +81,7 @@ UbuntuMinimalImageFixture = _ubuntu.UbuntuMinimalImageFixture
UbuntuServerStackFixture = _ubuntu.UbuntuServerStackFixture
UbuntuMinimalServerStackFixture = _ubuntu.UbuntuMinimalServerStackFixture
UbuntuExternalServerStackFixture = _ubuntu.UbuntuExternalServerStackFixture
UbuntuQosServerStackFixture = _ubuntu.UbuntuQosServerStackFixture
OctaviaLoadbalancerStackFixture = _octavia.OctaviaLoadbalancerStackFixture
OctaviaListenerStackFixture = _octavia.OctaviaListenerStackFixture

12
tobiko/openstack/stacks/_ubuntu.py
View File

@ -85,3 +85,15 @@ class UbuntuMinimalServerStackFixture(UbuntuServerStackFixture):
class UbuntuExternalServerStackFixture(UbuntuMinimalServerStackFixture,
_nova.ExternalServerStackFixture):
pass
class UbuntuQosServerImageFixture(UbuntuMinimalImageFixture,
glance.CustomizedGlanceImageFixture):
install_packages = ['iperf3']
class UbuntuQosServerStackFixture(UbuntuMinimalServerStackFixture,
_nova.QosServerStackFixture):
#: Glance image used to create a Nova server instance
image_fixture = tobiko.required_setup_fixture(UbuntuQosServerImageFixture)

4
tobiko/shell/iperf/_assert.py
View File

@ -50,6 +50,6 @@ def assert_bw_limit(ssh_client, ssh_server, **params):
LOG.debug('measured_bw = %f', measured_bw)
LOG.debug('bw_limit = %f', bw_limit)
# a 5% of upper deviation is allowed
testcase.assertLess(measured_bw, bw_limit * 1.05)
testcase.assertLess(measured_bw, bw_limit * 1.1)
# an 8% of lower deviation is allowed
testcase.assertGreater(measured_bw, bw_limit * 0.92)
testcase.assertGreater(measured_bw, bw_limit * 0.9)

38
tobiko/shell/iperf/_interface.py
View File

@ -24,43 +24,6 @@ from tobiko.shell import sh
LOG = log.getLogger(__name__)
def install_iperf(ssh_client):
def iperf_help():
cmd = 'iperf3 --help'
try:
return sh.execute(cmd,
expect_exit_status=None,
ssh_client=ssh_client)
except FileNotFoundError:
return sh.execute_result(command=cmd,
exit_status=127,
stdout='command not found')
result = iperf_help()
usage = ((result.stdout and str(result.stdout)) or
(result.stderr and str(result.stderr)) or "").strip()
if result.exit_status != 0 and 'command not found' in usage.lower():
install_command = '{install_tool} install -y iperf3'
install_tools = ('yum', 'apt')
for install_tool in install_tools:
try:
result = sh.execute(
command=install_command.format(install_tool=install_tool),
ssh_client=ssh_client,
sudo=True)
except sh.ShellError:
LOG.debug(f'Unable to install iperf3 using {install_tool}')
else:
LOG.debug(f'iperf3 successfully installed with {install_tool}')
break
if iperf_help().exit_status != 0:
raise RuntimeError('iperf3 command was not installed successfully')
elif result.exit_status != 0:
raise RuntimeError('Error executing iperf3 command')
else:
LOG.debug('iperf3 already installed')
def get_iperf_command(parameters, ssh_client):
interface = get_iperf_interface(ssh_client=ssh_client)
return interface.get_iperf_command(parameters)
@ -90,7 +53,6 @@ class IperfInterfaceManager(tobiko.SharedFixture):
except KeyError:
pass
install_iperf(ssh_client)
LOG.debug('Assign default iperf interface to SSH client %r',
ssh_client)
self.client_interfaces[ssh_client] = self.default_interface

3
tobiko/shell/iperf/_iperf.py
View File

@ -70,6 +70,5 @@ def execute_iperf_client(parameters, ssh_client):
ssh_client=ssh_client)
result = sh.execute(command=command,
ssh_client=ssh_client,
timeout=parameters.timeout + 5.,
expect_exit_status=None)
timeout=parameters.timeout + 5.)
return json.loads(result.stdout)

4
tobiko/shell/sh/__init__.py
View File

@ -25,6 +25,7 @@ from tobiko.shell.sh import _nameservers
from tobiko.shell.sh import _process
from tobiko.shell.sh import _ps
from tobiko.shell.sh import _reboot
from tobiko.shell.sh import _sftp
from tobiko.shell.sh import _ssh
from tobiko.shell.sh import _uptime
@ -80,6 +81,9 @@ crash_method = RebootHostMethod.CRASH
hard_reset_method = RebootHostMethod.HARD
soft_reset_method = RebootHostMethod.SOFT
put_file = _sftp.put_file
get_file = _sftp.get_file
ssh_process = _ssh.ssh_process
ssh_execute = _ssh.ssh_execute
SSHShellProcessFixture = _ssh.SSHShellProcessFixture

65
tobiko/shell/sh/_sftp.py Normal file
View File

@ -0,0 +1,65 @@
# Copyright (c) 2019 Red Hat, Inc.
#
# All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
from __future__ import absolute_import
import shutil
import typing
from oslo_log import log
import paramiko
from tobiko.shell import ssh
LOG = log.getLogger(__name__)
SSHClientType = typing.Union[None, ssh.SSHClientFixture, bool]
def sftp_client(ssh_client: SSHClientType) \
-> typing.Optional[paramiko.SFTPClient]:
if ssh_client is None:
ssh_client = ssh.ssh_proxy_client()
if isinstance(ssh_client, ssh.SSHClientFixture):
return ssh_client.connect().open_sftp()
assert ssh_client is None
return None
def put_file(local_file: str,
remote_file: str,
ssh_client: SSHClientType = None):
sftp = sftp_client(ssh_client)
if sftp is None:
LOG.debug(f"Copy local file: '{local_file}' -> '{remote_file}' ...")
shutil.copyfile(local_file, remote_file)
else:
LOG.debug(f"Put remote file: '{local_file}' -> '{remote_file}' ...")
with sftp:
sftp.put(local_file, remote_file)
def get_file(remote_file: str,
local_file: str,
ssh_client: SSHClientType = None):
sftp = sftp_client(ssh_client)
if sftp is None:
LOG.debug(f"Copy local file: '{remote_file}' -> '{local_file}' ...")
shutil.copyfile(remote_file, local_file)
else:
LOG.debug(f"Get remote file: '{remote_file}' -> '{local_file}' ...")
with sftp:
sftp.get(remote_file, local_file)

20
tobiko/tests/scenario/neutron/test_qos.py
View File

@ -32,7 +32,7 @@ class QoSBasicTest(testtools.TestCase):
"""Tests QoS basic functionality"""
#: Resources stack with QoS Policy and QoS Rules and Advanced server
stack = tobiko.required_setup_fixture(stacks.CentosQosServerStackFixture)
stack = tobiko.required_setup_fixture(stacks.UbuntuQosServerStackFixture)
def setUp(self):
"""Skip these tests if OVN is configured and OSP version is lower than
@ -44,22 +44,18 @@ class QoSBasicTest(testtools.TestCase):
containers.ovn_used_on_overcloud()):
self.skipTest("QoS not supported in this setup")
def test_qos_basic(self):
def test_network_qos_policy_id(self):
'''Verify QoS Policy attached to the network corresponds with the QoS
Policy previously created'''
self.assertEqual(self.stack.network_stack.qos_stack.qos_policy_id,
self.stack.network_stack.qos_policy_id)
def test_server_qos_policy_id(self):
self.assertIsNone(self.stack.port_details['qos_policy_id'])
def test_qos_bw_limit(self):
'''Verify BW limit using the iperf tool
The test is executed from the undercloud node (client) to the VM
instance (server)'''
if not tobiko.tripleo.has_undercloud():
# TODO(eolivare): this test does not support devstack environments
# yet and that should be fixed
tobiko.skip_test('test not supported on devstack environments')
ssh_client = None # localhost will act as client
ssh_server = self.stack.peer_ssh_client
iperf.assert_bw_limit(ssh_client, ssh_server)
'''
self.stack.wait_for_cloud_init_done()
iperf.assert_bw_limit(ssh_client=None, # localhost will act as client
ssh_server=self.stack.peer_ssh_client)