Refactoring novaClient instantiation and unittests

And adding infra for unittest execution.

Partial-Bug: 1951579
Signed-off-by: Thiago Brito <thiago.brito@windriver.com>
Change-Id: I58cefac9076db52333b41633bf2cbaa5441dc98c
This commit is contained in:
Thiago Brito 2021-11-19 18:41:26 -03:00
parent ed01aa74e1
commit 5d0b69af84
12 changed files with 271 additions and 24 deletions

2
.gitignore vendored
View File

@ -2,3 +2,5 @@
*.egg-info *.egg-info
*.swp *.swp
.idea .idea
__pycache__
*.py[cod]

View File

@ -6,12 +6,18 @@
- stx-tox-pylint - stx-tox-pylint
- utilities-ceph-manager-tox-bandit - utilities-ceph-manager-tox-bandit
- utilities-ceph-client-tox-bandit - utilities-ceph-client-tox-bandit
- utilities-pci-irq-affinity-agent-tox-py27
- utilities-pci-irq-affinity-agent-tox-py39
- utilities-pci-irq-affinity-agent-tox-flake8
gate: gate:
jobs: jobs:
- openstack-tox-linters - openstack-tox-linters
- stx-tox-pylint - stx-tox-pylint
- utilities-ceph-manager-tox-bandit - utilities-ceph-manager-tox-bandit
- utilities-ceph-client-tox-bandit - utilities-ceph-client-tox-bandit
- utilities-pci-irq-affinity-agent-tox-py27
- utilities-pci-irq-affinity-agent-tox-py39
- utilities-pci-irq-affinity-agent-tox-flake8
post: post:
jobs: jobs:
- stx-utilities-upload-git-mirror - stx-utilities-upload-git-mirror
@ -63,6 +69,44 @@
tox_envlist: bandit tox_envlist: bandit
tox_extra_args: -c ./ceph/python-cephclient/python-cephclient/tox.ini tox_extra_args: -c ./ceph/python-cephclient/python-cephclient/tox.ini
- job:
name: utilities-pci-irq-affinity-agent-tox-py27
parent: tox
description: |
Run py27 unittests for utilities/pci-irq-affinity-agent
nodeset: ubuntu-bionic
files:
- ./utilities/pci-irq-affinity-agent/*
vars:
tox_envlist: py27
tox_extra_args: -c ./utilities/pci-irq-affinity-agent/pci_irq_affinity/tox.ini
- job:
name: utilities-pci-irq-affinity-agent-tox-py39
voting: false
parent: tox
description: |
Run py39 unittests for utilities/pci-irq-affinity-agent
nodeset: debian-bullseye
files:
- ./utilities/pci-irq-affinity-agent/*
vars:
tox_envlist: py39
python_version: 3.9
tox_extra_args: -c ./utilities/pci-irq-affinity-agent/pci_irq_affinity/tox.ini
- job:
name: utilities-pci-irq-affinity-agent-tox-flake8
parent: tox
description: |
Run flake8 checks for utilities/pci-irq-affinity-agent
nodeset: ubuntu-bionic
files:
- ./utilities/pci-irq-affinity-agent/*
vars:
tox_envlist: flake8
tox_extra_args: -c ./utilities/pci-irq-affinity-agent/pci_irq_affinity/tox.ini
- secret: - secret:
name: stx-utilities-github-secret name: stx-utilities-github-secret
data: data:

View File

@ -0,0 +1,13 @@
# This is a cross-platform list tracking distribution packages needed for install and tests;
# see https://docs.openstack.org/infra/bindep/ for additional information.
build-essential [platform:dpkg test]
gcc [platform:rpm test]
libvirt-dev [platform:dpkg]
python3-dev [platform:dpkg (requirements python3)]
python3-devel [platform:rpm (requirements python3)]
python-dev [platform:dpkg test]
python3-all [platform:dpkg]
python3-all-dev [platform:dpkg]
python3 [platform:rpm test]
python3-devel [platform:rpm test]

View File

@ -12,15 +12,15 @@
""" Define pci_irq_affinity_provider class""" """ Define pci_irq_affinity_provider class"""
import os import os
import pci_irq_affinity.utils as pci_utils
from pci_irq_affinity.driver import AffinePciIrqDriver from pci_irq_affinity.driver import AffinePciIrqDriver
from pci_irq_affinity.nova_provider import novaClient
from pci_irq_affinity.log import LOG from pci_irq_affinity.log import LOG
from pci_irq_affinity import nova_provider
from pci_irq_affinity import utils as pci_utils
COMPUTE_IRQ = os.getenv("COMPUTE_IRQ", default="/proc/irq/") COMPUTE_IRQ = os.getenv("COMPUTE_IRQ", default="/proc/irq/")
class pci_irq_affinity_provider: class PciIrqAffinityProvider:
def __init__(self): def __init__(self):
self.affinePciIrqDriver = AffinePciIrqDriver() self.affinePciIrqDriver = AffinePciIrqDriver()
@ -90,9 +90,10 @@ class pci_irq_affinity_provider:
'task_state': None, 'task_state': None,
'deleted': False, 'deleted': False,
'all_tenants': True} 'all_tenants': True}
instances = novaClient.get_instances(filters) nova_client = nova_provider.get_nova_client()
instances = nova_client.get_instances(filters)
for inst in instances: for inst in instances:
self.affine_pci_dev_instance(inst, wait_for_irqs=False) self.affine_pci_dev_instance(inst, wait_for_irqs=False)
pciIrqAffinity = pci_irq_affinity_provider() pci_irq_affinity = PciIrqAffinityProvider()

View File

@ -23,11 +23,12 @@ from oslo_service import periodic_task
from oslo_service import service from oslo_service import service
import oslo_messaging import oslo_messaging
from pci_irq_affinity.affinity import pciIrqAffinity from pci_irq_affinity import affinity
from pci_irq_affinity.config import CONF from pci_irq_affinity.config import CONF
from pci_irq_affinity.log import LOG from pci_irq_affinity.log import LOG
from pci_irq_affinity.nova_provider import novaClient from pci_irq_affinity import nova_provider
import pci_irq_affinity.utils as pci_utils from pci_irq_affinity import utils as pci_utils
stay_on = True stay_on = True
@ -44,7 +45,8 @@ def process_signal_handler(signum, frame):
def get_inst(instance_uuid, callback): def get_inst(instance_uuid, callback):
# get instance info from nova # get instance info from nova
inst = novaClient.get_instance(instance_uuid) nova_client = nova_provider.get_nova_client()
inst = nova_client.get_instance(instance_uuid)
if inst is not None: if inst is not None:
LOG.debug("inst:%s" % inst) LOG.debug("inst:%s" % inst)
callback(inst) callback(inst)
@ -52,12 +54,12 @@ def get_inst(instance_uuid, callback):
def query_instance_callback(inst): def query_instance_callback(inst):
LOG.debug("query inst:%s" % inst) LOG.debug("query inst:%s" % inst)
pciIrqAffinity.affine_pci_dev_instance(inst) affinity.pci_irq_affinity.affine_pci_dev_instance(inst)
@periodic_task.periodic_task(spacing=CONF.parameters.pci_affine_interval) @periodic_task.periodic_task(spacing=CONF.parameters.pci_affine_interval)
def audit_affinity(self, context): def audit_affinity(self, context):
pciIrqAffinity.audit_pci_irq_affinity() affinity.pci_irq_affinity.audit_pci_irq_affinity()
def audit_work(srv, callback): def audit_work(srv, callback):
@ -150,7 +152,7 @@ class InstanceOfflineNotificationEndpoint(BaseInstanceEndpoint):
if instance_uuid: if instance_uuid:
LOG.info("Instance offline: uuid=%s, instance_host=%s, event_type=%s" % ( LOG.info("Instance offline: uuid=%s, instance_host=%s, event_type=%s" % (
instance_uuid, instance_host, event_type)) instance_uuid, instance_host, event_type))
pciIrqAffinity.reset_irq_affinity(instance_uuid) affinity.pci_irq_affinity.reset_irq_affinity(instance_uuid)
def rpc_work(srv): def rpc_work(srv):
@ -194,11 +196,12 @@ def process_main():
LOG.info("Enter PCIInterruptAffinity Agent") LOG.info("Enter PCIInterruptAffinity Agent")
nova_client = nova_provider.get_nova_client()
try: try:
signal.signal(signal.SIGTSTP, process_signal_handler) signal.signal(signal.SIGTSTP, process_signal_handler)
openstack_enabled = CONF.openstack.openstack_enabled openstack_enabled = CONF.openstack.openstack_enabled
if openstack_enabled: if openstack_enabled:
novaClient.open_libvirt_connect() nova_client.open_libvirt_connect()
audit_srv = audits_initialize() audit_srv = audits_initialize()
rabbit_client = start_rabbitmq_client() rabbit_client = start_rabbitmq_client()
@ -216,7 +219,7 @@ def process_main():
finally: finally:
LOG.error("process_main finalized!!!") LOG.error("process_main finalized!!!")
if openstack_enabled: if openstack_enabled:
novaClient.close_libvirt_connect() nova_client.close_libvirt_connect()
audit_srv.tg.stop() audit_srv.tg.stop()
rabbit_client.stop() rabbit_client.stop()

View File

@ -14,10 +14,10 @@
from oslo_service import loopingcall from oslo_service import loopingcall
from oslo_concurrency import lockutils from oslo_concurrency import lockutils
import pci_irq_affinity.utils as pci_utils
from pci_irq_affinity.config import CONF from pci_irq_affinity.config import CONF
from pci_irq_affinity.log import LOG from pci_irq_affinity.log import LOG
from pci_irq_affinity.nova_provider import novaClient from pci_irq_affinity import nova_provider
from pci_irq_affinity import utils as pci_utils
synchronized = lockutils.synchronized_with_prefix('pci_irq_affinity-') synchronized = lockutils.synchronized_with_prefix('pci_irq_affinity-')
@ -94,7 +94,8 @@ class AffinePciIrqDriver:
_msi_irqs = set() _msi_irqs = set()
# refresh instance info. # refresh instance info.
if refresh_need: if refresh_need:
_inst = novaClient.get_instance(inst.uuid) nova_client = nova_provider.get_nova_client()
_inst = nova_client.get_instance(inst.uuid)
if _inst is None: if _inst is None:
return return

View File

@ -14,18 +14,18 @@ This class wraps novaclient access interface and expose get_instance() and
get_instances() to other agent classes. get_instances() to other agent classes.
""" """
import keyring
import os import os
import socket import socket
import keyring
from keystoneauth1 import loading from keystoneauth1 import loading
from keystoneauth1 import session from keystoneauth1 import session
from novaclient import client from novaclient import client
from pci_irq_affinity.log import LOG
from pci_irq_affinity.config import CONF from pci_irq_affinity.config import CONF
from pci_irq_affinity import instance
from pci_irq_affinity import guest from pci_irq_affinity import guest
from pci_irq_affinity import instance
from pci_irq_affinity.log import LOG
class NovaProvider: class NovaProvider:
@ -142,7 +142,8 @@ class NovaProvider:
return instances return instances
def get_nova_client():
if CONF.openstack.openstack_enabled: if CONF.openstack.openstack_enabled:
novaClient = NovaProvider() return NovaProvider()
else: else:
novaClient = None return None

View File

@ -0,0 +1,19 @@
eventlet==0.33.0
keyring==18.0.1
keyrings.alt==3.2.0
pycrypto==2.6.1
keystoneauth1==4.0.1
kombu==4.6.11
pkgconfig==1.5.2
libvirt-python;python_version>="3.0" # GPLv2
libvirt-python<6.0.0;python_version<"3.0" # GPLv2
oslo.concurrency==3.31.0
oslo.config==7.0.0
oslo.i18n==3.25.1
oslo.log==3.45.2
oslo.messaging==10.5.0
oslo.service==1.41.1
python-daemon==2.3.0
python-novaclient==16.0.0
retrying==1.3.3
psutil==5.8.0

View File

@ -0,0 +1,8 @@
# hacking pulls in flake8
hacking>=1.1.0,<=2.0.0 # Apache-2.0
bashate >= 0.2
isort<5;python_version>="3.0"
pylint<2.1.0;python_version<"3.0" # GPLv2
pylint<2.3.0;python_version>="3.0" # GPLv2
pytest
mock

View File

@ -0,0 +1,73 @@
#
# Copyright (c) 2021 StarlingX.
#
# SPDX-License-Identifier: Apache-2.0
#
# All Rights Reserved.
#
import mock
import unittest
from pci_irq_affinity.config import CONF
from pci_irq_affinity import nova_provider
class TestNovaProvider(unittest.TestCase):
AUTH_URL = "http://keystone.local"
USERNAME = "my-name"
CACERT = "cacert.pem"
KEYRING_SERVICE = "CGCS"
PASSWORD = "my-pass"
def setUp(self):
CONF.set_override("auth_url", self.AUTH_URL, group="openstack")
CONF.set_override("username", self.USERNAME, group="openstack")
CONF.set_override("cacert", self.CACERT, group="openstack")
CONF.set_override("keyring_service", self.KEYRING_SERVICE,
group="openstack")
def tearDown(self):
CONF.clear_override("auth_url", group="openstack")
CONF.clear_override("username", group="openstack")
CONF.clear_override("cacert", group="openstack")
CONF.clear_override("keyring_service", group="openstack")
CONF.clear_override("password", group="openstack")
def test__get_keystone_creds(self):
CONF.set_override("password", "my-pass", group="openstack")
instance = nova_provider.NovaProvider()
creds = instance._get_keystone_creds()
expected = {
"auth_url": self.AUTH_URL,
"username": self.USERNAME,
"password": self.PASSWORD,
# "project_domain_name": None,
# "project_name": None,
# "user_domain_name": None
}
self.assertDictEqual(expected, creds)
@mock.patch.object(nova_provider, "keyring")
def test__get_keystone_creds_no_password(self, mock_keyring):
keyring_pass = "keyring-pass"
mock_keyring.get_password.return_value = keyring_pass
instance = nova_provider.NovaProvider()
mock_keyring.get_password.reset_mock()
creds = instance._get_keystone_creds()
expected = {
"auth_url": self.AUTH_URL,
"username": self.USERNAME,
"password": keyring_pass,
# "project_domain_name": None,
# "project_name": None,
}
self.assertDictEqual(expected, creds)
mock_keyring.get_password.assert_called_once_with(
self.KEYRING_SERVICE,
self.USERNAME,
)

View File

@ -0,0 +1,82 @@
[tox]
envlist = py27,flake8
minversion = 2.3
skipsdist = True
[testenv]
sitepackages = False
basepython = python2.7
install_command = pip install -vvv -U {opts} {packages}
setenv =
VIRTUAL_ENV={envdir}
OS_STDOUT_CAPTURE=1
OS_STDERR_CAPTURE=1
OS_DEBUG=1
OS_LOG_CAPTURE=1
deps =
-r{toxinidir}/requirements.txt
-r{toxinidir}/test-requirements.txt
whitelist_externals =
bash
commands =
pytest tests/
[testenv:py36]
basepython = python3.6
[testenv:py39]
basepython = python3.9
[flake8]
# Note: hacking pulls in flake8 2.5.5 which can not parse an ignore list spanning multiple lines
# F errors are high priority to fix. W are warnings. E series are pep8, H series are hacking
# F401 'FOO' imported but unused
# F841 local variable 'FOO' is assigned to but never used
# W291 trailing whitespace
# W391 blank line at end of file
# W503 line break before binary operator
# W504 line break after binary operator
# W605 invalid escape sequence '\s'
# E128 continuation line under-indented for visual indent
# E221 multiple spaces before operator
# E226 missing whitespace around arithmetic operator
# E227 missing whitespace around bitwise or shift operator
# E241 multiple spaces after ','
# E265 block comment should start with '# '
# E302 expected 2 blank lines, found 1
# E305 expected 2 blank lines after class or function definition, found 1
# E501 line too long
# E502 the backslash is redundant between brackets
# E702 multiple statements on one line (semicolon)
# E722 do not use bare 'except'
# H101 is TODO
# H104 File contains nothing but comments
# H201 no 'except:' at least use 'except Exception:'
# H238 old style class declaration, use new style (inherit from `object`)
# H306 imports not in alphabetical order (sys, re)
# H401 docstring should not start with a space
# H405 multi line docstring summary not separated with an empty line
ignore = F401,F841,W291,W391,W503,W504,W605,E128,E221,E226,E227,E241,E265,E302,E305,E501,E502,E702,E722,H101,H104,H201,H238,H306,H401,H405
# max-line-length is not referenced if E501 is suppressed
max-line-length=80
[testenv:flake8]
commands =
flake8
[testenv:pylint]
commands = pylint {posargs} pci_irq_affinity --rcfile=../../../pylint.rc
[testenv:bandit]
description = Dummy environment to allow bandit to be run in subdir tox
[testenv:bindep]
# Do not install any requirements. We want this to be fast and work even if
# system dependencies are missing, since it's used to tell you what system
# dependencies are missing! This also means that bindep must be installed
# separately, outside of the requirements files, and develop mode disabled
# explicitly to avoid unnecessarily installing the checked-out repo too (this
# further relies on "tox.skipsdist = True" above).
deps = bindep
commands = bindep test
usedevelop = False