From 5d0b69af8437c68552a8ab90b46efb12d3bb2896 Mon Sep 17 00:00:00 2001 From: Thiago Brito Date: Fri, 19 Nov 2021 18:41:26 -0300 Subject: [PATCH] Refactoring novaClient instantiation and unittests And adding infra for unittest execution. Partial-Bug: 1951579 Signed-off-by: Thiago Brito Change-Id: I58cefac9076db52333b41633bf2cbaa5441dc98c --- .gitignore | 2 + .zuul.yaml | 44 ++++++++++ .../pci_irq_affinity/bindep.txt | 13 +++ .../pci_irq_affinity/affinity.py | 11 +-- .../pci_irq_affinity/agent.py | 21 +++-- .../pci_irq_affinity/driver.py | 7 +- .../pci_irq_affinity/nova_provider.py | 15 ++-- .../pci_irq_affinity/requirements.txt | 19 +++++ .../pci_irq_affinity/test-requirements.txt | 8 ++ .../pci_irq_affinity/tests/__init__.py | 0 .../tests/test_nova_provider.py | 73 +++++++++++++++++ .../pci_irq_affinity/tox.ini | 82 +++++++++++++++++++ 12 files changed, 271 insertions(+), 24 deletions(-) create mode 100644 utilities/pci-irq-affinity-agent/pci_irq_affinity/bindep.txt create mode 100644 utilities/pci-irq-affinity-agent/pci_irq_affinity/requirements.txt create mode 100644 utilities/pci-irq-affinity-agent/pci_irq_affinity/test-requirements.txt create mode 100644 utilities/pci-irq-affinity-agent/pci_irq_affinity/tests/__init__.py create mode 100644 utilities/pci-irq-affinity-agent/pci_irq_affinity/tests/test_nova_provider.py create mode 100644 utilities/pci-irq-affinity-agent/pci_irq_affinity/tox.ini diff --git a/.gitignore b/.gitignore index 30984c31..81487f29 100644 --- a/.gitignore +++ b/.gitignore @@ -2,3 +2,5 @@ *.egg-info *.swp .idea +__pycache__ +*.py[cod] diff --git a/.zuul.yaml b/.zuul.yaml index a99ba8fd..5b2cd3be 100644 --- a/.zuul.yaml +++ b/.zuul.yaml @@ -6,12 +6,18 @@ - stx-tox-pylint - utilities-ceph-manager-tox-bandit - utilities-ceph-client-tox-bandit + - utilities-pci-irq-affinity-agent-tox-py27 + - utilities-pci-irq-affinity-agent-tox-py39 + - utilities-pci-irq-affinity-agent-tox-flake8 gate: jobs: - openstack-tox-linters - stx-tox-pylint - utilities-ceph-manager-tox-bandit - utilities-ceph-client-tox-bandit + - utilities-pci-irq-affinity-agent-tox-py27 + - utilities-pci-irq-affinity-agent-tox-py39 + - utilities-pci-irq-affinity-agent-tox-flake8 post: jobs: - stx-utilities-upload-git-mirror @@ -63,6 +69,44 @@ tox_envlist: bandit tox_extra_args: -c ./ceph/python-cephclient/python-cephclient/tox.ini +- job: + name: utilities-pci-irq-affinity-agent-tox-py27 + parent: tox + description: | + Run py27 unittests for utilities/pci-irq-affinity-agent + nodeset: ubuntu-bionic + files: + - ./utilities/pci-irq-affinity-agent/* + vars: + tox_envlist: py27 + tox_extra_args: -c ./utilities/pci-irq-affinity-agent/pci_irq_affinity/tox.ini + +- job: + name: utilities-pci-irq-affinity-agent-tox-py39 + voting: false + parent: tox + description: | + Run py39 unittests for utilities/pci-irq-affinity-agent + nodeset: debian-bullseye + files: + - ./utilities/pci-irq-affinity-agent/* + vars: + tox_envlist: py39 + python_version: 3.9 + tox_extra_args: -c ./utilities/pci-irq-affinity-agent/pci_irq_affinity/tox.ini + +- job: + name: utilities-pci-irq-affinity-agent-tox-flake8 + parent: tox + description: | + Run flake8 checks for utilities/pci-irq-affinity-agent + nodeset: ubuntu-bionic + files: + - ./utilities/pci-irq-affinity-agent/* + vars: + tox_envlist: flake8 + tox_extra_args: -c ./utilities/pci-irq-affinity-agent/pci_irq_affinity/tox.ini + - secret: name: stx-utilities-github-secret data: diff --git a/utilities/pci-irq-affinity-agent/pci_irq_affinity/bindep.txt b/utilities/pci-irq-affinity-agent/pci_irq_affinity/bindep.txt new file mode 100644 index 00000000..ba61889a --- /dev/null +++ b/utilities/pci-irq-affinity-agent/pci_irq_affinity/bindep.txt @@ -0,0 +1,13 @@ +# This is a cross-platform list tracking distribution packages needed for install and tests; +# see https://docs.openstack.org/infra/bindep/ for additional information. + +build-essential [platform:dpkg test] +gcc [platform:rpm test] +libvirt-dev [platform:dpkg] +python3-dev [platform:dpkg (requirements python3)] +python3-devel [platform:rpm (requirements python3)] +python-dev [platform:dpkg test] +python3-all [platform:dpkg] +python3-all-dev [platform:dpkg] +python3 [platform:rpm test] +python3-devel [platform:rpm test] diff --git a/utilities/pci-irq-affinity-agent/pci_irq_affinity/pci_irq_affinity/affinity.py b/utilities/pci-irq-affinity-agent/pci_irq_affinity/pci_irq_affinity/affinity.py index a4831199..26071a4b 100644 --- a/utilities/pci-irq-affinity-agent/pci_irq_affinity/pci_irq_affinity/affinity.py +++ b/utilities/pci-irq-affinity-agent/pci_irq_affinity/pci_irq_affinity/affinity.py @@ -12,15 +12,15 @@ """ Define pci_irq_affinity_provider class""" import os -import pci_irq_affinity.utils as pci_utils from pci_irq_affinity.driver import AffinePciIrqDriver -from pci_irq_affinity.nova_provider import novaClient from pci_irq_affinity.log import LOG +from pci_irq_affinity import nova_provider +from pci_irq_affinity import utils as pci_utils COMPUTE_IRQ = os.getenv("COMPUTE_IRQ", default="/proc/irq/") -class pci_irq_affinity_provider: +class PciIrqAffinityProvider: def __init__(self): self.affinePciIrqDriver = AffinePciIrqDriver() @@ -90,9 +90,10 @@ class pci_irq_affinity_provider: 'task_state': None, 'deleted': False, 'all_tenants': True} - instances = novaClient.get_instances(filters) + nova_client = nova_provider.get_nova_client() + instances = nova_client.get_instances(filters) for inst in instances: self.affine_pci_dev_instance(inst, wait_for_irqs=False) -pciIrqAffinity = pci_irq_affinity_provider() +pci_irq_affinity = PciIrqAffinityProvider() diff --git a/utilities/pci-irq-affinity-agent/pci_irq_affinity/pci_irq_affinity/agent.py b/utilities/pci-irq-affinity-agent/pci_irq_affinity/pci_irq_affinity/agent.py index 7d19898e..11c79fd5 100644 --- a/utilities/pci-irq-affinity-agent/pci_irq_affinity/pci_irq_affinity/agent.py +++ b/utilities/pci-irq-affinity-agent/pci_irq_affinity/pci_irq_affinity/agent.py @@ -23,11 +23,12 @@ from oslo_service import periodic_task from oslo_service import service import oslo_messaging -from pci_irq_affinity.affinity import pciIrqAffinity +from pci_irq_affinity import affinity from pci_irq_affinity.config import CONF from pci_irq_affinity.log import LOG -from pci_irq_affinity.nova_provider import novaClient -import pci_irq_affinity.utils as pci_utils +from pci_irq_affinity import nova_provider +from pci_irq_affinity import utils as pci_utils + stay_on = True @@ -44,7 +45,8 @@ def process_signal_handler(signum, frame): def get_inst(instance_uuid, callback): # get instance info from nova - inst = novaClient.get_instance(instance_uuid) + nova_client = nova_provider.get_nova_client() + inst = nova_client.get_instance(instance_uuid) if inst is not None: LOG.debug("inst:%s" % inst) callback(inst) @@ -52,12 +54,12 @@ def get_inst(instance_uuid, callback): def query_instance_callback(inst): LOG.debug("query inst:%s" % inst) - pciIrqAffinity.affine_pci_dev_instance(inst) + affinity.pci_irq_affinity.affine_pci_dev_instance(inst) @periodic_task.periodic_task(spacing=CONF.parameters.pci_affine_interval) def audit_affinity(self, context): - pciIrqAffinity.audit_pci_irq_affinity() + affinity.pci_irq_affinity.audit_pci_irq_affinity() def audit_work(srv, callback): @@ -150,7 +152,7 @@ class InstanceOfflineNotificationEndpoint(BaseInstanceEndpoint): if instance_uuid: LOG.info("Instance offline: uuid=%s, instance_host=%s, event_type=%s" % ( instance_uuid, instance_host, event_type)) - pciIrqAffinity.reset_irq_affinity(instance_uuid) + affinity.pci_irq_affinity.reset_irq_affinity(instance_uuid) def rpc_work(srv): @@ -194,11 +196,12 @@ def process_main(): LOG.info("Enter PCIInterruptAffinity Agent") + nova_client = nova_provider.get_nova_client() try: signal.signal(signal.SIGTSTP, process_signal_handler) openstack_enabled = CONF.openstack.openstack_enabled if openstack_enabled: - novaClient.open_libvirt_connect() + nova_client.open_libvirt_connect() audit_srv = audits_initialize() rabbit_client = start_rabbitmq_client() @@ -216,7 +219,7 @@ def process_main(): finally: LOG.error("process_main finalized!!!") if openstack_enabled: - novaClient.close_libvirt_connect() + nova_client.close_libvirt_connect() audit_srv.tg.stop() rabbit_client.stop() diff --git a/utilities/pci-irq-affinity-agent/pci_irq_affinity/pci_irq_affinity/driver.py b/utilities/pci-irq-affinity-agent/pci_irq_affinity/pci_irq_affinity/driver.py index d0cc2794..e7fe373b 100644 --- a/utilities/pci-irq-affinity-agent/pci_irq_affinity/pci_irq_affinity/driver.py +++ b/utilities/pci-irq-affinity-agent/pci_irq_affinity/pci_irq_affinity/driver.py @@ -14,10 +14,10 @@ from oslo_service import loopingcall from oslo_concurrency import lockutils -import pci_irq_affinity.utils as pci_utils from pci_irq_affinity.config import CONF from pci_irq_affinity.log import LOG -from pci_irq_affinity.nova_provider import novaClient +from pci_irq_affinity import nova_provider +from pci_irq_affinity import utils as pci_utils synchronized = lockutils.synchronized_with_prefix('pci_irq_affinity-') @@ -94,7 +94,8 @@ class AffinePciIrqDriver: _msi_irqs = set() # refresh instance info. if refresh_need: - _inst = novaClient.get_instance(inst.uuid) + nova_client = nova_provider.get_nova_client() + _inst = nova_client.get_instance(inst.uuid) if _inst is None: return diff --git a/utilities/pci-irq-affinity-agent/pci_irq_affinity/pci_irq_affinity/nova_provider.py b/utilities/pci-irq-affinity-agent/pci_irq_affinity/pci_irq_affinity/nova_provider.py index afd00729..2fb502f8 100644 --- a/utilities/pci-irq-affinity-agent/pci_irq_affinity/pci_irq_affinity/nova_provider.py +++ b/utilities/pci-irq-affinity-agent/pci_irq_affinity/pci_irq_affinity/nova_provider.py @@ -14,18 +14,18 @@ This class wraps novaclient access interface and expose get_instance() and get_instances() to other agent classes. """ +import keyring import os import socket -import keyring from keystoneauth1 import loading from keystoneauth1 import session from novaclient import client -from pci_irq_affinity.log import LOG from pci_irq_affinity.config import CONF -from pci_irq_affinity import instance from pci_irq_affinity import guest +from pci_irq_affinity import instance +from pci_irq_affinity.log import LOG class NovaProvider: @@ -142,7 +142,8 @@ class NovaProvider: return instances -if CONF.openstack.openstack_enabled: - novaClient = NovaProvider() -else: - novaClient = None +def get_nova_client(): + if CONF.openstack.openstack_enabled: + return NovaProvider() + else: + return None diff --git a/utilities/pci-irq-affinity-agent/pci_irq_affinity/requirements.txt b/utilities/pci-irq-affinity-agent/pci_irq_affinity/requirements.txt new file mode 100644 index 00000000..fd7f4e56 --- /dev/null +++ b/utilities/pci-irq-affinity-agent/pci_irq_affinity/requirements.txt @@ -0,0 +1,19 @@ +eventlet==0.33.0 +keyring==18.0.1 +keyrings.alt==3.2.0 +pycrypto==2.6.1 +keystoneauth1==4.0.1 +kombu==4.6.11 +pkgconfig==1.5.2 +libvirt-python;python_version>="3.0" # GPLv2 +libvirt-python<6.0.0;python_version<"3.0" # GPLv2 +oslo.concurrency==3.31.0 +oslo.config==7.0.0 +oslo.i18n==3.25.1 +oslo.log==3.45.2 +oslo.messaging==10.5.0 +oslo.service==1.41.1 +python-daemon==2.3.0 +python-novaclient==16.0.0 +retrying==1.3.3 +psutil==5.8.0 \ No newline at end of file diff --git a/utilities/pci-irq-affinity-agent/pci_irq_affinity/test-requirements.txt b/utilities/pci-irq-affinity-agent/pci_irq_affinity/test-requirements.txt new file mode 100644 index 00000000..4b743b38 --- /dev/null +++ b/utilities/pci-irq-affinity-agent/pci_irq_affinity/test-requirements.txt @@ -0,0 +1,8 @@ +# hacking pulls in flake8 +hacking>=1.1.0,<=2.0.0 # Apache-2.0 +bashate >= 0.2 +isort<5;python_version>="3.0" +pylint<2.1.0;python_version<"3.0" # GPLv2 +pylint<2.3.0;python_version>="3.0" # GPLv2 +pytest +mock diff --git a/utilities/pci-irq-affinity-agent/pci_irq_affinity/tests/__init__.py b/utilities/pci-irq-affinity-agent/pci_irq_affinity/tests/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/utilities/pci-irq-affinity-agent/pci_irq_affinity/tests/test_nova_provider.py b/utilities/pci-irq-affinity-agent/pci_irq_affinity/tests/test_nova_provider.py new file mode 100644 index 00000000..1291eef7 --- /dev/null +++ b/utilities/pci-irq-affinity-agent/pci_irq_affinity/tests/test_nova_provider.py @@ -0,0 +1,73 @@ +# +# Copyright (c) 2021 StarlingX. +# +# SPDX-License-Identifier: Apache-2.0 +# +# All Rights Reserved. +# + +import mock +import unittest + +from pci_irq_affinity.config import CONF +from pci_irq_affinity import nova_provider + + +class TestNovaProvider(unittest.TestCase): + AUTH_URL = "http://keystone.local" + USERNAME = "my-name" + CACERT = "cacert.pem" + KEYRING_SERVICE = "CGCS" + PASSWORD = "my-pass" + + def setUp(self): + CONF.set_override("auth_url", self.AUTH_URL, group="openstack") + CONF.set_override("username", self.USERNAME, group="openstack") + CONF.set_override("cacert", self.CACERT, group="openstack") + CONF.set_override("keyring_service", self.KEYRING_SERVICE, + group="openstack") + + def tearDown(self): + CONF.clear_override("auth_url", group="openstack") + CONF.clear_override("username", group="openstack") + CONF.clear_override("cacert", group="openstack") + CONF.clear_override("keyring_service", group="openstack") + CONF.clear_override("password", group="openstack") + + def test__get_keystone_creds(self): + CONF.set_override("password", "my-pass", group="openstack") + + instance = nova_provider.NovaProvider() + creds = instance._get_keystone_creds() + + expected = { + "auth_url": self.AUTH_URL, + "username": self.USERNAME, + "password": self.PASSWORD, + # "project_domain_name": None, + # "project_name": None, + # "user_domain_name": None + } + self.assertDictEqual(expected, creds) + + @mock.patch.object(nova_provider, "keyring") + def test__get_keystone_creds_no_password(self, mock_keyring): + keyring_pass = "keyring-pass" + mock_keyring.get_password.return_value = keyring_pass + instance = nova_provider.NovaProvider() + mock_keyring.get_password.reset_mock() + + creds = instance._get_keystone_creds() + + expected = { + "auth_url": self.AUTH_URL, + "username": self.USERNAME, + "password": keyring_pass, + # "project_domain_name": None, + # "project_name": None, + } + self.assertDictEqual(expected, creds) + mock_keyring.get_password.assert_called_once_with( + self.KEYRING_SERVICE, + self.USERNAME, + ) diff --git a/utilities/pci-irq-affinity-agent/pci_irq_affinity/tox.ini b/utilities/pci-irq-affinity-agent/pci_irq_affinity/tox.ini new file mode 100644 index 00000000..9b03209a --- /dev/null +++ b/utilities/pci-irq-affinity-agent/pci_irq_affinity/tox.ini @@ -0,0 +1,82 @@ +[tox] +envlist = py27,flake8 +minversion = 2.3 +skipsdist = True + +[testenv] +sitepackages = False +basepython = python2.7 +install_command = pip install -vvv -U {opts} {packages} +setenv = + VIRTUAL_ENV={envdir} + OS_STDOUT_CAPTURE=1 + OS_STDERR_CAPTURE=1 + OS_DEBUG=1 + OS_LOG_CAPTURE=1 +deps = + -r{toxinidir}/requirements.txt + -r{toxinidir}/test-requirements.txt +whitelist_externals = + bash +commands = + pytest tests/ + +[testenv:py36] +basepython = python3.6 + +[testenv:py39] +basepython = python3.9 + +[flake8] +# Note: hacking pulls in flake8 2.5.5 which can not parse an ignore list spanning multiple lines +# F errors are high priority to fix. W are warnings. E series are pep8, H series are hacking +# F401 'FOO' imported but unused +# F841 local variable 'FOO' is assigned to but never used +# W291 trailing whitespace +# W391 blank line at end of file +# W503 line break before binary operator +# W504 line break after binary operator +# W605 invalid escape sequence '\s' +# E128 continuation line under-indented for visual indent +# E221 multiple spaces before operator +# E226 missing whitespace around arithmetic operator +# E227 missing whitespace around bitwise or shift operator +# E241 multiple spaces after ',' +# E265 block comment should start with '# ' +# E302 expected 2 blank lines, found 1 +# E305 expected 2 blank lines after class or function definition, found 1 +# E501 line too long +# E502 the backslash is redundant between brackets +# E702 multiple statements on one line (semicolon) +# E722 do not use bare 'except' +# H101 is TODO +# H104 File contains nothing but comments +# H201 no 'except:' at least use 'except Exception:' +# H238 old style class declaration, use new style (inherit from `object`) +# H306 imports not in alphabetical order (sys, re) +# H401 docstring should not start with a space +# H405 multi line docstring summary not separated with an empty line +ignore = F401,F841,W291,W391,W503,W504,W605,E128,E221,E226,E227,E241,E265,E302,E305,E501,E502,E702,E722,H101,H104,H201,H238,H306,H401,H405 +# max-line-length is not referenced if E501 is suppressed +max-line-length=80 + +[testenv:flake8] +commands = + flake8 + +[testenv:pylint] +commands = pylint {posargs} pci_irq_affinity --rcfile=../../../pylint.rc + +[testenv:bandit] +description = Dummy environment to allow bandit to be run in subdir tox + +[testenv:bindep] +# Do not install any requirements. We want this to be fast and work even if +# system dependencies are missing, since it's used to tell you what system +# dependencies are missing! This also means that bindep must be installed +# separately, outside of the requirements files, and develop mode disabled +# explicitly to avoid unnecessarily installing the checked-out repo too (this +# further relies on "tox.skipsdist = True" above). +deps = bindep +commands = bindep test +usedevelop = False \ No newline at end of file