From c7a35dd275572a2113d3bef829ab4fa706e582eb Mon Sep 17 00:00:00 2001 From: Jim Gauld Date: Thu, 24 Oct 2019 13:01:12 -0400 Subject: [PATCH] Remove pm-qos-mgr daemon We observe poor cyclictest results on lowlatency nodes. This was due to inconsistent PM-QoS values set across all cpus. The PM-QoS value is now set uniformly across all cpus on lowlatency nodes. The pm-qos-mgr daemon is no longer needed. This update removes pm-qos-mgr daemon completely. Change-Id: Icf70c358f043a2fe7398ce8a59c6cd2ff14237d0 Partial-Bug: 1845735 Depends-On: https://review.opendev.org/691020 Signed-off-by: Jim Gauld --- centos_iso_image.inc | 2 - centos_pkg_dirs | 1 - tox.ini | 1 - utilities/pm-qos-mgr/centos/build_srpm.data | 2 - utilities/pm-qos-mgr/centos/pm-qos-mgr.spec | 66 ------ utilities/pm-qos-mgr/src/LICENSE | 202 ------------------ utilities/pm-qos-mgr/src/pm-qos-mgr.conf | 10 - utilities/pm-qos-mgr/src/pm-qos-mgr.service | 12 -- .../pm-qos-mgr/src/pm_qos_mgr/__init__.py | 0 .../pm-qos-mgr/src/pm_qos_mgr/pm_qos_mgr.py | 165 -------------- utilities/pm-qos-mgr/src/setup.cfg | 38 ---- utilities/pm-qos-mgr/src/setup.py | 12 -- 12 files changed, 511 deletions(-) delete mode 100644 utilities/pm-qos-mgr/centos/build_srpm.data delete mode 100644 utilities/pm-qos-mgr/centos/pm-qos-mgr.spec delete mode 100644 utilities/pm-qos-mgr/src/LICENSE delete mode 100644 utilities/pm-qos-mgr/src/pm-qos-mgr.conf delete mode 100644 utilities/pm-qos-mgr/src/pm-qos-mgr.service delete mode 100644 utilities/pm-qos-mgr/src/pm_qos_mgr/__init__.py delete mode 100644 utilities/pm-qos-mgr/src/pm_qos_mgr/pm_qos_mgr.py delete mode 100644 utilities/pm-qos-mgr/src/setup.cfg delete mode 100644 utilities/pm-qos-mgr/src/setup.py diff --git a/centos_iso_image.inc b/centos_iso_image.inc index 81fef845..80aa4cc0 100644 --- a/centos_iso_image.inc +++ b/centos_iso_image.inc @@ -1,5 +1,3 @@ -pm-qos-mgr - # worker-utils worker-utils diff --git a/centos_pkg_dirs b/centos_pkg_dirs index d6258553..603a1962 100644 --- a/centos_pkg_dirs +++ b/centos_pkg_dirs @@ -1,4 +1,3 @@ -utilities/pm-qos-mgr utilities/worker-utils ceph/ceph-manager ceph/python-cephclient diff --git a/tox.ini b/tox.ini index 5e22eec4..0a5a5b85 100644 --- a/tox.ini +++ b/tox.ini @@ -95,7 +95,6 @@ commands = pylint {posargs} \ utilities/logmgmt/logmgmt/logmgmt/ \ utilities/pci-irq-affinity-agent/pci_irq_affinity/pci_irq_affinity \ utilities/platform-util/platform-util/platform_util \ - utilities/pm-qos-mgr/src/pm_qos_mgr \ --rcfile=./pylint.rc diff --git a/utilities/pm-qos-mgr/centos/build_srpm.data b/utilities/pm-qos-mgr/centos/build_srpm.data deleted file mode 100644 index f579f0d2..00000000 --- a/utilities/pm-qos-mgr/centos/build_srpm.data +++ /dev/null @@ -1,2 +0,0 @@ -SRC_DIR="src" -TIS_PATCH_VER=2 diff --git a/utilities/pm-qos-mgr/centos/pm-qos-mgr.spec b/utilities/pm-qos-mgr/centos/pm-qos-mgr.spec deleted file mode 100644 index 9387da5e..00000000 --- a/utilities/pm-qos-mgr/centos/pm-qos-mgr.spec +++ /dev/null @@ -1,66 +0,0 @@ -%define debug_package %{nil} -%global pypi_name pm_qos_mgr - -Name: pm-qos-mgr -Version: 1.0 -Release: %{tis_patch_ver}%{?_tis_dist} -Summary: PM QoS CPU wakeup latency manager for kubelet cpu-manager -License: Apache-2.0 -Group: base -URL: unknown -Source0: %{name}-%{version}.tar.gz - -BuildRequires: git -BuildRequires: python-pbr >= 2.0.0 -BuildRequires: python-setuptools -BuildRequires: python2-pip -BuildRequires: systemd-devel - -Requires: python-pbr >= 2.0.0 -Requires: python-inotify -Requires: systemd - -%description -A daemon that monitors kubelet cpu-manager static cpu assignments -and modifies PM QoS CPU wakeup latency. - -%define pythonroot %{_libdir}/python2.7/site-packages -%define local_etc_pmond /etc/pmon.d/ - -%prep -%autosetup -n %{name}-%{version} -S git - -# Remove bundled egg-info -rm -rf *.egg-info - -%build -export PBR_VERSION=%{version} -%{__python} setup.py build - -%install -export PBR_VERSION=%{version} -%{__python} setup.py install --root=%{buildroot} \ - --install-lib=%{pythonroot} \ - --prefix=%{_prefix} \ - --install-data=%{_datadir} \ - --single-version-externally-managed - -install -d -m 755 %{buildroot}%{local_etc_pmond} -install -p -D -m 644 pm-qos-mgr.conf %{buildroot}%{local_etc_pmond}/pm-qos-mgr.conf - -install -p -D -m 664 pm-qos-mgr.service %{buildroot}%{_unitdir}/pm-qos-mgr.service - -%post -systemctl enable pm-qos-mgr.service - -%clean -rm -rf $RPM_BUILD_ROOT - -%files -%defattr(-,root,root,-) -%doc LICENSE -%{_bindir}/* -%{pythonroot}/%{pypi_name}/* -%{pythonroot}/%{pypi_name}-%{version}*.egg-info -%{local_etc_pmond}/pm-qos-mgr.conf -%{_unitdir}/* diff --git a/utilities/pm-qos-mgr/src/LICENSE b/utilities/pm-qos-mgr/src/LICENSE deleted file mode 100644 index d6456956..00000000 --- a/utilities/pm-qos-mgr/src/LICENSE +++ /dev/null @@ -1,202 +0,0 @@ - - Apache License - Version 2.0, January 2004 - http://www.apache.org/licenses/ - - TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION - - 1. Definitions. - - "License" shall mean the terms and conditions for use, reproduction, - and distribution as defined by Sections 1 through 9 of this document. - - "Licensor" shall mean the copyright owner or entity authorized by - the copyright owner that is granting the License. - - "Legal Entity" shall mean the union of the acting entity and all - other entities that control, are controlled by, or are under common - control with that entity. For the purposes of this definition, - "control" means (i) the power, direct or indirect, to cause the - direction or management of such entity, whether by contract or - otherwise, or (ii) ownership of fifty percent (50%) or more of the - outstanding shares, or (iii) beneficial ownership of such entity. - - "You" (or "Your") shall mean an individual or Legal Entity - exercising permissions granted by this License. - - "Source" form shall mean the preferred form for making modifications, - including but not limited to software source code, documentation - source, and configuration files. - - "Object" form shall mean any form resulting from mechanical - transformation or translation of a Source form, including but - not limited to compiled object code, generated documentation, - and conversions to other media types. - - "Work" shall mean the work of authorship, whether in Source or - Object form, made available under the License, as indicated by a - copyright notice that is included in or attached to the work - (an example is provided in the Appendix below). - - "Derivative Works" shall mean any work, whether in Source or Object - form, that is based on (or derived from) the Work and for which the - editorial revisions, annotations, elaborations, or other modifications - represent, as a whole, an original work of authorship. For the purposes - of this License, Derivative Works shall not include works that remain - separable from, or merely link (or bind by name) to the interfaces of, - the Work and Derivative Works thereof. - - "Contribution" shall mean any work of authorship, including - the original version of the Work and any modifications or additions - to that Work or Derivative Works thereof, that is intentionally - submitted to Licensor for inclusion in the Work by the copyright owner - or by an individual or Legal Entity authorized to submit on behalf of - the copyright owner. For the purposes of this definition, "submitted" - means any form of electronic, verbal, or written communication sent - to the Licensor or its representatives, including but not limited to - communication on electronic mailing lists, source code control systems, - and issue tracking systems that are managed by, or on behalf of, the - Licensor for the purpose of discussing and improving the Work, but - excluding communication that is conspicuously marked or otherwise - designated in writing by the copyright owner as "Not a Contribution." - - "Contributor" shall mean Licensor and any individual or Legal Entity - on behalf of whom a Contribution has been received by Licensor and - subsequently incorporated within the Work. - - 2. Grant of Copyright License. Subject to the terms and conditions of - this License, each Contributor hereby grants to You a perpetual, - worldwide, non-exclusive, no-charge, royalty-free, irrevocable - copyright license to reproduce, prepare Derivative Works of, - publicly display, publicly perform, sublicense, and distribute the - Work and such Derivative Works in Source or Object form. - - 3. Grant of Patent License. Subject to the terms and conditions of - this License, each Contributor hereby grants to You a perpetual, - worldwide, non-exclusive, no-charge, royalty-free, irrevocable - (except as stated in this section) patent license to make, have made, - use, offer to sell, sell, import, and otherwise transfer the Work, - where such license applies only to those patent claims licensable - by such Contributor that are necessarily infringed by their - Contribution(s) alone or by combination of their Contribution(s) - with the Work to which such Contribution(s) was submitted. If You - institute patent litigation against any entity (including a - cross-claim or counterclaim in a lawsuit) alleging that the Work - or a Contribution incorporated within the Work constitutes direct - or contributory patent infringement, then any patent licenses - granted to You under this License for that Work shall terminate - as of the date such litigation is filed. - - 4. Redistribution. You may reproduce and distribute copies of the - Work or Derivative Works thereof in any medium, with or without - modifications, and in Source or Object form, provided that You - meet the following conditions: - - (a) You must give any other recipients of the Work or - Derivative Works a copy of this License; and - - (b) You must cause any modified files to carry prominent notices - stating that You changed the files; and - - (c) You must retain, in the Source form of any Derivative Works - that You distribute, all copyright, patent, trademark, and - attribution notices from the Source form of the Work, - excluding those notices that do not pertain to any part of - the Derivative Works; and - - (d) If the Work includes a "NOTICE" text file as part of its - distribution, then any Derivative Works that You distribute must - include a readable copy of the attribution notices contained - within such NOTICE file, excluding those notices that do not - pertain to any part of the Derivative Works, in at least one - of the following places: within a NOTICE text file distributed - as part of the Derivative Works; within the Source form or - documentation, if provided along with the Derivative Works; or, - within a display generated by the Derivative Works, if and - wherever such third-party notices normally appear. The contents - of the NOTICE file are for informational purposes only and - do not modify the License. You may add Your own attribution - notices within Derivative Works that You distribute, alongside - or as an addendum to the NOTICE text from the Work, provided - that such additional attribution notices cannot be construed - as modifying the License. - - You may add Your own copyright statement to Your modifications and - may provide additional or different license terms and conditions - for use, reproduction, or distribution of Your modifications, or - for any such Derivative Works as a whole, provided Your use, - reproduction, and distribution of the Work otherwise complies with - the conditions stated in this License. - - 5. Submission of Contributions. Unless You explicitly state otherwise, - any Contribution intentionally submitted for inclusion in the Work - by You to the Licensor shall be under the terms and conditions of - this License, without any additional terms or conditions. - Notwithstanding the above, nothing herein shall supersede or modify - the terms of any separate license agreement you may have executed - with Licensor regarding such Contributions. - - 6. Trademarks. This License does not grant permission to use the trade - names, trademarks, service marks, or product names of the Licensor, - except as required for reasonable and customary use in describing the - origin of the Work and reproducing the content of the NOTICE file. - - 7. Disclaimer of Warranty. Unless required by applicable law or - agreed to in writing, Licensor provides the Work (and each - Contributor provides its Contributions) on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or - implied, including, without limitation, any warranties or conditions - of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A - PARTICULAR PURPOSE. You are solely responsible for determining the - appropriateness of using or redistributing the Work and assume any - risks associated with Your exercise of permissions under this License. - - 8. Limitation of Liability. In no event and under no legal theory, - whether in tort (including negligence), contract, or otherwise, - unless required by applicable law (such as deliberate and grossly - negligent acts) or agreed to in writing, shall any Contributor be - liable to You for damages, including any direct, indirect, special, - incidental, or consequential damages of any character arising as a - result of this License or out of the use or inability to use the - Work (including but not limited to damages for loss of goodwill, - work stoppage, computer failure or malfunction, or any and all - other commercial damages or losses), even if such Contributor - has been advised of the possibility of such damages. - - 9. Accepting Warranty or Additional Liability. While redistributing - the Work or Derivative Works thereof, You may choose to offer, - and charge a fee for, acceptance of support, warranty, indemnity, - or other liability obligations and/or rights consistent with this - License. However, in accepting such obligations, You may act only - on Your own behalf and on Your sole responsibility, not on behalf - of any other Contributor, and only if You agree to indemnify, - defend, and hold each Contributor harmless for any liability - incurred by, or claims asserted against, such Contributor by reason - of your accepting any such warranty or additional liability. - - END OF TERMS AND CONDITIONS - - APPENDIX: How to apply the Apache License to your work. - - To apply the Apache License to your work, attach the following - boilerplate notice, with the fields enclosed by brackets "[]" - replaced with your own identifying information. (Don't include - the brackets!) The text should be enclosed in the appropriate - comment syntax for the file format. We also recommend that a - file or class name and description of purpose be included on the - same "printed page" as the copyright notice for easier - identification within third-party archives. - - Copyright [yyyy] [name of copyright owner] - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. diff --git a/utilities/pm-qos-mgr/src/pm-qos-mgr.conf b/utilities/pm-qos-mgr/src/pm-qos-mgr.conf deleted file mode 100644 index e4f6a417..00000000 --- a/utilities/pm-qos-mgr/src/pm-qos-mgr.conf +++ /dev/null @@ -1,10 +0,0 @@ -[process] -process = pm-qos-mgr -pidfile = /var/run/pm-qos-mgr.pid -service = pm-qos-mgr -style = lsb ; ocf or lsb -severity = major ; minor, major, critical -restarts = 3 ; restarts before error assertion -interval = 5 ; number of seconds to wait between restarts -debounce = 20 ; number of seconds to wait before degrade clear -subfunction = last-config ; run it only after last config is run diff --git a/utilities/pm-qos-mgr/src/pm-qos-mgr.service b/utilities/pm-qos-mgr/src/pm-qos-mgr.service deleted file mode 100644 index 0a2a6c6f..00000000 --- a/utilities/pm-qos-mgr/src/pm-qos-mgr.service +++ /dev/null @@ -1,12 +0,0 @@ -[Unit] -Description=PM QoS CPU wakeup latency manager for kubelet cpu-manager -After=syslog.target network-online.target remote-fs.target sw-patch.service -Before=kubelet.service - -[Service] -Type=simple -ExecStart=/usr/bin/pm-qos-mgr -ExecStartPost=/bin/bash -c 'echo $MAINPID > /var/run/pm-qos-mgr.pid' - -[Install] -WantedBy=multi-user.target diff --git a/utilities/pm-qos-mgr/src/pm_qos_mgr/__init__.py b/utilities/pm-qos-mgr/src/pm_qos_mgr/__init__.py deleted file mode 100644 index e69de29b..00000000 diff --git a/utilities/pm-qos-mgr/src/pm_qos_mgr/pm_qos_mgr.py b/utilities/pm-qos-mgr/src/pm_qos_mgr/pm_qos_mgr.py deleted file mode 100644 index 02ab3265..00000000 --- a/utilities/pm-qos-mgr/src/pm_qos_mgr/pm_qos_mgr.py +++ /dev/null @@ -1,165 +0,0 @@ -# -# Copyright (c) 2019 Wind River Systems, Inc. -# -# SPDX-License-Identifier: Apache-2.0 -# -# Purpose: -# This manager watches for changes in /var/lib/kubelet/cpu_manager_state file. -# This sets appropriate PM QoS resume latency constraints for CPUs -# when kubelet cpu-manager is configured with 'static' policy. -# -# This parses the cpu_manager_state file, deduces the cpu-manager policy, -# and CPU lists of Guaranteed pods versus the remaining Default CPUs. -# Guaranteed pods with exclusive CPUs get "low" cpu wakeup latency policy. -# Default CPUs get "high" cpu wakeup latency policy. - -import itertools as it -import json -import logging -import logging.handlers -import os -import pyinotify -import subprocess -import sys - -# Global variables -statefile = '/var/lib/kubelet/cpu_manager_state' -pm_script = '/usr/bin/set-cpu-wakeup-latency.sh' - -LOG = logging.getLogger(__name__) - -def configure_logging(logger, level=logging.DEBUG): - """ Configure logger streams and format. """ - LOG.setLevel(level) - - syslog_facility = logging.handlers.SysLogHandler.LOG_DAEMON - ch = logging.handlers.SysLogHandler(address='/dev/log', - facility=syslog_facility) - ch.setLevel(level) - formatter = logging.Formatter('%(module)s[%(process)d]: %(message)s') - ch.setFormatter(formatter) - LOG.addHandler(ch) - - -def format_range_set(items): - """ Generate pretty-printed value of ranges, such as 3-6,12-17. """ - ranges = [] - for k, iterable in it.groupby(enumerate(sorted(items)), - lambda x: x[1] - x[0]): - rng = list(iterable) - if len(rng) == 1: - s = str(rng[0][1]) - else: - s = "%s-%s" % (rng[0][1], rng[-1][1]) - ranges.append(s) - return ','.join(ranges) - - -def range_to_list(csv_range=None): - """ Convert a string of comma separate ranges into an expanded list - of integers. e.g., '1-3,8-9,15' is converted to [1,2,3,8,9,15]. - """ - if not csv_range: - return [] - ranges = [(lambda L: range(L[0], L[-1] + 1))(map(int, r.split('-'))) - for r in csv_range.split(',')] - return [y for x in ranges for y in x] - - -class ProcessTransientFile(pyinotify.ProcessEvent): - def __init__(self, *args, **kw): - self.policy = None - self.cpusets = {'default': set(), - 'guaranteed': set()} - self.update_pm_qos_cpu_latency() - - def update_pm_qos_cpu_latency(self, event=None): - if self.policy is not None and self.policy != 'static': - return - if event is not None: - LOG.debug('%s, %s', event.pathname, event.maskname) - - # Read JSON formatted state file dictionary - state = {} - try: - with open(statefile, 'r') as f: - state = json.load(f) - except Exception as e: - LOG.error('Could not load: %s, error: %s.', statefile, e) - return - - self.policy = str(state['policyName']) - if self.policy != 'static': - return - - # Determine default cpuset - if 'defaultCpuSet' not in state: - LOG.error('Missing defaultCpuSet.', statefile) - return - default_cpuranges = str(state['defaultCpuSet']) - default_cpuset = set(range_to_list(csv_range=default_cpuranges)) - - # Determine guaranteed cpuset - guaranteed_cpuset = set() - if 'entries' in state: - for pod, cpus in state['entries'].items(): - cpulist = range_to_list(csv_range=cpus) - guaranteed_cpuset.update(cpulist) - guaranteed_cpuranges = format_range_set(guaranteed_cpuset) - - # Update PM QoS resume latency if the set of cpus have changed - if default_cpuset != self.cpusets['default']: - self.cpusets['default'] = default_cpuset.copy() - if default_cpuset: - pm_policy = 'high' - LOG.info('Set PM policy: %s, CPUs: %s', - pm_policy, default_cpuranges) - command = [pm_script, pm_policy, default_cpuranges] - proc = subprocess.Popen(command, stdout=subprocess.PIPE) - output, errors = proc.communicate() - if errors: - LOG.error('Problem with command: %s, error: %s', - command, errors) - - if guaranteed_cpuset != self.cpusets['guaranteed']: - self.cpusets['guaranteed'] = guaranteed_cpuset.copy() - if guaranteed_cpuset: - pm_policy = 'low' - LOG.info('Set PM policy: %s, CPUs: %s', - pm_policy, guaranteed_cpuranges) - command = [pm_script, pm_policy, guaranteed_cpuranges] - proc = subprocess.Popen(command, stdout=subprocess.PIPE) - output, errors = proc.communicate() - if errors: - LOG.error('Problem with command: %s, error: %s', - command, errors) - - def process_IN_MOVED_TO(self, event): - """ Handler for watched IN_MOVED_TO events. - - kubelet cpu-manager overwrites state-file by moving a temp file, - so this is the expected handler. - """ - self.update_pm_qos_cpu_latency(event) - - -def main(): - """ A shell command for pm-qos-daemon. """ - configure_logging(LOG, level=logging.INFO) - if os.geteuid() != 0: - LOG.error('Require sudo/root.') - sys.exit(1) - - LOG.info('Watching: %s', statefile) - watch_manager = pyinotify.WatchManager() - notifier = pyinotify.Notifier(watch_manager) - flags = pyinotify.IN_MOVED_TO - watch_manager.watch_transient_file(statefile, flags, ProcessTransientFile) - - try: - notifier.loop() - except pyinotify.NotifierError as err: - LOG.error('Problem with notifier.loop(), error: %s', err) - -if __name__ == "__main__": - main() diff --git a/utilities/pm-qos-mgr/src/setup.cfg b/utilities/pm-qos-mgr/src/setup.cfg deleted file mode 100644 index 63d503e4..00000000 --- a/utilities/pm-qos-mgr/src/setup.cfg +++ /dev/null @@ -1,38 +0,0 @@ -[metadata] -license_files = LICENSE -name = pm_qos_mgr -summary = PM QoS CPU wakeup latency manager for kubelet cpu-manager -author = StarlingX -author-email = starlingx-discuss@lists.starlingx.io -home-page = http://www.starlingx.io/ -classifier = - Environment :: OpenStack - Intended Audience :: Information Technology - Intended Audience :: System Administrators - License :: OSI Approved :: Apache Software License - Operating System :: POSIX :: Linux - Programming Language :: Python - Programming Language :: Python :: 2 - Programming Language :: Python :: 2.7 - Programming Language :: Python :: 3 - Programming Language :: Python :: 3.5 - -[global] -setup-hooks = - pbr.hooks.setup_hook - -[files] -packages = - pm_qos_mgr - -[entry_points] -console_scripts = - pm-qos-mgr = pm_qos_mgr.pm_qos_mgr:main - -[wheel] -universal = 1 - -[egg_info] -tag_build = -tag_date = 0 -tag_svn_revision = 0 diff --git a/utilities/pm-qos-mgr/src/setup.py b/utilities/pm-qos-mgr/src/setup.py deleted file mode 100644 index 6ca01bee..00000000 --- a/utilities/pm-qos-mgr/src/setup.py +++ /dev/null @@ -1,12 +0,0 @@ -# -# Copyright (c) 2019 Wind River Systems, Inc. -# -# SPDX-License-Identifier: Apache-2.0 -# - -import setuptools - -setuptools.setup( - setup_requires=['pbr>=2.0.0'], - pbr=True) -