diff --git a/.zuul.yaml b/.zuul.yaml index 7647884..b2d364a 100644 --- a/.zuul.yaml +++ b/.zuul.yaml @@ -5,12 +5,36 @@ check: jobs: - openstack-tox-linters + - kube-cpusets-tox-py27 + - kube-cpusets-tox-py36 + - kube-cpusets-tox-flake8 + - kube-cpusets-tox-pylint + - kube-memory-tox-py27 + - kube-memory-tox-py36 + - kube-memory-tox-flake8 + - kube-memory-tox-pylint gate: jobs: - openstack-tox-linters + - kube-cpusets-tox-py27 + - kube-cpusets-tox-py36 + - kube-cpusets-tox-flake8 + - kube-cpusets-tox-pylint + - kube-memory-tox-py27 + - kube-memory-tox-py36 + - kube-memory-tox-flake8 + - kube-memory-tox-pylint post: jobs: - stx-monitoring-upload-git-mirror + - kube-cpusets-tox-py27 + - kube-cpusets-tox-py36 + - kube-cpusets-tox-flake8 + - kube-cpusets-tox-pylint + - kube-memory-tox-py27 + - kube-memory-tox-py36 + - kube-memory-tox-flake8 + - kube-memory-tox-pylint - job: name: stx-monitoring-upload-git-mirror @@ -25,6 +49,119 @@ secret: stx-monitoring-github-secret pass-to-parent: true +- job: + name: kube-cpusets-tox-pylint + parent: openstack-tox-pylint + description: | + Run pylint test for kube-cpusets + required-projects: + - starlingx/config + nodeset: ubuntu-bionic + files: + - kube-cpusets/* + vars: + tox_envlist: pylint + tox_extra_args: -c kube-cpusets/kube-cpusets/tox.ini + +- job: + name: kube-cpusets-tox-flake8 + parent: tox + description: | + Run flake8 test for kube-cpusets + required-projects: + - starlingx/config + nodeset: ubuntu-bionic + files: + - kube-cpusets/kube-cpusets/* + vars: + tox_envlist: flake8 + tox_extra_args: -c kube-cpusets/kube-cpusets/tox.ini + +- job: + name: kube-cpusets-tox-py36 + parent: tox + description: | + Run py36 test for kube-cpusets + required-projects: + - starlingx/config + nodeset: ubuntu-bionic + files: + - kube-cpusets/kube-cpusets/* + vars: + tox_envlist: py36 + tox_extra_args: -c kube-cpusets/kube-cpusets/tox.ini + +- job: + name: kube-cpusets-tox-py27 + parent: tox + description: | + Run py27 test for kube-cpusets + nodeset: ubuntu-bionic + required-projects: + - starlingx/config + files: + - kube-cpusets/kube-cpusets/* + vars: + tox_envlist: py27 + tox_extra_args: -c kube-cpusets/kube-cpusets/tox.ini + +- job: + name: kube-memory-tox-pylint + parent: tox + required-projects: + - starlingx/config + nodeset: ubuntu-bionic + description: | + Run pylint test for kube-memory + files: + - kube-memory/kube-memory/* + vars: + tox_envlist: pylint + tox_extra_args: -c kube-memory/kube-memory/tox.ini + +- job: + name: kube-memory-tox-flake8 + parent: tox + description: | + Run flake8 test for kube-memory + nodeset: ubuntu-bionic + required-projects: + - starlingx/config + files: + - kube-memory/kube-memory/* + vars: + tox_envlist: flake8 + tox_extra_args: -c kube-memory/kube-memory/tox.ini + +- job: + name: kube-memory-tox-py36 + parent: tox + description: | + Run py36 test for kube-memory + nodeset: ubuntu-bionic + required-projects: + - starlingx/config + files: + - kube-memory/kube-memory/* + vars: + tox_envlist: py36 + tox_extra_args: -c kube-memory/kube-memory/tox.ini + +- job: + name: kube-memory-tox-py27 + parent: tox + description: | + Run py27 test for kube-memory + nodeset: ubuntu-bionic + required-projects: + - starlingx/config + files: + - kube-memory/kube-memory/* + vars: + tox_envlist: py27 + tox_extra_args: -c kube-memory/kube-memory/tox.ini + + - secret: name: stx-monitoring-github-secret data: diff --git a/centos_iso_image.inc b/centos_iso_image.inc index 35634e2..6314669 100644 --- a/centos_iso_image.inc +++ b/centos_iso_image.inc @@ -13,3 +13,6 @@ vm-topology # kube-cpusets kube-cpusets + +# kube-memory +kube-memory diff --git a/centos_pkg_dirs b/centos_pkg_dirs index d678a8f..6d64a04 100644 --- a/centos_pkg_dirs +++ b/centos_pkg_dirs @@ -3,3 +3,4 @@ influxdb-extensions monitor-tools vm-topology kube-cpusets +kube-memory diff --git a/kube-cpusets/kube-cpusets/requirements.txt b/kube-cpusets/kube-cpusets/requirements.txt index 2683dd4..9dbc011 100644 --- a/kube-cpusets/kube-cpusets/requirements.txt +++ b/kube-cpusets/kube-cpusets/requirements.txt @@ -1,4 +1,3 @@ # The order of packages is significant, because pip processes them in the order # of appearance. Changing the order has an impact on the overall integration # process, which may cause wedges in the gate later. - diff --git a/kube-cpusets/kube-cpusets/test-requirements.txt b/kube-cpusets/kube-cpusets/test-requirements.txt index dadd6c5..8f6b16c 100644 --- a/kube-cpusets/kube-cpusets/test-requirements.txt +++ b/kube-cpusets/kube-cpusets/test-requirements.txt @@ -1,11 +1,10 @@ # The order of packages is significant, because pip processes them in the order # of appearance. Changing the order has an impact on the overall integration # process, which may cause wedges in the gate later. - +flake8<3.8.0 hacking>=1.1.0,<=2.0.0 # Apache-2.0 - -coverage!=4.4,>=4.0 # Apache-2.0 +pylint<2.1.0;python_version<"3.0" # GPLv2 +pylint<2.4.0;python_version>="3.0" # GPLv2coverage!=4.4,>=4.0 # Apache-2.0 mock>=2.0.0 # BSD stestr>=1.0.0 # Apache-2.0 testtools>=2.2.0 # MIT - diff --git a/kube-cpusets/kube-cpusets/tox.ini b/kube-cpusets/kube-cpusets/tox.ini index d08bd57..32bbe27 100644 --- a/kube-cpusets/kube-cpusets/tox.ini +++ b/kube-cpusets/kube-cpusets/tox.ini @@ -1,16 +1,12 @@ -# -# Copyright (c) 2020 Wind River Systems, Inc. +# Copyright (c) 2021 Wind River Systems, Inc. # # SPDX-License-Identifier: Apache-2.0 # - [tox] envlist = flake8,py27,py36,pylint,cover minversion = 2.3.2 skipsdist = True - stxdir = {toxinidir}/../../.. - [testenv] setenv = VIRTUAL_ENV={envdir} LANG=en_US.UTF-8 @@ -24,46 +20,36 @@ setenv = VIRTUAL_ENV={envdir} PYTHONHASHSEED=0 PYTHONWARNINGS=default::DeprecationWarning PIP_DISABLE_PIP_VERSION_CHECK=1 - passenv = XDG_CACHE_HOME - sitepackages = False install_command = pip install \ -v -v -v \ -c{env:UPPER_CONSTRAINTS_FILE:https://opendev.org/openstack/requirements/raw/branch/stable/stein/upper-constraints.txt} \ {opts} {packages} - deps = -r{toxinidir}/requirements.txt -r{toxinidir}/test-requirements.txt -e{[tox]stxdir}/config/tsconfig/tsconfig - whitelist_externals = find sh - [testenv:stestr] commands = find . -name "*.pyc" -delete stestr run {posargs} stestr slowest - [testenv:py27] basepython = python2.7 commands = {[testenv:stestr]commands} - [testenv:py36] basepython = python3.6 commands = {[testenv:stestr]commands} - [bandit] exclude = tests - [testenv:bandit] basepython = python3 deps = -r{toxinidir}/test-requirements.txt bandit commands = bandit --ini tox.ini -n 5 -r kube_cpusets - [flake8] show-source = True ignore = @@ -73,8 +59,6 @@ exclude = .venv,.git,.tox,dist,doc,*lib/python*,*egg,build,release-tag-* # enable: H904 Delay string interpolations at logging calls (off by default). enable-extensions = H106 H203 H904 max-line-length = 120 - - [testenv:flake8] basepython = python3 deps = {[testenv]deps} @@ -87,17 +71,14 @@ commands = [testenv:pylint] deps = {[testenv]deps} pylint - -basepython = python2.7 +basepython = python3.6 sitepackages = False - commands = pylint kube_cpusets --rcfile=./pylint.rc [testenv:cover] setenv = PYTHON=coverage run --parallel-mode PYTHONDONTWRITEBYTECODE=True - commands = coverage erase find . -name "*.pyc" -delete stestr run {posargs} diff --git a/kube-memory/PKG-INFO b/kube-memory/PKG-INFO new file mode 100644 index 0000000..a668aa1 --- /dev/null +++ b/kube-memory/PKG-INFO @@ -0,0 +1,12 @@ +Metadata-Version: 1.1 +Name: kube-memory +Version: 1.0 +Summary: Display services and kubernetes containers memory usage +Home-page: +Author: Windriver +Author-email: info@windriver.com +License: Apache-2.0 + +Description: Display services and kubernetes containers memory usage + +Platform: UNKNOWN diff --git a/kube-memory/centos/build_srpm.data b/kube-memory/centos/build_srpm.data new file mode 100644 index 0000000..7cde07e --- /dev/null +++ b/kube-memory/centos/build_srpm.data @@ -0,0 +1,4 @@ +PACKAGE_NAME=kube-memory +VERSION=1.0 +SRC_DIR=$PKG_BASE/$PACKAGE_NAME +TIS_PATCH_VER=PKG_GITREVCOUNT diff --git a/kube-memory/centos/kube-memory.spec b/kube-memory/centos/kube-memory.spec new file mode 100644 index 0000000..7eac184 --- /dev/null +++ b/kube-memory/centos/kube-memory.spec @@ -0,0 +1,71 @@ +%global pypi_name kube-memory + +Summary: Display services and kubernetes containers memory usage +Name: kube-memory +Version: 1.0 +Release: %{tis_patch_ver}%{?_tis_dist} +License: Apache-2.0 +Group: base +Packager: Wind River + +URL: unknown +Source0: %{pypi_name}-%{version}.tar.gz + +BuildArch: noarch + +BuildRequires: python +BuildRequires: python-setuptools +BuildRequires: python2-pip +BuildRequires: python2-wheel + +Requires: python +Requires: python-devel +Requires: /usr/bin/env +Requires: /bin/bash + +%define debug_package %{nil} + +%description +Display services and kubernetes containers memory usage + +%define pythonroot /usr/lib64/python2.7/site-packages + +%prep +%autosetup -p 1 -n %{pypi_name}-%{version} +# Remove bundled egg-info +rm -rf %{pypi_name}.egg-info +# Let RPM handle the dependencies +rm -f requirements.txt +rm -f test-requirements.txt + +%build +%{__python2} setup.py build +%py2_build_wheel + +%install +%{__python2} setup.py install --skip-build --root %{buildroot} +mkdir -p $RPM_BUILD_ROOT/wheels +install -m 644 dist/*.whl $RPM_BUILD_ROOT/wheels/ +mkdir -p %{buildroot} +install -d 755 -d %{buildroot}%{_sysconfdir}/collect.d +install -m 755 collect_kube_memory.sh %{buildroot}%{_sysconfdir}/collect.d/collect_kube_memory + +%files +%defattr(-,root,root,-) +%doc LICENSE +%{_bindir}/kube-memory +%{python2_sitelib}/kube_memory +%{python2_sitelib}/*.egg-info +%{_sysconfdir}/collect.d/* + +%package wheels +Summary: %{name} wheels + +%description wheels +Contains python wheels for %{name} + +%files wheels +/wheels/* + +%clean +rm -rf $RPM_BUILD_ROOT diff --git a/kube-memory/kube-memory/.coveragerc b/kube-memory/kube-memory/.coveragerc new file mode 100644 index 0000000..540ccab --- /dev/null +++ b/kube-memory/kube-memory/.coveragerc @@ -0,0 +1,7 @@ +[run] +branch = True +source = kube_memory +omit = kube_memory/tests/* + +[report] +ignore_errors = True diff --git a/kube-memory/kube-memory/.stestr.conf b/kube-memory/kube-memory/.stestr.conf new file mode 100644 index 0000000..a50225e --- /dev/null +++ b/kube-memory/kube-memory/.stestr.conf @@ -0,0 +1,2 @@ +[DEFAULT] +test_path=kube_memory/tests diff --git a/kube-memory/kube-memory/LICENSE b/kube-memory/kube-memory/LICENSE new file mode 100644 index 0000000..d645695 --- /dev/null +++ b/kube-memory/kube-memory/LICENSE @@ -0,0 +1,202 @@ + + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright [yyyy] [name of copyright owner] + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. diff --git a/kube-memory/kube-memory/collect_kube_memory.sh b/kube-memory/kube-memory/collect_kube_memory.sh new file mode 100644 index 0000000..750014e --- /dev/null +++ b/kube-memory/kube-memory/collect_kube_memory.sh @@ -0,0 +1,23 @@ +#! /bin/bash +######################################################################### +# +# Copyright (c) 2021 Wind River Systems, Inc. +# +# SPDX-License-Identifier: Apache-2.0 +# +######################################################################### + +# Loads Up Utilities and Commands Variables + +source /usr/local/sbin/collect_parms +source /usr/local/sbin/collect_utils + +SERVICE="kube_memory" +LOGFILE="${extradir}/${SERVICE}.info" + +# This displays the total resident set size per namespace and container, +# the aggregate memory usage per system service, and the platform memory usage. +delimiter ${LOGFILE} "kube-memory" +kube-memory >>${LOGFILE} + +exit 0 diff --git a/kube-memory/kube-memory/kube_memory/__init__.py b/kube-memory/kube-memory/kube_memory/__init__.py new file mode 100644 index 0000000..8133a68 --- /dev/null +++ b/kube-memory/kube-memory/kube_memory/__init__.py @@ -0,0 +1,6 @@ +""" +Copyright (c) 2021 Wind River Systems, Inc. + +SPDX-License-Identifier: Apache-2.0 + +""" diff --git a/kube-memory/kube-memory/kube_memory/kube_memory.py b/kube-memory/kube-memory/kube_memory/kube_memory.py new file mode 100644 index 0000000..53d72d9 --- /dev/null +++ b/kube-memory/kube-memory/kube_memory/kube_memory.py @@ -0,0 +1,527 @@ +# +# Copyright (c) 2021 Wind River Systems, Inc. +# +# SPDX-License-Identifier: Apache-2.0 +# + +""" +This tool gathers memory usage information for all kubernetes containers and +system services displayed in cgroup memory that are running on the current +host. + +This displays the total resident set size per container. +This displays the aggregate memory usage per system service. + +Usage: kube-memory [-h] [--debug] +""" + +import argparse +import json +import logging +import os +import re +import subprocess +import sys + +import prettytable + +# Constants +MEMINFO = '/proc/meminfo' +MEMPATH = '/sys/fs/cgroup/memory/' +BYTES_IN_MEBIBYTE = 1048576 +KBYTE = 1024 +DECIMAL_DIGITS = 2 + +MEMORY = {} +MEMORY['cgroups'] = {} +MEMORY['namespaces'] = {} + +RESERVED_CONF = '/etc/platform/worker_reserved.conf' + +BASE_GROUPS = ['docker', 'system.slice', 'user.slice'] +K8S_NAMESPACE_SYSTEM = ['kube-system', 'armada', 'cert-manager', 'portieris', + 'vault', 'notification', 'platform-deployment-manager'] +K8S_NAMESPACE_ADDON = ['monitor', 'openstack'] + +# Used commands +AWK_CMD = ["awk", "$2>0{print$0}"] +GREP_CMD = ["grep", "-rs", "total_rss"] + +# logger +LOG = logging.getLogger(__name__) + + +def mem_to_mebibytes(n_bytes): + """Convert a string that represents memory in bytes into mebibytes(MiB) + + Output is displayed with precision of 3 decimal digits. + e.g., '1829108992' is converted to 1744.374. + """ + try: + mebibytes = (float(n_bytes) / BYTES_IN_MEBIBYTE) + return str(round(mebibytes, DECIMAL_DIGITS)) + except (ValueError, TypeError): + return "-" + + +def pid_from_container(container_id): + """Get pid for a given a container id. + + """ + + cmd = ['pgrep', '-f', container_id] + try: + return subprocess.check_output(cmd, stderr=subprocess.STDOUT) + except subprocess.CalledProcessError as error: + LOG.error('Could not get pid, error=%s', error) + return 1 + + +def get_memory_cgroups(): + """Get system-level service groups from cgroup memory + + Returns a list of each system service name. + """ + groups = [] + for dirs in os.listdir(MEMPATH): + if os.path.isdir(MEMPATH + dirs): + groups.append(dirs) + return groups + + +def get_meminfo(): + """Get contents of /proc/meminfo. + + Returns dictionary containing each meminfo field integer. + i.e., m[field] + """ + + mem_info = {} + re_keyval = re.compile(r'^\s*(\S+)\s*[=:]\s*(\d+)') + try: + with open(MEMINFO, 'r') as mem_file: + for line in mem_file: + match = re_keyval.search(line) + if match: + keyfile = match.group(1) + val = match.group(2) + mem_info[keyfile] = int(val) + except IOError as err: + LOG.error('%s: Cannot read meminfo, error=%s', + 'platform memory usage', err) + return mem_info + + return mem_info + + +def get_platform_reserved_memory(): + """Get platform reserved memory MiB by parsing worker_reserved.conf file. + + This value is provided by puppet resource file which is populated + via sysinv query. + + Returns total platform reserved MiB. + Returns 0.0 if worker_reserved.conf does not exist. + """ + re_keyval_arr = re.compile( + r'^\s*WORKER_BASE_RESERVED\s*[=:]\s*\(\s*(.*)\s*\)') + re_base_mem = re.compile(r'\"node\d+:(\d+)MB:\d+\"') + reserved_mebib = 0.0 + + # Match key=("value1" "value2" ... "valueN") + reserved_str = None + if os.path.exists(RESERVED_CONF): + try: + with open(RESERVED_CONF, 'r') as infile: + for line in infile: + match = re_keyval_arr.search(line) + if match: + reserved_str = match.group(1) + except Exception as err: + LOG.error( + '%s: Cannot parse file, error=%s', + 'platform memory usage', err) + return 0.0 + + # Parse and aggregate reserved memory from pattern like this: + # WORKER_BASE_MEMORY=("node0:1500MB:1" "node1:1500MB:1") + if reserved_str: + nodes = [ + int(x.group(1)) for x in re.finditer(re_base_mem, reserved_str)] + reserved_mebib = float(sum(x for x in nodes)) + return reserved_mebib + + +def pipe_command(*cmds, **kwargs): + """Creates a shell pipeline and run command + + :param *cmds(list): Lists of commands to be executed + :param **kwargs: cwd(str): Sets directory to execute subprocess.Popen() + :returns (str): Final command output + """ + cmd_list = [] + for cmd in cmds: + cmd_list.append(cmd) + + last_popen = [subprocess.Popen( + cmd_list[0], stdout=subprocess.PIPE, cwd=kwargs.pop("cwd", None))] + + for i in range(len(cmd_list) - 1): + last_popen.append(subprocess.Popen( + cmd_list[i + 1], stdin=last_popen[i].stdout, + stdout=subprocess.PIPE)) + last_popen[i].stdout.close() + return last_popen[-1].communicate()[0] + + +def gather_groups_memory(output_mem): + """Obtain total rss displayed in memory.stat for each group. + + :param output_mem(str): Total rss output + :returns (PrettyTable): Table with the final results. + """ + groups = get_memory_cgroups() + p_table = prettytable.PrettyTable( + ['Group', + 'Resident Set Size (MiB)' + ], caching=False) + p_table.align = 'l' + p_table.align['Resident Set Size (MiB)'] = 'r' + + # Get overall memory summary per group + total_rss = 0.0 + for group in groups: + for line in output_mem.split("\n"): + if group + "/memory.stat" in line: + total_rss += float(line.split()[1]) + rss_mem = mem_to_mebibytes(line.split()[1]) + MEMORY['cgroups'][group] = rss_mem + p_table.add_row( + [group, + rss_mem or '-', + ]) + break + + # Add overall rss memory + MEMORY['cgroups']['total_rss'] = mem_to_mebibytes(total_rss) + p_table.add_row( + ["Total cgroup-rss", + MEMORY['cgroups']['total_rss'] or '-', + ]) + return p_table + + +def gather_containers_memory(output_mem): + """Gather memory information for all kubernetes containers. + + :param output_mem(str): Total rss output + :returns (PrettyTable): Table with the final results. + """ + + # Get list of containers on this host + cmd = ['crictl', 'ps', '--output=json'] + try: + output = subprocess.check_output(cmd, stderr=subprocess.STDOUT) + LOG.debug('command: %s\n%s', ' '.join(cmd), output) + except subprocess.CalledProcessError as error: + LOG.error('Could not list containers, error=%s', error) + return 1 + + p_table = prettytable.PrettyTable( + ['namespace', + 'pod.name', + 'container.name', + 'container.id', + 'state', + 'QoS', + 'PID', + 'Resident Set Size (MiB)', + ], caching=False) + p_table.align = 'l' + p_table.align['Resident Set Size (MiB)'] = 'c' + + # Gather data for each container + c_json = json.loads(output) + + containers = {} + for cont in c_json['containers']: + containers[cont['id']] = { + 'name': cont['metadata']['name'], + 'pod.name': cont['labels']['io.kubernetes.pod.name'], + 'cont.name': cont['labels']['io.kubernetes.container.name'], + 'namespace': cont['labels']['io.kubernetes.pod.namespace'], + 'pod.SandboxId': cont['podSandboxId'], + 'state': cont['state'], + 'pod.uid': cont['labels']['io.kubernetes.pod.uid'], + } + + sandboxes = [] + for cid, cjson in sorted(containers.items(), + key=lambda kv: (kv[1]['namespace'], + kv[1]['name'], + kv[1]['cont.name'])): + cid_short = cid[0:13] + sbid_short = cjson['pod.SandboxId'][0:13] + pname = cjson['pod.name'] + cname = cjson['cont.name'] + namespace = cjson['namespace'] + cstate = cjson['state'] + + # Now that we have the container ids, get memory, PID and QoS info + rss_mem_cid = None + rss_mem_sbid = None + pid_cid = None + if namespace not in MEMORY['namespaces']: + MEMORY['namespaces'][namespace] = 0 + qos = "guaranteed" + for line in output_mem.split("\n"): + if cid_short in line: + rss_mem_cid = line.split()[1] + MEMORY['namespaces'][namespace] += float( + mem_to_mebibytes(rss_mem_cid)) + pid_cid = pid_from_container(cid_short) + qos_list = {"besteffort", "burstable"} + for c_qos in qos_list: + if c_qos in line: + qos = c_qos + break + elif sbid_short in line: + if sbid_short not in sandboxes: + rss_mem_sbid = line.split()[1] + sandboxes.append(sbid_short) + else: + rss_mem_sbid = "" + break + + # Display both container and sandbox rss memory + rss_mem = "container: " + mem_to_mebibytes(rss_mem_cid) + \ + "\nsandbox: " + mem_to_mebibytes(rss_mem_sbid) + + p_table.add_row( + [namespace, + pname, + cname, + cid_short, + cstate, + qos, + pid_cid, + rss_mem or '-', + ]) + return p_table + + +def sys_service_memory(): + """Break down memory per system service (system.slice) + + Returns a table with the final results. + """ + sort_cmd = ["sort", "-k", "2nr"] + + p_table = prettytable.PrettyTable( + ['Service', + 'Resident Set Size (MiB)', + ], caching=False) + p_table.align = 'l' + p_table.align['Resident Set Size (MiB)'] = 'r' + + try: + output = pipe_command(GREP_CMD, AWK_CMD, sort_cmd, + cwd=MEMPATH + "system.slice") + LOG.debug( + 'command: %s\n%s', + ' '.join(GREP_CMD + [MEMPATH] + AWK_CMD + sort_cmd), output) + except subprocess.CalledProcessError as error: + LOG.error('Could not get total_rss memory, error=%s', error) + return 1 + + for line in output.split("\n"): + service = line.split("memory.stat:total_rss ")[0] + rss_mem = line.split("memory.stat:total_rss ")[-1] + p_table.add_row( + [service, + mem_to_mebibytes(rss_mem), + ]) + + # Delete first row wich display total system.slice rss + p_table.del_row(0) + return p_table + + +def gather_info_and_display(): + """Gather memory info for all kubernetes containers and system services. + + This displays the total resident set size per container. + This displays the aggregate memory usage per system-level groupings. + """ + # Obtain total rss displayed in memory.stat for each group, + # container and service. + try: + output_mem = pipe_command(GREP_CMD, AWK_CMD, cwd=MEMPATH) + LOG.debug( + 'command: %s\n%s', + "grep -rs total_rss '/sys/fs/cgroup/memory/' " + "| awk '$2>0{print$0}' ", + output_mem) + except subprocess.CalledProcessError as error: + LOG.error('Could not get total_rss memory, error=%s', error) + return 1 + + mem_info = get_meminfo() + pt_groups = gather_groups_memory(output_mem) + pt_cont = gather_containers_memory(output_mem) + pt_serv = sys_service_memory() + + # Dump the tables out + print('\nPer groups memory usage:') + + # Get string to be printed and create list of elements separated by \n + list_of_table_lines = pt_groups.get_string().split('\n') + + # Use the first line (+---+-- ...) as horizontal rule to insert later + horizontal_line = list_of_table_lines[0] + + # Print the table, except last two lines ( "Total" row + final separator). + print("\n".join(list_of_table_lines[:-2])) + # Print separator, and finally the "Total" row. + print(horizontal_line) + print("\n".join(list_of_table_lines[-2:])) + + pt_namespc = prettytable.PrettyTable( + ['Namespace', + 'Resident Set Size (MiB)', + ], caching=False) + pt_namespc.align = 'l' + pt_namespc.align['Resident Set Size (MiB)'] = 'r' + + print('\nPer namespace memory usage:') + for n_s in MEMORY['namespaces']: + pt_namespc.add_row( + [n_s, + MEMORY['namespaces'][n_s], + ]) + print(pt_namespc) + + print('\nPer container memory usage:') + print(pt_cont) + + print('\nPer service memory usage:') + print(pt_serv) + + base_mebib = 0.0 + k8s_system = 0.0 + k8s_addon = 0.0 + platform_memory_percent = 0.0 + + # Calculate base memory usage (i.e., normal memory, exclude K8S and VMs) + # e.g., docker, system.slice, user.slice + for group in MEMORY['cgroups']: + if group in BASE_GROUPS: + base_mebib += float(MEMORY['cgroups'][group]) + + # K8S platform system usage (essential) and addons usage (non-essential) + for n_s in MEMORY['namespaces']: + if n_s in K8S_NAMESPACE_SYSTEM: + k8s_system += MEMORY['namespaces'][n_s] + elif n_s in K8S_NAMESPACE_ADDON: + k8s_addon += MEMORY['namespaces'][n_s] + + # Calculate platform memory usage + platform_mebib = base_mebib + k8s_system + + anon_mebib = float(mem_to_mebibytes( + mem_info['Active(anon)'] + mem_info['Inactive(anon)'])) * KBYTE + avail_mebib = float(mem_to_mebibytes( + mem_info['MemAvailable'])) * KBYTE + total_mebib = float(anon_mebib + avail_mebib) + + anon_percent = round(100 * anon_mebib / total_mebib, DECIMAL_DIGITS) + + reserved_mebib = get_platform_reserved_memory() + # Calculate platform memory in terms of percent reserved + if reserved_mebib > 0.0: + platform_memory_percent = round( + 100 * platform_mebib / reserved_mebib, DECIMAL_DIGITS) + + pt_platf = prettytable.PrettyTable( + ['Reserved', + 'Platform', + 'Base', + 'K8s Platform system', + 'k8s-addon' + ], caching=False) + pt_platf.align = 'l' + + pt_platf.add_row( + [reserved_mebib, + '{} ({}%)'.format(platform_mebib, platform_memory_percent), + base_mebib, + k8s_system, + k8s_addon + ]) + print('\nPlatform memory usage in MiB:') + print(pt_platf) + + pt_4k = prettytable.PrettyTable( + ['Anon', + 'Cgroup-rss', + 'Available', + 'Total' + ], caching=False) + pt_4k.align = 'l' + + pt_4k.add_row( + ['{} ({}%)'.format(anon_mebib, anon_percent), + MEMORY['cgroups']['total_rss'], + avail_mebib, + total_mebib + ]) + + print('\n4K memory usage in MiB:') + print(pt_4k) + + return 0 + + +def main(): + """Main program.""" + # Parse command line arguments + parser = argparse.ArgumentParser( + description='Display memory usage information ' + 'for all kubernetes containers ' + 'and services.') + parser.add_argument('--debug', + action='store_true', + help='display debug info') + args = parser.parse_args() + # Configure logging + if args.debug: + level = logging.DEBUG + else: + level = logging.INFO + out_hdlr = logging.StreamHandler(sys.stdout) + formatter = logging.Formatter( + '%(asctime)s %(process)s %(levelname)s %(module)s: %(message)s') + out_hdlr.setFormatter(formatter) + out_hdlr.setLevel(level) + LOG.addHandler(out_hdlr) + LOG.setLevel(level) + + # Limit access of this tool since some sysfs data requires root. + if os.geteuid() != 0: + LOG.error('Require sudo/root.') + sys.exit(1) + + try: + ret = gather_info_and_display() + sys.exit(ret) + + except KeyboardInterrupt as error: + LOG.info('caught: %r, shutting down', error) + sys.exit(0) + + except IOError: + sys.exit(0) + + except Exception as error: + LOG.error('exception: %r', error, exc_info=1) + sys.exit(-4) diff --git a/kube-memory/kube-memory/kube_memory/tests/__init__.py b/kube-memory/kube-memory/kube_memory/tests/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/kube-memory/kube-memory/kube_memory/tests/test_kube_memory.py b/kube-memory/kube-memory/kube_memory/tests/test_kube_memory.py new file mode 100644 index 0000000..e24f751 --- /dev/null +++ b/kube-memory/kube-memory/kube_memory/tests/test_kube_memory.py @@ -0,0 +1,13 @@ +# +# SPDX-License-Identifier: Apache-2.0 +# +# Copyright (c) 2021 Wind River Systems, Inc. +# + +import testtools + + +class KubeCpusetsTestCase(testtools.TestCase): + + def test_get_groups(self): + pass diff --git a/kube-memory/kube-memory/pylint.rc b/kube-memory/kube-memory/pylint.rc new file mode 100644 index 0000000..de30e02 --- /dev/null +++ b/kube-memory/kube-memory/pylint.rc @@ -0,0 +1,344 @@ +[MASTER] + +# Specify a configuration file. +#rcfile= + +# Python code to execute, usually for sys.path manipulation such as +# pygtk.require(). +#init-hook= + +# Profiled execution. +profile=no + +# Add files or directories to the blacklist. They should be base names, not +# paths. +ignore=CVS + +# Pickle collected data for later comparisons. +persistent=yes + +# List of plugins (as comma separated values of python modules names) to load, +# usually to register additional checkers. +# load-plugins= + +# DEPRECATED +include-ids=no + +# DEPRECATED +symbols=no + + +[MESSAGES CONTROL] + +# Enable the message, report, category or checker with the given id(s). You can +# either give multiple identifier separated by comma (,) or put this option +# multiple time. See also the "--disable" option for examples. +#enable= + +# Disable the message, report, category or checker with the given id(s). You +# can either give multiple identifiers separated by comma (,) or put this +# option multiple times (only on the command line, not in the configuration +# file where it should appear only once).You can also use "--disable=all" to +# disable everything first and then reenable specific checks. For example, if +# you want to run only the similarities checker, you can use "--disable=all +# --enable=similarities". If you want to run only the classes checker, but have +# no Warning level messages displayed, use"--disable=all --enable=classes +# --disable=W" +# Disable specific warnings: +# W0703: Catching too general exception +# Disable refactor for bad code smell: +# R0914: Too many local variables +# R0915: Too many statements +# Disable convention warnings: +# C0330: Wrong hanging indentation before block : inconsistent with flake8 +# C0325: Unnecessary parens after %r keyword : inconsistent with python 3 +# C0111: Missing %s docstring +disable= R0914, R0915, W0703, C0325, C0111, C0330 + +[REPORTS] + +# Set the output format. Available formats are text, parseable, colorized, msvs +# (visual studio) and html. You can also give a reporter class, eg +# mypackage.mymodule.MyReporterClass. +output-format=text + +# Put messages in a separate file for each module / package specified on the +# command line instead of printing them on stdout. Reports (if any) will be +# written in a file name "pylint_global.[txt|html]". +files-output=no + +# Tells whether to display a full report or only the messages +reports=yes + +# Python expression which should return a note less than 10 (10 is the highest +# note). You have access to the variables errors warning, statement which +# respectively contain the number of errors / warnings messages and the total +# number of statements analyzed. This is used by the global evaluation report +# (RP0004). +evaluation=10.0 - ((float(5 * error + warning + refactor + convention) / statement) * 10) + +# Add a comment according to your evaluation note. This is used by the global +# evaluation report (RP0004). +comment=no + +# Template used to display messages. This is a python new-style format string +# used to format the message information. See doc for all details +#msg-template= + + +[BASIC] + +# Required attributes for module, separated by a comma +required-attributes= + +# List of builtins function names that should not be used, separated by a comma +bad-functions=map,filter,apply,input,file + +# Good variable names which should always be accepted, separated by a comma +good-names=i,j,k,ex,Run,_ + +# Bad variable names which should always be refused, separated by a comma +bad-names=foo,bar,baz,toto,tutu,tata + +# Colon-delimited sets of names that determine each other's naming style when +# the name regexes allow several styles. +name-group= + +# Include a hint for the correct naming format with invalid-name +include-naming-hint=no + +# Regular expression matching correct function names +function-rgx=[a-z_][a-z0-9_]{2,30}$ + +# Naming hint for function names +function-name-hint=[a-z_][a-z0-9_]{2,30}$ + +# Regular expression matching correct variable names +variable-rgx=[a-z_][a-z0-9_]{2,30}$ + +# Naming hint for variable names +variable-name-hint=[a-z_][a-z0-9_]{2,30}$ + +# Regular expression matching correct constant names +const-rgx=(([A-Z_][A-Z0-9_]*)|(__.*__))$ + +# Naming hint for constant names +const-name-hint=(([A-Z_][A-Z0-9_]*)|(__.*__))$ + +# Regular expression matching correct attribute names +attr-rgx=[a-z_][a-z0-9_]{2,30}$ + +# Naming hint for attribute names +attr-name-hint=[a-z_][a-z0-9_]{2,30}$ + +# Regular expression matching correct argument names +argument-rgx=[a-z_][a-z0-9_]{2,30}$ + +# Naming hint for argument names +argument-name-hint=[a-z_][a-z0-9_]{2,30}$ + +# Regular expression matching correct class attribute names +class-attribute-rgx=([A-Za-z_][A-Za-z0-9_]{2,30}|(__.*__))$ + +# Naming hint for class attribute names +class-attribute-name-hint=([A-Za-z_][A-Za-z0-9_]{2,30}|(__.*__))$ + +# Regular expression matching correct inline iteration names +inlinevar-rgx=[A-Za-z_][A-Za-z0-9_]*$ + +# Naming hint for inline iteration names +inlinevar-name-hint=[A-Za-z_][A-Za-z0-9_]*$ + +# Regular expression matching correct class names +class-rgx=[A-Z_][a-zA-Z0-9]+$ + +# Naming hint for class names +class-name-hint=[A-Z_][a-zA-Z0-9]+$ + +# Regular expression matching correct module names +module-rgx=(([a-z_][a-z0-9_]*)|([A-Z][a-zA-Z0-9]+))$ + +# Naming hint for module names +module-name-hint=(([a-z_][a-z0-9_]*)|([A-Z][a-zA-Z0-9]+))$ + +# Regular expression matching correct method names +method-rgx=[a-z_][a-z0-9_]{2,30}$ + +# Naming hint for method names +method-name-hint=[a-z_][a-z0-9_]{2,30}$ + +# Regular expression which should only match function or class names that do +# not require a docstring. +no-docstring-rgx=__.*__ + +# Minimum line length for functions/classes that require docstrings, shorter +# ones are exempt. +docstring-min-length=-1 + + +[FORMAT] + +# Maximum number of characters on a single line. +max-line-length=80 + +# Regexp for a line that is allowed to be longer than the limit. +ignore-long-lines=^\s*(# )??$ + +# Allow the body of an if to be on the same line as the test if there is no +# else. +single-line-if-stmt=no + +# List of optional constructs for which whitespace checking is disabled +no-space-check=trailing-comma,dict-separator + +# Maximum number of lines in a module +max-module-lines=1000 + +# String used as indentation unit. This is usually " " (4 spaces) or "\t" (1 +# tab). +indent-string=' ' + +# Number of spaces of indent required inside a hanging or continued line. +indent-after-paren=4 + + +[LOGGING] + +# Logging modules to check that the string format arguments are in logging +# function parameter format +logging-modules=logging + + +[MISCELLANEOUS] + +# List of note tags to take in consideration, separated by a comma. +notes=FIXME,XXX,TODO + + +[SIMILARITIES] + +# Minimum lines number of a similarity. +min-similarity-lines=4 + +# Ignore comments when computing similarities. +ignore-comments=yes + +# Ignore docstrings when computing similarities. +ignore-docstrings=yes + +# Ignore imports when computing similarities. +ignore-imports=no + + +[TYPECHECK] + +# Tells whether missing members accessed in mixin class should be ignored. A +# mixin class is detected if its name ends with "mixin" (case insensitive). +ignore-mixin-members=yes + +# List of module names for which member attributes should not be checked +# (useful for modules/projects where namespaces are manipulated during runtime +# and thus existing member attributes cannot be deduced by static analysis +ignored-modules= + +# List of classes names for which member attributes should not be checked +# (useful for classes with attributes dynamically set). +ignored-classes=rpm,PKCS1_PSS + +# When zope mode is activated, add a predefined set of Zope acquired attributes +# to generated-members. +zope=no + +# List of members which are set dynamically and missed by pylint inference +# system, and so shouldn't trigger E0201 when accessed. Python regular +# expressions are accepted. +generated-members=REQUEST,acl_users,aq_parent + + +[VARIABLES] + +# Tells whether we should check for unused import in __init__ files. +init-import=no + +# A regular expression matching the name of dummy variables (i.e. expectedly +# not used). +dummy-variables-rgx=_$|dummy + +# List of additional names supposed to be defined in builtins. Remember that +# you should avoid to define new builtins when possible. +additional-builtins= + + +[CLASSES] + +# List of interface methods to ignore, separated by a comma. This is used for +# instance to not check methods defines in Zope's Interface base class. +ignore-iface-methods=isImplementedBy,deferred,extends,names,namesAndDescriptions,queryDescriptionFor,getBases,getDescriptionFor,getDoc,getName,getTaggedValue,getTaggedValueTags,isEqualOrExtendedBy,setTaggedValue,isImplementedByInstancesOf,adaptWith,is_implemented_by + +# List of method names used to declare (i.e. assign) instance attributes. +defining-attr-methods=__init__,__new__,setUp + +# List of valid names for the first argument in a class method. +valid-classmethod-first-arg=cls + +# List of valid names for the first argument in a metaclass class method. +valid-metaclass-classmethod-first-arg=mcs + + +[DESIGN] + +# Maximum number of arguments for function / method +max-args=5 + +# Argument names that match this expression will be ignored. Default to name +# with leading underscore +ignored-argument-names=_.* + +# Maximum number of locals for function / method body +max-locals=15 + +# Maximum number of return / yield for function / method body +max-returns=6 + +# Maximum number of branch for function / method body +max-branches=12 + +# Maximum number of statements in function / method body +max-statements=50 + +# Maximum number of parents for a class (see R0901). +max-parents=7 + +# Maximum number of attributes for a class (see R0902). +max-attributes=7 + +# Minimum number of public methods for a class (see R0903). +min-public-methods=2 + +# Maximum number of public methods for a class (see R0904). +max-public-methods=20 + + +[IMPORTS] + +# Deprecated modules which should not be used, separated by a comma +deprecated-modules=regsub,TERMIOS,Bastion,rexec + +# Create a graph of every (i.e. internal and external) dependencies in the +# given file (report RP0402 must not be disabled) +import-graph= + +# Create a graph of external dependencies in the given file (report RP0402 must +# not be disabled) +ext-import-graph= + +# Create a graph of internal dependencies in the given file (report RP0402 must +# not be disabled) +int-import-graph= + + +[EXCEPTIONS] + +# Exceptions that will emit a warning when being caught. Defaults to +# "Exception" +overgeneral-exceptions=Exception diff --git a/kube-memory/kube-memory/requirements.txt b/kube-memory/kube-memory/requirements.txt new file mode 100644 index 0000000..2683dd4 --- /dev/null +++ b/kube-memory/kube-memory/requirements.txt @@ -0,0 +1,4 @@ +# The order of packages is significant, because pip processes them in the order +# of appearance. Changing the order has an impact on the overall integration +# process, which may cause wedges in the gate later. + diff --git a/kube-memory/kube-memory/setup.cfg b/kube-memory/kube-memory/setup.cfg new file mode 100644 index 0000000..b08e14c --- /dev/null +++ b/kube-memory/kube-memory/setup.cfg @@ -0,0 +1,22 @@ +[metadata] +license_files = LICENSE +name = kube_memory +classifier = + Environment :: OpenStack + Intended Audience :: Information Technology + Intended Audience :: System Administrators + License :: OSI Approved :: Apache Software License + Operating System :: POSIX :: Linux + Programming Language :: Python + Programming Language :: Python :: 2 + Programming Language :: Python :: 2.7 + Programming Language :: Python :: 3 + Programming Language :: Python :: 3.5 + Programming Language :: Python :: 3.6 + +[files] +packages = + kube_memory + +[wheel] +universal = 1 diff --git a/kube-memory/kube-memory/setup.py b/kube-memory/kube-memory/setup.py new file mode 100644 index 0000000..0ef851f --- /dev/null +++ b/kube-memory/kube-memory/setup.py @@ -0,0 +1,18 @@ +#!/usr/bin/env python + +""" +Copyright (c) 2021 Wind River Systems, Inc. +SPDX-License-Identifier: Apache-2.0 +""" +import setuptools +setuptools.setup( + name='kube_memory', + version='1.0.0', + description='display services and kubernetes containers memory usage', + license='Apache-2.0', + packages=['kube_memory'], + entry_points={ + 'console_scripts': [ + 'kube-memory = kube_memory.kube_memory:main', + ]} +) diff --git a/kube-memory/kube-memory/test-requirements.txt b/kube-memory/kube-memory/test-requirements.txt new file mode 100644 index 0000000..8f6b16c --- /dev/null +++ b/kube-memory/kube-memory/test-requirements.txt @@ -0,0 +1,10 @@ +# The order of packages is significant, because pip processes them in the order +# of appearance. Changing the order has an impact on the overall integration +# process, which may cause wedges in the gate later. +flake8<3.8.0 +hacking>=1.1.0,<=2.0.0 # Apache-2.0 +pylint<2.1.0;python_version<"3.0" # GPLv2 +pylint<2.4.0;python_version>="3.0" # GPLv2coverage!=4.4,>=4.0 # Apache-2.0 +mock>=2.0.0 # BSD +stestr>=1.0.0 # Apache-2.0 +testtools>=2.2.0 # MIT diff --git a/kube-memory/kube-memory/tox.ini b/kube-memory/kube-memory/tox.ini new file mode 100644 index 0000000..1acfdf3 --- /dev/null +++ b/kube-memory/kube-memory/tox.ini @@ -0,0 +1,87 @@ +# +# Copyright (c) 2021 Wind River Systems, Inc. +# +# SPDX-License-Identifier: Apache-2.0 +# +[tox] +envlist = flake8,py27,py36,pylint,cover +minversion = 2.3.2 +skipsdist = True +stxdir = {toxinidir}/../../.. +[testenv] +setenv = VIRTUAL_ENV={envdir} + LANG=en_US.UTF-8 + LANGUAGE=en_US:en + LC_ALL=C + OS_STDERR_CAPTURE=1 + OS_STDOUT_CAPTURE=1 + OS_TEST_PATH=./kube_memory/tests + OS_TEST_TIMEOUT=60 + PYTHONDONTWRITEBYTECODE=1 + PYTHONHASHSEED=0 + PYTHONWARNINGS=default::DeprecationWarning + PIP_DISABLE_PIP_VERSION_CHECK=1 +passenv = + XDG_CACHE_HOME +sitepackages = False +install_command = pip install \ + -v -v -v \ + -c{env:UPPER_CONSTRAINTS_FILE:https://opendev.org/openstack/requirements/raw/branch/stable/stein/upper-constraints.txt} \ + {opts} {packages} +deps = -r{toxinidir}/requirements.txt + -r{toxinidir}/test-requirements.txt + -e{[tox]stxdir}/config/tsconfig/tsconfig +whitelist_externals = find + sh +[testenv:stestr] +commands = + find . -name "*.pyc" -delete + stestr run {posargs} + stestr slowest +[testenv:py27] +basepython = python2.7 +commands = {[testenv:stestr]commands} +[testenv:py36] +basepython = python3.6 +commands = {[testenv:stestr]commands} +[bandit] +exclude = tests +[testenv:bandit] +basepython = python3 +deps = -r{toxinidir}/test-requirements.txt + bandit +commands = bandit --ini tox.ini -n 5 -r kube_memory +[flake8] +show-source = True +ignore = +exclude = .venv,.git,.tox,dist,doc,*lib/python*,*egg,build,release-tag-* +# H106: Don't put vim configuration in source files (off by default). +# H203: Use assertIs(Not)None to check for None (off by default). +# enable: H904 Delay string interpolations at logging calls (off by default). +enable-extensions = H106 H203 H904 +max-line-length = 120 +[testenv:flake8] +basepython = python3 +deps = {[testenv]deps} + flake8-bugbear +usedevelop = False +#skip_install = True +commands = + flake8 {posargs} . +[testenv:pylint] +deps = {[testenv]deps} + pylint +basepython = python3.6 +sitepackages = False +commands = pylint kube_memory --rcfile=./pylint.rc +[testenv:cover] +setenv = + PYTHON=coverage run --parallel-mode + PYTHONDONTWRITEBYTECODE=True +commands = coverage erase + find . -name "*.pyc" -delete + stestr run {posargs} + coverage combine + coverage html -d cover + coverage xml -o cover/coverage.xml + coverage report diff --git a/tox.ini b/tox.ini index e27160b..c08cd85 100644 --- a/tox.ini +++ b/tox.ini @@ -1,5 +1,5 @@ [tox] -envlist = linters +envlist = linters,pylint minversion = 2.3 skipsdist = True sitepackages=False @@ -66,6 +66,11 @@ commands = {[testenv:bashate]commands} {[testenv:flake8]commands} +[testenv:pylint] +basepython = python3 +description = Dummy environment to allow pylint to be run in subdir tox +# deps = -r{toxinidir}/test-requirements.txt + [testenv:bandit] basepython = python3 description = Bandit code scan for *.py files under config folder