diff --git a/.zuul.yaml b/.zuul.yaml index 3b4d92e1..46530649 100644 --- a/.zuul.yaml +++ b/.zuul.yaml @@ -3,9 +3,13 @@ check: jobs: - stx-puppet-linters + - openstack-tox-pep8 + - openstack-tox-pylint gate: jobs: - stx-puppet-linters + - openstack-tox-pep8 + - openstack-tox-pylint post: jobs: - stx-stx-puppet-upload-git-mirror diff --git a/puppet-manifests/centos/puppet-manifests.spec b/puppet-manifests/centos/puppet-manifests.spec index 0094396c..c0e1e7ad 100644 --- a/puppet-manifests/centos/puppet-manifests.spec +++ b/puppet-manifests/centos/puppet-manifests.spec @@ -60,6 +60,9 @@ Requires: puppet-puppi Requires: puppet-vlan Requires: puppet-collectd +# python scripts +Requires: python2-ruamel-yaml + %description Platform puppet configuration files and manifests diff --git a/puppet-manifests/src/modules/platform/files/change_kube_apiserver_params.py b/puppet-manifests/src/modules/platform/files/change_kube_apiserver_params.py new file mode 100644 index 00000000..75afff85 --- /dev/null +++ b/puppet-manifests/src/modules/platform/files/change_kube_apiserver_params.py @@ -0,0 +1,71 @@ +# +# Copyright (c) 2020 Wind River Systems, Inc. +# +# SPDX-License-Identifier: Apache-2.0 +# +# This script edits a file containing a kubernetes cluster configmap. +# It currently adds/removes certain kube-apiserver startup parameters. +# If the script is run without a particular kube-apiserver parameter +# passed in as an argument, the existing kube-apiserver parameter will +# be removed. + +import argparse +import ruamel.yaml as yaml + +configmap_file = '/tmp/cluster_configmap.yaml' +parser = argparse.ArgumentParser() +parser.add_argument("--configmap_file") +parser.add_argument("--oidc_issuer_url") +parser.add_argument("--oidc_client_id") +parser.add_argument("--oidc_username_claim") +parser.add_argument("--oidc_groups_claim") +args = parser.parse_args() + +if args.configmap_file: + configmap_file = args.configmap_file + +with open(configmap_file, 'r') as dest: + configmap = yaml.load(dest, Loader=yaml.RoundTripLoader) + # cluster config is a single string, so we need to parse the string + # in order to modify it correctly + cluster_config = yaml.load(configmap['data']['ClusterConfiguration'], + Loader=yaml.RoundTripLoader) + +if args.oidc_issuer_url: + cluster_config['apiServer']['extraArgs']['oidc-issuer-url'] = \ + args.oidc_issuer_url +else: + if 'oidc-issuer-url' in cluster_config['apiServer']['extraArgs']: + del cluster_config['apiServer']['extraArgs']['oidc-issuer-url'] + +if args.oidc_client_id: + cluster_config['apiServer']['extraArgs']['oidc-client-id'] = \ + args.oidc_client_id +else: + if 'oidc-client-id' in cluster_config['apiServer']['extraArgs']: + del cluster_config['apiServer']['extraArgs']['oidc-client-id'] + +if args.oidc_username_claim: + cluster_config['apiServer']['extraArgs']['oidc-username-claim'] = \ + args.oidc_username_claim +else: + if 'oidc-username-claim' in cluster_config['apiServer']['extraArgs']: + del cluster_config['apiServer']['extraArgs']['oidc-username-claim'] + +if args.oidc_groups_claim: + cluster_config['apiServer']['extraArgs']['oidc-groups-claim'] = \ + args.oidc_groups_claim +else: + if 'oidc-groups-claim' in cluster_config['apiServer']['extraArgs']: + del cluster_config['apiServer']['extraArgs']['oidc-groups-claim'] + +cluster_config_string = yaml.dump(cluster_config, Dumper=yaml.RoundTripDumper, + default_flow_style=False) +# use yaml.scalarstring.PreservedScalarString to make sure the yaml is +# constructed with proper formatting and tabbing +cluster_config_string = yaml.scalarstring.PreservedScalarString( + cluster_config_string) +configmap['data']['ClusterConfiguration'] = cluster_config_string +with open(configmap_file, 'w') as dest: + yaml.dump(configmap, dest, Dumper=yaml.RoundTripDumper, + default_flow_style=False) diff --git a/puppet-manifests/src/modules/platform/manifests/kubernetes.pp b/puppet-manifests/src/modules/platform/manifests/kubernetes.pp index ac0fea0c..e9b7fe99 100644 --- a/puppet-manifests/src/modules/platform/manifests/kubernetes.pp +++ b/puppet-manifests/src/modules/platform/manifests/kubernetes.pp @@ -16,7 +16,11 @@ class platform::kubernetes::params ( $k8s_cpu_mgr_policy = 'none', $k8s_topology_mgr_policy = 'best-effort', $k8s_cni_bin_dir = '/usr/libexec/cni', - $join_cmd = undef + $join_cmd = undef, + $oidc_issuer_url = undef, + $oidc_client_id = undef, + $oidc_username_claim = undef, + $oidc_groups_claim = undef ) { } class platform::kubernetes::cgroup::params ( @@ -615,3 +619,59 @@ class platform::kubernetes::worker::upgrade_kubelet command => '/usr/local/sbin/pmon-restart kubelet' } } + +class platform::kubernetes::master::change_apiserver_parameters + inherits ::platform::kubernetes::params { + + $configmap_temp_file = '/tmp/cluster_configmap.yaml' + $configview_temp_file = '/tmp/kubeadm_config_view.yaml' + + file { $configmap_temp_file: + ensure => present, + owner => 'root', + group => 'root', + mode => '0600', + } + + -> file { $configview_temp_file: + ensure => present, + owner => 'root', + group => 'root', + mode => '0600', + } + + # Kubeadm stores the cluster configuration as a configmap in the cluster. + # We will change that configmap to include/remove kube-apiserver parameters. + # In order to restart kube-apiserver, we will use the "kubeadm init phase" + # command and feed it the output of "kubeadm config view". + # This keeps the configmap consistent and keeps kube-apiserver managed by kubeadm. + + -> exec { 'read kubeadm config map': + command => "kubectl --kubeconfig=/etc/kubernetes/admin.conf get configmap kubeadm-config -o yaml -n kube-system > ${configmap_temp_file}" # lint:ignore:140chars + } + + -> exec { 'update kube-apiserver params': + command => template('platform/kube-apiserver-change-params.erb') + } + + -> exec { 'patch kubeadm config map': + command => "kubectl --kubeconfig=/etc/kubernetes/admin.conf -n kube-system patch configmap kubeadm-config -p \"$(cat ${configmap_temp_file})\"" # lint:ignore:140chars + } + + -> exec { 'get patched configmap': + command => "kubeadm config view > ${configview_temp_file}" + } + + -> exec { 'update kube-apiserver parameters': + command => "kubeadm init phase control-plane apiserver --config ${configview_temp_file}" + } + + -> exec { 'remove temp configmap': + command => "rm ${configmap_temp_file}", + } + + -> exec { 'remove temp configview': + command => "rm ${configview_temp_file}", + } + +} diff --git a/puppet-manifests/src/modules/platform/templates/kube-apiserver-change-params.erb b/puppet-manifests/src/modules/platform/templates/kube-apiserver-change-params.erb new file mode 100644 index 00000000..f88f3ba5 --- /dev/null +++ b/puppet-manifests/src/modules/platform/templates/kube-apiserver-change-params.erb @@ -0,0 +1,14 @@ +python /usr/share/puppet/modules/platform/files/change_kube_apiserver_params.py \ +--configmap_file <%= @configmap_temp_file %> \ +<%- if @oidc_issuer_url -%> +--oidc_issuer_url <%= @oidc_issuer_url %> \ +<%- end -%> +<%- if @oidc_client_id -%> +--oidc_client_id <%= @oidc_client_id %> \ +<%- end -%> +<%- if @oidc_username_claim -%> +--oidc_username_claim <%= @oidc_username_claim %> \ +<%- end -%> +<%- if @oidc_groups_claim -%> +--oidc_groups_claim <%= @oidc_groups_claim %> \ +<%- end -%> diff --git a/pylint.rc b/pylint.rc new file mode 100755 index 00000000..34a963a7 --- /dev/null +++ b/pylint.rc @@ -0,0 +1,237 @@ +[MASTER] +# Specify a configuration file. +rcfile=pylint.rc + +# Python code to execute, usually for sys.path manipulation such as +# pygtk.require(). +#init-hook= + +# Add files or directories to the blacklist. Should be base names, not paths. +ignore= + +# Pickle collected data for later comparisons. +persistent=yes + +# List of plugins (as comma separated values of python modules names) to load, +# usually to register additional checkers. +load-plugins= + +# Use multiple processes to speed up Pylint. +jobs=4 + +# Allow loading of arbitrary C extensions. Extensions are imported into the +# active Python interpreter and may run arbitrary code. +unsafe-load-any-extension=no + +# A comma-separated list of package or module names from where C extensions may +# be loaded. Extensions are loading into the active Python interpreter and may +# run arbitrary code +extension-pkg-whitelist=lxml.etree,greenlet + + + +[MESSAGES CONTROL] +# Enable the message, report, category or checker with the given id(s). You can +# either give multiple identifier separated by comma (,) or put this option +# multiple time. +#enable= + +# Disable the message, report, category or checker with the given id(s). You +# can either give multiple identifier separated by comma (,) or put this option +# multiple time (only on the command line, not in the configuration file where +# it should appear only once). +# See "Messages Control" section of +# https://pylint.readthedocs.io/en/latest/user_guide +# We are disabling (C)onvention +disable=C, + +[REPORTS] +# Set the output format. Available formats are text, parseable, colorized, msvs +# (visual studio) and html +output-format=text + +# Put messages in a separate file for each module / package specified on the +# command line instead of printing them on stdout. Reports (if any) will be +# written in a file name "pylint_global.[txt|html]". +files-output=no + +# Tells whether to display a full report or only the messages +reports=yes + +# Python expression which should return a note less than 10 (10 is the highest +# note). You have access to the variables errors warning, statement which +# respectively contain the number of errors / warnings messages and the total +# number of statements analyzed. This is used by the global evaluation report +# (RP0004). +evaluation=10.0 - ((float(5 * error + warning + refactor + convention) / statement) * 10) + + +[SIMILARITIES] +# Minimum lines number of a similarity. +min-similarity-lines=4 + +# Ignore comments when computing similarities. +ignore-comments=yes + +# Ignore docstrings when computing similarities. +ignore-docstrings=yes + + +[FORMAT] +# Maximum number of characters on a single line. +max-line-length=85 + +# Maximum number of lines in a module +max-module-lines=1000 + +# String used as indentation unit. This is usually 4 spaces or "\t" (1 tab). +indent-string=' ' + + +[TYPECHECK] +# Tells whether missing members accessed in mixin class should be ignored. A +# mixin class is detected if its name ends with "mixin" (case insensitive). +ignore-mixin-members=yes + +# List of module names for which member attributes should not be checked +# (useful for modules/projects where namespaces are manipulated during runtime +# and thus existing member attributes cannot be deduced by static analysis +ignored-modules=distutils,eventlet.green.subprocess,six,six.moves + +# List of classes names for which member attributes should not be checked +# (useful for classes with attributes dynamically set). +# pylint is confused by sqlalchemy Table, as well as sqlalchemy Enum types +# ie: (unprovisioned, identity) +# LookupDict in requests library confuses pylint +ignored-classes=SQLObject, optparse.Values, thread._local, _thread._local, + Table, unprovisioned, identity, LookupDict + +# List of members which are set dynamically and missed by pylint inference +# system, and so shouldn't trigger E0201 when accessed. Python regular +# expressions are accepted. +generated-members=REQUEST,acl_users,aq_parent + + +[BASIC] +# List of builtins function names that should not be used, separated by a comma +bad-functions=map,filter,apply,input + +# Regular expression which should only match correct module names +module-rgx=(([a-z_][a-z0-9_]*)|([A-Z][a-zA-Z0-9]+))$ + +# Regular expression which should only match correct module level names +const-rgx=(([A-Z_][A-Z0-9_]*)|(__.*__))$ + +# Regular expression which should only match correct class names +class-rgx=[A-Z_][a-zA-Z0-9]+$ + +# Regular expression which should only match correct function names +function-rgx=[a-z_][a-z0-9_]{2,30}$ + +# Regular expression which should only match correct method names +method-rgx=[a-z_][a-z0-9_]{2,30}$ + +# Regular expression which should only match correct instance attribute names +attr-rgx=[a-z_][a-z0-9_]{2,30}$ + +# Regular expression which should only match correct argument names +argument-rgx=[a-z_][a-z0-9_]{2,30}$ + +# Regular expression which should only match correct variable names +variable-rgx=[a-z_][a-z0-9_]{2,30}$ + +# Regular expression which should only match correct list comprehension / +# generator expression variable names +inlinevar-rgx=[A-Za-z_][A-Za-z0-9_]*$ + +# Good variable names which should always be accepted, separated by a comma +good-names=i,j,k,ex,Run,_ + +# Bad variable names which should always be refused, separated by a comma +bad-names=foo,bar,baz,toto,tutu,tata + +# Regular expression which should only match functions or classes name which do +# not require a docstring +no-docstring-rgx=__.*__ + + +[MISCELLANEOUS] +# List of note tags to take in consideration, separated by a comma. +notes=FIXME,XXX,TODO + + +[VARIABLES] +# Tells whether we should check for unused import in __init__ files. +init-import=no + +# A regular expression matching the beginning of the name of dummy variables +# (i.e. not used). +dummy-variables-rgx=_|dummy + +# List of additional names supposed to be defined in builtins. Remember that +# you should avoid to define new builtins when possible. +additional-builtins= + + +[IMPORTS] +# Deprecated modules which should not be used, separated by a comma +deprecated-modules=regsub,string,TERMIOS,Bastion,rexec + +# Create a graph of every (i.e. internal and external) dependencies in the +# given file (report RP0402 must not be disabled) +import-graph= + +# Create a graph of external dependencies in the given file (report RP0402 must +# not be disabled) +ext-import-graph= + +# Create a graph of internal dependencies in the given file (report RP0402 must +# not be disabled) +int-import-graph= + + +[DESIGN] +# Maximum number of arguments for function / method +max-args=5 + +# Argument names that match this expression will be ignored. Default to name +# with leading underscore +ignored-argument-names=_.* + +# Maximum number of locals for function / method body +max-locals=15 + +# Maximum number of return / yield for function / method body +max-returns=6 + +# Maximum number of branch for function / method body +max-branchs=12 + +# Maximum number of statements in function / method body +max-statements=50 + +# Maximum number of parents for a class (see R0901). +max-parents=7 + +# Maximum number of attributes for a class (see R0902). +max-attributes=7 + +# Minimum number of public methods for a class (see R0903). +min-public-methods=2 + +# Maximum number of public methods for a class (see R0904). +max-public-methods=20 + + +[CLASSES] +# List of method names used to declare (i.e. assign) instance attributes. +defining-attr-methods=__init__,__new__,setUp + +# List of valid names for the first argument in a class method. +valid-classmethod-first-arg=cls + + +[EXCEPTIONS] +# Exceptions that will emit a warning when being caught. Defaults to +# "Exception" +overgeneral-exceptions=Exception diff --git a/tox.ini b/tox.ini index 6e79cfd1..e5029101 100644 --- a/tox.ini +++ b/tox.ini @@ -1,5 +1,5 @@ [tox] -envlist = linters +envlist = linters,pep8,pylint minversion = 2.3 skipsdist = True sitepackages=False @@ -52,3 +52,32 @@ commands = | xargs -0 puppet-lint --fail-on-warnings {[testenv:linters]skip_tests}" {[testenv:bashate]commands} +[testenv:pep8] +basepython = python3 +usedevelop = False +description = + Run style checks. + + +commands = + flake8 puppet-manifests/src/modules/platform/files + +[testenv:pylint] +basepython = python3 +sitepackages = False + +deps = {[testenv]deps} + ruamel.yaml + pylint +commands = + pylint {posargs} --rcfile=./pylint.rc puppet-manifests + +[flake8] +# E123, E125 skipped as they are invalid PEP-8. +# E501 skipped because some of the code files include templates +# that end up quite wide +# H405: multi line docstring summary not separated with an empty line +show-source = True +ignore = E123,E125,E501,H405,W504 +exclude = .venv,.git,.tox,dist,doc,*lib/python*,*egg,build,release-tag-* +