From 28fce9925078e9c7b961234d59fa1cdba929423f Mon Sep 17 00:00:00 2001 From: Jorge Saffe Date: Wed, 17 Aug 2022 17:54:02 -0400 Subject: [PATCH] Kubernetes custom configuration support: runtime. This changes allow users for global customization of kubelet and control plane components during runtime process. * Validations have been relaxed to enable creation of new sections in kubernetes service through service-parameter. e.i.: kube_apiserver, kube_scheduler, kube_controllerManager, kubelet. * Validations have been relaxed to enable creation of new parameters in kubernetes service through service-parameter. * Upgrade script has been added in order to migrate the parameters (oidc_issuer_url, oidc_client_id, oidc_username_claim, oidc_groups_claim) to (oidc-issuer-url, oidc-client-id, oidc-username-claim, oidc-groups-claim) Test Plan: * Fresh Install: AIO-SX, Standard. * B&R and Upgrade: AIO-SX, Standard. * Create, modify, delete supported parameters and verify changes. * Add and apply not supported parameters and verify kube-apiserver auto restore process. * Validate launch example Pods, for both simplex and duplex systems. Story: 2009766 Task: 44378 Depends-On: https://review.opendev.org/c/starlingx/stx-puppet/+/827760 Signed-off-by: Jorge Saffe Change-Id: I8e1311a78bf9e1419d76d4d19777a847a53e82f4 --- .../98-sysinv-k8s-apiserver-param-upgrade.py | 139 ++++++++++++++++++ .../api/controllers/v1/service_parameter.py | 27 ++-- .../sysinv/sysinv/sysinv/common/constants.py | 21 ++- .../sysinv/sysinv/common/service_parameter.py | 104 ++++++++++--- .../sysinv/sysinv/puppet/service_parameter.py | 10 +- 5 files changed, 257 insertions(+), 44 deletions(-) create mode 100644 controllerconfig/controllerconfig/upgrade-scripts/98-sysinv-k8s-apiserver-param-upgrade.py diff --git a/controllerconfig/controllerconfig/upgrade-scripts/98-sysinv-k8s-apiserver-param-upgrade.py b/controllerconfig/controllerconfig/upgrade-scripts/98-sysinv-k8s-apiserver-param-upgrade.py new file mode 100644 index 0000000000..be85328b97 --- /dev/null +++ b/controllerconfig/controllerconfig/upgrade-scripts/98-sysinv-k8s-apiserver-param-upgrade.py @@ -0,0 +1,139 @@ +#!/usr/bin/env python +# Copyright (c) 2022 Wind River Systems, Inc. +# +# SPDX-License-Identifier: Apache-2.0 +# +# This script will upgrade kubernetes service parameters +# from release 22.06 to 22.12 +# +# Note: this can be removed in the release after STX8.0 + +import datetime +import sys +import ruamel.yaml as yaml + +from oslo_utils import uuidutils +import psycopg2 +from psycopg2.extras import RealDictCursor +from psycopg2.extras import DictCursor + +from controllerconfig.common import log + + +LOG = log.get_logger(__name__) + +K8S_SERVICE = 'kubernetes' +K8S_APISERVER_SECTION = 'kube_apiserver' +K8S_CONTROLLER_MANAGER_SECTION = 'kube_controller_manager' +K8S_SCHEDULER_SECTION = 'kube_scheduler' +K8S_KUBELET_SECTION = 'kubelet' + +SYSINV_K8S_SECTIONS = { + 'apiserver': K8S_APISERVER_SECTION, + 'controllermanager': K8S_CONTROLLER_MANAGER_SECTION, + 'scheduler': K8S_SCHEDULER_SECTION, + 'kubelet': K8S_KUBELET_SECTION} + + +def get_service_parameters(db_conn, K8S_SERVICE, K8S_SECTION): + with db_conn.cursor(cursor_factory=RealDictCursor) as cur: + cur.execute("select name, uuid, value, personality, resource from " + "service_parameter where service='{}' and " + "section='{}'".format(K8S_SERVICE, K8S_SECTION)) + return cur.fetchall() + + +def add_service_parameter(db_conn, name, value, service, section, + personality=None, resource=None): + with db_conn.cursor(cursor_factory=DictCursor) as cur: + cur.execute( + "INSERT INTO service_parameter " + "(created_at, uuid, name, value, service, " + "section, personality, resource) " + "VALUES (%s, %s, %s, %s, %s, %s, %s, %s);", + (datetime.datetime.now(), uuidutils.generate_uuid(), + name, value, service, section, personality, resource)) + LOG.info("Adding %s=%s to db [%s]." % (name, value, section)) + + +def main(): + action = None + from_release = None + to_release = None + arg = 1 + + while arg < len(sys.argv): + if arg == 1: + from_release = sys.argv[arg] + elif arg == 2: + to_release = sys.argv[arg] + elif arg == 3: + action = sys.argv[arg] + else: + LOG.error("Invalid option %s." % sys.argv[arg]) + return 1 + arg += 1 + + log.configure() + + LOG.info("%s invoked with from_release = %s to_release = %s action = %s" + % (sys.argv[0], from_release, to_release, action)) + if action == "migrate" and from_release == '22.06': + try: + db_conn = psycopg2.connect("dbname=sysinv user=postgres") + with db_conn: + upgrade_k8s_apiserver_parameters(db_conn) + return 0 + except Exception as ex: + LOG.exception(ex) + return 1 + + +def upgrade_k8s_apiserver_parameters(db_conn): + """This method will take each parameter from dict params and update its + name. key is the current value for instance, oidc_issuer_url. + And, its value is the new name for instance, oidc-issuer-url. + """ + k8s_bootstrap_parameters =\ + "/opt/platform/config/22.12/last_kube_extra_config_bootstrap.yaml" + + try: + with open(k8s_bootstrap_parameters, 'r') as file: + cluster_cfg = yaml.load(file, Loader=yaml.RoundTripLoader) + except FileNotFoundError as e: + msg = str('Loading k8s bootstrap parameters from file. {}'.format(e)) + LOG.error(msg) + return 1 + + # ------------------------------------------------------------------------- + # Save new params into sysinv + # ------------------------------------------------------------------------- + for kubeadm_section in [ + 'apiserver_extra_args', 'controllermanager_extra_args', + 'scheduler_extra_args', 'kubelet_configurations']: + + # current parameters stored in sysinv db + sysinv_section = SYSINV_K8S_SECTIONS.get(kubeadm_section.split('_')[0]) + sysinv_params = get_service_parameters( + db_conn, K8S_SERVICE, sysinv_section) + sysinv_params_names = [param.get('name') for param in sysinv_params] + + # new parameters to store into sysinv db (loaded from 22.06) + for param_name, param_value in cluster_cfg[kubeadm_section].items(): + if param_name not in sysinv_params_names: + try: + # add new parameter to sysinv + add_service_parameter( + db_conn, param_name, param_value, + K8S_SERVICE, sysinv_section) + except Exception as e: + LOG.error("[%s] Adding %s=%s to db [Detail: %s]." % ( + sysinv_section, param_name, param_value, e)) + else: + LOG.info("Skipping %s pre existent param." % (param_name)) + + LOG.info("k8s service-parameters upgrade completed") + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/sysinv/sysinv/sysinv/sysinv/api/controllers/v1/service_parameter.py b/sysinv/sysinv/sysinv/sysinv/api/controllers/v1/service_parameter.py index b57f788b20..e751bc0816 100644 --- a/sysinv/sysinv/sysinv/sysinv/api/controllers/v1/service_parameter.py +++ b/sysinv/sysinv/sysinv/sysinv/api/controllers/v1/service_parameter.py @@ -732,10 +732,22 @@ class ServiceParameterController(rest.RestController): oidc_username_claim and oidc_groups_claim)): msg = _("Unable to apply service parameters. Please choose one of " "the valid Kubernetes OIDC parameter setups: (None) or " - "(oidc_issuer_url, oidc_client_id, oidc_username_claim) or " - "(the previous 3 plus oidc_groups_claim)") + "(oidc-issuer-url, oidc-client-id, oidc-username-claim) or " + "(the previous 3 plus oidc-groups-claim)") raise wsme.exc.ClientSideError(msg) + # Verify if the file configured in parameter audit_policy_file is + # cluster configuration. + # + # Configure audit-policy-file mountPath in kube-apiserver + # Allow to end-users to add audit policy file configuration + # in apiserver_extra_volumes variable. + # Add a default audit-policy-file configuration in extraVolumes section + # in kube-apiserver. + # + # This validation is required since if the file does not exist in the + # kube-apiserver pod then after puppet applies the new configuration + # kube-apiserver will fail to start. try: audit_policy_file = pecan.request.dbapi.service_parameter_get_one( service=constants.SERVICE_TYPE_KUBERNETES, @@ -745,17 +757,6 @@ class ServiceParameterController(rest.RestController): audit_policy_file = None if audit_policy_file: - # Verify if the file configured in parameter audit_policy_file is - # cluster configuration. - # - # Task 44831 and 45202 configure audit-policy-file mountPath in kube-apiserver - # Task 44831: Allow to end-users to add audit policy file configuration - # in apiserver_extra_volumes variable. - # Task 45202: Add a default audit-policy-file configuration in extraVolumes section - # in kube-apiserver. - # - # This validation is required since if the file does not exist in the kube-apiserver pod - # then after puppet applies the new configuration kube-apiserver will fail to start. kube_operator = kubernetes.KubeOperator() try: config_map = kube_operator.kube_read_config_map( diff --git a/sysinv/sysinv/sysinv/sysinv/common/constants.py b/sysinv/sysinv/sysinv/sysinv/common/constants.py index dac700b95e..b54f4afb1a 100644 --- a/sysinv/sysinv/sysinv/sysinv/common/constants.py +++ b/sysinv/sysinv/sysinv/sysinv/common/constants.py @@ -1140,6 +1140,7 @@ DEFAULT_REGISTRIES_INFO = { # kubernetes parameters SERVICE_PARAM_SECTION_KUBERNETES_CONFIG = 'config' SERVICE_PARAM_NAME_KUBERNETES_POD_MAX_PIDS = 'pod_max_pids' +SERVICE_PARAM_NAME_KUBERNETES_AUTOMATIC_RECOVERY = 'automatic_recovery' # Platform pods use under 20 in steady state, but allow extra room. SERVICE_PARAM_KUBERNETES_POD_MAX_PIDS_MIN = 100 # Account for uncontrolled changes in applications (e.g. stx-openstack) by @@ -1152,14 +1153,20 @@ SERVICE_PARAM_SECTION_KUBERNETES_CERTIFICATES = 'certificates' SERVICE_PARAM_NAME_KUBERNETES_API_SAN_LIST = 'apiserver_certsan' SERVICE_PARAM_SECTION_KUBERNETES_APISERVER = 'kube_apiserver' -SERVICE_PARAM_NAME_OIDC_ISSUER_URL = 'oidc_issuer_url' -SERVICE_PARAM_NAME_OIDC_CLIENT_ID = 'oidc_client_id' -SERVICE_PARAM_NAME_OIDC_USERNAME_CLAIM = 'oidc_username_claim' -SERVICE_PARAM_NAME_OIDC_GROUPS_CLAIM = 'oidc_groups_claim' -SERVICE_PARAM_NAME_ADMISSION_PLUGINS = 'admission_plugins' -SERVICE_PARAM_NAME_AUDIT_POLICY_FILE = 'audit-policy-file' +SERVICE_PARAM_SECTION_KUBERNETES_CONTROLLER_MANAGER = 'kube_controller_manager' +SERVICE_PARAM_SECTION_KUBERNETES_SCHEDULER = 'kube_scheduler' +SERVICE_PARAM_SECTION_KUBERNETES_KUBELET = 'kubelet' -VALID_ADMISSION_PLUGINS = ['PodSecurityPolicy'] +SERVICE_PARAM_NAME_OIDC_ISSUER_URL = 'oidc-issuer-url' +SERVICE_PARAM_NAME_OIDC_CLIENT_ID = 'oidc-client-id' +SERVICE_PARAM_NAME_OIDC_USERNAME_CLAIM = 'oidc-username-claim' +SERVICE_PARAM_NAME_OIDC_GROUPS_CLAIM = 'oidc-groups-claim' +SERVICE_PARAM_DEPRECATED_NAME_OIDC_ISSUER_URL = 'oidc_issuer_url' +SERVICE_PARAM_DEPRECATED_NAME_OIDC_CLIENT_ID = 'oidc_client_id' +SERVICE_PARAM_DEPRECATED_NAME_OIDC_USERNAME_CLAIM = 'oidc_username_claim' +SERVICE_PARAM_DEPRECATED_NAME_OIDC_GROUPS_CLAIM = 'oidc_groups_claim' + +SERVICE_PARAM_NAME_AUDIT_POLICY_FILE = 'audit-policy-file' # ptp service parameters SERVICE_PARAM_SECTION_PTP_GLOBAL = 'global' diff --git a/sysinv/sysinv/sysinv/sysinv/common/service_parameter.py b/sysinv/sysinv/sysinv/sysinv/common/service_parameter.py index c0ac3ec95c..e07095b37b 100644 --- a/sysinv/sysinv/sysinv/sysinv/common/service_parameter.py +++ b/sysinv/sysinv/sysinv/sysinv/common/service_parameter.py @@ -168,6 +168,13 @@ def _validate_oidc_issuer_url(name, value): "Parameter '%s' must be a valid address or domain." % name)) +def _deprecated_oidc_params(name, value): + """Check oidc deprecated parameters""" + msg = "This parameter '{}' is deprecated you must use a valid parameter like " \ + "(oidc-issuer-url, oidc-client-id, oidc-username-claim, oidc-groups-claim).".format(name) + raise wsme.exc.ClientSideError(_(msg)) + + def _validate_cri_class_format(name, value): """ Validate string into cri runtimeClassName:runtimeBinary format, @@ -385,19 +392,6 @@ def _validate_domain(name, value): (name, value))) -def _validate_admission_plugins(name, value): - """Check if specified plugins are supported""" - if not value: - raise wsme.exc.ClientSideError(_( - "Please specify at least 1 plugin")) - - plugins = value.split(',') - for plugin in plugins: - if plugin not in constants.VALID_ADMISSION_PLUGINS: - raise wsme.exc.ClientSideError(_( - "Invalid admission plugin: '%s'" % plugin)) - - def _validate_pod_max_pids(name, value): """Check if specified value is supported""" _validate_range(name, value, @@ -813,15 +807,19 @@ KUBERNETES_CERTIFICATES_PARAMETER_DATA_FORMAT = { KUBERNETES_CONFIG_PARAMETER_OPTIONAL = [ constants.SERVICE_PARAM_NAME_KUBERNETES_POD_MAX_PIDS, + constants.SERVICE_PARAM_NAME_KUBERNETES_AUTOMATIC_RECOVERY ] KUBERNETES_CONFIG_PARAMETER_VALIDATOR = { constants.SERVICE_PARAM_NAME_KUBERNETES_POD_MAX_PIDS: _validate_pod_max_pids, + constants.SERVICE_PARAM_NAME_KUBERNETES_AUTOMATIC_RECOVERY: _validate_boolean } KUBERNETES_CONFIG_PARAMETER_RESOURCE = { constants.SERVICE_PARAM_NAME_KUBERNETES_POD_MAX_PIDS: 'platform::kubernetes::params::k8s_pod_max_pids', + constants.SERVICE_PARAM_NAME_KUBERNETES_AUTOMATIC_RECOVERY: + 'platform::kubernetes::config::params::automatic_recovery', } KUBERNETES_APISERVER_PARAMETER_OPTIONAL = [ @@ -829,31 +827,78 @@ KUBERNETES_APISERVER_PARAMETER_OPTIONAL = [ constants.SERVICE_PARAM_NAME_OIDC_CLIENT_ID, constants.SERVICE_PARAM_NAME_OIDC_USERNAME_CLAIM, constants.SERVICE_PARAM_NAME_OIDC_GROUPS_CLAIM, - constants.SERVICE_PARAM_NAME_ADMISSION_PLUGINS, - constants.SERVICE_PARAM_NAME_AUDIT_POLICY_FILE + constants.SERVICE_PARAM_NAME_AUDIT_POLICY_FILE, + constants.SERVICE_PARAM_NAME_WILDCARD, ] KUBERNETES_APISERVER_PARAMETER_VALIDATOR = { constants.SERVICE_PARAM_NAME_OIDC_ISSUER_URL: _validate_oidc_issuer_url, - constants.SERVICE_PARAM_NAME_ADMISSION_PLUGINS: _validate_admission_plugins, + constants.SERVICE_PARAM_DEPRECATED_NAME_OIDC_ISSUER_URL: _deprecated_oidc_params, + constants.SERVICE_PARAM_DEPRECATED_NAME_OIDC_CLIENT_ID: _deprecated_oidc_params, + constants.SERVICE_PARAM_DEPRECATED_NAME_OIDC_USERNAME_CLAIM: _deprecated_oidc_params, + constants.SERVICE_PARAM_DEPRECATED_NAME_OIDC_GROUPS_CLAIM: _deprecated_oidc_params, + constants.SERVICE_PARAM_NAME_WILDCARD: _validate_not_empty, constants.SERVICE_PARAM_NAME_AUDIT_POLICY_FILE: _validate_not_empty } KUBERNETES_APISERVER_PARAMETER_RESOURCE = { constants.SERVICE_PARAM_NAME_OIDC_ISSUER_URL: - 'platform::kubernetes::params::oidc_issuer_url', + 'platform::kubernetes::kube_apiserver::params::oidc_issuer_url', constants.SERVICE_PARAM_NAME_OIDC_CLIENT_ID: - 'platform::kubernetes::params::oidc_client_id', + 'platform::kubernetes::kube_apiserver::params::oidc_client_id', constants.SERVICE_PARAM_NAME_OIDC_USERNAME_CLAIM: - 'platform::kubernetes::params::oidc_username_claim', + 'platform::kubernetes::kube_apiserver::params::oidc_username_claim', constants.SERVICE_PARAM_NAME_OIDC_GROUPS_CLAIM: - 'platform::kubernetes::params::oidc_groups_claim', - constants.SERVICE_PARAM_NAME_ADMISSION_PLUGINS: - 'platform::kubernetes::params::admission_plugins', + 'platform::kubernetes::kube_apiserver::params::oidc_groups_claim', + constants.SERVICE_PARAM_NAME_WILDCARD: + 'platform::kubernetes::kube_apiserver::params', constants.SERVICE_PARAM_NAME_AUDIT_POLICY_FILE: 'platform::kubernetes::params::audit_policy_file' } +KUBERNETES_CONTROLLER_MANAGER_PARAMETER_OPTIONAL = [ + constants.SERVICE_PARAM_NAME_OIDC_ISSUER_URL, + constants.SERVICE_PARAM_NAME_OIDC_CLIENT_ID, + constants.SERVICE_PARAM_NAME_OIDC_USERNAME_CLAIM, + constants.SERVICE_PARAM_NAME_OIDC_GROUPS_CLAIM, + constants.SERVICE_PARAM_NAME_WILDCARD +] + +KUBERNETES_CONTROLLER_MANAGER_PARAMETER_VALIDATOR = { + constants.SERVICE_PARAM_NAME_WILDCARD: _validate_not_empty +} + +KUBERNETES_CONTROLLER_MANAGER_PARAMETER_RESOURCE = { + constants.SERVICE_PARAM_NAME_WILDCARD: + 'platform::kubernetes::kube_controller_manager::params', +} + +KUBERNETES_SCHEDULER_PARAMETER_OPTIONAL = [ + constants.SERVICE_PARAM_NAME_WILDCARD +] + +KUBERNETES_SCHEDULER_PARAMETER_VALIDATOR = { + constants.SERVICE_PARAM_NAME_WILDCARD: _validate_not_empty +} + +KUBERNETES_SCHEDULER_PARAMETER_RESOURCE = { + constants.SERVICE_PARAM_NAME_WILDCARD: + 'platform::kubernetes::kube_scheduler::params', +} + +KUBERNETES_KUBELET_PARAMETER_OPTIONAL = [ + constants.SERVICE_PARAM_NAME_WILDCARD +] + +KUBERNETES_KUBELET_PARAMETER_VALIDATOR = { + constants.SERVICE_PARAM_NAME_WILDCARD: _validate_not_empty +} + +KUBERNETES_KUBELET_PARAMETER_RESOURCE = { + constants.SERVICE_PARAM_NAME_WILDCARD: + 'platform::kubernetes::kubelet::params', +} + HTTPD_PORT_PARAMETER_OPTIONAL = [ constants.SERVICE_PARAM_HTTP_PORT_HTTP, constants.SERVICE_PARAM_HTTP_PORT_HTTPS, @@ -1064,6 +1109,21 @@ SERVICE_PARAMETER_SCHEMA = { SERVICE_PARAM_VALIDATOR: KUBERNETES_CONFIG_PARAMETER_VALIDATOR, SERVICE_PARAM_RESOURCE: KUBERNETES_CONFIG_PARAMETER_RESOURCE, }, + constants.SERVICE_PARAM_SECTION_KUBERNETES_CONTROLLER_MANAGER: { + SERVICE_PARAM_OPTIONAL: KUBERNETES_CONTROLLER_MANAGER_PARAMETER_OPTIONAL, + SERVICE_PARAM_VALIDATOR: KUBERNETES_CONTROLLER_MANAGER_PARAMETER_VALIDATOR, + SERVICE_PARAM_RESOURCE: KUBERNETES_CONTROLLER_MANAGER_PARAMETER_RESOURCE + }, + constants.SERVICE_PARAM_SECTION_KUBERNETES_SCHEDULER: { + SERVICE_PARAM_OPTIONAL: KUBERNETES_SCHEDULER_PARAMETER_OPTIONAL, + SERVICE_PARAM_VALIDATOR: KUBERNETES_SCHEDULER_PARAMETER_VALIDATOR, + SERVICE_PARAM_RESOURCE: KUBERNETES_SCHEDULER_PARAMETER_RESOURCE, + }, + constants.SERVICE_PARAM_SECTION_KUBERNETES_KUBELET: { + SERVICE_PARAM_OPTIONAL: KUBERNETES_KUBELET_PARAMETER_OPTIONAL, + SERVICE_PARAM_VALIDATOR: KUBERNETES_KUBELET_PARAMETER_VALIDATOR, + SERVICE_PARAM_RESOURCE: KUBERNETES_KUBELET_PARAMETER_RESOURCE, + }, }, constants.SERVICE_TYPE_PTP: { constants.SERVICE_PARAM_SECTION_PTP_GLOBAL: { diff --git a/sysinv/sysinv/sysinv/sysinv/puppet/service_parameter.py b/sysinv/sysinv/sysinv/sysinv/puppet/service_parameter.py index bc64f0f708..75f2e4f357 100644 --- a/sysinv/sysinv/sysinv/sysinv/puppet/service_parameter.py +++ b/sysinv/sysinv/sysinv/sysinv/puppet/service_parameter.py @@ -1,10 +1,11 @@ # -# Copyright (c) 2017-2021 Wind River Systems, Inc. +# Copyright (c) 2017-2022 Wind River Systems, Inc. # # SPDX-License-Identifier: Apache-2.0 # from oslo_log import log as logging +from sysinv.common import constants from sysinv.common import service_parameter from sysinv.puppet import base @@ -70,7 +71,12 @@ class ServiceParamPuppet(base.BasePuppet): resource = schema[service_parameter.SERVICE_PARAM_RESOURCE].get(param.name) if resource is None: - continue + has_wildcard = (constants.SERVICE_PARAM_NAME_WILDCARD in + schema[service_parameter.SERVICE_PARAM_RESOURCE]) + if not has_wildcard: + continue + resource = "{}::{}".format(schema[service_parameter.SERVICE_PARAM_RESOURCE] + .get(constants.SERVICE_PARAM_NAME_WILDCARD), param.name) formatter = None