Merge "Kubernetes custom configuration support: runtime."

This commit is contained in:
Zuul 2022-09-13 21:29:48 +00:00 committed by Gerrit Code Review
commit f5a9020fa5
5 changed files with 257 additions and 44 deletions

View File

@ -0,0 +1,139 @@
#!/usr/bin/env python
# Copyright (c) 2022 Wind River Systems, Inc.
#
# SPDX-License-Identifier: Apache-2.0
#
# This script will upgrade kubernetes service parameters
# from release 22.06 to 22.12
#
# Note: this can be removed in the release after STX8.0
import datetime
import sys
import ruamel.yaml as yaml
from oslo_utils import uuidutils
import psycopg2
from psycopg2.extras import RealDictCursor
from psycopg2.extras import DictCursor
from controllerconfig.common import log
LOG = log.get_logger(__name__)
K8S_SERVICE = 'kubernetes'
K8S_APISERVER_SECTION = 'kube_apiserver'
K8S_CONTROLLER_MANAGER_SECTION = 'kube_controller_manager'
K8S_SCHEDULER_SECTION = 'kube_scheduler'
K8S_KUBELET_SECTION = 'kubelet'
SYSINV_K8S_SECTIONS = {
'apiserver': K8S_APISERVER_SECTION,
'controllermanager': K8S_CONTROLLER_MANAGER_SECTION,
'scheduler': K8S_SCHEDULER_SECTION,
'kubelet': K8S_KUBELET_SECTION}
def get_service_parameters(db_conn, K8S_SERVICE, K8S_SECTION):
with db_conn.cursor(cursor_factory=RealDictCursor) as cur:
cur.execute("select name, uuid, value, personality, resource from "
"service_parameter where service='{}' and "
"section='{}'".format(K8S_SERVICE, K8S_SECTION))
return cur.fetchall()
def add_service_parameter(db_conn, name, value, service, section,
personality=None, resource=None):
with db_conn.cursor(cursor_factory=DictCursor) as cur:
cur.execute(
"INSERT INTO service_parameter "
"(created_at, uuid, name, value, service, "
"section, personality, resource) "
"VALUES (%s, %s, %s, %s, %s, %s, %s, %s);",
(datetime.datetime.now(), uuidutils.generate_uuid(),
name, value, service, section, personality, resource))
LOG.info("Adding %s=%s to db [%s]." % (name, value, section))
def main():
action = None
from_release = None
to_release = None
arg = 1
while arg < len(sys.argv):
if arg == 1:
from_release = sys.argv[arg]
elif arg == 2:
to_release = sys.argv[arg]
elif arg == 3:
action = sys.argv[arg]
else:
LOG.error("Invalid option %s." % sys.argv[arg])
return 1
arg += 1
log.configure()
LOG.info("%s invoked with from_release = %s to_release = %s action = %s"
% (sys.argv[0], from_release, to_release, action))
if action == "migrate" and from_release == '22.06':
try:
db_conn = psycopg2.connect("dbname=sysinv user=postgres")
with db_conn:
upgrade_k8s_apiserver_parameters(db_conn)
return 0
except Exception as ex:
LOG.exception(ex)
return 1
def upgrade_k8s_apiserver_parameters(db_conn):
"""This method will take each parameter from dict params and update its
name. key is the current value for instance, oidc_issuer_url.
And, its value is the new name for instance, oidc-issuer-url.
"""
k8s_bootstrap_parameters =\
"/opt/platform/config/22.12/last_kube_extra_config_bootstrap.yaml"
try:
with open(k8s_bootstrap_parameters, 'r') as file:
cluster_cfg = yaml.load(file, Loader=yaml.RoundTripLoader)
except FileNotFoundError as e:
msg = str('Loading k8s bootstrap parameters from file. {}'.format(e))
LOG.error(msg)
return 1
# -------------------------------------------------------------------------
# Save new params into sysinv
# -------------------------------------------------------------------------
for kubeadm_section in [
'apiserver_extra_args', 'controllermanager_extra_args',
'scheduler_extra_args', 'kubelet_configurations']:
# current parameters stored in sysinv db
sysinv_section = SYSINV_K8S_SECTIONS.get(kubeadm_section.split('_')[0])
sysinv_params = get_service_parameters(
db_conn, K8S_SERVICE, sysinv_section)
sysinv_params_names = [param.get('name') for param in sysinv_params]
# new parameters to store into sysinv db (loaded from 22.06)
for param_name, param_value in cluster_cfg[kubeadm_section].items():
if param_name not in sysinv_params_names:
try:
# add new parameter to sysinv
add_service_parameter(
db_conn, param_name, param_value,
K8S_SERVICE, sysinv_section)
except Exception as e:
LOG.error("[%s] Adding %s=%s to db [Detail: %s]." % (
sysinv_section, param_name, param_value, e))
else:
LOG.info("Skipping %s pre existent param." % (param_name))
LOG.info("k8s service-parameters upgrade completed")
if __name__ == "__main__":
sys.exit(main())

View File

@ -732,10 +732,22 @@ class ServiceParameterController(rest.RestController):
oidc_username_claim and oidc_groups_claim)):
msg = _("Unable to apply service parameters. Please choose one of "
"the valid Kubernetes OIDC parameter setups: (None) or "
"(oidc_issuer_url, oidc_client_id, oidc_username_claim) or "
"(the previous 3 plus oidc_groups_claim)")
"(oidc-issuer-url, oidc-client-id, oidc-username-claim) or "
"(the previous 3 plus oidc-groups-claim)")
raise wsme.exc.ClientSideError(msg)
# Verify if the file configured in parameter audit_policy_file is
# cluster configuration.
#
# Configure audit-policy-file mountPath in kube-apiserver
# Allow to end-users to add audit policy file configuration
# in apiserver_extra_volumes variable.
# Add a default audit-policy-file configuration in extraVolumes section
# in kube-apiserver.
#
# This validation is required since if the file does not exist in the
# kube-apiserver pod then after puppet applies the new configuration
# kube-apiserver will fail to start.
try:
audit_policy_file = pecan.request.dbapi.service_parameter_get_one(
service=constants.SERVICE_TYPE_KUBERNETES,
@ -745,17 +757,6 @@ class ServiceParameterController(rest.RestController):
audit_policy_file = None
if audit_policy_file:
# Verify if the file configured in parameter audit_policy_file is
# cluster configuration.
#
# Task 44831 and 45202 configure audit-policy-file mountPath in kube-apiserver
# Task 44831: Allow to end-users to add audit policy file configuration
# in apiserver_extra_volumes variable.
# Task 45202: Add a default audit-policy-file configuration in extraVolumes section
# in kube-apiserver.
#
# This validation is required since if the file does not exist in the kube-apiserver pod
# then after puppet applies the new configuration kube-apiserver will fail to start.
kube_operator = kubernetes.KubeOperator()
try:
config_map = kube_operator.kube_read_config_map(

View File

@ -1140,6 +1140,7 @@ DEFAULT_REGISTRIES_INFO = {
# kubernetes parameters
SERVICE_PARAM_SECTION_KUBERNETES_CONFIG = 'config'
SERVICE_PARAM_NAME_KUBERNETES_POD_MAX_PIDS = 'pod_max_pids'
SERVICE_PARAM_NAME_KUBERNETES_AUTOMATIC_RECOVERY = 'automatic_recovery'
# Platform pods use under 20 in steady state, but allow extra room.
SERVICE_PARAM_KUBERNETES_POD_MAX_PIDS_MIN = 100
# Account for uncontrolled changes in applications (e.g. stx-openstack) by
@ -1152,14 +1153,20 @@ SERVICE_PARAM_SECTION_KUBERNETES_CERTIFICATES = 'certificates'
SERVICE_PARAM_NAME_KUBERNETES_API_SAN_LIST = 'apiserver_certsan'
SERVICE_PARAM_SECTION_KUBERNETES_APISERVER = 'kube_apiserver'
SERVICE_PARAM_NAME_OIDC_ISSUER_URL = 'oidc_issuer_url'
SERVICE_PARAM_NAME_OIDC_CLIENT_ID = 'oidc_client_id'
SERVICE_PARAM_NAME_OIDC_USERNAME_CLAIM = 'oidc_username_claim'
SERVICE_PARAM_NAME_OIDC_GROUPS_CLAIM = 'oidc_groups_claim'
SERVICE_PARAM_NAME_ADMISSION_PLUGINS = 'admission_plugins'
SERVICE_PARAM_NAME_AUDIT_POLICY_FILE = 'audit-policy-file'
SERVICE_PARAM_SECTION_KUBERNETES_CONTROLLER_MANAGER = 'kube_controller_manager'
SERVICE_PARAM_SECTION_KUBERNETES_SCHEDULER = 'kube_scheduler'
SERVICE_PARAM_SECTION_KUBERNETES_KUBELET = 'kubelet'
VALID_ADMISSION_PLUGINS = ['PodSecurityPolicy']
SERVICE_PARAM_NAME_OIDC_ISSUER_URL = 'oidc-issuer-url'
SERVICE_PARAM_NAME_OIDC_CLIENT_ID = 'oidc-client-id'
SERVICE_PARAM_NAME_OIDC_USERNAME_CLAIM = 'oidc-username-claim'
SERVICE_PARAM_NAME_OIDC_GROUPS_CLAIM = 'oidc-groups-claim'
SERVICE_PARAM_DEPRECATED_NAME_OIDC_ISSUER_URL = 'oidc_issuer_url'
SERVICE_PARAM_DEPRECATED_NAME_OIDC_CLIENT_ID = 'oidc_client_id'
SERVICE_PARAM_DEPRECATED_NAME_OIDC_USERNAME_CLAIM = 'oidc_username_claim'
SERVICE_PARAM_DEPRECATED_NAME_OIDC_GROUPS_CLAIM = 'oidc_groups_claim'
SERVICE_PARAM_NAME_AUDIT_POLICY_FILE = 'audit-policy-file'
# ptp service parameters
SERVICE_PARAM_SECTION_PTP_GLOBAL = 'global'

View File

@ -168,6 +168,13 @@ def _validate_oidc_issuer_url(name, value):
"Parameter '%s' must be a valid address or domain." % name))
def _deprecated_oidc_params(name, value):
"""Check oidc deprecated parameters"""
msg = "This parameter '{}' is deprecated you must use a valid parameter like " \
"(oidc-issuer-url, oidc-client-id, oidc-username-claim, oidc-groups-claim).".format(name)
raise wsme.exc.ClientSideError(_(msg))
def _validate_cri_class_format(name, value):
"""
Validate string into cri runtimeClassName:runtimeBinary format,
@ -385,19 +392,6 @@ def _validate_domain(name, value):
(name, value)))
def _validate_admission_plugins(name, value):
"""Check if specified plugins are supported"""
if not value:
raise wsme.exc.ClientSideError(_(
"Please specify at least 1 plugin"))
plugins = value.split(',')
for plugin in plugins:
if plugin not in constants.VALID_ADMISSION_PLUGINS:
raise wsme.exc.ClientSideError(_(
"Invalid admission plugin: '%s'" % plugin))
def _validate_pod_max_pids(name, value):
"""Check if specified value is supported"""
_validate_range(name, value,
@ -813,15 +807,19 @@ KUBERNETES_CERTIFICATES_PARAMETER_DATA_FORMAT = {
KUBERNETES_CONFIG_PARAMETER_OPTIONAL = [
constants.SERVICE_PARAM_NAME_KUBERNETES_POD_MAX_PIDS,
constants.SERVICE_PARAM_NAME_KUBERNETES_AUTOMATIC_RECOVERY
]
KUBERNETES_CONFIG_PARAMETER_VALIDATOR = {
constants.SERVICE_PARAM_NAME_KUBERNETES_POD_MAX_PIDS: _validate_pod_max_pids,
constants.SERVICE_PARAM_NAME_KUBERNETES_AUTOMATIC_RECOVERY: _validate_boolean
}
KUBERNETES_CONFIG_PARAMETER_RESOURCE = {
constants.SERVICE_PARAM_NAME_KUBERNETES_POD_MAX_PIDS:
'platform::kubernetes::params::k8s_pod_max_pids',
constants.SERVICE_PARAM_NAME_KUBERNETES_AUTOMATIC_RECOVERY:
'platform::kubernetes::config::params::automatic_recovery',
}
KUBERNETES_APISERVER_PARAMETER_OPTIONAL = [
@ -829,31 +827,78 @@ KUBERNETES_APISERVER_PARAMETER_OPTIONAL = [
constants.SERVICE_PARAM_NAME_OIDC_CLIENT_ID,
constants.SERVICE_PARAM_NAME_OIDC_USERNAME_CLAIM,
constants.SERVICE_PARAM_NAME_OIDC_GROUPS_CLAIM,
constants.SERVICE_PARAM_NAME_ADMISSION_PLUGINS,
constants.SERVICE_PARAM_NAME_AUDIT_POLICY_FILE
constants.SERVICE_PARAM_NAME_AUDIT_POLICY_FILE,
constants.SERVICE_PARAM_NAME_WILDCARD,
]
KUBERNETES_APISERVER_PARAMETER_VALIDATOR = {
constants.SERVICE_PARAM_NAME_OIDC_ISSUER_URL: _validate_oidc_issuer_url,
constants.SERVICE_PARAM_NAME_ADMISSION_PLUGINS: _validate_admission_plugins,
constants.SERVICE_PARAM_DEPRECATED_NAME_OIDC_ISSUER_URL: _deprecated_oidc_params,
constants.SERVICE_PARAM_DEPRECATED_NAME_OIDC_CLIENT_ID: _deprecated_oidc_params,
constants.SERVICE_PARAM_DEPRECATED_NAME_OIDC_USERNAME_CLAIM: _deprecated_oidc_params,
constants.SERVICE_PARAM_DEPRECATED_NAME_OIDC_GROUPS_CLAIM: _deprecated_oidc_params,
constants.SERVICE_PARAM_NAME_WILDCARD: _validate_not_empty,
constants.SERVICE_PARAM_NAME_AUDIT_POLICY_FILE: _validate_not_empty
}
KUBERNETES_APISERVER_PARAMETER_RESOURCE = {
constants.SERVICE_PARAM_NAME_OIDC_ISSUER_URL:
'platform::kubernetes::params::oidc_issuer_url',
'platform::kubernetes::kube_apiserver::params::oidc_issuer_url',
constants.SERVICE_PARAM_NAME_OIDC_CLIENT_ID:
'platform::kubernetes::params::oidc_client_id',
'platform::kubernetes::kube_apiserver::params::oidc_client_id',
constants.SERVICE_PARAM_NAME_OIDC_USERNAME_CLAIM:
'platform::kubernetes::params::oidc_username_claim',
'platform::kubernetes::kube_apiserver::params::oidc_username_claim',
constants.SERVICE_PARAM_NAME_OIDC_GROUPS_CLAIM:
'platform::kubernetes::params::oidc_groups_claim',
constants.SERVICE_PARAM_NAME_ADMISSION_PLUGINS:
'platform::kubernetes::params::admission_plugins',
'platform::kubernetes::kube_apiserver::params::oidc_groups_claim',
constants.SERVICE_PARAM_NAME_WILDCARD:
'platform::kubernetes::kube_apiserver::params',
constants.SERVICE_PARAM_NAME_AUDIT_POLICY_FILE:
'platform::kubernetes::params::audit_policy_file'
}
KUBERNETES_CONTROLLER_MANAGER_PARAMETER_OPTIONAL = [
constants.SERVICE_PARAM_NAME_OIDC_ISSUER_URL,
constants.SERVICE_PARAM_NAME_OIDC_CLIENT_ID,
constants.SERVICE_PARAM_NAME_OIDC_USERNAME_CLAIM,
constants.SERVICE_PARAM_NAME_OIDC_GROUPS_CLAIM,
constants.SERVICE_PARAM_NAME_WILDCARD
]
KUBERNETES_CONTROLLER_MANAGER_PARAMETER_VALIDATOR = {
constants.SERVICE_PARAM_NAME_WILDCARD: _validate_not_empty
}
KUBERNETES_CONTROLLER_MANAGER_PARAMETER_RESOURCE = {
constants.SERVICE_PARAM_NAME_WILDCARD:
'platform::kubernetes::kube_controller_manager::params',
}
KUBERNETES_SCHEDULER_PARAMETER_OPTIONAL = [
constants.SERVICE_PARAM_NAME_WILDCARD
]
KUBERNETES_SCHEDULER_PARAMETER_VALIDATOR = {
constants.SERVICE_PARAM_NAME_WILDCARD: _validate_not_empty
}
KUBERNETES_SCHEDULER_PARAMETER_RESOURCE = {
constants.SERVICE_PARAM_NAME_WILDCARD:
'platform::kubernetes::kube_scheduler::params',
}
KUBERNETES_KUBELET_PARAMETER_OPTIONAL = [
constants.SERVICE_PARAM_NAME_WILDCARD
]
KUBERNETES_KUBELET_PARAMETER_VALIDATOR = {
constants.SERVICE_PARAM_NAME_WILDCARD: _validate_not_empty
}
KUBERNETES_KUBELET_PARAMETER_RESOURCE = {
constants.SERVICE_PARAM_NAME_WILDCARD:
'platform::kubernetes::kubelet::params',
}
HTTPD_PORT_PARAMETER_OPTIONAL = [
constants.SERVICE_PARAM_HTTP_PORT_HTTP,
constants.SERVICE_PARAM_HTTP_PORT_HTTPS,
@ -1064,6 +1109,21 @@ SERVICE_PARAMETER_SCHEMA = {
SERVICE_PARAM_VALIDATOR: KUBERNETES_CONFIG_PARAMETER_VALIDATOR,
SERVICE_PARAM_RESOURCE: KUBERNETES_CONFIG_PARAMETER_RESOURCE,
},
constants.SERVICE_PARAM_SECTION_KUBERNETES_CONTROLLER_MANAGER: {
SERVICE_PARAM_OPTIONAL: KUBERNETES_CONTROLLER_MANAGER_PARAMETER_OPTIONAL,
SERVICE_PARAM_VALIDATOR: KUBERNETES_CONTROLLER_MANAGER_PARAMETER_VALIDATOR,
SERVICE_PARAM_RESOURCE: KUBERNETES_CONTROLLER_MANAGER_PARAMETER_RESOURCE
},
constants.SERVICE_PARAM_SECTION_KUBERNETES_SCHEDULER: {
SERVICE_PARAM_OPTIONAL: KUBERNETES_SCHEDULER_PARAMETER_OPTIONAL,
SERVICE_PARAM_VALIDATOR: KUBERNETES_SCHEDULER_PARAMETER_VALIDATOR,
SERVICE_PARAM_RESOURCE: KUBERNETES_SCHEDULER_PARAMETER_RESOURCE,
},
constants.SERVICE_PARAM_SECTION_KUBERNETES_KUBELET: {
SERVICE_PARAM_OPTIONAL: KUBERNETES_KUBELET_PARAMETER_OPTIONAL,
SERVICE_PARAM_VALIDATOR: KUBERNETES_KUBELET_PARAMETER_VALIDATOR,
SERVICE_PARAM_RESOURCE: KUBERNETES_KUBELET_PARAMETER_RESOURCE,
},
},
constants.SERVICE_TYPE_PTP: {
constants.SERVICE_PARAM_SECTION_PTP_GLOBAL: {

View File

@ -1,10 +1,11 @@
#
# Copyright (c) 2017-2021 Wind River Systems, Inc.
# Copyright (c) 2017-2022 Wind River Systems, Inc.
#
# SPDX-License-Identifier: Apache-2.0
#
from oslo_log import log as logging
from sysinv.common import constants
from sysinv.common import service_parameter
from sysinv.puppet import base
@ -70,7 +71,12 @@ class ServiceParamPuppet(base.BasePuppet):
resource = schema[service_parameter.SERVICE_PARAM_RESOURCE].get(param.name)
if resource is None:
continue
has_wildcard = (constants.SERVICE_PARAM_NAME_WILDCARD in
schema[service_parameter.SERVICE_PARAM_RESOURCE])
if not has_wildcard:
continue
resource = "{}::{}".format(schema[service_parameter.SERVICE_PARAM_RESOURCE]
.get(constants.SERVICE_PARAM_NAME_WILDCARD), param.name)
formatter = None