From df52f9c9ea08b67158491583d47a8a10113de129 Mon Sep 17 00:00:00 2001 From: Diogo Guerra Date: Fri, 8 Nov 2019 14:17:35 +0100 Subject: [PATCH] [k8s] Update metrics-server Magnum allows to use CONTAINER_INFRA_PREFIX to specify a local repository from which we can pull container images. This repository defaults to the upstream one that is specified in the metrics helm chart. * This patch allows for the usage of CONTAINER_INFRA_PREFIX to correctly configure the pull of the metric-server container image from the specified repo. * Add label metrics_server_chart_tag to allow user to specify stable/metrics-server chart tag to use * Add label metrics_server_enabled to allow enable/disable of component (defaults: true) Story: 2004816 Task: 37390 Change-Id: Idc315937a82317b76349bbe8466d900d00194953 Signed-off-by: Diogo Guerra --- doc/source/user/index.rst | 17 ++++++++++ .../fragments/install-helm-modules.sh | 13 +++++--- .../fragments/write-heat-params-master.sh | 2 ++ .../kubernetes/fragments/write-heat-params.sh | 2 ++ .../kubernetes/helm/metrics-server.sh | 33 ++++++++++++++----- .../drivers/heat/k8s_fedora_template_def.py | 2 ++ .../templates/kubecluster.yaml | 12 +++++++ .../templates/kubemaster.yaml | 10 ++++++ .../templates/kubecluster.yaml | 12 +++++++ .../templates/kubemaster.yaml | 10 ++++++ .../unit/drivers/test_template_definition.py | 12 +++++++ ...tall-metrics-service-e7a5459417504a75.yaml | 10 ++++++ 12 files changed, 121 insertions(+), 14 deletions(-) create mode 100644 releasenotes/notes/helm-install-metrics-service-e7a5459417504a75.yaml diff --git a/doc/source/user/index.rst b/doc/source/user/index.rst index f33a2a638c..2c638342e3 100644 --- a/doc/source/user/index.rst +++ b/doc/source/user/index.rst @@ -309,6 +309,11 @@ the table are linked to more details elsewhere in the user guide. | `heapster_enabled`_ | - true | true | | | - false | | +---------------------------------------+--------------------+---------------+ +| `metrics_server_chart_tag`_ | see below | see below | ++---------------------------------------+--------------------+---------------+ +| `metrics_server_enabled`_ | - true | true | +| | - false | | ++---------------------------------------+--------------------+---------------+ | `monitoring_enabled`_ | - true | false | | | - false | | +---------------------------------------+--------------------+---------------+ @@ -1184,6 +1189,7 @@ _`container_infra_prefix` * docker.io/traefik:v1.7.10 * gcr.io/google_containers/kubernetes-dashboard-amd64:v1.5.1 * gcr.io/google-containers/hyperkube:v1.12.1 + * gcr.io/google_containers/metrics-server-amd64:v0.3.6 * quay.io/coreos/configmap-reload:v0.0.1 * quay.io/coreos/prometheus-config-reloader:v0.30.1 * quay.io/coreos/prometheus-operator:v0.30.1 @@ -1205,6 +1211,17 @@ _`heapster_enabled` Ussuri default: true Train default: true +_`metrics_server_chart_tag` + Add metrics_server_chart_tag to select the version of the + stable/metrics-server chart to install. + Ussuri default: v2.8.8 + +_`metrics_server_enabled` + metrics_server_enabled is used to enable disable the installation of + the metrics server. To use this service tiller_enabled must be true. + Train default: true + Stein default: true + _`cloud_provider_tag` This label allows users to select `a specific release for the openstack cloud provider diff --git a/magnum/drivers/common/templates/kubernetes/fragments/install-helm-modules.sh b/magnum/drivers/common/templates/kubernetes/fragments/install-helm-modules.sh index 20d11c8d13..e8009dad10 100644 --- a/magnum/drivers/common/templates/kubernetes/fragments/install-helm-modules.sh +++ b/magnum/drivers/common/templates/kubernetes/fragments/install-helm-modules.sh @@ -17,13 +17,16 @@ if [ "$(echo ${TILLER_ENABLED} | tr '[:upper:]' '[:lower:]')" != "true" ]; then echo "Use --labels tiller_enabled=True to allow for tiller dependent resources to be installed" else HELM_MODULES_PATH="/srv/magnum/kubernetes/helm" - chmod +x ${HELM_MODULES_PATH}/* + mkdir -p ${HELM_MODULES_PATH} helm_modules=(${HELM_MODULES_PATH}/*) - for module in "${helm_modules[@]}"; do - echo "" - kubectl apply -f ${module} - done + # Only run kubectl if we have modules to install + if [ "${helm_modules}" != "${HELM_MODULES_PATH}/*" ]; then + for module in "${helm_modules[@]}"; do + echo "Applying ${module}." + kubectl apply -f ${module} + done + fi fi printf "Finished running ${step}\n" diff --git a/magnum/drivers/common/templates/kubernetes/fragments/write-heat-params-master.sh b/magnum/drivers/common/templates/kubernetes/fragments/write-heat-params-master.sh index abd5a6cf82..8f8a77dce3 100644 --- a/magnum/drivers/common/templates/kubernetes/fragments/write-heat-params-master.sh +++ b/magnum/drivers/common/templates/kubernetes/fragments/write-heat-params-master.sh @@ -9,6 +9,8 @@ HEAT_PARAMS=/etc/sysconfig/heat-params cat > ${HEAT_PARAMS} < ${HEAT_PARAMS} < ${HELM_MODULE_CONFIG_FILE} +### Configuration +############################################################################### +CHART_NAME="metrics-server" + +if [ "$(echo ${METRICS_SERVER_ENABLED} | tr '[:upper:]' '[:lower:]')" = "true" ]; then + + HELM_MODULE_CONFIG_FILE="/srv/magnum/kubernetes/helm/${CHART_NAME}.yaml" + [ -f ${HELM_MODULE_CONFIG_FILE} ] || { + echo "Writing File: ${HELM_MODULE_CONFIG_FILE}" + mkdir -p $(dirname ${HELM_MODULE_CONFIG_FILE}) + cat << EOF > ${HELM_MODULE_CONFIG_FILE} --- kind: ConfigMap apiVersion: v1 @@ -37,9 +45,12 @@ data: echo "${CHART_NAME} already installed on server. Continue..." exit 0 else - helm install stable/${CHART_NAME} --namespace kube-system --name ${CHART_NAME} --version v${CHART_VERSION} + helm install stable/${CHART_NAME} --namespace kube-system --name ${CHART_NAME} --version ${METRICS_SERVER_CHART_TAG} --values /opt/magnum/install-${CHART_NAME}-values.yaml fi + install-${CHART_NAME}-values.yaml: | + image: + repository: ${CONTAINER_INFRA_PREFIX:-gcr.io/google_containers/}metrics-server-amd64 --- apiVersion: batch/v1 @@ -80,4 +91,8 @@ spec: secret: secretName: helm-client-secret EOF -} + } + +fi + +printf "Finished running ${step}\n" diff --git a/magnum/drivers/heat/k8s_fedora_template_def.py b/magnum/drivers/heat/k8s_fedora_template_def.py index a302674497..f7c497c9da 100644 --- a/magnum/drivers/heat/k8s_fedora_template_def.py +++ b/magnum/drivers/heat/k8s_fedora_template_def.py @@ -86,6 +86,8 @@ class K8sFedoraTemplateDefinition(k8s_template_def.K8sTemplateDefinition): 'heat_container_agent_tag', 'keystone_auth_enabled', 'k8s_keystone_auth_tag', 'heapster_enabled', + 'metrics_server_enabled', + 'metrics_server_chart_tag', 'monitoring_enabled', 'prometheus_operator_chart_tag', 'prometheus_adapter_enabled', diff --git a/magnum/drivers/k8s_fedora_atomic_v1/templates/kubecluster.yaml b/magnum/drivers/k8s_fedora_atomic_v1/templates/kubecluster.yaml index d74b112e67..562828c71b 100644 --- a/magnum/drivers/k8s_fedora_atomic_v1/templates/kubecluster.yaml +++ b/magnum/drivers/k8s_fedora_atomic_v1/templates/kubecluster.yaml @@ -122,6 +122,16 @@ parameters: description: enable/disable the use of heapster default: true + metrics_server_enabled: + type: boolean + description: enable/disable the use of metrics-server + default: true + + metrics_server_chart_tag: + type: string + description: tag of the stable/metrics-server chart to install + default: v2.8.8 + minion_flavor: type: string default: m1.small @@ -1005,6 +1015,8 @@ resources: nodegroup_role: {get_param: master_role} nodegroup_name: {get_param: master_nodegroup_name} heapster_enabled: {get_param: heapster_enabled} + metrics_server_enabled: {get_param: metrics_server_enabled} + metrics_server_chart_tag: {get_param: metrics_server_chart_tag} prometheus_monitoring: {get_param: prometheus_monitoring} grafana_admin_passwd: {get_param: grafana_admin_passwd} api_public_address: {get_attr: [api_lb, floating_address]} diff --git a/magnum/drivers/k8s_fedora_atomic_v1/templates/kubemaster.yaml b/magnum/drivers/k8s_fedora_atomic_v1/templates/kubemaster.yaml index 0fd065df42..de9ac053cf 100644 --- a/magnum/drivers/k8s_fedora_atomic_v1/templates/kubemaster.yaml +++ b/magnum/drivers/k8s_fedora_atomic_v1/templates/kubemaster.yaml @@ -31,6 +31,14 @@ parameters: type: boolean description: enable/disable the use of heapster + metrics_server_enabled: + type: boolean + description: enable/disable the use of metrics-server + + metrics_server_chart_tag: + type: string + description: tag of the stable/metrics-server chart to install + ssh_key_name: type: string description: name of ssh key to be provisioned on our server @@ -627,6 +635,8 @@ resources: params: "$INSTANCE_NAME": {get_param: name} "$HEAPSTER_ENABLED": {get_param: heapster_enabled} + "$METRICS_SERVER_ENABLED": {get_param: metrics_server_enabled} + "$METRICS_SERVER_CHART_TAG": {get_param: metrics_server_chart_tag} "$PROMETHEUS_MONITORING": {get_param: prometheus_monitoring} "$KUBE_API_PUBLIC_ADDRESS": {get_attr: [api_address_switch, public_ip]} "$KUBE_API_PRIVATE_ADDRESS": {get_attr: [api_address_switch, private_ip]} diff --git a/magnum/drivers/k8s_fedora_coreos_v1/templates/kubecluster.yaml b/magnum/drivers/k8s_fedora_coreos_v1/templates/kubecluster.yaml index e4e97441c1..378987de60 100644 --- a/magnum/drivers/k8s_fedora_coreos_v1/templates/kubecluster.yaml +++ b/magnum/drivers/k8s_fedora_coreos_v1/templates/kubecluster.yaml @@ -122,6 +122,16 @@ parameters: description: enable/disable the use of heapster default: true + metrics_server_enabled: + type: boolean + description: enable/disable the use of metrics-server + default: true + + metrics_server_chart_tag: + type: string + description: tag of the stable/metrics-server chart to install + default: v2.8.8 + minion_flavor: type: string default: m1.small @@ -1007,6 +1017,8 @@ resources: nodegroup_role: {get_param: master_role} nodegroup_name: {get_param: master_nodegroup_name} heapster_enabled: {get_param: heapster_enabled} + metrics_server_enabled: {get_param: metrics_server_enabled} + metrics_server_chart_tag: {get_param: metrics_server_chart_tag} prometheus_monitoring: {get_param: prometheus_monitoring} grafana_admin_passwd: {get_param: grafana_admin_passwd} api_public_address: {get_attr: [api_lb, floating_address]} diff --git a/magnum/drivers/k8s_fedora_coreos_v1/templates/kubemaster.yaml b/magnum/drivers/k8s_fedora_coreos_v1/templates/kubemaster.yaml index ab43ddb694..a92d4073c0 100644 --- a/magnum/drivers/k8s_fedora_coreos_v1/templates/kubemaster.yaml +++ b/magnum/drivers/k8s_fedora_coreos_v1/templates/kubemaster.yaml @@ -31,6 +31,14 @@ parameters: type: boolean description: enable/disable the use of heapster + metrics_server_enabled: + type: boolean + description: enable/disable the use of metrics-server + + metrics_server_chart_tag: + type: string + description: tag of the stable/metrics-server chart to install + ssh_key_name: type: string description: name of ssh key to be provisioned on our server @@ -633,6 +641,8 @@ resources: params: "$INSTANCE_NAME": {get_param: name} "$HEAPSTER_ENABLED": {get_param: heapster_enabled} + "$METRICS_SERVER_ENABLED": {get_param: metrics_server_enabled} + "$METRICS_SERVER_CHART_TAG": {get_param: metrics_server_chart_tag} "$PROMETHEUS_MONITORING": {get_param: prometheus_monitoring} "$KUBE_API_PUBLIC_ADDRESS": {get_attr: [api_address_switch, public_ip]} "$KUBE_API_PRIVATE_ADDRESS": {get_attr: [api_address_switch, private_ip]} diff --git a/magnum/tests/unit/drivers/test_template_definition.py b/magnum/tests/unit/drivers/test_template_definition.py index a8b62859a3..22fbcc4c45 100644 --- a/magnum/tests/unit/drivers/test_template_definition.py +++ b/magnum/tests/unit/drivers/test_template_definition.py @@ -466,6 +466,10 @@ class AtomicK8sTemplateDefinitionTestCase(BaseK8sTemplateDefinitionTestCase): flannel_backend = mock_cluster.labels.get('flannel_backend') heapster_enabled = mock_cluster.labels.get( 'heapster_enabled') + metrics_server_enabled = mock_cluster.labels.get( + 'metrics_server_enabled') + metrics_server_chart_tag = mock_cluster.labels.get( + 'metrics_server_chart_tag') system_pods_initial_delay = mock_cluster.labels.get( 'system_pods_initial_delay') system_pods_timeout = mock_cluster.labels.get( @@ -594,6 +598,8 @@ class AtomicK8sTemplateDefinitionTestCase(BaseK8sTemplateDefinitionTestCase): 'flannel_network_subnetlen': flannel_subnet, 'flannel_backend': flannel_backend, 'heapster_enabled': heapster_enabled, + 'metrics_server_enabled': metrics_server_enabled, + 'metrics_server_chart_tag': metrics_server_chart_tag, 'system_pods_initial_delay': system_pods_initial_delay, 'system_pods_timeout': system_pods_timeout, 'admission_control_list': admission_control_list, @@ -924,6 +930,10 @@ class AtomicK8sTemplateDefinitionTestCase(BaseK8sTemplateDefinitionTestCase): flannel_backend = mock_cluster.labels.get('flannel_backend') heapster_enabled = mock_cluster.labels.get( 'heapster_enabled') + metrics_server_enabled = mock_cluster.labels.get( + 'metrics_server_enabled') + metrics_server_chart_tag = mock_cluster.labels.get( + 'metrics_server_chart_tag') system_pods_initial_delay = mock_cluster.labels.get( 'system_pods_initial_delay') system_pods_timeout = mock_cluster.labels.get( @@ -1052,6 +1062,8 @@ class AtomicK8sTemplateDefinitionTestCase(BaseK8sTemplateDefinitionTestCase): 'flannel_network_subnetlen': flannel_subnet, 'flannel_backend': flannel_backend, 'heapster_enabled': heapster_enabled, + 'metrics_server_enabled': metrics_server_enabled, + 'metrics_server_chart_tag': metrics_server_chart_tag, 'system_pods_initial_delay': system_pods_initial_delay, 'system_pods_timeout': system_pods_timeout, 'fixed_network_name': fixed_network_name, diff --git a/releasenotes/notes/helm-install-metrics-service-e7a5459417504a75.yaml b/releasenotes/notes/helm-install-metrics-service-e7a5459417504a75.yaml new file mode 100644 index 0000000000..2c3ea46d93 --- /dev/null +++ b/releasenotes/notes/helm-install-metrics-service-e7a5459417504a75.yaml @@ -0,0 +1,10 @@ +--- +features: + - | + Installs the metrics-server service that is replacing kubernetes + deprecated heapster as a cluster wide metrics reporting service used by + schedulling, HPA and others. This service is installed and configured + using helm and so tiller_enabled flag must be True. The label + metrics_server_chart_tag can be used to specify the stable/metrics-server + chart tag to be used. The label metrics_server_enabled is used to enable + disable the installation of the metrics server (default: true).