From 7f4d92d6a3551b6b4ea2838fe9f9fc23775c3fa1 Mon Sep 17 00:00:00 2001 From: Spyros Trigazis Date: Wed, 30 May 2018 14:24:57 +0200 Subject: [PATCH] k8s_fedora: Add cloud_provider_enabled label Add 'cloud_provider_enabled' label for the k8s_fedora_atomic driver. Defaults to true. For specific kubernetes versions if 'cinder' is selected as a 'volume_driver', it is implied that the cloud provider will be enabled since they are combined. The motivation for this change is that in environments with high load to the OpenStack APIs, users might want to disable the cloud provider. story: 1775358 task: 20253 Change-Id: I2920f699654af1f4ba45644ab60a04a3f70918fe (cherry picked from commit 974399a912b02ebe1587ad932a405ca8b44dc947) --- doc/source/user/index.rst | 9 +++++++++ .../fragments/configure-kubernetes-master.sh | 4 ++-- .../fragments/configure-kubernetes-minion.sh | 2 +- .../fragments/write-heat-params-master.yaml | 1 + .../fragments/write-heat-params.yaml | 1 + magnum/drivers/heat/k8s_fedora_template_def.py | 16 +++++++++++++++- .../templates/kubecluster.yaml | 7 +++++++ .../templates/kubemaster.yaml | 5 +++++ .../templates/kubeminion.yaml | 5 +++++ .../unit/drivers/test_template_definition.py | 18 ++++++++++++++++++ ..._cloud_provider_label-ed79295041bc46a8.yaml | 7 +++++++ 11 files changed, 71 insertions(+), 4 deletions(-) create mode 100644 releasenotes/notes/enable_cloud_provider_label-ed79295041bc46a8.yaml diff --git a/doc/source/user/index.rst b/doc/source/user/index.rst index c2e05a1287..ada6038d99 100644 --- a/doc/source/user/index.rst +++ b/doc/source/user/index.rst @@ -380,6 +380,9 @@ the table are linked to more details elsewhere in the user guide. | `cgroup_driver`_ | - systemd | "systemd" | | | - cgroupfs | | +---------------------------------------+--------------------+---------------+ +| `cloud_provider_enabled`_ | - true | true | +| | - false | | ++---------------------------------------+--------------------+---------------+ Cluster ------- @@ -1200,6 +1203,12 @@ _`cgroup_driver` should be identical to the Cgroup driver that Docker has been started with. +_`cloud_provider_enabled` + Add 'cloud_provider_enabled' label for the k8s_fedora_atomic driver. Defaults + to true. For specific kubernetes versions if 'cinder' is selected as a + 'volume_driver', it is implied that the cloud provider will be enabled since + they are combined. + External load balancer for services ----------------------------------- diff --git a/magnum/drivers/common/templates/kubernetes/fragments/configure-kubernetes-master.sh b/magnum/drivers/common/templates/kubernetes/fragments/configure-kubernetes-master.sh index 752257a90d..fd071fe049 100644 --- a/magnum/drivers/common/templates/kubernetes/fragments/configure-kubernetes-master.sh +++ b/magnum/drivers/common/templates/kubernetes/fragments/configure-kubernetes-master.sh @@ -81,7 +81,7 @@ if [ -n "${ADMISSION_CONTROL_LIST}" ] && [ "${TLS_DISABLED}" == "False" ]; then KUBE_ADMISSION_CONTROL="--admission-control=NodeRestriction,${ADMISSION_CONTROL_LIST}" fi -if [ -n "$TRUST_ID" ]; then +if [ -n "$TRUST_ID" && "$(echo $CLOUD_PROVIDER_ENABLED | tr '[:upper:]' '[:lower:]')" == "true" ]; then KUBE_API_ARGS="$KUBE_API_ARGS --cloud-config=/etc/kubernetes/kube_openstack_config --cloud-provider=openstack" fi @@ -101,7 +101,7 @@ if [ -n "${ADMISSION_CONTROL_LIST}" ] && [ "${TLS_DISABLED}" == "False" ]; then KUBE_CONTROLLER_MANAGER_ARGS="$KUBE_CONTROLLER_MANAGER_ARGS --service-account-private-key-file=$CERT_DIR/server.key --root-ca-file=$CERT_DIR/ca.crt" fi -if [ -n "$TRUST_ID" ]; then +if [ -n "$TRUST_ID" && "$(echo $CLOUD_PROVIDER_ENABLED | tr '[:upper:]' '[:lower:]')" == "true" ]; then KUBE_CONTROLLER_MANAGER_ARGS="$KUBE_CONTROLLER_MANAGER_ARGS --cloud-config=/etc/kubernetes/kube_openstack_config --cloud-provider=openstack" fi diff --git a/magnum/drivers/common/templates/kubernetes/fragments/configure-kubernetes-minion.sh b/magnum/drivers/common/templates/kubernetes/fragments/configure-kubernetes-minion.sh index ba9cda5147..da9dc954d6 100644 --- a/magnum/drivers/common/templates/kubernetes/fragments/configure-kubernetes-minion.sh +++ b/magnum/drivers/common/templates/kubernetes/fragments/configure-kubernetes-minion.sh @@ -119,7 +119,7 @@ KUBELET_ARGS="${KUBELET_ARGS} --address=${KUBE_NODE_IP} --port=10250 --read-only KUBELET_ARGS="${KUBELET_ARGS} --cluster_dns=${DNS_SERVICE_IP} --cluster_domain=${DNS_CLUSTER_DOMAIN}" KUBELET_ARGS="${KUBELET_ARGS} ${KUBELET_OPTIONS}" -if [ -n "$TRUST_ID" ]; then +if [ -n "$TRUST_ID" && "$(echo $CLOUD_PROVIDER_ENABLED | tr '[:upper:]' '[:lower:]')" == "true" ]; then KUBELET_ARGS="$KUBELET_ARGS --cloud-provider=openstack --cloud-config=/etc/kubernetes/kube_openstack_config" fi diff --git a/magnum/drivers/common/templates/kubernetes/fragments/write-heat-params-master.yaml b/magnum/drivers/common/templates/kubernetes/fragments/write-heat-params-master.yaml index d9fa4b51e6..288100b377 100644 --- a/magnum/drivers/common/templates/kubernetes/fragments/write-heat-params-master.yaml +++ b/magnum/drivers/common/templates/kubernetes/fragments/write-heat-params-master.yaml @@ -52,6 +52,7 @@ write_files: TRUSTEE_PASSWORD="$TRUSTEE_PASSWORD" TRUST_ID="$TRUST_ID" AUTH_URL="$AUTH_URL" + CLOUD_PROVIDER_ENABLED="$CLOUD_PROVIDER_ENABLED" INSECURE_REGISTRY_URL="$INSECURE_REGISTRY_URL" CONTAINER_INFRA_PREFIX="$CONTAINER_INFRA_PREFIX" SYSTEM_PODS_INITIAL_DELAY="$SYSTEM_PODS_INITIAL_DELAY" diff --git a/magnum/drivers/common/templates/kubernetes/fragments/write-heat-params.yaml b/magnum/drivers/common/templates/kubernetes/fragments/write-heat-params.yaml index 67553bce10..00f83a4d73 100644 --- a/magnum/drivers/common/templates/kubernetes/fragments/write-heat-params.yaml +++ b/magnum/drivers/common/templates/kubernetes/fragments/write-heat-params.yaml @@ -45,6 +45,7 @@ write_files: TRUSTEE_USER_ID="$TRUSTEE_USER_ID" TRUSTEE_PASSWORD="$TRUSTEE_PASSWORD" TRUST_ID="$TRUST_ID" + CLOUD_PROVIDER_ENABLED="$CLOUD_PROVIDER_ENABLED" INSECURE_REGISTRY_URL="$INSECURE_REGISTRY_URL" CONTAINER_INFRA_PREFIX="$CONTAINER_INFRA_PREFIX" DNS_SERVICE_IP="$DNS_SERVICE_IP" diff --git a/magnum/drivers/heat/k8s_fedora_template_def.py b/magnum/drivers/heat/k8s_fedora_template_def.py index 16884f08ea..ebbe5b8eff 100644 --- a/magnum/drivers/heat/k8s_fedora_template_def.py +++ b/magnum/drivers/heat/k8s_fedora_template_def.py @@ -13,10 +13,12 @@ from oslo_log import log as logging from oslo_utils import strutils +from magnum.common import exception from magnum.common.x509 import operations as x509 from magnum.conductor.handlers.common import cert_manager from magnum.drivers.heat import k8s_template_def from magnum.drivers.heat import template_def +from magnum.i18n import _ from oslo_config import cfg CONF = cfg.CONF @@ -91,12 +93,24 @@ class K8sFedoraTemplateDefinition(k8s_template_def.K8sTemplateDefinition): extra_params["pods_network_cidr"] = \ cluster.labels.get('calico_ipv4pool', '192.168.0.0/16') + # check cloud provider and cinder options. If cinder is selected, + # the cloud provider needs to be enabled. + cloud_provider_enabled = cluster.labels.get( + 'cloud_provider_enabled', 'true').lower() + if (cluster_template.volume_driver == 'cinder' + and cloud_provider_enabled == 'false'): + raise exception.InvalidParameterValue(_( + '"cinder" volume driver needs "cloud_provider_enabled" label ' + 'to be true or unset.')) + label_list = ['kube_tag', 'container_infra_prefix', 'availability_zone', 'cgroup_driver', 'calico_tag', 'calico_cni_tag', 'calico_kube_controllers_tag', 'calico_ipv4pool', - 'etcd_tag', 'flannel_tag'] + 'etcd_tag', 'flannel_tag', + 'cloud_provider_enabled'] + for label in label_list: label_value = cluster.labels.get(label) if label_value: diff --git a/magnum/drivers/k8s_fedora_atomic_v1/templates/kubecluster.yaml b/magnum/drivers/k8s_fedora_atomic_v1/templates/kubecluster.yaml index 727d83a50e..d7fa0fb404 100644 --- a/magnum/drivers/k8s_fedora_atomic_v1/templates/kubecluster.yaml +++ b/magnum/drivers/k8s_fedora_atomic_v1/templates/kubecluster.yaml @@ -475,6 +475,11 @@ parameters: whether or not to use Octavia for LoadBalancer type service. default: False + cloud_provider_enabled: + type: boolean + description: Enable or disable the openstack kubernetes cloud provider + default: true + resources: ###################################################################### @@ -670,6 +675,7 @@ resources: trustee_password: {get_param: trustee_password} trust_id: {get_param: trust_id} auth_url: {get_param: auth_url} + cloud_provider_enabled: {get_param: cloud_provider_enabled} insecure_registry_url: {get_param: insecure_registry_url} container_infra_prefix: {get_param: container_infra_prefix} etcd_lb_vip: {get_attr: [etcd_lb, address]} @@ -759,6 +765,7 @@ resources: trustee_password: {get_param: trustee_password} trustee_domain_id: {get_param: trustee_domain_id} trust_id: {get_param: trust_id} + cloud_provider_enabled: {get_param: cloud_provider_enabled} insecure_registry_url: {get_param: insecure_registry_url} container_infra_prefix: {get_param: container_infra_prefix} dns_service_ip: {get_param: dns_service_ip} diff --git a/magnum/drivers/k8s_fedora_atomic_v1/templates/kubemaster.yaml b/magnum/drivers/k8s_fedora_atomic_v1/templates/kubemaster.yaml index fee552a40a..77f743ecc1 100644 --- a/magnum/drivers/k8s_fedora_atomic_v1/templates/kubemaster.yaml +++ b/magnum/drivers/k8s_fedora_atomic_v1/templates/kubemaster.yaml @@ -373,6 +373,10 @@ parameters: the index of master node, index 0 means the master node is the primary, bootstrapping node. + cloud_provider_enabled: + type: boolean + description: Enable or disable the openstack kubernetes cloud provider + resources: master_wait_handle: @@ -461,6 +465,7 @@ resources: "$TRUSTEE_USER_ID": {get_param: trustee_user_id} "$TRUSTEE_PASSWORD": {get_param: trustee_password} "$TRUST_ID": {get_param: trust_id} + "$CLOUD_PROVIDER_ENABLED": {get_param: cloud_provider_enabled} "$INSECURE_REGISTRY_URL": {get_param: insecure_registry_url} "$CONTAINER_INFRA_PREFIX": {get_param: container_infra_prefix} "$ETCD_LB_VIP": {get_param: etcd_lb_vip} diff --git a/magnum/drivers/k8s_fedora_atomic_v1/templates/kubeminion.yaml b/magnum/drivers/k8s_fedora_atomic_v1/templates/kubeminion.yaml index f134afd8f7..26475aa729 100644 --- a/magnum/drivers/k8s_fedora_atomic_v1/templates/kubeminion.yaml +++ b/magnum/drivers/k8s_fedora_atomic_v1/templates/kubeminion.yaml @@ -272,6 +272,10 @@ parameters: whether or not to use Octavia for LoadBalancer type service. default: False + cloud_provider_enabled: + type: boolean + description: Enable or disable the openstack kubernetes cloud provider + resources: minion_wait_handle: @@ -337,6 +341,7 @@ resources: $TRUSTEE_PASSWORD: {get_param: trustee_password} $TRUST_ID: {get_param: trust_id} $AUTH_URL: {get_param: auth_url} + $CLOUD_PROVIDER_ENABLED: {get_param: cloud_provider_enabled} $INSECURE_REGISTRY_URL: {get_param: insecure_registry_url} $CONTAINER_INFRA_PREFIX: {get_param: container_infra_prefix} $DNS_SERVICE_IP: {get_param: dns_service_ip} diff --git a/magnum/tests/unit/drivers/test_template_definition.py b/magnum/tests/unit/drivers/test_template_definition.py index caf3ce3e0f..167d055825 100644 --- a/magnum/tests/unit/drivers/test_template_definition.py +++ b/magnum/tests/unit/drivers/test_template_definition.py @@ -360,6 +360,8 @@ class AtomicK8sTemplateDefinitionTestCase(BaseK8sTemplateDefinitionTestCase): 'kubescheduler_options') kubeproxy_options = mock_cluster.labels.get( 'kubeproxy_options') + cloud_provider_enabled = mock_cluster.labels.get( + 'cloud_provider_enabled') k8s_def = k8sa_tdef.AtomicK8sTemplateDefinition() @@ -387,6 +389,7 @@ class AtomicK8sTemplateDefinitionTestCase(BaseK8sTemplateDefinitionTestCase): 'kubecontroller_options': kubecontroller_options, 'kubescheduler_options': kubescheduler_options, 'kubeproxy_options': kubeproxy_options, + 'cloud_provider_enabled': cloud_provider_enabled, 'username': 'fake_user', 'magnum_url': mock_osc.magnum_url.return_value, 'region_name': mock_osc.cinder_region_name.return_value, @@ -412,6 +415,18 @@ class AtomicK8sTemplateDefinitionTestCase(BaseK8sTemplateDefinitionTestCase): mock_cluster, **expected_kwargs) + mock_cluster_template.volume_driver = 'cinder' + mock_cluster.labels = {'cloud_provider_enabled': 'false'} + k8s_def = k8sa_tdef.AtomicK8sTemplateDefinition() + self.assertRaises( + exception.InvalidParameterValue, + k8s_def.get_params, + mock_context, + mock_cluster_template, + mock_cluster, + scale_manager=mock_scale_manager + ) + @mock.patch('magnum.common.keystone.is_octavia_enabled') @mock.patch('magnum.common.clients.OpenStackClients') @mock.patch('magnum.drivers.heat.template_def' @@ -504,6 +519,8 @@ class AtomicK8sTemplateDefinitionTestCase(BaseK8sTemplateDefinitionTestCase): 'kubescheduler_options') kubeproxy_options = mock_cluster.labels.get( 'kubeproxy_options') + cloud_provider_enabled = mock_cluster.labels.get( + 'cloud_provider_enabled') k8s_def = k8sa_tdef.AtomicK8sTemplateDefinition() @@ -531,6 +548,7 @@ class AtomicK8sTemplateDefinitionTestCase(BaseK8sTemplateDefinitionTestCase): 'kubecontroller_options': kubecontroller_options, 'kubescheduler_options': kubescheduler_options, 'kubeproxy_options': kubeproxy_options, + 'cloud_provider_enabled': cloud_provider_enabled, 'username': 'fake_user', 'magnum_url': mock_osc.magnum_url.return_value, 'region_name': mock_osc.cinder_region_name.return_value, diff --git a/releasenotes/notes/enable_cloud_provider_label-ed79295041bc46a8.yaml b/releasenotes/notes/enable_cloud_provider_label-ed79295041bc46a8.yaml new file mode 100644 index 0000000000..20a7bdc25e --- /dev/null +++ b/releasenotes/notes/enable_cloud_provider_label-ed79295041bc46a8.yaml @@ -0,0 +1,7 @@ +--- +features: + - | + Add 'cloud_provider_enabled' label for the k8s_fedora_atomic driver. + Defaults to true. For specific kubernetes versions if 'cinder' is + selected as a 'volume_driver', it is implied that the cloud provider + will be enabled since they are combined.