openstack
/
magnum
Code Issues Proposed changes

k8s: allow passing extra options to kube daemons 

Define a set of new labels to pass additional options to the kubernetes
daemons - kubelet_options, kubeapi_options, kubescheduler_options,
kubecontroller_options, kubeproxy_options.

In all cases the default value is "", meaning no extra options are
passed to the daemons.

Change-Id: Idabe33b1365c7530edc53d1a81dee3c857a4ea47
Closes-Bug: #1701223
This commit is contained in:
Ricardo Rocha 2017-07-06 13:00:10 +00:00 committed by Spyros Trigazis
parent 0b18989a50
commit 4efb58b28d
11 changed files with 215 additions and 3 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

42
doc/source/user/index.rst
View File

@ -357,6 +357,19 @@ the table are linked to more details elsewhere in the user guide.
+---------------------------------------+--------------------+---------------+
| `ingress_controller_role`_ | see below | "ingress" |
+---------------------------------------+--------------------+---------------+
| `kubelet_options`_ | extra kubelet args | "" |
+---------------------------------------+--------------------+---------------+
| `kubeapi_options`_ | extra kubeapi args | "" |
+---------------------------------------+--------------------+---------------+
| `kubescheduler_options`_ | extra kubescheduler| "" |
| | args | |
+---------------------------------------+--------------------+---------------+
| `kubecontroller_options`_ | extra | "" |
| | kubecontroller args| |
+---------------------------------------+--------------------+---------------+
| `kubeproxy_options`_ | extra kubeproxy | "" |
| | args | |
+---------------------------------------+--------------------+---------------+
Cluster
-------
@ -1121,6 +1134,35 @@ _`kube_dashboard_enabled`
_`cert_manager_api`
This label enables the kubernetes `certificate manager api
<https://kubernetes.io/docs/tasks/tls/managing-tls-in-a-cluster/>`_.
_`kubelet_options`
This label can hold any additional options to be passed to the kubelet.
For more details, refer to the `kubelet admin guide
<https://kubernetes.io/docs/admin/kubelet//>`_.
By default no additional options are passed.
_`kubeproxy_options`
This label can hold any additional options to be passed to the kube proxy.
For more details, refer to the `kube proxy admin guide
<https://kubernetes.io/docs/admin/kube-proxy//>`_.
By default no additional options are passed.
_`kubecontroller_options`
This label can hold any additional options to be passed to the kube controller manager.
For more details, refer to the `kube controller manager admin guide
<https://kubernetes.io/docs/admin/kube-controller-manager//>`_.
By default no additional options are passed.
_`kubeapi_options`
This label can hold any additional options to be passed to the kube api server.
For more details, refer to the `kube api admin guide
<https://kubernetes.io/docs/admin/kube-apiserver//>`_.
By default no additional options are passed.
_`kubescheduler_options`
This label can hold any additional options to be passed to the kube scheduler.
For more details, refer to the `kube scheduler admin guide
<https://kubernetes.io/docs/admin/kube-scheduler//>`_.
By default no additional options are passed.
External load balancer for services
-----------------------------------

7
magnum/drivers/common/templates/kubernetes/fragments/configure-kubernetes-master.sh
View File

@ -18,6 +18,7 @@ CERT_DIR=/etc/kubernetes/certs
KUBE_API_ARGS="--runtime-config=api/all=true"
KUBE_API_ARGS="$KUBE_API_ARGS --kubelet-preferred-address-types=InternalIP,Hostname,ExternalIP"
KUBE_API_ARGS="$KUBE_API_ARGS $KUBEAPI_OPTIONS"
if [ "$TLS_DISABLED" == "True" ]; then
KUBE_API_ADDRESS="--insecure-bind-address=0.0.0.0 --insecure-port=$KUBE_API_PORT"
else
@ -49,6 +50,7 @@ sed -i '
# Add controller manager args
KUBE_CONTROLLER_MANAGER_ARGS="--leader-elect=true"
KUBE_CONTROLLER_MANAGER_ARGS="$KUBE_CONTROLLER_MANAGER_ARGS $KUBECONTROLLER_OPTIONS"
if [ -n "${ADMISSION_CONTROL_LIST}" ] && [ "${TLS_DISABLED}" == "False" ]; then
KUBE_CONTROLLER_MANAGER_ARGS="$KUBE_CONTROLLER_MANAGER_ARGS --service-account-private-key-file=$CERT_DIR/server.key --root-ca-file=$CERT_DIR/ca.crt"
fi
@ -68,6 +70,11 @@ sed -i '
sed -i '/^KUBE_SCHEDULER_ARGS=/ s/=.*/="--leader-elect=true"/' /etc/kubernetes/scheduler
HOSTNAME_OVERRIDE=$(hostname --short | sed 's/\.novalocal//')
KUBELET_ARGS="--register-node=true --register-schedulable=false --pod-manifest-path=/etc/kubernetes/manifests --hostname-override=${HOSTNAME_OVERRIDE}"
KUBELET_ARGS="${KUBELET_ARGS} --cluster_dns=${DNS_SERVICE_IP} --cluster_domain=${DNS_CLUSTER_DOMAIN}"
KUBELET_ARGS="${KUBELET_ARGS} ${KUBELET_OPTIONS}"
# For using default log-driver, other options should be ignored
sed -i 's/\-\-log\-driver\=journald//g' /etc/sysconfig/docker

1
magnum/drivers/common/templates/kubernetes/fragments/configure-kubernetes-minion.sh
View File

@ -114,6 +114,7 @@ sed -i '
mkdir -p /etc/kubernetes/manifests
KUBELET_ARGS="--pod-manifest-path=/etc/kubernetes/manifests --cadvisor-port=4194 --kubeconfig ${KUBELET_KUBECONFIG} --hostname-override=${HOSTNAME_OVERRIDE}"
KUBELET_ARGS="${KUBELET_ARGS} --cluster_dns=${DNS_SERVICE_IP} --cluster_domain=${DNS_CLUSTER_DOMAIN}"
KUBELET_ARGS="${KUBELET_ARGS} ${KUBELET_OPTIONS}"
if [ -n "$TRUST_ID" ]; then
KUBELET_ARGS="$KUBELET_ARGS --cloud-provider=openstack --cloud-config=/etc/kubernetes/kube_openstack_config"

5
magnum/drivers/common/templates/kubernetes/fragments/write-heat-params-master.yaml
View File

@ -61,3 +61,8 @@ write_files:
CALICO_IPV4POOL="$CALICO_IPV4POOL"
INGRESS_CONTROLLER="$INGRESS_CONTROLLER"
INGRESS_CONTROLLER_ROLE="$INGRESS_CONTROLLER_ROLE"
KUBELET_OPTIONS="$KUBELET_OPTIONS"
KUBECONTROLLER_OPTIONS="$KUBECONTROLLER_OPTIONS"
KUBEAPI_OPTIONS="$KUBEAPI_OPTIONS"
KUBEPROXY_OPTIONS="$KUBEPROXY_OPTIONS"
KUBESCHEDULER_OPTIONS="$KUBESCHEDULER_OPTIONS"

2
magnum/drivers/common/templates/kubernetes/fragments/write-heat-params.yaml
View File

@ -45,3 +45,5 @@ write_files:
CONTAINER_INFRA_PREFIX="$CONTAINER_INFRA_PREFIX"
DNS_SERVICE_IP="$DNS_SERVICE_IP"
DNS_CLUSTER_DOMAIN="$DNS_CLUSTER_DOMAIN"
KUBELET_OPTIONS="$KUBELET_OPTIONS"
KUBEPROXY_OPTIONS="$KUBEPROXY_OPTIONS"

7
magnum/drivers/heat/k8s_template_def.py
View File

@ -116,7 +116,12 @@ class K8sTemplateDefinition(template_def.BaseTemplateDefinition):
'etcd_volume_size',
'cert_manager_api',
'ingress_controller',
'ingress_controller_role']
'ingress_controller_role',
'kubelet_options',
'kubeapi_options',
'kubeproxy_options',
'kubecontroller_options',
'kubescheduler_options']
for label in label_list:
extra_params[label] = cluster.labels.get(label)

37
magnum/drivers/k8s_fedora_atomic_v1/templates/kubecluster.yaml
View File

@ -417,6 +417,36 @@ parameters:
node role where the ingress controller backend should run
default: "ingress"
kubelet_options:
type: string
description: >
additional options to be passed to the kubelet
default: ""
kubeapi_options:
type: string
description: >
additional options to be passed to the api
default: ""
kubecontroller_options:
type: string
description: >
additional options to be passed to the controller manager
default: ""
kubeproxy_options:
type: string
description: >
additional options to be passed to the kube proxy
default: ""
kubescheduler_options:
type: string
description: >
additional options to be passed to the scheduler
default: ""
resources:
######################################################################
@ -620,6 +650,11 @@ resources:
pods_network_cidr: {get_param: pods_network_cidr}
ingress_controller: {get_param: ingress_controller}
ingress_controller_role: {get_param: ingress_controller_role}
kubelet_options: {get_param: kubelet_options}
kubeapi_options: {get_param: kubeapi_options}
kubeproxy_options: {get_param: kubeproxy_options}
kubecontroller_options: {get_param: kubecontroller_options}
kubescheduler_options: {get_param: kubescheduler_options}
######################################################################
#
@ -692,6 +727,8 @@ resources:
nodes_server_group_id: {get_resource: nodes_server_group}
availability_zone: {get_param: availability_zone}
pods_network_cidr: {get_param: pods_network_cidr}
kubelet_options: {get_param: kubelet_options}
kubeproxy_options: {get_param: kubeproxy_options}
outputs:

30
magnum/drivers/k8s_fedora_atomic_v1/templates/kubemaster.yaml
View File

@ -313,6 +313,31 @@ parameters:
description: >
node role where the ingress controller should run
kubelet_options:
type: string
description: >
additional options to be passed to the kubelet
kubeapi_options:
type: string
description: >
additional options to be passed to the api
kubecontroller_options:
type: string
description: >
additional options to be passed to the controller manager
kubeproxy_options:
type: string
description: >
additional options to be passed to the kube proxy
kubescheduler_options:
type: string
description: >
additional options to be passed to the scheduler
resources:
master_wait_handle:
@ -408,6 +433,11 @@ resources:
"$CALICO_IPV4POOL": {get_param: calico_ipv4pool}
"$INGRESS_CONTROLLER": {get_param: ingress_controller}
"$INGRESS_CONTROLLER_ROLE": {get_param: ingress_controller_role}
"$KUBELET_OPTIONS": {get_param: kubelet_options}
"$KUBEAPI_OPTIONS": {get_param: kubeapi_options}
"$KUBECONTROLLER_OPTIONS": {get_param: kubecontroller_options}
"$KUBEPROXY_OPTIONS": {get_param: kubeproxy_options}
"$KUBESCHEDULER_OPTIONS": {get_param: kubescheduler_options}
install_openstack_ca:
type: OS::Heat::SoftwareConfig

12
magnum/drivers/k8s_fedora_atomic_v1/templates/kubeminion.yaml
View File

@ -245,6 +245,16 @@ parameters:
type: string
description: Configure the IP pool/range from which pod IPs will be chosen
kubelet_options:
type: string
description: >
additional options to be passed to the kubelet
kubeproxy_options:
type: string
description: >
additional options to be passed to the kube proxy
resources:
minion_wait_handle:
@ -310,6 +320,8 @@ resources:
$CONTAINER_INFRA_PREFIX: {get_param: container_infra_prefix}
$DNS_SERVICE_IP: {get_param: dns_service_ip}
$DNS_CLUSTER_DOMAIN: {get_param: dns_cluster_domain}
$KUBELET_OPTIONS: {get_param: kubelet_options}
$KUBEPROXY_OPTIONS: {get_param: kubeproxy_options}
install_openstack_ca:
type: OS::Heat::SoftwareConfig

45
magnum/tests/unit/conductor/handlers/test_k8s_cluster_conductor.py
View File

@ -100,7 +100,12 @@ class TestClusterConductorWithK8s(base.TestCase):
'availability_zone': 'az_1',
'cert_manager_api': 'False',
'ingress_controller': 'i-controller',
'ingress_controller_role': 'i-controller-role'},
'ingress_controller_role': 'i-controller-role',
'kubelet_options': '--kubelet',
'kubeapi_options': '--kubeapi',
'kubecontroller_options': '--kubecontroller',
'kubescheduler_options': '--kubescheduler',
'kubeproxy_options': '--kubeproxy'},
'master_flavor_id': 'master_flavor_id',
'flavor_id': 'flavor_id',
}
@ -183,7 +188,13 @@ class TestClusterConductorWithK8s(base.TestCase):
'availability_zone': 'az_1',
'cert_manager_api': 'False',
'ingress_controller': 'i-controller',
'ingress_controller_role': 'i-controller-role'},
'ingress_controller_role': 'i-controller-role',
'kubelet_options': '--kubelet',
'kubeapi_options': '--kubeapi',
'kubecontroller_options': '--kubecontroller',
'kubescheduler_options': '--kubescheduler',
'kubeproxy_options': '--kubeproxy',
},
'http_proxy': 'http_proxy',
'https_proxy': 'https_proxy',
'no_proxy': 'no_proxy',
@ -243,6 +254,11 @@ class TestClusterConductorWithK8s(base.TestCase):
'cert_manager_api': 'False',
'ingress_controller': 'i-controller',
'ingress_controller_role': 'i-controller-role',
'kubelet_options': '--kubelet',
'kubeapi_options': '--kubeapi',
'kubecontroller_options': '--kubecontroller',
'kubescheduler_options': '--kubescheduler',
'kubeproxy_options': '--kubeproxy',
}
if missing_attr is not None:
expected.pop(mapping[missing_attr], None)
@ -344,6 +360,11 @@ class TestClusterConductorWithK8s(base.TestCase):
'cert_manager_api': 'False',
'ingress_controller': 'i-controller',
'ingress_controller_role': 'i-controller-role',
'kubelet_options': '--kubelet',
'kubeapi_options': '--kubeapi',
'kubecontroller_options': '--kubecontroller',
'kubescheduler_options': '--kubescheduler',
'kubeproxy_options': '--kubeproxy',
}
self.assertEqual(expected, definition)
@ -432,6 +453,11 @@ class TestClusterConductorWithK8s(base.TestCase):
'cert_manager_api': 'False',
'ingress_controller': 'i-controller',
'ingress_controller_role': 'i-controller-role',
'kubelet_options': '--kubelet',
'kubeapi_options': '--kubeapi',
'kubecontroller_options': '--kubecontroller',
'kubescheduler_options': '--kubescheduler',
'kubeproxy_options': '--kubeproxy',
}
self.assertEqual(expected, definition)
self.assertEqual(
@ -513,6 +539,11 @@ class TestClusterConductorWithK8s(base.TestCase):
'cert_manager_api': 'False',
'ingress_controller': 'i-controller',
'ingress_controller_role': 'i-controller-role',
'kubelet_options': '--kubelet',
'kubeapi_options': '--kubeapi',
'kubecontroller_options': '--kubecontroller',
'kubescheduler_options': '--kubescheduler',
'kubeproxy_options': '--kubeproxy',
}
self.assertEqual(expected, definition)
self.assertEqual(
@ -589,6 +620,11 @@ class TestClusterConductorWithK8s(base.TestCase):
'cert_manager_api': 'False',
'ingress_controller': 'i-controller',
'ingress_controller_role': 'i-controller-role',
'kubelet_options': '--kubelet',
'kubeapi_options': '--kubeapi',
'kubecontroller_options': '--kubecontroller',
'kubescheduler_options': '--kubescheduler',
'kubeproxy_options': '--kubeproxy',
}
self.assertEqual(expected, definition)
self.assertEqual(
@ -766,6 +802,11 @@ class TestClusterConductorWithK8s(base.TestCase):
'cert_manager_api': 'False',
'ingress_controller': 'i-controller',
'ingress_controller_role': 'i-controller-role',
'kubelet_options': '--kubelet',
'kubeapi_options': '--kubeapi',
'kubecontroller_options': '--kubecontroller',
'kubescheduler_options': '--kubescheduler',
'kubeproxy_options': '--kubeproxy',
}
self.assertEqual(expected, definition)
self.assertEqual(

30
magnum/tests/unit/drivers/test_template_definition.py
View File

@ -292,6 +292,16 @@ class AtomicK8sTemplateDefinitionTestCase(BaseTemplateDefinitionTestCase):
'ingress_controller')
ingress_controller_role = mock_cluster.labels.get(
'ingress_controller_role')
kubelet_options = mock_cluster.labels.get(
'kubelet_options')
kubeapi_options = mock_cluster.labels.get(
'kubeapi_options')
kubecontroller_options = mock_cluster.labels.get(
'kubecontroller_options')
kubescheduler_options = mock_cluster.labels.get(
'kubescheduler_options')
kubeproxy_options = mock_cluster.labels.get(
'kubeproxy_options')
k8s_def = k8sa_tdef.AtomicK8sTemplateDefinition()
@ -312,6 +322,11 @@ class AtomicK8sTemplateDefinitionTestCase(BaseTemplateDefinitionTestCase):
'kube_dashboard_enabled': kube_dashboard_enabled,
'docker_volume_type': docker_volume_type,
'etcd_volume_size': etcd_volume_size,
'kubelet_options': kubelet_options,
'kubeapi_options': kubeapi_options,
'kubecontroller_options': kubecontroller_options,
'kubescheduler_options': kubescheduler_options,
'kubeproxy_options': kubeproxy_options,
'username': 'fake_user',
'magnum_url': mock_osc.magnum_url.return_value,
'region_name': mock_osc.cinder_region_name.return_value,
@ -405,6 +420,16 @@ class AtomicK8sTemplateDefinitionTestCase(BaseTemplateDefinitionTestCase):
'ingress_controller')
ingress_controller_role = mock_cluster.labels.get(
'ingress_controller_role')
kubelet_options = mock_cluster.labels.get(
'kubelet_options')
kubeapi_options = mock_cluster.labels.get(
'kubeapi_options')
kubecontroller_options = mock_cluster.labels.get(
'kubecontroller_options')
kubescheduler_options = mock_cluster.labels.get(
'kubescheduler_options')
kubeproxy_options = mock_cluster.labels.get(
'kubeproxy_options')
k8s_def = k8sa_tdef.AtomicK8sTemplateDefinition()
@ -425,6 +450,11 @@ class AtomicK8sTemplateDefinitionTestCase(BaseTemplateDefinitionTestCase):
'kube_dashboard_enabled': kube_dashboard_enabled,
'docker_volume_type': docker_volume_type,
'etcd_volume_size': etcd_volume_size,
'kubelet_options': kubelet_options,
'kubeapi_options': kubeapi_options,
'kubecontroller_options': kubecontroller_options,
'kubescheduler_options': kubescheduler_options,
'kubeproxy_options': kubeproxy_options,
'username': 'fake_user',
'magnum_url': mock_osc.magnum_url.return_value,
'region_name': mock_osc.cinder_region_name.return_value,