diff --git a/doc/source/user/index.rst b/doc/source/user/index.rst index a22b659dea..09b1117ef0 100644 --- a/doc/source/user/index.rst +++ b/doc/source/user/index.rst @@ -356,6 +356,9 @@ the table are linked to more details elsewhere in the user guide. | `kubeproxy_options`_ | extra kubeproxy | "" | | | args | | +---------------------------------------+--------------------+---------------+ +| `cgroup_driver`_ | - systemd | "systemd" | +| | - cgroupfs | | ++---------------------------------------+--------------------+---------------+ Cluster ------- @@ -1142,6 +1145,11 @@ _`influx_grafana_dashboard_enabled` label is set, an influxdb and grafana instance will be deployed, heapster will push data to influx and grafana will project them. +_`cgroup_driver` + This label tells kubelet which Cgroup driver to use. Ideally this + should be identical to the Cgroup driver that Docker has been + started with. + External load balancer for services ----------------------------------- diff --git a/magnum/drivers/common/templates/kubernetes/fragments/configure-kubernetes-master.sh b/magnum/drivers/common/templates/kubernetes/fragments/configure-kubernetes-master.sh index 37ec12aea4..08cf75070b 100644 --- a/magnum/drivers/common/templates/kubernetes/fragments/configure-kubernetes-master.sh +++ b/magnum/drivers/common/templates/kubernetes/fragments/configure-kubernetes-master.sh @@ -1,4 +1,4 @@ -#!/bin/sh +#!/bin/sh -x . /etc/sysconfig/heat-params @@ -140,7 +140,22 @@ EOF KUBELET_ARGS="${KUBELET_ARGS} --client-ca-file=${CERT_DIR}/ca.crt --tls-cert-file=${CERT_DIR}/kubelet.crt --tls-private-key-file=${CERT_DIR}/kubelet.key --kubeconfig ${KUBELET_KUBECONFIG}" # specified cgroup driver - KUBELET_ARGS="${KUBELET_ARGS} --cgroup-driver=systemd" + KUBELET_ARGS="${KUBELET_ARGS} --cgroup-driver=${CGROUP_DRIVER}" + + systemctl disable docker + if cat /usr/lib/systemd/system/docker.service | grep 'native.cgroupdriver'; then + cp /usr/lib/systemd/system/docker.service /etc/systemd/system/ + sed -i "s/\(native.cgroupdriver=\)\w\+/\1$CGROUP_DRIVER/" \ + /etc/systemd/system/docker.service + else + cat > /etc/systemd/system/docker.service.d/cgroupdriver.conf << EOF +ExecStart=---exec-opt native.cgroupdriver=$CGROUP_DRIVER +EOF + + fi + + systemctl daemon-reload + systemctl enable docker if [ -z "${KUBE_NODE_IP}" ]; then KUBE_NODE_IP=$(curl -s http://169.254.169.254/latest/meta-data/local-ipv4) diff --git a/magnum/drivers/common/templates/kubernetes/fragments/configure-kubernetes-minion.sh b/magnum/drivers/common/templates/kubernetes/fragments/configure-kubernetes-minion.sh index 01350c2250..29975a76d0 100644 --- a/magnum/drivers/common/templates/kubernetes/fragments/configure-kubernetes-minion.sh +++ b/magnum/drivers/common/templates/kubernetes/fragments/configure-kubernetes-minion.sh @@ -136,8 +136,25 @@ if [ -n "${INSECURE_REGISTRY_URL}" ]; then echo "INSECURE_REGISTRY='--insecure-registry ${INSECURE_REGISTRY_URL}'" >> /etc/sysconfig/docker fi +KUBELET_ARGS="${KUBELET_ARGS} --client-ca-file=${CERT_DIR}/ca.crt --tls-cert-file=${CERT_DIR}/kubelet.crt --tls-private-key-file=${CERT_DIR}/kubelet.key" + # specified cgroup driver -KUBELET_ARGS="${KUBELET_ARGS} --client-ca-file=${CERT_DIR}/ca.crt --tls-cert-file=${CERT_DIR}/kubelet.crt --tls-private-key-file=${CERT_DIR}/kubelet.key --cgroup-driver=systemd" +KUBELET_ARGS="${KUBELET_ARGS} --cgroup-driver=${CGROUP_DRIVER}" + +systemctl disable docker +if cat /usr/lib/systemd/system/docker.service | grep 'native.cgroupdriver'; then + cp /usr/lib/systemd/system/docker.service /etc/systemd/system/ + sed -i "s/\(native.cgroupdriver=\)\w\+/\1$CGROUP_DRIVER/" \ + /etc/systemd/system/docker.service +else + cat > /etc/systemd/system/docker.service.d/cgroupdriver.conf << EOF +ExecStart=---exec-opt native.cgroupdriver=$CGROUP_DRIVER +EOF + +fi + +systemctl daemon-reload +systemctl enable docker cat > /etc/kubernetes/get_require_kubeconfig.sh < + cgroup driver name that kubelet should use, ideally the same as + the docker cgroup driver. + default: "systemd" + wait_condition_timeout: type: number description: > @@ -627,6 +634,7 @@ resources: docker_volume_size: {get_param: docker_volume_size} docker_volume_type: {get_param: docker_volume_type} docker_storage_driver: {get_param: docker_storage_driver} + cgroup_driver: {get_param: cgroup_driver} wait_condition_timeout: {get_param: wait_condition_timeout} network_driver: {get_param: network_driver} flannel_network_cidr: {get_param: flannel_network_cidr} @@ -723,6 +731,7 @@ resources: docker_volume_size: {get_param: docker_volume_size} docker_volume_type: {get_param: docker_volume_type} docker_storage_driver: {get_param: docker_storage_driver} + cgroup_driver: {get_param: cgroup_driver} wait_condition_timeout: {get_param: wait_condition_timeout} registry_enabled: {get_param: registry_enabled} registry_port: {get_param: registry_port} diff --git a/magnum/drivers/k8s_fedora_atomic_v1/templates/kubemaster.yaml b/magnum/drivers/k8s_fedora_atomic_v1/templates/kubemaster.yaml index 62718041a4..df52c98232 100644 --- a/magnum/drivers/k8s_fedora_atomic_v1/templates/kubemaster.yaml +++ b/magnum/drivers/k8s_fedora_atomic_v1/templates/kubemaster.yaml @@ -61,6 +61,13 @@ parameters: description: docker storage driver name default: "devicemapper" + cgroup_driver: + type: string + description: > + cgroup driver name that kubelet should use, ideally the same as + the docker cgroup driver. + default: "systemd" + volume_driver: type: string description: volume driver to use for container storage @@ -415,6 +422,7 @@ resources: "$DOCKER_VOLUME": {get_resource: docker_volume} "$DOCKER_VOLUME_SIZE": {get_param: docker_volume_size} "$DOCKER_STORAGE_DRIVER": {get_param: docker_storage_driver} + "$CGROUP_DRIVER": {get_param: cgroup_driver} "$NETWORK_DRIVER": {get_param: network_driver} "$FLANNEL_NETWORK_CIDR": {get_param: flannel_network_cidr} "$FLANNEL_NETWORK_SUBNETLEN": {get_param: flannel_network_subnetlen} diff --git a/magnum/drivers/k8s_fedora_atomic_v1/templates/kubeminion.yaml b/magnum/drivers/k8s_fedora_atomic_v1/templates/kubeminion.yaml index 09a311edd9..5facff1a95 100644 --- a/magnum/drivers/k8s_fedora_atomic_v1/templates/kubeminion.yaml +++ b/magnum/drivers/k8s_fedora_atomic_v1/templates/kubeminion.yaml @@ -51,6 +51,13 @@ parameters: description: docker storage driver name default: "devicemapper" + cgroup_driver: + type: string + description: > + cgroup driver name that kubelet should use, ideally the same as + the docker cgroup driver. + default: "systemd" + tls_disabled: type: boolean description: whether or not to enable TLS @@ -301,6 +308,7 @@ resources: $DOCKER_VOLUME: {get_resource: docker_volume} $DOCKER_VOLUME_SIZE: {get_param: docker_volume_size} $DOCKER_STORAGE_DRIVER: {get_param: docker_storage_driver} + $CGROUP_DRIVER: {get_param: cgroup_driver} $NETWORK_DRIVER: {get_param: network_driver} $REGISTRY_ENABLED: {get_param: registry_enabled} $REGISTRY_PORT: {get_param: registry_port} diff --git a/magnum/tests/unit/drivers/test_template_definition.py b/magnum/tests/unit/drivers/test_template_definition.py index f7f3363632..594362abd9 100644 --- a/magnum/tests/unit/drivers/test_template_definition.py +++ b/magnum/tests/unit/drivers/test_template_definition.py @@ -331,6 +331,8 @@ class AtomicK8sTemplateDefinitionTestCase(BaseTemplateDefinitionTestCase): pods_network_cidr = flannel_cidr elif mock_cluster_template.network_driver == 'calico': pods_network_cidr = calico_ipv4pool + cgroup_driver = mock_cluster.labels.get( + 'cgroup_driver') ingress_controller = mock_cluster.labels.get( 'ingress_controller') ingress_controller_role = mock_cluster.labels.get( @@ -386,6 +388,7 @@ class AtomicK8sTemplateDefinitionTestCase(BaseTemplateDefinitionTestCase): 'calico_cni_tag': calico_cni_tag, 'calico_kube_controllers_tag': calico_kube_controllers_tag, 'calico_ipv4pool': calico_ipv4pool, + 'cgroup_driver': cgroup_driver, 'pods_network_cidr': pods_network_cidr, 'ingress_controller': ingress_controller, 'ingress_controller_role': ingress_controller_role, @@ -472,6 +475,8 @@ class AtomicK8sTemplateDefinitionTestCase(BaseTemplateDefinitionTestCase): pods_network_cidr = flannel_cidr elif mock_cluster_template.network_driver == 'calico': pods_network_cidr = calico_ipv4pool + cgroup_driver = mock_cluster.labels.get( + 'cgroup_driver') ingress_controller = mock_cluster.labels.get( 'ingress_controller') ingress_controller_role = mock_cluster.labels.get( @@ -529,6 +534,7 @@ class AtomicK8sTemplateDefinitionTestCase(BaseTemplateDefinitionTestCase): 'calico_cni_tag': calico_cni_tag, 'calico_kube_controllers_tag': calico_kube_controllers_tag, 'calico_ipv4pool': calico_ipv4pool, + 'cgroup_driver': cgroup_driver, 'pods_network_cidr': pods_network_cidr, 'ingress_controller': ingress_controller, 'ingress_controller_role': ingress_controller_role,