From f82749457c0da6879ba5d189a0fcd8d7a7914602 Mon Sep 17 00:00:00 2001 From: Spyros Trigazis Date: Fri, 30 Sep 2016 15:10:52 +0200 Subject: [PATCH] Make cinder volume optional In the swarm_atomic and k8s_atomic drivers container images are stored in a dedicated cinder volume per cluster node. It is proven that this architecture can be a scalability bottleneck. Make the use of cinder volumes for container images and opt-in option. If docker-volume-size is not specified no cinder volumes will be created. Before, if docker-volume-size wasn't specified the default value was 25. To use cinder volumes for container storage the user will interact with magnum as before, (meaning the valid values are integers starting from 1). Closes-Bug: #1638006 Change-Id: I3394c62a43bbf950b7cf0b86a71b1d9b0481d68f --- doc/source/userguide.rst | 76 +++++++--------- .../drivers/common/k8s_fedora_template_def.py | 28 +++--- .../common/swarm_fedora_template_def.py | 15 ++- .../templates/environments/no_volume.yaml | 4 + .../templates/environments/with_volume.yaml | 4 + .../fragments/configure-docker-storage.sh | 46 +++++----- .../configure_docker_storage_driver_atomic.sh | 16 ++-- .../fragments/write-heat-params-master.yaml | 1 + .../fragments/write-heat-params.yaml | 1 + .../fragments/write-heat-params-master.yaml | 1 + .../fragments/write-heat-params-node.yaml | 1 + .../templates/kubecluster.yaml | 2 +- .../templates/kubemaster.yaml | 5 +- .../templates/kubeminion.yaml | 5 +- .../templates/kubecluster.yaml | 4 +- .../templates/kubemaster.yaml | 7 ++ .../templates/kubeminion.yaml | 7 ++ .../templates/cluster.yaml | 2 +- .../templates/swarmmaster.yaml | 5 +- .../templates/swarmnode.yaml | 5 +- .../unit/api/controllers/v1/test_baymodel.py | 1 - .../controllers/v1/test_cluster_template.py | 1 - .../handlers/test_k8s_cluster_conductor.py | 91 +++++++++++++++---- .../handlers/test_swarm_cluster_conductor.py | 15 ++- .../no-cinder-volume-87b9339e066c30a0.yaml | 10 ++ 25 files changed, 232 insertions(+), 121 deletions(-) create mode 100644 magnum/drivers/common/templates/environments/no_volume.yaml create mode 100644 magnum/drivers/common/templates/environments/with_volume.yaml create mode 100644 releasenotes/notes/no-cinder-volume-87b9339e066c30a0.yaml diff --git a/doc/source/userguide.rst b/doc/source/userguide.rst index 1f318b4b18..5879105d88 100644 --- a/doc/source/userguide.rst +++ b/doc/source/userguide.rst @@ -206,11 +206,11 @@ They are loosely grouped as: mandatory, infrastructure, COE specific. is 'None'. --docker-volume-size \ - The size in GB for the local storage on each server for the Docker - daemon to cache the images and host the containers. Cinder volumes - provide the storage. The default is 25 GB. For the 'devicemapper' - storage driver, the minimum value is 3GB. For the 'overlay' storage - driver, the minimum value is 1GB. + If specified, container images will be stored in a cinder volume of the + specified size in GB. Each cluster node will have a volume attached of + the above size. If not specified, images will be stored in the compute + instance's local disk. For the 'devicemapper' storage driver, the minimum + value is 3GB. For the 'overlay' storage driver, the minimum value is 1GB. --docker-storage-driver \ The name of a driver to manage the storage for the images and the @@ -359,8 +359,8 @@ Network needed. Storage - Cinder provides the block storage that is used for both hosting the - containers as well as persistent storage for the containers. + Cinder provides the block storage that can be used to host the + containers and as persistent storage for the containers. Security Barbican provides the storage of secrets such as certificates used @@ -981,14 +981,8 @@ Volume driver (volume-driver) Storage driver (docker-storage-driver) Specified in the ClusterTemplate to select the Docker storage driver. The supported storage drivers are 'devicemapper' and 'overlay', with - 'devicemapper' being the default. You may get better performance with - the overlay driver depending on your use patterns, with the requirement - that SELinux must be disabled inside the containers, although it still runs - in enforcing mode on the cluster servers. Magnum will create a Cinder volume - for each node, mount it on the node and configure it as a logical - volume named 'docker'. The Docker daemon will run the selected device - driver to manage this logical volume and host the container writable - layer there. Refer to the `Storage`_ section for more details. + 'devicemapper' being the default. Refer to the `Storage`_ section for more + details. Image (image-id) Specified in the ClusterTemplate to indicate the image to boot the servers. @@ -1126,15 +1120,8 @@ Volume driver (volume-driver) Storage driver (docker-storage-driver) Specified in the ClusterTemplate to select the Docker storage driver. The supported storage driver are 'devicemapper' and 'overlay', with - 'devicemapper' being the default. You may get better performance with - the 'overlay' driver depending on your use patterns, with the requirement - that SELinux must be disabled inside the containers, although it still runs - in enforcing mode on the cluster servers. Magnum will create a Cinder volume - for each node and attach it as a device. Then depending on the driver, - additional configuration is performed to make the volume available to - the particular driver. For instance, 'devicemapper' uses LVM; therefore - Magnum will create physical volume and logical volume using the attached - device. Refer to the `Storage`_ section for more details. + 'devicemapper' being the default. Refer to the `Storage`_ section for more + details. Image (image-id) Specified in the ClusterTemplate to indicate the image to boot the servers @@ -2132,25 +2119,32 @@ configured in the Docker daemon through a number of storage options. When the container is removed, the storage allocated to the particular container is also deleted. -To manage this space in a flexible manner independent of the Nova -instance flavor, Magnum creates a separate Cinder block volume for each -node in the cluster, mounts it to the node and configures it to be used as -ephemeral storage. Users can specify the size of the Cinder volume with -the ClusterTemplate attribute 'docker-volume-size'. The default size is 5GB. -Currently the block size is fixed at cluster creation time, but future -lifecycle operations may allow modifying the block size during the -life of the cluster. +Magnum can manage the containers' filesystem in two ways, storing them +on the local disk of the compute instances or in a separate Cinder block +volume for each node in the cluster, mounts it to the node and +configures it to be used as ephemeral storage. Users can specify the +size of the Cinder volume with the ClusterTemplate attribute +'docker-volume-size'. Currently the block size is fixed at cluster +creation time, but future lifecycle operations may allow modifying the +block size during the life of the cluster. -To use the Cinder block storage, there is a number of Docker -storage drivers available. Only 'devicemapper' is supported as the -storage driver but other drivers such as 'OverlayFS' are being -considered. There are important trade-off between the choices -for the storage drivers that should be considered. For instance, -'OperlayFS' may offer better performance, but it may not support -the filesystem metadata needed to use SELinux, which is required -to support strong isolation between containers running in the same -cluster. Using the 'devicemapper' driver does allow the use of SELinux. +Both local disk and the Cinder block storage can be used with a number +of Docker storage drivers available. +* 'devicemapper': When used with a dedicated Cinder volume it is + configured using direct-lvm and offers very good performance. If it's + used with the compute instance's local disk uses a loopback device + offering poor performance and it's not recommended for production + environments. Using the 'devicemapper' driver does allow the use of + SELinux. + +* 'overlay' When used with a dedicated Cinder volume offers as good + or better performance than devicemapper. If used on the local disk of + the compute instance (especially with high IOPS drives) you can get + significant performance gains. However, for kernel versions less than + 4.9, SELinux must be disabled inside the containers resulting in worse + container isolation, although it still runs in enforcing mode on the + cluster compute instances. Persistent storage ------------------ diff --git a/magnum/drivers/common/k8s_fedora_template_def.py b/magnum/drivers/common/k8s_fedora_template_def.py index 67d4671333..b7cc9c1c0d 100644 --- a/magnum/drivers/common/k8s_fedora_template_def.py +++ b/magnum/drivers/common/k8s_fedora_template_def.py @@ -80,17 +80,19 @@ class K8sFedoraTemplateDefinition(k8s_template_def.K8sTemplateDefinition): def get_env_files(self, cluster_template): env_files = [] - if cluster_template.master_lb_enabled: - env_files.append( - template_def.COMMON_ENV_PATH + 'with_master_lb.yaml') - else: - env_files.append( - template_def.COMMON_ENV_PATH + 'no_master_lb.yaml') - if cluster_template.floating_ip_enabled: - env_files.append( - template_def.COMMON_ENV_PATH + 'enable_floating_ip.yaml') - else: - env_files.append( - template_def.COMMON_ENV_PATH + 'disable_floating_ip.yaml') - return env_files + if cluster_template.docker_volume_size is None: + env_files.append('no_volume.yaml') + else: + env_files.append('with_volume.yaml') + + if cluster_template.master_lb_enabled: + env_files.append('with_master_lb.yaml') + else: + env_files.append('no_master_lb.yaml') + if cluster_template.floating_ip_enabled: + env_files.append('enable_floating_ip.yaml') + else: + env_files.append('disable_floating_ip.yaml') + + return [template_def.COMMON_ENV_PATH + ef for ef in env_files] diff --git a/magnum/drivers/common/swarm_fedora_template_def.py b/magnum/drivers/common/swarm_fedora_template_def.py index c5c596ab0e..737570252b 100644 --- a/magnum/drivers/common/swarm_fedora_template_def.py +++ b/magnum/drivers/common/swarm_fedora_template_def.py @@ -113,7 +113,16 @@ class SwarmFedoraTemplateDefinition(template_def.BaseTemplateDefinition): **kwargs) def get_env_files(self, cluster_template): - if cluster_template.master_lb_enabled: - return [template_def.COMMON_ENV_PATH + 'with_master_lb.yaml'] + env_files = [] + + if cluster_template.docker_volume_size is None: + env_files.append('no_volume.yaml') else: - return [template_def.COMMON_ENV_PATH + 'no_master_lb.yaml'] + env_files.append('with_volume.yaml') + + if cluster_template.master_lb_enabled: + env_files.append('with_master_lb.yaml') + else: + env_files.append('no_master_lb.yaml') + + return [template_def.COMMON_ENV_PATH + ef for ef in env_files] diff --git a/magnum/drivers/common/templates/environments/no_volume.yaml b/magnum/drivers/common/templates/environments/no_volume.yaml new file mode 100644 index 0000000000..8e2dc31add --- /dev/null +++ b/magnum/drivers/common/templates/environments/no_volume.yaml @@ -0,0 +1,4 @@ +# Environment file to NOT use a cinder volume to store containers +resource_registry: + "Magnum::Optional::Cinder::Volume": "OS::Heat::None" + "Magnum::Optional::Cinder::VolumeAttachment": "OS::Heat::None" diff --git a/magnum/drivers/common/templates/environments/with_volume.yaml b/magnum/drivers/common/templates/environments/with_volume.yaml new file mode 100644 index 0000000000..e67f289446 --- /dev/null +++ b/magnum/drivers/common/templates/environments/with_volume.yaml @@ -0,0 +1,4 @@ +# Environment file to use a cinder volume to store containers +resource_registry: + "Magnum::Optional::Cinder::Volume": "OS::Cinder::Volume" + "Magnum::Optional::Cinder::VolumeAttachment": "OS::Cinder::VolumeAttachment" diff --git a/magnum/drivers/common/templates/fragments/configure-docker-storage.sh b/magnum/drivers/common/templates/fragments/configure-docker-storage.sh index e52dbc5058..104c3ac5eb 100644 --- a/magnum/drivers/common/templates/fragments/configure-docker-storage.sh +++ b/magnum/drivers/common/templates/fragments/configure-docker-storage.sh @@ -2,30 +2,32 @@ . /etc/sysconfig/heat-params -if [ "$ENABLE_CINDER" == "False" ]; then - # FIXME(yuanying): Use ephemeral disk for docker storage - # Currently Ironic doesn't support cinder volumes, - # so we must use preserved ephemeral disk instead of a cinder volume. - device_path=$(readlink -f /dev/disk/by-label/ephemeral0) -else - attempts=60 - while [ ${attempts} -gt 0 ]; do - device_name=$(ls /dev/disk/by-id | grep ${DOCKER_VOLUME:0:20}$) - if [ -n "${device_name}" ]; then - break - fi - echo "waiting for disk device" - sleep 0.5 - udevadm trigger - let attempts-- - done +if [ -n "$DOCKER_VOLUME_SIZE" ] && [ "$DOCKER_VOLUME_SIZE" -gt 0 ]; then + if [ "$ENABLE_CINDER" == "False" ]; then + # FIXME(yuanying): Use ephemeral disk for docker storage + # Currently Ironic doesn't support cinder volumes, + # so we must use preserved ephemeral disk instead of a cinder volume. + device_path=$(readlink -f /dev/disk/by-label/ephemeral0) + else + attempts=60 + while [ ${attempts} -gt 0 ]; do + device_name=$(ls /dev/disk/by-id | grep ${DOCKER_VOLUME:0:20}$) + if [ -n "${device_name}" ]; then + break + fi + echo "waiting for disk device" + sleep 0.5 + udevadm trigger + let attempts-- + done - if [ -z "${device_name}" ]; then - echo "ERROR: disk device does not exist" >&2 - exit 1 + if [ -z "${device_name}" ]; then + echo "ERROR: disk device does not exist" >&2 + exit 1 + fi + + device_path=/dev/disk/by-id/${device_name} fi - - device_path=/dev/disk/by-id/${device_name} fi $configure_docker_storage_driver diff --git a/magnum/drivers/common/templates/fragments/configure_docker_storage_driver_atomic.sh b/magnum/drivers/common/templates/fragments/configure_docker_storage_driver_atomic.sh index 9a9eb80e6c..a28dc8f73e 100644 --- a/magnum/drivers/common/templates/fragments/configure_docker_storage_driver_atomic.sh +++ b/magnum/drivers/common/templates/fragments/configure_docker_storage_driver_atomic.sh @@ -15,9 +15,11 @@ configure_overlay () { rm -rf /var/lib/docker/* - mkfs.xfs -f ${device_path} - echo "${device_path} /var/lib/docker xfs defaults 0 0" >> /etc/fstab - mount -a + if [ -n "$DOCKER_VOLUME_SIZE" ] && [ "$DOCKER_VOLUME_SIZE" -gt 0 ]; then + mkfs.xfs -f ${device_path} + echo "${device_path} /var/lib/docker xfs defaults 0 0" >> /etc/fstab + mount -a + fi echo "STORAGE_DRIVER=overlay" > /etc/sysconfig/docker-storage-setup @@ -31,8 +33,10 @@ configure_overlay () { configure_devicemapper () { clear_docker_storage_congiguration - pvcreate -f ${device_path} - vgcreate docker ${device_path} + if [ -n "$DOCKER_VOLUME_SIZE" ] && [ "$DOCKER_VOLUME_SIZE" -gt 0 ]; then + pvcreate -f ${device_path} + vgcreate docker ${device_path} - echo "VG=docker" > /etc/sysconfig/docker-storage-setup + echo "VG=docker" > /etc/sysconfig/docker-storage-setup + fi } diff --git a/magnum/drivers/common/templates/kubernetes/fragments/write-heat-params-master.yaml b/magnum/drivers/common/templates/kubernetes/fragments/write-heat-params-master.yaml index e0d555a198..4651ab3db3 100644 --- a/magnum/drivers/common/templates/kubernetes/fragments/write-heat-params-master.yaml +++ b/magnum/drivers/common/templates/kubernetes/fragments/write-heat-params-master.yaml @@ -13,6 +13,7 @@ write_files: KUBE_ALLOW_PRIV="$KUBE_ALLOW_PRIV" ENABLE_CINDER="$ENABLE_CINDER" DOCKER_VOLUME="$DOCKER_VOLUME" + DOCKER_VOLUME_SIZE="$DOCKER_VOLUME_SIZE" DOCKER_STORAGE_DRIVER="$DOCKER_STORAGE_DRIVER" NETWORK_DRIVER="$NETWORK_DRIVER" FLANNEL_NETWORK_CIDR="$FLANNEL_NETWORK_CIDR" diff --git a/magnum/drivers/common/templates/kubernetes/fragments/write-heat-params.yaml b/magnum/drivers/common/templates/kubernetes/fragments/write-heat-params.yaml index 3801e9a190..d455a23e4b 100644 --- a/magnum/drivers/common/templates/kubernetes/fragments/write-heat-params.yaml +++ b/magnum/drivers/common/templates/kubernetes/fragments/write-heat-params.yaml @@ -13,6 +13,7 @@ write_files: ETCD_SERVER_IP="$ETCD_SERVER_IP" ENABLE_CINDER="$ENABLE_CINDER" DOCKER_VOLUME="$DOCKER_VOLUME" + DOCKER_VOLUME_SIZE="$DOCKER_VOLUME_SIZE" DOCKER_STORAGE_DRIVER="$DOCKER_STORAGE_DRIVER" NETWORK_DRIVER="$NETWORK_DRIVER" REGISTRY_ENABLED="$REGISTRY_ENABLED" diff --git a/magnum/drivers/common/templates/swarm/fragments/write-heat-params-master.yaml b/magnum/drivers/common/templates/swarm/fragments/write-heat-params-master.yaml index 1f8d5232b6..0a3504cbef 100644 --- a/magnum/drivers/common/templates/swarm/fragments/write-heat-params-master.yaml +++ b/magnum/drivers/common/templates/swarm/fragments/write-heat-params-master.yaml @@ -10,6 +10,7 @@ write_files: WAIT_CURL="$WAIT_CURL" ETCD_DISCOVERY_URL="$ETCD_DISCOVERY_URL" DOCKER_VOLUME="$DOCKER_VOLUME" + DOCKER_VOLUME_SIZE="$DOCKER_VOLUME_SIZE" DOCKER_STORAGE_DRIVER="$DOCKER_STORAGE_DRIVER" HTTP_PROXY="$HTTP_PROXY" HTTPS_PROXY="$HTTPS_PROXY" diff --git a/magnum/drivers/common/templates/swarm/fragments/write-heat-params-node.yaml b/magnum/drivers/common/templates/swarm/fragments/write-heat-params-node.yaml index e870b96443..d201654905 100644 --- a/magnum/drivers/common/templates/swarm/fragments/write-heat-params-node.yaml +++ b/magnum/drivers/common/templates/swarm/fragments/write-heat-params-node.yaml @@ -9,6 +9,7 @@ write_files: WAIT_HANDLE_TOKEN="$WAIT_HANDLE_TOKEN" WAIT_CURL="$WAIT_CURL" DOCKER_VOLUME="$DOCKER_VOLUME" + DOCKER_VOLUME_SIZE="$DOCKER_VOLUME_SIZE" DOCKER_STORAGE_DRIVER="$DOCKER_STORAGE_DRIVER" HTTP_PROXY="$HTTP_PROXY" HTTPS_PROXY="$HTTPS_PROXY" diff --git a/magnum/drivers/k8s_fedora_atomic_v1/templates/kubecluster.yaml b/magnum/drivers/k8s_fedora_atomic_v1/templates/kubecluster.yaml index b303f88612..641af0f7f1 100644 --- a/magnum/drivers/k8s_fedora_atomic_v1/templates/kubecluster.yaml +++ b/magnum/drivers/k8s_fedora_atomic_v1/templates/kubecluster.yaml @@ -92,7 +92,7 @@ parameters: description: > size of a cinder volume to allocate to docker for container/image storage - default: 25 + default: 0 docker_storage_driver: type: string diff --git a/magnum/drivers/k8s_fedora_atomic_v1/templates/kubemaster.yaml b/magnum/drivers/k8s_fedora_atomic_v1/templates/kubemaster.yaml index 74131fb4c4..66d779a0f6 100644 --- a/magnum/drivers/k8s_fedora_atomic_v1/templates/kubemaster.yaml +++ b/magnum/drivers/k8s_fedora_atomic_v1/templates/kubemaster.yaml @@ -230,6 +230,7 @@ resources: "$KUBE_NODE_IP": {get_attr: [kube_master_eth0, fixed_ips, 0, ip_address]} "$KUBE_ALLOW_PRIV": {get_param: kube_allow_priv} "$DOCKER_VOLUME": {get_resource: docker_volume} + "$DOCKER_VOLUME_SIZE": {get_param: docker_volume_size} "$DOCKER_STORAGE_DRIVER": {get_param: docker_storage_driver} "$NETWORK_DRIVER": {get_param: network_driver} "$FLANNEL_NETWORK_CIDR": {get_param: flannel_network_cidr} @@ -442,12 +443,12 @@ resources: # docker_volume: - type: OS::Cinder::Volume + type: Magnum::Optional::Cinder::Volume properties: size: {get_param: docker_volume_size} docker_volume_attach: - type: OS::Cinder::VolumeAttachment + type: Magnum::Optional::Cinder::VolumeAttachment properties: instance_uuid: {get_resource: kube_master} volume_id: {get_resource: docker_volume} diff --git a/magnum/drivers/k8s_fedora_atomic_v1/templates/kubeminion.yaml b/magnum/drivers/k8s_fedora_atomic_v1/templates/kubeminion.yaml index 66e7631976..a2f8963446 100644 --- a/magnum/drivers/k8s_fedora_atomic_v1/templates/kubeminion.yaml +++ b/magnum/drivers/k8s_fedora_atomic_v1/templates/kubeminion.yaml @@ -227,6 +227,7 @@ resources: $KUBE_NODE_IP: {get_attr: [kube_minion_eth0, fixed_ips, 0, ip_address]} $ETCD_SERVER_IP: {get_param: etcd_server_ip} $DOCKER_VOLUME: {get_resource: docker_volume} + $DOCKER_VOLUME_SIZE: {get_param: docker_volume_size} $DOCKER_STORAGE_DRIVER: {get_param: docker_storage_driver} $NETWORK_DRIVER: {get_param: network_driver} $REGISTRY_ENABLED: {get_param: registry_enabled} @@ -410,12 +411,12 @@ resources: # docker_volume: - type: OS::Cinder::Volume + type: Magnum::Optional::Cinder::Volume properties: size: {get_param: docker_volume_size} docker_volume_attach: - type: OS::Cinder::VolumeAttachment + type: Magnum::Optional::Cinder::VolumeAttachment properties: instance_uuid: {get_resource: kube-minion} volume_id: {get_resource: docker_volume} diff --git a/magnum/drivers/k8s_fedora_ironic_v1/templates/kubecluster.yaml b/magnum/drivers/k8s_fedora_ironic_v1/templates/kubecluster.yaml index 57278fbf8d..baec083fcd 100644 --- a/magnum/drivers/k8s_fedora_ironic_v1/templates/kubecluster.yaml +++ b/magnum/drivers/k8s_fedora_ironic_v1/templates/kubecluster.yaml @@ -100,7 +100,7 @@ parameters: description: > size of a cinder volume to allocate to docker for container/image storage - default: 25 + default: 0 docker_storage_driver: type: string @@ -430,6 +430,7 @@ resources: master_flavor: {get_param: master_flavor} external_network: {get_param: external_network} kube_allow_priv: {get_param: kube_allow_priv} + docker_volume_size: {get_param: docker_volume_size} docker_storage_driver: {get_param: docker_storage_driver} wait_condition_timeout: {get_param: wait_condition_timeout} network_driver: {get_param: network_driver} @@ -486,6 +487,7 @@ resources: etcd_server_ip: {get_attr: [etcd_address_switch, private_ip]} external_network: {get_param: external_network} kube_allow_priv: {get_param: kube_allow_priv} + docker_volume_size: {get_param: docker_volume_size} docker_storage_driver: {get_param: docker_storage_driver} wait_condition_timeout: {get_param: wait_condition_timeout} registry_enabled: {get_param: registry_enabled} diff --git a/magnum/drivers/k8s_fedora_ironic_v1/templates/kubemaster.yaml b/magnum/drivers/k8s_fedora_ironic_v1/templates/kubemaster.yaml index 67597f8e96..27f4fe2e55 100644 --- a/magnum/drivers/k8s_fedora_ironic_v1/templates/kubemaster.yaml +++ b/magnum/drivers/k8s_fedora_ironic_v1/templates/kubemaster.yaml @@ -35,6 +35,12 @@ parameters: constraints: - allowed_values: ["true", "false"] + docker_volume_size: + type: number + description: > + size of a cinder volume to allocate to docker for container/image + storage + docker_storage_driver: type: string description: docker storage driver name @@ -222,6 +228,7 @@ resources: "$KUBE_API_PORT": {get_param: kubernetes_port} "$KUBE_ALLOW_PRIV": {get_param: kube_allow_priv} "$DOCKER_VOLUME": 'None' + "$DOCKER_VOLUME_SIZE": {get_param: docker_volume_size} "$DOCKER_STORAGE_DRIVER": {get_param: docker_storage_driver} "$NETWORK_DRIVER": {get_param: network_driver} "$FLANNEL_NETWORK_CIDR": {get_param: flannel_network_cidr} diff --git a/magnum/drivers/k8s_fedora_ironic_v1/templates/kubeminion.yaml b/magnum/drivers/k8s_fedora_ironic_v1/templates/kubeminion.yaml index 1dc74598c5..38d67667d6 100644 --- a/magnum/drivers/k8s_fedora_ironic_v1/templates/kubeminion.yaml +++ b/magnum/drivers/k8s_fedora_ironic_v1/templates/kubeminion.yaml @@ -30,6 +30,12 @@ parameters: constraints: - allowed_values: ["true", "false"] + docker_volume_size: + type: number + description: > + size of a cinder volume to allocate to docker for container/image + storage + docker_storage_driver: type: string description: docker storage driver name @@ -219,6 +225,7 @@ resources: $KUBE_API_PORT: {get_param: kubernetes_port} $ETCD_SERVER_IP: {get_param: etcd_server_ip} $DOCKER_VOLUME: 'None' + $DOCKER_VOLUME_SIZE: {get_param: docker_volume_size} $DOCKER_STORAGE_DRIVER: {get_param: docker_storage_driver} $NETWORK_DRIVER: {get_param: network_driver} $REGISTRY_ENABLED: {get_param: registry_enabled} diff --git a/magnum/drivers/swarm_fedora_atomic_v1/templates/cluster.yaml b/magnum/drivers/swarm_fedora_atomic_v1/templates/cluster.yaml index 63885dffec..871fd0b500 100644 --- a/magnum/drivers/swarm_fedora_atomic_v1/templates/cluster.yaml +++ b/magnum/drivers/swarm_fedora_atomic_v1/templates/cluster.yaml @@ -118,7 +118,7 @@ parameters: description: > size of a cinder volume to allocate to docker for container/image storage - default: 25 + default: 0 docker_storage_driver: type: string diff --git a/magnum/drivers/swarm_fedora_atomic_v1/templates/swarmmaster.yaml b/magnum/drivers/swarm_fedora_atomic_v1/templates/swarmmaster.yaml index 72d2900a6a..dcffa9d720 100644 --- a/magnum/drivers/swarm_fedora_atomic_v1/templates/swarmmaster.yaml +++ b/magnum/drivers/swarm_fedora_atomic_v1/templates/swarmmaster.yaml @@ -211,6 +211,7 @@ resources: "$WAIT_HANDLE_TOKEN": {get_attr: [master_wait_handle, token]} "$WAIT_CURL": {get_attr: [master_wait_handle, curl_cli]} "$DOCKER_VOLUME": {get_resource: docker_volume} + "$DOCKER_VOLUME_SIZE": {get_param: docker_volume_size} "$DOCKER_STORAGE_DRIVER": {get_param: docker_storage_driver} "$ETCD_DISCOVERY_URL": {get_param: discovery_url} "$HTTP_PROXY": {get_param: http_proxy} @@ -445,12 +446,12 @@ resources: # docker_volume: - type: OS::Cinder::Volume + type: Magnum::Optional::Cinder::Volume properties: size: {get_param: docker_volume_size} docker_volume_attach: - type: OS::Cinder::VolumeAttachment + type: Magnum::Optional::Cinder::VolumeAttachment properties: instance_uuid: {get_resource: swarm_master} volume_id: {get_resource: docker_volume} diff --git a/magnum/drivers/swarm_fedora_atomic_v1/templates/swarmnode.yaml b/magnum/drivers/swarm_fedora_atomic_v1/templates/swarmnode.yaml index 12bf495cdd..c60be2dbd9 100644 --- a/magnum/drivers/swarm_fedora_atomic_v1/templates/swarmnode.yaml +++ b/magnum/drivers/swarm_fedora_atomic_v1/templates/swarmnode.yaml @@ -189,6 +189,7 @@ resources: "$WAIT_HANDLE_TOKEN": {get_attr: [node_wait_handle, token]} "$WAIT_CURL": {get_attr: [node_wait_handle, curl_cli]} "$DOCKER_VOLUME": {get_resource: docker_volume} + "$DOCKER_VOLUME_SIZE": {get_param: docker_volume_size} "$DOCKER_STORAGE_DRIVER": {get_param: docker_storage_driver} "$HTTP_PROXY": {get_param: http_proxy} "$HTTPS_PROXY": {get_param: https_proxy} @@ -385,12 +386,12 @@ resources: # docker_volume: - type: OS::Cinder::Volume + type: Magnum::Optional::Cinder::Volume properties: size: {get_param: docker_volume_size} docker_volume_attach: - type: OS::Cinder::VolumeAttachment + type: Magnum::Optional::Cinder::VolumeAttachment properties: instance_uuid: {get_resource: swarm_node} volume_id: {get_resource: docker_volume} diff --git a/magnum/tests/unit/api/controllers/v1/test_baymodel.py b/magnum/tests/unit/api/controllers/v1/test_baymodel.py index 2f93bd62e5..20324d02d1 100644 --- a/magnum/tests/unit/api/controllers/v1/test_baymodel.py +++ b/magnum/tests/unit/api/controllers/v1/test_baymodel.py @@ -545,7 +545,6 @@ class TestPost(api_base.FunctionalTest): self._create_baymodel_raises_app_error(coe='osomatsu') def test_create_baymodel_with_invalid_docker_volume_size(self): - self._create_baymodel_raises_app_error(docker_volume_size=0) self._create_baymodel_raises_app_error(docker_volume_size=-1) self._create_baymodel_raises_app_error( docker_volume_size=1, diff --git a/magnum/tests/unit/api/controllers/v1/test_cluster_template.py b/magnum/tests/unit/api/controllers/v1/test_cluster_template.py index 194fddeb7f..1a3c6e9961 100644 --- a/magnum/tests/unit/api/controllers/v1/test_cluster_template.py +++ b/magnum/tests/unit/api/controllers/v1/test_cluster_template.py @@ -604,7 +604,6 @@ class TestPost(api_base.FunctionalTest): self._create_model_raises_app_error(coe='osomatsu') def test_create_cluster_template_with_invalid_docker_volume_size(self): - self._create_model_raises_app_error(docker_volume_size=0) self._create_model_raises_app_error(docker_volume_size=-1) self._create_model_raises_app_error( docker_volume_size=1, diff --git a/magnum/tests/unit/conductor/handlers/test_k8s_cluster_conductor.py b/magnum/tests/unit/conductor/handlers/test_k8s_cluster_conductor.py index 21015e2599..72d443996a 100644 --- a/magnum/tests/unit/conductor/handlers/test_k8s_cluster_conductor.py +++ b/magnum/tests/unit/conductor/handlers/test_k8s_cluster_conductor.py @@ -192,7 +192,8 @@ class TestClusterConductorWithK8s(base.TestCase): self.assertEqual(expected, definition) self.assertEqual( - ['../../common/templates/environments/no_master_lb.yaml', + ['../../common/templates/environments/with_volume.yaml', + '../../common/templates/environments/no_master_lb.yaml', '../../common/templates/environments/disable_floating_ip.yaml'], env_files) @@ -267,7 +268,75 @@ class TestClusterConductorWithK8s(base.TestCase): self.assertEqual(expected, definition) self.assertEqual( - ['../../common/templates/environments/no_master_lb.yaml', + ['../../common/templates/environments/with_volume.yaml', + '../../common/templates/environments/no_master_lb.yaml', + '../../common/templates/environments/disable_floating_ip.yaml'], + env_files) + + @patch('requests.get') + @patch('magnum.objects.ClusterTemplate.get_by_uuid') + @patch('magnum.drivers.common.driver.Driver.get_driver') + def test_extract_template_definition_only_required( + self, + mock_driver, + mock_objects_cluster_template_get_by_uuid, + mock_get): + + not_required = ['image_id', 'flavor_id', 'dns_nameserver', + 'docker_volume_size', 'fixed_network', 'http_proxy', + 'https_proxy', 'no_proxy', 'network_driver', + 'master_flavor_id', 'docker_storage_driver', + 'volume_driver'] + for key in not_required: + self.cluster_template_dict[key] = None + self.cluster_dict['discovery_url'] = 'https://discovery.etcd.io/test' + + cluster_template = objects.ClusterTemplate( + self.context, **self.cluster_template_dict) + mock_objects_cluster_template_get_by_uuid.return_value = \ + cluster_template + expected_result = str('{"action":"get","node":{"key":"test","value":' + '"1","modifiedIndex":10,"createdIndex":10}}') + mock_resp = mock.MagicMock() + mock_resp.text = expected_result + mock_get.return_value = mock_resp + mock_driver.return_value = k8s_dr.Driver() + cluster = objects.Cluster(self.context, **self.cluster_dict) + + (template_path, + definition, + env_files) = driver._extract_template_definition(self.context, + cluster) + + expected = { + 'auth_url': 'http://192.168.10.10:5000/v3', + 'cluster_uuid': '5d12f6fd-a196-4bf0-ae4c-1f639a523a52', + 'discovery_url': 'https://discovery.etcd.io/test', + 'external_network': 'external_network_id', + 'flannel_backend': 'vxlan', + 'flannel_network_cidr': '10.101.0.0/16', + 'flannel_network_subnetlen': '26', + 'insecure_registry_url': '10.0.0.1:5000', + 'kube_version': 'fake-version', + 'magnum_url': 'http://127.0.0.1:9511/v1', + 'number_of_masters': 1, + 'number_of_minions': 1, + 'region_name': 'RegionOne', + 'registry_enabled': False, + 'ssh_key_name': 'keypair_id', + 'tenant_name': 'fake_tenant', + 'tls_disabled': False, + 'trust_id': 'bd11efc5-d4e2-4dac-bbce-25e348ddf7de', + 'trustee_domain_id': 'trustee_domain_id', + 'trustee_password': 'fake_trustee_password', + 'trustee_user_id': '7b489f04-b458-4541-8179-6a48a553e656', + 'trustee_username': 'fake_trustee', + 'username': 'fake_user' + } + self.assertEqual(expected, definition) + self.assertEqual( + ['../../common/templates/environments/no_volume.yaml', + '../../common/templates/environments/no_master_lb.yaml', '../../common/templates/environments/disable_floating_ip.yaml'], env_files) @@ -438,21 +507,6 @@ class TestClusterConductorWithK8s(base.TestCase): mock_get, missing_attr='flavor_id') - @patch('requests.get') - @patch('magnum.objects.ClusterTemplate.get_by_uuid') - @patch('magnum.drivers.common.driver.Driver.get_driver') - def test_extract_template_definition_without_docker_volume_size( - self, - mock_driver, - mock_objects_cluster_template_get_by_uuid, - mock_get): - mock_driver.return_value = k8s_dr.Driver() - self._test_extract_template_definition( - mock_driver, - mock_objects_cluster_template_get_by_uuid, - mock_get, - missing_attr='docker_volume_size') - @patch('requests.get') @patch('magnum.objects.ClusterTemplate.get_by_uuid') @patch('magnum.drivers.common.driver.Driver.get_driver') @@ -594,7 +648,8 @@ class TestClusterConductorWithK8s(base.TestCase): } self.assertEqual(expected, definition) self.assertEqual( - ['../../common/templates/environments/no_master_lb.yaml', + ['../../common/templates/environments/with_volume.yaml', + '../../common/templates/environments/no_master_lb.yaml', '../../common/templates/environments/disable_floating_ip.yaml'], env_files) reqget.assert_called_once_with('http://etcd/test?size=1') diff --git a/magnum/tests/unit/conductor/handlers/test_swarm_cluster_conductor.py b/magnum/tests/unit/conductor/handlers/test_swarm_cluster_conductor.py index c20868d379..2bb8c597f6 100644 --- a/magnum/tests/unit/conductor/handlers/test_swarm_cluster_conductor.py +++ b/magnum/tests/unit/conductor/handlers/test_swarm_cluster_conductor.py @@ -146,7 +146,8 @@ class TestClusterConductorWithSwarm(base.TestCase): } self.assertEqual(expected, definition) self.assertEqual( - ['../../common/templates/environments/no_master_lb.yaml'], + ['../../common/templates/environments/with_volume.yaml', + '../../common/templates/environments/no_master_lb.yaml'], env_files) @patch('requests.get') @@ -217,7 +218,8 @@ class TestClusterConductorWithSwarm(base.TestCase): } self.assertEqual(expected, definition) self.assertEqual( - ['../../common/templates/environments/no_master_lb.yaml'], + ['../../common/templates/environments/with_volume.yaml', + '../../common/templates/environments/no_master_lb.yaml'], env_files) @patch('requests.get') @@ -280,7 +282,8 @@ class TestClusterConductorWithSwarm(base.TestCase): } self.assertEqual(expected, definition) self.assertEqual( - ['../../common/templates/environments/no_master_lb.yaml'], + ['../../common/templates/environments/no_volume.yaml', + '../../common/templates/environments/no_master_lb.yaml'], env_files) @patch('requests.get') @@ -345,7 +348,8 @@ class TestClusterConductorWithSwarm(base.TestCase): } self.assertEqual(expected, definition) self.assertEqual( - ['../../common/templates/environments/with_master_lb.yaml'], + ['../../common/templates/environments/with_volume.yaml', + '../../common/templates/environments/with_master_lb.yaml'], env_files) @patch('requests.get') @@ -411,7 +415,8 @@ class TestClusterConductorWithSwarm(base.TestCase): } self.assertEqual(expected, definition) self.assertEqual( - ['../../common/templates/environments/with_master_lb.yaml'], + ['../../common/templates/environments/with_volume.yaml', + '../../common/templates/environments/with_master_lb.yaml'], env_files) @patch('magnum.conductor.utils.retrieve_cluster_template') diff --git a/releasenotes/notes/no-cinder-volume-87b9339e066c30a0.yaml b/releasenotes/notes/no-cinder-volume-87b9339e066c30a0.yaml new file mode 100644 index 0000000000..06300fdabe --- /dev/null +++ b/releasenotes/notes/no-cinder-volume-87b9339e066c30a0.yaml @@ -0,0 +1,10 @@ +--- +prelude: > + Currently, the swarm and the kubernetes drivers use + a dedicated cinder volume to store the container + images. It was been observed that one cinder volume + per node is a bottleneck for large clusters. +fixes: + - Make the dedicated cinder volume per node an opt-in + option. By default, no cinder volumes will be created + unless the user passes the docker-volume-size argument.