diff --git a/doc/source/userguide.rst b/doc/source/userguide.rst index dbd7d6fbf0..092785cc63 100644 --- a/doc/source/userguide.rst +++ b/doc/source/userguide.rst @@ -337,6 +337,9 @@ the table are linked to more details elsewhere in the user guide. +---------------------------------------+--------------------+---------------+ | `docker_volume_type`_ | see below | see below | +---------------------------------------+--------------------+---------------+ +| `etcd_volume_size`_ | etcd storage | 0 | +| | volume size | | ++---------------------------------------+--------------------+---------------+ ======= Cluster @@ -1060,6 +1063,10 @@ _`admission_control_list` The default value corresponds to the one recommended in this doc for our current Kubernetes version. +_`etcd_volume_size` + This label sets the size of a volume holding the etcd storage data. + The default value is 0, meaning the etcd data is not persisted (no volume). + External load balancer for services ----------------------------------- diff --git a/magnum/drivers/common/templates/environments/no_etcd_volume.yaml b/magnum/drivers/common/templates/environments/no_etcd_volume.yaml new file mode 100644 index 0000000000..edd710ed7e --- /dev/null +++ b/magnum/drivers/common/templates/environments/no_etcd_volume.yaml @@ -0,0 +1,4 @@ +# Environment file to not use a cinder volume for etcd storage +resource_registry: + "Magnum::Optional::Etcd::Volume": "OS::Heat::None" + "Magnum::Optional::Etcd::VolumeAttachment": "OS::Heat::None" diff --git a/magnum/drivers/common/templates/environments/with_etcd_volume.yaml b/magnum/drivers/common/templates/environments/with_etcd_volume.yaml new file mode 100644 index 0000000000..92027c491c --- /dev/null +++ b/magnum/drivers/common/templates/environments/with_etcd_volume.yaml @@ -0,0 +1,4 @@ +# Environment file to use a volume for etcd storage +resource_registry: + "Magnum::Optional::Etcd::Volume": "OS::Cinder::Volume" + "Magnum::Optional::Etcd::VolumeAttachment": "OS::Cinder::VolumeAttachment" diff --git a/magnum/drivers/common/templates/kubernetes/fragments/configure-etcd.sh b/magnum/drivers/common/templates/kubernetes/fragments/configure-etcd.sh index 8ce0ba5797..85422ac76e 100644 --- a/magnum/drivers/common/templates/kubernetes/fragments/configure-etcd.sh +++ b/magnum/drivers/common/templates/kubernetes/fragments/configure-etcd.sh @@ -2,6 +2,38 @@ . /etc/sysconfig/heat-params +if [ -n "$ETCD_VOLUME_SIZE" ] && [ "$ETCD_VOLUME_SIZE" -gt 0 ]; then + + attempts=60 + while [ ${attempts} -gt 0 ]; do + device_name=$(ls /dev/disk/by-id | grep ${ETCD_VOLUME:0:20}$) + if [ -n "${device_name}" ]; then + break + fi + echo "waiting for disk device" + sleep 0.5 + udevadm trigger + let attempts-- + done + + if [ -z "${device_name}" ]; then + echo "ERROR: disk device does not exist" >&2 + exit 1 + fi + + device_path=/dev/disk/by-id/${device_name} + fstype=$(blkid -s TYPE -o value ${device_path}) + if [ "${fstype}" != "xfs" ]; then + mkfs.xfs -f ${device_path} + fi + mkdir -p /var/lib/etcd + echo "${device_path} /var/lib/etcd xfs defaults 0 0" >> /etc/fstab + mount -a + chown -R etcd.etcd /var/lib/etcd + chmod 755 /var/lib/etcd + +fi + if [ -z "$KUBE_NODE_IP" ]; then # FIXME(yuanying): Set KUBE_NODE_IP correctly KUBE_NODE_IP=$(curl -s http://169.254.169.254/latest/meta-data/local-ipv4) diff --git a/magnum/drivers/common/templates/kubernetes/fragments/write-heat-params-master.yaml b/magnum/drivers/common/templates/kubernetes/fragments/write-heat-params-master.yaml index 77b006ef4e..c6a0049e06 100644 --- a/magnum/drivers/common/templates/kubernetes/fragments/write-heat-params-master.yaml +++ b/magnum/drivers/common/templates/kubernetes/fragments/write-heat-params-master.yaml @@ -13,6 +13,8 @@ write_files: KUBE_NODE_IP="$KUBE_NODE_IP" KUBE_ALLOW_PRIV="$KUBE_ALLOW_PRIV" ENABLE_CINDER="$ENABLE_CINDER" + ETCD_VOLUME="$ETCD_VOLUME" + ETCD_VOLUME_SIZE="$ETCD_VOLUME_SIZE" DOCKER_VOLUME="$DOCKER_VOLUME" DOCKER_VOLUME_SIZE="$DOCKER_VOLUME_SIZE" DOCKER_STORAGE_DRIVER="$DOCKER_STORAGE_DRIVER" diff --git a/magnum/drivers/heat/k8s_fedora_template_def.py b/magnum/drivers/heat/k8s_fedora_template_def.py index 437a429832..f580bc9449 100644 --- a/magnum/drivers/heat/k8s_fedora_template_def.py +++ b/magnum/drivers/heat/k8s_fedora_template_def.py @@ -88,6 +88,7 @@ class K8sFedoraTemplateDefinition(k8s_template_def.K8sTemplateDefinition): env_files = [] template_def.add_priv_net_env_file(env_files, cluster_template) + template_def.add_etcd_volume_env_file(env_files, cluster_template) template_def.add_volume_env_file(env_files, cluster) template_def.add_lb_env_file(env_files, cluster_template) template_def.add_fip_env_file(env_files, cluster_template) diff --git a/magnum/drivers/heat/k8s_template_def.py b/magnum/drivers/heat/k8s_template_def.py index 1ed73114f8..bff1644c92 100644 --- a/magnum/drivers/heat/k8s_template_def.py +++ b/magnum/drivers/heat/k8s_template_def.py @@ -112,7 +112,8 @@ class K8sTemplateDefinition(template_def.BaseTemplateDefinition): 'admission_control_list', 'prometheus_monitoring', 'grafana_admin_passwd', - 'kube_dashboard_enabled'] + 'kube_dashboard_enabled', + 'etcd_volume_size'] for label in label_list: extra_params[label] = cluster_template.labels.get(label) diff --git a/magnum/drivers/heat/template_def.py b/magnum/drivers/heat/template_def.py index 7000d30c83..6eaf36b50d 100644 --- a/magnum/drivers/heat/template_def.py +++ b/magnum/drivers/heat/template_def.py @@ -337,6 +337,13 @@ def add_volume_env_file(env_files, cluster): env_files.append(COMMON_ENV_PATH + 'with_volume.yaml') +def add_etcd_volume_env_file(env_files, cluster_template): + if int(cluster_template.labels.get('etcd_volume_size', 0)) < 1: + env_files.append(COMMON_ENV_PATH + 'no_etcd_volume.yaml') + else: + env_files.append(COMMON_ENV_PATH + 'with_etcd_volume.yaml') + + def add_fip_env_file(env_files, cluster_template): if cluster_template.floating_ip_enabled: env_files.append(COMMON_ENV_PATH + 'enable_floating_ip.yaml') diff --git a/magnum/drivers/k8s_fedora_atomic_v1/templates/kubecluster.yaml b/magnum/drivers/k8s_fedora_atomic_v1/templates/kubecluster.yaml index 8d942291b0..2e988b28c2 100644 --- a/magnum/drivers/k8s_fedora_atomic_v1/templates/kubecluster.yaml +++ b/magnum/drivers/k8s_fedora_atomic_v1/templates/kubecluster.yaml @@ -130,6 +130,12 @@ parameters: constraints: - allowed_values: ["true", "false"] + etcd_volume_size: + type: number + description: > + size of the cinder volume for etcd storage + default: 0 + docker_volume_size: type: number description: > @@ -467,6 +473,7 @@ resources: master_flavor: {get_param: master_flavor} external_network: {get_param: external_network} kube_allow_priv: {get_param: kube_allow_priv} + etcd_volume_size: {get_param: etcd_volume_size} docker_volume_size: {get_param: docker_volume_size} docker_volume_type: {get_param: docker_volume_type} docker_storage_driver: {get_param: docker_storage_driver} diff --git a/magnum/drivers/k8s_fedora_atomic_v1/templates/kubemaster.yaml b/magnum/drivers/k8s_fedora_atomic_v1/templates/kubemaster.yaml index 57783c4ef1..32a911a29f 100644 --- a/magnum/drivers/k8s_fedora_atomic_v1/templates/kubemaster.yaml +++ b/magnum/drivers/k8s_fedora_atomic_v1/templates/kubemaster.yaml @@ -35,6 +35,11 @@ parameters: constraints: - allowed_values: ["true", "false"] + etcd_volume_size: + type: number + description: > + size of a cinder volume to allocate for etcd storage + docker_volume_size: type: number description: > @@ -290,6 +295,8 @@ resources: "$KUBE_NODE_PUBLIC_IP": {get_attr: [kube_master_floating, floating_ip_address]} "$KUBE_NODE_IP": {get_attr: [kube_master_eth0, fixed_ips, 0, ip_address]} "$KUBE_ALLOW_PRIV": {get_param: kube_allow_priv} + "$ETCD_VOLUME": {get_resource: etcd_volume} + "$ETCD_VOLUME_SIZE": {get_param: etcd_volume_size} "$DOCKER_VOLUME": {get_resource: docker_volume} "$DOCKER_VOLUME_SIZE": {get_param: docker_volume_size} "$DOCKER_STORAGE_DRIVER": {get_param: docker_storage_driver} @@ -545,6 +552,24 @@ resources: subnet: { get_param: fixed_subnet } protocol_port: 2379 + ###################################################################### + # + # etcd storage. This allocates a cinder volume and attaches it + # to the master. + # + + etcd_volume: + type: Magnum::Optional::Etcd::Volume + properties: + size: {get_param: etcd_volume_size} + + etcd_volume_attach: + type: Magnum::Optional::Etcd::VolumeAttachment + properties: + instance_uuid: {get_resource: kube-master} + volume_id: {get_resource: etcd_volume} + mountpoint: /dev/vdc + ###################################################################### # # docker storage. This allocates a cinder volume and attaches it diff --git a/magnum/drivers/k8s_fedora_ironic_v1/templates/kubecluster.yaml b/magnum/drivers/k8s_fedora_ironic_v1/templates/kubecluster.yaml index 14a7015fb9..29b75b875f 100644 --- a/magnum/drivers/k8s_fedora_ironic_v1/templates/kubecluster.yaml +++ b/magnum/drivers/k8s_fedora_ironic_v1/templates/kubecluster.yaml @@ -128,6 +128,12 @@ parameters: constraints: - allowed_values: ["true", "false"] + etcd_volume_size: + type: number + description: > + size of the cinder volume for etcd storage + default: 0 + docker_volume_size: type: number description: > diff --git a/magnum/tests/unit/conductor/handlers/test_k8s_cluster_conductor.py b/magnum/tests/unit/conductor/handlers/test_k8s_cluster_conductor.py index 90ada97b13..598f8127b4 100644 --- a/magnum/tests/unit/conductor/handlers/test_k8s_cluster_conductor.py +++ b/magnum/tests/unit/conductor/handlers/test_k8s_cluster_conductor.py @@ -55,7 +55,8 @@ class TestClusterConductorWithK8s(base.TestCase): 'prometheus_monitoring': 'False', 'grafana_admin_passwd': 'fake_pwd', 'kube_dashboard_enabled': 'True', - 'docker_volume_type': 'lvmdriver-1'}, + 'docker_volume_type': 'lvmdriver-1', + 'etcd_volume_size': '0'}, 'tls_disabled': False, 'server_type': 'vm', 'registry_enabled': False, @@ -158,7 +159,8 @@ class TestClusterConductorWithK8s(base.TestCase): 'prometheus_monitoring': 'False', 'grafana_admin_passwd': 'fake_pwd', 'kube_dashboard_enabled': 'True', - 'docker_volume_type': 'lvmdriver-1'}, + 'docker_volume_type': 'lvmdriver-1', + 'etcd_volume_size': '0'}, 'http_proxy': 'http_proxy', 'https_proxy': 'https_proxy', 'no_proxy': 'no_proxy', @@ -184,6 +186,7 @@ class TestClusterConductorWithK8s(base.TestCase): 'docker_volume_type': 'lvmdriver-1', 'docker_storage_driver': 'devicemapper', 'discovery_url': 'https://discovery.etcd.io/test', + 'etcd_volume_size': '0', 'flannel_network_cidr': '10.101.0.0/16', 'flannel_network_subnetlen': '26', 'flannel_backend': 'vxlan', @@ -218,6 +221,7 @@ class TestClusterConductorWithK8s(base.TestCase): self.assertEqual(expected, definition) self.assertEqual( ['../../common/templates/environments/no_private_network.yaml', + '../../common/templates/environments/no_etcd_volume.yaml', '../../common/templates/environments/with_volume.yaml', '../../common/templates/environments/no_master_lb.yaml', '../../common/templates/environments/disable_floating_ip.yaml', @@ -266,6 +270,7 @@ class TestClusterConductorWithK8s(base.TestCase): 'docker_storage_driver': 'devicemapper', 'docker_volume_size': 20, 'docker_volume_type': 'lvmdriver-1', + 'etcd_volume_size': '0', 'external_network': 'external_network_id', 'fixed_network': 'fixed_network', 'fixed_subnet': 'fixed_subnet', @@ -309,6 +314,7 @@ class TestClusterConductorWithK8s(base.TestCase): self.assertEqual(expected, definition) self.assertEqual( ['../../common/templates/environments/no_private_network.yaml', + '../../common/templates/environments/no_etcd_volume.yaml', '../../common/templates/environments/with_volume.yaml', '../../common/templates/environments/no_master_lb.yaml', '../../common/templates/environments/disable_floating_ip.yaml', @@ -366,6 +372,7 @@ class TestClusterConductorWithK8s(base.TestCase): 'grafana_admin_passwd': 'fake_pwd', 'kube_dashboard_enabled': 'True', 'docker_volume_type': 'lvmdriver-1', + 'etcd_volume_size': '0', 'insecure_registry_url': '10.0.0.1:5000', 'kube_version': 'fake-version', 'magnum_url': 'http://127.0.0.1:9511/v1', @@ -386,6 +393,7 @@ class TestClusterConductorWithK8s(base.TestCase): self.assertEqual(expected, definition) self.assertEqual( ['../../common/templates/environments/with_private_network.yaml', + '../../common/templates/environments/no_etcd_volume.yaml', '../../common/templates/environments/with_volume.yaml', '../../common/templates/environments/no_master_lb.yaml', '../../common/templates/environments/disable_floating_ip.yaml', @@ -432,6 +440,7 @@ class TestClusterConductorWithK8s(base.TestCase): 'network_driver': 'network_driver', 'volume_driver': 'volume_driver', 'discovery_url': 'https://discovery.etcd.io/test', + 'etcd_volume_size': '0', 'http_proxy': 'http_proxy', 'https_proxy': 'https_proxy', 'no_proxy': 'no_proxy', @@ -502,6 +511,7 @@ class TestClusterConductorWithK8s(base.TestCase): 'network_driver': 'network_driver', 'volume_driver': 'volume_driver', 'discovery_url': 'http://tokentest/h1/h2/h3', + 'etcd_volume_size': '0', 'http_proxy': 'http_proxy', 'https_proxy': 'https_proxy', 'no_proxy': 'no_proxy', @@ -699,6 +709,7 @@ class TestClusterConductorWithK8s(base.TestCase): 'docker_volume_type': 'lvmdriver-1', 'docker_storage_driver': 'devicemapper', 'discovery_url': 'https://address/token', + 'etcd_volume_size': '0', 'http_proxy': 'http_proxy', 'https_proxy': 'https_proxy', 'no_proxy': 'no_proxy', @@ -730,6 +741,7 @@ class TestClusterConductorWithK8s(base.TestCase): self.assertEqual(expected, definition) self.assertEqual( ['../../common/templates/environments/no_private_network.yaml', + '../../common/templates/environments/no_etcd_volume.yaml', '../../common/templates/environments/with_volume.yaml', '../../common/templates/environments/no_master_lb.yaml', '../../common/templates/environments/disable_floating_ip.yaml', diff --git a/magnum/tests/unit/drivers/test_template_definition.py b/magnum/tests/unit/drivers/test_template_definition.py index dd71f10546..890474db3c 100644 --- a/magnum/tests/unit/drivers/test_template_definition.py +++ b/magnum/tests/unit/drivers/test_template_definition.py @@ -268,6 +268,8 @@ class AtomicK8sTemplateDefinitionTestCase(BaseTemplateDefinitionTestCase): 'kube_dashboard_enabled') docker_volume_type = mock_cluster_template.labels.get( 'docker_volume_type') + etcd_volume_size = mock_cluster_template.labels.get( + 'etcd_volume_size') k8s_def = k8sa_tdef.AtomicK8sTemplateDefinition() @@ -287,6 +289,7 @@ class AtomicK8sTemplateDefinitionTestCase(BaseTemplateDefinitionTestCase): 'grafana_admin_passwd': grafana_admin_passwd, 'kube_dashboard_enabled': kube_dashboard_enabled, 'docker_volume_type': docker_volume_type, + 'etcd_volume_size': etcd_volume_size, 'username': 'fake_user', 'tenant_name': 'fake_tenant', 'magnum_url': mock_osc.magnum_url.return_value, @@ -345,6 +348,8 @@ class AtomicK8sTemplateDefinitionTestCase(BaseTemplateDefinitionTestCase): 'kube_dashboard_enabled') docker_volume_type = mock_cluster_template.labels.get( 'docker_volume_type') + etcd_volume_size = mock_cluster_template.labels.get( + 'etcd_volume_size') k8s_def = k8sa_tdef.AtomicK8sTemplateDefinition() @@ -364,6 +369,7 @@ class AtomicK8sTemplateDefinitionTestCase(BaseTemplateDefinitionTestCase): 'grafana_admin_passwd': grafana_admin_passwd, 'kube_dashboard_enabled': kube_dashboard_enabled, 'docker_volume_type': docker_volume_type, + 'etcd_volume_size': etcd_volume_size, 'username': 'fake_user', 'tenant_name': 'fake_tenant', 'magnum_url': mock_osc.magnum_url.return_value,