k8s-fedora: Add etcd_volume_size label

Allow setting the size of a volume for etcd storage. Default is 0 which matches the current behavior - no persistency. Related-Bug: #1697655 Change-Id: I8a30df63684133a902ae209ba6c124da2a567d3f
2017-06-13 11:19:38 +00:00 · 2017-06-13 11:19:38 +00:00 · 2875c97bff
parent 2e7e5a7796
commit 2875c97bff
13 changed files with 117 additions and 3 deletions
--- a/doc/source/userguide.rst
+++ b/doc/source/userguide.rst
@ -337,6 +337,9 @@ the table are linked to more details elsewhere in the user guide.
 +---------------------------------------+--------------------+---------------+
 | `docker_volume_type`_                 | see below          | see below     |
 +---------------------------------------+--------------------+---------------+
+| `etcd_volume_size`_                   | etcd storage       | 0             |
+|                                       | volume size        |               |
+---------------------------------------+--------------------+---------------+

 =======
 Cluster
@ -1060,6 +1063,10 @@ _`admission_control_list`
  The default value corresponds to the one recommended in this doc
  for our current Kubernetes version.

+_`etcd_volume_size`
+  This label sets the size of a volume holding the etcd storage data.
+  The default value is 0, meaning the etcd data is not persisted (no volume).
+
 External load balancer for services
 -----------------------------------

--- a/magnum/drivers/common/templates/environments/no_etcd_volume.yaml
+++ b/magnum/drivers/common/templates/environments/no_etcd_volume.yaml
@ -0,0 +1,4 @@
+# Environment file to not use a cinder volume for etcd storage
+resource_registry:
+  "Magnum::Optional::Etcd::Volume": "OS::Heat::None"
+  "Magnum::Optional::Etcd::VolumeAttachment": "OS::Heat::None"
--- a/magnum/drivers/common/templates/environments/with_etcd_volume.yaml
+++ b/magnum/drivers/common/templates/environments/with_etcd_volume.yaml
@ -0,0 +1,4 @@
+# Environment file to use a volume for etcd storage
+resource_registry:
+  "Magnum::Optional::Etcd::Volume": "OS::Cinder::Volume"
+  "Magnum::Optional::Etcd::VolumeAttachment": "OS::Cinder::VolumeAttachment"
--- a/magnum/drivers/common/templates/kubernetes/fragments/configure-etcd.sh
+++ b/magnum/drivers/common/templates/kubernetes/fragments/configure-etcd.sh
@ -2,6 +2,38 @@

 . /etc/sysconfig/heat-params

+if [ -n "$ETCD_VOLUME_SIZE" ] && [ "$ETCD_VOLUME_SIZE" -gt 0 ]; then
+
+    attempts=60
+    while [ ${attempts} -gt 0 ]; do
+        device_name=$(ls /dev/disk/by-id | grep ${ETCD_VOLUME:0:20}$)
+        if [ -n "${device_name}" ]; then
+            break
+        fi
+        echo "waiting for disk device"
+        sleep 0.5
+        udevadm trigger
+        let attempts--
+    done
+
+    if [ -z "${device_name}" ]; then
+        echo "ERROR: disk device does not exist" >&2
+        exit 1
+    fi
+
+    device_path=/dev/disk/by-id/${device_name}
+    fstype=$(blkid -s TYPE -o value ${device_path})
+    if [ "${fstype}" != "xfs" ]; then
+        mkfs.xfs -f ${device_path}
+    fi
+    mkdir -p /var/lib/etcd
+    echo "${device_path} /var/lib/etcd xfs defaults 0 0" >> /etc/fstab
+    mount -a
+    chown -R etcd.etcd /var/lib/etcd
+    chmod 755 /var/lib/etcd
+
+fi
+
 if [ -z "$KUBE_NODE_IP" ]; then
    # FIXME(yuanying): Set KUBE_NODE_IP correctly
    KUBE_NODE_IP=$(curl -s http://169.254.169.254/latest/meta-data/local-ipv4)
--- a/magnum/drivers/common/templates/kubernetes/fragments/write-heat-params-master.yaml
+++ b/magnum/drivers/common/templates/kubernetes/fragments/write-heat-params-master.yaml
@ -13,6 +13,8 @@ write_files:
      KUBE_NODE_IP="$KUBE_NODE_IP"
      KUBE_ALLOW_PRIV="$KUBE_ALLOW_PRIV"
      ENABLE_CINDER="$ENABLE_CINDER"
+      ETCD_VOLUME="$ETCD_VOLUME"
+      ETCD_VOLUME_SIZE="$ETCD_VOLUME_SIZE"
      DOCKER_VOLUME="$DOCKER_VOLUME"
      DOCKER_VOLUME_SIZE="$DOCKER_VOLUME_SIZE"
      DOCKER_STORAGE_DRIVER="$DOCKER_STORAGE_DRIVER"
--- a/magnum/drivers/heat/k8s_fedora_template_def.py
+++ b/magnum/drivers/heat/k8s_fedora_template_def.py
@ -88,6 +88,7 @@ class K8sFedoraTemplateDefinition(k8s_template_def.K8sTemplateDefinition):
        env_files = []

        template_def.add_priv_net_env_file(env_files, cluster_template)
+        template_def.add_etcd_volume_env_file(env_files, cluster_template)
        template_def.add_volume_env_file(env_files, cluster)
        template_def.add_lb_env_file(env_files, cluster_template)
        template_def.add_fip_env_file(env_files, cluster_template)
--- a/magnum/drivers/heat/k8s_template_def.py
+++ b/magnum/drivers/heat/k8s_template_def.py
@ -112,7 +112,8 @@ class K8sTemplateDefinition(template_def.BaseTemplateDefinition):
                      'admission_control_list',
                      'prometheus_monitoring',
                      'grafana_admin_passwd',
-                      'kube_dashboard_enabled']
+                      'kube_dashboard_enabled',
+                      'etcd_volume_size']

        for label in label_list:
            extra_params[label] = cluster_template.labels.get(label)
--- a/magnum/drivers/heat/template_def.py
+++ b/magnum/drivers/heat/template_def.py
@ -337,6 +337,13 @@ def add_volume_env_file(env_files, cluster):
        env_files.append(COMMON_ENV_PATH + 'with_volume.yaml')


+def add_etcd_volume_env_file(env_files, cluster_template):
+    if int(cluster_template.labels.get('etcd_volume_size', 0)) < 1:
+        env_files.append(COMMON_ENV_PATH + 'no_etcd_volume.yaml')
+    else:
+        env_files.append(COMMON_ENV_PATH + 'with_etcd_volume.yaml')
+
+
 def add_fip_env_file(env_files, cluster_template):
    if cluster_template.floating_ip_enabled:
        env_files.append(COMMON_ENV_PATH + 'enable_floating_ip.yaml')
--- a/magnum/drivers/k8s_fedora_atomic_v1/templates/kubecluster.yaml
+++ b/magnum/drivers/k8s_fedora_atomic_v1/templates/kubecluster.yaml
@ -130,6 +130,12 @@ parameters:
    constraints:
      - allowed_values: ["true", "false"]

+  etcd_volume_size:
+    type: number
+    description: >
+      size of the cinder volume for etcd storage
+    default: 0
+
  docker_volume_size:
    type: number
    description: >
@ -467,6 +473,7 @@ resources:
          master_flavor: {get_param: master_flavor}
          external_network: {get_param: external_network}
          kube_allow_priv: {get_param: kube_allow_priv}
+          etcd_volume_size: {get_param: etcd_volume_size}
          docker_volume_size: {get_param: docker_volume_size}
          docker_volume_type: {get_param: docker_volume_type}
          docker_storage_driver: {get_param: docker_storage_driver}
--- a/magnum/drivers/k8s_fedora_atomic_v1/templates/kubemaster.yaml
+++ b/magnum/drivers/k8s_fedora_atomic_v1/templates/kubemaster.yaml
@ -35,6 +35,11 @@ parameters:
    constraints:
      - allowed_values: ["true", "false"]

+  etcd_volume_size:
+    type: number
+    description: >
+      size of a cinder volume to allocate for etcd storage
+
  docker_volume_size:
    type: number
    description: >
@ -290,6 +295,8 @@ resources:
            "$KUBE_NODE_PUBLIC_IP": {get_attr: [kube_master_floating, floating_ip_address]}
            "$KUBE_NODE_IP": {get_attr: [kube_master_eth0, fixed_ips, 0, ip_address]}
            "$KUBE_ALLOW_PRIV": {get_param: kube_allow_priv}
+            "$ETCD_VOLUME": {get_resource: etcd_volume}
+            "$ETCD_VOLUME_SIZE": {get_param: etcd_volume_size}
            "$DOCKER_VOLUME": {get_resource: docker_volume}
            "$DOCKER_VOLUME_SIZE": {get_param: docker_volume_size}
            "$DOCKER_STORAGE_DRIVER": {get_param: docker_storage_driver}
@ -545,6 +552,24 @@ resources:
      subnet: { get_param: fixed_subnet }
      protocol_port: 2379

+  ######################################################################
+  #
+  # etcd storage.  This allocates a cinder volume and attaches it
+  # to the master.
+  #
+
+  etcd_volume:
+    type: Magnum::Optional::Etcd::Volume
+    properties:
+      size: {get_param: etcd_volume_size}
+
+  etcd_volume_attach:
+    type: Magnum::Optional::Etcd::VolumeAttachment
+    properties:
+      instance_uuid: {get_resource: kube-master}
+      volume_id: {get_resource: etcd_volume}
+      mountpoint: /dev/vdc
+
  ######################################################################
  #
  # docker storage.  This allocates a cinder volume and attaches it
--- a/magnum/drivers/k8s_fedora_ironic_v1/templates/kubecluster.yaml
+++ b/magnum/drivers/k8s_fedora_ironic_v1/templates/kubecluster.yaml
@ -128,6 +128,12 @@ parameters:
    constraints:
      - allowed_values: ["true", "false"]

+  etcd_volume_size:
+    type: number
+    description: >
+      size of the cinder volume for etcd storage
+    default: 0
+
  docker_volume_size:
    type: number
    description: >
--- a/magnum/tests/unit/conductor/handlers/test_k8s_cluster_conductor.py
+++ b/magnum/tests/unit/conductor/handlers/test_k8s_cluster_conductor.py
@ -55,7 +55,8 @@ class TestClusterConductorWithK8s(base.TestCase):
                       'prometheus_monitoring': 'False',
                       'grafana_admin_passwd': 'fake_pwd',
                       'kube_dashboard_enabled': 'True',
-                       'docker_volume_type': 'lvmdriver-1'},
+                       'docker_volume_type': 'lvmdriver-1',
+                       'etcd_volume_size': '0'},
            'tls_disabled': False,
            'server_type': 'vm',
            'registry_enabled': False,
@ -158,7 +159,8 @@ class TestClusterConductorWithK8s(base.TestCase):
                       'prometheus_monitoring': 'False',
                       'grafana_admin_passwd': 'fake_pwd',
                       'kube_dashboard_enabled': 'True',
-                       'docker_volume_type': 'lvmdriver-1'},
+                       'docker_volume_type': 'lvmdriver-1',
+                       'etcd_volume_size': '0'},
            'http_proxy': 'http_proxy',
            'https_proxy': 'https_proxy',
            'no_proxy': 'no_proxy',
@ -184,6 +186,7 @@ class TestClusterConductorWithK8s(base.TestCase):
            'docker_volume_type': 'lvmdriver-1',
            'docker_storage_driver': 'devicemapper',
            'discovery_url': 'https://discovery.etcd.io/test',
+            'etcd_volume_size': '0',
            'flannel_network_cidr': '10.101.0.0/16',
            'flannel_network_subnetlen': '26',
            'flannel_backend': 'vxlan',
@ -218,6 +221,7 @@ class TestClusterConductorWithK8s(base.TestCase):
        self.assertEqual(expected, definition)
        self.assertEqual(
            ['../../common/templates/environments/no_private_network.yaml',
+             '../../common/templates/environments/no_etcd_volume.yaml',
             '../../common/templates/environments/with_volume.yaml',
             '../../common/templates/environments/no_master_lb.yaml',
             '../../common/templates/environments/disable_floating_ip.yaml',
@ -266,6 +270,7 @@ class TestClusterConductorWithK8s(base.TestCase):
            'docker_storage_driver': 'devicemapper',
            'docker_volume_size': 20,
            'docker_volume_type': 'lvmdriver-1',
+            'etcd_volume_size': '0',
            'external_network': 'external_network_id',
            'fixed_network': 'fixed_network',
            'fixed_subnet': 'fixed_subnet',
@ -309,6 +314,7 @@ class TestClusterConductorWithK8s(base.TestCase):
        self.assertEqual(expected, definition)
        self.assertEqual(
            ['../../common/templates/environments/no_private_network.yaml',
+             '../../common/templates/environments/no_etcd_volume.yaml',
             '../../common/templates/environments/with_volume.yaml',
             '../../common/templates/environments/no_master_lb.yaml',
             '../../common/templates/environments/disable_floating_ip.yaml',
@ -366,6 +372,7 @@ class TestClusterConductorWithK8s(base.TestCase):
            'grafana_admin_passwd': 'fake_pwd',
            'kube_dashboard_enabled': 'True',
            'docker_volume_type': 'lvmdriver-1',
+            'etcd_volume_size': '0',
            'insecure_registry_url': '10.0.0.1:5000',
            'kube_version': 'fake-version',
            'magnum_url': 'http://127.0.0.1:9511/v1',
@ -386,6 +393,7 @@ class TestClusterConductorWithK8s(base.TestCase):
        self.assertEqual(expected, definition)
        self.assertEqual(
            ['../../common/templates/environments/with_private_network.yaml',
+             '../../common/templates/environments/no_etcd_volume.yaml',
             '../../common/templates/environments/with_volume.yaml',
             '../../common/templates/environments/no_master_lb.yaml',
             '../../common/templates/environments/disable_floating_ip.yaml',
@ -432,6 +440,7 @@ class TestClusterConductorWithK8s(base.TestCase):
            'network_driver': 'network_driver',
            'volume_driver': 'volume_driver',
            'discovery_url': 'https://discovery.etcd.io/test',
+            'etcd_volume_size': '0',
            'http_proxy': 'http_proxy',
            'https_proxy': 'https_proxy',
            'no_proxy': 'no_proxy',
@ -502,6 +511,7 @@ class TestClusterConductorWithK8s(base.TestCase):
            'network_driver': 'network_driver',
            'volume_driver': 'volume_driver',
            'discovery_url': 'http://tokentest/h1/h2/h3',
+            'etcd_volume_size': '0',
            'http_proxy': 'http_proxy',
            'https_proxy': 'https_proxy',
            'no_proxy': 'no_proxy',
@ -699,6 +709,7 @@ class TestClusterConductorWithK8s(base.TestCase):
            'docker_volume_type': 'lvmdriver-1',
            'docker_storage_driver': 'devicemapper',
            'discovery_url': 'https://address/token',
+            'etcd_volume_size': '0',
            'http_proxy': 'http_proxy',
            'https_proxy': 'https_proxy',
            'no_proxy': 'no_proxy',
@ -730,6 +741,7 @@ class TestClusterConductorWithK8s(base.TestCase):
        self.assertEqual(expected, definition)
        self.assertEqual(
            ['../../common/templates/environments/no_private_network.yaml',
+             '../../common/templates/environments/no_etcd_volume.yaml',
             '../../common/templates/environments/with_volume.yaml',
             '../../common/templates/environments/no_master_lb.yaml',
             '../../common/templates/environments/disable_floating_ip.yaml',
--- a/magnum/tests/unit/drivers/test_template_definition.py
+++ b/magnum/tests/unit/drivers/test_template_definition.py
@ -268,6 +268,8 @@ class AtomicK8sTemplateDefinitionTestCase(BaseTemplateDefinitionTestCase):
            'kube_dashboard_enabled')
        docker_volume_type = mock_cluster_template.labels.get(
            'docker_volume_type')
+        etcd_volume_size = mock_cluster_template.labels.get(
+            'etcd_volume_size')

        k8s_def = k8sa_tdef.AtomicK8sTemplateDefinition()

@ -287,6 +289,7 @@ class AtomicK8sTemplateDefinitionTestCase(BaseTemplateDefinitionTestCase):
            'grafana_admin_passwd': grafana_admin_passwd,
            'kube_dashboard_enabled': kube_dashboard_enabled,
            'docker_volume_type': docker_volume_type,
+            'etcd_volume_size': etcd_volume_size,
            'username': 'fake_user',
            'tenant_name': 'fake_tenant',
            'magnum_url': mock_osc.magnum_url.return_value,
@ -345,6 +348,8 @@ class AtomicK8sTemplateDefinitionTestCase(BaseTemplateDefinitionTestCase):
            'kube_dashboard_enabled')
        docker_volume_type = mock_cluster_template.labels.get(
            'docker_volume_type')
+        etcd_volume_size = mock_cluster_template.labels.get(
+            'etcd_volume_size')

        k8s_def = k8sa_tdef.AtomicK8sTemplateDefinition()

@ -364,6 +369,7 @@ class AtomicK8sTemplateDefinitionTestCase(BaseTemplateDefinitionTestCase):
            'grafana_admin_passwd': grafana_admin_passwd,
            'kube_dashboard_enabled': kube_dashboard_enabled,
            'docker_volume_type': docker_volume_type,
+            'etcd_volume_size': etcd_volume_size,
            'username': 'fake_user',
            'tenant_name': 'fake_tenant',
            'magnum_url': mock_osc.magnum_url.return_value,