k8s-fedora: Add etcd_volume_size label

Allow setting the size of a volume for etcd storage.

Default is 0 which matches the current behavior - no persistency.

Related-Bug: #1697655

Change-Id: I8a30df63684133a902ae209ba6c124da2a567d3f

doc/source/userguide.rst

@@ -337,6 +337,9 @@ the table are linked to more details elsewhere in the user guide.
 +---------------------------------------+--------------------+---------------+
 | `docker_volume_type`_                 | see below          | see below     |
 +---------------------------------------+--------------------+---------------+
+| `etcd_volume_size`_                   | etcd storage       | 0             |
+|                                       | volume size        |               |
++---------------------------------------+--------------------+---------------+
 
 =======
 Cluster
@@ -1060,6 +1063,10 @@ _`admission_control_list`
   The default value corresponds to the one recommended in this doc
   for our current Kubernetes version.
 
+_`etcd_volume_size`
+  This label sets the size of a volume holding the etcd storage data.
+  The default value is 0, meaning the etcd data is not persisted (no volume).
+
 External load balancer for services
 -----------------------------------
 

magnum/drivers/common/templates/environments/no_etcd_volume.yaml

@@ -0,0 +1,4 @@
+# Environment file to not use a cinder volume for etcd storage
+resource_registry:
+  "Magnum::Optional::Etcd::Volume": "OS::Heat::None"
+  "Magnum::Optional::Etcd::VolumeAttachment": "OS::Heat::None"

magnum/drivers/common/templates/environments/with_etcd_volume.yaml

@@ -0,0 +1,4 @@
+# Environment file to use a volume for etcd storage
+resource_registry:
+  "Magnum::Optional::Etcd::Volume": "OS::Cinder::Volume"
+  "Magnum::Optional::Etcd::VolumeAttachment": "OS::Cinder::VolumeAttachment"

magnum/drivers/common/templates/kubernetes/fragments/configure-etcd.sh

@@ -2,6 +2,38 @@
 
 . /etc/sysconfig/heat-params
 
+if [ -n "$ETCD_VOLUME_SIZE" ] && [ "$ETCD_VOLUME_SIZE" -gt 0 ]; then
+
+    attempts=60
+    while [ ${attempts} -gt 0 ]; do
+        device_name=$(ls /dev/disk/by-id | grep ${ETCD_VOLUME:0:20}$)
+        if [ -n "${device_name}" ]; then
+            break
+        fi
+        echo "waiting for disk device"
+        sleep 0.5
+        udevadm trigger
+        let attempts--
+    done
+
+    if [ -z "${device_name}" ]; then
+        echo "ERROR: disk device does not exist" >&2
+        exit 1
+    fi
+
+    device_path=/dev/disk/by-id/${device_name}
+    fstype=$(blkid -s TYPE -o value ${device_path})
+    if [ "${fstype}" != "xfs" ]; then
+        mkfs.xfs -f ${device_path}
+    fi
+    mkdir -p /var/lib/etcd
+    echo "${device_path} /var/lib/etcd xfs defaults 0 0" >> /etc/fstab
+    mount -a
+    chown -R etcd.etcd /var/lib/etcd
+    chmod 755 /var/lib/etcd
+
+fi
+
 if [ -z "$KUBE_NODE_IP" ]; then
     # FIXME(yuanying): Set KUBE_NODE_IP correctly
     KUBE_NODE_IP=$(curl -s http://169.254.169.254/latest/meta-data/local-ipv4)

magnum/drivers/common/templates/kubernetes/fragments/write-heat-params-master.yaml

@@ -13,6 +13,8 @@ write_files:
       KUBE_NODE_IP="$KUBE_NODE_IP"
       KUBE_ALLOW_PRIV="$KUBE_ALLOW_PRIV"
       ENABLE_CINDER="$ENABLE_CINDER"
+      ETCD_VOLUME="$ETCD_VOLUME"
+      ETCD_VOLUME_SIZE="$ETCD_VOLUME_SIZE"
       DOCKER_VOLUME="$DOCKER_VOLUME"
       DOCKER_VOLUME_SIZE="$DOCKER_VOLUME_SIZE"
       DOCKER_STORAGE_DRIVER="$DOCKER_STORAGE_DRIVER"

magnum/drivers/heat/k8s_fedora_template_def.py

@@ -88,6 +88,7 @@ class K8sFedoraTemplateDefinition(k8s_template_def.K8sTemplateDefinition):
         env_files = []
 
         template_def.add_priv_net_env_file(env_files, cluster_template)
+        template_def.add_etcd_volume_env_file(env_files, cluster_template)
         template_def.add_volume_env_file(env_files, cluster)
         template_def.add_lb_env_file(env_files, cluster_template)
         template_def.add_fip_env_file(env_files, cluster_template)

magnum/drivers/heat/k8s_template_def.py

@@ -112,7 +112,8 @@ class K8sTemplateDefinition(template_def.BaseTemplateDefinition):
                       'admission_control_list',
                       'prometheus_monitoring',
                       'grafana_admin_passwd',
-                      'kube_dashboard_enabled']
+                      'kube_dashboard_enabled',
+                      'etcd_volume_size']
 
         for label in label_list:
             extra_params[label] = cluster_template.labels.get(label)

magnum/drivers/heat/template_def.py

@@ -337,6 +337,13 @@ def add_volume_env_file(env_files, cluster):
         env_files.append(COMMON_ENV_PATH + 'with_volume.yaml')
 
 
+def add_etcd_volume_env_file(env_files, cluster_template):
+    if int(cluster_template.labels.get('etcd_volume_size', 0)) < 1:
+        env_files.append(COMMON_ENV_PATH + 'no_etcd_volume.yaml')
+    else:
+        env_files.append(COMMON_ENV_PATH + 'with_etcd_volume.yaml')
+
+
 def add_fip_env_file(env_files, cluster_template):
     if cluster_template.floating_ip_enabled:
         env_files.append(COMMON_ENV_PATH + 'enable_floating_ip.yaml')

magnum/drivers/k8s_fedora_atomic_v1/templates/kubecluster.yaml

@@ -130,6 +130,12 @@ parameters:
     constraints:
       - allowed_values: ["true", "false"]
 
+  etcd_volume_size:
+    type: number
+    description: >
+      size of the cinder volume for etcd storage
+    default: 0
+
   docker_volume_size:
     type: number
     description: >
@@ -467,6 +473,7 @@ resources:
           master_flavor: {get_param: master_flavor}
           external_network: {get_param: external_network}
           kube_allow_priv: {get_param: kube_allow_priv}
+          etcd_volume_size: {get_param: etcd_volume_size}
           docker_volume_size: {get_param: docker_volume_size}
           docker_volume_type: {get_param: docker_volume_type}
           docker_storage_driver: {get_param: docker_storage_driver}

magnum/drivers/k8s_fedora_atomic_v1/templates/kubemaster.yaml

@@ -35,6 +35,11 @@ parameters:
     constraints:
       - allowed_values: ["true", "false"]
 
+  etcd_volume_size:
+    type: number
+    description: >
+      size of a cinder volume to allocate for etcd storage
+
   docker_volume_size:
     type: number
     description: >
@@ -290,6 +295,8 @@ resources:
             "$KUBE_NODE_PUBLIC_IP": {get_attr: [kube_master_floating, floating_ip_address]}
             "$KUBE_NODE_IP": {get_attr: [kube_master_eth0, fixed_ips, 0, ip_address]}
             "$KUBE_ALLOW_PRIV": {get_param: kube_allow_priv}
+            "$ETCD_VOLUME": {get_resource: etcd_volume}
+            "$ETCD_VOLUME_SIZE": {get_param: etcd_volume_size}
             "$DOCKER_VOLUME": {get_resource: docker_volume}
             "$DOCKER_VOLUME_SIZE": {get_param: docker_volume_size}
             "$DOCKER_STORAGE_DRIVER": {get_param: docker_storage_driver}
@@ -545,6 +552,24 @@ resources:
       subnet: { get_param: fixed_subnet }
       protocol_port: 2379
 
+  ######################################################################
+  #
+  # etcd storage.  This allocates a cinder volume and attaches it
+  # to the master.
+  #
+
+  etcd_volume:
+    type: Magnum::Optional::Etcd::Volume
+    properties:
+      size: {get_param: etcd_volume_size}
+
+  etcd_volume_attach:
+    type: Magnum::Optional::Etcd::VolumeAttachment
+    properties:
+      instance_uuid: {get_resource: kube-master}
+      volume_id: {get_resource: etcd_volume}
+      mountpoint: /dev/vdc
+
   ######################################################################
   #
   # docker storage.  This allocates a cinder volume and attaches it

magnum/drivers/k8s_fedora_ironic_v1/templates/kubecluster.yaml

@@ -128,6 +128,12 @@ parameters:
     constraints:
       - allowed_values: ["true", "false"]
 
+  etcd_volume_size:
+    type: number
+    description: >
+      size of the cinder volume for etcd storage
+    default: 0
+
   docker_volume_size:
     type: number
     description: >

magnum/tests/unit/conductor/handlers/test_k8s_cluster_conductor.py

@@ -55,7 +55,8 @@ class TestClusterConductorWithK8s(base.TestCase):
                        'prometheus_monitoring': 'False',
                        'grafana_admin_passwd': 'fake_pwd',
                        'kube_dashboard_enabled': 'True',
-                       'docker_volume_type': 'lvmdriver-1'},
+                       'docker_volume_type': 'lvmdriver-1',
+                       'etcd_volume_size': '0'},
             'tls_disabled': False,
             'server_type': 'vm',
             'registry_enabled': False,
@@ -158,7 +159,8 @@ class TestClusterConductorWithK8s(base.TestCase):
                        'prometheus_monitoring': 'False',
                        'grafana_admin_passwd': 'fake_pwd',
                        'kube_dashboard_enabled': 'True',
-                       'docker_volume_type': 'lvmdriver-1'},
+                       'docker_volume_type': 'lvmdriver-1',
+                       'etcd_volume_size': '0'},
             'http_proxy': 'http_proxy',
             'https_proxy': 'https_proxy',
             'no_proxy': 'no_proxy',
@@ -184,6 +186,7 @@ class TestClusterConductorWithK8s(base.TestCase):
             'docker_volume_type': 'lvmdriver-1',
             'docker_storage_driver': 'devicemapper',
             'discovery_url': 'https://discovery.etcd.io/test',
+            'etcd_volume_size': '0',
             'flannel_network_cidr': '10.101.0.0/16',
             'flannel_network_subnetlen': '26',
             'flannel_backend': 'vxlan',
@@ -218,6 +221,7 @@ class TestClusterConductorWithK8s(base.TestCase):
         self.assertEqual(expected, definition)
         self.assertEqual(
             ['../../common/templates/environments/no_private_network.yaml',
+             '../../common/templates/environments/no_etcd_volume.yaml',
              '../../common/templates/environments/with_volume.yaml',
              '../../common/templates/environments/no_master_lb.yaml',
              '../../common/templates/environments/disable_floating_ip.yaml',
@@ -266,6 +270,7 @@ class TestClusterConductorWithK8s(base.TestCase):
             'docker_storage_driver': 'devicemapper',
             'docker_volume_size': 20,
             'docker_volume_type': 'lvmdriver-1',
+            'etcd_volume_size': '0',
             'external_network': 'external_network_id',
             'fixed_network': 'fixed_network',
             'fixed_subnet': 'fixed_subnet',
@@ -309,6 +314,7 @@ class TestClusterConductorWithK8s(base.TestCase):
         self.assertEqual(expected, definition)
         self.assertEqual(
             ['../../common/templates/environments/no_private_network.yaml',
+             '../../common/templates/environments/no_etcd_volume.yaml',
              '../../common/templates/environments/with_volume.yaml',
              '../../common/templates/environments/no_master_lb.yaml',
              '../../common/templates/environments/disable_floating_ip.yaml',
@@ -366,6 +372,7 @@ class TestClusterConductorWithK8s(base.TestCase):
             'grafana_admin_passwd': 'fake_pwd',
             'kube_dashboard_enabled': 'True',
             'docker_volume_type': 'lvmdriver-1',
+            'etcd_volume_size': '0',
             'insecure_registry_url': '10.0.0.1:5000',
             'kube_version': 'fake-version',
             'magnum_url': 'http://127.0.0.1:9511/v1',
@@ -386,6 +393,7 @@ class TestClusterConductorWithK8s(base.TestCase):
         self.assertEqual(expected, definition)
         self.assertEqual(
             ['../../common/templates/environments/with_private_network.yaml',
+             '../../common/templates/environments/no_etcd_volume.yaml',
              '../../common/templates/environments/with_volume.yaml',
              '../../common/templates/environments/no_master_lb.yaml',
              '../../common/templates/environments/disable_floating_ip.yaml',
@@ -432,6 +440,7 @@ class TestClusterConductorWithK8s(base.TestCase):
             'network_driver': 'network_driver',
             'volume_driver': 'volume_driver',
             'discovery_url': 'https://discovery.etcd.io/test',
+            'etcd_volume_size': '0',
             'http_proxy': 'http_proxy',
             'https_proxy': 'https_proxy',
             'no_proxy': 'no_proxy',
@@ -502,6 +511,7 @@ class TestClusterConductorWithK8s(base.TestCase):
             'network_driver': 'network_driver',
             'volume_driver': 'volume_driver',
             'discovery_url': 'http://tokentest/h1/h2/h3',
+            'etcd_volume_size': '0',
             'http_proxy': 'http_proxy',
             'https_proxy': 'https_proxy',
             'no_proxy': 'no_proxy',
@@ -699,6 +709,7 @@ class TestClusterConductorWithK8s(base.TestCase):
             'docker_volume_type': 'lvmdriver-1',
             'docker_storage_driver': 'devicemapper',
             'discovery_url': 'https://address/token',
+            'etcd_volume_size': '0',
             'http_proxy': 'http_proxy',
             'https_proxy': 'https_proxy',
             'no_proxy': 'no_proxy',
@@ -730,6 +741,7 @@ class TestClusterConductorWithK8s(base.TestCase):
         self.assertEqual(expected, definition)
         self.assertEqual(
             ['../../common/templates/environments/no_private_network.yaml',
+             '../../common/templates/environments/no_etcd_volume.yaml',
              '../../common/templates/environments/with_volume.yaml',
              '../../common/templates/environments/no_master_lb.yaml',
              '../../common/templates/environments/disable_floating_ip.yaml',

magnum/tests/unit/drivers/test_template_definition.py

@@ -268,6 +268,8 @@ class AtomicK8sTemplateDefinitionTestCase(BaseTemplateDefinitionTestCase):
             'kube_dashboard_enabled')
         docker_volume_type = mock_cluster_template.labels.get(
             'docker_volume_type')
+        etcd_volume_size = mock_cluster_template.labels.get(
+            'etcd_volume_size')
 
         k8s_def = k8sa_tdef.AtomicK8sTemplateDefinition()
 
@@ -287,6 +289,7 @@ class AtomicK8sTemplateDefinitionTestCase(BaseTemplateDefinitionTestCase):
             'grafana_admin_passwd': grafana_admin_passwd,
             'kube_dashboard_enabled': kube_dashboard_enabled,
             'docker_volume_type': docker_volume_type,
+            'etcd_volume_size': etcd_volume_size,
             'username': 'fake_user',
             'tenant_name': 'fake_tenant',
             'magnum_url': mock_osc.magnum_url.return_value,
@@ -345,6 +348,8 @@ class AtomicK8sTemplateDefinitionTestCase(BaseTemplateDefinitionTestCase):
             'kube_dashboard_enabled')
         docker_volume_type = mock_cluster_template.labels.get(
             'docker_volume_type')
+        etcd_volume_size = mock_cluster_template.labels.get(
+            'etcd_volume_size')
 
         k8s_def = k8sa_tdef.AtomicK8sTemplateDefinition()
 
@@ -364,6 +369,7 @@ class AtomicK8sTemplateDefinitionTestCase(BaseTemplateDefinitionTestCase):
             'grafana_admin_passwd': grafana_admin_passwd,
             'kube_dashboard_enabled': kube_dashboard_enabled,
             'docker_volume_type': docker_volume_type,
+            'etcd_volume_size': etcd_volume_size,
             'username': 'fake_user',
             'tenant_name': 'fake_tenant',
             'magnum_url': mock_osc.magnum_url.return_value,