DNM Add boot from volume support
This is proof that what I proposed works. During the discussion I was getting comments that using conditions, it does not work. DO NOT MERGE this PatchSet. Support boot from volume for Kubernetes all nodes (master and worker) so that user can create a big size root volume, which could be more flexible than using docker_volume_size. And user can specify the volume type so that user can leverage high performance storage, e.g. NVMe etc. And a new label etcd_volme_type is added as well so that user can set volume type for etcd volume. If the boot_volume_type or etcd_volume_type are not passed by labels, Magnum will try to read them from config option default_boot_volume_type and default_etcd_volume_type. A random volume type from Cinder will be used if those options are not set. Task: 30374 Story: 2005386 Co-Authored-By: Lingxian Kong <anlin.kong@gmail.com> Co-Authored-By: Feilong Wang<flwang@catalyst.net.nz> Change-Id: I39dd456bfa285bf06dd948d11c86867fc03d5afb
This commit is contained in:
parent
df3d5a3150
commit
6a239a87c9
|
@ -346,9 +346,15 @@ the table are linked to more details elsewhere in the user guide.
|
|||
+---------------------------------------+--------------------+---------------+
|
||||
| `docker_volume_type`_ | see below | see below |
|
||||
+---------------------------------------+--------------------+---------------+
|
||||
| `boot_volume_size`_ | see below | see below |
|
||||
+---------------------------------------+--------------------+---------------+
|
||||
| `boot_volume_type`_ | see below | see below |
|
||||
+---------------------------------------+--------------------+---------------+
|
||||
| `etcd_volume_size`_ | etcd storage | 0 |
|
||||
| | volume size | |
|
||||
+---------------------------------------+--------------------+---------------+
|
||||
| `etcd_volume_type`_ | see below | see below |
|
||||
+---------------------------------------+--------------------+---------------+
|
||||
| `container_infra_prefix`_ | see below | "" |
|
||||
+---------------------------------------+--------------------+---------------+
|
||||
| `availability_zone`_ | AZ for the cluster | "" |
|
||||
|
@ -1114,10 +1120,26 @@ _`admission_control_list`
|
|||
The default value corresponds to the one recommended in this doc
|
||||
for our current Kubernetes version.
|
||||
|
||||
_`boot_volume_size`
|
||||
This label sets the size of a boot volume for instances, this is useful if
|
||||
your flavors are boot from volume only. The default value is 20, setting
|
||||
by config option default_boot_volume_size
|
||||
|
||||
_`boot_volume_type`
|
||||
This label sets the volume type of a boot volume for instances, this is
|
||||
useful if your flavors are boot from volume only. The default value is '',
|
||||
meaning that no volume will be created on boot. If it's not setting by
|
||||
config option default_boot_volume_type, then Magnum will random select
|
||||
one from the list of Cinder volume types.
|
||||
|
||||
_`etcd_volume_size`
|
||||
This label sets the size of a volume holding the etcd storage data.
|
||||
The default value is 0, meaning the etcd data is not persisted (no volume).
|
||||
|
||||
_`etcd_volume_type`
|
||||
This label sets the volume type of a volume holding the etcd storage data.
|
||||
The default value is '', meaning the etcd data is not persisted (no volume).
|
||||
|
||||
_`container_infra_prefix`
|
||||
Prefix of all container images used in the cluster (kubernetes components,
|
||||
coredns, kubernetes-dashboard, node-exporter). For example,
|
||||
|
|
|
@ -117,6 +117,7 @@ python-barbicanclient==4.5.2
|
|||
python-dateutil==2.7.0
|
||||
python-editor==1.0.3
|
||||
python-glanceclient==2.8.0
|
||||
python-cinderclient==2.2.0
|
||||
python-heatclient==1.10.0
|
||||
python-keystoneclient==3.8.0
|
||||
python-mimeparse==1.6.0
|
||||
|
|
|
@ -0,0 +1,46 @@
|
|||
# Copyright 2019 Catalyst Cloud Ltd.
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
from oslo_config import cfg
|
||||
from oslo_log import log as logging
|
||||
|
||||
from magnum.common import clients
|
||||
from magnum.common import exception
|
||||
|
||||
LOG = logging.getLogger(__name__)
|
||||
CONF = cfg.CONF
|
||||
|
||||
|
||||
def get_default_docker_volume_type(context):
|
||||
return (CONF.cinder.default_docker_volume_type or
|
||||
_get_random_volume_type(context))
|
||||
|
||||
|
||||
def get_default_boot_volume_type(context):
|
||||
return (CONF.cinder.default_boot_volume_type or
|
||||
_get_random_volume_type(context))
|
||||
|
||||
|
||||
def get_default_etcd_volume_type(context):
|
||||
return (CONF.cinder.default_etcd_volume_type or
|
||||
_get_random_volume_type(context))
|
||||
|
||||
|
||||
def _get_random_volume_type(context):
|
||||
c_client = clients.OpenStackClients(context).cinder()
|
||||
volume_types = c_client.volume_types.list()
|
||||
if volume_types:
|
||||
return volume_types[0].name
|
||||
else:
|
||||
raise exception.VolumeTypeNotFound()
|
|
@ -13,6 +13,7 @@
|
|||
# under the License.
|
||||
|
||||
from barbicanclient.v1 import client as barbicanclient
|
||||
from cinderclient.v2 import client as cinder_client
|
||||
from glanceclient import client as glanceclient
|
||||
from heatclient import client as heatclient
|
||||
from keystoneauth1.exceptions import catalog
|
||||
|
@ -41,6 +42,7 @@ class OpenStackClients(object):
|
|||
self._nova = None
|
||||
self._neutron = None
|
||||
self._octavia = None
|
||||
self._cinder = None
|
||||
|
||||
def url_for(self, **kwargs):
|
||||
return self.keystone().session.get_endpoint(**kwargs)
|
||||
|
@ -207,3 +209,24 @@ class OpenStackClients(object):
|
|||
}
|
||||
self._neutron = neutronclient.Client(**args)
|
||||
return self._neutron
|
||||
|
||||
@exception.wrap_keystone_exception
|
||||
def cinder(self):
|
||||
if self._cinder:
|
||||
return self._cinder
|
||||
endpoint_type = self._get_client_option('cinder', 'endpoint_type')
|
||||
region_name = self._get_client_option('cinder', 'region_name')
|
||||
cinderclient_version = self._get_client_option('cinder', 'api_version')
|
||||
endpoint = self.url_for(service_type='block-storage',
|
||||
interface=endpoint_type,
|
||||
region_name=region_name)
|
||||
args = {
|
||||
'cacert': self._get_client_option('cinder', 'ca_file'),
|
||||
'insecure': self._get_client_option('cinder', 'insecure')
|
||||
}
|
||||
|
||||
session = self.keystone().session
|
||||
self._cinder = cinder_client.Client(cinderclient_version,
|
||||
session=session,
|
||||
endpoint_override=endpoint, **args)
|
||||
return self._cinder
|
||||
|
|
|
@ -279,6 +279,12 @@ class OperationInProgress(Invalid):
|
|||
"progress.")
|
||||
|
||||
|
||||
class VolumeTypeNotFound(ResourceNotFound):
|
||||
"""The code here changed to 400 according to the latest document."""
|
||||
message = _("Valid volume type could not be found.")
|
||||
code = 400
|
||||
|
||||
|
||||
class ImageNotFound(ResourceNotFound):
|
||||
"""The code here changed to 400 according to the latest document."""
|
||||
message = _("Image %(image_id)s could not be found.")
|
||||
|
|
|
@ -28,12 +28,53 @@ cinder_opts = [
|
|||
help=_('The default docker volume_type to use for volumes '
|
||||
'used for docker storage. To use the cinder volumes '
|
||||
'for docker storage, you need to select a default '
|
||||
'value.'))]
|
||||
'value. Otherwise, Magnum will select random one from '
|
||||
'Cinder volume type list.')),
|
||||
cfg.StrOpt('default_etcd_volume_type',
|
||||
default='',
|
||||
help=_('The default etcd volume_type to use for volumes '
|
||||
'used for etcd storage. To use the cinder volumes '
|
||||
'for etcd storage, you need to select a default '
|
||||
'value. Otherwise, Magnum will select random one from '
|
||||
'Cinder volume type list.')),
|
||||
cfg.StrOpt('default_boot_volume_type',
|
||||
default='',
|
||||
help=_('The default boot volume_type to use for volumes '
|
||||
'used for VM of COE. To use the cinder volumes '
|
||||
'for VM of COE, you need to select a default '
|
||||
'value. Otherwise, Magnum will select random one from '
|
||||
'Cinder volume type list.')),
|
||||
cfg.IntOpt('default_boot_volume_size',
|
||||
default=20,
|
||||
help=_('The default volume size to use for volumes '
|
||||
'used for VM of COE.'))
|
||||
]
|
||||
|
||||
cinder_client_opts = [
|
||||
cfg.StrOpt('region_name',
|
||||
help=_('Region in Identity service catalog to use for '
|
||||
'communication with the OpenStack service.'))]
|
||||
'communication with the OpenStack service.')),
|
||||
cfg.StrOpt('endpoint_type',
|
||||
default='publicURL',
|
||||
help=_('Type of endpoint in Identity service catalog to use '
|
||||
'for communication with the OpenStack service.')),
|
||||
cfg.StrOpt('api_version',
|
||||
default='2',
|
||||
help=_('Version of Cinder API to use in cinderclient.'))
|
||||
]
|
||||
|
||||
common_security_opts = [
|
||||
cfg.StrOpt('ca_file',
|
||||
help=_('Optional CA cert file to use in SSL connections.')),
|
||||
cfg.StrOpt('cert_file',
|
||||
help=_('Optional PEM-formatted certificate chain file.')),
|
||||
cfg.StrOpt('key_file',
|
||||
help=_('Optional PEM-formatted file that contains the '
|
||||
'private key.')),
|
||||
cfg.BoolOpt('insecure',
|
||||
default=False,
|
||||
help=_("If set, then the server's certificate will not "
|
||||
"be verified."))]
|
||||
|
||||
|
||||
def register_opts(conf):
|
||||
|
@ -41,6 +82,7 @@ def register_opts(conf):
|
|||
conf.register_group(cinder_client_group)
|
||||
conf.register_opts(cinder_opts, group=cinder_group)
|
||||
conf.register_opts(cinder_client_opts, group=cinder_client_group)
|
||||
conf.register_opts(common_security_opts, group=cinder_client_group)
|
||||
|
||||
|
||||
def list_opts():
|
||||
|
|
|
@ -15,6 +15,7 @@ import json
|
|||
from oslo_log import log as logging
|
||||
from oslo_utils import strutils
|
||||
|
||||
from magnum.common import cinder
|
||||
from magnum.common import exception
|
||||
from magnum.common.x509 import operations as x509
|
||||
from magnum.conductor.handlers.common import cert_manager
|
||||
|
@ -91,11 +92,7 @@ class K8sFedoraTemplateDefinition(k8s_template_def.K8sTemplateDefinition):
|
|||
osc = self.get_osc(context)
|
||||
extra_params['region_name'] = osc.cinder_region_name()
|
||||
|
||||
# set docker_volume_type
|
||||
# use the configuration default if None provided
|
||||
docker_volume_type = cluster.labels.get(
|
||||
'docker_volume_type', CONF.cinder.default_docker_volume_type)
|
||||
extra_params['docker_volume_type'] = docker_volume_type
|
||||
self._set_volumes(context, cluster, extra_params)
|
||||
|
||||
extra_params['nodes_affinity_policy'] = \
|
||||
CONF.cluster.nodes_affinity_policy
|
||||
|
@ -163,6 +160,7 @@ class K8sFedoraTemplateDefinition(k8s_template_def.K8sTemplateDefinition):
|
|||
|
||||
self._set_cert_manager_params(cluster, extra_params)
|
||||
self._get_keystone_auth_default_policy(extra_params)
|
||||
self._set_volumes(context, cluster, extra_params)
|
||||
|
||||
return super(K8sFedoraTemplateDefinition,
|
||||
self).get_params(context, cluster_template, cluster,
|
||||
|
@ -214,6 +212,28 @@ class K8sFedoraTemplateDefinition(k8s_template_def.K8sTemplateDefinition):
|
|||
.replace("$PROJECT_ID", extra_params["project_id"])
|
||||
extra_params["keystone_auth_default_policy"] = washed_policy
|
||||
|
||||
def _set_volumes(self, context, cluster, extra_params):
|
||||
# set docker_volume_type
|
||||
docker_volume_type = cluster.labels.get(
|
||||
'docker_volume_type',
|
||||
cinder.get_default_docker_volume_type(context))
|
||||
extra_params['docker_volume_type'] = docker_volume_type
|
||||
|
||||
# set etcd_volume_type
|
||||
etcd_volume_type = cluster.labels.get(
|
||||
'etcd_volume_type', cinder.get_default_etcd_volume_type(context))
|
||||
extra_params['etcd_volume_type'] = etcd_volume_type
|
||||
|
||||
# set boot_volume_type
|
||||
boot_volume_type = cluster.labels.get(
|
||||
'boot_volume_type', cinder.get_default_boot_volume_type(context))
|
||||
extra_params['boot_volume_type'] = boot_volume_type
|
||||
|
||||
# set boot_volume_size
|
||||
boot_volume_size = cluster.labels.get(
|
||||
'boot_volume_size', CONF.cinder.default_boot_volume_size)
|
||||
extra_params['boot_volume_size'] = boot_volume_size
|
||||
|
||||
def get_env_files(self, cluster_template, cluster):
|
||||
env_files = []
|
||||
|
||||
|
@ -223,5 +243,6 @@ class K8sFedoraTemplateDefinition(k8s_template_def.K8sTemplateDefinition):
|
|||
template_def.add_volume_env_file(env_files, cluster)
|
||||
template_def.add_lb_env_file(env_files, cluster_template)
|
||||
template_def.add_fip_env_file(env_files, cluster_template, cluster)
|
||||
template_def.add_server_env_file(env_files, cluster)
|
||||
|
||||
return env_files
|
||||
|
|
|
@ -530,3 +530,13 @@ def add_priv_net_env_file(env_files, cluster_template, cluster):
|
|||
env_files.append(COMMON_ENV_PATH + 'no_private_network.yaml')
|
||||
else:
|
||||
env_files.append(COMMON_ENV_PATH + 'with_private_network.yaml')
|
||||
|
||||
|
||||
def add_server_env_file(env_files, cluster):
|
||||
try:
|
||||
boot_volume_size = int(cluster.labels.get(
|
||||
'boot_volume_size',
|
||||
CONF.cinder.default_boot_volume_size
|
||||
))
|
||||
except ValueError:
|
||||
boot_volume_size = 0
|
||||
|
|
|
@ -1,4 +1,4 @@
|
|||
heat_template_version: 2014-10-16
|
||||
heat_template_version: 2015-04-30
|
||||
|
||||
description: >
|
||||
This template will boot a Kubernetes cluster with one or more
|
||||
|
@ -143,12 +143,27 @@ parameters:
|
|||
constraints:
|
||||
- allowed_values: ["true", "false"]
|
||||
|
||||
boot_volume_size:
|
||||
type: number
|
||||
description: >
|
||||
size of the cinder boot volume for nodes root volume
|
||||
|
||||
boot_volume_type:
|
||||
type: string
|
||||
description: >
|
||||
type of the cinder boot volume for nodes root volume
|
||||
|
||||
etcd_volume_size:
|
||||
type: number
|
||||
description: >
|
||||
size of the cinder volume for etcd storage
|
||||
default: 0
|
||||
|
||||
etcd_volume_type:
|
||||
type: string
|
||||
description: >
|
||||
type of a cinder volume for etcd storage
|
||||
|
||||
docker_volume_size:
|
||||
type: number
|
||||
description: >
|
||||
|
@ -878,7 +893,10 @@ resources:
|
|||
master_flavor: {get_param: master_flavor}
|
||||
external_network: {get_param: external_network}
|
||||
kube_allow_priv: {get_param: kube_allow_priv}
|
||||
boot_volume_size: {get_param: boot_volume_size}
|
||||
boot_volume_type: {get_param: boot_volume_type}
|
||||
etcd_volume_size: {get_param: etcd_volume_size}
|
||||
etcd_volume_type: {get_param: etcd_volume_type}
|
||||
docker_volume_size: {get_param: docker_volume_size}
|
||||
docker_volume_type: {get_param: docker_volume_type}
|
||||
docker_storage_driver: {get_param: docker_storage_driver}
|
||||
|
@ -1062,6 +1080,8 @@ resources:
|
|||
etcd_server_ip: {get_attr: [etcd_address_lb_switch, private_ip]}
|
||||
external_network: {get_param: external_network}
|
||||
kube_allow_priv: {get_param: kube_allow_priv}
|
||||
boot_volume_size: {get_param: boot_volume_size}
|
||||
boot_volume_type: {get_param: boot_volume_type}
|
||||
docker_volume_size: {get_param: docker_volume_size}
|
||||
docker_volume_type: {get_param: docker_volume_type}
|
||||
docker_storage_driver: {get_param: docker_storage_driver}
|
||||
|
|
|
@ -1,10 +1,19 @@
|
|||
heat_template_version: 2014-10-16
|
||||
heat_template_version: queens
|
||||
|
||||
description: >
|
||||
This is a nested stack that defines a single Kubernetes master, This stack is
|
||||
included by an ResourceGroup resource in the parent template
|
||||
(kubecluster.yaml).
|
||||
|
||||
conditions:
|
||||
bfv:
|
||||
not:
|
||||
equals:
|
||||
- get_param: boot_volume_size
|
||||
- 0
|
||||
withoutbfv:
|
||||
not: bfv
|
||||
|
||||
parameters:
|
||||
|
||||
name:
|
||||
|
@ -39,11 +48,27 @@ parameters:
|
|||
constraints:
|
||||
- allowed_values: ["true", "false"]
|
||||
|
||||
boot_volume_size:
|
||||
type: number
|
||||
description: >
|
||||
size of the cinder boot volume for nodes root volume
|
||||
default: 0
|
||||
|
||||
boot_volume_type:
|
||||
type: string
|
||||
description: >
|
||||
type of the cinder boot volume for nodes root volume
|
||||
|
||||
etcd_volume_size:
|
||||
type: number
|
||||
description: >
|
||||
size of a cinder volume to allocate for etcd storage
|
||||
|
||||
etcd_volume_type:
|
||||
type: string
|
||||
description: >
|
||||
type of a cinder volume to allocate for etcd storage
|
||||
|
||||
docker_volume_size:
|
||||
type: number
|
||||
description: >
|
||||
|
@ -676,18 +701,27 @@ resources:
|
|||
properties:
|
||||
signal_transport: HEAT_SIGNAL
|
||||
config: {get_resource: master_config}
|
||||
server: {get_resource: kube-master}
|
||||
server: {if: ["bfv", {get_resource: kube-master-bfv}, {get_resource: kube-master}]}
|
||||
actions: ['CREATE']
|
||||
|
||||
######################################################################
|
||||
#
|
||||
# a single kubernetes master.
|
||||
#
|
||||
kube_node_volume:
|
||||
condition: bfv
|
||||
type: OS::Cinder::Volume
|
||||
properties:
|
||||
image: {get_param: server_image}
|
||||
size: {get_param: boot_volume_size}
|
||||
volume_type: {get_param: boot_volume_type}
|
||||
|
||||
|
||||
# do NOT use "_" (underscore) in the Nova server name
|
||||
# it creates a mismatch between the generated Nova name and its hostname
|
||||
# which can lead to weird problems
|
||||
kube-master:
|
||||
condition: withoutbfv
|
||||
type: OS::Nova::Server
|
||||
properties:
|
||||
name: {get_param: name}
|
||||
|
@ -702,6 +736,25 @@ resources:
|
|||
scheduler_hints: { group: { get_param: nodes_server_group_id }}
|
||||
availability_zone: {get_param: availability_zone}
|
||||
|
||||
kube-master-bfv:
|
||||
condition: bfv
|
||||
type: OS::Nova::Server
|
||||
properties:
|
||||
name: {get_param: name}
|
||||
block_device_mapping_v2:
|
||||
- boot_index: 0
|
||||
volume_id: {get_resource: kube_node_volume}
|
||||
delete_on_termination: true
|
||||
flavor: {get_param: master_flavor}
|
||||
key_name: {get_param: ssh_key_name}
|
||||
user_data_format: SOFTWARE_CONFIG
|
||||
software_config_transport: POLL_SERVER_HEAT
|
||||
user_data: {get_resource: agent_config}
|
||||
networks:
|
||||
- port: {get_resource: kube_master_eth0}
|
||||
scheduler_hints: { group: { get_param: nodes_server_group_id }}
|
||||
availability_zone: {get_param: availability_zone}
|
||||
|
||||
kube_master_eth0:
|
||||
type: OS::Neutron::Port
|
||||
properties:
|
||||
|
@ -746,11 +799,12 @@ resources:
|
|||
type: Magnum::Optional::Etcd::Volume
|
||||
properties:
|
||||
size: {get_param: etcd_volume_size}
|
||||
volume_type: {get_param: etcd_volume_type}
|
||||
|
||||
etcd_volume_attach:
|
||||
type: Magnum::Optional::Etcd::VolumeAttachment
|
||||
properties:
|
||||
instance_uuid: {get_resource: kube-master}
|
||||
instance_uuid: {if: ["bfv", {get_resource: kube-master-bfv}, {get_resource: kube-master}]}
|
||||
volume_id: {get_resource: etcd_volume}
|
||||
mountpoint: /dev/vdc
|
||||
|
||||
|
@ -769,7 +823,7 @@ resources:
|
|||
docker_volume_attach:
|
||||
type: Magnum::Optional::Cinder::VolumeAttachment
|
||||
properties:
|
||||
instance_uuid: {get_resource: kube-master}
|
||||
instance_uuid: {if: ["bfv", {get_resource: kube-master-bfv}, {get_resource: kube-master}]}
|
||||
volume_id: {get_resource: docker_volume}
|
||||
mountpoint: /dev/vdb
|
||||
|
||||
|
@ -787,7 +841,7 @@ resources:
|
|||
properties:
|
||||
signal_transport: HEAT_SIGNAL
|
||||
config: {get_resource: upgrade_kubernetes}
|
||||
server: {get_resource: kube-master}
|
||||
server: {if: ["bfv", {get_resource: kube-master-bfv}, {get_resource: kube-master}]}
|
||||
actions: ['UPDATE']
|
||||
input_values:
|
||||
kube_tag_input: {get_param: kube_tag}
|
||||
|
@ -795,7 +849,7 @@ resources:
|
|||
outputs:
|
||||
|
||||
OS::stack_id:
|
||||
value: { get_resource: kube-master }
|
||||
value: {if: ["bfv", {get_resource: kube-master-bfv}, {get_resource: kube-master}]}
|
||||
|
||||
kube_master_ip:
|
||||
value: {get_attr: [kube_master_eth0, fixed_ips, 0, ip_address]}
|
||||
|
|
|
@ -1,10 +1,19 @@
|
|||
heat_template_version: 2014-10-16
|
||||
heat_template_version: queens
|
||||
|
||||
description: >
|
||||
This is a nested stack that defines a single Kubernetes minion, This stack is
|
||||
included by an AutoScalingGroup resource in the parent template
|
||||
(kubecluster.yaml).
|
||||
|
||||
conditions:
|
||||
bfv:
|
||||
not:
|
||||
equals:
|
||||
- get_param: boot_volume_size
|
||||
- 0
|
||||
withoutbfv:
|
||||
not: bfv
|
||||
|
||||
parameters:
|
||||
|
||||
name:
|
||||
|
@ -34,6 +43,16 @@ parameters:
|
|||
constraints:
|
||||
- allowed_values: ["true", "false"]
|
||||
|
||||
boot_volume_size:
|
||||
type: number
|
||||
description: >
|
||||
size of the cinder boot volume
|
||||
|
||||
boot_volume_type:
|
||||
type: string
|
||||
description: >
|
||||
type of the cinder boot volume
|
||||
|
||||
docker_volume_size:
|
||||
type: number
|
||||
description: >
|
||||
|
@ -399,27 +418,55 @@ resources:
|
|||
properties:
|
||||
signal_transport: HEAT_SIGNAL
|
||||
config: {get_resource: node_config}
|
||||
server: {get_resource: kube-minion}
|
||||
server: {if: ["bfv", {get_resource: kube-minion-bfv}, {get_resource: kube-minion}]}
|
||||
actions: ['CREATE']
|
||||
|
||||
######################################################################
|
||||
#
|
||||
# a single kubernetes minion.
|
||||
#
|
||||
kube_node_volume:
|
||||
condition: bfv
|
||||
type: OS::Cinder::Volume
|
||||
properties:
|
||||
image: {get_param: server_image}
|
||||
size: {get_param: boot_volume_size}
|
||||
volume_type: {get_param: boot_volume_type}
|
||||
|
||||
|
||||
# do NOT use "_" (underscore) in the Nova server name
|
||||
# it creates a mismatch between the generated Nova name and its hostname
|
||||
# which can lead to weird problems
|
||||
kube-minion:
|
||||
condition: withoutbfv
|
||||
type: OS::Nova::Server
|
||||
properties:
|
||||
name: {get_param: name}
|
||||
image: {get_param: server_image}
|
||||
flavor: {get_param: minion_flavor}
|
||||
key_name: {get_param: ssh_key_name}
|
||||
user_data: {get_resource: agent_config}
|
||||
user_data_format: SOFTWARE_CONFIG
|
||||
software_config_transport: POLL_SERVER_HEAT
|
||||
user_data: {get_resource: agent_config}
|
||||
networks:
|
||||
- port: {get_resource: kube_minion_eth0}
|
||||
scheduler_hints: { group: { get_param: nodes_server_group_id }}
|
||||
availability_zone: {get_param: availability_zone}
|
||||
|
||||
kube-minion-bfv:
|
||||
condition: bfv
|
||||
type: OS::Nova::Server
|
||||
properties:
|
||||
name: {get_param: name}
|
||||
block_device_mapping_v2:
|
||||
- boot_index: 0
|
||||
volume_id: {get_resource: kube_node_volume}
|
||||
delete_on_termination: true
|
||||
flavor: {get_param: minion_flavor}
|
||||
key_name: {get_param: ssh_key_name}
|
||||
user_data_format: SOFTWARE_CONFIG
|
||||
software_config_transport: POLL_SERVER_HEAT
|
||||
user_data: {get_resource: agent_config}
|
||||
networks:
|
||||
- port: {get_resource: kube_minion_eth0}
|
||||
scheduler_hints: { group: { get_param: nodes_server_group_id }}
|
||||
|
@ -458,7 +505,7 @@ resources:
|
|||
docker_volume_attach:
|
||||
type: Magnum::Optional::Cinder::VolumeAttachment
|
||||
properties:
|
||||
instance_uuid: {get_resource: kube-minion}
|
||||
instance_uuid: {if: ["bfv", {get_resource: kube-minion-bfv}, {get_resource: kube-minion}]}
|
||||
volume_id: {get_resource: docker_volume}
|
||||
mountpoint: /dev/vdb
|
||||
|
||||
|
@ -476,7 +523,7 @@ resources:
|
|||
properties:
|
||||
signal_transport: HEAT_SIGNAL
|
||||
config: {get_resource: upgrade_kubernetes}
|
||||
server: {get_resource: kube-minion}
|
||||
server: {if: ["bfv", {get_resource: kube-minion-bfv}, {get_resource: kube-minion}]}
|
||||
actions: ['UPDATE']
|
||||
input_values:
|
||||
kube_tag_input: {get_param: kube_tag}
|
||||
|
@ -505,6 +552,6 @@ outputs:
|
|||
######################################################################
|
||||
|
||||
OS::stack_id:
|
||||
value: { get_resource: kube-minion }
|
||||
value: {if: ["bfv", {get_resource: kube-minion-bfv}, {get_resource: kube-minion}]}
|
||||
description: >
|
||||
This is the Nova server id of the node.
|
||||
|
|
|
@ -114,7 +114,8 @@ class TestClusterConductorWithK8s(base.TestCase):
|
|||
'kubescheduler_options': '--kubescheduler',
|
||||
'kubeproxy_options': '--kubeproxy',
|
||||
'influx_grafana_dashboard_enabled': 'True',
|
||||
'service_cluster_ip_range': '10.254.0.0/16'},
|
||||
'service_cluster_ip_range': '10.254.0.0/16',
|
||||
'boot_volume_size': '60'},
|
||||
'master_flavor_id': 'master_flavor_id',
|
||||
'flavor_id': 'flavor_id',
|
||||
'project_id': 'project_id',
|
||||
|
@ -175,6 +176,10 @@ class TestClusterConductorWithK8s(base.TestCase):
|
|||
self.mock_enable_octavia = octavia_patcher.start()
|
||||
self.mock_enable_octavia.return_value = False
|
||||
self.addCleanup(octavia_patcher.stop)
|
||||
CONF.set_override('default_boot_volume_type',
|
||||
'lvmdriver-1', group='cinder')
|
||||
CONF.set_override('default_etcd_volume_type',
|
||||
'lvmdriver-1', group='cinder')
|
||||
|
||||
@patch('requests.get')
|
||||
@patch('magnum.objects.ClusterTemplate.get_by_uuid')
|
||||
|
@ -262,6 +267,8 @@ class TestClusterConductorWithK8s(base.TestCase):
|
|||
'kube_dashboard_enabled': 'True',
|
||||
'influx_grafana_dashboard_enabled': 'True',
|
||||
'docker_volume_type': 'lvmdriver-1',
|
||||
'boot_volume_type': 'lvmdriver-1',
|
||||
'etcd_volume_type': 'lvmdriver-1',
|
||||
'etcd_volume_size': None,
|
||||
'availability_zone': 'az_1',
|
||||
'cert_manager_api': 'False',
|
||||
|
@ -349,7 +356,10 @@ class TestClusterConductorWithK8s(base.TestCase):
|
|||
'max_node_count': 2,
|
||||
'master_image': 'image_id',
|
||||
'minion_image': 'image_id',
|
||||
'keystone_auth_default_policy': self.keystone_auth_default_policy
|
||||
'keystone_auth_default_policy': self.keystone_auth_default_policy,
|
||||
'boot_volume_size': '60',
|
||||
'boot_volume_type': 'lvmdriver-1',
|
||||
'etcd_volume_type': 'lvmdriver-1'
|
||||
}
|
||||
if missing_attr is not None:
|
||||
expected.pop(mapping[missing_attr], None)
|
||||
|
@ -369,6 +379,7 @@ class TestClusterConductorWithK8s(base.TestCase):
|
|||
'../../common/templates/environments/no_master_lb.yaml',
|
||||
'../../common/templates/environments/disable_floating_ip.yaml',
|
||||
'../../common/templates/environments/disable_lb_floating_ip.yaml',
|
||||
'../../common/templates/environments/server_with_root_volume.yaml'
|
||||
],
|
||||
env_files)
|
||||
|
||||
|
@ -489,7 +500,10 @@ class TestClusterConductorWithK8s(base.TestCase):
|
|||
'max_node_count': 2,
|
||||
'master_image': 'image_id',
|
||||
'minion_image': 'image_id',
|
||||
'keystone_auth_default_policy': self.keystone_auth_default_policy
|
||||
'keystone_auth_default_policy': self.keystone_auth_default_policy,
|
||||
'boot_volume_size': '60',
|
||||
'boot_volume_type': 'lvmdriver-1',
|
||||
'etcd_volume_type': 'lvmdriver-1'
|
||||
}
|
||||
|
||||
self.assertEqual(expected, definition)
|
||||
|
@ -499,7 +513,8 @@ class TestClusterConductorWithK8s(base.TestCase):
|
|||
'../../common/templates/environments/with_volume.yaml',
|
||||
'../../common/templates/environments/no_master_lb.yaml',
|
||||
'../../common/templates/environments/disable_floating_ip.yaml',
|
||||
'../../common/templates/environments/disable_lb_floating_ip.yaml'
|
||||
'../../common/templates/environments/disable_lb_floating_ip.yaml',
|
||||
'../../common/templates/environments/server_with_root_volume.yaml'
|
||||
],
|
||||
env_files)
|
||||
|
||||
|
@ -611,7 +626,10 @@ class TestClusterConductorWithK8s(base.TestCase):
|
|||
'max_node_count': 2,
|
||||
'master_image': None,
|
||||
'minion_image': None,
|
||||
'keystone_auth_default_policy': self.keystone_auth_default_policy
|
||||
'keystone_auth_default_policy': self.keystone_auth_default_policy,
|
||||
'boot_volume_size': '60',
|
||||
'boot_volume_type': 'lvmdriver-1',
|
||||
'etcd_volume_type': 'lvmdriver-1'
|
||||
}
|
||||
self.assertEqual(expected, definition)
|
||||
self.assertEqual(
|
||||
|
@ -620,7 +638,8 @@ class TestClusterConductorWithK8s(base.TestCase):
|
|||
'../../common/templates/environments/with_volume.yaml',
|
||||
'../../common/templates/environments/no_master_lb.yaml',
|
||||
'../../common/templates/environments/disable_floating_ip.yaml',
|
||||
'../../common/templates/environments/disable_lb_floating_ip.yaml'
|
||||
'../../common/templates/environments/disable_lb_floating_ip.yaml',
|
||||
'../../common/templates/environments/server_with_root_volume.yaml'
|
||||
],
|
||||
env_files)
|
||||
|
||||
|
@ -1044,7 +1063,10 @@ class TestClusterConductorWithK8s(base.TestCase):
|
|||
'max_node_count': 2,
|
||||
'master_image': 'image_id',
|
||||
'minion_image': 'image_id',
|
||||
'keystone_auth_default_policy': self.keystone_auth_default_policy
|
||||
'keystone_auth_default_policy': self.keystone_auth_default_policy,
|
||||
'boot_volume_size': '60',
|
||||
'boot_volume_type': 'lvmdriver-1',
|
||||
'etcd_volume_type': 'lvmdriver-1'
|
||||
}
|
||||
self.assertEqual(expected, definition)
|
||||
self.assertEqual(
|
||||
|
@ -1054,6 +1076,7 @@ class TestClusterConductorWithK8s(base.TestCase):
|
|||
'../../common/templates/environments/no_master_lb.yaml',
|
||||
'../../common/templates/environments/disable_floating_ip.yaml',
|
||||
'../../common/templates/environments/disable_lb_floating_ip.yaml',
|
||||
'../../common/templates/environments/server_with_root_volume.yaml'
|
||||
],
|
||||
env_files)
|
||||
reqget.assert_called_once_with('http://etcd/test?size=1')
|
||||
|
|
|
@ -323,6 +323,48 @@ class TemplateDefinitionTestCase(base.TestCase):
|
|||
self.assertEqual(definition.get_scale_params(mock_context,
|
||||
mock_cluster), {})
|
||||
|
||||
def test_add_server_env_file(self):
|
||||
CONF.set_override('default_boot_volume_size',
|
||||
10,
|
||||
group='cinder')
|
||||
mock_cluster = mock.MagicMock(labels={"boot_volume_size": 5})
|
||||
env_files = []
|
||||
|
||||
cmn_tdef.add_server_env_file(env_files, mock_cluster)
|
||||
|
||||
self.assertEqual(
|
||||
[cmn_tdef.COMMON_ENV_PATH + 'server_with_root_volume.yaml'],
|
||||
env_files
|
||||
)
|
||||
|
||||
def test_add_server_env_file_no_cluster_label(self):
|
||||
CONF.set_override('default_boot_volume_size',
|
||||
10,
|
||||
group='cinder')
|
||||
mock_cluster = mock.MagicMock(labels={})
|
||||
env_files = []
|
||||
|
||||
cmn_tdef.add_server_env_file(env_files, mock_cluster)
|
||||
|
||||
self.assertEqual(
|
||||
[cmn_tdef.COMMON_ENV_PATH + 'server_with_root_volume.yaml'],
|
||||
env_files
|
||||
)
|
||||
|
||||
def test_add_server_env_file_no_root_volume(self):
|
||||
CONF.set_override('default_boot_volume_size',
|
||||
0,
|
||||
group='cinder')
|
||||
mock_cluster = mock.MagicMock(labels={})
|
||||
env_files = []
|
||||
|
||||
cmn_tdef.add_server_env_file(env_files, mock_cluster)
|
||||
|
||||
self.assertEqual(
|
||||
[cmn_tdef.COMMON_ENV_PATH + 'server_with_image.yaml'],
|
||||
env_files
|
||||
)
|
||||
|
||||
|
||||
@six.add_metaclass(abc.ABCMeta)
|
||||
class BaseK8sTemplateDefinitionTestCase(base.TestCase):
|
||||
|
@ -479,6 +521,8 @@ class AtomicK8sTemplateDefinitionTestCase(BaseK8sTemplateDefinitionTestCase):
|
|||
'influx_grafana_dashboard_enabled')
|
||||
docker_volume_type = mock_cluster.labels.get(
|
||||
'docker_volume_type')
|
||||
boot_volume_size = mock_cluster.labels.get(
|
||||
'boot_volume_size')
|
||||
etcd_volume_size = mock_cluster.labels.get(
|
||||
'etcd_volume_size')
|
||||
kube_tag = mock_cluster.labels.get('kube_tag')
|
||||
|
@ -568,6 +612,9 @@ class AtomicK8sTemplateDefinitionTestCase(BaseK8sTemplateDefinitionTestCase):
|
|||
npd_enabled = mock_cluster.labels.get('npd_enabled')
|
||||
master_image = mock_cluster_template.image_id
|
||||
minion_image = mock_cluster_template.image_id
|
||||
boot_volume_size = mock_cluster.labels.get('boot_volume_size')
|
||||
boot_volume_type = mock_cluster.labels.get('boot_volume_type')
|
||||
etcd_volume_type = mock_cluster.labels.get('etcd_volume_type')
|
||||
|
||||
k8s_def = k8sa_tdef.AtomicK8sTemplateDefinition()
|
||||
|
||||
|
@ -587,6 +634,7 @@ class AtomicK8sTemplateDefinitionTestCase(BaseK8sTemplateDefinitionTestCase):
|
|||
'influx_grafana_dashboard_enabled':
|
||||
influx_grafana_dashboard_enabled,
|
||||
'docker_volume_type': docker_volume_type,
|
||||
'boot_volume_size': boot_volume_size,
|
||||
'etcd_volume_size': etcd_volume_size,
|
||||
'kubelet_options': kubelet_options,
|
||||
'kubeapi_options': kubeapi_options,
|
||||
|
@ -650,6 +698,9 @@ class AtomicK8sTemplateDefinitionTestCase(BaseK8sTemplateDefinitionTestCase):
|
|||
'kube_version': kube_tag,
|
||||
'master_kube_tag': kube_tag,
|
||||
'minion_kube_tag': kube_tag,
|
||||
'boot_volume_size': boot_volume_size,
|
||||
'boot_volume_type': boot_volume_type,
|
||||
'etcd_volume_type': etcd_volume_type
|
||||
}}
|
||||
mock_get_params.assert_called_once_with(mock_context,
|
||||
mock_cluster_template,
|
||||
|
@ -908,6 +959,8 @@ class AtomicK8sTemplateDefinitionTestCase(BaseK8sTemplateDefinitionTestCase):
|
|||
'influx_grafana_dashboard_enabled')
|
||||
docker_volume_type = mock_cluster.labels.get(
|
||||
'docker_volume_type')
|
||||
boot_volume_size = mock_cluster.labels.get(
|
||||
'boot_volume_size')
|
||||
etcd_volume_size = mock_cluster.labels.get(
|
||||
'etcd_volume_size')
|
||||
kube_tag = mock_cluster.labels.get('kube_tag')
|
||||
|
@ -997,6 +1050,9 @@ class AtomicK8sTemplateDefinitionTestCase(BaseK8sTemplateDefinitionTestCase):
|
|||
npd_enabled = mock_cluster.labels.get('npd_enabled')
|
||||
master_image = mock_cluster_template.image_id
|
||||
minion_image = mock_cluster_template.image_id
|
||||
boot_volume_size = mock_cluster.labels.get('boot_volume_size')
|
||||
boot_volume_type = mock_cluster.labels.get('boot_volume_type')
|
||||
etcd_volume_type = mock_cluster.labels.get('etcd_volume_type')
|
||||
|
||||
k8s_def = k8sa_tdef.AtomicK8sTemplateDefinition()
|
||||
|
||||
|
@ -1017,6 +1073,7 @@ class AtomicK8sTemplateDefinitionTestCase(BaseK8sTemplateDefinitionTestCase):
|
|||
'influx_grafana_dashboard_enabled':
|
||||
influx_grafana_dashboard_enabled,
|
||||
'docker_volume_type': docker_volume_type,
|
||||
'boot_volume_size': boot_volume_size,
|
||||
'etcd_volume_size': etcd_volume_size,
|
||||
'kubelet_options': kubelet_options,
|
||||
'kubeapi_options': kubeapi_options,
|
||||
|
@ -1081,6 +1138,9 @@ class AtomicK8sTemplateDefinitionTestCase(BaseK8sTemplateDefinitionTestCase):
|
|||
'kube_version': kube_tag,
|
||||
'master_kube_tag': kube_tag,
|
||||
'minion_kube_tag': kube_tag,
|
||||
'boot_volume_size': boot_volume_size,
|
||||
'boot_volume_type': boot_volume_type,
|
||||
'etcd_volume_type': etcd_volume_type
|
||||
}}
|
||||
mock_get_params.assert_called_once_with(mock_context,
|
||||
mock_cluster_template,
|
||||
|
|
|
@ -0,0 +1,13 @@
|
|||
---
|
||||
features:
|
||||
- |
|
||||
Support boot from volume for Kubernetes all nodes (master and worker)
|
||||
so that user can create a big size root volume, which could be more
|
||||
flexible than using docker_volume_size. And user can specify the
|
||||
volume type so that user can leverage high performance storage, e.g.
|
||||
NVMe etc. And a new label etcd_volme_type is added as well so that
|
||||
user can set volume type for etcd volume. If the boot_volume_type
|
||||
or etcd_volume_type are not passed by labels, Magnum will try to
|
||||
read them from config option default_boot_volume_type and
|
||||
default_etcd_volume_type. A random volume type from Cinder will
|
||||
be used if those options are not set.
|
|
@ -42,6 +42,7 @@ pbr!=2.1.0,>=2.0.0 # Apache-2.0
|
|||
pecan!=1.0.2,!=1.0.3,!=1.0.4,!=1.2,>=1.0.0 # BSD
|
||||
pycadf!=2.0.0,>=1.1.0 # Apache-2.0
|
||||
python-barbicanclient>=4.5.2 # Apache-2.0
|
||||
python-cinderclient>=2.2.0 # Apache-2.0
|
||||
python-glanceclient>=2.8.0 # Apache-2.0
|
||||
python-heatclient>=1.10.0 # Apache-2.0
|
||||
python-neutronclient>=6.7.0 # Apache-2.0
|
||||
|
|
Loading…
Reference in New Issue