You can not select more than 25 topics
Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.
636 lines
20 KiB
636 lines
20 KiB
heat_template_version: queens |
|
|
|
description: > |
|
This is a nested stack that defines a single Kubernetes minion, This stack is |
|
included by an AutoScalingGroup resource in the parent template |
|
(kubecluster.yaml). |
|
|
|
parameters: |
|
|
|
name: |
|
type: string |
|
description: server name |
|
|
|
server_image: |
|
type: string |
|
description: glance image used to boot the server |
|
|
|
minion_flavor: |
|
type: string |
|
description: flavor to use when booting the server |
|
|
|
nodegroup_role: |
|
type: string |
|
description: the role of the nodegroup |
|
|
|
nodegroup_name: |
|
type: string |
|
description: the name of the nodegroup where the node belongs |
|
|
|
ssh_key_name: |
|
type: string |
|
description: name of ssh key to be provisioned on our server |
|
|
|
external_network: |
|
type: string |
|
description: uuid/name of a network to use for floating ip addresses |
|
|
|
kube_allow_priv: |
|
type: string |
|
description: > |
|
whether or not kubernetes should permit privileged containers. |
|
constraints: |
|
- allowed_values: ["true", "false"] |
|
|
|
boot_volume_size: |
|
type: number |
|
description: > |
|
size of the cinder boot volume |
|
|
|
boot_volume_type: |
|
type: string |
|
description: > |
|
type of the cinder boot volume |
|
|
|
docker_volume_size: |
|
type: number |
|
description: > |
|
size of a cinder volume to allocate to docker for container/image |
|
storage |
|
|
|
docker_volume_type: |
|
type: string |
|
description: > |
|
type of a cinder volume to allocate to docker for container/image |
|
storage |
|
|
|
docker_storage_driver: |
|
type: string |
|
description: docker storage driver name |
|
default: "devicemapper" |
|
|
|
cgroup_driver: |
|
type: string |
|
description: > |
|
cgroup driver name that kubelet should use, ideally the same as |
|
the docker cgroup driver. |
|
default: "cgroupfs" |
|
|
|
tls_disabled: |
|
type: boolean |
|
description: whether or not to enable TLS |
|
|
|
verify_ca: |
|
type: boolean |
|
description: whether or not to validate certificate authority |
|
|
|
kubernetes_port: |
|
type: number |
|
description: > |
|
The port which are used by kube-apiserver to provide Kubernetes |
|
service. |
|
|
|
cluster_uuid: |
|
type: string |
|
description: identifier for the cluster this template is generating |
|
|
|
magnum_url: |
|
type: string |
|
description: endpoint to retrieve TLS certs from |
|
|
|
prometheus_monitoring: |
|
type: boolean |
|
description: > |
|
whether or not to have the node-exporter running on the node |
|
|
|
kube_master_ip: |
|
type: string |
|
description: IP address of the Kubernetes master server. |
|
|
|
etcd_server_ip: |
|
type: string |
|
description: IP address of the Etcd server. |
|
|
|
fixed_network: |
|
type: string |
|
description: Network from which to allocate fixed addresses. |
|
|
|
fixed_subnet: |
|
type: string |
|
description: Subnet from which to allocate fixed addresses. |
|
|
|
network_driver: |
|
type: string |
|
description: network driver to use for instantiating container networks |
|
|
|
flannel_network_cidr: |
|
type: string |
|
description: network range for flannel overlay network |
|
|
|
wait_condition_timeout: |
|
type: number |
|
description : > |
|
timeout for the Wait Conditions |
|
|
|
registry_enabled: |
|
type: boolean |
|
description: > |
|
Indicates whether the docker registry is enabled. |
|
|
|
registry_port: |
|
type: number |
|
description: port of registry service |
|
|
|
swift_region: |
|
type: string |
|
description: region of swift service |
|
|
|
registry_container: |
|
type: string |
|
description: > |
|
name of swift container which docker registry stores images in |
|
|
|
registry_insecure: |
|
type: boolean |
|
description: > |
|
indicates whether to skip TLS verification between registry and backend storage |
|
|
|
registry_chunksize: |
|
type: number |
|
description: > |
|
size fo the data segments for the swift dynamic large objects |
|
|
|
secgroup_kube_minion_id: |
|
type: string |
|
description: ID of the security group for kubernetes minion. |
|
|
|
volume_driver: |
|
type: string |
|
description: volume driver to use for container storage |
|
|
|
region_name: |
|
type: string |
|
description: A logically separate section of the cluster |
|
|
|
username: |
|
type: string |
|
description: > |
|
user account |
|
|
|
password: |
|
type: string |
|
description: > |
|
user password, not set in current implementation, only used to |
|
fill in for Kubernetes config file |
|
hidden: true |
|
|
|
http_proxy: |
|
type: string |
|
description: http proxy address for docker |
|
|
|
https_proxy: |
|
type: string |
|
description: https proxy address for docker |
|
|
|
no_proxy: |
|
type: string |
|
description: no proxies for docker |
|
|
|
hyperkube_prefix: |
|
type: string |
|
description: prefix to use for hyperkube images |
|
|
|
kube_tag: |
|
type: string |
|
description: tag of the k8s containers used to provision the kubernetes cluster |
|
|
|
kube_version: |
|
type: string |
|
description: version of kubernetes used for kubernetes cluster |
|
|
|
trustee_domain_id: |
|
type: string |
|
description: domain id of the trustee |
|
|
|
trustee_user_id: |
|
type: string |
|
description: user id of the trustee |
|
|
|
trustee_username: |
|
type: string |
|
description: username of the trustee |
|
|
|
trustee_password: |
|
type: string |
|
description: password of the trustee |
|
hidden: true |
|
|
|
trust_id: |
|
type: string |
|
description: id of the trust which is used by the trustee |
|
hidden: true |
|
|
|
auth_url: |
|
type: string |
|
description: > |
|
url for keystone, must be v2 since k8s backend only support v2 |
|
at this point |
|
|
|
insecure_registry_url: |
|
type: string |
|
description: insecure registry url |
|
|
|
container_infra_prefix: |
|
type: string |
|
description: > |
|
prefix of container images used in the cluster, kubernetes components, |
|
kubernetes-dashboard, coredns etc |
|
|
|
dns_service_ip: |
|
type: string |
|
description: > |
|
address used by Kubernetes DNS service |
|
|
|
dns_cluster_domain: |
|
type: string |
|
description: > |
|
domain name for cluster DNS |
|
|
|
openstack_ca: |
|
type: string |
|
description: The OpenStack CA certificate to install on the node. |
|
|
|
nodes_server_group_id: |
|
type: string |
|
description: ID of the server group for kubernetes cluster nodes. |
|
|
|
availability_zone: |
|
type: string |
|
description: > |
|
availability zone for master and nodes |
|
default: "" |
|
|
|
pods_network_cidr: |
|
type: string |
|
description: Configure the IP pool/range from which pod IPs will be chosen |
|
|
|
kubelet_options: |
|
type: string |
|
description: > |
|
additional options to be passed to the kubelet |
|
|
|
kubeproxy_options: |
|
type: string |
|
description: > |
|
additional options to be passed to the kube proxy |
|
|
|
octavia_enabled: |
|
type: boolean |
|
description: > |
|
whether or not to use Octavia for LoadBalancer type service. |
|
default: False |
|
|
|
cloud_provider_enabled: |
|
type: boolean |
|
description: Enable or disable the openstack kubernetes cloud provider |
|
|
|
heat_container_agent_tag: |
|
type: string |
|
description: tag of the heat_container_agent system container |
|
|
|
auto_healing_enabled: |
|
type: boolean |
|
description: > |
|
true if the auto healing feature should be enabled |
|
|
|
auto_healing_controller: |
|
type: string |
|
description: > |
|
The service to be deployed for auto-healing. |
|
default: "draino" |
|
|
|
npd_enabled: |
|
type: boolean |
|
description: > |
|
true if the npd service should be launched |
|
default: |
|
true |
|
|
|
ostree_remote: |
|
type: string |
|
description: The ostree remote branch to upgrade |
|
default: '' |
|
|
|
ostree_commit: |
|
type: string |
|
description: The ostree commit to deploy |
|
default: '' |
|
|
|
use_podman: |
|
type: boolean |
|
description: > |
|
if true, run system containers for kubernetes, etcd and heat-agent |
|
|
|
selinux_mode: |
|
type: string |
|
description: > |
|
Choose SELinux mode |
|
|
|
container_runtime: |
|
type: string |
|
description: The container runtime to install |
|
|
|
containerd_version: |
|
type: string |
|
description: The containerd version to download from https://storage.googleapis.com/cri-containerd-release/ |
|
|
|
containerd_tarball_url: |
|
type: string |
|
description: Url location of the containerd tarball. |
|
|
|
containerd_tarball_sha256: |
|
type: string |
|
description: sha256 of the target containerd tarball. |
|
|
|
conditions: |
|
|
|
image_based: {equals: [{get_param: boot_volume_size}, 0]} |
|
volume_based: |
|
not: |
|
equals: |
|
- get_param: boot_volume_size |
|
- 0 |
|
|
|
|
|
resources: |
|
|
|
agent_config: |
|
type: OS::Heat::SoftwareConfig |
|
properties: |
|
group: ungrouped |
|
config: |
|
list_join: |
|
- "\n" |
|
- |
|
- str_replace: |
|
template: {get_file: ../../common/templates/fragments/atomic-install-openstack-ca.sh} |
|
params: |
|
$OPENSTACK_CA: {get_param: openstack_ca} |
|
- str_replace: |
|
template: {get_file: ../../common/templates/kubernetes/fragments/start-container-agent.sh} |
|
params: |
|
$CONTAINER_INFRA_PREFIX: {get_param: container_infra_prefix} |
|
$HEAT_CONTAINER_AGENT_TAG: {get_param: heat_container_agent_tag} |
|
$USE_PODMAN: {get_param: use_podman} |
|
$HTTP_PROXY: {get_param: http_proxy} |
|
$HTTPS_PROXY: {get_param: https_proxy} |
|
$NO_PROXY: {get_param: no_proxy} |
|
- str_replace: |
|
template: {get_file: ../../common/templates/kubernetes/fragments/disable-selinux.sh} |
|
params: |
|
$SELINUX_MODE: {get_param: selinux_mode} |
|
|
|
###################################################################### |
|
# |
|
# software configs. these are components that are combined into |
|
# a multipart MIME user-data archive. |
|
# |
|
|
|
node_config: |
|
type: OS::Heat::SoftwareConfig |
|
properties: |
|
group: script |
|
config: |
|
list_join: |
|
- "\n" |
|
- |
|
- "#!/bin/bash" |
|
- str_replace: |
|
template: {get_file: ../../common/templates/kubernetes/fragments/write-heat-params.sh} |
|
params: |
|
$INSTANCE_NAME: {get_param: name} |
|
$PROMETHEUS_MONITORING: {get_param: prometheus_monitoring} |
|
$KUBE_ALLOW_PRIV: {get_param: kube_allow_priv} |
|
$KUBE_MASTER_IP: {get_param: kube_master_ip} |
|
$KUBE_API_PORT: {get_param: kubernetes_port} |
|
$KUBE_NODE_PUBLIC_IP: {get_attr: [kube_minion_floating, floating_ip_address]} |
|
$KUBE_NODE_IP: {get_attr: [kube_minion_eth0, fixed_ips, 0, ip_address]} |
|
$ETCD_SERVER_IP: {get_param: etcd_server_ip} |
|
$DOCKER_VOLUME: {get_resource: docker_volume} |
|
$DOCKER_VOLUME_SIZE: {get_param: docker_volume_size} |
|
$DOCKER_STORAGE_DRIVER: {get_param: docker_storage_driver} |
|
$CGROUP_DRIVER: {get_param: cgroup_driver} |
|
$NETWORK_DRIVER: {get_param: network_driver} |
|
$REGISTRY_ENABLED: {get_param: registry_enabled} |
|
$REGISTRY_PORT: {get_param: registry_port} |
|
$SWIFT_REGION: {get_param: swift_region} |
|
$REGISTRY_CONTAINER: {get_param: registry_container} |
|
$REGISTRY_INSECURE: {get_param: registry_insecure} |
|
$REGISTRY_CHUNKSIZE: {get_param: registry_chunksize} |
|
$TLS_DISABLED: {get_param: tls_disabled} |
|
$VERIFY_CA: {get_param: verify_ca} |
|
$CLUSTER_UUID: {get_param: cluster_uuid} |
|
$MAGNUM_URL: {get_param: magnum_url} |
|
$USERNAME: {get_param: username} |
|
$PASSWORD: {get_param: password} |
|
$VOLUME_DRIVER: {get_param: volume_driver} |
|
$REGION_NAME: {get_param: region_name} |
|
$HTTP_PROXY: {get_param: http_proxy} |
|
$HTTPS_PROXY: {get_param: https_proxy} |
|
$NO_PROXY: {get_param: no_proxy} |
|
$HYPERKUBE_PREFIX: {get_param: hyperkube_prefix} |
|
$KUBE_TAG: {get_param: kube_tag} |
|
$FLANNEL_NETWORK_CIDR: {get_param: flannel_network_cidr} |
|
$PODS_NETWORK_CIDR: {get_param: pods_network_cidr} |
|
$KUBE_VERSION: {get_param: kube_version} |
|
$TRUSTEE_USER_ID: {get_param: trustee_user_id} |
|
$TRUSTEE_USERNAME: {get_param: trustee_username} |
|
$TRUSTEE_PASSWORD: {get_param: trustee_password} |
|
$TRUSTEE_DOMAIN_ID: {get_param: trustee_domain_id} |
|
$TRUST_ID: {get_param: trust_id} |
|
$AUTH_URL: {get_param: auth_url} |
|
$CLOUD_PROVIDER_ENABLED: {get_param: cloud_provider_enabled} |
|
$INSECURE_REGISTRY_URL: {get_param: insecure_registry_url} |
|
$CONTAINER_INFRA_PREFIX: {get_param: container_infra_prefix} |
|
$DNS_SERVICE_IP: {get_param: dns_service_ip} |
|
$DNS_CLUSTER_DOMAIN: {get_param: dns_cluster_domain} |
|
$KUBELET_OPTIONS: {get_param: kubelet_options} |
|
$KUBEPROXY_OPTIONS: {get_param: kubeproxy_options} |
|
$OCTAVIA_ENABLED: {get_param: octavia_enabled} |
|
$HEAT_CONTAINER_AGENT_TAG: {get_param: heat_container_agent_tag} |
|
$AUTO_HEALING_ENABLED: {get_param: auto_healing_enabled} |
|
$AUTO_HEALING_CONTROLLER: {get_param: auto_healing_controller} |
|
$NPD_ENABLED: {get_param: npd_enabled} |
|
$NODEGROUP_ROLE: {get_param: nodegroup_role} |
|
$NODEGROUP_NAME: {get_param: nodegroup_name} |
|
$USE_PODMAN: {get_param: use_podman} |
|
$CONTAINER_RUNTIME: {get_param: container_runtime} |
|
$CONTAINERD_VERSION: {get_param: containerd_version} |
|
$CONTAINERD_TARBALL_URL: {get_param: containerd_tarball_url} |
|
$CONTAINERD_TARBALL_SHA256: {get_param: containerd_tarball_sha256} |
|
- get_file: ../../common/templates/kubernetes/fragments/install-cri.sh |
|
- get_file: ../../common/templates/kubernetes/fragments/install-clients.sh |
|
- get_file: ../../common/templates/kubernetes/fragments/write-kube-os-config.sh |
|
- get_file: ../../common/templates/kubernetes/fragments/make-cert-client.sh |
|
- get_file: ../../common/templates/fragments/configure-docker-registry.sh |
|
- get_file: ../../common/templates/kubernetes/fragments/configure-kubernetes-minion.sh |
|
- get_file: ../../common/templates/kubernetes/fragments/add-proxy.sh |
|
- str_replace: |
|
template: {get_file: ../../common/templates/fragments/configure-docker-storage.sh} |
|
params: |
|
$configure_docker_storage_driver: {get_file: ../../common/templates/fragments/configure_docker_storage_driver_atomic.sh} |
|
- get_file: ../../common/templates/kubernetes/fragments/enable-services-minion.sh |
|
- get_file: ../../common/templates/fragments/enable-docker-registry.sh |
|
|
|
node_config_deployment: |
|
type: OS::Heat::SoftwareDeployment |
|
properties: |
|
signal_transport: HEAT_SIGNAL |
|
config: {get_resource: node_config} |
|
server: {if: ["volume_based", {get_resource: kube-minion-bfv}, {get_resource: kube-minion}]} |
|
actions: ['CREATE'] |
|
|
|
###################################################################### |
|
# |
|
# a single kubernetes minion. |
|
# |
|
|
|
kube_node_volume: |
|
type: OS::Cinder::Volume |
|
condition: volume_based |
|
properties: |
|
image: {get_param: server_image} |
|
size: {get_param: boot_volume_size} |
|
volume_type: {get_param: boot_volume_type} |
|
availability_zone: {get_param: availability_zone} |
|
|
|
# do NOT use "_" (underscore) in the Nova server name |
|
# it creates a mismatch between the generated Nova name and its hostname |
|
# which can lead to weird problems |
|
kube-minion: |
|
condition: image_based |
|
type: OS::Nova::Server |
|
properties: |
|
name: {get_param: name} |
|
flavor: {get_param: minion_flavor} |
|
image: {get_param: server_image} |
|
key_name: {get_param: ssh_key_name} |
|
user_data: {get_resource: agent_config} |
|
user_data_format: SOFTWARE_CONFIG |
|
software_config_transport: POLL_SERVER_HEAT |
|
networks: |
|
- port: {get_resource: kube_minion_eth0} |
|
scheduler_hints: { group: { get_param: nodes_server_group_id }} |
|
availability_zone: {get_param: availability_zone} |
|
|
|
kube-minion-bfv: |
|
condition: volume_based |
|
type: OS::Nova::Server |
|
properties: |
|
name: {get_param: name} |
|
flavor: {get_param: minion_flavor} |
|
key_name: {get_param: ssh_key_name} |
|
user_data: {get_resource: agent_config} |
|
user_data_format: SOFTWARE_CONFIG |
|
software_config_transport: POLL_SERVER_HEAT |
|
networks: |
|
- port: {get_resource: kube_minion_eth0} |
|
scheduler_hints: { group: { get_param: nodes_server_group_id }} |
|
availability_zone: {get_param: availability_zone} |
|
block_device_mapping_v2: |
|
- boot_index: 0 |
|
volume_id: {get_resource: kube_node_volume} |
|
|
|
kube_minion_eth0: |
|
type: OS::Neutron::Port |
|
properties: |
|
network: {get_param: fixed_network} |
|
security_groups: |
|
- get_param: secgroup_kube_minion_id |
|
fixed_ips: |
|
- subnet: {get_param: fixed_subnet} |
|
allowed_address_pairs: |
|
- ip_address: {get_param: pods_network_cidr} |
|
replacement_policy: AUTO |
|
|
|
kube_minion_floating: |
|
type: Magnum::Optional::KubeMinion::Neutron::FloatingIP |
|
properties: |
|
floating_network: {get_param: external_network} |
|
port_id: {get_resource: kube_minion_eth0} |
|
depends_on: kube-minion |
|
|
|
###################################################################### |
|
# |
|
# docker storage. This allocates a cinder volume and attaches it |
|
# to the minion. |
|
# |
|
|
|
docker_volume: |
|
type: Magnum::Optional::Cinder::Volume |
|
properties: |
|
size: {get_param: docker_volume_size} |
|
volume_type: {get_param: docker_volume_type} |
|
availability_zone: {get_param: availability_zone} |
|
|
|
docker_volume_attach: |
|
type: Magnum::Optional::Cinder::VolumeAttachment |
|
properties: |
|
instance_uuid: {if: ["volume_based", {get_resource: kube-minion-bfv}, {get_resource: kube-minion}]} |
|
volume_id: {get_resource: docker_volume} |
|
mountpoint: /dev/vdb |
|
|
|
upgrade_kubernetes: |
|
type: OS::Heat::SoftwareConfig |
|
properties: |
|
group: script |
|
inputs: |
|
- name: kube_tag_input |
|
- name: ostree_remote_input |
|
- name: ostree_commit_input |
|
config: |
|
list_join: |
|
- "\n" |
|
- |
|
- "#!/bin/bash" |
|
- get_file: ../../common/templates/kubernetes/fragments/upgrade-kubernetes.sh |
|
|
|
upgrade_kubernetes_deployment: |
|
type: OS::Heat::SoftwareDeployment |
|
properties: |
|
signal_transport: HEAT_SIGNAL |
|
config: {get_resource: upgrade_kubernetes} |
|
server: {if: ["volume_based", {get_resource: kube-minion-bfv}, {get_resource: kube-minion}]} |
|
actions: ['UPDATE'] |
|
input_values: |
|
kube_tag_input: {get_param: kube_tag} |
|
ostree_remote_input: {get_param: ostree_remote} |
|
ostree_commit_input: {get_param: ostree_commit} |
|
|
|
outputs: |
|
|
|
kube_minion_ip: |
|
value: {get_attr: [kube_minion_eth0, fixed_ips, 0, ip_address]} |
|
description: > |
|
This is the "public" IP address of the Kubernetes minion node. |
|
|
|
kube_minion_external_ip: |
|
value: {get_attr: [kube_minion_floating, floating_ip_address]} |
|
description: > |
|
This is the "public" IP address of the Kubernetes minion node. |
|
|
|
###################################################################### |
|
# |
|
# NOTE(flwang): Returning the minion node server ID here so that |
|
# consumer can send API request to Heat to remove a particular |
|
# node with removal_policies. Otherwise, the consumer (e.g. AutoScaler) |
|
# has to use index to do the remove which is confusing out of the |
|
# OpenStack world. |
|
# https://storyboard.openstack.org/#!/story/2005054 |
|
# |
|
###################################################################### |
|
|
|
OS::stack_id: |
|
value: {if: ["volume_based", {get_resource: kube-minion-bfv}, {get_resource: kube-minion}]} |
|
description: > |
|
This is the Nova server id of the node.
|
|
|