magnum/magnum/drivers/k8s_fedora_coreos_v1/templates/kubeminion.yaml

616 lines
19 KiB
YAML

heat_template_version: queens
description: >
This is a nested stack that defines a single Kubernetes minion, This stack is
included by an AutoScalingGroup resource in the parent template
(kubecluster.yaml).
parameters:
name:
type: string
description: server name
server_image:
type: string
description: glance image used to boot the server
minion_flavor:
type: string
description: flavor to use when booting the server
nodegroup_role:
type: string
description: the role of the nodegroup
nodegroup_name:
type: string
description: the name of the nodegroup where the node belongs
ssh_key_name:
type: string
description: name of ssh key to be provisioned on our server
ssh_public_key:
type: string
description: name of ssh key to be provisioned on our server
external_network:
type: string
description: uuid/name of a network to use for floating ip addresses
kube_allow_priv:
type: string
description: >
whether or not kubernetes should permit privileged containers.
constraints:
- allowed_values: ["true", "false"]
boot_volume_size:
type: number
description: >
size of the cinder boot volume
boot_volume_type:
type: string
description: >
type of the cinder boot volume
docker_volume_size:
type: number
description: >
size of a cinder volume to allocate to docker for container/image
storage
docker_volume_type:
type: string
description: >
type of a cinder volume to allocate to docker for container/image
storage
docker_storage_driver:
type: string
description: docker storage driver name
default: "devicemapper"
cgroup_driver:
type: string
description: >
cgroup driver name that kubelet should use, ideally the same as
the docker cgroup driver.
default: "cgroupfs"
tls_disabled:
type: boolean
description: whether or not to enable TLS
verify_ca:
type: boolean
description: whether or not to validate certificate authority
kubernetes_port:
type: number
description: >
The port which are used by kube-apiserver to provide Kubernetes
service.
cluster_uuid:
type: string
description: identifier for the cluster this template is generating
magnum_url:
type: string
description: endpoint to retrieve TLS certs from
prometheus_monitoring:
type: boolean
description: >
whether or not to have the node-exporter running on the node
kube_master_ip:
type: string
description: IP address of the Kubernetes master server.
etcd_server_ip:
type: string
description: IP address of the Etcd server.
fixed_network:
type: string
description: Network from which to allocate fixed addresses.
fixed_subnet:
type: string
description: Subnet from which to allocate fixed addresses.
network_driver:
type: string
description: network driver to use for instantiating container networks
flannel_network_cidr:
type: string
description: network range for flannel overlay network
wait_condition_timeout:
type: number
description : >
timeout for the Wait Conditions
registry_enabled:
type: boolean
description: >
Indicates whether the docker registry is enabled.
registry_port:
type: number
description: port of registry service
swift_region:
type: string
description: region of swift service
registry_container:
type: string
description: >
name of swift container which docker registry stores images in
registry_insecure:
type: boolean
description: >
indicates whether to skip TLS verification between registry and backend storage
registry_chunksize:
type: number
description: >
size fo the data segments for the swift dynamic large objects
secgroup_kube_minion_id:
type: string
description: ID of the security group for kubernetes minion.
volume_driver:
type: string
description: volume driver to use for container storage
region_name:
type: string
description: A logically separate section of the cluster
username:
type: string
description: >
user account
password:
type: string
description: >
user password, not set in current implementation, only used to
fill in for Kubernetes config file
hidden: true
http_proxy:
type: string
description: http proxy address for docker
https_proxy:
type: string
description: https proxy address for docker
no_proxy:
type: string
description: no proxies for docker
kube_tag:
type: string
description: tag of the k8s containers used to provision the kubernetes cluster
kube_version:
type: string
description: version of kubernetes used for kubernetes cluster
trustee_domain_id:
type: string
description: domain id of the trustee
trustee_user_id:
type: string
description: user id of the trustee
trustee_username:
type: string
description: username of the trustee
trustee_password:
type: string
description: password of the trustee
hidden: true
trust_id:
type: string
description: id of the trust which is used by the trustee
hidden: true
auth_url:
type: string
description: >
url for keystone, must be v2 since k8s backend only support v2
at this point
insecure_registry_url:
type: string
description: insecure registry url
container_infra_prefix:
type: string
description: >
prefix of container images used in the cluster, kubernetes components,
kubernetes-dashboard, coredns etc
dns_service_ip:
type: string
description: >
address used by Kubernetes DNS service
dns_cluster_domain:
type: string
description: >
domain name for cluster DNS
openstack_ca:
type: string
description: The OpenStack CA certificate to install on the node.
nodes_server_group_id:
type: string
description: ID of the server group for kubernetes cluster nodes.
availability_zone:
type: string
description: >
availability zone for master and nodes
default: ""
pods_network_cidr:
type: string
description: Configure the IP pool/range from which pod IPs will be chosen
kubelet_options:
type: string
description: >
additional options to be passed to the kubelet
kubeproxy_options:
type: string
description: >
additional options to be passed to the kube proxy
octavia_enabled:
type: boolean
description: >
whether or not to use Octavia for LoadBalancer type service.
default: False
cloud_provider_enabled:
type: boolean
description: Enable or disable the openstack kubernetes cloud provider
heat_container_agent_tag:
type: string
description: tag of the heat_container_agent system container
auto_healing_enabled:
type: boolean
description: >
true if the auto healing feature should be enabled
auto_healing_controller:
type: string
description: >
The service to be deployed for auto-healing.
default: "draino"
npd_enabled:
type: boolean
description: >
true if the npd service should be launched
default:
true
ostree_remote:
type: string
description: The ostree remote branch to upgrade
ostree_commit:
type: string
description: The ostree commit to deploy
use_podman:
type: boolean
description: >
If true, run system containers for kubernetes, etcd and heat-agent
container_runtime:
type: string
description: The container runtime to install
containerd_version:
type: string
description: The containerd version to download from https://storage.googleapis.com/cri-containerd-release/
containerd_tarball_url:
type: string
description: Url location of the containerd tarball.
containerd_tarball_sha256:
type: string
description: sha256 of the target containerd tarball.
conditions:
image_based: {equals: [{get_param: boot_volume_size}, 0]}
volume_based:
not:
equals:
- get_param: boot_volume_size
- 0
resources:
agent_config:
type: OS::Heat::SoftwareConfig
properties:
group: ungrouped
config:
list_join:
- "\n"
-
- str_replace:
template: {get_file: user_data.json}
params:
$HOSTNAME: {get_param: name}
$SSH_KEY_VALUE: {get_param: ssh_public_key}
$OPENSTACK_CA: {get_param: openstack_ca}
$CONTAINER_INFRA_PREFIX:
if:
- equals:
- get_param: container_infra_prefix
- ""
- "docker.io/openstackmagnum/"
- get_param: container_infra_prefix
$HEAT_CONTAINER_AGENT_TAG: {get_param: heat_container_agent_tag}
$HTTP_PROXY: {get_param: http_proxy}
$HTTPS_PROXY: {get_param: https_proxy}
$NO_PROXY: {get_param: no_proxy}
######################################################################
#
# software configs. these are components that are combined into
# a multipart MIME user-data archive.
#
node_config:
type: OS::Heat::SoftwareConfig
properties:
group: script
config:
list_join:
- "\n"
-
- str_replace:
template: {get_file: ../../common/templates/kubernetes/fragments/write-heat-params.sh}
params:
$INSTANCE_NAME: {get_param: name}
$PROMETHEUS_MONITORING: {get_param: prometheus_monitoring}
$KUBE_ALLOW_PRIV: {get_param: kube_allow_priv}
$KUBE_MASTER_IP: {get_param: kube_master_ip}
$KUBE_API_PORT: {get_param: kubernetes_port}
$KUBE_NODE_PUBLIC_IP: {get_attr: [kube_minion_floating, floating_ip_address]}
$KUBE_NODE_IP: {get_attr: [kube_minion_eth0, fixed_ips, 0, ip_address]}
$ETCD_SERVER_IP: {get_param: etcd_server_ip}
$DOCKER_VOLUME: {get_resource: docker_volume}
$DOCKER_VOLUME_SIZE: {get_param: docker_volume_size}
$DOCKER_STORAGE_DRIVER: {get_param: docker_storage_driver}
$CGROUP_DRIVER: {get_param: cgroup_driver}
$NETWORK_DRIVER: {get_param: network_driver}
$REGISTRY_ENABLED: {get_param: registry_enabled}
$REGISTRY_PORT: {get_param: registry_port}
$SWIFT_REGION: {get_param: swift_region}
$REGISTRY_CONTAINER: {get_param: registry_container}
$REGISTRY_INSECURE: {get_param: registry_insecure}
$REGISTRY_CHUNKSIZE: {get_param: registry_chunksize}
$TLS_DISABLED: {get_param: tls_disabled}
$VERIFY_CA: {get_param: verify_ca}
$CLUSTER_UUID: {get_param: cluster_uuid}
$MAGNUM_URL: {get_param: magnum_url}
$USERNAME: {get_param: username}
$PASSWORD: {get_param: password}
$VOLUME_DRIVER: {get_param: volume_driver}
$REGION_NAME: {get_param: region_name}
$HTTP_PROXY: {get_param: http_proxy}
$HTTPS_PROXY: {get_param: https_proxy}
$NO_PROXY: {get_param: no_proxy}
$KUBE_TAG: {get_param: kube_tag}
$FLANNEL_NETWORK_CIDR: {get_param: flannel_network_cidr}
$PODS_NETWORK_CIDR: {get_param: pods_network_cidr}
$KUBE_VERSION: {get_param: kube_version}
$TRUSTEE_USER_ID: {get_param: trustee_user_id}
$TRUSTEE_PASSWORD: {get_param: trustee_password}
$TRUST_ID: {get_param: trust_id}
$AUTH_URL: {get_param: auth_url}
$CLOUD_PROVIDER_ENABLED: {get_param: cloud_provider_enabled}
$INSECURE_REGISTRY_URL: {get_param: insecure_registry_url}
$CONTAINER_INFRA_PREFIX: {get_param: container_infra_prefix}
$DNS_SERVICE_IP: {get_param: dns_service_ip}
$DNS_CLUSTER_DOMAIN: {get_param: dns_cluster_domain}
$KUBELET_OPTIONS: {get_param: kubelet_options}
$KUBEPROXY_OPTIONS: {get_param: kubeproxy_options}
$OCTAVIA_ENABLED: {get_param: octavia_enabled}
$HEAT_CONTAINER_AGENT_TAG: {get_param: heat_container_agent_tag}
$AUTO_HEALING_ENABLED: {get_param: auto_healing_enabled}
$AUTO_HEALING_CONTROLLER: {get_param: auto_healing_controller}
$NPD_ENABLED: {get_param: npd_enabled}
$NODEGROUP_ROLE: {get_param: nodegroup_role}
$NODEGROUP_NAME: {get_param: nodegroup_name}
$USE_PODMAN: {get_param: use_podman}
$CONTAINER_RUNTIME: {get_param: container_runtime}
$CONTAINERD_VERSION: {get_param: containerd_version}
$CONTAINERD_TARBALL_URL: {get_param: containerd_tarball_url}
$CONTAINERD_TARBALL_SHA256: {get_param: containerd_tarball_sha256}
- get_file: ../../common/templates/kubernetes/fragments/install-cri.sh
- get_file: ../../common/templates/kubernetes/fragments/write-kube-os-config.sh
- get_file: ../../common/templates/kubernetes/fragments/make-cert-client.sh
- get_file: ../../common/templates/fragments/configure-docker-registry.sh
- get_file: ../../common/templates/kubernetes/fragments/configure-kubernetes-minion.sh
- get_file: ../../common/templates/kubernetes/fragments/add-proxy.sh
# TODO add docker_storage_setup
- get_file: ../../common/templates/kubernetes/fragments/enable-services-minion.sh
- get_file: ../../common/templates/fragments/enable-docker-registry.sh
node_config_deployment:
type: OS::Heat::SoftwareDeployment
properties:
signal_transport: HEAT_SIGNAL
config: {get_resource: node_config}
server: {if: ["volume_based", {get_resource: kube-minion-bfv}, {get_resource: kube-minion}]}
actions: ['CREATE']
######################################################################
#
# a single kubernetes minion.
#
kube_node_volume:
type: OS::Cinder::Volume
condition: volume_based
properties:
image: {get_param: server_image}
size: {get_param: boot_volume_size}
volume_type: {get_param: boot_volume_type}
availability_zone: {get_param: availability_zone}
# do NOT use "_" (underscore) in the Nova server name
# it creates a mismatch between the generated Nova name and its hostname
# which can lead to weird problems
kube-minion:
condition: image_based
type: OS::Nova::Server
properties:
name: {get_param: name}
flavor: {get_param: minion_flavor}
image: {get_param: server_image}
user_data: {get_resource: agent_config}
user_data_format: SOFTWARE_CONFIG
software_config_transport: POLL_SERVER_HEAT
networks:
- port: {get_resource: kube_minion_eth0}
scheduler_hints: { group: { get_param: nodes_server_group_id }}
availability_zone: {get_param: availability_zone}
kube-minion-bfv:
condition: volume_based
type: OS::Nova::Server
properties:
name: {get_param: name}
flavor: {get_param: minion_flavor}
user_data: {get_resource: agent_config}
user_data_format: SOFTWARE_CONFIG
software_config_transport: POLL_SERVER_HEAT
networks:
- port: {get_resource: kube_minion_eth0}
scheduler_hints: { group: { get_param: nodes_server_group_id }}
availability_zone: {get_param: availability_zone}
block_device_mapping_v2:
- boot_index: 0
volume_id: {get_resource: kube_node_volume}
delete_on_termination: true
kube_minion_eth0:
type: OS::Neutron::Port
properties:
network: {get_param: fixed_network}
security_groups:
- get_param: secgroup_kube_minion_id
fixed_ips:
- subnet: {get_param: fixed_subnet}
allowed_address_pairs:
- ip_address: {get_param: pods_network_cidr}
replacement_policy: AUTO
kube_minion_floating:
type: Magnum::Optional::KubeMinion::Neutron::FloatingIP
properties:
floating_network: {get_param: external_network}
port_id: {get_resource: kube_minion_eth0}
depends_on: kube-minion
######################################################################
#
# docker storage. This allocates a cinder volume and attaches it
# to the minion.
#
docker_volume:
type: Magnum::Optional::Cinder::Volume
properties:
size: {get_param: docker_volume_size}
volume_type: {get_param: docker_volume_type}
availability_zone: {get_param: availability_zone}
docker_volume_attach:
type: Magnum::Optional::Cinder::VolumeAttachment
properties:
instance_uuid: {if: ["volume_based", {get_resource: kube-minion-bfv}, {get_resource: kube-minion}]}
volume_id: {get_resource: docker_volume}
mountpoint: /dev/vdb
upgrade_kubernetes:
type: OS::Heat::SoftwareConfig
properties:
group: script
inputs:
- name: kube_tag_input
- name: ostree_remote_input
- name: ostree_commit_input
config:
get_file: ../../common/templates/kubernetes/fragments/upgrade-kubernetes.sh
upgrade_kubernetes_deployment:
type: OS::Heat::SoftwareDeployment
properties:
signal_transport: HEAT_SIGNAL
config: {get_resource: upgrade_kubernetes}
server: {if: ["volume_based", {get_resource: kube-minion-bfv}, {get_resource: kube-minion}]}
actions: ['UPDATE']
input_values:
kube_tag_input: {get_param: kube_tag}
ostree_remote_input: {get_param: ostree_remote}
ostree_commit_input: {get_param: ostree_commit}
outputs:
kube_minion_ip:
value: {get_attr: [kube_minion_eth0, fixed_ips, 0, ip_address]}
description: >
This is the "public" IP address of the Kubernetes minion node.
kube_minion_external_ip:
value: {get_attr: [kube_minion_floating, floating_ip_address]}
description: >
This is the "public" IP address of the Kubernetes minion node.
######################################################################
#
# NOTE(flwang): Returning the minion node server ID here so that
# consumer can send API request to Heat to remove a particular
# node with removal_policies. Otherwise, the consumer (e.g. AutoScaler)
# has to use index to do the remove which is confusing out of the
# OpenStack world.
# https://storyboard.openstack.org/#!/story/2005054
#
######################################################################
OS::stack_id:
value: {if: ["volume_based", {get_resource: kube-minion-bfv}, {get_resource: kube-minion}]}
description: >
This is the Nova server id of the node.