heat_template_version: 2014-10-16 description: > This is a nested stack that defines a single Kubernetes minion, This stack is included by an AutoScalingGroup resource in the parent template (kubecluster.yaml). parameters: name: type: string description: server name server_image: type: string description: glance image used to boot the server minion_flavor: type: string description: flavor to use when booting the server ssh_key_name: type: string description: name of ssh key to be provisioned on our server external_network: type: string description: uuid/name of a network to use for floating ip addresses kube_allow_priv: type: string description: > whether or not kubernetes should permit privileged containers. constraints: - allowed_values: ["true", "false"] docker_volume_size: type: number description: > size of a cinder volume to allocate to docker for container/image storage docker_volume_type: type: string description: > type of a cinder volume to allocate to docker for container/image storage docker_storage_driver: type: string description: docker storage driver name default: "devicemapper" cgroup_driver: type: string description: > cgroup driver name that kubelet should use, ideally the same as the docker cgroup driver. default: "cgroupfs" tls_disabled: type: boolean description: whether or not to enable TLS verify_ca: type: boolean description: whether or not to validate certificate authority kubernetes_port: type: number description: > The port which are used by kube-apiserver to provide Kubernetes service. cluster_uuid: type: string description: identifier for the cluster this template is generating magnum_url: type: string description: endpoint to retrieve TLS certs from prometheus_monitoring: type: boolean description: > whether or not to have the node-exporter running on the node kube_master_ip: type: string description: IP address of the Kubernetes master server. etcd_server_ip: type: string description: IP address of the Etcd server. fixed_network: type: string description: Network from which to allocate fixed addresses. fixed_subnet: type: string description: Subnet from which to allocate fixed addresses. network_driver: type: string description: network driver to use for instantiating container networks flannel_network_cidr: type: string description: network range for flannel overlay network wait_condition_timeout: type: number description : > timeout for the Wait Conditions registry_enabled: type: boolean description: > Indicates whether the docker registry is enabled. registry_port: type: number description: port of registry service swift_region: type: string description: region of swift service registry_container: type: string description: > name of swift container which docker registry stores images in registry_insecure: type: boolean description: > indicates whether to skip TLS verification between registry and backend storage registry_chunksize: type: number description: > size fo the data segments for the swift dynamic large objects secgroup_kube_minion_id: type: string description: ID of the security group for kubernetes minion. volume_driver: type: string description: volume driver to use for container storage region_name: type: string description: A logically separate section of the cluster username: type: string description: > user account password: type: string description: > user password, not set in current implementation, only used to fill in for Kubernetes config file hidden: true http_proxy: type: string description: http proxy address for docker https_proxy: type: string description: https proxy address for docker no_proxy: type: string description: no proxies for docker kube_tag: type: string description: tag of the k8s containers used to provision the kubernetes cluster kube_version: type: string description: version of kubernetes used for kubernetes cluster trustee_domain_id: type: string description: domain id of the trustee trustee_user_id: type: string description: user id of the trustee trustee_username: type: string description: username of the trustee trustee_password: type: string description: password of the trustee hidden: true trust_id: type: string description: id of the trust which is used by the trustee hidden: true auth_url: type: string description: > url for keystone, must be v2 since k8s backend only support v2 at this point insecure_registry_url: type: string description: insecure registry url container_infra_prefix: type: string description: > prefix of container images used in the cluster, kubernetes components, kubernetes-dashboard, coredns etc dns_service_ip: type: string description: > address used by Kubernetes DNS service dns_cluster_domain: type: string description: > domain name for cluster DNS openstack_ca: type: string description: The OpenStack CA certificate to install on the node. nodes_server_group_id: type: string description: ID of the server group for kubernetes cluster nodes. availability_zone: type: string description: > availability zone for master and nodes default: "" pods_network_cidr: type: string description: Configure the IP pool/range from which pod IPs will be chosen kubelet_options: type: string description: > additional options to be passed to the kubelet kubeproxy_options: type: string description: > additional options to be passed to the kube proxy octavia_enabled: type: boolean description: > whether or not to use Octavia for LoadBalancer type service. default: False cloud_provider_enabled: type: boolean description: Enable or disable the openstack kubernetes cloud provider heat_container_agent_tag: type: string description: tag of the heat_container_agent system container auto_healing_enabled: type: boolean description: > true if the auto healing feature should be enabled resources: agent_config: type: OS::Heat::SoftwareConfig properties: group: ungrouped config: list_join: - "\n" - - str_replace: template: {get_file: ../../common/templates/fragments/atomic-install-openstack-ca.sh} params: $OPENSTACK_CA: {get_param: openstack_ca} - str_replace: template: {get_file: ../../common/templates/kubernetes/fragments/start-container-agent.sh} params: $CONTAINER_INFRA_PREFIX: {get_param: container_infra_prefix} $HEAT_CONTAINER_AGENT_TAG: {get_param: heat_container_agent_tag} - get_file: ../../common/templates/kubernetes/fragments/disable-selinux.sh ###################################################################### # # software configs. these are components that are combined into # a multipart MIME user-data archive. # node_config: type: OS::Heat::SoftwareConfig properties: group: script config: list_join: - "\n" - - str_replace: template: {get_file: ../../common/templates/kubernetes/fragments/write-heat-params.sh} params: $PROMETHEUS_MONITORING: {get_param: prometheus_monitoring} $KUBE_ALLOW_PRIV: {get_param: kube_allow_priv} $KUBE_MASTER_IP: {get_param: kube_master_ip} $KUBE_API_PORT: {get_param: kubernetes_port} $KUBE_NODE_PUBLIC_IP: {get_attr: [kube_minion_floating, floating_ip_address]} $KUBE_NODE_IP: {get_attr: [kube_minion_eth0, fixed_ips, 0, ip_address]} $ETCD_SERVER_IP: {get_param: etcd_server_ip} $DOCKER_VOLUME: {get_resource: docker_volume} $DOCKER_VOLUME_SIZE: {get_param: docker_volume_size} $DOCKER_STORAGE_DRIVER: {get_param: docker_storage_driver} $CGROUP_DRIVER: {get_param: cgroup_driver} $NETWORK_DRIVER: {get_param: network_driver} $REGISTRY_ENABLED: {get_param: registry_enabled} $REGISTRY_PORT: {get_param: registry_port} $SWIFT_REGION: {get_param: swift_region} $REGISTRY_CONTAINER: {get_param: registry_container} $REGISTRY_INSECURE: {get_param: registry_insecure} $REGISTRY_CHUNKSIZE: {get_param: registry_chunksize} $TLS_DISABLED: {get_param: tls_disabled} $VERIFY_CA: {get_param: verify_ca} $CLUSTER_UUID: {get_param: cluster_uuid} $MAGNUM_URL: {get_param: magnum_url} $USERNAME: {get_param: username} $PASSWORD: {get_param: password} $VOLUME_DRIVER: {get_param: volume_driver} $REGION_NAME: {get_param: region_name} $HTTP_PROXY: {get_param: http_proxy} $HTTPS_PROXY: {get_param: https_proxy} $NO_PROXY: {get_param: no_proxy} $KUBE_TAG: {get_param: kube_tag} $FLANNEL_NETWORK_CIDR: {get_param: flannel_network_cidr} $PODS_NETWORK_CIDR: {get_param: pods_network_cidr} $KUBE_VERSION: {get_param: kube_version} $TRUSTEE_USER_ID: {get_param: trustee_user_id} $TRUSTEE_PASSWORD: {get_param: trustee_password} $TRUST_ID: {get_param: trust_id} $AUTH_URL: {get_param: auth_url} $CLOUD_PROVIDER_ENABLED: {get_param: cloud_provider_enabled} $INSECURE_REGISTRY_URL: {get_param: insecure_registry_url} $CONTAINER_INFRA_PREFIX: {get_param: container_infra_prefix} $DNS_SERVICE_IP: {get_param: dns_service_ip} $DNS_CLUSTER_DOMAIN: {get_param: dns_cluster_domain} $KUBELET_OPTIONS: {get_param: kubelet_options} $KUBEPROXY_OPTIONS: {get_param: kubeproxy_options} $OCTAVIA_ENABLED: {get_param: octavia_enabled} $HEAT_CONTAINER_AGENT_TAG: {get_param: heat_container_agent_tag} $AUTO_HEALING_ENABLED: {get_param: auto_healing_enabled} - get_file: ../../common/templates/kubernetes/fragments/write-kube-os-config.sh - get_file: ../../common/templates/kubernetes/fragments/make-cert-client.sh - get_file: ../../common/templates/fragments/configure-docker-registry.sh - get_file: ../../common/templates/kubernetes/fragments/configure-kubernetes-minion.sh - get_file: ../../common/templates/kubernetes/fragments/add-proxy.sh - str_replace: template: {get_file: ../../common/templates/fragments/configure-docker-storage.sh} params: $configure_docker_storage_driver: {get_file: ../../common/templates/fragments/configure_docker_storage_driver_atomic.sh} - get_file: ../../common/templates/kubernetes/fragments/enable-services-minion.sh - get_file: ../../common/templates/fragments/enable-docker-registry.sh node_config_deployment: type: OS::Heat::SoftwareDeployment properties: signal_transport: HEAT_SIGNAL config: {get_resource: node_config} server: {get_resource: kube-minion} actions: ['CREATE'] ###################################################################### # # a single kubernetes minion. # # do NOT use "_" (underscore) in the Nova server name # it creates a mismatch between the generated Nova name and its hostname # which can lead to weird problems kube-minion: type: OS::Nova::Server properties: name: {get_param: name} image: {get_param: server_image} flavor: {get_param: minion_flavor} key_name: {get_param: ssh_key_name} user_data: {get_resource: agent_config} user_data_format: SOFTWARE_CONFIG software_config_transport: POLL_SERVER_HEAT networks: - port: {get_resource: kube_minion_eth0} scheduler_hints: { group: { get_param: nodes_server_group_id }} availability_zone: {get_param: availability_zone} kube_minion_eth0: type: OS::Neutron::Port properties: network: {get_param: fixed_network} security_groups: - get_param: secgroup_kube_minion_id fixed_ips: - subnet: {get_param: fixed_subnet} allowed_address_pairs: - ip_address: {get_param: pods_network_cidr} replacement_policy: AUTO kube_minion_floating: type: Magnum::Optional::KubeMinion::Neutron::FloatingIP properties: floating_network: {get_param: external_network} port_id: {get_resource: kube_minion_eth0} ###################################################################### # # docker storage. This allocates a cinder volume and attaches it # to the minion. # docker_volume: type: Magnum::Optional::Cinder::Volume properties: size: {get_param: docker_volume_size} volume_type: {get_param: docker_volume_type} docker_volume_attach: type: Magnum::Optional::Cinder::VolumeAttachment properties: instance_uuid: {get_resource: kube-minion} volume_id: {get_resource: docker_volume} mountpoint: /dev/vdb upgrade_kubernetes: type: OS::Heat::SoftwareConfig properties: group: script inputs: - name: kube_tag_input config: get_file: ../../common/templates/kubernetes/fragments/upgrade-kubernetes.sh upgrade_kubernetes_deployment: type: OS::Heat::SoftwareDeployment properties: signal_transport: HEAT_SIGNAL config: {get_resource: upgrade_kubernetes} server: {get_resource: kube-minion} actions: ['UPDATE'] input_values: kube_tag_input: {get_param: kube_tag} outputs: kube_minion_ip: value: {get_attr: [kube_minion_eth0, fixed_ips, 0, ip_address]} description: > This is the "public" IP address of the Kubernetes minion node. kube_minion_external_ip: value: {get_attr: [kube_minion_floating, floating_ip_address]} description: > This is the "public" IP address of the Kubernetes minion node. ###################################################################### # # NOTE(flwang): Returning the minion node server ID here so that # consumer can send API request to Heat to remove a particular # node with removal_policies. Otherwise, the consumer (e.g. AutoScaler) # has to use index to do the remove which is confusing out of the # OpenStack world. # https://storyboard.openstack.org/#!/story/2005054 # ###################################################################### OS::stack_id: value: { get_resource: kube-minion } description: > This is the Nova server id of the node.