heat_template_version: 2014-10-16 description: > This is a nested stack that defines a single Kubernetes master, This stack is included by an ResourceGroup resource in the parent template (kubecluster.yaml). parameters: name: type: string description: server name server_image: type: string description: glance image used to boot the server master_flavor: type: string description: flavor to use when booting the server ssh_key_name: type: string description: name of ssh key to be provisioned on our server external_network: type: string description: uuid of a network to use for floating ip addresses portal_network_cidr: type: string description: > address range used by kubernetes for service portals kube_allow_priv: type: string description: > whether or not kubernetes should permit privileged containers. constraints: - allowed_values: ["true", "false"] etcd_volume_size: type: number description: > size of a cinder volume to allocate for etcd storage docker_volume_size: type: number description: > size of a cinder volume to allocate to docker for container/image storage docker_volume_type: type: string description: > type of a cinder volume to allocate to docker for container/image storage docker_storage_driver: type: string description: docker storage driver name default: "devicemapper" cgroup_driver: type: string description: > cgroup driver name that kubelet should use, ideally the same as the docker cgroup driver. default: "cgroupfs" volume_driver: type: string description: volume driver to use for container storage region_name: type: string description: A logically separate section of the cluster flannel_network_cidr: type: string description: network range for flannel overlay network flannel_network_subnetlen: type: number description: size of subnet assigned to each master flannel_backend: type: string description: > specify the backend for flannel, default udp backend constraints: - allowed_values: ["udp", "vxlan", "host-gw"] system_pods_initial_delay: type: number description: > health check, time to wait for system pods (podmaster, scheduler) to boot (in seconds) default: 30 system_pods_timeout: type: number description: > health check, timeout for system pods (podmaster, scheduler) to answer. (in seconds) default: 5 admission_control_list: type: string description: > List of admission control plugins to activate discovery_url: type: string description: > Discovery URL used for bootstrapping the etcd cluster. tls_disabled: type: boolean description: whether or not to enable TLS traefik_ingress_controller_tag: type: string description: tag of the traefik containers to be used. kube_dashboard_enabled: type: boolean description: whether or not to disable kubernetes dashboard influx_grafana_dashboard_enabled: type: boolean description: Enable influxdb with grafana dashboard for data from heapster verify_ca: type: boolean description: whether or not to validate certificate authority kubernetes_port: type: number description: > The port which are used by kube-apiserver to provide Kubernetes service. cluster_uuid: type: string description: identifier for the cluster this template is generating magnum_url: type: string description: endpoint to retrieve TLS certs from prometheus_monitoring: type: boolean description: > whether or not to have prometheus and grafana deployed grafana_admin_passwd: type: string hidden: true description: > admin user password for the Grafana monitoring interface api_public_address: type: string description: Public IP address of the Kubernetes master server. default: "" api_private_address: type: string description: Private IP address of the Kubernetes master server. default: "" fixed_network: type: string description: Network from which to allocate fixed addresses. fixed_subnet: type: string description: Subnet from which to allocate fixed addresses. network_driver: type: string description: network driver to use for instantiating container networks secgroup_kube_master_id: type: string description: ID of the security group for kubernetes master. api_pool_id: type: string description: ID of the load balancer pool of k8s API server. etcd_pool_id: type: string description: ID of the load balancer pool of etcd server. auth_url: type: string description: > url for kubernetes to authenticate username: type: string description: > user account password: type: string description: > user password http_proxy: type: string description: http proxy address for docker https_proxy: type: string description: https proxy address for docker no_proxy: type: string description: no proxies for docker kube_tag: type: string description: tag of the k8s containers used to provision the kubernetes cluster cloud_provider_tag: type: string description: tag of the kubernetes/cloud-provider-openstack https://hub.docker.com/r/k8scloudprovider/openstack-cloud-controller-manager/tags/ cloud_provider_enabled: type: boolean description: Enable or disable the openstack kubernetes cloud provider etcd_tag: type: string description: tag of the etcd system container coredns_tag: type: string description: tag of the coredns container flannel_tag: type: string description: tag of the flannel system containers flannel_cni_tag: type: string description: tag of the flannel cni container kube_version: type: string description: version of kubernetes used for kubernetes cluster kube_dashboard_version: type: string description: version of kubernetes dashboard used for kubernetes cluster trustee_user_id: type: string description: user id of the trustee trustee_password: type: string description: password of the trustee hidden: true trust_id: type: string description: id of the trust which is used by the trustee hidden: true insecure_registry_url: type: string description: insecure registry url container_infra_prefix: type: string description: > prefix of container images used in the cluster, kubernetes components, kubernetes-dashboard, coredns etc etcd_lb_vip: type: string description: > etcd lb vip private used to generate certs on master. default: "" dns_service_ip: type: string description: > address used by Kubernetes DNS service dns_cluster_domain: type: string description: > domain name for cluster DNS openstack_ca: type: string description: The OpenStack CA certificate to install on the node. nodes_server_group_id: type: string description: ID of the server group for kubernetes cluster nodes. availability_zone: type: string description: > availability zone for master and nodes default: "" ca_key: type: string description: key of internal ca for the kube certificate api manager hidden: true cert_manager_api: type: boolean description: true if the kubernetes cert api manager should be enabled default: false calico_tag: type: string description: tag of the calico containers used to provision the calico node calico_cni_tag: type: string description: tag of the cni used to provision the calico node calico_kube_controllers_tag: type: string description: tag of the kube_controllers used to provision the calico node calico_ipv4pool: type: string description: Configure the IP pool from which Pod IPs will be chosen pods_network_cidr: type: string description: Configure the IP pool/range from which pod IPs will be chosen ingress_controller: type: string description: > ingress controller backend to use ingress_controller_role: type: string description: > node role where the ingress controller should run octavia_ingress_controller_tag: type: string description: Octavia ingress controller docker image tag. kubelet_options: type: string description: > additional options to be passed to the kubelet kubeapi_options: type: string description: > additional options to be passed to the api kubecontroller_options: type: string description: > additional options to be passed to the controller manager kubeproxy_options: type: string description: > additional options to be passed to the kube proxy kubescheduler_options: type: string description: > additional options to be passed to the scheduler octavia_enabled: type: boolean description: > whether or not to use Octavia for LoadBalancer type service. default: False kube_service_account_key: type: string hidden: true description: > The signed cert will be used to verify the k8s service account tokens during authentication. kube_service_account_private_key: type: string hidden: true description: > The private key will be used to sign generated k8s service account tokens. prometheus_tag: type: string description: tag of prometheus container grafana_tag: type: string description: tag of grafana container heat_container_agent_tag: type: string description: tag of the heat_container_agent system container keystone_auth_enabled: type: boolean description: > true if the keystone authN and authZ should be enabled default: false k8s_keystone_auth_tag: type: string description: tag of the k8s_keystone_auth container monitoring_enabled: type: boolean description: Enable or disable prometheus-operator monitoring solution. default: false project_id: type: string description: > project id of current project tiller_enabled: type: string description: Whether to enable tiller or not tiller_tag: type: string description: tag of tiller container tiller_namespace: type: string description: namespace where tiller will be installed auto_healing_enabled: type: boolean description: > true if the auto healing feature should be enabled auto_scaling_enabled: type: boolean description: > true if the auto scaling feature should be enabled node_problem_detector_tag: type: string description: tag of the node problem detector container nginx_ingress_controller_tag: type: string description: nginx ingress controller docker image tag draino_tag: type: string description: tag of the draino container autoscaler_tag: type: string description: tag of the autoscaler container min_node_count: type: number description: > minimum node count of cluster workers when doing scale down max_node_count: type: number description: > maximum node count of cluster workers when doing scale up resources: ###################################################################### # # resource that exposes the IPs of either the kube master or the API # LBaaS pool depending on whether LBaaS is enabled for the cluster. # api_address_switch: type: Magnum::ApiGatewaySwitcher properties: pool_public_ip: {get_param: api_public_address} pool_private_ip: {get_param: api_private_address} master_public_ip: {get_attr: [kube_master_floating, floating_ip_address]} master_private_ip: {get_attr: [kube_master_eth0, fixed_ips, 0, ip_address]} ###################################################################### # # software configs. these are components that are combined into # a multipart MIME user-data archive. # agent_config: type: OS::Heat::SoftwareConfig properties: group: ungrouped config: list_join: - "\n" - - str_replace: template: {get_file: ../../common/templates/fragments/atomic-install-openstack-ca.sh} params: $OPENSTACK_CA: {get_param: openstack_ca} - str_replace: template: {get_file: ../../common/templates/kubernetes/fragments/start-container-agent.sh} params: $CONTAINER_INFRA_PREFIX: {get_param: container_infra_prefix} $HEAT_CONTAINER_AGENT_TAG: {get_param: heat_container_agent_tag} - get_file: ../../common/templates/kubernetes/fragments/disable-selinux.sh master_config: type: OS::Heat::SoftwareConfig properties: group: script config: list_join: - "\n" - - str_replace: template: {get_file: ../../common/templates/kubernetes/fragments/write-heat-params-master.sh} params: "$PROMETHEUS_MONITORING": {get_param: prometheus_monitoring} "$KUBE_API_PUBLIC_ADDRESS": {get_attr: [api_address_switch, public_ip]} "$KUBE_API_PRIVATE_ADDRESS": {get_attr: [api_address_switch, private_ip]} "$KUBE_API_PORT": {get_param: kubernetes_port} "$KUBE_NODE_PUBLIC_IP": {get_attr: [kube_master_floating, floating_ip_address]} "$KUBE_NODE_IP": {get_attr: [kube_master_eth0, fixed_ips, 0, ip_address]} "$KUBE_ALLOW_PRIV": {get_param: kube_allow_priv} "$ETCD_VOLUME": {get_resource: etcd_volume} "$ETCD_VOLUME_SIZE": {get_param: etcd_volume_size} "$DOCKER_VOLUME": {get_resource: docker_volume} "$DOCKER_VOLUME_SIZE": {get_param: docker_volume_size} "$DOCKER_STORAGE_DRIVER": {get_param: docker_storage_driver} "$CGROUP_DRIVER": {get_param: cgroup_driver} "$NETWORK_DRIVER": {get_param: network_driver} "$FLANNEL_NETWORK_CIDR": {get_param: flannel_network_cidr} "$FLANNEL_NETWORK_SUBNETLEN": {get_param: flannel_network_subnetlen} "$FLANNEL_BACKEND": {get_param: flannel_backend} "$SYSTEM_PODS_INITIAL_DELAY": {get_param: system_pods_initial_delay} "$SYSTEM_PODS_TIMEOUT": {get_param: system_pods_timeout} "$PODS_NETWORK_CIDR": {get_param: pods_network_cidr} "$PORTAL_NETWORK_CIDR": {get_param: portal_network_cidr} "$ADMISSION_CONTROL_LIST": {get_param: admission_control_list} "$ETCD_DISCOVERY_URL": {get_param: discovery_url} "$AUTH_URL": {get_param: auth_url} "$USERNAME": {get_param: username} "$PASSWORD": {get_param: password} "$CLUSTER_SUBNET": {get_param: fixed_subnet} "$TLS_DISABLED": {get_param: tls_disabled} "$TRAEFIK_INGRESS_CONTROLLER_TAG": {get_param: traefik_ingress_controller_tag} "$KUBE_DASHBOARD_ENABLED": {get_param: kube_dashboard_enabled} "$INFLUX_GRAFANA_DASHBOARD_ENABLED": {get_param: influx_grafana_dashboard_enabled} "$VERIFY_CA": {get_param: verify_ca} "$CLUSTER_UUID": {get_param: cluster_uuid} "$MAGNUM_URL": {get_param: magnum_url} "$VOLUME_DRIVER": {get_param: volume_driver} "$REGION_NAME": {get_param: region_name} "$HTTP_PROXY": {get_param: http_proxy} "$HTTPS_PROXY": {get_param: https_proxy} "$NO_PROXY": {get_param: no_proxy} "$KUBE_TAG": {get_param: kube_tag} "$CLOUD_PROVIDER_TAG": {get_param: cloud_provider_tag} "$CLOUD_PROVIDER_ENABLED": {get_param: cloud_provider_enabled} "$ETCD_TAG": {get_param: etcd_tag} "$COREDNS_TAG": {get_param: coredns_tag} "$FLANNEL_TAG": {get_param: flannel_tag} "$FLANNEL_CNI_TAG": {get_param: flannel_cni_tag} "$KUBE_VERSION": {get_param: kube_version} "$KUBE_DASHBOARD_VERSION": {get_param: kube_dashboard_version} "$TRUSTEE_USER_ID": {get_param: trustee_user_id} "$TRUSTEE_PASSWORD": {get_param: trustee_password} "$TRUST_ID": {get_param: trust_id} "$INSECURE_REGISTRY_URL": {get_param: insecure_registry_url} "$CONTAINER_INFRA_PREFIX": {get_param: container_infra_prefix} "$ETCD_LB_VIP": {get_param: etcd_lb_vip} "$DNS_SERVICE_IP": {get_param: dns_service_ip} "$DNS_CLUSTER_DOMAIN": {get_param: dns_cluster_domain} "$CERT_MANAGER_API": {get_param: cert_manager_api} "$CA_KEY": {get_param: ca_key} "$CALICO_TAG": {get_param: calico_tag} "$CALICO_CNI_TAG": {get_param: calico_cni_tag} "$CALICO_KUBE_CONTROLLERS_TAG": {get_param: calico_kube_controllers_tag} "$CALICO_IPV4POOL": {get_param: calico_ipv4pool} "$INGRESS_CONTROLLER": {get_param: ingress_controller} "$INGRESS_CONTROLLER_ROLE": {get_param: ingress_controller_role} "$OCTAVIA_INGRESS_CONTROLLER_TAG": {get_param: octavia_ingress_controller_tag} "$KUBELET_OPTIONS": {get_param: kubelet_options} "$KUBEAPI_OPTIONS": {get_param: kubeapi_options} "$KUBECONTROLLER_OPTIONS": {get_param: kubecontroller_options} "$KUBEPROXY_OPTIONS": {get_param: kubeproxy_options} "$KUBESCHEDULER_OPTIONS": {get_param: kubescheduler_options} "$OCTAVIA_ENABLED": {get_param: octavia_enabled} "$KUBE_SERVICE_ACCOUNT_KEY": {get_param: kube_service_account_key} "$KUBE_SERVICE_ACCOUNT_PRIVATE_KEY": {get_param: kube_service_account_private_key} "$PROMETHEUS_TAG": {get_param: prometheus_tag} "$GRAFANA_TAG": {get_param: grafana_tag} "$HEAT_CONTAINER_AGENT_TAG": {get_param: heat_container_agent_tag} "$KEYSTONE_AUTH_ENABLED": {get_param: keystone_auth_enabled} "$K8S_KEYSTONE_AUTH_TAG": {get_param: k8s_keystone_auth_tag} "$MONITORING_ENABLED": {get_param: monitoring_enabled} "$PROJECT_ID": {get_param: project_id} "$EXTERNAL_NETWORK_ID": {get_param: external_network} "$TILLER_ENABLED": {get_param: tiller_enabled} "$TILLER_TAG": {get_param: tiller_tag} "$TILLER_NAMESPACE": {get_param: tiller_namespace} "$NODE_PROBLEM_DETECTOR_TAG": {get_param: node_problem_detector_tag} "$NGINX_INGRESS_CONTROLLER_TAG": {get_param: nginx_ingress_controller_tag} "$AUTO_HEALING_ENABLED": {get_param: auto_healing_enabled} "$AUTO_SCALING_ENABLED": {get_param: auto_scaling_enabled} "$DRAINO_TAG": {get_param: draino_tag} "$AUTOSCALER_TAG": {get_param: autoscaler_tag} "$MIN_NODE_COUNT": {get_param: min_node_count} "$MAX_NODE_COUNT": {get_param: max_node_count} - get_file: ../../common/templates/kubernetes/fragments/make-cert.sh - get_file: ../../common/templates/kubernetes/fragments/configure-etcd.sh - get_file: ../../common/templates/kubernetes/fragments/write-kube-os-config.sh - get_file: ../../common/templates/kubernetes/fragments/configure-kubernetes-master.sh - str_replace: template: {get_file: ../../common/templates/fragments/configure-docker-storage.sh} params: $configure_docker_storage_driver: {get_file: ../../common/templates/fragments/configure_docker_storage_driver_atomic.sh} - get_file: ../../common/templates/kubernetes/fragments/enable-services-master.sh - get_file: ../../common/templates/kubernetes/fragments/add-proxy.sh master_config_deployment: type: OS::Heat::SoftwareDeployment properties: signal_transport: HEAT_SIGNAL config: {get_resource: master_config} server: {get_resource: kube-master} actions: ['CREATE'] ###################################################################### # # a single kubernetes master. # # do NOT use "_" (underscore) in the Nova server name # it creates a mismatch between the generated Nova name and its hostname # which can lead to weird problems kube-master: type: OS::Nova::Server properties: name: {get_param: name} image: {get_param: server_image} flavor: {get_param: master_flavor} key_name: {get_param: ssh_key_name} user_data_format: SOFTWARE_CONFIG software_config_transport: POLL_SERVER_HEAT user_data: {get_resource: agent_config} networks: - port: {get_resource: kube_master_eth0} scheduler_hints: { group: { get_param: nodes_server_group_id }} availability_zone: {get_param: availability_zone} kube_master_eth0: type: OS::Neutron::Port properties: network: {get_param: fixed_network} security_groups: - {get_param: secgroup_kube_master_id} fixed_ips: - subnet: {get_param: fixed_subnet} allowed_address_pairs: - ip_address: {get_param: pods_network_cidr} replacement_policy: AUTO kube_master_floating: type: Magnum::Optional::KubeMaster::Neutron::FloatingIP properties: floating_network: {get_param: external_network} port_id: {get_resource: kube_master_eth0} api_pool_member: type: Magnum::Optional::Neutron::LBaaS::PoolMember properties: pool: {get_param: api_pool_id} address: {get_attr: [kube_master_eth0, fixed_ips, 0, ip_address]} subnet: { get_param: fixed_subnet } protocol_port: {get_param: kubernetes_port} etcd_pool_member: type: Magnum::Optional::Neutron::LBaaS::PoolMember properties: pool: {get_param: etcd_pool_id} address: {get_attr: [kube_master_eth0, fixed_ips, 0, ip_address]} subnet: { get_param: fixed_subnet } protocol_port: 2379 ###################################################################### # # etcd storage. This allocates a cinder volume and attaches it # to the master. # etcd_volume: type: Magnum::Optional::Etcd::Volume properties: size: {get_param: etcd_volume_size} etcd_volume_attach: type: Magnum::Optional::Etcd::VolumeAttachment properties: instance_uuid: {get_resource: kube-master} volume_id: {get_resource: etcd_volume} mountpoint: /dev/vdc ###################################################################### # # docker storage. This allocates a cinder volume and attaches it # to the minion. # docker_volume: type: Magnum::Optional::Cinder::Volume properties: size: {get_param: docker_volume_size} volume_type: {get_param: docker_volume_type} docker_volume_attach: type: Magnum::Optional::Cinder::VolumeAttachment properties: instance_uuid: {get_resource: kube-master} volume_id: {get_resource: docker_volume} mountpoint: /dev/vdb upgrade_kubernetes: type: OS::Heat::SoftwareConfig properties: group: script inputs: - name: kube_tag_input config: get_file: ../../common/templates/kubernetes/fragments/upgrade-kubernetes.sh upgrade_kubernetes_deployment: type: OS::Heat::SoftwareDeployment properties: signal_transport: HEAT_SIGNAL config: {get_resource: upgrade_kubernetes} server: {get_resource: kube-master} actions: ['UPDATE'] input_values: kube_tag_input: {get_param: kube_tag} outputs: OS::stack_id: value: { get_resource: kube-master } kube_master_ip: value: {get_attr: [kube_master_eth0, fixed_ips, 0, ip_address]} description: > This is the "private" IP address of the Kubernetes master node. kube_master_external_ip: value: {get_attr: [kube_master_floating, floating_ip_address]} description: > This is the "public" IP address of the Kubernetes master node.