heat_template_version: 2014-10-16 description: > This is a nested stack that defines a single Kubernetes master, This stack is included by an ResourceGroup resource in the parent template (kubecluster.yaml). parameters: name: type: string description: server name server_image: type: string description: glance image used to boot the server master_flavor: type: string description: flavor to use when booting the server ssh_key_name: type: string description: name of ssh key to be provisioned on our server external_network: type: string description: uuid/name of a network to use for floating ip addresses portal_network_cidr: type: string description: > address range used by kubernetes for service portals kube_allow_priv: type: string description: > whether or not kubernetes should permit privileged containers. constraints: - allowed_values: ["true", "false"] etcd_volume_size: type: number description: > size of a cinder volume to allocate for etcd storage docker_volume_size: type: number description: > size of a cinder volume to allocate to docker for container/image storage docker_volume_type: type: string description: > type of a cinder volume to allocate to docker for container/image storage docker_storage_driver: type: string description: docker storage driver name default: "devicemapper" volume_driver: type: string description: volume driver to use for container storage flannel_network_cidr: type: string description: network range for flannel overlay network flannel_network_subnetlen: type: number description: size of subnet assigned to each master flannel_backend: type: string description: > specify the backend for flannel, default udp backend constraints: - allowed_values: ["udp", "vxlan", "host-gw"] system_pods_initial_delay: type: number description: > health check, time to wait for system pods (podmaster, scheduler) to boot (in seconds) default: 30 system_pods_timeout: type: number description: > health check, timeout for system pods (podmaster, scheduler) to answer. (in seconds) default: 5 admission_control_list: type: string description: > List of admission control plugins to activate discovery_url: type: string description: > Discovery URL used for bootstrapping the etcd cluster. tls_disabled: type: boolean description: whether or not to enable TLS kube_dashboard_enabled: type: boolean description: whether or not to disable kubernetes dashboard influx_grafana_dashboard_enabled: type: boolean description: Enable influxdb with grafana dashboard for data from heapster verify_ca: type: boolean description: whether or not to validate certificate authority kubernetes_port: type: number description: > The port which are used by kube-apiserver to provide Kubernetes service. cluster_uuid: type: string description: identifier for the cluster this template is generating magnum_url: type: string description: endpoint to retrieve TLS certs from prometheus_monitoring: type: boolean description: > whether or not to have prometheus and grafana deployed grafana_admin_passwd: type: string hidden: true description: > admin user password for the Grafana monitoring interface api_public_address: type: string description: Public IP address of the Kubernetes master server. default: "" api_private_address: type: string description: Private IP address of the Kubernetes master server. default: "" fixed_network: type: string description: Network from which to allocate fixed addresses. fixed_subnet: type: string description: Subnet from which to allocate fixed addresses. network_driver: type: string description: network driver to use for instantiating container networks wait_condition_timeout: type: number description : > timeout for the Wait Conditions secgroup_kube_master_id: type: string description: ID of the security group for kubernetes master. api_pool_id: type: string description: ID of the load balancer pool of k8s API server. etcd_pool_id: type: string description: ID of the load balancer pool of etcd server. auth_url: type: string description: > url for kubernetes to authenticate username: type: string description: > user account password: type: string description: > user password http_proxy: type: string description: http proxy address for docker https_proxy: type: string description: https proxy address for docker no_proxy: type: string description: no proxies for docker kube_tag: type: string description: tag of the k8s containers used to provision the kubernetes cluster etcd_tag: type: string description: tag of the etcd system container kube_version: type: string description: version of kubernetes used for kubernetes cluster kube_dashboard_version: type: string description: version of kubernetes dashboard used for kubernetes cluster trustee_user_id: type: string description: user id of the trustee trustee_password: type: string description: password of the trustee hidden: true trust_id: type: string description: id of the trust which is used by the trustee hidden: true insecure_registry_url: type: string description: insecure registry url container_infra_prefix: type: string description: > prefix of container images used in the cluster, kubernetes components, kubernetes-dashboard, coredns etc etcd_lb_vip: type: string description: > etcd lb vip private used to generate certs on master. default: "" dns_service_ip: type: string description: > address used by Kubernetes DNS service dns_cluster_domain: type: string description: > domain name for cluster DNS openstack_ca: type: string description: The OpenStack CA certificate to install on the node. nodes_server_group_id: type: string description: ID of the server group for kubernetes cluster nodes. availability_zone: type: string description: > availability zone for master and nodes default: "" ca_key: type: string description: key of internal ca for the kube certificate api manager hidden: true cert_manager_api: type: boolean description: true if the kubernetes cert api manager should be enabled default: false calico_tag: type: string description: tag of the calico containers used to provision the calico node calico_cni_tag: type: string description: tag of the cni used to provision the calico node calico_kube_controllers_tag: type: string description: tag of the kube_controllers used to provision the calico node calico_ipv4pool: type: string description: Configure the IP pool from which Pod IPs will be chosen pods_network_cidr: type: string description: Configure the IP pool/range from which pod IPs will be chosen ingress_controller: type: string description: > ingress controller backend to use ingress_controller_role: type: string description: > node role where the ingress controller should run kubelet_options: type: string description: > additional options to be passed to the kubelet kubeapi_options: type: string description: > additional options to be passed to the api kubecontroller_options: type: string description: > additional options to be passed to the controller manager kubeproxy_options: type: string description: > additional options to be passed to the kube proxy kubescheduler_options: type: string description: > additional options to be passed to the scheduler octavia_enabled: type: boolean description: > whether or not to use Octavia for LoadBalancer type service. default: False container_runtime: type: string description: > Container runtime to use with Kubernetes. hyperkube_image: type: string description: > Docker registry used for hyperkube image resources: master_wait_handle: type: OS::Heat::WaitConditionHandle master_wait_condition: type: OS::Heat::WaitCondition depends_on: kube-master properties: handle: {get_resource: master_wait_handle} timeout: {get_param: wait_condition_timeout} ###################################################################### # # resource that exposes the IPs of either the kube master or the API # LBaaS pool depending on whether LBaaS is enabled for the cluster. # api_address_switch: type: Magnum::ApiGatewaySwitcher properties: pool_public_ip: {get_param: api_public_address} pool_private_ip: {get_param: api_private_address} master_public_ip: {get_attr: [kube_master_floating, floating_ip_address]} master_private_ip: {get_attr: [kube_master_eth0, fixed_ips, 0, ip_address]} ###################################################################### # # software configs. these are components that are combined into # a multipart MIME user-data archive. # write_heat_params: type: OS::Heat::SoftwareConfig properties: group: ungrouped config: str_replace: template: {get_file: fragments/write-heat-params-master.yaml} params: "$KUBE_API_PUBLIC_ADDRESS": {get_attr: [api_address_switch, public_ip]} "$KUBE_API_PRIVATE_ADDRESS": {get_attr: [api_address_switch, private_ip]} "$KUBE_NODE_PUBLIC_IP": {get_attr: [kube_master_floating, floating_ip_address]} "$KUBE_NODE_IP": {get_attr: [kube_master_eth0, fixed_ips, 0, ip_address]} "$KUBE_ALLOW_PRIV": {get_param: kube_allow_priv} "$ETCD_VOLUME": {get_resource: etcd_volume} "$ETCD_VOLUME_SIZE": {get_param: etcd_volume_size} "$DOCKER_VOLUME": {get_resource: docker_volume} "$DOCKER_VOLUME_SIZE": {get_param: docker_volume_size} "$FLANNEL_NETWORK_CIDR": {get_param: flannel_network_cidr} "$FLANNEL_NETWORK_SUBNETLEN": {get_param: flannel_network_subnetlen} "$FLANNEL_BACKEND": {get_param: flannel_backend} "$SYSTEM_PODS_INITIAL_DELAY": {get_param: system_pods_initial_delay} "$SYSTEM_PODS_TIMEOUT": {get_param: system_pods_timeout} "$PORTAL_NETWORK_CIDR": {get_param: portal_network_cidr} "$ADMISSION_CONTROL_LIST": {get_param: admission_control_list} "$CLUSTER_SUBNET": {get_param: fixed_subnet} "$ETCD_DISCOVERY_URL": {get_param: discovery_url} "$WAIT_CURL": {get_attr: [master_wait_handle, curl_cli]} "$NETWORK_DRIVER": {get_param: network_driver} "$KUBE_API_PORT": {get_param: kubernetes_port} "$TLS_DISABLED": {get_param: tls_disabled} "$VERIFY_CA": {get_param: verify_ca} "$KUBE_DASHBOARD_ENABLED": {get_param: kube_dashboard_enabled} "$INFLUX_GRAFANA_DASHBOARD_ENABLED": {get_param: influx_grafana_dashboard_enabled} "$KUBE_VERSION": {get_param: kube_version} "$KUBE_DASHBOARD_VERSION": {get_param: kube_dashboard_version} "$CLUSTER_UUID": {get_param: cluster_uuid} "$MAGNUM_URL": {get_param: magnum_url} "$HTTP_PROXY": {get_param: http_proxy} "$HTTPS_PROXY": {get_param: https_proxy} "$NO_PROXY": {get_param: no_proxy} "$TRUSTEE_USER_ID": {get_param: trustee_user_id} "$TRUSTEE_PASSWORD": {get_param: trustee_password} "$TRUST_ID": {get_param: trust_id} "$AUTH_URL": {get_param: auth_url} "$KUBE_CERTS_PATH": "/etc/kubernetes/ssl" "$HOST_CERTS_PATH": "/usr/share/ca-certificates" "$HYPERKUBE_IMAGE_REPO": str_replace: template: insecure_registry_urlhyperkube_image params: insecure_registry_url: { get_param: insecure_registry_url } hyperkube_image: { get_param: hyperkube_image } "$INSECURE_REGISTRY_URL": {get_param: insecure_registry_url} "$CONTAINER_RUNTIME": {get_param: container_runtime} "$ETCD_LB_VIP": {get_param: etcd_lb_vip} "$DNS_SERVICE_IP": {get_param: dns_service_ip} "$DNS_CLUSTER_DOMAIN": {get_param: dns_cluster_domain} "$OCTAVIA_ENABLED": {get_param: octavia_enabled} write_kubeconfig: type: OS::Heat::SoftwareConfig properties: group: ungrouped config: {get_file: fragments/write-master-kubeconfig.yaml} enable_docker_mount: type: OS::Heat::SoftwareConfig properties: group: ungrouped config: {get_file: fragments/enable-docker-mount.yaml} add_ext_ca_certs: type: OS::Heat::SoftwareConfig properties: group: ungrouped config: str_replace: params: $OPENSTACK_CA: {get_param: openstack_ca} template: {get_file: fragments/add-ext-ca-certs.yaml} configure_etcd: type: OS::Heat::SoftwareConfig properties: group: ungrouped config: {get_file: fragments/configure-etcd.yaml} make_cert: type: OS::Heat::SoftwareConfig properties: group: ungrouped config: {get_file: fragments/make-cert.yaml} write_network_config: type: OS::Heat::SoftwareConfig properties: group: ungrouped config: {get_file: fragments/write-network-config.yaml} enable_network_service: type: OS::Heat::SoftwareConfig properties: group: ungrouped config: {get_file: fragments/enable-network-service.yaml} enable_kubelet: type: OS::Heat::SoftwareConfig properties: group: ungrouped config: {get_file: fragments/enable-kubelet-master.yaml} enable_kube_apiserver: type: OS::Heat::SoftwareConfig properties: group: ungrouped config: {get_file: fragments/enable-kube-apiserver.yaml} create_kube_namespace: type: OS::Heat::SoftwareConfig properties: group: ungrouped config: {get_file: fragments/create-kube-namespace.yaml} enable_kube_proxy: type: OS::Heat::SoftwareConfig properties: group: ungrouped config: {get_file: fragments/enable-kube-proxy-master.yaml} enable_kube_controller_manager: type: OS::Heat::SoftwareConfig properties: group: ungrouped config: {get_file: fragments/enable-kube-controller-manager.yaml} enable_kube_scheduler: type: OS::Heat::SoftwareConfig properties: group: ungrouped config: {get_file: fragments/enable-kube-scheduler.yaml} enable_kube_dashboard: type: OS::Heat::SoftwareConfig properties: group: ungrouped config: {get_file: fragments/enable-kube-dashboard.yaml} wc_notify: type: OS::Heat::SoftwareConfig properties: group: ungrouped config: {get_file: fragments/wc-notify.yaml} add_proxy: type: OS::Heat::SoftwareConfig properties: group: ungrouped config: {get_file: fragments/add-proxy.yaml} configure_docker: type: OS::Heat::SoftwareConfig properties: group: ungrouped config: {get_file: fragments/configure-docker.yaml} enable_coredns: type: OS::Heat::SoftwareConfig properties: group: ungrouped config: {get_file: fragments/enable-coredns.yaml} kube_master_init: type: OS::Heat::SoftwareConfig properties: group: ungrouped config: str_replace: template: | $add_ext_ca_certs $write_heat_params $write_kubeconfig $enable_docker_mount $make_cert $configure_docker $add_proxy $configure_etcd $write_network_config $enable_network_service $enable_kubelet $enable_kube_apiserver $create_kube_namespace $enable_kube_proxy $enable_kube_controller_manager $enable_kube_scheduler $enable_kube_dashboard $enable_coredns $wc_notify coreos: units: - name: "add-ext-ca-certs.service" command: "start" - name: "make-cert.service" command: "start" - name: "enable-docker-mount.service" command: "start" - name: "configure-docker.service" command: "start" - name: "add-proxy.service" command: "start" - name: "configure-etcd.service" command: "start" - name: "write-network-config.service" command: "start" - name: "enable-network-service.service" command: "start" - name: "enable-kubelet.service" command: "start" - name: "enable-kube-apiserver.service" command: "start" - name: "create-kube-namespace.service" command: "start" - name: "enable-kube-proxy.service" command: "start" - name: "enable-kube-controller-manager.service" command: "start" - name: "enable-kube-scheduler.service" command: "start" - name: "enable-kube-dashboard.service" command: "start" - name: "enable-coredns.service" command: "start" - name: "wc-notify.service" command: "start" params: "$add_ext_ca_certs": {get_attr: [add_ext_ca_certs, config]} "$write_heat_params": {get_attr: [write_heat_params, config]} "$write_kubeconfig": {get_attr: [write_kubeconfig, config]} "$enable_docker_mount": {get_attr: [enable_docker_mount, config]} "$make_cert": {get_attr: [make_cert, config]} "$configure_docker": {get_attr: [configure_docker, config]} "$add_proxy": {get_attr: [add_proxy, config]} "$configure_etcd": {get_attr: [configure_etcd, config]} "$write_network_config": {get_attr: [write_network_config, config]} "$enable_network_service": {get_attr: [enable_network_service, config]} "$enable_kubelet": {get_attr: [enable_kubelet, config]} "$enable_kube_apiserver": {get_attr: [enable_kube_apiserver, config]} "$create_kube_namespace": {get_attr: [create_kube_namespace, config]} "$enable_kube_proxy": {get_attr: [enable_kube_proxy, config]} "$enable_kube_controller_manager": {get_attr: [enable_kube_controller_manager, config]} "$enable_kube_scheduler": {get_attr: [enable_kube_scheduler, config]} "$enable_kube_dashboard": {get_attr: [enable_kube_dashboard, config]} "$enable_coredns": {get_attr: [enable_coredns, config]} "$wc_notify": {get_attr: [wc_notify, config]} ###################################################################### # # a single kubernetes master. # # do NOT use "_" (underscore) in the Nova server name # it creates a mismatch between the generated Nova name and its hostname # which can lead to weird problems kube-master: type: OS::Nova::Server properties: name: {get_param: name} image: {get_param: server_image} flavor: {get_param: master_flavor} key_name: {get_param: ssh_key_name} user_data_format: RAW user_data: {get_resource: kube_master_init} networks: - port: {get_resource: kube_master_eth0} scheduler_hints: { group: { get_param: nodes_server_group_id }} kube_master_eth0: type: OS::Neutron::Port properties: network: {get_param: fixed_network} security_groups: - {get_param: secgroup_kube_master_id} fixed_ips: - subnet: {get_param: fixed_subnet} allowed_address_pairs: - ip_address: {get_param: flannel_network_cidr} replacement_policy: AUTO kube_master_floating: type: Magnum::Optional::KubeMaster::Neutron::FloatingIP properties: floating_network: {get_param: external_network} port_id: {get_resource: kube_master_eth0} api_pool_member: type: Magnum::Optional::Neutron::LBaaS::PoolMember properties: pool: {get_param: api_pool_id} address: {get_attr: [kube_master_eth0, fixed_ips, 0, ip_address]} subnet: { get_param: fixed_subnet } protocol_port: {get_param: kubernetes_port} etcd_pool_member: type: Magnum::Optional::Neutron::LBaaS::PoolMember properties: pool: {get_param: etcd_pool_id} address: {get_attr: [kube_master_eth0, fixed_ips, 0, ip_address]} subnet: { get_param: fixed_subnet } protocol_port: 2379 ###################################################################### # # etcd storage. This allocates a cinder volume and attaches it # to the master. # etcd_volume: type: Magnum::Optional::Etcd::Volume properties: size: {get_param: etcd_volume_size} etcd_volume_attach: type: Magnum::Optional::Etcd::VolumeAttachment properties: instance_uuid: {get_resource: kube-master} volume_id: {get_resource: etcd_volume} mountpoint: /dev/vdc ###################################################################### # # docker storage. This allocates a cinder volume and attaches it # to the minion. # docker_volume: type: Magnum::Optional::Cinder::Volume properties: size: {get_param: docker_volume_size} volume_type: {get_param: docker_volume_type} docker_volume_attach: type: Magnum::Optional::Cinder::VolumeAttachment properties: instance_uuid: {get_resource: kube-master} volume_id: {get_resource: docker_volume} mountpoint: /dev/vdb outputs: kube_master_ip: value: {get_attr: [kube_master_eth0, fixed_ips, 0, ip_address]} description: > This is the "private" IP address of the Kubernetes master node. kube_master_external_ip: value: {get_attr: [kube_master_floating, floating_ip_address]} description: > This is the "public" IP address of the Kubernetes master node.