openstack
/
magnum


			
							heat_template_version: queens

description: >
  This template will boot a Kubernetes cluster with one or more
  minions (as specified by the number_of_minions parameter, which
  defaults to 1).

conditions:
  create_cluster_resources:
    equals:
    - get_param: is_cluster_stack
    - true

  is_master:
    and:
    - equals:
      - get_param: master_role
      - "master"
    - equals:
      - get_param: worker_role
      - ""

  is_worker:
    not:
      equals:
      - get_param: worker_role
      - ""

  master_only:
    or:
    - create_cluster_resources
    - is_master

  worker_only:
    or:
    - create_cluster_resources
    - is_worker

parameters:

  # needs to become a list if we want to join master nodes?
  existing_master_private_ip:
    type: string
    default: ""

  is_cluster_stack:
    type: boolean
    default: false

  master_role:
    type: string
    default: ""

  worker_role:
    type: string
    default: ""

  existing_security_group:
    type: string
    default: ""

  ssh_key_name:
    type: string
    description: name of ssh key to be provisioned on our server
    default: ""

  ssh_public_key:
    type: string
    description: The public ssh key to add in all nodes
    default: ""

  external_network:
    type: string
    description: uuid of a network to use for floating ip addresses

  fixed_network:
    type: string
    description: uuid/name of an existing network to use to provision machines
    default: ""

  fixed_network_name:
    type: string
    description: name of a private network to use to provision machines
    default: "private"

  fixed_subnet:
    type: string
    description: uuid/name of an existing subnet to use to provision machines
    default: ""

  master_image:
    type: string
    description: glance image used to boot the server
    # When creating a new minion nodegroup this will not
    # be provided by magnum. So make it default to ""
    default: ""

  minion_image:
    type: string
    description: glance image used to boot the server
    # When creating a new master nodegroup this will not
    # be provided by magnum. So make it default to ""
    default: ""

  master_flavor:
    type: string
    default: m1.small
    description: flavor to use when booting the server for master nodes

  master_nodegroup_name:
    type: string
    default: ""
    description: the name of the nodegroup where the node belongs

  worker_nodegroup_name:
    type: string
    default: ""
    description: the name of the nodegroup where the node belongs

  minion_flavor:
    type: string
    default: m1.small
    description: flavor to use when booting the server for minions

  prometheus_monitoring:
    type: boolean
    default: false
    description: >
      whether or not to have the grafana-prometheus-cadvisor monitoring setup

  grafana_admin_passwd:
    type: string
    default: admin
    hidden: true
    description: >
      admin user password for the Grafana monitoring interface

  dns_nameserver:
    type: comma_delimited_list
    description: address of a DNS nameserver reachable in your environment
    default: 8.8.8.8

  number_of_masters:
    type: number
    description: how many kubernetes masters to spawn
    default: 1

  number_of_minions:
    type: number
    description: how many kubernetes minions to spawn
    default: 1

  fixed_network_cidr:
    type: string
    description: network range for fixed ip network
    default: 10.0.0.0/24

  portal_network_cidr:
    type: string
    description: >
      address range used by kubernetes for service portals
    default: 10.254.0.0/16

  network_driver:
    type: string
    description: network driver to use for instantiating container networks
    default: flannel

  flannel_network_cidr:
    type: string
    description: network range for flannel overlay network
    default: 10.100.0.0/16

  flannel_network_subnetlen:
    type: number
    description: size of subnet assigned to each minion
    default: 24

  flannel_backend:
    type: string
    description: >
      specify the backend for flannel, default vxlan backend
    default: "vxlan"
    constraints:
      - allowed_values: ["udp", "vxlan", "host-gw"]

  system_pods_initial_delay:
    type: number
    description: >
      health check, time to wait for system pods (podmaster, scheduler) to boot
      (in seconds)
    default: 30

  system_pods_timeout:
    type: number
    description: >
      health check, timeout for system pods (podmaster, scheduler) to answer.
      (in seconds)
    default: 5

  admission_control_list:
    type: string
    description: >
      List of admission control plugins to activate
    default: "NamespaceLifecycle,LimitRanger,ServiceAccount,DefaultStorageClass,DefaultTolerationSeconds,MutatingAdmissionWebhook,ValidatingAdmissionWebhook,ResourceQuota"

  kube_allow_priv:
    type: string
    description: >
      whether or not kubernetes should permit privileged containers.
    default: "true"
    constraints:
      - allowed_values: ["true", "false"]

  boot_volume_size:
    type: number
    description: >
      size of the cinder boot volume for nodes root volume

  boot_volume_type:
    type: string
    description: >
      type of the cinder boot volume for nodes root volume

  etcd_volume_size:
    type: number
    description: >
      size of the cinder volume for etcd storage
    default: 0

  etcd_volume_type:
    type: string
    description: >
      type of a cinder volume for etcd storage

  docker_volume_size:
    type: number
    description: >
      size of a cinder volume to allocate to docker for container/image
      storage
    default: 0

  docker_volume_type:
    type: string
    description: >
      type of a cinder volume to allocate to docker for container/image
      storage

  docker_storage_driver:
    type: string
    description: docker storage driver name
    default: "devicemapper"

  cgroup_driver:
    type: string
    description: >
      cgroup driver name that kubelet should use, ideally the same as
      the docker cgroup driver.
    default: "cgroupfs"

  traefik_ingress_controller_tag:
    type: string
    description: tag of the traefik containers to be used.
    default: v1.7.10

  wait_condition_timeout:
    type: number
    description: >
      timeout for the Wait Conditions
    default: 6000

  minions_to_remove:
    type: comma_delimited_list
    description: >
      List of minions to be removed when doing an update. Individual minion may
      be referenced several ways: (1) The resource name (e.g. ['1', '3']),
      (2) The private IP address ['10.0.0.4', '10.0.0.6']. Note: the list should
      be empty when doing an create.
    default: []

  discovery_url:
    type: string
    description: >
      Discovery URL used for bootstrapping the etcd cluster.

  registry_enabled:
    type: boolean
    description: >
      Indicates whether the docker registry is enabled.
    default: false

  registry_port:
    type: number
    description: port of registry service
    default: 5000

  swift_region:
    type: string
    description: region of swift service
    default: ""

  registry_container:
    type: string
    description: >
      name of swift container which docker registry stores images in
    default: "container"

  registry_insecure:
    type: boolean
    description: >
      indicates whether to skip TLS verification between registry and backend storage
    default: true

  registry_chunksize:
    type: number
    description: >
      size fo the data segments for the swift dynamic large objects
    default: 5242880

  volume_driver:
    type: string
    description: volume driver to use for container storage
    default: ""

  region_name:
    type: string
    description: A logically separate section of the cluster

  username:
    type: string
    description: >
      user account

  password:
    type: string
    description: >
      user password, not set in current implementation, only used to
      fill in for Kubernetes config file
    default:
      ChangeMe
    hidden: true

  loadbalancing_protocol:
    type: string
    description: >
      The protocol which is used for load balancing. If you want to change
      tls_disabled option to 'True', please change this to "HTTP".
    default: TCP
    constraints:
      - allowed_values: ["TCP", "HTTP"]

  tls_disabled:
    type: boolean
    description: whether or not to disable TLS
    default: False

  kube_dashboard_enabled:
    type: boolean
    description: whether or not to enable kubernetes dashboard
    default: True

  influx_grafana_dashboard_enabled:
    type: boolean
    description: Enable influxdb with grafana dashboard for data from heapster
    default: False

  verify_ca:
    type: boolean
    description: whether or not to validate certificate authority

  kubernetes_port:
    type: number
    description: >
      The port which are used by kube-apiserver to provide Kubernetes
      service.
    default: 6443

  cluster_uuid:
    type: string
    description: identifier for the cluster this template is generating

  magnum_url:
    type: string
    description: endpoint to retrieve TLS certs from

  http_proxy:
    type: string
    description: http proxy address for docker
    default: ""

  https_proxy:
    type: string
    description: https proxy address for docker
    default: ""

  no_proxy:
    type: string
    description: no proxies for docker
    default: ""

  trustee_domain_id:
    type: string
    description: domain id of the trustee

  trustee_user_id:
    type: string
    description: user id of the trustee

  trustee_username:
    type: string
    description: username of the trustee

  trustee_password:
    type: string
    description: password of the trustee
    hidden: true

  trust_id:
    type: string
    description: id of the trust which is used by the trustee
    hidden: true

  auth_url:
    type: string
    description: url for keystone

  kube_tag:
    type: string
    description: tag of the k8s containers used to provision the kubernetes cluster
    default: v1.14.3

  master_kube_tag:
    type: string
    description: tag of the k8s containers used to provision the kubernetes cluster
    default: v1.14.3

  minion_kube_tag:
    type: string
    description: tag of the k8s containers used to provision the kubernetes cluster
    default: v1.14.3

  # FIXME update cloud_provider_tag when a fix for PVC is released
  # https://github.com/kubernetes/cloud-provider-openstack/pull/405
  cloud_provider_tag:
    type: string
    description:
      tag of the kubernetes/cloud-provider-openstack
      https://hub.docker.com/r/k8scloudprovider/openstack-cloud-controller-manager/tags/
    default: v1.14.0

  cloud_provider_enabled:
    type: boolean
    description: Enable or disable the openstack kubernetes cloud provider

  etcd_tag:
    type: string
    description: tag of the etcd system container
    default: 3.2.26

  coredns_tag:
    type: string
    description: tag for coredns
    default: 1.3.1

  flannel_tag:
    type: string
    description: tag of the flannel container
    default: v0.11.0-amd64

  flannel_cni_tag:
    type: string
    description: tag of the flannel cni container
    default: v0.3.0

  kube_version:
    type: string
    description: version of kubernetes used for kubernetes cluster
    default: v1.14.3

  kube_dashboard_version:
    type: string
    description: version of kubernetes dashboard used for kubernetes cluster
    default: v1.8.3

  insecure_registry_url:
    type: string
    description: insecure registry url
    default: ""

  container_infra_prefix:
    type: string
    description: >
      prefix of container images used in the cluster, kubernetes components,
      kubernetes-dashboard, coredns etc
    constraints:
      - allowed_pattern: "^$|.*/"
    default: ""

  dns_service_ip:
    type: string
    description: >
      address used by Kubernetes DNS service
    default: 10.254.0.10

  dns_cluster_domain:
    type: string
    description: >
      domain name for cluster DNS
    default: "cluster.local"

  openstack_ca:
    type: string
    hidden: true
    description: The OpenStack CA certificate to install on the node.

  nodes_affinity_policy:
    type: string
    description: >
      affinity policy for nodes server group
    constraints:
      - allowed_values: ["affinity", "anti-affinity", "soft-affinity",
                         "soft-anti-affinity"]

  availability_zone:
    type: string
    description: >
      availability zone for master and nodes
    default: ""

  cert_manager_api:
    type: boolean
    description: true if the kubernetes cert api manager should be enabled
    default: false

  ca_key:
    type: string
    description: key of internal ca for the kube certificate api manager
    default: ""
    hidden: true

  calico_tag:
    type: string
    description: tag of the calico containers used to provision the calico node
    default: v3.3.6

  calico_kube_controllers_tag:
    type: string
    description: tag of the kube_controllers used to provision the calico node
    default: v1.0.3

  calico_ipv4pool:
    type: string
    description: Configure the IP pool from which Pod IPs will be chosen
    default: "192.168.0.0/16"

  pods_network_cidr:
    type: string
    description: Configure the IP pool/range from which pod IPs will be chosen

  ingress_controller:
    type: string
    description: >
      ingress controller backend to use
    default: ""

  ingress_controller_role:
    type: string
    description: >
      node role where the ingress controller backend should run
    default: "ingress"

  octavia_ingress_controller_tag:
    type: string
    description: Octavia ingress controller docker image tag.
    default: "1.13.2-alpha"

  kubelet_options:
    type: string
    description: >
      additional options to be passed to the kubelet
    default: ""

  kubeapi_options:
    type: string
    description: >
      additional options to be passed to the api
    default: ""

  kubecontroller_options:
    type: string
    description: >
      additional options to be passed to the controller manager
    default: ""

  kubeproxy_options:
    type: string
    description: >
      additional options to be passed to the kube proxy
    default: ""

  kubescheduler_options:
    type: string
    description: >
      additional options to be passed to the scheduler
    default: ""

  octavia_enabled:
    type: boolean
    description: >
      whether or not to use Octavia for LoadBalancer type service.
    default: False

  kube_service_account_key:
    type: string
    hidden: true
    description: >
      The signed cert will be used to verify the k8s service account tokens
      during authentication.

  kube_service_account_private_key:
    type: string
    hidden: true
    description: >
      The private key will be used to sign generated k8s service account
      tokens.

  prometheus_tag:
    type: string
    description: tag of the prometheus container
    default: v1.8.2

  grafana_tag:
    type: string
    description: tag of grafana container
    default: 5.1.5

  heat_container_agent_tag:
    type: string
    description: tag of the heat_container_agent system container
    default: train-dev

  keystone_auth_enabled:
    type: boolean
    description: >
      true if the keystone authN and authZ should be enabled
    default:
      true

  keystone_auth_default_policy:
    type: string
    description: Json read from /etc/magnum/keystone_auth_default_policy.json
    default: ""

  k8s_keystone_auth_tag:
    type: string
    description: tag of the k8s_keystone_auth container
    default: v1.14.0

  monitoring_enabled:
    type: boolean
    description: Enable or disable prometheus-operator monitoring solution.
    default: false

  prometheus_operator_chart_tag:
    type: string
    description: The stable/prometheus-operator chart version to use.
    default: 5.12.3

  project_id:
    type: string
    description: >
      project id of current project

  tiller_enabled:
    type: boolean
    description: Choose whether to install tiller or not.
    default: false

  tiller_tag:
    type: string
    description: tag of tiller container
    default: "v2.12.3"

  tiller_namespace:
    type: string
    description: namespace where tiller will be installed.
    default: "magnum-tiller"

  auto_healing_enabled:
    type: boolean
    description: >
      true if the auto healing feature should be enabled
    default:
      false

  auto_healing_controller:
    type: string
    description: >
      The service to be deployed for auto-healing.
    default: "draino"

  magnum_auto_healer_tag:
    type: string
    description: tag of the magnum-auto-healer service.
    default: "v1.15.0"

  auto_scaling_enabled:
    type: boolean
    description: >
      true if the auto scaling feature should be enabled
    default:
      false

  node_problem_detector_tag:
    type: string
    description: tag of the node problem detector container
    default: v0.6.2

  nginx_ingress_controller_tag:
    type: string
    description: nginx ingress controller docker image tag
    default: 0.23.0

  draino_tag:
    type: string
    description: tag of the draino container
    default: abf028a

  autoscaler_tag:
    type: string
    description: tag of the autoscaler container
    default: v1.0

  min_node_count:
    type: number
    description: >
      minimum node count of cluster workers when doing scale down
    default: 1

  max_node_count:
    type: number
    description: >
      maximum node count of cluster workers when doing scale up

  update_max_batch_size:
    type: number
    description: >
      max batch size when doing rolling upgrade
    default: 1

  npd_enabled:
    type: boolean
    description: >
      true if the npd service should be launched
    default:
      true

resources:

  ######################################################################
  #
  # network resources.  allocate a network and router for our server.
  # Important: the Load Balancer feature in Kubernetes requires that
  # the name for the fixed_network must be "private" for the
  # address lookup in Kubernetes to work properly
  #

  network:
    condition: create_cluster_resources
    type: ../../common/templates/network.yaml
    properties:
      existing_network: {get_param: fixed_network}
      existing_subnet: {get_param: fixed_subnet}
      private_network_cidr: {get_param: fixed_network_cidr}
      dns_nameserver: {get_param: dns_nameserver}
      external_network: {get_param: external_network}
      private_network_name: {get_param: fixed_network_name}

  api_lb:
    condition: create_cluster_resources
    type: ../../common/templates/lb_api.yaml
    properties:
      fixed_subnet: {get_attr: [network, fixed_subnet]}
      external_network: {get_param: external_network}
      protocol: {get_param: loadbalancing_protocol}
      port: {get_param: kubernetes_port}

  etcd_lb:
    condition: create_cluster_resources
    type: ../../common/templates/lb_etcd.yaml
    properties:
      fixed_subnet: {get_attr: [network, fixed_subnet]}
      protocol: {get_param: loadbalancing_protocol}
      port: 2379

  ######################################################################
  #
  # security groups.  we need to permit network traffic of various
  # sorts.
  #

  secgroup_kube_master:
    condition: create_cluster_resources
    type: OS::Neutron::SecurityGroup
    properties:
      rules:
        - protocol: icmp
        - protocol: tcp
          port_range_min: 22
          port_range_max: 22
        - protocol: tcp
          port_range_min: 7080
          port_range_max: 7080
        - protocol: tcp
          port_range_min: 8080
          port_range_max: 8080
        - protocol: tcp
          port_range_min: 2379
          port_range_max: 2379
        - protocol: tcp
          port_range_min: 2380
          port_range_max: 2380
        - protocol: tcp
          port_range_min: 6443
          port_range_max: 6443
        - protocol: tcp
          port_range_min: 9100
          port_range_max: 9100
        - protocol: tcp
          port_range_min: 10250
          port_range_max: 10250
        - protocol: tcp
          port_range_min: 30000
          port_range_max: 32767
        - protocol: udp
          port_range_min: 8472
          port_range_max: 8472

  secgroup_kube_minion:
    condition: create_cluster_resources
    type: OS::Neutron::SecurityGroup
    properties:
      rules:
        - protocol: icmp
        # Default port range for external service ports.
        # In future, if the option `manage-security-groups` for ccm works
        # well, we could remove this rule here.
        # The PR in ccm is
        # https://github.com/kubernetes/cloud-provider-openstack/pull/491
        - protocol: tcp
          port_range_min: 22
          port_range_max: 22
        - protocol: tcp
          port_range_min: 30000
          port_range_max: 32767
        # allow any traffic from master nodes
        - protocol: tcp
          port_range_min: 1
          port_range_max: 65535
          remote_mode: 'remote_group_id'
          remote_group_id: {get_resource: secgroup_kube_master}
        - protocol: udp
          port_range_min: 1
          port_range_max: 65535
          remote_mode: 'remote_group_id'
          remote_group_id: {get_resource: secgroup_kube_master}

  # allow any traffic between worker nodes
  secgroup_rule_tcp_kube_minion:
    condition: create_cluster_resources
    type: OS::Neutron::SecurityGroupRule
    properties:
      protocol: tcp
      port_range_min: 1
      port_range_max: 65535
      security_group: {get_resource: secgroup_kube_minion}
      remote_group: {get_resource: secgroup_kube_minion}
  secgroup_rule_udp_kube_minion:
    condition: create_cluster_resources
    type: OS::Neutron::SecurityGroupRule
    properties:
      protocol: udp
      port_range_min: 1
      port_range_max: 65535
      security_group: {get_resource: secgroup_kube_minion}
      remote_group: {get_resource: secgroup_kube_minion}

  ######################################################################
  #
  # resources that expose the IPs of either the kube master or a given
  # LBaaS pool depending on whether LBaaS is enabled for the cluster.
  #

  api_address_lb_switch:
    condition: create_cluster_resources
    type: Magnum::ApiGatewaySwitcher
    properties:
      pool_public_ip: {get_attr: [api_lb, floating_address]}
      pool_private_ip: {get_attr: [api_lb, address]}
      master_public_ip: {get_attr: [kube_masters, resource.0.kube_master_external_ip]}
      master_private_ip: {get_attr: [kube_masters, resource.0.kube_master_ip]}

  etcd_address_lb_switch:
    condition: create_cluster_resources
    type: Magnum::ApiGatewaySwitcher
    properties:
      pool_private_ip: {get_attr: [etcd_lb, address]}
      master_private_ip: {get_attr: [kube_masters, resource.0.kube_master_ip]}

  ######################################################################
  #
  # resources that expose the IPs of either floating ip or a given
  # fixed ip depending on whether FloatingIP is enabled for the cluster.
  #

  api_address_floating_switch:
    condition: create_cluster_resources
    type: Magnum::FloatingIPAddressSwitcher
    properties:
      public_ip: {get_attr: [api_address_lb_switch, public_ip]}
      private_ip: {get_attr: [api_address_lb_switch, private_ip]}

  ######################################################################
  #
  # resources that expose one server group for each master and worker nodes
  # separately.
  #

  master_nodes_server_group:
    condition: master_only
    type: OS::Nova::ServerGroup
    properties:
      policies: [{get_param: nodes_affinity_policy}]

  worker_nodes_server_group:
    condition: worker_only
    type: OS::Nova::ServerGroup
    properties:
      policies: [{get_param: nodes_affinity_policy}]

  ######################################################################
  #
  # kubernetes masters. This is a resource group that will create
  # <number_of_masters> masters.
  #

  kube_masters:
    condition: master_only
    type: OS::Heat::ResourceGroup
    depends_on:
      - network
    update_policy:
      rolling_update: {max_batch_size: {get_param: update_max_batch_size}, pause_time: 30}
    properties:
      count: {get_param: number_of_masters}
      resource_def:
        type: kubemaster.yaml
        properties:
          name:
            list_join:
              - '-'
              - [{ get_param: 'OS::stack_name' }, 'master', '%index%']
          nodegroup_role: {get_param: master_role}
          nodegroup_name: {get_param: master_nodegroup_name}
          prometheus_monitoring: {get_param: prometheus_monitoring}
          grafana_admin_passwd: {get_param: grafana_admin_passwd}
          api_public_address: {get_attr: [api_lb, floating_address]}
          api_private_address: {get_attr: [api_lb, address]}
          ssh_key_name: {get_param: ssh_key_name}
          server_image: {get_param: master_image}
          master_flavor: {get_param: master_flavor}
          external_network: {get_param: external_network}
          kube_allow_priv: {get_param: kube_allow_priv}
          boot_volume_size: {get_param: boot_volume_size}
          boot_volume_type: {get_param: boot_volume_type}
          etcd_volume_size: {get_param: etcd_volume_size}
          etcd_volume_type: {get_param: etcd_volume_type}
          docker_volume_size: {get_param: docker_volume_size}
          docker_volume_type: {get_param: docker_volume_type}
          docker_storage_driver: {get_param: docker_storage_driver}
          cgroup_driver: {get_param: cgroup_driver}
          network_driver: {get_param: network_driver}
          flannel_network_cidr: {get_param: flannel_network_cidr}
          flannel_network_subnetlen: {get_param: flannel_network_subnetlen}
          flannel_backend: {get_param: flannel_backend}
          system_pods_initial_delay: {get_param: system_pods_initial_delay}
          system_pods_timeout: {get_param: system_pods_timeout}
          portal_network_cidr: {get_param: portal_network_cidr}
          admission_control_list: {get_param: admission_control_list}
          discovery_url: {get_param: discovery_url}
          cluster_uuid: {get_param: cluster_uuid}
          magnum_url: {get_param: magnum_url}
          traefik_ingress_controller_tag: {get_param: traefik_ingress_controller_tag}
          volume_driver: {get_param: volume_driver}
          region_name: {get_param: region_name}
          fixed_network: {get_attr: [network, fixed_network]}
          fixed_network_name: {get_param: fixed_network_name}
          fixed_subnet: {get_attr: [network, fixed_subnet]}
          api_pool_id: {get_attr: [api_lb, pool_id]}
          etcd_pool_id: {get_attr: [etcd_lb, pool_id]}
          username: {get_param: username}
          password: {get_param: password}
          kubernetes_port: {get_param: kubernetes_port}
          tls_disabled: {get_param: tls_disabled}
          kube_dashboard_enabled: {get_param: kube_dashboard_enabled}
          influx_grafana_dashboard_enabled: {get_param: influx_grafana_dashboard_enabled}
          verify_ca: {get_param: verify_ca}
          secgroup_kube_master_id: {get_resource: secgroup_kube_master}
          http_proxy: {get_param: http_proxy}
          https_proxy: {get_param: https_proxy}
          no_proxy: {get_param: no_proxy}
          kube_tag: {get_param: master_kube_tag}
          cloud_provider_tag: {get_param: cloud_provider_tag}
          cloud_provider_enabled: {get_param: cloud_provider_enabled}
          kube_version: {get_param: kube_version}
          etcd_tag: {get_param: etcd_tag}
          coredns_tag: {get_param: coredns_tag}
          flannel_tag: {get_param: flannel_tag}
          flannel_cni_tag: {get_param: flannel_cni_tag}
          kube_dashboard_version: {get_param: kube_dashboard_version}
          trustee_user_id: {get_param: trustee_user_id}
          trustee_password: {get_param: trustee_password}
          trust_id: {get_param: trust_id}
          auth_url: {get_param: auth_url}
          insecure_registry_url: {get_param: insecure_registry_url}
          container_infra_prefix: {get_param: container_infra_prefix}
          etcd_lb_vip: {get_attr: [etcd_lb, address]}
          dns_service_ip: {get_param: dns_service_ip}
          dns_cluster_domain: {get_param: dns_cluster_domain}
          openstack_ca: {get_param: openstack_ca}
          nodes_server_group_id: {get_resource: master_nodes_server_group}
          availability_zone: {get_param: availability_zone}
          ca_key: {get_param: ca_key}
          cert_manager_api: {get_param: cert_manager_api}
          calico_tag: {get_param: calico_tag}
          calico_kube_controllers_tag: {get_param: calico_kube_controllers_tag}
          calico_ipv4pool: {get_param: calico_ipv4pool}
          pods_network_cidr: {get_param: pods_network_cidr}
          ingress_controller: {get_param: ingress_controller}
          ingress_controller_role: {get_param: ingress_controller_role}
          octavia_ingress_controller_tag: {get_param: octavia_ingress_controller_tag}
          kubelet_options: {get_param: kubelet_options}
          kubeapi_options: {get_param: kubeapi_options}
          kubeproxy_options: {get_param: kubeproxy_options}
          kubecontroller_options: {get_param: kubecontroller_options}
          kubescheduler_options: {get_param: kubescheduler_options}
          octavia_enabled: {get_param: octavia_enabled}
          kube_service_account_key: {get_param: kube_service_account_key}
          kube_service_account_private_key: {get_param: kube_service_account_private_key}
          prometheus_tag: {get_param: prometheus_tag}
          grafana_tag: {get_param: grafana_tag}
          heat_container_agent_tag: {get_param: heat_container_agent_tag}
          keystone_auth_enabled: {get_param: keystone_auth_enabled}
          k8s_keystone_auth_tag: {get_param: k8s_keystone_auth_tag}
          monitoring_enabled: {get_param: monitoring_enabled}
          prometheus_operator_chart_tag: {get_param: prometheus_operator_chart_tag}
          project_id: {get_param: project_id}
          tiller_enabled: {get_param: tiller_enabled}
          tiller_tag: {get_param: tiller_tag}
          tiller_namespace: {get_param: tiller_namespace}
          node_problem_detector_tag: {get_param: node_problem_detector_tag}
          nginx_ingress_controller_tag: {get_param: nginx_ingress_controller_tag}
          auto_healing_enabled: {get_param: auto_healing_enabled}
          auto_healing_controller: {get_param: auto_healing_controller}
          magnum_auto_healer_tag: {get_param: magnum_auto_healer_tag}
          auto_scaling_enabled: {get_param: auto_scaling_enabled}
          draino_tag: {get_param: draino_tag}
          autoscaler_tag: {get_param: autoscaler_tag}
          min_node_count: {get_param: min_node_count}
          max_node_count: {get_param: max_node_count}
          npd_enabled: {get_param: npd_enabled}

  kube_cluster_config:
    condition: create_cluster_resources
    type: OS::Heat::SoftwareConfig
    properties:
      group: script
      config:
        list_join:
          - "\n"
          -
            - str_replace:
                template: {get_file: ../../common/templates/kubernetes/fragments/enable-cert-api-manager.sh}
                params:
                  "$CA_KEY": {get_param: ca_key}
            - get_file: ../../common/templates/kubernetes/fragments/kube-apiserver-to-kubelet-role.sh
            - get_file: ../../common/templates/kubernetes/fragments/core-dns-service.sh
            - get_file: ../../common/templates/kubernetes/fragments/calico-service.sh
            - get_file: ../../common/templates/kubernetes/fragments/flannel-service.sh
            - get_file: ../../common/templates/kubernetes/fragments/enable-helm-tiller.sh
            - str_replace:
                template: {get_file: ../../common/templates/kubernetes/fragments/enable-prometheus-monitoring.sh}
                params:
                  "$ADMIN_PASSWD": {get_param: grafana_admin_passwd}
            - str_replace:
                params:
                  $enable-ingress-traefik: {get_file: ../../common/templates/kubernetes/fragments/enable-ingress-traefik.sh}
                  $enable-ingress-octavia: {get_file: ../../common/templates/kubernetes/fragments/enable-ingress-octavia.sh}
                template: {get_file: ../../common/templates/kubernetes/fragments/enable-ingress-controller.sh}
            - get_file: ../../common/templates/kubernetes/fragments/kube-dashboard-service.sh
            - str_replace:
                template: {get_file: ../../common/templates/kubernetes/fragments/enable-keystone-auth.sh}
                params:
                  "$KEYSTONE_AUTH_DEFAULT_POLICY": {get_param: keystone_auth_default_policy}
            - get_file: ../../common/templates/kubernetes/fragments/enable-auto-healing.sh
            - get_file: ../../common/templates/kubernetes/fragments/enable-auto-scaling.sh
            # Helm Based Installation Configuration Scripts
            - get_file: ../../common/templates/kubernetes/helm/metrics-server.sh
            - str_replace:
                template: {get_file:  ../../common/templates/kubernetes/helm/prometheus-operator.sh}
                params:
                  "${ADMIN_PASSWD}": {get_param: grafana_admin_passwd}
            - get_file: ../../common/templates/kubernetes/helm/ingress-nginx.sh
            - get_file: ../../common/templates/kubernetes/fragments/install-helm-modules.sh

  kube_cluster_deploy:
    condition: create_cluster_resources
    type: OS::Heat::SoftwareDeployment
    properties:
      actions: ['CREATE']
      signal_transport: HEAT_SIGNAL
      config:
        get_resource: kube_cluster_config
      server:
        get_attr: [kube_masters, resource.0]


  ######################################################################
  #
  # kubernetes minions. This is an resource group that will initially
  # create <number_of_minions> minions, and needs to be manually scaled.
  #

  kube_minions:
    condition: worker_only
    type: OS::Heat::ResourceGroup
    depends_on:
      - network
    update_policy:
      rolling_update: {max_batch_size: {get_param: update_max_batch_size}, pause_time: 30}
    properties:
      count: {get_param: number_of_minions}
      removal_policies: [{resource_list: {get_param: minions_to_remove}}]
      resource_def:
        type: kubeminion.yaml
        properties:
          name:
            list_join:
              - '-'
              - [{ get_param: 'OS::stack_name' }, 'node', '%index%']
          prometheus_monitoring: {get_param: prometheus_monitoring}
          nodegroup_role: {get_param: worker_role}
          nodegroup_name: {get_param: worker_nodegroup_name}
          ssh_key_name: {get_param: ssh_key_name}
          server_image: {get_param: minion_image}
          minion_flavor: {get_param: minion_flavor}
          fixed_network:
            if:
              - create_cluster_resources
              - get_attr: [network, fixed_network]
              - get_param: fixed_network
          fixed_subnet:
            if:
              - create_cluster_resources
              - get_attr: [network, fixed_subnet]
              - get_param: fixed_subnet
          network_driver: {get_param: network_driver}
          flannel_network_cidr: {get_param: flannel_network_cidr}
          kube_master_ip:
            if:
              - create_cluster_resources
              - get_attr: [api_address_lb_switch, private_ip]
              - get_param: existing_master_private_ip
          etcd_server_ip:
            if:
              - create_cluster_resources
              - get_attr: [etcd_address_lb_switch, private_ip]
              - get_param: existing_master_private_ip
          external_network: {get_param: external_network}
          kube_allow_priv: {get_param: kube_allow_priv}
          boot_volume_size: {get_param: boot_volume_size}
          boot_volume_type: {get_param: boot_volume_type}
          docker_volume_size: {get_param: docker_volume_size}
          docker_volume_type: {get_param: docker_volume_type}
          docker_storage_driver: {get_param: docker_storage_driver}
          cgroup_driver: {get_param: cgroup_driver}
          wait_condition_timeout: {get_param: wait_condition_timeout}
          registry_enabled: {get_param: registry_enabled}
          registry_port: {get_param: registry_port}
          swift_region: {get_param: swift_region}
          registry_container: {get_param: registry_container}
          registry_insecure: {get_param: registry_insecure}
          registry_chunksize: {get_param: registry_chunksize}
          cluster_uuid: {get_param: cluster_uuid}
          magnum_url: {get_param: magnum_url}
          volume_driver: {get_param: volume_driver}
          region_name: {get_param: region_name}
          auth_url: {get_param: auth_url}
          username: {get_param: username}
          password: {get_param: password}
          kubernetes_port: {get_param: kubernetes_port}
          tls_disabled: {get_param: tls_disabled}
          verify_ca: {get_param: verify_ca}
          secgroup_kube_minion_id:
            if:
              - create_cluster_resources
              - get_resource: secgroup_kube_minion
              - get_param: existing_security_group
          http_proxy: {get_param: http_proxy}
          https_proxy: {get_param: https_proxy}
          no_proxy: {get_param: no_proxy}
          kube_tag: {get_param: minion_kube_tag}
          kube_version: {get_param: kube_version}
          trustee_user_id: {get_param: trustee_user_id}
          trustee_username: {get_param: trustee_username}
          trustee_password: {get_param: trustee_password}
          trustee_domain_id: {get_param: trustee_domain_id}
          trust_id: {get_param: trust_id}
          cloud_provider_enabled: {get_param: cloud_provider_enabled}
          insecure_registry_url: {get_param: insecure_registry_url}
          container_infra_prefix: {get_param: container_infra_prefix}
          dns_service_ip: {get_param: dns_service_ip}
          dns_cluster_domain: {get_param: dns_cluster_domain}
          openstack_ca: {get_param: openstack_ca}
          nodes_server_group_id: {get_resource: worker_nodes_server_group}
          availability_zone: {get_param: availability_zone}
          pods_network_cidr: {get_param: pods_network_cidr}
          kubelet_options: {get_param: kubelet_options}
          kubeproxy_options: {get_param: kubeproxy_options}
          octavia_enabled: {get_param: octavia_enabled}
          heat_container_agent_tag: {get_param: heat_container_agent_tag}
          auto_healing_enabled: {get_param: auto_healing_enabled}
          npd_enabled: {get_param: npd_enabled}
          auto_healing_controller: {get_param: auto_healing_controller}

outputs:

  api_address:
    condition: create_cluster_resources
    value:
      str_replace:
        template: api_ip_address
        params:
          api_ip_address: {get_attr: [api_address_floating_switch, ip_address]}
    description: >
      This is the API endpoint of the Kubernetes cluster. Use this to access
      the Kubernetes API.

  registry_address:
    condition: create_cluster_resources
    value:
      str_replace:
        template: localhost:port
        params:
          port: {get_param: registry_port}
    description:
      This is the url of docker registry server where you can store docker
      images.

  kube_masters_private:
    condition: master_only
    value: {get_attr: [kube_masters, kube_master_ip]}
    description: >
      This is a list of the "private" IP addresses of all the Kubernetes masters.

  kube_masters:
    condition: master_only
    value: {get_attr: [kube_masters, kube_master_external_ip]}
    description: >
      This is a list of the "public" IP addresses of all the Kubernetes masters.
      Use these IP addresses to log in to the Kubernetes masters via ssh.

  kube_minions_private:
    condition: worker_only
    value: {get_attr: [kube_minions, kube_minion_ip]}
    description: >
      This is a list of the "private" IP addresses of all the Kubernetes minions.

  kube_minions:
    condition: worker_only
    value: {get_attr: [kube_minions, kube_minion_external_ip]}
    description: >
      This is a list of the "public" IP addresses of all the Kubernetes minions.
      Use these IP addresses to log in to the Kubernetes minions via ssh.