Container Infrastructure Management Service for OpenStack
You can not select more than 25 topics Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.
 
 
 
 
 

1526 lines
48 KiB

heat_template_version: queens
description: >
This template will boot a Kubernetes cluster with one or more
minions (as specified by the number_of_minions parameter, which
defaults to 1).
conditions:
create_cluster_resources:
equals:
- get_param: is_cluster_stack
- true
is_master:
and:
- equals:
- get_param: master_role
- "master"
- equals:
- get_param: worker_role
- ""
is_worker:
not:
equals:
- get_param: worker_role
- ""
master_only:
or:
- create_cluster_resources
- is_master
worker_only:
or:
- create_cluster_resources
- is_worker
calico_v3_3:
yaql:
expression: $.data.calico_tag.startsWith("v3.3.")
data:
calico_tag: {get_param: calico_tag}
parameters:
# needs to become a list if we want to join master nodes?
existing_master_private_ip:
type: string
default: ""
is_cluster_stack:
type: boolean
default: false
master_role:
type: string
default: ""
worker_role:
type: string
default: ""
existing_security_group:
type: string
default: ""
ssh_key_name:
type: string
description: name of ssh key to be provisioned on our server
default: ""
ssh_public_key:
type: string
description: The public ssh key to add in all nodes
default: ""
external_network:
type: string
description: uuid of a network to use for floating ip addresses
fixed_network:
type: string
description: uuid/name of an existing network to use to provision machines
default: ""
fixed_network_name:
type: string
description: name of a private network to use to provision machines
fixed_subnet:
type: string
description: uuid/name of an existing subnet to use to provision machines
default: ""
master_image:
type: string
description: glance image used to boot the server
# When creating a new minion nodegroup this will not
# be provided by magnum. So make it default to ""
default: ""
minion_image:
type: string
description: glance image used to boot the server
# When creating a new master nodegroup this will not
# be provided by magnum. So make it default to ""
default: ""
master_flavor:
type: string
default: m1.small
description: flavor to use when booting the server for master nodes
master_nodegroup_name:
type: string
default: ""
description: the name of the nodegroup where the node belongs
worker_nodegroup_name:
type: string
default: ""
description: the name of the nodegroup where the node belongs
heapster_enabled:
type: boolean
description: enable/disable the use of heapster
default: false
metrics_server_enabled:
type: boolean
description: enable/disable the use of metrics-server
default: true
metrics_server_chart_tag:
type: string
description: tag of the stable/metrics-server chart to install
default: v2.8.8
minion_flavor:
type: string
default: m1.small
description: flavor to use when booting the server for minions
prometheus_monitoring:
type: boolean
default: false
description: >
whether or not to have the grafana-prometheus-cadvisor monitoring setup
grafana_admin_passwd:
type: string
default: admin
hidden: true
description: >
admin user password for the Grafana monitoring interface
dns_nameserver:
type: comma_delimited_list
description: address of a DNS nameserver reachable in your environment
default: 8.8.8.8
number_of_masters:
type: number
description: how many kubernetes masters to spawn
default: 1
number_of_minions:
type: number
description: how many kubernetes minions to spawn
default: 1
fixed_subnet_cidr:
type: string
description: network range for fixed ip network
default: 10.0.0.0/24
portal_network_cidr:
type: string
description: >
address range used by kubernetes for service portals
default: 10.254.0.0/16
network_driver:
type: string
description: network driver to use for instantiating container networks
default: flannel
flannel_network_cidr:
type: string
description: network range for flannel overlay network
default: 10.100.0.0/16
flannel_network_subnetlen:
type: number
description: size of subnet assigned to each minion
default: 24
flannel_backend:
type: string
description: >
specify the backend for flannel, default vxlan backend
default: "vxlan"
constraints:
- allowed_values: ["udp", "vxlan", "host-gw"]
system_pods_initial_delay:
type: number
description: >
health check, time to wait for system pods (podmaster, scheduler) to boot
(in seconds)
default: 30
system_pods_timeout:
type: number
description: >
health check, timeout for system pods (podmaster, scheduler) to answer.
(in seconds)
default: 5
admission_control_list:
type: string
description: >
List of admission control plugins to activate
default: "PodSecurityPolicy,NamespaceLifecycle,LimitRanger,ServiceAccount,ResourceQuota,TaintNodesByCondition,Priority,DefaultTolerationSeconds,DefaultStorageClass,StorageObjectInUseProtection,PersistentVolumeClaimResize,MutatingAdmissionWebhook,ValidatingAdmissionWebhook,RuntimeClass"
kube_allow_priv:
type: string
description: >
whether or not kubernetes should permit privileged containers.
default: "true"
constraints:
- allowed_values: ["true", "false"]
boot_volume_size:
type: number
description: >
size of the cinder boot volume for nodes root volume
boot_volume_type:
type: string
description: >
type of the cinder boot volume for nodes root volume
etcd_volume_size:
type: number
description: >
size of the cinder volume for etcd storage
default: 0
etcd_volume_type:
type: string
description: >
type of a cinder volume for etcd storage
docker_volume_size:
type: number
description: >
size of a cinder volume to allocate to docker for container/image
storage
default: 0
docker_volume_type:
type: string
description: >
type of a cinder volume to allocate to docker for container/image
storage
docker_storage_driver:
type: string
description: docker storage driver name
default: "overlay2"
constraints:
- allowed_pattern: "^(?!devicemapper$).*"
cgroup_driver:
type: string
description: >
cgroup driver name that kubelet should use, ideally the same as
the docker cgroup driver.
default: "cgroupfs"
traefik_ingress_controller_tag:
type: string
description: tag of the traefik containers to be used.
default: v1.7.19
wait_condition_timeout:
type: number
description: >
timeout for the Wait Conditions
default: 6000
minions_to_remove:
type: comma_delimited_list
description: >
List of minions to be removed when doing an update. Individual minion may
be referenced several ways: (1) The resource name (e.g. ['1', '3']),
(2) The private IP address ['10.0.0.4', '10.0.0.6']. Note: the list should
be empty when doing an create.
default: []
discovery_url:
type: string
description: >
Discovery URL used for bootstrapping the etcd cluster.
registry_enabled:
type: boolean
description: >
Indicates whether the docker registry is enabled.
default: false
registry_port:
type: number
description: port of registry service
default: 5000
swift_region:
type: string
description: region of swift service
default: ""
registry_container:
type: string
description: >
name of swift container which docker registry stores images in
default: "container"
registry_insecure:
type: boolean
description: >
indicates whether to skip TLS verification between registry and backend storage
default: true
registry_chunksize:
type: number
description: >
size fo the data segments for the swift dynamic large objects
default: 5242880
volume_driver:
type: string
description: volume driver to use for container storage
default: ""
region_name:
type: string
description: A logically separate section of the cluster
username:
type: string
description: >
user account
password:
type: string
description: >
user password, not set in current implementation, only used to
fill in for Kubernetes config file
default:
ChangeMe
hidden: true
loadbalancing_protocol:
type: string
description: >
The protocol which is used for load balancing. If you want to change
tls_disabled option to 'True', please change this to "HTTP".
default: TCP
constraints:
- allowed_values: ["TCP", "HTTP"]
tls_disabled:
type: boolean
description: whether or not to disable TLS
default: False
kube_dashboard_enabled:
type: boolean
description: whether or not to enable kubernetes dashboard
default: True
influx_grafana_dashboard_enabled:
type: boolean
description: Enable influxdb with grafana dashboard for data from heapster
default: False
verify_ca:
type: boolean
description: whether or not to validate certificate authority
kubernetes_port:
type: number
description: >
The port which are used by kube-apiserver to provide Kubernetes
service.
default: 6443
cluster_uuid:
type: string
description: identifier for the cluster this template is generating
magnum_url:
type: string
description: endpoint to retrieve TLS certs from
http_proxy:
type: string
description: http proxy address for docker
default: ""
https_proxy:
type: string
description: https proxy address for docker
default: ""
no_proxy:
type: string
description: no proxies for docker
default: ""
trustee_domain_id:
type: string
description: domain id of the trustee
trustee_user_id:
type: string
description: user id of the trustee
trustee_username:
type: string
description: username of the trustee
trustee_password:
type: string
description: password of the trustee
hidden: true
trust_id:
type: string
description: id of the trust which is used by the trustee
hidden: true
auth_url:
type: string
description: url for keystone
kube_tag:
type: string
description:
tag of the k8s containers used to provision the kubernetes cluster
https://github.com/kubernetes/kubernetes/releases
default: v1.18.2
master_kube_tag:
type: string
description:
tag of the k8s containers used to provision the kubernetes cluster
https://github.com/kubernetes/kubernetes/releases
default: v1.18.2
minion_kube_tag:
type: string
description:
tag of the k8s containers used to provision the kubernetes cluster
https://github.com/kubernetes/kubernetes/releases
default: v1.18.2
cloud_provider_tag:
type: string
description:
tag of the k8scloudprovider/openstack-cloud-controller-manager
https://hub.docker.com/r/k8scloudprovider/openstack-cloud-controller-manager/tags/
default: v1.18.0
cloud_provider_enabled:
type: boolean
description: Enable or disable the openstack kubernetes cloud provider
etcd_tag:
type: string
description: tag of the etcd system container
default: v3.4.6
coredns_tag:
type: string
description: tag for coredns
default: 1.6.6
flannel_tag:
type: string
description: tag of the flannel container
default: v0.12.0-amd64
flannel_cni_tag:
type: string
description: tag of the flannel cni container
default: v0.3.0
kube_version:
type: string
description: version of kubernetes used for kubernetes cluster
default: v1.18.2
kube_dashboard_version:
type: string
description: version of kubernetes dashboard used for kubernetes cluster
default: v2.0.0
metrics_scraper_tag:
type: string
description: >
Tag of metrics-scraper for kubernetes dashboard.
default: v1.0.4
insecure_registry_url:
type: string
description: insecure registry url
default: ""
container_infra_prefix:
type: string
description: >
prefix of container images used in the cluster, kubernetes components,
kubernetes-dashboard, coredns etc
constraints:
- allowed_pattern: "^$|.*/"
default: ""
dns_service_ip:
type: string
description: >
address used by Kubernetes DNS service
default: 10.254.0.10
dns_cluster_domain:
type: string
description: >
domain name for cluster DNS
default: "cluster.local"
openstack_ca:
type: string
hidden: true
description: The OpenStack CA certificate to install on the node.
nodes_affinity_policy:
type: string
description: >
affinity policy for nodes server group
constraints:
- allowed_values: ["affinity", "anti-affinity", "soft-affinity",
"soft-anti-affinity"]
availability_zone:
type: string
description: >
availability zone for master and nodes
default: ""
cert_manager_api:
type: boolean
description: true if the kubernetes cert api manager should be enabled
default: false
ca_key:
type: string
description: key of internal ca for the kube certificate api manager
default: ""
hidden: true
calico_tag:
type: string
description: tag of the calico containers used to provision the calico node
default: v3.13.1
calico_kube_controllers_tag:
type: string
description: tag of the kube_controllers used to provision the calico node
default: v1.0.3
calico_ipv4pool:
type: string
description: Configure the IP pool from which Pod IPs will be chosen
default: "10.100.0.0/16"
calico_ipv4pool_ipip:
type: string
description: IPIP Mode to use for the IPv4 POOL created at start up
default: "Off"
constraints:
- allowed_values: ["Always", "CrossSubnet", "Never", "Off"]
pods_network_cidr:
type: string
description: Configure the IP pool/range from which pod IPs will be chosen
ingress_controller:
type: string
description: >
ingress controller backend to use
default: ""
ingress_controller_role:
type: string
description: >
node role where the ingress controller backend should run
default: "ingress"
octavia_ingress_controller_tag:
type: string
description: Octavia ingress controller docker image tag.
tag of the k8scloudprovider/octavia-ingress-controller container
https://hub.docker.com/r/k8scloudprovider/octavia-ingress-controller/tags/
default: v1.18.0
kubelet_options:
type: string
description: >
additional options to be passed to the kubelet
default: ""
kubeapi_options:
type: string
description: >
additional options to be passed to the api
default: ""
kubecontroller_options:
type: string
description: >
additional options to be passed to the controller manager
default: ""
kubeproxy_options:
type: string
description: >
additional options to be passed to the kube proxy
default: ""
kubescheduler_options:
type: string
description: >
additional options to be passed to the scheduler
default: ""
octavia_enabled:
type: boolean
description: >
whether or not to use Octavia for LoadBalancer type service.
default: False
kube_service_account_key:
type: string
hidden: true
description: >
The signed cert will be used to verify the k8s service account tokens
during authentication.
kube_service_account_private_key:
type: string
hidden: true
description: >
The private key will be used to sign generated k8s service account
tokens.
prometheus_tag:
type: string
description: tag of the prometheus container
default: v1.8.2
grafana_tag:
type: string
description: tag of grafana container
default: 5.1.5
heat_container_agent_tag:
type: string
description: tag of the heat_container_agent system container
default: victoria-dev
keystone_auth_enabled:
type: boolean
description: >
true if the keystone authN and authZ should be enabled
default:
true
keystone_auth_default_policy:
type: string
description: Json read from /etc/magnum/keystone_auth_default_policy.json
default: ""
k8s_keystone_auth_tag:
type: string
description:
tag of the k8scloudprovider/k8s-keystone-auth container
https://hub.docker.com/r/k8scloudprovider/k8s-keystone-auth/tags/
default: v1.18.0
monitoring_enabled:
type: boolean
description: Enable or disable prometheus-operator monitoring solution.
default: false
prometheus_operator_chart_tag:
type: string
description: The stable/prometheus-operator chart version to use.
default: v8.12.13
prometheus_adapter_enabled:
type: boolean
description: Enable or disable prometheus-adapter custom metrics.
default: true
prometheus_adapter_chart_tag:
type: string
description: The stable/prometheus-adapter chart version to use.
default: 1.4.0
prometheus_adapter_configmap:
type: string
description: The prometheus adapter rules ConfigMap name to use as overwrite.
default: ""
project_id:
type: string
description: >
project id of current project
tiller_enabled:
type: boolean
description: Choose whether to install tiller or not.
default: false
tiller_tag:
type: string
description: tag of tiller container
default: "v2.16.7"
tiller_namespace:
type: string
description: namespace where tiller will be installed.
default: "magnum-tiller"
helm_client_url:
type: string
description: url of helm client tarball
default: ""
helm_client_sha256:
type: string
description: sha256 of helm client tarball
default: "018f9908cb950701a5d59e757653a790c66d8eda288625dbb185354ca6f41f6b"
helm_client_tag:
type: string
description: >
release tag of helm client
https://github.com/helm/helm/releases
default: "v3.2.1"
auto_healing_enabled:
type: boolean
description: >
true if the auto healing feature should be enabled
default:
false
auto_healing_controller:
type: string
description: >
The service to be deployed for auto-healing.
default: "draino"
magnum_auto_healer_tag:
type: string
description:
tag of the k8scloudprovider/magnum-auto-healer container
https://hub.docker.com/r/k8scloudprovider/magnum-auto-healer/tags/
default: v1.18.0
auto_scaling_enabled:
type: boolean
description: >
true if the auto scaling feature should be enabled
default:
false
cinder_csi_enabled:
type: boolean
description: >
true if the cinder csi feature should be enabled
default:
true
cinder_csi_plugin_tag:
type: string
description: tag of cinder csi plugin
tag of the k8scloudprovider/cinder-csi-plugin container
https://hub.docker.com/r/k8scloudprovider/cinder-csi-plugin/tags/
default: v1.18.0
csi_attacher_tag:
type: string
description: tag of csi attacher
default: v2.0.0
csi_provisioner_tag:
type: string
description: tag of csi provisioner
default: v1.4.0
csi_snapshotter_tag:
type: string
description: tag of csi snapshotter
default: v1.2.2
csi_resizer_tag:
type: string
description: tag of csi resizer
default: v0.3.0
csi_node_driver_registrar_tag:
type: string
description: tag of csi node driver registrar
default: v1.1.0
node_problem_detector_tag:
type: string
description: tag of the node problem detector container
default: v0.6.2
nginx_ingress_controller_tag:
type: string
description: nginx ingress controller docker image tag
default: 0.32.0
nginx_ingress_controller_chart_tag:
type: string
description: nginx ingress controller helm chart tag
default: v1.36.3
draino_tag:
type: string
description: tag of the draino container
default: abf028a
autoscaler_tag:
type: string
description: tag of the autoscaler container
tag of the openstackmagnum/cluster-autoscaler container
https://hub.docker.com/r/openstackmagnum/cluster-autoscaler/tags/
default: v1.18.1
min_node_count:
type: number
description: >
minimum node count of cluster workers when doing scale down
default: 1
max_node_count:
type: number
description: >
maximum node count of cluster workers when doing scale up
update_max_batch_size:
type: number
description: >
max batch size when doing rolling upgrade
default: 1
npd_enabled:
type: boolean
description: >
true if the npd service should be launched
default:
true
ostree_remote:
type: string
description: This parameter is ignored for k8s_fedora_coreos.
default: ''
ostree_commit:
type: string
description: This parameter is ignored for k8s_fedora_coreos.
default: ''
use_podman:
type: boolean
description: >
If true, run system containers for kubernetes, etcd and heat-agent
default:
true
constraints:
- allowed_values: [true]
selinux_mode:
type: string
description: >
Choose SELinux mode
default: "enforcing"
constraints:
- allowed_values: ["enforcing", "permissive", "disabled"]
kube_image_digest:
type: string
description: >
The digest of the image which should match the given kube_tag
default: ''
container_runtime:
type: string
description: The container runtime to install
default: 'host-docker'
containerd_version:
type: string
description: The containerd version to download from https://storage.googleapis.com/cri-containerd-release/
default: '1.2.8'
containerd_tarball_url:
type: string
description: Url location of the containerd tarball.
default: ''
containerd_tarball_sha256:
type: string
description: sha256 of the target containerd tarball.
default: '1f2f0fb928179df90492a83c326a194b8e9d992538498efb44cbb6ef15465627'
post_install_manifest_url:
type: string
description: >
Post install manifest URL used to setup some cloud provider/vendor
specific configs
default: ""
master_lb_allowed_cidrs:
type: comma_delimited_list
description: The allowed CIDR list for master load balancer
default: []
resources:
######################################################################
#
# network resources. allocate a network and router for our server.
# Important: the Load Balancer feature in Kubernetes requires that
# the name for the fixed_network must be "private" for the
# address lookup in Kubernetes to work properly
#
network:
condition: create_cluster_resources
type: ../../common/templates/network.yaml
properties:
existing_network: {get_param: fixed_network}
existing_subnet: {get_param: fixed_subnet}
private_network_cidr: {get_param: fixed_subnet_cidr}
dns_nameserver: {get_param: dns_nameserver}
external_network: {get_param: external_network}
private_network_name: {get_param: fixed_network_name}
api_lb:
condition: create_cluster_resources
type: ../../common/templates/lb_api.yaml
properties:
fixed_subnet: {get_attr: [network, fixed_subnet]}
external_network: {get_param: external_network}
protocol: {get_param: loadbalancing_protocol}
port: {get_param: kubernetes_port}
allowed_cidrs: {get_param: master_lb_allowed_cidrs}
etcd_lb:
condition: create_cluster_resources
type: ../../common/templates/lb_etcd.yaml
properties:
fixed_subnet: {get_attr: [network, fixed_subnet]}
protocol: {get_param: loadbalancing_protocol}
port: 2379
allowed_cidrs: {get_param: master_lb_allowed_cidrs}
######################################################################
#
# security groups. we need to permit network traffic of various
# sorts.
#
secgroup_kube_master:
condition: create_cluster_resources
type: OS::Neutron::SecurityGroup
properties:
rules:
- protocol: icmp
- protocol: tcp
port_range_min: 22
port_range_max: 22
- protocol: tcp
port_range_min: 53
port_range_max: 53
- protocol: udp
port_range_min: 53
port_range_max: 53
- protocol: tcp
port_range_min: 179
port_range_max: 179
- protocol: tcp
port_range_min: 7080
port_range_max: 7080
- protocol: tcp
port_range_min: 8080
port_range_max: 8080
- protocol: tcp
port_range_min: 2379
port_range_max: 2379
- protocol: tcp
port_range_min: 2380
port_range_max: 2380
- protocol: tcp
port_range_min: 6443
port_range_max: 6443
- protocol: tcp
port_range_min: 9100
port_range_max: 9100
- protocol: tcp
port_range_min: 10250
port_range_max: 10250
- protocol: tcp
port_range_min: 30000
port_range_max: 32767
- protocol: udp
port_range_min: 8472
port_range_max: 8472
secgroup_kube_minion:
condition: create_cluster_resources
type: OS::Neutron::SecurityGroup
properties:
rules:
- protocol: icmp
# Default port range for external service ports.
# In future, if the option `manage-security-groups` for ccm works
# well, we could remove this rule here.
# The PR in ccm is
# https://github.com/kubernetes/cloud-provider-openstack/pull/491
- protocol: tcp
port_range_min: 22
port_range_max: 22
- protocol: tcp
port_range_min: 30000
port_range_max: 32767
# allow any traffic from master nodes
- protocol: tcp
port_range_min: 1
port_range_max: 65535
remote_mode: 'remote_group_id'
remote_group_id: {get_resource: secgroup_kube_master}
- protocol: udp
port_range_min: 1
port_range_max: 65535
remote_mode: 'remote_group_id'
remote_group_id: {get_resource: secgroup_kube_master}
# allow any traffic between worker nodes
secgroup_rule_tcp_kube_minion:
condition: create_cluster_resources
type: OS::Neutron::SecurityGroupRule
properties:
protocol: tcp
port_range_min: 1
port_range_max: 65535
security_group: {get_resource: secgroup_kube_minion}
remote_group: {get_resource: secgroup_kube_minion}
secgroup_rule_udp_kube_minion:
condition: create_cluster_resources
type: OS::Neutron::SecurityGroupRule
properties:
protocol: udp
port_range_min: 1
port_range_max: 65535
security_group: {get_resource: secgroup_kube_minion}
remote_group: {get_resource: secgroup_kube_minion}
######################################################################
#
# resources that expose the IPs of either the kube master or a given
# LBaaS pool depending on whether LBaaS is enabled for the cluster.
#
api_address_lb_switch:
condition: create_cluster_resources
type: Magnum::ApiGatewaySwitcher
properties:
pool_public_ip: {get_attr: [api_lb, floating_address]}
pool_private_ip: {get_attr: [api_lb, address]}
master_public_ip: {get_attr: [kube_masters, resource.0.kube_master_external_ip]}
master_private_ip: {get_attr: [kube_masters, resource.0.kube_master_ip]}
etcd_address_lb_switch:
condition: create_cluster_resources
type: Magnum::ApiGatewaySwitcher
properties:
pool_private_ip: {get_attr: [etcd_lb, address]}
master_private_ip: {get_attr: [kube_masters, resource.0.kube_master_ip]}
######################################################################
#
# resources that expose the IPs of either floating ip or a given
# fixed ip depending on whether FloatingIP is enabled for the cluster.
#
api_address_floating_switch:
condition: create_cluster_resources
type: Magnum::FloatingIPAddressSwitcher
properties:
public_ip: {get_attr: [api_address_lb_switch, public_ip]}
private_ip: {get_attr: [api_address_lb_switch, private_ip]}
######################################################################
#
# resources that expose one server group for each master and worker nodes
# separately.
#
master_nodes_server_group:
condition: master_only
type: OS::Nova::ServerGroup
properties:
policies: [{get_param: nodes_affinity_policy}]
worker_nodes_server_group:
condition: worker_only
type: OS::Nova::ServerGroup
properties:
policies: [{get_param: nodes_affinity_policy}]
######################################################################
#
# kubernetes masters. This is a resource group that will create
# <number_of_masters> masters.
#
kube_masters:
condition: master_only
type: OS::Heat::ResourceGroup
depends_on:
- network
update_policy:
rolling_update: {max_batch_size: {get_param: update_max_batch_size}, pause_time: 30}
properties:
count: {get_param: number_of_masters}
resource_def:
type: kubemaster.yaml
properties:
name:
list_join:
- '-'
- [{ get_param: 'OS::stack_name' }, 'master', '%index%']
nodegroup_role: {get_param: master_role}
nodegroup_name: {get_param: master_nodegroup_name}
heapster_enabled: {get_param: heapster_enabled}
metrics_server_enabled: {get_param: metrics_server_enabled}
metrics_server_chart_tag: {get_param: metrics_server_chart_tag}
prometheus_monitoring: {get_param: prometheus_monitoring}
api_public_address: {get_attr: [api_lb, floating_address]}
api_private_address: {get_attr: [api_lb, address]}
ssh_key_name: {get_param: ssh_key_name}
ssh_public_key: {get_param: ssh_public_key}
server_image: {get_param: master_image}
master_flavor: {get_param: master_flavor}
external_network: {get_param: external_network}
kube_allow_priv: {get_param: kube_allow_priv}
boot_volume_size: {get_param: boot_volume_size}
boot_volume_type: {get_param: boot_volume_type}
etcd_volume_size: {get_param: etcd_volume_size}
etcd_volume_type: {get_param: etcd_volume_type}
docker_volume_size: {get_param: docker_volume_size}
docker_volume_type: {get_param: docker_volume_type}
docker_storage_driver: {get_param: docker_storage_driver}
cgroup_driver: {get_param: cgroup_driver}
network_driver: {get_param: network_driver}
flannel_network_cidr: {get_param: flannel_network_cidr}
flannel_network_subnetlen: {get_param: flannel_network_subnetlen}
flannel_backend: {get_param: flannel_backend}
system_pods_initial_delay: {get_param: system_pods_initial_delay}
system_pods_timeout: {get_param: system_pods_timeout}
portal_network_cidr: {get_param: portal_network_cidr}
admission_control_list: {get_param: admission_control_list}
discovery_url: {get_param: discovery_url}
cluster_uuid: {get_param: cluster_uuid}
magnum_url: {get_param: magnum_url}
traefik_ingress_controller_tag: {get_param: traefik_ingress_controller_tag}
volume_driver: {get_param: volume_driver}
region_name: {get_param: region_name}
fixed_network: {get_attr: [network, fixed_network]}
fixed_network_name: {get_param: fixed_network_name}
fixed_subnet: {get_attr: [network, fixed_subnet]}
api_pool_id: {get_attr: [api_lb, pool_id]}
etcd_pool_id: {get_attr: [etcd_lb, pool_id]}
username: {get_param: username}
password: {get_param: password}
kubernetes_port: {get_param: kubernetes_port}
tls_disabled: {get_param: tls_disabled}
kube_dashboard_enabled: {get_param: kube_dashboard_enabled}
influx_grafana_dashboard_enabled: {get_param: influx_grafana_dashboard_enabled}
verify_ca: {get_param: verify_ca}
secgroup_kube_master_id: {get_resource: secgroup_kube_master}
http_proxy: {get_param: http_proxy}
https_proxy: {get_param: https_proxy}
no_proxy: {get_param: no_proxy}
kube_tag: {get_param: master_kube_tag}
cloud_provider_tag: {get_param: cloud_provider_tag}
cloud_provider_enabled: {get_param: cloud_provider_enabled}
kube_version: {get_param: kube_version}
etcd_tag: {get_param: etcd_tag}
coredns_tag: {get_param: coredns_tag}
flannel_tag: {get_param: flannel_tag}
flannel_cni_tag: {get_param: flannel_cni_tag}
kube_dashboard_version: {get_param: kube_dashboard_version}
trustee_user_id: {get_param: trustee_user_id}
trustee_password: {get_param: trustee_password}
trust_id: {get_param: trust_id}
auth_url: {get_param: auth_url}
insecure_registry_url: {get_param: insecure_registry_url}
container_infra_prefix: {get_param: container_infra_prefix}
etcd_lb_vip: {get_attr: [etcd_lb, address]}
dns_service_ip: {get_param: dns_service_ip}
dns_cluster_domain: {get_param: dns_cluster_domain}
openstack_ca: {get_param: openstack_ca}
nodes_server_group_id: {get_resource: master_nodes_server_group}
availability_zone: {get_param: availability_zone}
ca_key: {get_param: ca_key}
cert_manager_api: {get_param: cert_manager_api}
calico_tag: {get_param: calico_tag}
calico_kube_controllers_tag: {get_param: calico_kube_controllers_tag}
calico_ipv4pool: {get_param: calico_ipv4pool}
calico_ipv4pool_ipip: {get_param: calico_ipv4pool_ipip}
pods_network_cidr: {get_param: pods_network_cidr}
ingress_controller: {get_param: ingress_controller}
ingress_controller_role: {get_param: ingress_controller_role}
octavia_ingress_controller_tag: {get_param: octavia_ingress_controller_tag}
kubelet_options: {get_param: kubelet_options}
kubeapi_options: {get_param: kubeapi_options}
kubeproxy_options: {get_param: kubeproxy_options}
kubecontroller_options: {get_param: kubecontroller_options}
kubescheduler_options: {get_param: kubescheduler_options}
octavia_enabled: {get_param: octavia_enabled}
kube_service_account_key: {get_param: kube_service_account_key}
kube_service_account_private_key: {get_param: kube_service_account_private_key}
prometheus_tag: {get_param: prometheus_tag}
grafana_tag: {get_param: grafana_tag}
heat_container_agent_tag: {get_param: heat_container_agent_tag}
keystone_auth_enabled: {get_param: keystone_auth_enabled}
k8s_keystone_auth_tag: {get_param: k8s_keystone_auth_tag}
monitoring_enabled: {get_param: monitoring_enabled}
prometheus_operator_chart_tag: {get_param: prometheus_operator_chart_tag}
prometheus_adapter_enabled: {get_param: prometheus_adapter_enabled}
prometheus_adapter_chart_tag: {get_param: prometheus_adapter_chart_tag}
prometheus_adapter_configmap: {get_param: prometheus_adapter_configmap}
project_id: {get_param: project_id}
tiller_enabled: {get_param: tiller_enabled}
tiller_tag: {get_param: tiller_tag}
tiller_namespace: {get_param: tiller_namespace}
helm_client_url: {get_param: helm_client_url}
helm_client_sha256: {get_param: helm_client_sha256}
helm_client_tag: {get_param: helm_client_tag}
node_problem_detector_tag: {get_param: node_problem_detector_tag}
nginx_ingress_controller_tag: {get_param: nginx_ingress_controller_tag}
nginx_ingress_controller_chart_tag: {get_param: nginx_ingress_controller_chart_tag}
auto_healing_enabled: {get_param: auto_healing_enabled}
auto_healing_controller: {get_param: auto_healing_controller}
magnum_auto_healer_tag: {get_param: magnum_auto_healer_tag}
auto_scaling_enabled: {get_param: auto_scaling_enabled}
cinder_csi_enabled: {get_param: cinder_csi_enabled}
cinder_csi_plugin_tag: {get_param: cinder_csi_plugin_tag}
csi_attacher_tag: {get_param: csi_attacher_tag}
csi_provisioner_tag: {get_param: csi_provisioner_tag}
csi_snapshotter_tag: {get_param: csi_snapshotter_tag}
csi_resizer_tag: {get_param: csi_resizer_tag}
csi_node_driver_registrar_tag: {get_param: csi_node_driver_registrar_tag}
draino_tag: {get_param: draino_tag}
autoscaler_tag: {get_param: autoscaler_tag}
min_node_count: {get_param: min_node_count}
max_node_count: {get_param: max_node_count}
npd_enabled: {get_param: npd_enabled}
ostree_remote: {get_param: ostree_remote}
ostree_commit: {get_param: ostree_commit}
use_podman: {get_param: use_podman}
selinux_mode: {get_param: selinux_mode}
kube_image_digest: {get_param: kube_image_digest}
container_runtime: {get_param: container_runtime}
containerd_version: {get_param: containerd_version}
containerd_tarball_url: {get_param: containerd_tarball_url}
containerd_tarball_sha256: {get_param: containerd_tarball_sha256}
post_install_manifest_url: {get_param: post_install_manifest_url}
metrics_scraper_tag: {get_param: metrics_scraper_tag}
kube_cluster_config:
condition: create_cluster_resources
type: OS::Heat::SoftwareConfig
properties:
group: script
config:
list_join:
- "\n"
-
- "#!/bin/bash"
- "source /etc/bashrc"
- get_file: ../../common/templates/kubernetes/fragments/kube-apiserver-to-kubelet-role.sh
- get_file: ../../common/templates/kubernetes/fragments/core-dns-service.sh
- if:
- calico_v3_3
- get_file: ../../common/templates/kubernetes/fragments/calico-service-v3-3-x.sh
- get_file: ../../common/templates/kubernetes/fragments/calico-service.sh
- get_file: ../../common/templates/kubernetes/fragments/flannel-service.sh
- get_file: ../../common/templates/kubernetes/fragments/enable-helm-tiller.sh
- str_replace:
template: {get_file: ../../common/templates/kubernetes/fragments/enable-prometheus-monitoring.sh}
params:
"${GRAFANA_ADMIN_PASSWD}": {get_param: grafana_admin_passwd}
- str_replace:
params:
$enable-ingress-traefik: {get_file: ../../common/templates/kubernetes/fragments/enable-ingress-traefik.sh}
$enable-ingress-octavia: {get_file: ../../common/templates/kubernetes/fragments/enable-ingress-octavia.sh}
template: {get_file: ../../common/templates/kubernetes/fragments/enable-ingress-controller.sh}
- get_file: ../../common/templates/kubernetes/fragments/kube-dashboard-service.sh
- str_replace:
template: {get_file: ../../common/templates/kubernetes/fragments/enable-keystone-auth.sh}
params:
"$KEYSTONE_AUTH_DEFAULT_POLICY": {get_param: keystone_auth_default_policy}
- get_file: ../../common/templates/kubernetes/fragments/enable-auto-healing.sh
- get_file: ../../common/templates/kubernetes/fragments/enable-auto-scaling.sh
- get_file: ../../common/templates/kubernetes/fragments/enable-cinder-csi.sh
# Helm Based Installation Configuration Scripts
- get_file: ../../common/templates/kubernetes/helm/metrics-server.sh
- str_replace:
template: {get_file: ../../common/templates/kubernetes/helm/prometheus-operator.sh}
params:
"${GRAFANA_ADMIN_PASSWD}": {get_param: grafana_admin_passwd}
"${KUBE_MASTERS_PRIVATE}": {get_attr: [kube_masters, kube_master_ip]}
- get_file: ../../common/templates/kubernetes/helm/prometheus-adapter.sh
- get_file: ../../common/templates/kubernetes/helm/ingress-nginx.sh
- get_file: ../../common/templates/kubernetes/fragments/install-helm-modules.sh
kube_cluster_deploy:
condition: create_cluster_resources
type: OS::Heat::SoftwareDeployment
properties:
actions: ['CREATE']
signal_transport: HEAT_SIGNAL
config:
get_resource: kube_cluster_config
server:
get_attr: [kube_masters, resource.0]
######################################################################
#
# kubernetes minions. This is an resource group that will initially
# create <number_of_minions> minions, and needs to be manually scaled.
#
kube_minions:
condition: worker_only
type: OS::Heat::ResourceGroup
depends_on:
- network
update_policy:
rolling_update: {max_batch_size: {get_param: update_max_batch_size}, pause_time: 30}
properties:
count: {get_param: number_of_minions}
removal_policies: [{resource_list: {get_param: minions_to_remove}}]
resource_def:
type: kubeminion.yaml
properties:
name:
list_join:
- '-'
- [{ get_param: 'OS::stack_name' }, 'node', '%index%']
prometheus_monitoring: {get_param: prometheus_monitoring}
nodegroup_role: {get_param: worker_role}
nodegroup_name: {get_param: worker_nodegroup_name}
ssh_key_name: {get_param: ssh_key_name}
ssh_public_key: {get_param: ssh_public_key}
server_image: {get_param: minion_image}
minion_flavor: {get_param: minion_flavor}
fixed_network:
if:
- create_cluster_resources
- get_attr: [network, fixed_network]
- get_param: fixed_network
fixed_subnet:
if:
- create_cluster_resources
- get_attr: [network, fixed_subnet]
- get_param: fixed_subnet
network_driver: {get_param: network_driver}
flannel_network_cidr: {get_param: flannel_network_cidr}
kube_master_ip:
if:
- create_cluster_resources
- get_attr: [api_address_lb_switch, private_ip]
- get_param: existing_master_private_ip
etcd_server_ip:
if:
- create_cluster_resources
- get_attr: [etcd_address_lb_switch, private_ip]
- get_param: existing_master_private_ip
external_network: {get_param: external_network}
kube_allow_priv: {get_param: kube_allow_priv}
boot_volume_size: {get_param: boot_volume_size}
boot_volume_type: {get_param: boot_volume_type}
docker_volume_size: {get_param: docker_volume_size}
docker_volume_type: {get_param: docker_volume_type}
docker_storage_driver: {get_param: docker_storage_driver}
cgroup_driver: {get_param: cgroup_driver}
wait_condition_timeout: {get_param: wait_condition_timeout}
registry_enabled: {get_param: registry_enabled}
registry_port: {get_param: registry_port}
swift_region: {get_param: swift_region}
registry_container: {get_param: registry_container}
registry_insecure: {get_param: registry_insecure}
registry_chunksize: {get_param: registry_chunksize}
cluster_uuid: {get_param: cluster_uuid}
magnum_url: {get_param: magnum_url}
volume_driver: {get_param: volume_driver}
region_name: {get_param: region_name}
auth_url: {get_param: auth_url}
username: {get_param: username}
password: {get_param: password}
kubernetes_port: {get_param: kubernetes_port}
tls_disabled: {get_param: tls_disabled}
verify_ca: {get_param: verify_ca}
secgroup_kube_minion_id:
if:
- create_cluster_resources
- get_resource: secgroup_kube_minion
- get_param: existing_security_group
http_proxy: {get_param: http_proxy}
https_proxy: {get_param: https_proxy}
no_proxy: {get_param: no_proxy}
kube_tag: {get_param: minion_kube_tag}
kube_version: {get_param: kube_version}
trustee_user_id: {get_param: trustee_user_id}
trustee_username: {get_param: trustee_username}
trustee_password: {get_param: trustee_password}
trustee_domain_id: {get_param: trustee_domain_id}
trust_id: {get_param: trust_id}
cloud_provider_enabled: {get_param: cloud_provider_enabled}
insecure_registry_url: {get_param: insecure_registry_url}
container_infra_prefix: {get_param: container_infra_prefix}
dns_service_ip: {get_param: dns_service_ip}
dns_cluster_domain: {get_param: dns_cluster_domain}
openstack_ca: {get_param: openstack_ca}
nodes_server_group_id: {get_resource: worker_nodes_server_group}
availability_zone: {get_param: availability_zone}
pods_network_cidr: {get_param: pods_network_cidr}
kubelet_options: {get_param: kubelet_options}
kubeproxy_options: {get_param: kubeproxy_options}
octavia_enabled: {get_param: octavia_enabled}
heat_container_agent_tag: {get_param: heat_container_agent_tag}
auto_healing_enabled: {get_param: auto_healing_enabled}
npd_enabled: {get_param: npd_enabled}
auto_healing_controller: {get_param: auto_healing_controller}
ostree_remote: {get_param: ostree_remote}
ostree_commit: {get_param: ostree_commit}
use_podman: {get_param: use_podman}
selinux_mode: {get_param: selinux_mode}
container_runtime: {get_param: container_runtime}
containerd_version: {get_param: containerd_version}
containerd_tarball_url: {get_param: containerd_tarball_url}
containerd_tarball_sha256: {get_param: containerd_tarball_sha256}
kube_service_account_key: {get_param: kube_service_account_key}
kube_service_account_private_key: {get_param: kube_service_account_private_key}
outputs:
api_address:
condition: create_cluster_resources
value:
str_replace:
template: api_ip_address
params:
api_ip_address: {get_attr: [api_address_floating_switch, ip_address]}
description: >
This is the API endpoint of the Kubernetes cluster. Use this to access
the Kubernetes API.
registry_address:
condition: create_cluster_resources
value:
str_replace:
template: localhost:port
params:
port: {get_param: registry_port}
description:
This is the url of docker registry server where you can store docker
images.
kube_masters_private:
condition: master_only
value: {get_attr: [kube_masters, kube_master_ip]}
description: >
This is a list of the "private" IP addresses of all the Kubernetes masters.
kube_masters:
condition: master_only
value: {get_attr: [kube_masters, kube_master_external_ip]}
description: >
This is a list of the "public" IP addresses of all the Kubernetes masters.
Use these IP addresses to log in to the Kubernetes masters via ssh.
kube_minions_private:
condition: worker_only
value: {get_attr: [kube_minions, kube_minion_ip]}
description: >
This is a list of the "private" IP addresses of all the Kubernetes minions.
kube_minions:
condition: worker_only
value: {get_attr: [kube_minions, kube_minion_external_ip]}
description: >
This is a list of the "public" IP addresses of all the Kubernetes minions.
Use these IP addresses to log in to the Kubernetes minions via ssh.