be0609ce88
Currently, there is no guarantee to make sure all nodes of one cluster are created on different compute hosts. So it would be nice if we can create a server group and set it with anti-affinity policy to get a better HA for cluster. This patch is proposing to create a server group for master and minion nodes with soft-anti-affinity policy by default. Closes-Bug: #1737802 Change-Id: Icc7a73ef55296a58bf00719ca4d1cdcc304fab86
381 lines
11 KiB
YAML
381 lines
11 KiB
YAML
heat_template_version: 2014-10-16
|
|
|
|
description: >
|
|
This is a nested stack that defines a single Kubernetes minion,
|
|
based on a CoreOS cloud image. This stack is included by a ResourceGroup
|
|
resource in the parent template (kubecluster.yaml).
|
|
|
|
parameters:
|
|
|
|
name:
|
|
type: string
|
|
description: server name
|
|
|
|
server_image:
|
|
type: string
|
|
description: glance image used to boot the server
|
|
|
|
minion_flavor:
|
|
type: string
|
|
description: flavor to use when booting the server
|
|
|
|
ssh_key_name:
|
|
type: string
|
|
description: name of ssh key to be provisioned on our server
|
|
|
|
external_network:
|
|
type: string
|
|
description: uuid/name of a network to use for floating ip addresses
|
|
|
|
kube_allow_priv:
|
|
type: string
|
|
description: >
|
|
whether or not kubernetes should permit privileged containers.
|
|
constraints:
|
|
- allowed_values: ["true", "false"]
|
|
|
|
network_driver:
|
|
type: string
|
|
description: network driver to use for instantiating container networks
|
|
|
|
tls_disabled:
|
|
type: boolean
|
|
description: whether or not to enable TLS
|
|
|
|
verify_ca:
|
|
type: boolean
|
|
description: whether or not to validate certificate authority
|
|
|
|
kubernetes_port:
|
|
type: number
|
|
description: >
|
|
The port which are used by kube-apiserver to provide Kubernetes
|
|
service.
|
|
default: 6443
|
|
|
|
cluster_uuid:
|
|
type: string
|
|
description: identifier for the cluster this template is generating
|
|
|
|
magnum_url:
|
|
type: string
|
|
description: endpoint to retrieve TLS certs from
|
|
|
|
kube_version:
|
|
type: string
|
|
description: version of kubernetes used for kubernetes cluster
|
|
|
|
hyperkube_image:
|
|
type: string
|
|
description: >
|
|
Docker registry used for hyperkube image
|
|
|
|
kube_master_ip:
|
|
type: string
|
|
description: IP address of the Kubernetes master server.
|
|
|
|
etcd_server_ip:
|
|
type: string
|
|
description: IP address of the Etcd server.
|
|
|
|
fixed_network:
|
|
type: string
|
|
description: Network from which to allocate fixed addresses.
|
|
|
|
fixed_subnet:
|
|
type: string
|
|
description: Subnet from which to allocate fixed addresses.
|
|
|
|
flannel_network_cidr:
|
|
type: string
|
|
description: network range for flannel overlay network
|
|
|
|
wait_condition_timeout:
|
|
type: number
|
|
description: >
|
|
timeout for the Wait Conditions
|
|
|
|
secgroup_kube_minion_id:
|
|
type: string
|
|
description: ID of the security group for kubernetes minion.
|
|
|
|
http_proxy:
|
|
type: string
|
|
description: http proxy address for docker
|
|
|
|
https_proxy:
|
|
type: string
|
|
description: https proxy address for docker
|
|
|
|
no_proxy:
|
|
type: string
|
|
description: no proxies for docker
|
|
|
|
trustee_user_id:
|
|
type: string
|
|
description: user id of the trustee
|
|
default: ""
|
|
|
|
trustee_password:
|
|
type: string
|
|
description: password of the trustee
|
|
default: ""
|
|
hidden: true
|
|
|
|
trust_id:
|
|
type: string
|
|
description: id of the trust which is used by the trustee
|
|
default: ""
|
|
hidden: true
|
|
|
|
auth_url:
|
|
type: string
|
|
description: url for keystone
|
|
|
|
insecure_registry_url:
|
|
type: string
|
|
description: insecure registry url
|
|
|
|
container_runtime:
|
|
type: string
|
|
description: >
|
|
Container runtime to use with Kubernetes.
|
|
|
|
prometheus_monitoring:
|
|
type: boolean
|
|
description: >
|
|
whether or not to have the node-exporter running on the node
|
|
|
|
dns_service_ip:
|
|
type: string
|
|
description: >
|
|
address used by Kubernetes DNS service
|
|
|
|
dns_cluster_domain:
|
|
type: string
|
|
description: >
|
|
domain name for cluster DNS
|
|
|
|
openstack_ca:
|
|
type: string
|
|
description: The OpenStack CA certificate to install on the node.
|
|
|
|
nodes_server_group_id:
|
|
type: string
|
|
description: ID of the server group for kubernetes cluster nodes.
|
|
|
|
resources:
|
|
|
|
minion_wait_handle:
|
|
type: OS::Heat::WaitConditionHandle
|
|
|
|
minion_wait_condition:
|
|
type: OS::Heat::WaitCondition
|
|
depends_on: kube-minion
|
|
properties:
|
|
handle: {get_resource: minion_wait_handle}
|
|
timeout: {get_param: wait_condition_timeout}
|
|
|
|
######################################################################
|
|
#
|
|
# software configs. these are components that are combined into
|
|
# a multipart MIME user-data archive.
|
|
#
|
|
|
|
write_heat_params:
|
|
type: OS::Heat::SoftwareConfig
|
|
properties:
|
|
group: ungrouped
|
|
config:
|
|
str_replace:
|
|
template: {get_file: fragments/write-heat-params.yaml}
|
|
params:
|
|
"$KUBE_ALLOW_PRIV": {get_param: kube_allow_priv}
|
|
"$KUBE_MASTER_IP": {get_param: kube_master_ip}
|
|
"$KUBE_NODE_PUBLIC_IP": {get_attr: [kube_minion_floating, floating_ip_address]}
|
|
"$KUBE_NODE_IP": {get_attr: [kube_minion_eth0, fixed_ips, 0, ip_address]}
|
|
"$WAIT_CURL": {get_attr: [minion_wait_handle, curl_cli]}
|
|
"$KUBE_API_PORT": {get_param: kubernetes_port}
|
|
"$TLS_DISABLED": {get_param: tls_disabled}
|
|
"$VERIFY_CA": {get_param: verify_ca}
|
|
"$NETWORK_DRIVER": {get_param: network_driver}
|
|
"$ETCD_SERVER_IP": {get_param: etcd_server_ip}
|
|
"$KUBE_VERSION": {get_param: kube_version}
|
|
"$CLUSTER_UUID": {get_param: cluster_uuid}
|
|
"$MAGNUM_URL": {get_param: magnum_url}
|
|
"$HTTP_PROXY": {get_param: http_proxy}
|
|
"$HTTPS_PROXY": {get_param: https_proxy}
|
|
"$NO_PROXY": {get_param: no_proxy}
|
|
"$TRUSTEE_USER_ID": {get_param: trustee_user_id}
|
|
"$TRUSTEE_PASSWORD": {get_param: trustee_password}
|
|
"$TRUST_ID": {get_param: trust_id}
|
|
"$AUTH_URL": {get_param: auth_url}
|
|
"$KUBE_CERTS_PATH": "/etc/kubernetes/ssl"
|
|
"$HOST_CERTS_PATH": "/usr/share/ca-certificates"
|
|
"$HYPERKUBE_IMAGE_REPO":
|
|
str_replace:
|
|
template: insecure_registry_urlhyperkube_image
|
|
params:
|
|
insecure_registry_url: { get_param: insecure_registry_url }
|
|
hyperkube_image: { get_param: hyperkube_image }
|
|
"$INSECURE_REGISTRY_URL": {get_param: insecure_registry_url}
|
|
"$CONTAINER_RUNTIME": {get_param: container_runtime}
|
|
"$DNS_SERVICE_IP": {get_param: dns_service_ip}
|
|
"$DNS_CLUSTER_DOMAIN": {get_param: dns_cluster_domain}
|
|
|
|
add_ext_ca_certs:
|
|
type: OS::Heat::SoftwareConfig
|
|
properties:
|
|
group: ungrouped
|
|
config:
|
|
str_replace:
|
|
params:
|
|
$OPENSTACK_CA: {get_param: openstack_ca}
|
|
template: {get_file: fragments/add-ext-ca-certs.yaml}
|
|
|
|
write_kubeconfig:
|
|
type: OS::Heat::SoftwareConfig
|
|
properties:
|
|
group: ungrouped
|
|
config: {get_file: fragments/write-kubeconfig.yaml}
|
|
|
|
make_cert:
|
|
type: OS::Heat::SoftwareConfig
|
|
properties:
|
|
group: ungrouped
|
|
config: {get_file: fragments/make-cert-client.yaml}
|
|
|
|
enable_network_service:
|
|
type: OS::Heat::SoftwareConfig
|
|
properties:
|
|
group: ungrouped
|
|
config: {get_file: fragments/enable-network-service-client.yaml}
|
|
|
|
enable_kubelet:
|
|
type: OS::Heat::SoftwareConfig
|
|
properties:
|
|
group: ungrouped
|
|
config: {get_file: fragments/enable-kubelet-minion.yaml}
|
|
|
|
enable_kube_proxy:
|
|
type: OS::Heat::SoftwareConfig
|
|
properties:
|
|
group: ungrouped
|
|
config: {get_file: fragments/enable-kube-proxy-minion.yaml}
|
|
|
|
wc_notify:
|
|
type: OS::Heat::SoftwareConfig
|
|
properties:
|
|
group: ungrouped
|
|
config: {get_file: fragments/wc-notify.yaml}
|
|
|
|
add_proxy:
|
|
type: OS::Heat::SoftwareConfig
|
|
properties:
|
|
group: ungrouped
|
|
config: {get_file: fragments/add-proxy.yaml}
|
|
|
|
configure_docker:
|
|
type: OS::Heat::SoftwareConfig
|
|
properties:
|
|
group: ungrouped
|
|
config: {get_file: fragments/configure-docker.yaml}
|
|
|
|
kube_minion_init:
|
|
type: OS::Heat::SoftwareConfig
|
|
properties:
|
|
group: ungrouped
|
|
config:
|
|
str_replace:
|
|
template: |
|
|
$add_ext_ca_certs
|
|
$write_heat_params
|
|
$write_kubeconfig
|
|
$make_cert
|
|
$configure_docker
|
|
$add_proxy
|
|
$enable_network_service
|
|
$enable_kubelet
|
|
$enable_kube_proxy
|
|
$wc_notify
|
|
coreos:
|
|
units:
|
|
- name: "add-ext-ca-certs.service"
|
|
command: "start"
|
|
- name: "make-cert.service"
|
|
command: "start"
|
|
- name: "configure-docker.service"
|
|
command: "start"
|
|
- name: "add-proxy.service"
|
|
command: "start"
|
|
- name: "enable-network-service.service"
|
|
command: "start"
|
|
- name: "enable-kubelet.service"
|
|
command: "start"
|
|
- name: "enable-kube-proxy.service"
|
|
command: "start"
|
|
- name: "wc-notify.service"
|
|
command: "start"
|
|
params:
|
|
"$add_ext_ca_certs": {get_attr: [add_ext_ca_certs, config]}
|
|
"$write_heat_params": {get_attr: [write_heat_params, config]}
|
|
"$write_kubeconfig": {get_attr: [write_kubeconfig, config]}
|
|
"$make_cert": {get_attr: [make_cert, config]}
|
|
"$configure_docker": {get_attr: [configure_docker, config]}
|
|
"$add_proxy": {get_attr: [add_proxy, config]}
|
|
"$enable_network_service": {get_attr: [enable_network_service, config]}
|
|
"$enable_kubelet": {get_attr: [enable_kubelet, config]}
|
|
"$enable_kube_proxy": {get_attr: [enable_kube_proxy, config]}
|
|
"$wc_notify": {get_attr: [wc_notify, config]}
|
|
|
|
# do NOT use "_" (underscore) in the Nova server name
|
|
# it creates a mismatch between the generated Nova name and its hostname
|
|
# which can lead to weird problems
|
|
kube-minion:
|
|
type: OS::Nova::Server
|
|
properties:
|
|
name: {get_param: name}
|
|
image: {get_param: server_image}
|
|
flavor: {get_param: minion_flavor}
|
|
key_name: {get_param: ssh_key_name}
|
|
user_data_format: RAW
|
|
user_data: {get_resource: kube_minion_init}
|
|
networks:
|
|
- port: {get_resource: kube_minion_eth0}
|
|
scheduler_hints: { group: { get_param: nodes_server_group_id }}
|
|
|
|
kube_minion_eth0:
|
|
type: OS::Neutron::Port
|
|
properties:
|
|
network: {get_param: fixed_network}
|
|
security_groups:
|
|
- {get_param: secgroup_kube_minion_id}
|
|
fixed_ips:
|
|
- subnet: {get_param: fixed_subnet}
|
|
allowed_address_pairs:
|
|
- ip_address: {get_param: flannel_network_cidr}
|
|
replacement_policy: AUTO
|
|
|
|
kube_minion_floating:
|
|
type: Magnum::Optional::KubeMinion::Neutron::FloatingIP
|
|
properties:
|
|
floating_network: {get_param: external_network}
|
|
port_id: {get_resource: kube_minion_eth0}
|
|
|
|
outputs:
|
|
|
|
kube_minion_ip:
|
|
value: {get_attr: [kube_minion_eth0, fixed_ips, 0, ip_address]}
|
|
description: >
|
|
This is the "private" IP address of the Kubernetes minion node.
|
|
|
|
kube_minion_external_ip:
|
|
value: {get_attr: [kube_minion_floating, floating_ip_address]}
|
|
description: >
|
|
This is the "public" IP address of the Kubernetes minion node.
|
|
|
|
OS::stack_id:
|
|
value: {get_param: "OS::stack_id"}
|
|
description: >
|
|
This is a id of the stack which creates from this template.
|