Container Infrastructure Management Service for OpenStack
You can not select more than 25 topics Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.
 
 
 
 
 

569 lines
18 KiB

  1. heat_template_version: queens
  2. description: >
  3. This is a nested stack that defines a single Kubernetes minion, This stack is
  4. included by an AutoScalingGroup resource in the parent template
  5. (kubecluster.yaml).
  6. parameters:
  7. name:
  8. type: string
  9. description: server name
  10. server_image:
  11. type: string
  12. description: glance image used to boot the server
  13. minion_flavor:
  14. type: string
  15. description: flavor to use when booting the server
  16. nodegroup_role:
  17. type: string
  18. description: the role of the nodegroup
  19. nodegroup_name:
  20. type: string
  21. description: the name of the nodegroup where the node belongs
  22. ssh_key_name:
  23. type: string
  24. description: name of ssh key to be provisioned on our server
  25. external_network:
  26. type: string
  27. description: uuid/name of a network to use for floating ip addresses
  28. kube_allow_priv:
  29. type: string
  30. description: >
  31. whether or not kubernetes should permit privileged containers.
  32. constraints:
  33. - allowed_values: ["true", "false"]
  34. boot_volume_size:
  35. type: number
  36. description: >
  37. size of the cinder boot volume
  38. boot_volume_type:
  39. type: string
  40. description: >
  41. type of the cinder boot volume
  42. docker_volume_size:
  43. type: number
  44. description: >
  45. size of a cinder volume to allocate to docker for container/image
  46. storage
  47. docker_volume_type:
  48. type: string
  49. description: >
  50. type of a cinder volume to allocate to docker for container/image
  51. storage
  52. docker_storage_driver:
  53. type: string
  54. description: docker storage driver name
  55. default: "devicemapper"
  56. cgroup_driver:
  57. type: string
  58. description: >
  59. cgroup driver name that kubelet should use, ideally the same as
  60. the docker cgroup driver.
  61. default: "cgroupfs"
  62. tls_disabled:
  63. type: boolean
  64. description: whether or not to enable TLS
  65. verify_ca:
  66. type: boolean
  67. description: whether or not to validate certificate authority
  68. kubernetes_port:
  69. type: number
  70. description: >
  71. The port which are used by kube-apiserver to provide Kubernetes
  72. service.
  73. cluster_uuid:
  74. type: string
  75. description: identifier for the cluster this template is generating
  76. magnum_url:
  77. type: string
  78. description: endpoint to retrieve TLS certs from
  79. prometheus_monitoring:
  80. type: boolean
  81. description: >
  82. whether or not to have the node-exporter running on the node
  83. kube_master_ip:
  84. type: string
  85. description: IP address of the Kubernetes master server.
  86. etcd_server_ip:
  87. type: string
  88. description: IP address of the Etcd server.
  89. fixed_network:
  90. type: string
  91. description: Network from which to allocate fixed addresses.
  92. fixed_subnet:
  93. type: string
  94. description: Subnet from which to allocate fixed addresses.
  95. network_driver:
  96. type: string
  97. description: network driver to use for instantiating container networks
  98. flannel_network_cidr:
  99. type: string
  100. description: network range for flannel overlay network
  101. wait_condition_timeout:
  102. type: number
  103. description : >
  104. timeout for the Wait Conditions
  105. registry_enabled:
  106. type: boolean
  107. description: >
  108. Indicates whether the docker registry is enabled.
  109. registry_port:
  110. type: number
  111. description: port of registry service
  112. swift_region:
  113. type: string
  114. description: region of swift service
  115. registry_container:
  116. type: string
  117. description: >
  118. name of swift container which docker registry stores images in
  119. registry_insecure:
  120. type: boolean
  121. description: >
  122. indicates whether to skip TLS verification between registry and backend storage
  123. registry_chunksize:
  124. type: number
  125. description: >
  126. size fo the data segments for the swift dynamic large objects
  127. secgroup_kube_minion_id:
  128. type: string
  129. description: ID of the security group for kubernetes minion.
  130. volume_driver:
  131. type: string
  132. description: volume driver to use for container storage
  133. region_name:
  134. type: string
  135. description: A logically separate section of the cluster
  136. username:
  137. type: string
  138. description: >
  139. user account
  140. password:
  141. type: string
  142. description: >
  143. user password, not set in current implementation, only used to
  144. fill in for Kubernetes config file
  145. hidden: true
  146. http_proxy:
  147. type: string
  148. description: http proxy address for docker
  149. https_proxy:
  150. type: string
  151. description: https proxy address for docker
  152. no_proxy:
  153. type: string
  154. description: no proxies for docker
  155. kube_tag:
  156. type: string
  157. description: tag of the k8s containers used to provision the kubernetes cluster
  158. kube_version:
  159. type: string
  160. description: version of kubernetes used for kubernetes cluster
  161. trustee_domain_id:
  162. type: string
  163. description: domain id of the trustee
  164. trustee_user_id:
  165. type: string
  166. description: user id of the trustee
  167. trustee_username:
  168. type: string
  169. description: username of the trustee
  170. trustee_password:
  171. type: string
  172. description: password of the trustee
  173. hidden: true
  174. trust_id:
  175. type: string
  176. description: id of the trust which is used by the trustee
  177. hidden: true
  178. auth_url:
  179. type: string
  180. description: >
  181. url for keystone, must be v2 since k8s backend only support v2
  182. at this point
  183. insecure_registry_url:
  184. type: string
  185. description: insecure registry url
  186. container_infra_prefix:
  187. type: string
  188. description: >
  189. prefix of container images used in the cluster, kubernetes components,
  190. kubernetes-dashboard, coredns etc
  191. dns_service_ip:
  192. type: string
  193. description: >
  194. address used by Kubernetes DNS service
  195. dns_cluster_domain:
  196. type: string
  197. description: >
  198. domain name for cluster DNS
  199. openstack_ca:
  200. type: string
  201. description: The OpenStack CA certificate to install on the node.
  202. nodes_server_group_id:
  203. type: string
  204. description: ID of the server group for kubernetes cluster nodes.
  205. availability_zone:
  206. type: string
  207. description: >
  208. availability zone for master and nodes
  209. default: ""
  210. pods_network_cidr:
  211. type: string
  212. description: Configure the IP pool/range from which pod IPs will be chosen
  213. kubelet_options:
  214. type: string
  215. description: >
  216. additional options to be passed to the kubelet
  217. kubeproxy_options:
  218. type: string
  219. description: >
  220. additional options to be passed to the kube proxy
  221. octavia_enabled:
  222. type: boolean
  223. description: >
  224. whether or not to use Octavia for LoadBalancer type service.
  225. default: False
  226. cloud_provider_enabled:
  227. type: boolean
  228. description: Enable or disable the openstack kubernetes cloud provider
  229. heat_container_agent_tag:
  230. type: string
  231. description: tag of the heat_container_agent system container
  232. auto_healing_enabled:
  233. type: boolean
  234. description: >
  235. true if the auto healing feature should be enabled
  236. auto_healing_controller:
  237. type: string
  238. description: >
  239. The service to be deployed for auto-healing.
  240. default: "draino"
  241. npd_enabled:
  242. type: boolean
  243. description: >
  244. true if the npd service should be launched
  245. default:
  246. true
  247. conditions:
  248. image_based: {equals: [{get_param: boot_volume_size}, 0]}
  249. volume_based:
  250. not:
  251. equals:
  252. - get_param: boot_volume_size
  253. - 0
  254. resources:
  255. agent_config:
  256. type: OS::Heat::SoftwareConfig
  257. properties:
  258. group: ungrouped
  259. config:
  260. list_join:
  261. - "\n"
  262. -
  263. - str_replace:
  264. template: {get_file: ../../common/templates/fragments/atomic-install-openstack-ca.sh}
  265. params:
  266. $OPENSTACK_CA: {get_param: openstack_ca}
  267. - str_replace:
  268. template: {get_file: ../../common/templates/kubernetes/fragments/start-container-agent.sh}
  269. params:
  270. $CONTAINER_INFRA_PREFIX: {get_param: container_infra_prefix}
  271. $HEAT_CONTAINER_AGENT_TAG: {get_param: heat_container_agent_tag}
  272. - get_file: ../../common/templates/kubernetes/fragments/disable-selinux.sh
  273. ######################################################################
  274. #
  275. # software configs. these are components that are combined into
  276. # a multipart MIME user-data archive.
  277. #
  278. node_config:
  279. type: OS::Heat::SoftwareConfig
  280. properties:
  281. group: script
  282. config:
  283. list_join:
  284. - "\n"
  285. -
  286. - str_replace:
  287. template: {get_file: ../../common/templates/kubernetes/fragments/write-heat-params.sh}
  288. params:
  289. $INSTANCE_NAME: {get_param: name}
  290. $PROMETHEUS_MONITORING: {get_param: prometheus_monitoring}
  291. $KUBE_ALLOW_PRIV: {get_param: kube_allow_priv}
  292. $KUBE_MASTER_IP: {get_param: kube_master_ip}
  293. $KUBE_API_PORT: {get_param: kubernetes_port}
  294. $KUBE_NODE_PUBLIC_IP: {get_attr: [kube_minion_floating, floating_ip_address]}
  295. $KUBE_NODE_IP: {get_attr: [kube_minion_eth0, fixed_ips, 0, ip_address]}
  296. $ETCD_SERVER_IP: {get_param: etcd_server_ip}
  297. $DOCKER_VOLUME: {get_resource: docker_volume}
  298. $DOCKER_VOLUME_SIZE: {get_param: docker_volume_size}
  299. $DOCKER_STORAGE_DRIVER: {get_param: docker_storage_driver}
  300. $CGROUP_DRIVER: {get_param: cgroup_driver}
  301. $NETWORK_DRIVER: {get_param: network_driver}
  302. $REGISTRY_ENABLED: {get_param: registry_enabled}
  303. $REGISTRY_PORT: {get_param: registry_port}
  304. $SWIFT_REGION: {get_param: swift_region}
  305. $REGISTRY_CONTAINER: {get_param: registry_container}
  306. $REGISTRY_INSECURE: {get_param: registry_insecure}
  307. $REGISTRY_CHUNKSIZE: {get_param: registry_chunksize}
  308. $TLS_DISABLED: {get_param: tls_disabled}
  309. $VERIFY_CA: {get_param: verify_ca}
  310. $CLUSTER_UUID: {get_param: cluster_uuid}
  311. $MAGNUM_URL: {get_param: magnum_url}
  312. $USERNAME: {get_param: username}
  313. $PASSWORD: {get_param: password}
  314. $VOLUME_DRIVER: {get_param: volume_driver}
  315. $REGION_NAME: {get_param: region_name}
  316. $HTTP_PROXY: {get_param: http_proxy}
  317. $HTTPS_PROXY: {get_param: https_proxy}
  318. $NO_PROXY: {get_param: no_proxy}
  319. $KUBE_TAG: {get_param: kube_tag}
  320. $FLANNEL_NETWORK_CIDR: {get_param: flannel_network_cidr}
  321. $PODS_NETWORK_CIDR: {get_param: pods_network_cidr}
  322. $KUBE_VERSION: {get_param: kube_version}
  323. $TRUSTEE_USER_ID: {get_param: trustee_user_id}
  324. $TRUSTEE_PASSWORD: {get_param: trustee_password}
  325. $TRUST_ID: {get_param: trust_id}
  326. $AUTH_URL: {get_param: auth_url}
  327. $CLOUD_PROVIDER_ENABLED: {get_param: cloud_provider_enabled}
  328. $INSECURE_REGISTRY_URL: {get_param: insecure_registry_url}
  329. $CONTAINER_INFRA_PREFIX: {get_param: container_infra_prefix}
  330. $DNS_SERVICE_IP: {get_param: dns_service_ip}
  331. $DNS_CLUSTER_DOMAIN: {get_param: dns_cluster_domain}
  332. $KUBELET_OPTIONS: {get_param: kubelet_options}
  333. $KUBEPROXY_OPTIONS: {get_param: kubeproxy_options}
  334. $OCTAVIA_ENABLED: {get_param: octavia_enabled}
  335. $HEAT_CONTAINER_AGENT_TAG: {get_param: heat_container_agent_tag}
  336. $AUTO_HEALING_ENABLED: {get_param: auto_healing_enabled}
  337. $AUTO_HEALING_CONTROLLER: {get_param: auto_healing_controller}
  338. $NPD_ENABLED: {get_param: npd_enabled}
  339. $NODEGROUP_ROLE: {get_param: nodegroup_role}
  340. $NODEGROUP_NAME: {get_param: nodegroup_name}
  341. - get_file: ../../common/templates/kubernetes/fragments/write-kube-os-config.sh
  342. - get_file: ../../common/templates/kubernetes/fragments/make-cert-client.sh
  343. - get_file: ../../common/templates/fragments/configure-docker-registry.sh
  344. - get_file: ../../common/templates/kubernetes/fragments/configure-kubernetes-minion.sh
  345. - get_file: ../../common/templates/kubernetes/fragments/add-proxy.sh
  346. - str_replace:
  347. template: {get_file: ../../common/templates/fragments/configure-docker-storage.sh}
  348. params:
  349. $configure_docker_storage_driver: {get_file: ../../common/templates/fragments/configure_docker_storage_driver_atomic.sh}
  350. - get_file: ../../common/templates/kubernetes/fragments/enable-services-minion.sh
  351. - get_file: ../../common/templates/fragments/enable-docker-registry.sh
  352. node_config_deployment:
  353. type: OS::Heat::SoftwareDeployment
  354. properties:
  355. signal_transport: HEAT_SIGNAL
  356. config: {get_resource: node_config}
  357. server: {if: ["volume_based", {get_resource: kube-minion-bfv}, {get_resource: kube-minion}]}
  358. actions: ['CREATE']
  359. ######################################################################
  360. #
  361. # a single kubernetes minion.
  362. #
  363. kube_node_volume:
  364. type: OS::Cinder::Volume
  365. condition: volume_based
  366. properties:
  367. image: {get_param: server_image}
  368. size: {get_param: boot_volume_size}
  369. volume_type: {get_param: boot_volume_type}
  370. # do NOT use "_" (underscore) in the Nova server name
  371. # it creates a mismatch between the generated Nova name and its hostname
  372. # which can lead to weird problems
  373. kube-minion:
  374. condition: image_based
  375. type: OS::Nova::Server
  376. properties:
  377. name: {get_param: name}
  378. flavor: {get_param: minion_flavor}
  379. image: {get_param: server_image}
  380. key_name: {get_param: ssh_key_name}
  381. user_data: {get_resource: agent_config}
  382. user_data_format: SOFTWARE_CONFIG
  383. software_config_transport: POLL_SERVER_HEAT
  384. networks:
  385. - port: {get_resource: kube_minion_eth0}
  386. scheduler_hints: { group: { get_param: nodes_server_group_id }}
  387. availability_zone: {get_param: availability_zone}
  388. kube-minion-bfv:
  389. condition: volume_based
  390. type: OS::Nova::Server
  391. properties:
  392. name: {get_param: name}
  393. flavor: {get_param: minion_flavor}
  394. key_name: {get_param: ssh_key_name}
  395. user_data: {get_resource: agent_config}
  396. user_data_format: SOFTWARE_CONFIG
  397. software_config_transport: POLL_SERVER_HEAT
  398. networks:
  399. - port: {get_resource: kube_minion_eth0}
  400. scheduler_hints: { group: { get_param: nodes_server_group_id }}
  401. availability_zone: {get_param: availability_zone}
  402. block_device_mapping_v2:
  403. - boot_index: 0
  404. volume_id: {get_resource: kube_node_volume}
  405. delete_on_termination: true
  406. kube_minion_eth0:
  407. type: OS::Neutron::Port
  408. properties:
  409. network: {get_param: fixed_network}
  410. security_groups:
  411. - get_param: secgroup_kube_minion_id
  412. fixed_ips:
  413. - subnet: {get_param: fixed_subnet}
  414. allowed_address_pairs:
  415. - ip_address: {get_param: pods_network_cidr}
  416. replacement_policy: AUTO
  417. kube_minion_floating:
  418. type: Magnum::Optional::KubeMinion::Neutron::FloatingIP
  419. properties:
  420. floating_network: {get_param: external_network}
  421. port_id: {get_resource: kube_minion_eth0}
  422. depends_on: kube-minion
  423. ######################################################################
  424. #
  425. # docker storage. This allocates a cinder volume and attaches it
  426. # to the minion.
  427. #
  428. docker_volume:
  429. type: Magnum::Optional::Cinder::Volume
  430. properties:
  431. size: {get_param: docker_volume_size}
  432. volume_type: {get_param: docker_volume_type}
  433. docker_volume_attach:
  434. type: Magnum::Optional::Cinder::VolumeAttachment
  435. properties:
  436. instance_uuid: {if: ["volume_based", {get_resource: kube-minion-bfv}, {get_resource: kube-minion}]}
  437. volume_id: {get_resource: docker_volume}
  438. mountpoint: /dev/vdb
  439. upgrade_kubernetes:
  440. type: OS::Heat::SoftwareConfig
  441. properties:
  442. group: script
  443. inputs:
  444. - name: kube_tag_input
  445. config:
  446. get_file: ../../common/templates/kubernetes/fragments/upgrade-kubernetes.sh
  447. upgrade_kubernetes_deployment:
  448. type: OS::Heat::SoftwareDeployment
  449. properties:
  450. signal_transport: HEAT_SIGNAL
  451. config: {get_resource: upgrade_kubernetes}
  452. server: {if: ["volume_based", {get_resource: kube-minion-bfv}, {get_resource: kube-minion}]}
  453. actions: ['UPDATE']
  454. input_values:
  455. kube_tag_input: {get_param: kube_tag}
  456. outputs:
  457. kube_minion_ip:
  458. value: {get_attr: [kube_minion_eth0, fixed_ips, 0, ip_address]}
  459. description: >
  460. This is the "public" IP address of the Kubernetes minion node.
  461. kube_minion_external_ip:
  462. value: {get_attr: [kube_minion_floating, floating_ip_address]}
  463. description: >
  464. This is the "public" IP address of the Kubernetes minion node.
  465. ######################################################################
  466. #
  467. # NOTE(flwang): Returning the minion node server ID here so that
  468. # consumer can send API request to Heat to remove a particular
  469. # node with removal_policies. Otherwise, the consumer (e.g. AutoScaler)
  470. # has to use index to do the remove which is confusing out of the
  471. # OpenStack world.
  472. # https://storyboard.openstack.org/#!/story/2005054
  473. #
  474. ######################################################################
  475. OS::stack_id:
  476. value: {if: ["volume_based", {get_resource: kube-minion-bfv}, {get_resource: kube-minion}]}
  477. description: >
  478. This is the Nova server id of the node.