Container Infrastructure Management Service for OpenStack
You can not select more than 25 topics Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.
 
 
 
 
 

564 lines
17 KiB

  1. heat_template_version: queens
  2. description: >
  3. This is a nested stack that defines a single Kubernetes minion, This stack is
  4. included by an AutoScalingGroup resource in the parent template
  5. (kubecluster.yaml).
  6. parameters:
  7. name:
  8. type: string
  9. description: server name
  10. server_image:
  11. type: string
  12. description: glance image used to boot the server
  13. minion_flavor:
  14. type: string
  15. description: flavor to use when booting the server
  16. nodegroup_role:
  17. type: string
  18. description: the role of the nodegroup
  19. nodegroup_name:
  20. type: string
  21. description: the name of the nodegroup where the node belongs
  22. ssh_key_name:
  23. type: string
  24. description: name of ssh key to be provisioned on our server
  25. ssh_public_key:
  26. type: string
  27. description: name of ssh key to be provisioned on our server
  28. external_network:
  29. type: string
  30. description: uuid/name of a network to use for floating ip addresses
  31. kube_allow_priv:
  32. type: string
  33. description: >
  34. whether or not kubernetes should permit privileged containers.
  35. constraints:
  36. - allowed_values: ["true", "false"]
  37. boot_volume_size:
  38. type: number
  39. description: >
  40. size of the cinder boot volume
  41. boot_volume_type:
  42. type: string
  43. description: >
  44. type of the cinder boot volume
  45. docker_volume_size:
  46. type: number
  47. description: >
  48. size of a cinder volume to allocate to docker for container/image
  49. storage
  50. docker_volume_type:
  51. type: string
  52. description: >
  53. type of a cinder volume to allocate to docker for container/image
  54. storage
  55. docker_storage_driver:
  56. type: string
  57. description: docker storage driver name
  58. default: "devicemapper"
  59. cgroup_driver:
  60. type: string
  61. description: >
  62. cgroup driver name that kubelet should use, ideally the same as
  63. the docker cgroup driver.
  64. default: "cgroupfs"
  65. tls_disabled:
  66. type: boolean
  67. description: whether or not to enable TLS
  68. verify_ca:
  69. type: boolean
  70. description: whether or not to validate certificate authority
  71. kubernetes_port:
  72. type: number
  73. description: >
  74. The port which are used by kube-apiserver to provide Kubernetes
  75. service.
  76. cluster_uuid:
  77. type: string
  78. description: identifier for the cluster this template is generating
  79. magnum_url:
  80. type: string
  81. description: endpoint to retrieve TLS certs from
  82. prometheus_monitoring:
  83. type: boolean
  84. description: >
  85. whether or not to have the node-exporter running on the node
  86. kube_master_ip:
  87. type: string
  88. description: IP address of the Kubernetes master server.
  89. etcd_server_ip:
  90. type: string
  91. description: IP address of the Etcd server.
  92. fixed_network:
  93. type: string
  94. description: Network from which to allocate fixed addresses.
  95. fixed_subnet:
  96. type: string
  97. description: Subnet from which to allocate fixed addresses.
  98. network_driver:
  99. type: string
  100. description: network driver to use for instantiating container networks
  101. flannel_network_cidr:
  102. type: string
  103. description: network range for flannel overlay network
  104. wait_condition_timeout:
  105. type: number
  106. description : >
  107. timeout for the Wait Conditions
  108. registry_enabled:
  109. type: boolean
  110. description: >
  111. Indicates whether the docker registry is enabled.
  112. registry_port:
  113. type: number
  114. description: port of registry service
  115. swift_region:
  116. type: string
  117. description: region of swift service
  118. registry_container:
  119. type: string
  120. description: >
  121. name of swift container which docker registry stores images in
  122. registry_insecure:
  123. type: boolean
  124. description: >
  125. indicates whether to skip TLS verification between registry and backend storage
  126. registry_chunksize:
  127. type: number
  128. description: >
  129. size fo the data segments for the swift dynamic large objects
  130. secgroup_kube_minion_id:
  131. type: string
  132. description: ID of the security group for kubernetes minion.
  133. volume_driver:
  134. type: string
  135. description: volume driver to use for container storage
  136. region_name:
  137. type: string
  138. description: A logically separate section of the cluster
  139. username:
  140. type: string
  141. description: >
  142. user account
  143. password:
  144. type: string
  145. description: >
  146. user password, not set in current implementation, only used to
  147. fill in for Kubernetes config file
  148. hidden: true
  149. http_proxy:
  150. type: string
  151. description: http proxy address for docker
  152. https_proxy:
  153. type: string
  154. description: https proxy address for docker
  155. no_proxy:
  156. type: string
  157. description: no proxies for docker
  158. kube_tag:
  159. type: string
  160. description: tag of the k8s containers used to provision the kubernetes cluster
  161. kube_version:
  162. type: string
  163. description: version of kubernetes used for kubernetes cluster
  164. trustee_domain_id:
  165. type: string
  166. description: domain id of the trustee
  167. trustee_user_id:
  168. type: string
  169. description: user id of the trustee
  170. trustee_username:
  171. type: string
  172. description: username of the trustee
  173. trustee_password:
  174. type: string
  175. description: password of the trustee
  176. hidden: true
  177. trust_id:
  178. type: string
  179. description: id of the trust which is used by the trustee
  180. hidden: true
  181. auth_url:
  182. type: string
  183. description: >
  184. url for keystone, must be v2 since k8s backend only support v2
  185. at this point
  186. insecure_registry_url:
  187. type: string
  188. description: insecure registry url
  189. container_infra_prefix:
  190. type: string
  191. description: >
  192. prefix of container images used in the cluster, kubernetes components,
  193. kubernetes-dashboard, coredns etc
  194. dns_service_ip:
  195. type: string
  196. description: >
  197. address used by Kubernetes DNS service
  198. dns_cluster_domain:
  199. type: string
  200. description: >
  201. domain name for cluster DNS
  202. openstack_ca:
  203. type: string
  204. description: The OpenStack CA certificate to install on the node.
  205. nodes_server_group_id:
  206. type: string
  207. description: ID of the server group for kubernetes cluster nodes.
  208. availability_zone:
  209. type: string
  210. description: >
  211. availability zone for master and nodes
  212. default: ""
  213. pods_network_cidr:
  214. type: string
  215. description: Configure the IP pool/range from which pod IPs will be chosen
  216. kubelet_options:
  217. type: string
  218. description: >
  219. additional options to be passed to the kubelet
  220. kubeproxy_options:
  221. type: string
  222. description: >
  223. additional options to be passed to the kube proxy
  224. octavia_enabled:
  225. type: boolean
  226. description: >
  227. whether or not to use Octavia for LoadBalancer type service.
  228. default: False
  229. cloud_provider_enabled:
  230. type: boolean
  231. description: Enable or disable the openstack kubernetes cloud provider
  232. heat_container_agent_tag:
  233. type: string
  234. description: tag of the heat_container_agent system container
  235. auto_healing_enabled:
  236. type: boolean
  237. description: >
  238. true if the auto healing feature should be enabled
  239. auto_healing_controller:
  240. type: string
  241. description: >
  242. The service to be deployed for auto-healing.
  243. default: "draino"
  244. npd_enabled:
  245. type: boolean
  246. description: >
  247. true if the npd service should be launched
  248. default:
  249. true
  250. conditions:
  251. image_based: {equals: [{get_param: boot_volume_size}, 0]}
  252. volume_based:
  253. not:
  254. equals:
  255. - get_param: boot_volume_size
  256. - 0
  257. resources:
  258. agent_config:
  259. type: OS::Heat::SoftwareConfig
  260. properties:
  261. group: ungrouped
  262. config:
  263. list_join:
  264. - "\n"
  265. -
  266. - str_replace:
  267. template: {get_file: user_data.json}
  268. params:
  269. $HOSTNAME: {get_param: name}
  270. $SSH_KEY_VALUE: {get_param: ssh_public_key}
  271. $OPENSTACK_CA: {get_param: openstack_ca}
  272. ######################################################################
  273. #
  274. # software configs. these are components that are combined into
  275. # a multipart MIME user-data archive.
  276. #
  277. node_config:
  278. type: OS::Heat::SoftwareConfig
  279. properties:
  280. group: script
  281. config:
  282. list_join:
  283. - "\n"
  284. -
  285. - str_replace:
  286. template: {get_file: ../../common/templates/kubernetes/fragments/write-heat-params.sh}
  287. params:
  288. $INSTANCE_NAME: {get_param: name}
  289. $PROMETHEUS_MONITORING: {get_param: prometheus_monitoring}
  290. $KUBE_ALLOW_PRIV: {get_param: kube_allow_priv}
  291. $KUBE_MASTER_IP: {get_param: kube_master_ip}
  292. $KUBE_API_PORT: {get_param: kubernetes_port}
  293. $KUBE_NODE_PUBLIC_IP: {get_attr: [kube_minion_floating, floating_ip_address]}
  294. $KUBE_NODE_IP: {get_attr: [kube_minion_eth0, fixed_ips, 0, ip_address]}
  295. $ETCD_SERVER_IP: {get_param: etcd_server_ip}
  296. $DOCKER_VOLUME: {get_resource: docker_volume}
  297. $DOCKER_VOLUME_SIZE: {get_param: docker_volume_size}
  298. $DOCKER_STORAGE_DRIVER: {get_param: docker_storage_driver}
  299. $CGROUP_DRIVER: {get_param: cgroup_driver}
  300. $NETWORK_DRIVER: {get_param: network_driver}
  301. $REGISTRY_ENABLED: {get_param: registry_enabled}
  302. $REGISTRY_PORT: {get_param: registry_port}
  303. $SWIFT_REGION: {get_param: swift_region}
  304. $REGISTRY_CONTAINER: {get_param: registry_container}
  305. $REGISTRY_INSECURE: {get_param: registry_insecure}
  306. $REGISTRY_CHUNKSIZE: {get_param: registry_chunksize}
  307. $TLS_DISABLED: {get_param: tls_disabled}
  308. $VERIFY_CA: {get_param: verify_ca}
  309. $CLUSTER_UUID: {get_param: cluster_uuid}
  310. $MAGNUM_URL: {get_param: magnum_url}
  311. $USERNAME: {get_param: username}
  312. $PASSWORD: {get_param: password}
  313. $VOLUME_DRIVER: {get_param: volume_driver}
  314. $REGION_NAME: {get_param: region_name}
  315. $HTTP_PROXY: {get_param: http_proxy}
  316. $HTTPS_PROXY: {get_param: https_proxy}
  317. $NO_PROXY: {get_param: no_proxy}
  318. $KUBE_TAG: {get_param: kube_tag}
  319. $FLANNEL_NETWORK_CIDR: {get_param: flannel_network_cidr}
  320. $PODS_NETWORK_CIDR: {get_param: pods_network_cidr}
  321. $KUBE_VERSION: {get_param: kube_version}
  322. $TRUSTEE_USER_ID: {get_param: trustee_user_id}
  323. $TRUSTEE_PASSWORD: {get_param: trustee_password}
  324. $TRUST_ID: {get_param: trust_id}
  325. $AUTH_URL: {get_param: auth_url}
  326. $CLOUD_PROVIDER_ENABLED: {get_param: cloud_provider_enabled}
  327. $INSECURE_REGISTRY_URL: {get_param: insecure_registry_url}
  328. $CONTAINER_INFRA_PREFIX: {get_param: container_infra_prefix}
  329. $DNS_SERVICE_IP: {get_param: dns_service_ip}
  330. $DNS_CLUSTER_DOMAIN: {get_param: dns_cluster_domain}
  331. $KUBELET_OPTIONS: {get_param: kubelet_options}
  332. $KUBEPROXY_OPTIONS: {get_param: kubeproxy_options}
  333. $OCTAVIA_ENABLED: {get_param: octavia_enabled}
  334. $HEAT_CONTAINER_AGENT_TAG: {get_param: heat_container_agent_tag}
  335. $AUTO_HEALING_ENABLED: {get_param: auto_healing_enabled}
  336. $AUTO_HEALING_CONTROLLER: {get_param: auto_healing_controller}
  337. $NPD_ENABLED: {get_param: npd_enabled}
  338. $NODEGROUP_ROLE: {get_param: nodegroup_role}
  339. $NODEGROUP_NAME: {get_param: nodegroup_name}
  340. - get_file: ../../common/templates/kubernetes/fragments/write-kube-os-config.sh
  341. - get_file: ../../common/templates/kubernetes/fragments/make-cert-client.sh
  342. - get_file: ../../common/templates/fragments/configure-docker-registry.sh
  343. - get_file: ../../common/templates/kubernetes/fragments/configure-kubernetes-minion.sh
  344. - get_file: ../../common/templates/kubernetes/fragments/add-proxy.sh
  345. # TODO add docker_storage_setup
  346. - get_file: ../../common/templates/kubernetes/fragments/enable-services-minion.sh
  347. - get_file: ../../common/templates/fragments/enable-docker-registry.sh
  348. node_config_deployment:
  349. type: OS::Heat::SoftwareDeployment
  350. properties:
  351. signal_transport: HEAT_SIGNAL
  352. config: {get_resource: node_config}
  353. server: {if: ["volume_based", {get_resource: kube-minion-bfv}, {get_resource: kube-minion}]}
  354. actions: ['CREATE']
  355. ######################################################################
  356. #
  357. # a single kubernetes minion.
  358. #
  359. kube_node_volume:
  360. type: OS::Cinder::Volume
  361. condition: volume_based
  362. properties:
  363. image: {get_param: server_image}
  364. size: {get_param: boot_volume_size}
  365. volume_type: {get_param: boot_volume_type}
  366. # do NOT use "_" (underscore) in the Nova server name
  367. # it creates a mismatch between the generated Nova name and its hostname
  368. # which can lead to weird problems
  369. kube-minion:
  370. condition: image_based
  371. type: OS::Nova::Server
  372. properties:
  373. name: {get_param: name}
  374. flavor: {get_param: minion_flavor}
  375. image: {get_param: server_image}
  376. user_data: {get_resource: agent_config}
  377. user_data_format: SOFTWARE_CONFIG
  378. software_config_transport: POLL_SERVER_HEAT
  379. networks:
  380. - port: {get_resource: kube_minion_eth0}
  381. scheduler_hints: { group: { get_param: nodes_server_group_id }}
  382. availability_zone: {get_param: availability_zone}
  383. kube-minion-bfv:
  384. condition: volume_based
  385. type: OS::Nova::Server
  386. properties:
  387. name: {get_param: name}
  388. flavor: {get_param: minion_flavor}
  389. user_data: {get_resource: agent_config}
  390. user_data_format: SOFTWARE_CONFIG
  391. software_config_transport: POLL_SERVER_HEAT
  392. networks:
  393. - port: {get_resource: kube_minion_eth0}
  394. scheduler_hints: { group: { get_param: nodes_server_group_id }}
  395. availability_zone: {get_param: availability_zone}
  396. block_device_mapping_v2:
  397. - boot_index: 0
  398. volume_id: {get_resource: kube_node_volume}
  399. delete_on_termination: true
  400. kube_minion_eth0:
  401. type: OS::Neutron::Port
  402. properties:
  403. network: {get_param: fixed_network}
  404. security_groups:
  405. - get_param: secgroup_kube_minion_id
  406. fixed_ips:
  407. - subnet: {get_param: fixed_subnet}
  408. allowed_address_pairs:
  409. - ip_address: {get_param: pods_network_cidr}
  410. replacement_policy: AUTO
  411. kube_minion_floating:
  412. type: Magnum::Optional::KubeMinion::Neutron::FloatingIP
  413. properties:
  414. floating_network: {get_param: external_network}
  415. port_id: {get_resource: kube_minion_eth0}
  416. depends_on: kube-minion
  417. ######################################################################
  418. #
  419. # docker storage. This allocates a cinder volume and attaches it
  420. # to the minion.
  421. #
  422. docker_volume:
  423. type: Magnum::Optional::Cinder::Volume
  424. properties:
  425. size: {get_param: docker_volume_size}
  426. volume_type: {get_param: docker_volume_type}
  427. docker_volume_attach:
  428. type: Magnum::Optional::Cinder::VolumeAttachment
  429. properties:
  430. instance_uuid: {if: ["volume_based", {get_resource: kube-minion-bfv}, {get_resource: kube-minion}]}
  431. volume_id: {get_resource: docker_volume}
  432. mountpoint: /dev/vdb
  433. upgrade_kubernetes:
  434. type: OS::Heat::SoftwareConfig
  435. properties:
  436. group: script
  437. inputs:
  438. - name: kube_tag_input
  439. config:
  440. get_file: ../../common/templates/kubernetes/fragments/upgrade-kubernetes.sh
  441. upgrade_kubernetes_deployment:
  442. type: OS::Heat::SoftwareDeployment
  443. properties:
  444. signal_transport: HEAT_SIGNAL
  445. config: {get_resource: upgrade_kubernetes}
  446. server: {if: ["volume_based", {get_resource: kube-minion-bfv}, {get_resource: kube-minion}]}
  447. actions: ['UPDATE']
  448. input_values:
  449. kube_tag_input: {get_param: kube_tag}
  450. outputs:
  451. kube_minion_ip:
  452. value: {get_attr: [kube_minion_eth0, fixed_ips, 0, ip_address]}
  453. description: >
  454. This is the "public" IP address of the Kubernetes minion node.
  455. kube_minion_external_ip:
  456. value: {get_attr: [kube_minion_floating, floating_ip_address]}
  457. description: >
  458. This is the "public" IP address of the Kubernetes minion node.
  459. ######################################################################
  460. #
  461. # NOTE(flwang): Returning the minion node server ID here so that
  462. # consumer can send API request to Heat to remove a particular
  463. # node with removal_policies. Otherwise, the consumer (e.g. AutoScaler)
  464. # has to use index to do the remove which is confusing out of the
  465. # OpenStack world.
  466. # https://storyboard.openstack.org/#!/story/2005054
  467. #
  468. ######################################################################
  469. OS::stack_id:
  470. value: {if: ["volume_based", {get_resource: kube-minion-bfv}, {get_resource: kube-minion}]}
  471. description: >
  472. This is the Nova server id of the node.