Container Infrastructure Management Service for OpenStack
You can not select more than 25 topics Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.
 
 
 
 
 

1285 lines
40 KiB

  1. heat_template_version: queens
  2. description: >
  3. This template will boot a Kubernetes cluster with one or more
  4. minions (as specified by the number_of_minions parameter, which
  5. defaults to 1).
  6. conditions:
  7. create_cluster_resources:
  8. equals:
  9. - get_param: is_cluster_stack
  10. - true
  11. is_master:
  12. and:
  13. - equals:
  14. - get_param: master_role
  15. - "master"
  16. - equals:
  17. - get_param: worker_role
  18. - ""
  19. is_worker:
  20. not:
  21. equals:
  22. - get_param: worker_role
  23. - ""
  24. master_only:
  25. or:
  26. - create_cluster_resources
  27. - is_master
  28. worker_only:
  29. or:
  30. - create_cluster_resources
  31. - is_worker
  32. parameters:
  33. # needs to become a list if we want to join master nodes?
  34. existing_master_private_ip:
  35. type: string
  36. default: ""
  37. is_cluster_stack:
  38. type: boolean
  39. default: false
  40. master_role:
  41. type: string
  42. default: ""
  43. worker_role:
  44. type: string
  45. default: ""
  46. existing_security_group:
  47. type: string
  48. default: ""
  49. ssh_key_name:
  50. type: string
  51. description: name of ssh key to be provisioned on our server
  52. default: ""
  53. ssh_public_key:
  54. type: string
  55. description: The public ssh key to add in all nodes
  56. default: ""
  57. external_network:
  58. type: string
  59. description: uuid of a network to use for floating ip addresses
  60. fixed_network:
  61. type: string
  62. description: uuid/name of an existing network to use to provision machines
  63. default: ""
  64. fixed_network_name:
  65. type: string
  66. description: name of a private network to use to provision machines
  67. default: "private"
  68. fixed_subnet:
  69. type: string
  70. description: uuid/name of an existing subnet to use to provision machines
  71. default: ""
  72. master_image:
  73. type: string
  74. description: glance image used to boot the server
  75. # When creating a new minion nodegroup this will not
  76. # be provided by magnum. So make it default to ""
  77. default: ""
  78. minion_image:
  79. type: string
  80. description: glance image used to boot the server
  81. # When creating a new master nodegroup this will not
  82. # be provided by magnum. So make it default to ""
  83. default: ""
  84. master_flavor:
  85. type: string
  86. default: m1.small
  87. description: flavor to use when booting the server for master nodes
  88. master_nodegroup_name:
  89. type: string
  90. default: ""
  91. description: the name of the nodegroup where the node belongs
  92. worker_nodegroup_name:
  93. type: string
  94. default: ""
  95. description: the name of the nodegroup where the node belongs
  96. minion_flavor:
  97. type: string
  98. default: m1.small
  99. description: flavor to use when booting the server for minions
  100. prometheus_monitoring:
  101. type: boolean
  102. default: false
  103. description: >
  104. whether or not to have the grafana-prometheus-cadvisor monitoring setup
  105. grafana_admin_passwd:
  106. type: string
  107. default: admin
  108. hidden: true
  109. description: >
  110. admin user password for the Grafana monitoring interface
  111. dns_nameserver:
  112. type: comma_delimited_list
  113. description: address of a DNS nameserver reachable in your environment
  114. default: 8.8.8.8
  115. number_of_masters:
  116. type: number
  117. description: how many kubernetes masters to spawn
  118. default: 1
  119. number_of_minions:
  120. type: number
  121. description: how many kubernetes minions to spawn
  122. default: 1
  123. fixed_network_cidr:
  124. type: string
  125. description: network range for fixed ip network
  126. default: 10.0.0.0/24
  127. portal_network_cidr:
  128. type: string
  129. description: >
  130. address range used by kubernetes for service portals
  131. default: 10.254.0.0/16
  132. network_driver:
  133. type: string
  134. description: network driver to use for instantiating container networks
  135. default: flannel
  136. flannel_network_cidr:
  137. type: string
  138. description: network range for flannel overlay network
  139. default: 10.100.0.0/16
  140. flannel_network_subnetlen:
  141. type: number
  142. description: size of subnet assigned to each minion
  143. default: 24
  144. flannel_backend:
  145. type: string
  146. description: >
  147. specify the backend for flannel, default vxlan backend
  148. default: "vxlan"
  149. constraints:
  150. - allowed_values: ["udp", "vxlan", "host-gw"]
  151. system_pods_initial_delay:
  152. type: number
  153. description: >
  154. health check, time to wait for system pods (podmaster, scheduler) to boot
  155. (in seconds)
  156. default: 30
  157. system_pods_timeout:
  158. type: number
  159. description: >
  160. health check, timeout for system pods (podmaster, scheduler) to answer.
  161. (in seconds)
  162. default: 5
  163. admission_control_list:
  164. type: string
  165. description: >
  166. List of admission control plugins to activate
  167. default: "NamespaceLifecycle,LimitRanger,ServiceAccount,DefaultStorageClass,DefaultTolerationSeconds,MutatingAdmissionWebhook,ValidatingAdmissionWebhook,ResourceQuota"
  168. kube_allow_priv:
  169. type: string
  170. description: >
  171. whether or not kubernetes should permit privileged containers.
  172. default: "true"
  173. constraints:
  174. - allowed_values: ["true", "false"]
  175. boot_volume_size:
  176. type: number
  177. description: >
  178. size of the cinder boot volume for nodes root volume
  179. boot_volume_type:
  180. type: string
  181. description: >
  182. type of the cinder boot volume for nodes root volume
  183. etcd_volume_size:
  184. type: number
  185. description: >
  186. size of the cinder volume for etcd storage
  187. default: 0
  188. etcd_volume_type:
  189. type: string
  190. description: >
  191. type of a cinder volume for etcd storage
  192. docker_volume_size:
  193. type: number
  194. description: >
  195. size of a cinder volume to allocate to docker for container/image
  196. storage
  197. default: 0
  198. docker_volume_type:
  199. type: string
  200. description: >
  201. type of a cinder volume to allocate to docker for container/image
  202. storage
  203. docker_storage_driver:
  204. type: string
  205. description: docker storage driver name
  206. default: "devicemapper"
  207. cgroup_driver:
  208. type: string
  209. description: >
  210. cgroup driver name that kubelet should use, ideally the same as
  211. the docker cgroup driver.
  212. default: "cgroupfs"
  213. traefik_ingress_controller_tag:
  214. type: string
  215. description: tag of the traefik containers to be used.
  216. default: v1.7.10
  217. wait_condition_timeout:
  218. type: number
  219. description: >
  220. timeout for the Wait Conditions
  221. default: 6000
  222. minions_to_remove:
  223. type: comma_delimited_list
  224. description: >
  225. List of minions to be removed when doing an update. Individual minion may
  226. be referenced several ways: (1) The resource name (e.g. ['1', '3']),
  227. (2) The private IP address ['10.0.0.4', '10.0.0.6']. Note: the list should
  228. be empty when doing an create.
  229. default: []
  230. discovery_url:
  231. type: string
  232. description: >
  233. Discovery URL used for bootstrapping the etcd cluster.
  234. registry_enabled:
  235. type: boolean
  236. description: >
  237. Indicates whether the docker registry is enabled.
  238. default: false
  239. registry_port:
  240. type: number
  241. description: port of registry service
  242. default: 5000
  243. swift_region:
  244. type: string
  245. description: region of swift service
  246. default: ""
  247. registry_container:
  248. type: string
  249. description: >
  250. name of swift container which docker registry stores images in
  251. default: "container"
  252. registry_insecure:
  253. type: boolean
  254. description: >
  255. indicates whether to skip TLS verification between registry and backend storage
  256. default: true
  257. registry_chunksize:
  258. type: number
  259. description: >
  260. size fo the data segments for the swift dynamic large objects
  261. default: 5242880
  262. volume_driver:
  263. type: string
  264. description: volume driver to use for container storage
  265. default: ""
  266. region_name:
  267. type: string
  268. description: A logically separate section of the cluster
  269. username:
  270. type: string
  271. description: >
  272. user account
  273. password:
  274. type: string
  275. description: >
  276. user password, not set in current implementation, only used to
  277. fill in for Kubernetes config file
  278. default:
  279. ChangeMe
  280. hidden: true
  281. loadbalancing_protocol:
  282. type: string
  283. description: >
  284. The protocol which is used for load balancing. If you want to change
  285. tls_disabled option to 'True', please change this to "HTTP".
  286. default: TCP
  287. constraints:
  288. - allowed_values: ["TCP", "HTTP"]
  289. tls_disabled:
  290. type: boolean
  291. description: whether or not to disable TLS
  292. default: False
  293. kube_dashboard_enabled:
  294. type: boolean
  295. description: whether or not to enable kubernetes dashboard
  296. default: True
  297. influx_grafana_dashboard_enabled:
  298. type: boolean
  299. description: Enable influxdb with grafana dashboard for data from heapster
  300. default: False
  301. verify_ca:
  302. type: boolean
  303. description: whether or not to validate certificate authority
  304. kubernetes_port:
  305. type: number
  306. description: >
  307. The port which are used by kube-apiserver to provide Kubernetes
  308. service.
  309. default: 6443
  310. cluster_uuid:
  311. type: string
  312. description: identifier for the cluster this template is generating
  313. magnum_url:
  314. type: string
  315. description: endpoint to retrieve TLS certs from
  316. http_proxy:
  317. type: string
  318. description: http proxy address for docker
  319. default: ""
  320. https_proxy:
  321. type: string
  322. description: https proxy address for docker
  323. default: ""
  324. no_proxy:
  325. type: string
  326. description: no proxies for docker
  327. default: ""
  328. trustee_domain_id:
  329. type: string
  330. description: domain id of the trustee
  331. trustee_user_id:
  332. type: string
  333. description: user id of the trustee
  334. trustee_username:
  335. type: string
  336. description: username of the trustee
  337. trustee_password:
  338. type: string
  339. description: password of the trustee
  340. hidden: true
  341. trust_id:
  342. type: string
  343. description: id of the trust which is used by the trustee
  344. hidden: true
  345. auth_url:
  346. type: string
  347. description: url for keystone
  348. kube_tag:
  349. type: string
  350. description: tag of the k8s containers used to provision the kubernetes cluster
  351. default: v1.14.3
  352. master_kube_tag:
  353. type: string
  354. description: tag of the k8s containers used to provision the kubernetes cluster
  355. default: v1.14.3
  356. minion_kube_tag:
  357. type: string
  358. description: tag of the k8s containers used to provision the kubernetes cluster
  359. default: v1.14.3
  360. # FIXME update cloud_provider_tag when a fix for PVC is released
  361. # https://github.com/kubernetes/cloud-provider-openstack/pull/405
  362. cloud_provider_tag:
  363. type: string
  364. description:
  365. tag of the kubernetes/cloud-provider-openstack
  366. https://hub.docker.com/r/k8scloudprovider/openstack-cloud-controller-manager/tags/
  367. default: v1.14.0
  368. cloud_provider_enabled:
  369. type: boolean
  370. description: Enable or disable the openstack kubernetes cloud provider
  371. etcd_tag:
  372. type: string
  373. description: tag of the etcd system container
  374. default: 3.2.26
  375. coredns_tag:
  376. type: string
  377. description: tag for coredns
  378. default: 1.3.1
  379. flannel_tag:
  380. type: string
  381. description: tag of the flannel container
  382. default: v0.11.0-amd64
  383. flannel_cni_tag:
  384. type: string
  385. description: tag of the flannel cni container
  386. default: v0.3.0
  387. kube_version:
  388. type: string
  389. description: version of kubernetes used for kubernetes cluster
  390. default: v1.14.3
  391. kube_dashboard_version:
  392. type: string
  393. description: version of kubernetes dashboard used for kubernetes cluster
  394. default: v1.8.3
  395. insecure_registry_url:
  396. type: string
  397. description: insecure registry url
  398. default: ""
  399. container_infra_prefix:
  400. type: string
  401. description: >
  402. prefix of container images used in the cluster, kubernetes components,
  403. kubernetes-dashboard, coredns etc
  404. constraints:
  405. - allowed_pattern: "^$|.*/"
  406. default: ""
  407. dns_service_ip:
  408. type: string
  409. description: >
  410. address used by Kubernetes DNS service
  411. default: 10.254.0.10
  412. dns_cluster_domain:
  413. type: string
  414. description: >
  415. domain name for cluster DNS
  416. default: "cluster.local"
  417. openstack_ca:
  418. type: string
  419. hidden: true
  420. description: The OpenStack CA certificate to install on the node.
  421. nodes_affinity_policy:
  422. type: string
  423. description: >
  424. affinity policy for nodes server group
  425. constraints:
  426. - allowed_values: ["affinity", "anti-affinity", "soft-affinity",
  427. "soft-anti-affinity"]
  428. availability_zone:
  429. type: string
  430. description: >
  431. availability zone for master and nodes
  432. default: ""
  433. cert_manager_api:
  434. type: boolean
  435. description: true if the kubernetes cert api manager should be enabled
  436. default: false
  437. ca_key:
  438. type: string
  439. description: key of internal ca for the kube certificate api manager
  440. default: ""
  441. hidden: true
  442. calico_tag:
  443. type: string
  444. description: tag of the calico containers used to provision the calico node
  445. default: v3.3.6
  446. calico_kube_controllers_tag:
  447. type: string
  448. description: tag of the kube_controllers used to provision the calico node
  449. default: v1.0.3
  450. calico_ipv4pool:
  451. type: string
  452. description: Configure the IP pool from which Pod IPs will be chosen
  453. default: "192.168.0.0/16"
  454. pods_network_cidr:
  455. type: string
  456. description: Configure the IP pool/range from which pod IPs will be chosen
  457. ingress_controller:
  458. type: string
  459. description: >
  460. ingress controller backend to use
  461. default: ""
  462. ingress_controller_role:
  463. type: string
  464. description: >
  465. node role where the ingress controller backend should run
  466. default: "ingress"
  467. octavia_ingress_controller_tag:
  468. type: string
  469. description: Octavia ingress controller docker image tag.
  470. default: "1.13.2-alpha"
  471. kubelet_options:
  472. type: string
  473. description: >
  474. additional options to be passed to the kubelet
  475. default: ""
  476. kubeapi_options:
  477. type: string
  478. description: >
  479. additional options to be passed to the api
  480. default: ""
  481. kubecontroller_options:
  482. type: string
  483. description: >
  484. additional options to be passed to the controller manager
  485. default: ""
  486. kubeproxy_options:
  487. type: string
  488. description: >
  489. additional options to be passed to the kube proxy
  490. default: ""
  491. kubescheduler_options:
  492. type: string
  493. description: >
  494. additional options to be passed to the scheduler
  495. default: ""
  496. octavia_enabled:
  497. type: boolean
  498. description: >
  499. whether or not to use Octavia for LoadBalancer type service.
  500. default: False
  501. kube_service_account_key:
  502. type: string
  503. hidden: true
  504. description: >
  505. The signed cert will be used to verify the k8s service account tokens
  506. during authentication.
  507. kube_service_account_private_key:
  508. type: string
  509. hidden: true
  510. description: >
  511. The private key will be used to sign generated k8s service account
  512. tokens.
  513. prometheus_tag:
  514. type: string
  515. description: tag of the prometheus container
  516. default: v1.8.2
  517. grafana_tag:
  518. type: string
  519. description: tag of grafana container
  520. default: 5.1.5
  521. heat_container_agent_tag:
  522. type: string
  523. description: tag of the heat_container_agent system container
  524. default: train-dev
  525. keystone_auth_enabled:
  526. type: boolean
  527. description: >
  528. true if the keystone authN and authZ should be enabled
  529. default:
  530. true
  531. keystone_auth_default_policy:
  532. type: string
  533. description: Json read from /etc/magnum/keystone_auth_default_policy.json
  534. default: ""
  535. k8s_keystone_auth_tag:
  536. type: string
  537. description: tag of the k8s_keystone_auth container
  538. default: v1.14.0
  539. monitoring_enabled:
  540. type: boolean
  541. description: Enable or disable prometheus-operator monitoring solution.
  542. default: false
  543. prometheus_operator_chart_tag:
  544. type: string
  545. description: The stable/prometheus-operator chart version to use.
  546. default: 5.12.3
  547. project_id:
  548. type: string
  549. description: >
  550. project id of current project
  551. tiller_enabled:
  552. type: boolean
  553. description: Choose whether to install tiller or not.
  554. default: false
  555. tiller_tag:
  556. type: string
  557. description: tag of tiller container
  558. default: "v2.12.3"
  559. tiller_namespace:
  560. type: string
  561. description: namespace where tiller will be installed.
  562. default: "magnum-tiller"
  563. auto_healing_enabled:
  564. type: boolean
  565. description: >
  566. true if the auto healing feature should be enabled
  567. default:
  568. false
  569. auto_healing_controller:
  570. type: string
  571. description: >
  572. The service to be deployed for auto-healing.
  573. default: "draino"
  574. magnum_auto_healer_tag:
  575. type: string
  576. description: tag of the magnum-auto-healer service.
  577. default: "v1.15.0"
  578. auto_scaling_enabled:
  579. type: boolean
  580. description: >
  581. true if the auto scaling feature should be enabled
  582. default:
  583. false
  584. node_problem_detector_tag:
  585. type: string
  586. description: tag of the node problem detector container
  587. default: v0.6.2
  588. nginx_ingress_controller_tag:
  589. type: string
  590. description: nginx ingress controller docker image tag
  591. default: 0.23.0
  592. draino_tag:
  593. type: string
  594. description: tag of the draino container
  595. default: abf028a
  596. autoscaler_tag:
  597. type: string
  598. description: tag of the autoscaler container
  599. default: v1.0
  600. min_node_count:
  601. type: number
  602. description: >
  603. minimum node count of cluster workers when doing scale down
  604. default: 1
  605. max_node_count:
  606. type: number
  607. description: >
  608. maximum node count of cluster workers when doing scale up
  609. update_max_batch_size:
  610. type: number
  611. description: >
  612. max batch size when doing rolling upgrade
  613. default: 1
  614. npd_enabled:
  615. type: boolean
  616. description: >
  617. true if the npd service should be launched
  618. default:
  619. true
  620. resources:
  621. ######################################################################
  622. #
  623. # network resources. allocate a network and router for our server.
  624. # Important: the Load Balancer feature in Kubernetes requires that
  625. # the name for the fixed_network must be "private" for the
  626. # address lookup in Kubernetes to work properly
  627. #
  628. network:
  629. condition: create_cluster_resources
  630. type: ../../common/templates/network.yaml
  631. properties:
  632. existing_network: {get_param: fixed_network}
  633. existing_subnet: {get_param: fixed_subnet}
  634. private_network_cidr: {get_param: fixed_network_cidr}
  635. dns_nameserver: {get_param: dns_nameserver}
  636. external_network: {get_param: external_network}
  637. private_network_name: {get_param: fixed_network_name}
  638. api_lb:
  639. condition: create_cluster_resources
  640. type: ../../common/templates/lb_api.yaml
  641. properties:
  642. fixed_subnet: {get_attr: [network, fixed_subnet]}
  643. external_network: {get_param: external_network}
  644. protocol: {get_param: loadbalancing_protocol}
  645. port: {get_param: kubernetes_port}
  646. etcd_lb:
  647. condition: create_cluster_resources
  648. type: ../../common/templates/lb_etcd.yaml
  649. properties:
  650. fixed_subnet: {get_attr: [network, fixed_subnet]}
  651. protocol: {get_param: loadbalancing_protocol}
  652. port: 2379
  653. ######################################################################
  654. #
  655. # security groups. we need to permit network traffic of various
  656. # sorts.
  657. #
  658. secgroup_kube_master:
  659. condition: create_cluster_resources
  660. type: OS::Neutron::SecurityGroup
  661. properties:
  662. rules:
  663. - protocol: icmp
  664. - protocol: tcp
  665. port_range_min: 22
  666. port_range_max: 22
  667. - protocol: tcp
  668. port_range_min: 7080
  669. port_range_max: 7080
  670. - protocol: tcp
  671. port_range_min: 8080
  672. port_range_max: 8080
  673. - protocol: tcp
  674. port_range_min: 2379
  675. port_range_max: 2379
  676. - protocol: tcp
  677. port_range_min: 2380
  678. port_range_max: 2380
  679. - protocol: tcp
  680. port_range_min: 6443
  681. port_range_max: 6443
  682. - protocol: tcp
  683. port_range_min: 9100
  684. port_range_max: 9100
  685. - protocol: tcp
  686. port_range_min: 10250
  687. port_range_max: 10250
  688. - protocol: tcp
  689. port_range_min: 30000
  690. port_range_max: 32767
  691. - protocol: udp
  692. port_range_min: 8472
  693. port_range_max: 8472
  694. secgroup_kube_minion:
  695. condition: create_cluster_resources
  696. type: OS::Neutron::SecurityGroup
  697. properties:
  698. rules:
  699. - protocol: icmp
  700. # Default port range for external service ports.
  701. # In future, if the option `manage-security-groups` for ccm works
  702. # well, we could remove this rule here.
  703. # The PR in ccm is
  704. # https://github.com/kubernetes/cloud-provider-openstack/pull/491
  705. - protocol: tcp
  706. port_range_min: 22
  707. port_range_max: 22
  708. - protocol: tcp
  709. port_range_min: 30000
  710. port_range_max: 32767
  711. # allow any traffic from master nodes
  712. - protocol: tcp
  713. port_range_min: 1
  714. port_range_max: 65535
  715. remote_mode: 'remote_group_id'
  716. remote_group_id: {get_resource: secgroup_kube_master}
  717. - protocol: udp
  718. port_range_min: 1
  719. port_range_max: 65535
  720. remote_mode: 'remote_group_id'
  721. remote_group_id: {get_resource: secgroup_kube_master}
  722. # allow any traffic between worker nodes
  723. secgroup_rule_tcp_kube_minion:
  724. condition: create_cluster_resources
  725. type: OS::Neutron::SecurityGroupRule
  726. properties:
  727. protocol: tcp
  728. port_range_min: 1
  729. port_range_max: 65535
  730. security_group: {get_resource: secgroup_kube_minion}
  731. remote_group: {get_resource: secgroup_kube_minion}
  732. secgroup_rule_udp_kube_minion:
  733. condition: create_cluster_resources
  734. type: OS::Neutron::SecurityGroupRule
  735. properties:
  736. protocol: udp
  737. port_range_min: 1
  738. port_range_max: 65535
  739. security_group: {get_resource: secgroup_kube_minion}
  740. remote_group: {get_resource: secgroup_kube_minion}
  741. ######################################################################
  742. #
  743. # resources that expose the IPs of either the kube master or a given
  744. # LBaaS pool depending on whether LBaaS is enabled for the cluster.
  745. #
  746. api_address_lb_switch:
  747. condition: create_cluster_resources
  748. type: Magnum::ApiGatewaySwitcher
  749. properties:
  750. pool_public_ip: {get_attr: [api_lb, floating_address]}
  751. pool_private_ip: {get_attr: [api_lb, address]}
  752. master_public_ip: {get_attr: [kube_masters, resource.0.kube_master_external_ip]}
  753. master_private_ip: {get_attr: [kube_masters, resource.0.kube_master_ip]}
  754. etcd_address_lb_switch:
  755. condition: create_cluster_resources
  756. type: Magnum::ApiGatewaySwitcher
  757. properties:
  758. pool_private_ip: {get_attr: [etcd_lb, address]}
  759. master_private_ip: {get_attr: [kube_masters, resource.0.kube_master_ip]}
  760. ######################################################################
  761. #
  762. # resources that expose the IPs of either floating ip or a given
  763. # fixed ip depending on whether FloatingIP is enabled for the cluster.
  764. #
  765. api_address_floating_switch:
  766. condition: create_cluster_resources
  767. type: Magnum::FloatingIPAddressSwitcher
  768. properties:
  769. public_ip: {get_attr: [api_address_lb_switch, public_ip]}
  770. private_ip: {get_attr: [api_address_lb_switch, private_ip]}
  771. ######################################################################
  772. #
  773. # resources that expose one server group for each master and worker nodes
  774. # separately.
  775. #
  776. master_nodes_server_group:
  777. condition: master_only
  778. type: OS::Nova::ServerGroup
  779. properties:
  780. policies: [{get_param: nodes_affinity_policy}]
  781. worker_nodes_server_group:
  782. condition: worker_only
  783. type: OS::Nova::ServerGroup
  784. properties:
  785. policies: [{get_param: nodes_affinity_policy}]
  786. ######################################################################
  787. #
  788. # kubernetes masters. This is a resource group that will create
  789. # <number_of_masters> masters.
  790. #
  791. kube_masters:
  792. condition: master_only
  793. type: OS::Heat::ResourceGroup
  794. depends_on:
  795. - network
  796. update_policy:
  797. rolling_update: {max_batch_size: {get_param: update_max_batch_size}, pause_time: 30}
  798. properties:
  799. count: {get_param: number_of_masters}
  800. resource_def:
  801. type: kubemaster.yaml
  802. properties:
  803. name:
  804. list_join:
  805. - '-'
  806. - [{ get_param: 'OS::stack_name' }, 'master', '%index%']
  807. nodegroup_role: {get_param: master_role}
  808. nodegroup_name: {get_param: master_nodegroup_name}
  809. prometheus_monitoring: {get_param: prometheus_monitoring}
  810. grafana_admin_passwd: {get_param: grafana_admin_passwd}
  811. api_public_address: {get_attr: [api_lb, floating_address]}
  812. api_private_address: {get_attr: [api_lb, address]}
  813. ssh_key_name: {get_param: ssh_key_name}
  814. server_image: {get_param: master_image}
  815. master_flavor: {get_param: master_flavor}
  816. external_network: {get_param: external_network}
  817. kube_allow_priv: {get_param: kube_allow_priv}
  818. boot_volume_size: {get_param: boot_volume_size}
  819. boot_volume_type: {get_param: boot_volume_type}
  820. etcd_volume_size: {get_param: etcd_volume_size}
  821. etcd_volume_type: {get_param: etcd_volume_type}
  822. docker_volume_size: {get_param: docker_volume_size}
  823. docker_volume_type: {get_param: docker_volume_type}
  824. docker_storage_driver: {get_param: docker_storage_driver}
  825. cgroup_driver: {get_param: cgroup_driver}
  826. network_driver: {get_param: network_driver}
  827. flannel_network_cidr: {get_param: flannel_network_cidr}
  828. flannel_network_subnetlen: {get_param: flannel_network_subnetlen}
  829. flannel_backend: {get_param: flannel_backend}
  830. system_pods_initial_delay: {get_param: system_pods_initial_delay}
  831. system_pods_timeout: {get_param: system_pods_timeout}
  832. portal_network_cidr: {get_param: portal_network_cidr}
  833. admission_control_list: {get_param: admission_control_list}
  834. discovery_url: {get_param: discovery_url}
  835. cluster_uuid: {get_param: cluster_uuid}
  836. magnum_url: {get_param: magnum_url}
  837. traefik_ingress_controller_tag: {get_param: traefik_ingress_controller_tag}
  838. volume_driver: {get_param: volume_driver}
  839. region_name: {get_param: region_name}
  840. fixed_network: {get_attr: [network, fixed_network]}
  841. fixed_network_name: {get_param: fixed_network_name}
  842. fixed_subnet: {get_attr: [network, fixed_subnet]}
  843. api_pool_id: {get_attr: [api_lb, pool_id]}
  844. etcd_pool_id: {get_attr: [etcd_lb, pool_id]}
  845. username: {get_param: username}
  846. password: {get_param: password}
  847. kubernetes_port: {get_param: kubernetes_port}
  848. tls_disabled: {get_param: tls_disabled}
  849. kube_dashboard_enabled: {get_param: kube_dashboard_enabled}
  850. influx_grafana_dashboard_enabled: {get_param: influx_grafana_dashboard_enabled}
  851. verify_ca: {get_param: verify_ca}
  852. secgroup_kube_master_id: {get_resource: secgroup_kube_master}
  853. http_proxy: {get_param: http_proxy}
  854. https_proxy: {get_param: https_proxy}
  855. no_proxy: {get_param: no_proxy}
  856. kube_tag: {get_param: master_kube_tag}
  857. cloud_provider_tag: {get_param: cloud_provider_tag}
  858. cloud_provider_enabled: {get_param: cloud_provider_enabled}
  859. kube_version: {get_param: kube_version}
  860. etcd_tag: {get_param: etcd_tag}
  861. coredns_tag: {get_param: coredns_tag}
  862. flannel_tag: {get_param: flannel_tag}
  863. flannel_cni_tag: {get_param: flannel_cni_tag}
  864. kube_dashboard_version: {get_param: kube_dashboard_version}
  865. trustee_user_id: {get_param: trustee_user_id}
  866. trustee_password: {get_param: trustee_password}
  867. trust_id: {get_param: trust_id}
  868. auth_url: {get_param: auth_url}
  869. insecure_registry_url: {get_param: insecure_registry_url}
  870. container_infra_prefix: {get_param: container_infra_prefix}
  871. etcd_lb_vip: {get_attr: [etcd_lb, address]}
  872. dns_service_ip: {get_param: dns_service_ip}
  873. dns_cluster_domain: {get_param: dns_cluster_domain}
  874. openstack_ca: {get_param: openstack_ca}
  875. nodes_server_group_id: {get_resource: master_nodes_server_group}
  876. availability_zone: {get_param: availability_zone}
  877. ca_key: {get_param: ca_key}
  878. cert_manager_api: {get_param: cert_manager_api}
  879. calico_tag: {get_param: calico_tag}
  880. calico_kube_controllers_tag: {get_param: calico_kube_controllers_tag}
  881. calico_ipv4pool: {get_param: calico_ipv4pool}
  882. pods_network_cidr: {get_param: pods_network_cidr}
  883. ingress_controller: {get_param: ingress_controller}
  884. ingress_controller_role: {get_param: ingress_controller_role}
  885. octavia_ingress_controller_tag: {get_param: octavia_ingress_controller_tag}
  886. kubelet_options: {get_param: kubelet_options}
  887. kubeapi_options: {get_param: kubeapi_options}
  888. kubeproxy_options: {get_param: kubeproxy_options}
  889. kubecontroller_options: {get_param: kubecontroller_options}
  890. kubescheduler_options: {get_param: kubescheduler_options}
  891. octavia_enabled: {get_param: octavia_enabled}
  892. kube_service_account_key: {get_param: kube_service_account_key}
  893. kube_service_account_private_key: {get_param: kube_service_account_private_key}
  894. prometheus_tag: {get_param: prometheus_tag}
  895. grafana_tag: {get_param: grafana_tag}
  896. heat_container_agent_tag: {get_param: heat_container_agent_tag}
  897. keystone_auth_enabled: {get_param: keystone_auth_enabled}
  898. k8s_keystone_auth_tag: {get_param: k8s_keystone_auth_tag}
  899. monitoring_enabled: {get_param: monitoring_enabled}
  900. prometheus_operator_chart_tag: {get_param: prometheus_operator_chart_tag}
  901. project_id: {get_param: project_id}
  902. tiller_enabled: {get_param: tiller_enabled}
  903. tiller_tag: {get_param: tiller_tag}
  904. tiller_namespace: {get_param: tiller_namespace}
  905. node_problem_detector_tag: {get_param: node_problem_detector_tag}
  906. nginx_ingress_controller_tag: {get_param: nginx_ingress_controller_tag}
  907. auto_healing_enabled: {get_param: auto_healing_enabled}
  908. auto_healing_controller: {get_param: auto_healing_controller}
  909. magnum_auto_healer_tag: {get_param: magnum_auto_healer_tag}
  910. auto_scaling_enabled: {get_param: auto_scaling_enabled}
  911. draino_tag: {get_param: draino_tag}
  912. autoscaler_tag: {get_param: autoscaler_tag}
  913. min_node_count: {get_param: min_node_count}
  914. max_node_count: {get_param: max_node_count}
  915. npd_enabled: {get_param: npd_enabled}
  916. kube_cluster_config:
  917. condition: create_cluster_resources
  918. type: OS::Heat::SoftwareConfig
  919. properties:
  920. group: script
  921. config:
  922. list_join:
  923. - "\n"
  924. -
  925. - str_replace:
  926. template: {get_file: ../../common/templates/kubernetes/fragments/enable-cert-api-manager.sh}
  927. params:
  928. "$CA_KEY": {get_param: ca_key}
  929. - get_file: ../../common/templates/kubernetes/fragments/kube-apiserver-to-kubelet-role.sh
  930. - get_file: ../../common/templates/kubernetes/fragments/core-dns-service.sh
  931. - get_file: ../../common/templates/kubernetes/fragments/calico-service.sh
  932. - get_file: ../../common/templates/kubernetes/fragments/flannel-service.sh
  933. - get_file: ../../common/templates/kubernetes/fragments/enable-helm-tiller.sh
  934. - str_replace:
  935. template: {get_file: ../../common/templates/kubernetes/fragments/enable-prometheus-monitoring.sh}
  936. params:
  937. "$ADMIN_PASSWD": {get_param: grafana_admin_passwd}
  938. - str_replace:
  939. params:
  940. $enable-ingress-traefik: {get_file: ../../common/templates/kubernetes/fragments/enable-ingress-traefik.sh}
  941. $enable-ingress-octavia: {get_file: ../../common/templates/kubernetes/fragments/enable-ingress-octavia.sh}
  942. template: {get_file: ../../common/templates/kubernetes/fragments/enable-ingress-controller.sh}
  943. - get_file: ../../common/templates/kubernetes/fragments/kube-dashboard-service.sh
  944. - str_replace:
  945. template: {get_file: ../../common/templates/kubernetes/fragments/enable-keystone-auth.sh}
  946. params:
  947. "$KEYSTONE_AUTH_DEFAULT_POLICY": {get_param: keystone_auth_default_policy}
  948. - get_file: ../../common/templates/kubernetes/fragments/enable-auto-healing.sh
  949. - get_file: ../../common/templates/kubernetes/fragments/enable-auto-scaling.sh
  950. # Helm Based Installation Configuration Scripts
  951. - get_file: ../../common/templates/kubernetes/helm/metrics-server.sh
  952. - str_replace:
  953. template: {get_file: ../../common/templates/kubernetes/helm/prometheus-operator.sh}
  954. params:
  955. "${ADMIN_PASSWD}": {get_param: grafana_admin_passwd}
  956. - get_file: ../../common/templates/kubernetes/helm/ingress-nginx.sh
  957. - get_file: ../../common/templates/kubernetes/fragments/install-helm-modules.sh
  958. kube_cluster_deploy:
  959. condition: create_cluster_resources
  960. type: OS::Heat::SoftwareDeployment
  961. properties:
  962. actions: ['CREATE']
  963. signal_transport: HEAT_SIGNAL
  964. config:
  965. get_resource: kube_cluster_config
  966. server:
  967. get_attr: [kube_masters, resource.0]
  968. ######################################################################
  969. #
  970. # kubernetes minions. This is an resource group that will initially
  971. # create <number_of_minions> minions, and needs to be manually scaled.
  972. #
  973. kube_minions:
  974. condition: worker_only
  975. type: OS::Heat::ResourceGroup
  976. depends_on:
  977. - network
  978. update_policy:
  979. rolling_update: {max_batch_size: {get_param: update_max_batch_size}, pause_time: 30}
  980. properties:
  981. count: {get_param: number_of_minions}
  982. removal_policies: [{resource_list: {get_param: minions_to_remove}}]
  983. resource_def:
  984. type: kubeminion.yaml
  985. properties:
  986. name:
  987. list_join:
  988. - '-'
  989. - [{ get_param: 'OS::stack_name' }, 'node', '%index%']
  990. prometheus_monitoring: {get_param: prometheus_monitoring}
  991. nodegroup_role: {get_param: worker_role}
  992. nodegroup_name: {get_param: worker_nodegroup_name}
  993. ssh_key_name: {get_param: ssh_key_name}
  994. server_image: {get_param: minion_image}
  995. minion_flavor: {get_param: minion_flavor}
  996. fixed_network:
  997. if:
  998. - create_cluster_resources
  999. - get_attr: [network, fixed_network]
  1000. - get_param: fixed_network
  1001. fixed_subnet:
  1002. if:
  1003. - create_cluster_resources
  1004. - get_attr: [network, fixed_subnet]
  1005. - get_param: fixed_subnet
  1006. network_driver: {get_param: network_driver}
  1007. flannel_network_cidr: {get_param: flannel_network_cidr}
  1008. kube_master_ip:
  1009. if:
  1010. - create_cluster_resources
  1011. - get_attr: [api_address_lb_switch, private_ip]
  1012. - get_param: existing_master_private_ip
  1013. etcd_server_ip:
  1014. if:
  1015. - create_cluster_resources
  1016. - get_attr: [etcd_address_lb_switch, private_ip]
  1017. - get_param: existing_master_private_ip
  1018. external_network: {get_param: external_network}
  1019. kube_allow_priv: {get_param: kube_allow_priv}
  1020. boot_volume_size: {get_param: boot_volume_size}
  1021. boot_volume_type: {get_param: boot_volume_type}
  1022. docker_volume_size: {get_param: docker_volume_size}
  1023. docker_volume_type: {get_param: docker_volume_type}
  1024. docker_storage_driver: {get_param: docker_storage_driver}
  1025. cgroup_driver: {get_param: cgroup_driver}
  1026. wait_condition_timeout: {get_param: wait_condition_timeout}
  1027. registry_enabled: {get_param: registry_enabled}
  1028. registry_port: {get_param: registry_port}
  1029. swift_region: {get_param: swift_region}
  1030. registry_container: {get_param: registry_container}
  1031. registry_insecure: {get_param: registry_insecure}
  1032. registry_chunksize: {get_param: registry_chunksize}
  1033. cluster_uuid: {get_param: cluster_uuid}
  1034. magnum_url: {get_param: magnum_url}
  1035. volume_driver: {get_param: volume_driver}
  1036. region_name: {get_param: region_name}
  1037. auth_url: {get_param: auth_url}
  1038. username: {get_param: username}
  1039. password: {get_param: password}
  1040. kubernetes_port: {get_param: kubernetes_port}
  1041. tls_disabled: {get_param: tls_disabled}
  1042. verify_ca: {get_param: verify_ca}
  1043. secgroup_kube_minion_id:
  1044. if:
  1045. - create_cluster_resources
  1046. - get_resource: secgroup_kube_minion
  1047. - get_param: existing_security_group
  1048. http_proxy: {get_param: http_proxy}
  1049. https_proxy: {get_param: https_proxy}
  1050. no_proxy: {get_param: no_proxy}
  1051. kube_tag: {get_param: minion_kube_tag}
  1052. kube_version: {get_param: kube_version}
  1053. trustee_user_id: {get_param: trustee_user_id}
  1054. trustee_username: {get_param: trustee_username}
  1055. trustee_password: {get_param: trustee_password}
  1056. trustee_domain_id: {get_param: trustee_domain_id}
  1057. trust_id: {get_param: trust_id}
  1058. cloud_provider_enabled: {get_param: cloud_provider_enabled}
  1059. insecure_registry_url: {get_param: insecure_registry_url}
  1060. container_infra_prefix: {get_param: container_infra_prefix}
  1061. dns_service_ip: {get_param: dns_service_ip}
  1062. dns_cluster_domain: {get_param: dns_cluster_domain}
  1063. openstack_ca: {get_param: openstack_ca}
  1064. nodes_server_group_id: {get_resource: worker_nodes_server_group}
  1065. availability_zone: {get_param: availability_zone}
  1066. pods_network_cidr: {get_param: pods_network_cidr}
  1067. kubelet_options: {get_param: kubelet_options}
  1068. kubeproxy_options: {get_param: kubeproxy_options}
  1069. octavia_enabled: {get_param: octavia_enabled}
  1070. heat_container_agent_tag: {get_param: heat_container_agent_tag}
  1071. auto_healing_enabled: {get_param: auto_healing_enabled}
  1072. npd_enabled: {get_param: npd_enabled}
  1073. auto_healing_controller: {get_param: auto_healing_controller}
  1074. outputs:
  1075. api_address:
  1076. condition: create_cluster_resources
  1077. value:
  1078. str_replace:
  1079. template: api_ip_address
  1080. params:
  1081. api_ip_address: {get_attr: [api_address_floating_switch, ip_address]}
  1082. description: >
  1083. This is the API endpoint of the Kubernetes cluster. Use this to access
  1084. the Kubernetes API.
  1085. registry_address:
  1086. condition: create_cluster_resources
  1087. value:
  1088. str_replace:
  1089. template: localhost:port
  1090. params:
  1091. port: {get_param: registry_port}
  1092. description:
  1093. This is the url of docker registry server where you can store docker
  1094. images.
  1095. kube_masters_private:
  1096. condition: master_only
  1097. value: {get_attr: [kube_masters, kube_master_ip]}
  1098. description: >
  1099. This is a list of the "private" IP addresses of all the Kubernetes masters.
  1100. kube_masters:
  1101. condition: master_only
  1102. value: {get_attr: [kube_masters, kube_master_external_ip]}
  1103. description: >
  1104. This is a list of the "public" IP addresses of all the Kubernetes masters.
  1105. Use these IP addresses to log in to the Kubernetes masters via ssh.
  1106. kube_minions_private:
  1107. condition: worker_only
  1108. value: {get_attr: [kube_minions, kube_minion_ip]}
  1109. description: >
  1110. This is a list of the "private" IP addresses of all the Kubernetes minions.
  1111. kube_minions:
  1112. condition: worker_only
  1113. value: {get_attr: [kube_minions, kube_minion_external_ip]}
  1114. description: >
  1115. This is a list of the "public" IP addresses of all the Kubernetes minions.
  1116. Use these IP addresses to log in to the Kubernetes minions via ssh.