A Kubernetes Operator for Zuul
You can not select more than 25 topics Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.
 
 
 
 
 

606 lines
24 KiB

  1. {- Zuul CR kubernetes resources
  2. The evaluation of that file is a function that takes the cr inputs as an argument,
  3. and returns the list of kubernetes of objects.
  4. Unless cert-manager usage is enabled, the resources expect those secrets to be available:
  5. * `${name}-gearman-tls` with:
  6. * `ca.crt`
  7. * `tls.crt`
  8. * `tls.key`
  9. * `${name}-registry-tls` with:
  10. * `tls.crt`
  11. * `tls.key`
  12. The resources expect those secrets to be available:
  13. * `${name}-zookeeper-tls` with:
  14. * `ca.crt`
  15. * `tls.crt`
  16. * `tls.key`
  17. * `zk.pem` the keystore
  18. * `${name}-registry-user-rw` with:
  19. * `secret` a password
  20. * `username` the user name with write access
  21. * `password` the user password
  22. Unless the input.database db uri is provided, the resources expect this secret to be available:
  23. * `${name}-database-password` the internal database password.
  24. -}
  25. let Prelude = ../Prelude.dhall
  26. let Kubernetes = ../Kubernetes.dhall
  27. let CertManager = ../CertManager.dhall
  28. let Schemas = ./input.dhall
  29. let F = ./functions.dhall
  30. let Input = Schemas.Input.Type
  31. let JobVolume = Schemas.JobVolume.Type
  32. let UserSecret = Schemas.UserSecret.Type
  33. let Volume = F.Volume
  34. in \(input : Input)
  35. -> let zk-conf =
  36. merge
  37. { None =
  38. { ServiceVolumes =
  39. [ Volume::{
  40. , name = "${input.name}-secret-zk"
  41. , dir = "/conf-tls"
  42. , files =
  43. [ { path = "zoo.cfg"
  44. , content = ./files/zoo.cfg.dhall "/conf" "/conf"
  45. }
  46. ]
  47. }
  48. ]
  49. , ClientVolumes =
  50. [ Volume::{
  51. , name = "${input.name}-zookeeper-tls"
  52. , dir = "/etc/zookeeper-tls"
  53. }
  54. ]
  55. , Zuul =
  56. ''
  57. hosts=zk:2281
  58. tls_cert=/etc/zookeeper-tls/tls.crt
  59. tls_key=/etc/zookeeper-tls/tls.key
  60. tls_ca=/etc/zookeeper-tls/ca.crt
  61. ''
  62. , Nodepool =
  63. ''
  64. zookeeper-servers:
  65. - host: zk
  66. port: 2281
  67. zookeeper-tls:
  68. cert: /etc/zookeeper-tls/tls.crt
  69. key: /etc/zookeeper-tls/tls.key
  70. ca: /etc/zookeeper-tls/ca.crt
  71. ''
  72. , Env = [] : List Kubernetes.EnvVar.Type
  73. }
  74. , Some =
  75. \(some : UserSecret)
  76. -> let empty = [] : List Volume.Type
  77. in { ServiceVolumes = empty
  78. , ClientVolumes = empty
  79. , Zuul = "hosts=%(ZUUL_ZK_HOSTS)"
  80. , Nodepool =
  81. ''
  82. zookeeper-servers:
  83. - hosts: %(ZUUL_ZK_HOSTS)"
  84. ''
  85. , Env =
  86. F.mkEnvVarSecret
  87. [ { name = "ZUUL_ZK_HOSTS"
  88. , secret = some.secretName
  89. , key = F.defaultText some.key "hosts"
  90. }
  91. ]
  92. }
  93. }
  94. input.zookeeper
  95. let db-internal-password-env =
  96. \(env-name : Text)
  97. -> F.mkEnvVarSecret
  98. [ { name = env-name
  99. , secret = "${input.name}-database-password"
  100. , key = "password"
  101. }
  102. ]
  103. let org =
  104. merge
  105. { None = "docker.io/zuul", Some = \(prefix : Text) -> prefix }
  106. input.imagePrefix
  107. let version = "latest"
  108. let image = \(name : Text) -> "${org}/${name}:${version}"
  109. let set-image =
  110. \(default-name : Text)
  111. -> \(input-name : Optional Text)
  112. -> { image =
  113. merge
  114. { None = Some default-name
  115. , Some = \(_ : Text) -> input-name
  116. }
  117. input-name
  118. }
  119. let etc-zuul =
  120. Volume::{
  121. , name = input.name ++ "-secret-zuul"
  122. , dir = "/etc/zuul"
  123. , files =
  124. [ { path = "zuul.conf"
  125. , content = ./files/zuul.conf.dhall input zk-conf.Zuul
  126. }
  127. ]
  128. }
  129. let etc-zuul-registry =
  130. Volume::{
  131. , name = input.name ++ "-secret-registry"
  132. , dir = "/etc/zuul"
  133. , files =
  134. [ { path = "registry.yaml"
  135. , content =
  136. let public-url =
  137. F.defaultText
  138. input.registry.public-url
  139. "https://registry:9000"
  140. in ./files/registry.yaml.dhall public-url
  141. }
  142. ]
  143. }
  144. let etc-nodepool =
  145. Volume::{
  146. , name = input.name ++ "-secret-nodepool"
  147. , dir = "/etc/nodepool"
  148. , files =
  149. [ { path = "nodepool.yaml"
  150. , content = ./files/nodepool.yaml.dhall zk-conf.Nodepool
  151. }
  152. ]
  153. }
  154. let Components =
  155. { CertManager =
  156. let issuer =
  157. { kind = "Issuer"
  158. , group = "cert-manager.io"
  159. , name = "${input.name}-ca"
  160. }
  161. let registry-enabled =
  162. Natural/isZero (F.defaultNat input.registry.count 0)
  163. == False
  164. let registry-cert =
  165. if registry-enabled
  166. then [ CertManager.Certificate::{
  167. , metadata =
  168. F.mkObjectMeta
  169. "${input.name}-registry-tls"
  170. ( F.mkComponentLabel
  171. input.name
  172. "cert-registry"
  173. )
  174. , spec = CertManager.CertificateSpec::{
  175. , secretName = "${input.name}-registry-tls"
  176. , issuerRef = issuer
  177. , dnsNames = Some [ "registry" ]
  178. , usages = Some
  179. [ "server auth", "client auth" ]
  180. }
  181. }
  182. ]
  183. else [] : List CertManager.Certificate.Type
  184. in { Issuers =
  185. [ CertManager.Issuer::{
  186. , metadata =
  187. F.mkObjectMeta
  188. "${input.name}-selfsigning"
  189. ( F.mkComponentLabel
  190. input.name
  191. "issuer-selfsigning"
  192. )
  193. , spec = CertManager.IssuerSpec::{
  194. , selfSigned = Some {=}
  195. }
  196. }
  197. , CertManager.Issuer::{
  198. , metadata =
  199. F.mkObjectMeta
  200. "${input.name}-ca"
  201. (F.mkComponentLabel input.name "issuer-ca")
  202. , spec = CertManager.IssuerSpec::{
  203. , ca = Some { secretName = "${input.name}-ca" }
  204. }
  205. }
  206. ]
  207. , Certificates =
  208. [ CertManager.Certificate::{
  209. , metadata =
  210. F.mkObjectMeta
  211. "${input.name}-ca"
  212. (F.mkComponentLabel input.name "cert-ca")
  213. , spec = CertManager.CertificateSpec::{
  214. , secretName = "${input.name}-ca"
  215. , isCA = Some True
  216. , commonName = Some "selfsigned-root-ca"
  217. , issuerRef =
  218. issuer
  219. // { name = "${input.name}-selfsigning" }
  220. , usages = Some
  221. [ "server auth", "client auth", "cert sign" ]
  222. }
  223. }
  224. , CertManager.Certificate::{
  225. , metadata =
  226. F.mkObjectMeta
  227. "${input.name}-gearman-tls"
  228. ( F.mkComponentLabel
  229. input.name
  230. "cert-gearman"
  231. )
  232. , spec = CertManager.CertificateSpec::{
  233. , secretName = "${input.name}-gearman-tls"
  234. , issuerRef = issuer
  235. , dnsNames = Some [ "gearman" ]
  236. , usages = Some [ "server auth", "client auth" ]
  237. }
  238. }
  239. ]
  240. # registry-cert
  241. }
  242. , Backend =
  243. { Database =
  244. merge
  245. { None =
  246. ./components/Database.dhall
  247. input.name
  248. db-internal-password-env
  249. , Some =
  250. \(some : UserSecret)
  251. -> F.KubernetesComponent.default
  252. }
  253. input.database
  254. , ZooKeeper =
  255. merge
  256. { None =
  257. ./components/ZooKeeper.dhall
  258. input.name
  259. (zk-conf.ClientVolumes # zk-conf.ServiceVolumes)
  260. , Some =
  261. \(some : UserSecret)
  262. -> F.KubernetesComponent.default
  263. }
  264. input.zookeeper
  265. }
  266. , Zuul =
  267. let zuul-image =
  268. \(name : Text) -> set-image (image "zuul-${name}")
  269. let zuul-env =
  270. F.mkEnvVarValue (toMap { HOME = "/var/lib/zuul" })
  271. let db-secret-env =
  272. merge
  273. { None = db-internal-password-env "ZUUL_DB_PASSWORD"
  274. , Some =
  275. \(some : UserSecret)
  276. -> F.mkEnvVarSecret
  277. [ { name = "ZUUL_DB_URI"
  278. , secret = some.secretName
  279. , key = F.defaultText some.key "db_uri"
  280. }
  281. ]
  282. }
  283. input.database
  284. let {- executor and merger do not need database info, but they fail to parse config without the env variable
  285. -} db-nosecret-env =
  286. F.mkEnvVarValue (toMap { ZUUL_DB_PASSWORD = "unused" })
  287. let zuul-data-dir =
  288. [ Volume::{ name = "zuul-data", dir = "/var/lib/zuul" }
  289. ]
  290. let sched-config =
  291. Volume::{
  292. , name = input.scheduler.config.secretName
  293. , dir = "/etc/zuul-scheduler"
  294. }
  295. let gearman-config =
  296. Volume::{
  297. , name = input.name ++ "-gearman-tls"
  298. , dir = "/etc/zuul-gearman"
  299. }
  300. let executor-ssh-key =
  301. Volume::{
  302. , name = input.executor.ssh_key.secretName
  303. , dir = "/etc/zuul-executor"
  304. }
  305. let zuul-volumes =
  306. [ etc-zuul, gearman-config ] # zk-conf.ClientVolumes
  307. in { Scheduler =
  308. ./components/Scheduler.dhall
  309. input.name
  310. ( input.scheduler
  311. // zuul-image "scheduler" input.scheduler.image
  312. )
  313. zuul-data-dir
  314. (zuul-volumes # [ sched-config ])
  315. (zuul-env # db-secret-env # zk-conf.Env)
  316. , Executor =
  317. ./components/Executor.dhall
  318. input.name
  319. ( input.executor
  320. // zuul-image "executor" input.executor.image
  321. )
  322. zuul-data-dir
  323. (zuul-volumes # [ executor-ssh-key ])
  324. (zuul-env # db-nosecret-env)
  325. input.jobVolumes
  326. , Web =
  327. ./components/Web.dhall
  328. input.name
  329. (input.web // zuul-image "web" input.web.image)
  330. zuul-data-dir
  331. zuul-volumes
  332. (zuul-env # db-secret-env # zk-conf.Env)
  333. , Merger =
  334. ./components/Merger.dhall
  335. input.name
  336. ( input.merger
  337. // zuul-image "merger" input.merger.image
  338. )
  339. zuul-data-dir
  340. zuul-volumes
  341. (zuul-env # db-nosecret-env)
  342. , Registry =
  343. ./components/Registry.dhall
  344. input.name
  345. ( input.registry
  346. // zuul-image "registry" input.registry.image
  347. )
  348. zuul-data-dir
  349. [ etc-zuul-registry ]
  350. , Preview =
  351. ./components/Preview.dhall
  352. input.name
  353. ( input.preview
  354. // zuul-image "preview" input.preview.image
  355. )
  356. zuul-data-dir
  357. }
  358. , Nodepool =
  359. let nodepool-image =
  360. \(name : Text) -> Some (image ("nodepool-" ++ name))
  361. let nodepool-data-dir =
  362. [ Volume::{
  363. , name = "nodepool-data"
  364. , dir = "/var/lib/nodepool"
  365. }
  366. ]
  367. let nodepool-config =
  368. Volume::{
  369. , name = input.launcher.config.secretName
  370. , dir = "/etc/nodepool-config"
  371. }
  372. let openstack-config =
  373. merge
  374. { None = [] : List Volume.Type
  375. , Some =
  376. \(some : UserSecret)
  377. -> [ Volume::{
  378. , name = some.secretName
  379. , dir = "/etc/nodepool-openstack"
  380. }
  381. ]
  382. }
  383. input.externalConfig.openstack
  384. let kubernetes-config =
  385. merge
  386. { None = [] : List Volume.Type
  387. , Some =
  388. \(some : UserSecret)
  389. -> [ Volume::{
  390. , name = some.secretName
  391. , dir = "/etc/nodepool-kubernetes"
  392. }
  393. ]
  394. }
  395. input.externalConfig.kubernetes
  396. let nodepool-env =
  397. F.mkEnvVarValue
  398. ( toMap
  399. { HOME = "/var/lib/nodepool"
  400. , OS_CLIENT_CONFIG_FILE =
  401. "/etc/nodepool-openstack/"
  402. ++ F.defaultKey
  403. input.externalConfig.openstack
  404. "clouds.yaml"
  405. , KUBECONFIG =
  406. "/etc/nodepool-kubernetes/"
  407. ++ F.defaultKey
  408. input.externalConfig.kubernetes
  409. "kube.config"
  410. }
  411. )
  412. let nodepool-volumes =
  413. [ etc-nodepool, nodepool-config ]
  414. # openstack-config
  415. # kubernetes-config
  416. # zk-conf.ClientVolumes
  417. let shard-config =
  418. "cat /etc/nodepool/nodepool.yaml /etc/nodepool-config/*.yaml > /var/lib/nodepool/config.yaml; "
  419. in { Launcher = F.KubernetesComponent::{
  420. , Deployment = Some
  421. ( F.mkDeployment
  422. input.name
  423. F.Component::{
  424. , name = "launcher"
  425. , count = 1
  426. , data-dir = nodepool-data-dir
  427. , volumes = nodepool-volumes
  428. , container = Kubernetes.Container::{
  429. , name = "launcher"
  430. , image = nodepool-image "launcher"
  431. , args = Some
  432. [ "sh"
  433. , "-c"
  434. , shard-config
  435. ++ "nodepool-launcher -d -c /var/lib/nodepool/config.yaml"
  436. ]
  437. , imagePullPolicy = Some "IfNotPresent"
  438. , env = Some nodepool-env
  439. , volumeMounts = Some
  440. ( F.mkVolumeMount
  441. (nodepool-volumes # nodepool-data-dir)
  442. )
  443. }
  444. }
  445. )
  446. }
  447. }
  448. }
  449. let mkSecret =
  450. \(volume : Volume.Type)
  451. -> Kubernetes.Resource.Secret
  452. Kubernetes.Secret::{
  453. , metadata = Kubernetes.ObjectMeta::{ name = volume.name }
  454. , stringData = Some
  455. ( Prelude.List.map
  456. { path : Text, content : Text }
  457. { mapKey : Text, mapValue : Text }
  458. ( \(config : { path : Text, content : Text })
  459. -> { mapKey = config.path
  460. , mapValue = config.content
  461. }
  462. )
  463. volume.files
  464. )
  465. }
  466. let {- This function transforms the different types into the Kubernetes.Resource
  467. union to enable using them inside a single List array
  468. -} mkUnion =
  469. \(component : F.KubernetesComponent.Type)
  470. -> let empty = [] : List Kubernetes.Resource
  471. in merge
  472. { None = empty
  473. , Some =
  474. \(some : Kubernetes.Service.Type)
  475. -> [ Kubernetes.Resource.Service some ]
  476. }
  477. component.Service
  478. # merge
  479. { None = empty
  480. , Some =
  481. \(some : Kubernetes.StatefulSet.Type)
  482. -> [ Kubernetes.Resource.StatefulSet some ]
  483. }
  484. component.StatefulSet
  485. # merge
  486. { None = empty
  487. , Some =
  488. \(some : Kubernetes.Deployment.Type)
  489. -> [ Kubernetes.Resource.Deployment some ]
  490. }
  491. component.Deployment
  492. let {- This function transform the Kubernetes.Resources type into the new Union
  493. that combines Kubernetes and CertManager resources
  494. -} transformKubernetesResource =
  495. Prelude.List.map
  496. Kubernetes.Resource
  497. CertManager.Union
  498. ( \(resource : Kubernetes.Resource)
  499. -> CertManager.Union.Kubernetes resource
  500. )
  501. let {- if cert-manager is enabled, then includes and transforms the CertManager types
  502. into the new Union that combines Kubernetes and CertManager resources
  503. -} all-certificates =
  504. if input.withCertManager
  505. then Prelude.List.map
  506. CertManager.Issuer.Type
  507. CertManager.Union
  508. CertManager.Union.Issuer
  509. Components.CertManager.Issuers
  510. # Prelude.List.map
  511. CertManager.Certificate.Type
  512. CertManager.Union
  513. CertManager.Union.Certificate
  514. Components.CertManager.Certificates
  515. else [] : List CertManager.Union
  516. in { Components = Components
  517. , List =
  518. { apiVersion = "v1"
  519. , kind = "List"
  520. , items =
  521. all-certificates
  522. # transformKubernetesResource
  523. ( Prelude.List.map
  524. Volume.Type
  525. Kubernetes.Resource
  526. mkSecret
  527. ( zk-conf.ServiceVolumes
  528. # [ etc-zuul, etc-nodepool, etc-zuul-registry ]
  529. )
  530. # mkUnion Components.Backend.Database
  531. # mkUnion Components.Backend.ZooKeeper
  532. # mkUnion Components.Zuul.Scheduler
  533. # mkUnion Components.Zuul.Executor
  534. # mkUnion Components.Zuul.Web
  535. # mkUnion Components.Zuul.Merger
  536. # mkUnion Components.Zuul.Registry
  537. # mkUnion Components.Zuul.Preview
  538. # mkUnion Components.Nodepool.Launcher
  539. )
  540. }
  541. }