A Kubernetes Operator for Zuul
You can not select more than 25 topics Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.
 
 
 
 
 

593 lines
22 KiB

  1. {- Zuul CR kubernetes resources
  2. The evaluation of that file is a function that takes the cr inputs as an argument,
  3. and returns the list of kubernetes of objects.
  4. Unless cert-manager usage is enabled, the resources expect those secrets to be available:
  5. * `${name}-gearman-tls` with:
  6. * `ca.crt`
  7. * `tls.crt`
  8. * `tls.key`
  9. * `${name}-registry-tls` with:
  10. * `tls.crt`
  11. * `tls.key`
  12. The resources expect those secrets to be available:
  13. * `${name}-zookeeper-tls` with:
  14. * `ca.crt`
  15. * `tls.crt`
  16. * `tls.key`
  17. * `zk.pem` the keystore
  18. * `${name}-registry-user-rw` with:
  19. * `secret` a password
  20. * `username` the user name with write access
  21. * `password` the user password
  22. Unless the input.database db uri is provided, the resources expect this secret to be available:
  23. * `${name}-database-password` the internal database password.
  24. -}
  25. let Prelude = ../Prelude.dhall
  26. let Kubernetes = ../Kubernetes.dhall
  27. let CertManager = ../CertManager.dhall
  28. let Schemas = ./input.dhall
  29. let F = ./functions.dhall
  30. let Input = Schemas.Input.Type
  31. let JobVolume = Schemas.JobVolume.Type
  32. let UserSecret = Schemas.UserSecret.Type
  33. let Volume = F.Volume
  34. in \(input : Input) ->
  35. let zk-conf =
  36. merge
  37. { None =
  38. { ServiceVolumes =
  39. [ Volume::{
  40. , name = "${input.name}-secret-zk"
  41. , dir = "/conf-tls"
  42. , files =
  43. [ { path = "zoo.cfg"
  44. , content = ./files/zoo.cfg.dhall "/conf" "/conf"
  45. }
  46. ]
  47. }
  48. ]
  49. , ClientVolumes =
  50. [ Volume::{
  51. , name = "${input.name}-zookeeper-tls"
  52. , dir = "/etc/zookeeper-tls"
  53. }
  54. ]
  55. , Zuul =
  56. ''
  57. hosts=zk:2281
  58. tls_cert=/etc/zookeeper-tls/tls.crt
  59. tls_key=/etc/zookeeper-tls/tls.key
  60. tls_ca=/etc/zookeeper-tls/ca.crt
  61. ''
  62. , Nodepool =
  63. ''
  64. zookeeper-servers:
  65. - host: zk
  66. port: 2281
  67. zookeeper-tls:
  68. cert: /etc/zookeeper-tls/tls.crt
  69. key: /etc/zookeeper-tls/tls.key
  70. ca: /etc/zookeeper-tls/ca.crt
  71. ''
  72. , Env = [] : List Kubernetes.EnvVar.Type
  73. }
  74. , Some =
  75. \(some : UserSecret) ->
  76. let empty = [] : List Volume.Type
  77. in { ServiceVolumes = empty
  78. , ClientVolumes = empty
  79. , Zuul = "hosts=%(ZUUL_ZK_HOSTS)"
  80. , Nodepool =
  81. ''
  82. zookeeper-servers:
  83. - hosts: %(ZUUL_ZK_HOSTS)"
  84. ''
  85. , Env =
  86. F.mkEnvVarSecret
  87. [ { name = "ZUUL_ZK_HOSTS"
  88. , secret = some.secretName
  89. , key = F.defaultText some.key "hosts"
  90. }
  91. ]
  92. }
  93. }
  94. input.zookeeper
  95. let db-internal-password-env =
  96. \(env-name : Text) ->
  97. F.mkEnvVarSecret
  98. [ { name = env-name
  99. , secret = "${input.name}-database-password"
  100. , key = "password"
  101. }
  102. ]
  103. let org =
  104. merge
  105. { None = "docker.io/zuul", Some = \(prefix : Text) -> prefix }
  106. input.imagePrefix
  107. let version = "latest"
  108. let image = \(name : Text) -> "${org}/${name}:${version}"
  109. let set-image =
  110. \(default-name : Text) ->
  111. \(input-name : Optional Text) ->
  112. { image =
  113. merge
  114. { None = Some default-name
  115. , Some = \(_ : Text) -> input-name
  116. }
  117. input-name
  118. }
  119. let etc-zuul =
  120. Volume::{
  121. , name = input.name ++ "-secret-zuul"
  122. , dir = "/etc/zuul"
  123. , files =
  124. [ { path = "zuul.conf"
  125. , content = ./files/zuul.conf.dhall input zk-conf.Zuul
  126. }
  127. ]
  128. }
  129. let etc-zuul-registry =
  130. Volume::{
  131. , name = input.name ++ "-secret-registry"
  132. , dir = "/etc/zuul"
  133. , files =
  134. [ { path = "registry.yaml"
  135. , content =
  136. let public-url =
  137. F.defaultText
  138. input.registry.public-url
  139. "https://registry:9000"
  140. in ./files/registry.yaml.dhall public-url
  141. }
  142. ]
  143. }
  144. let etc-nodepool =
  145. Volume::{
  146. , name = input.name ++ "-secret-nodepool"
  147. , dir = "/etc/nodepool"
  148. , files =
  149. [ { path = "nodepool.yaml"
  150. , content = ./files/nodepool.yaml.dhall zk-conf.Nodepool
  151. }
  152. ]
  153. }
  154. let Components =
  155. { CertManager =
  156. let issuer =
  157. { kind = "Issuer"
  158. , group = "cert-manager.io"
  159. , name = "${input.name}-ca"
  160. }
  161. let registry-enabled =
  162. Natural/isZero (F.defaultNat input.registry.count 0)
  163. == False
  164. let registry-cert =
  165. if registry-enabled
  166. then [ CertManager.Certificate::{
  167. , metadata =
  168. F.mkObjectMeta
  169. "${input.name}-registry-tls"
  170. ( F.mkComponentLabel
  171. input.name
  172. "cert-registry"
  173. )
  174. , spec = CertManager.CertificateSpec::{
  175. , secretName = "${input.name}-registry-tls"
  176. , issuerRef = issuer
  177. , dnsNames = Some [ "registry" ]
  178. , usages = Some [ "server auth", "client auth" ]
  179. }
  180. }
  181. ]
  182. else [] : List CertManager.Certificate.Type
  183. in { Issuers =
  184. [ CertManager.Issuer::{
  185. , metadata =
  186. F.mkObjectMeta
  187. "${input.name}-selfsigning"
  188. ( F.mkComponentLabel
  189. input.name
  190. "issuer-selfsigning"
  191. )
  192. , spec = CertManager.IssuerSpec::{
  193. , selfSigned = Some {=}
  194. }
  195. }
  196. , CertManager.Issuer::{
  197. , metadata =
  198. F.mkObjectMeta
  199. "${input.name}-ca"
  200. (F.mkComponentLabel input.name "issuer-ca")
  201. , spec = CertManager.IssuerSpec::{
  202. , ca = Some { secretName = "${input.name}-ca" }
  203. }
  204. }
  205. ]
  206. , Certificates =
  207. [ CertManager.Certificate::{
  208. , metadata =
  209. F.mkObjectMeta
  210. "${input.name}-ca"
  211. (F.mkComponentLabel input.name "cert-ca")
  212. , spec = CertManager.CertificateSpec::{
  213. , secretName = "${input.name}-ca"
  214. , isCA = Some True
  215. , commonName = Some "selfsigned-root-ca"
  216. , issuerRef =
  217. issuer
  218. // { name = "${input.name}-selfsigning" }
  219. , usages = Some
  220. [ "server auth", "client auth", "cert sign" ]
  221. }
  222. }
  223. , CertManager.Certificate::{
  224. , metadata =
  225. F.mkObjectMeta
  226. "${input.name}-gearman-tls"
  227. (F.mkComponentLabel input.name "cert-gearman")
  228. , spec = CertManager.CertificateSpec::{
  229. , secretName = "${input.name}-gearman-tls"
  230. , issuerRef = issuer
  231. , dnsNames = Some [ "gearman" ]
  232. , usages = Some [ "server auth", "client auth" ]
  233. }
  234. }
  235. ]
  236. # registry-cert
  237. }
  238. , Backend =
  239. { Database =
  240. merge
  241. { None =
  242. ./components/Database.dhall
  243. input.name
  244. db-internal-password-env
  245. , Some =
  246. \(some : UserSecret) -> F.KubernetesComponent.default
  247. }
  248. input.database
  249. , ZooKeeper =
  250. merge
  251. { None =
  252. ./components/ZooKeeper.dhall
  253. input.name
  254. (zk-conf.ClientVolumes # zk-conf.ServiceVolumes)
  255. , Some =
  256. \(some : UserSecret) -> F.KubernetesComponent.default
  257. }
  258. input.zookeeper
  259. }
  260. , Zuul =
  261. let zuul-image =
  262. \(name : Text) -> set-image (image "zuul-${name}")
  263. let zuul-env =
  264. F.mkEnvVarValue (toMap { HOME = "/var/lib/zuul" })
  265. let db-secret-env =
  266. merge
  267. { None = db-internal-password-env "ZUUL_DB_PASSWORD"
  268. , Some =
  269. \(some : UserSecret) ->
  270. F.mkEnvVarSecret
  271. [ { name = "ZUUL_DB_URI"
  272. , secret = some.secretName
  273. , key = F.defaultText some.key "db_uri"
  274. }
  275. ]
  276. }
  277. input.database
  278. let {- executor and merger do not need database info, but they fail to parse config without the env variable
  279. -} db-nosecret-env =
  280. F.mkEnvVarValue (toMap { ZUUL_DB_PASSWORD = "unused" })
  281. let zuul-data-dir =
  282. [ Volume::{ name = "zuul-data", dir = "/var/lib/zuul" } ]
  283. let sched-config =
  284. Volume::{
  285. , name = input.scheduler.config.secretName
  286. , dir = "/etc/zuul-scheduler"
  287. }
  288. let gearman-config =
  289. Volume::{
  290. , name = input.name ++ "-gearman-tls"
  291. , dir = "/etc/zuul-gearman"
  292. }
  293. let executor-ssh-key =
  294. Volume::{
  295. , name = input.executor.ssh_key.secretName
  296. , dir = "/etc/zuul-executor"
  297. }
  298. let zuul-volumes =
  299. [ etc-zuul, gearman-config ] # zk-conf.ClientVolumes
  300. in { Scheduler =
  301. ./components/Scheduler.dhall
  302. input.name
  303. ( input.scheduler
  304. // zuul-image "scheduler" input.scheduler.image
  305. )
  306. zuul-data-dir
  307. (zuul-volumes # [ sched-config ])
  308. (zuul-env # db-secret-env # zk-conf.Env)
  309. , Executor =
  310. ./components/Executor.dhall
  311. input.name
  312. ( input.executor
  313. // zuul-image "executor" input.executor.image
  314. )
  315. zuul-data-dir
  316. (zuul-volumes # [ executor-ssh-key ])
  317. (zuul-env # db-nosecret-env)
  318. input.jobVolumes
  319. , Web =
  320. ./components/Web.dhall
  321. input.name
  322. (input.web // zuul-image "web" input.web.image)
  323. zuul-data-dir
  324. zuul-volumes
  325. (zuul-env # db-secret-env # zk-conf.Env)
  326. , Merger =
  327. ./components/Merger.dhall
  328. input.name
  329. ( input.merger
  330. // zuul-image "merger" input.merger.image
  331. )
  332. zuul-data-dir
  333. zuul-volumes
  334. (zuul-env # db-nosecret-env)
  335. , Registry =
  336. ./components/Registry.dhall
  337. input.name
  338. ( input.registry
  339. // zuul-image "registry" input.registry.image
  340. )
  341. zuul-data-dir
  342. [ etc-zuul-registry ]
  343. , Preview =
  344. ./components/Preview.dhall
  345. input.name
  346. ( input.preview
  347. // zuul-image "preview" input.preview.image
  348. )
  349. zuul-data-dir
  350. }
  351. , Nodepool =
  352. let nodepool-image =
  353. \(name : Text) -> Some (image ("nodepool-" ++ name))
  354. let nodepool-data-dir =
  355. [ Volume::{
  356. , name = "nodepool-data"
  357. , dir = "/var/lib/nodepool"
  358. }
  359. ]
  360. let nodepool-config =
  361. Volume::{
  362. , name = input.launcher.config.secretName
  363. , dir = "/etc/nodepool-config"
  364. }
  365. let openstack-config =
  366. merge
  367. { None = [] : List Volume.Type
  368. , Some =
  369. \(some : UserSecret) ->
  370. [ Volume::{
  371. , name = some.secretName
  372. , dir = "/etc/nodepool-openstack"
  373. }
  374. ]
  375. }
  376. input.externalConfig.openstack
  377. let kubernetes-config =
  378. merge
  379. { None = [] : List Volume.Type
  380. , Some =
  381. \(some : UserSecret) ->
  382. [ Volume::{
  383. , name = some.secretName
  384. , dir = "/etc/nodepool-kubernetes"
  385. }
  386. ]
  387. }
  388. input.externalConfig.kubernetes
  389. let nodepool-env =
  390. F.mkEnvVarValue
  391. ( toMap
  392. { HOME = "/var/lib/nodepool"
  393. , OS_CLIENT_CONFIG_FILE =
  394. "/etc/nodepool-openstack/"
  395. ++ F.defaultKey
  396. input.externalConfig.openstack
  397. "clouds.yaml"
  398. , KUBECONFIG =
  399. "/etc/nodepool-kubernetes/"
  400. ++ F.defaultKey
  401. input.externalConfig.kubernetes
  402. "kube.config"
  403. }
  404. )
  405. let nodepool-volumes =
  406. [ etc-nodepool, nodepool-config ]
  407. # openstack-config
  408. # kubernetes-config
  409. # zk-conf.ClientVolumes
  410. let shard-config =
  411. "cat /etc/nodepool/nodepool.yaml /etc/nodepool-config/*.yaml > /var/lib/nodepool/config.yaml; "
  412. in { Launcher = F.KubernetesComponent::{
  413. , Deployment = Some
  414. ( F.mkDeployment
  415. input.name
  416. F.Component::{
  417. , name = "launcher"
  418. , count = 1
  419. , data-dir = nodepool-data-dir
  420. , volumes = nodepool-volumes
  421. , container = Kubernetes.Container::{
  422. , name = "launcher"
  423. , image = nodepool-image "launcher"
  424. , args = Some
  425. [ "sh"
  426. , "-c"
  427. , shard-config
  428. ++ "nodepool-launcher -d -c /var/lib/nodepool/config.yaml"
  429. ]
  430. , imagePullPolicy = Some "IfNotPresent"
  431. , env = Some nodepool-env
  432. , volumeMounts = Some
  433. ( F.mkVolumeMount
  434. (nodepool-volumes # nodepool-data-dir)
  435. )
  436. }
  437. }
  438. )
  439. }
  440. }
  441. }
  442. let mkSecret =
  443. \(volume : Volume.Type) ->
  444. Kubernetes.Resource.Secret
  445. Kubernetes.Secret::{
  446. , metadata = Kubernetes.ObjectMeta::{ name = volume.name }
  447. , stringData = Some
  448. ( Prelude.List.map
  449. { path : Text, content : Text }
  450. { mapKey : Text, mapValue : Text }
  451. ( \(config : { path : Text, content : Text }) ->
  452. { mapKey = config.path, mapValue = config.content }
  453. )
  454. volume.files
  455. )
  456. }
  457. let {- This function transforms the different types into the Kubernetes.Resource
  458. union to enable using them inside a single List array
  459. -} mkUnion =
  460. \(component : F.KubernetesComponent.Type) ->
  461. let empty = [] : List Kubernetes.Resource
  462. in merge
  463. { None = empty
  464. , Some =
  465. \(some : Kubernetes.Service.Type) ->
  466. [ Kubernetes.Resource.Service some ]
  467. }
  468. component.Service
  469. # merge
  470. { None = empty
  471. , Some =
  472. \(some : Kubernetes.StatefulSet.Type) ->
  473. [ Kubernetes.Resource.StatefulSet some ]
  474. }
  475. component.StatefulSet
  476. # merge
  477. { None = empty
  478. , Some =
  479. \(some : Kubernetes.Deployment.Type) ->
  480. [ Kubernetes.Resource.Deployment some ]
  481. }
  482. component.Deployment
  483. let {- This function transform the Kubernetes.Resources type into the new Union
  484. that combines Kubernetes and CertManager resources
  485. -} transformKubernetesResource =
  486. Prelude.List.map
  487. Kubernetes.Resource
  488. CertManager.Union
  489. ( \(resource : Kubernetes.Resource) ->
  490. CertManager.Union.Kubernetes resource
  491. )
  492. let {- if cert-manager is enabled, then includes and transforms the CertManager types
  493. into the new Union that combines Kubernetes and CertManager resources
  494. -} all-certificates =
  495. if input.withCertManager
  496. then Prelude.List.map
  497. CertManager.Issuer.Type
  498. CertManager.Union
  499. CertManager.Union.Issuer
  500. Components.CertManager.Issuers
  501. # Prelude.List.map
  502. CertManager.Certificate.Type
  503. CertManager.Union
  504. CertManager.Union.Certificate
  505. Components.CertManager.Certificates
  506. else [] : List CertManager.Union
  507. in { Components
  508. , List =
  509. { apiVersion = "v1"
  510. , kind = "List"
  511. , items =
  512. all-certificates
  513. # transformKubernetesResource
  514. ( Prelude.List.map
  515. Volume.Type
  516. Kubernetes.Resource
  517. mkSecret
  518. ( zk-conf.ServiceVolumes
  519. # [ etc-zuul, etc-nodepool, etc-zuul-registry ]
  520. )
  521. # mkUnion Components.Backend.Database
  522. # mkUnion Components.Backend.ZooKeeper
  523. # mkUnion Components.Zuul.Scheduler
  524. # mkUnion Components.Zuul.Executor
  525. # mkUnion Components.Zuul.Web
  526. # mkUnion Components.Zuul.Merger
  527. # mkUnion Components.Zuul.Registry
  528. # mkUnion Components.Zuul.Preview
  529. # mkUnion Components.Nodepool.Launcher
  530. )
  531. }
  532. }