Use kopf operator framework

This switches from the ansible/dhall operator framework to kopf,
an operator framework written in pure Python.  This allows us to:

* Build the operator application as a Python app.
* Build the operator image using the opendev python builder images.
* Run the operator as a Python CLI program "zuul-operator".
* Write procedural Python code to handle operator tasks (such as
  creating new nodepool launchers when providers are added).
* Use Jinja for templating config files and k8s resource files
  (direct pythonic manipulation of resources is an option too).

The new CR nearly matches the existing one, with some minor differences.

Some missing features and documentation are added in the commits
immediately following; they should be reviewed and merged as a unit.

Also, fx waiting for scheduler to settle in functional test since
we changed this log line in Zuul.

Change-Id: Ib37b67e3444b7cd44692d48eee77775ee9049e9f

Change-Id: I70ec31ecd8fe264118215944022b2e7b513dced9
changes/39/785039/26
James E. Blair 1 year ago
parent 0366b867bf
commit eff9f360f7
  1. 1
      .gitignore
  2. 1
      .zuul.yaml
  3. 85
      build/Dockerfile
  4. 59
      conf/CertManager.dhall
  5. 3
      conf/Kubernetes.dhall
  6. 28
      conf/Prelude.dhall
  7. 44
      conf/zuul/components/Database.dhall
  8. 67
      conf/zuul/components/Executor.dhall
  9. 30
      conf/zuul/components/Merger.dhall
  10. 39
      conf/zuul/components/Preview.dhall
  11. 67
      conf/zuul/components/Registry.dhall
  12. 38
      conf/zuul/components/Scheduler.dhall
  13. 37
      conf/zuul/components/Web.dhall
  14. 50
      conf/zuul/components/ZooKeeper.dhall
  15. 11
      conf/zuul/files/nodepool.yaml.dhall
  16. 20
      conf/zuul/files/registry.yaml.dhall
  17. 23
      conf/zuul/files/zoo.cfg.dhall
  18. 192
      conf/zuul/files/zuul.conf.dhall
  19. 294
      conf/zuul/functions.dhall
  20. 175
      conf/zuul/input.dhall
  21. 592
      conf/zuul/resources.dhall
  22. 9
      deploy/crds/zuul-ci_v1alpha1_zuul_cr.yaml
  23. 24
      deploy/operator.yaml
  24. 50
      deploy/rbac.yaml
  25. 3
      playbooks/files/ansible.cfg
  26. 46
      playbooks/files/cr_spec.yaml
  27. 2
      playbooks/files/hosts.yaml
  28. 10
      playbooks/files/local-vars.yaml
  29. 60
      playbooks/files/local.yaml
  30. 20
      playbooks/files/update-operator.sh
  31. 8
      playbooks/zuul-operator-functional/pre-k8s.yaml
  32. 34
      playbooks/zuul-operator-functional/run.yaml
  33. 6
      playbooks/zuul-operator-functional/tasks/apply_cr.yaml
  34. 4
      playbooks/zuul-operator-functional/tasks/create_config.yaml
  35. 6
      playbooks/zuul-operator-functional/tasks/create_test_secrets.yaml
  36. 2
      playbooks/zuul-operator-functional/tasks/test_cert_manager.yaml
  37. 10
      playbooks/zuul-operator-functional/tasks/wait_services.yaml
  38. 17
      playbooks/zuul-operator-functional/test.yaml
  39. 6
      requirements.txt
  40. 16
      roles/zuul-ensure-database-password/tasks/main.yaml
  41. 41
      roles/zuul-ensure-gearman-tls/tasks/main.yaml
  42. 52
      roles/zuul-ensure-registry-tls/tasks/main.yaml
  43. 352
      roles/zuul-ensure-zookeeper-tls/files/openssl.cnf
  44. 103
      roles/zuul-ensure-zookeeper-tls/files/zk-ca.sh
  45. 30
      roles/zuul-ensure-zookeeper-tls/tasks/main.yaml
  46. 4
      roles/zuul-lookup-conf/tasks/main.yaml
  47. 9
      roles/zuul-reconfigure-tenant-when-conf-changed/tasks/main.yaml
  48. 41
      roles/zuul-restart-when-zuul-conf-changed/library/dump_zuul_changes.py
  49. 60
      roles/zuul-restart-when-zuul-conf-changed/library/load_zuul_changes.py
  50. 38
      roles/zuul-restart-when-zuul-conf-changed/module_utils/gearlib.py
  51. 49
      roles/zuul-restart-when-zuul-conf-changed/tasks/main.yaml
  52. 19
      roles/zuul/defaults/main.yaml
  53. 83
      roles/zuul/library/autoscale_gearman.py
  54. 61
      roles/zuul/library/dhall_to_json.py
  55. 65
      roles/zuul/library/json_to_dhall.py
  56. 60
      roles/zuul/tasks/main.yaml
  57. 59
      setup.cfg
  58. 22
      setup.py
  59. 5
      watches.yaml
  60. 15
      zuul_operator/__init__.py
  61. 61
      zuul_operator/certmanager.py
  62. 51
      zuul_operator/cmd.py
  63. 82
      zuul_operator/objects.py
  64. 155
      zuul_operator/operator.py
  65. 106
      zuul_operator/pxc.py
  66. 0
      zuul_operator/templates/__init__.py
  67. 36
      zuul_operator/templates/cert-authority.yaml
  68. 26537
      zuul_operator/templates/cert-manager.yaml
  69. 67
      zuul_operator/templates/nodepool-launcher.yaml
  70. 445
      zuul_operator/templates/pxc-cluster.yaml
  71. 193
      zuul_operator/templates/pxc-crd.yaml
  72. 22
      zuul_operator/templates/pxc-create-db.yaml
  73. 168
      zuul_operator/templates/pxc-operator.yaml
  74. 364
      zuul_operator/templates/zookeeper.yaml
  75. 35
      zuul_operator/templates/zuul.conf
  76. 318
      zuul_operator/templates/zuul.yaml
  77. 101
      zuul_operator/utils.py
  78. 34
      zuul_operator/version.py
  79. 48
      zuul_operator/zookeeper.py
  80. 340
      zuul_operator/zuul.py

1
.gitignore vendored

@ -7,3 +7,4 @@
id_rsa
id_rsa.pub
*.patch
*.egg-info/

@ -20,7 +20,6 @@
nodeset: ubuntu-bionic
vars:
namespace: 'default'
withCertManager: true
- job:
description: Image and buildset registry job

@ -1,59 +1,26 @@
FROM quay.io/operator-framework/ansible-operator:v1.4.2
# dhall versions and digests
ARG DHALL_VERSION=1.33.1
ARG DHALL_JSON_VERSION=1.7.0
ARG DHALL_JSON_DIGEST=cc9fc70e492d35a3986183b589a435653e782f67cda51d33a935dff1ddd15aec
ARG DHALL_LANG_REF=v17.0.0
ARG DHALL_KUBE_REF=v4.0.0
# kubectl versions and digests
ARG KUBECTL_VERSION=v1.17.0
ARG KUBECTL_DIGEST=a5eb7e2e44d858d96410937a4e4c82f9087c9d120cb2b9e92462878eda59d578
# Install extra requirements
USER root
# Install gear to connect to the scheduler gearman
RUN pip3 install --upgrade gear
# Install collections
RUN ansible-galaxy collection install community.kubernetes && chmod -R ug+rwx ${HOME}/.ansible
# unarchive: bzip2 and tar
# generate zuul ssh-keys or certificate: openssh and openssl
# manage configuration: git
RUN dnf install -y bzip2 tar openssh openssl git
# Install kubectl to mitigate https://github.com/operator-framework/operator-sdk/issues/2204
RUN curl -OL https://dl.k8s.io/$KUBECTL_VERSION/kubernetes-client-linux-amd64.tar.gz \
&& echo "$KUBECTL_DIGEST kubernetes-client-linux-amd64.tar.gz" | sha256sum -c \
&& tar -xf kubernetes-client-linux-amd64.tar.gz --strip-components=3 -z --mode='a+x' -C /usr/bin \
&& rm kubernetes-client-linux-amd64.tar.gz
# Install dhall-to-json
RUN curl -OL https://github.com/dhall-lang/dhall-haskell/releases/download/$DHALL_VERSION/dhall-json-$DHALL_JSON_VERSION-x86_64-linux.tar.bz2 \
&& echo "$DHALL_JSON_DIGEST dhall-json-$DHALL_JSON_VERSION-x86_64-linux.tar.bz2" | sha256sum -c \
&& tar -xf dhall-json-$DHALL_JSON_VERSION-x86_64-linux.tar.bz2 --strip-components=2 -j --mode='a+x' -C /usr/bin \
&& rm dhall-json-$DHALL_JSON_VERSION-x86_64-linux.tar.bz2
# Back to the default operator user
USER 1001
# Install dhall libraries
RUN git clone --branch $DHALL_LANG_REF --depth 1 https://github.com/dhall-lang/dhall-lang /opt/ansible/dhall-lang \
&& git clone --branch $DHALL_KUBE_REF --depth 1 https://github.com/dhall-lang/dhall-kubernetes /opt/ansible/dhall-kubernetes
ENV DHALL_PRELUDE=/opt/ansible/dhall-lang/Prelude/package.dhall
ENV DHALL_KUBERNETES=/opt/ansible/dhall-kubernetes/package.dhall
# Copy configuration
COPY conf/ /opt/ansible/conf/
# Cache dhall objects
RUN echo 'let Prelude = ~/conf/Prelude.dhall let Kubernetes = ~/conf/Kubernetes.dhall in "OK"' | \
env DHALL_PRELUDE=/opt/ansible/dhall-lang/Prelude/package.dhall \
DHALL_KUBERNETES=/opt/ansible/dhall-kubernetes/package.dhall dhall-to-json
# Copy ansible operator requirements
COPY watches.yaml ${HOME}/watches.yaml
COPY roles ${HOME}/roles
# Copyright (c) 2020 Red Hat, Inc.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
# implied.
# See the License for the specific language governing permissions and
# limitations under the License.
FROM docker.io/opendevorg/python-builder:3.8 as builder
COPY . /tmp/src
RUN assemble
FROM docker.io/opendevorg/python-base:3.8
COPY --from=builder /output/ /output
RUN /output/install-from-bindep
ENTRYPOINT ["/usr/local/bin/zuul-operator"]

@ -1,59 +0,0 @@
{- A local cert manager package that extends the Kubernetes binding
TODO: Use union combinaison once it is available, see https://github.com/dhall-lang/dhall-lang/issues/175
TODO: Check with the dhall kubernetes community if the new type could be contributed,
though it currently only covers what is needed for zuul.
-}
let Kubernetes = ./Kubernetes.dhall
let IssuerSpec =
{ Type = { selfSigned : Optional {}, ca : Optional { secretName : Text } }
, default = { selfSigned = None {}, ca = None { secretName : Text } }
}
let Issuer =
{ Type =
{ apiVersion : Text
, kind : Text
, metadata : Kubernetes.ObjectMeta.Type
, spec : IssuerSpec.Type
}
, default = { apiVersion = "cert-manager.io/v1alpha2", kind = "Issuer" }
}
let CertificateSpec =
{ Type =
{ secretName : Text
, isCA : Optional Bool
, usages : Optional (List Text)
, commonName : Optional Text
, dnsNames : Optional (List Text)
, issuerRef : { name : Text, kind : Text, group : Text }
}
, default =
{ isCA = None Bool
, usages = None (List Text)
, commonName = None Text
, dnsNames = None (List Text)
}
}
let Certificate =
{ Type =
{ apiVersion : Text
, kind : Text
, metadata : Kubernetes.ObjectMeta.Type
, spec : CertificateSpec.Type
}
, default =
{ apiVersion = "cert-manager.io/v1alpha3", kind = "Certificate" }
}
let Union =
< Kubernetes : Kubernetes.Resource
| Issuer : Issuer.Type
| Certificate : Certificate.Type
>
in { IssuerSpec, Issuer, CertificateSpec, Certificate, Union }

@ -1,3 +0,0 @@
{- Import the kubernetes types, see the ./Prelude.dhall file for documentation -}
env:DHALL_KUBERNETES
? https://raw.githubusercontent.com/dhall-lang/dhall-kubernetes/v4.0.0/package.dhall sha256:d9eac5668d5ed9cb3364c0a39721d4694e4247dad16d8a82827e4619ee1d6188

@ -1,28 +0,0 @@
{- This file provides a central `Prelude` import for the rest of the library to
use so that the integrity check only needs to be updated in one place
whenever upgrading the interpreter.
This allows the user to provide their own Prelude import using the
`DHALL_PRELUDE` environment variable, like this:
```
$ export DHALL_PRELUDE='https://prelude.dhall-lang.org/package.dhall sha256:...'
```
Note that overriding the Prelude in this way only works if this repository
is imported locally. Remote imports do not have access to environment
variables and any attempt to import one will fall back to the next available
import. To learn more, read:
* https://docs.dhall-lang.org/discussions/Safety-guarantees.html#cross-site-scripting-xss
This file also provides an import without the integrity check as a slower
fallback if the user is using a different version of the Dhall interpreter.
This pattern is documented in the dhall-nethack repo:
* https://github.com/dhall-lang/dhall-nethack/blob/master/Prelude.dhall
-}
env:DHALL_PRELUDE
? https://prelude.dhall-lang.org/v17.0.0/package.dhall sha256:10db3c919c25e9046833df897a8ffe2701dc390fa0893d958c3430524be5a43e
? https://prelude.dhall-lang.org/v17.0.0/package.dhall

@ -1,44 +0,0 @@
let Kubernetes = ../../Kubernetes.dhall
let F = ../functions.dhall
let db-volumes = [ F.Volume::{ name = "pg-data", dir = "/var/lib/pg/" } ]
in \(app-name : Text) ->
\ ( db-internal-password-env
: forall (env-name : Text) -> List Kubernetes.EnvVar.Type
) ->
F.KubernetesComponent::{
, Service = Some (F.mkService app-name "db" "pg" 5432)
, StatefulSet = Some
( F.mkStatefulSet
app-name
F.Component::{
, name = "db"
, count = 1
, data-dir = db-volumes
, claim-size = 1
, container = Kubernetes.Container::{
, name = "db"
, image = Some "docker.io/library/postgres:12.1"
, imagePullPolicy = Some "IfNotPresent"
, ports = Some
[ Kubernetes.ContainerPort::{
, name = Some "pg"
, containerPort = 5432
}
]
, env = Some
( F.mkEnvVarValue
( toMap
{ POSTGRES_USER = "zuul"
, PGDATA = "/var/lib/pg/data"
}
)
# db-internal-password-env "POSTGRES_PASSWORD"
)
, volumeMounts = Some (F.mkVolumeMount db-volumes)
}
}
)
}

@ -1,67 +0,0 @@
let Kubernetes = ../../Kubernetes.dhall
let F = ../functions.dhall
let InputExecutor = (../input.dhall).Executor.Type
let JobVolume = (../input.dhall).JobVolume.Type
in \(app-name : Text) ->
\(input-executor : InputExecutor) ->
\(data-dir : List F.Volume.Type) ->
\(volumes : List F.Volume.Type) ->
\(env : List Kubernetes.EnvVar.Type) ->
\(jobVolumes : Optional (List JobVolume)) ->
F.KubernetesComponent::{
, Service = Some (F.mkService app-name "executor" "finger" 7900)
, StatefulSet = Some
( F.mkStatefulSet
app-name
F.Component::{
, name = "executor"
, count = 1
, data-dir
, volumes
, extra-volumes =
let job-volumes =
F.mkJobVolume
Kubernetes.Volume.Type
(\(job-volume : JobVolume) -> job-volume.volume)
jobVolumes
in job-volumes
, claim-size = 0
, container = Kubernetes.Container::{
, name = "executor"
, image = input-executor.image
, imagePullPolicy = Some "IfNotPresent"
, ports = Some
[ Kubernetes.ContainerPort::{
, name = Some "finger"
, containerPort = 7900
}
]
, env = Some env
, volumeMounts =
let job-volumes-mount =
F.mkJobVolume
F.Volume.Type
( \(job-volume : JobVolume) ->
F.Volume::{
, name = job-volume.volume.name
, dir = job-volume.dir
}
)
jobVolumes
in Some
( F.mkVolumeMount
(data-dir # volumes # job-volumes-mount)
)
, securityContext = Some Kubernetes.SecurityContext::{
, privileged = Some True
}
}
}
)
}

@ -1,30 +0,0 @@
let Kubernetes = ../../Kubernetes.dhall
let F = ../functions.dhall
let InputMerger = (../input.dhall).Merger.Type
in \(app-name : Text) ->
\(input-merger : InputMerger) ->
\(data-dir : List F.Volume.Type) ->
\(volumes : List F.Volume.Type) ->
\(env : List Kubernetes.EnvVar.Type) ->
F.KubernetesComponent::{
, Deployment = Some
( F.mkDeployment
app-name
F.Component::{
, name = "merger"
, count = 1
, data-dir
, volumes
, container = Kubernetes.Container::{
, name = "merger"
, image = input-merger.image
, imagePullPolicy = Some "IfNotPresent"
, env = Some env
, volumeMounts = Some (F.mkVolumeMount (data-dir # volumes))
}
}
)
}

@ -1,39 +0,0 @@
let Kubernetes = ../../Kubernetes.dhall
let F = ../functions.dhall
let InputPreview = (../input.dhall).Preview.Type
in \(app-name : Text) ->
\(input-preview : InputPreview) ->
\(data-dir : List F.Volume.Type) ->
F.KubernetesComponent::{
, Service = Some (F.mkService app-name "preview" "preview" 80)
, Deployment = Some
( F.mkDeployment
app-name
F.Component::{
, name = "preview"
, count = F.defaultNat input-preview.count 0
, data-dir
, container = Kubernetes.Container::{
, name = "preview"
, image = input-preview.image
, imagePullPolicy = Some "IfNotPresent"
, ports = Some
[ Kubernetes.ContainerPort::{
, name = Some "preview"
, containerPort = 80
}
]
, env = Some
[ Kubernetes.EnvVar::{
, name = "ZUUL_API_URL"
, value = Some "http://web:9000"
}
]
, volumeMounts = Some (F.mkVolumeMount data-dir)
}
}
)
}

@ -1,67 +0,0 @@
let Prelude = ../../Prelude.dhall
let Kubernetes = ../../Kubernetes.dhall
let F = ../functions.dhall
let InputRegistry = (../input.dhall).Registry.Type
let registry-volumes =
\(app-name : Text) ->
[ F.Volume::{
, name = app-name ++ "-registry-tls"
, dir = "/etc/zuul-registry"
}
]
let registry-env =
\(app-name : Text) ->
F.mkEnvVarSecret
( Prelude.List.map
Text
F.EnvSecret
( \(key : Text) ->
{ name = "ZUUL_REGISTRY_${key}"
, key
, secret = "${app-name}-registry-user-rw"
}
)
[ "secret", "username", "password" ]
)
in \(app-name : Text) ->
\(input-registry : InputRegistry) ->
\(data-dir : List F.Volume.Type) ->
\(volumes : List F.Volume.Type) ->
F.KubernetesComponent::{
, Service = Some (F.mkService app-name "registry" "registry" 9000)
, StatefulSet = Some
( F.mkStatefulSet
app-name
F.Component::{
, name = "registry"
, count = F.defaultNat input-registry.count 0
, data-dir
, volumes = volumes # registry-volumes app-name
, claim-size = F.defaultNat input-registry.storage-size 20
, container = Kubernetes.Container::{
, name = "registry"
, image = input-registry.image
, args = Some
[ "zuul-registry", "-c", "/etc/zuul/registry.yaml", "serve" ]
, imagePullPolicy = Some "IfNotPresent"
, ports = Some
[ Kubernetes.ContainerPort::{
, name = Some "registry"
, containerPort = 9000
}
]
, env = Some (registry-env app-name)
, volumeMounts = Some
( F.mkVolumeMount
(data-dir # volumes # registry-volumes app-name)
)
}
}
)
}

@ -1,38 +0,0 @@
let Kubernetes = ../../Kubernetes.dhall
let F = ../functions.dhall
let InputScheduler = (../input.dhall).Scheduler.Type
in \(app-name : Text) ->
\(input-scheduler : InputScheduler) ->
\(data-dir : List F.Volume.Type) ->
\(volumes : List F.Volume.Type) ->
\(env : List Kubernetes.EnvVar.Type) ->
F.KubernetesComponent::{
, Service = Some (F.mkService app-name "scheduler" "gearman" 4730)
, StatefulSet = Some
( F.mkStatefulSet
app-name
F.Component::{
, name = "scheduler"
, count = 1
, data-dir
, volumes
, claim-size = 5
, container = Kubernetes.Container::{
, name = "scheduler"
, image = input-scheduler.image
, imagePullPolicy = Some "IfNotPresent"
, ports = Some
[ Kubernetes.ContainerPort::{
, name = Some "gearman"
, containerPort = 4730
}
]
, env = Some env
, volumeMounts = Some (F.mkVolumeMount (data-dir # volumes))
}
}
)
}

@ -1,37 +0,0 @@
let Kubernetes = ../../Kubernetes.dhall
let F = ../functions.dhall
let InputWeb = (../input.dhall).Web.Type
in \(app-name : Text) ->
\(input-web : InputWeb) ->
\(data-dir : List F.Volume.Type) ->
\(volumes : List F.Volume.Type) ->
\(env : List Kubernetes.EnvVar.Type) ->
F.KubernetesComponent::{
, Service = Some (F.mkService app-name "web" "api" 9000)
, Deployment = Some
( F.mkDeployment
app-name
F.Component::{
, name = "web"
, count = 1
, data-dir
, volumes
, container = Kubernetes.Container::{
, name = "web"
, image = input-web.image
, imagePullPolicy = Some "IfNotPresent"
, ports = Some
[ Kubernetes.ContainerPort::{
, name = Some "api"
, containerPort = 9000
}
]
, env = Some env
, volumeMounts = Some (F.mkVolumeMount (data-dir # volumes))
}
}
)
}

@ -1,50 +0,0 @@
{- This function returns the ZooKeeper component in case the user doesn't provide it's own service.
The volumes list should contains the zoo
-}
let Kubernetes = ../../Kubernetes.dhall
let F = ../functions.dhall
let data-volumes =
[ F.Volume::{ name = "zk-log", dir = "/var/log/zookeeper/" }
, F.Volume::{ name = "zk-dat", dir = "/var/lib/zookeeper/" }
]
in \(app-name : Text) ->
\(client-conf : List F.Volume.Type) ->
F.KubernetesComponent::{
, Service = Some (F.mkService app-name "zk" "zk" 2281)
, StatefulSet = Some
( F.mkStatefulSet
app-name
F.Component::{
, name = "zk"
, count = 1
, data-dir = data-volumes
, volumes = client-conf
, claim-size = 1
, container = Kubernetes.Container::{
, name = "zk"
, command = Some
[ "sh"
, "-c"
, "cp /conf-tls/zoo.cfg /conf/ && "
++ "cp /etc/zookeeper-tls/zk.pem /conf/zk.pem && "
++ "cp /etc/zookeeper-tls/ca.crt /conf/ca.pem && "
++ "chown zookeeper /conf/zoo.cfg /conf/zk.pem /conf/ca.pem && "
++ "exec /docker-entrypoint.sh zkServer.sh start-foreground"
]
, image = Some "docker.io/library/zookeeper"
, imagePullPolicy = Some "IfNotPresent"
, ports = Some
[ Kubernetes.ContainerPort::{
, name = Some "zk"
, containerPort = 2281
}
]
, volumeMounts = Some
(F.mkVolumeMount (data-volumes # client-conf))
}
}
)
}

@ -1,11 +0,0 @@
{- This function converts a zk-host Text to a nodepool.yaml file content
TODO: replace opaque Text by structured zk host list and tls configuration
-}
\(zk-host : Text) ->
''
${zk-host}
webapp:
port: 5000
''

@ -1,20 +0,0 @@
{- This function converts a public-url Text to a registry.yaml file content
-}
\(public-url : Text) ->
''
registry:
address: '0.0.0.0'
port: 9000
public-url: ${public-url}
tls-cert: /etc/zuul-registry/tls.crt
tls-key: /etc/zuul-registry/tls.key
secret: "%(ZUUL_REGISTRY_secret)"
storage:
driver: filesystem
root: /var/lib/zuul
users:
- name: "%(ZUUL_REGISTRY_username)"
pass: "%(ZUUL_REGISTRY_password)"
access: write
''

@ -1,23 +0,0 @@
{- This function converts a client-dir and server-dir Text to a zoo.cfg file content
-}
\(client-dir : Text) ->
\(server-dir : Text) ->
''
dataDir=/data
dataLogDir=/datalog
tickTime=2000
initLimit=5
syncLimit=2
autopurge.snapRetainCount=3
autopurge.purgeInterval=0
maxClientCnxns=60
standaloneEnabled=true
admin.enableServer=true
server.1=0.0.0.0:2888:3888
# TLS configuration
secureClientPort=2281
serverCnxnFactory=org.apache.zookeeper.server.NettyServerCnxnFactory
ssl.keyStore.location=${server-dir}/zk.pem
ssl.trustStore.location=${client-dir}/ca.pem
''

@ -1,192 +0,0 @@
{- This method renders the zuul.conf.
TODO: replace input schemas by the required attributes.
-}
\(input : (../input.dhall).Input.Type) ->
\(zk-hosts : Text) ->
let Prelude = ../../Prelude.dhall
let Schemas = ../input.dhall
let F = ../functions.dhall
let {- This is a high level method. It takes:
* a Connection type such as `Schemas.Gerrit.Type`,
* an Optional List of that type
* a function that goes from that type to a zuul.conf text blob
Then it returns a text blob for all the connections
-} mkConns =
\(type : Type) ->
\(list : Optional (List type)) ->
\(f : type -> Text) ->
F.newlineSep
( merge
{ None = [] : List Text, Some = Prelude.List.map type Text f }
list
)
let merger-email =
F.defaultText input.merger.git_user_email "${input.name}@localhost"
let merger-user = F.defaultText input.merger.git_user_name "Zuul"
let executor-key-name = F.defaultText input.executor.ssh_key.key "id_rsa"
let sched-config = F.defaultText input.scheduler.config.key "main.yaml"
let web-url = F.defaultText input.web.status_url "http://web:9000"
let extra-kube-path = "/etc/nodepool-kubernetes/"
let db-uri =
merge
{ None = "postgresql://zuul:%(ZUUL_DB_PASSWORD)s@db/zuul"
, Some = \(some : Schemas.UserSecret.Type) -> "%(ZUUL_DB_URI)s"
}
input.database
let gerrits-conf =
mkConns
Schemas.Gerrit.Type
input.connections.gerrits
( \(gerrit : Schemas.Gerrit.Type) ->
let key = F.defaultText gerrit.sshkey.key "id_rsa"
let server = F.defaultText gerrit.server gerrit.name
in ''
[connection ${gerrit.name}]
driver=gerrit
server=${server}
sshkey=/etc/zuul-gerrit-${gerrit.name}/${key}
user=${gerrit.user}
baseurl=${gerrit.baseurl}
''
)
let githubs-conf =
mkConns
Schemas.GitHub.Type
input.connections.githubs
( \(github : Schemas.GitHub.Type) ->
let key = F.defaultText github.app_key.key "github_rsa"
in ''
[connection ${github.name}]
driver=github
server=github.com
app_id={github.app_id}
app_key=/etc/zuul-github-${github.name}/${key}
''
)
let gits-conf =
mkConns
Schemas.Git.Type
input.connections.gits
( \(git : Schemas.Git.Type) ->
''
[connection ${git.name}]
driver=git
baseurl=${git.baseurl}
''
)
let mqtts-conf =
mkConns
Schemas.Mqtt.Type
input.connections.mqtts
( \(mqtt : Schemas.Mqtt.Type) ->
let user =
merge
{ None = "", Some = \(some : Text) -> "user=${some}" }
mqtt.user
let password =
merge
{ None = ""
, Some =
\(some : Schemas.UserSecret.Type) ->
"password=%(ZUUL_MQTT_PASSWORD)"
}
mqtt.password
in ''
[connection ${mqtt.name}]
driver=mqtt
server=${mqtt.server}
${user}
${password}
''
)
let job-volumes =
F.mkJobVolume
Text
( \(job-volume : Schemas.JobVolume.Type) ->
let {- TODO: add support for abritary lists of path per (context, access)
-} context =
merge
{ trusted = "trusted", untrusted = "untrusted" }
job-volume.context
let access =
merge
{ None = "ro"
, Some =
\(access : < ro | rw >) ->
merge { ro = "ro", rw = "rw" } access
}
job-volume.access
in "${context}_${access}_paths=${job-volume.path}"
)
input.jobVolumes
in ''
[gearman]
server=scheduler
ssl_ca=/etc/zuul-gearman/ca.crt
ssl_cert=/etc/zuul-gearman/tls.crt
ssl_key=/etc/zuul-gearman/tls.key
[gearman_server]
start=true
ssl_ca=/etc/zuul-gearman/ca.crt
ssl_cert=/etc/zuul-gearman/tls.crt
ssl_key=/etc/zuul-gearman/tls.key
[zookeeper]
${zk-hosts}
[merger]
git_user_email=${merger-email}
git_user_name=${merger-user}
[scheduler]
tenant_config=/etc/zuul-scheduler/${sched-config}
[web]
listen_address=0.0.0.0
root=${web-url}
[executor]
private_key_file=/etc/zuul-executor/${executor-key-name}
manage_ansible=false
''
++ Prelude.Text.concatSep "\n" job-volumes
++ ''
[connection "sql"]
driver=sql
dburi=${db-uri}
''
++ gits-conf
++ gerrits-conf
++ githubs-conf
++ mqtts-conf

@ -1,294 +0,0 @@
{- Common functions -}
let Prelude = ../Prelude.dhall
let Kubernetes = ../Kubernetes.dhall
let Schemas = ./input.dhall
let JobVolume = Schemas.JobVolume.Type
let UserSecret = Schemas.UserSecret.Type
let {- This methods process the optional input.job-volumes list. It takes:
* the desired output type
* a function that goes from JobVolume to the output type
* the input.job-volumes spec attribute
Then it returns a list of the output type
-} mkJobVolume =
\(OutputType : Type) ->
\(f : JobVolume -> OutputType) ->
\(job-volumes : Optional (List JobVolume)) ->
merge
{ None = [] : List OutputType
, Some = Prelude.List.map JobVolume OutputType f
}
job-volumes
let defaultNat =
\(value : Optional Natural) ->
\(default : Natural) ->
merge { None = default, Some = \(some : Natural) -> some } value
let defaultText =
\(value : Optional Text) ->
\(default : Text) ->
merge { None = default, Some = \(some : Text) -> some } value
let defaultKey =
\(secret : Optional UserSecret) ->
\(default : Text) ->
merge
{ None = default
, Some = \(some : UserSecret) -> defaultText some.key default
}
secret
let mkAppLabels =
\(app-name : Text) ->
[ { mapKey = "app.kubernetes.io/name", mapValue = app-name }
, { mapKey = "app.kubernetes.io/instance", mapValue = app-name }
, { mapKey = "app.kubernetes.io/part-of", mapValue = "zuul" }
]
let mkComponentLabel =
\(app-name : Text) ->
\(component-name : Text) ->
mkAppLabels app-name
# [ { mapKey = "app.kubernetes.io/component"
, mapValue = component-name
}
]
let Label = { mapKey : Text, mapValue : Text }
let Labels = List Label
let mkObjectMeta =
\(name : Text) ->
\(labels : Labels) ->
Kubernetes.ObjectMeta::{ name, labels = Some labels }
let mkSelector =
\(labels : Labels) ->
Kubernetes.LabelSelector::{ matchLabels = Some labels }
let mkService =
\(app-name : Text) ->
\(name : Text) ->
\(port-name : Text) ->
\(port : Natural) ->
let labels = mkComponentLabel app-name name
in Kubernetes.Service::{
, metadata = mkObjectMeta name labels
, spec = Some Kubernetes.ServiceSpec::{
, type = Some "ClusterIP"
, selector = Some labels
, ports = Some
[ Kubernetes.ServicePort::{
, name = Some port-name
, protocol = Some "TCP"
, targetPort = Some (Kubernetes.IntOrString.String port-name)
, port
}
]
}
}
let EnvSecret = { name : Text, secret : Text, key : Text }
let File = { path : Text, content : Text }
let Volume =
{ Type = { name : Text, dir : Text, files : List File }
, default.files = [] : List File
}
let {- A high level description of a component such as the scheduler or the launcher
-} Component =
{ Type =
{ name : Text
, count : Natural
, container : Kubernetes.Container.Type
, data-dir : List Volume.Type
, volumes : List Volume.Type
, extra-volumes : List Kubernetes.Volume.Type
, claim-size : Natural
}
, default =
{ data-dir = [] : List Volume.Type
, volumes = [] : List Volume.Type
, extra-volumes = [] : List Kubernetes.Volume.Type
, claim-size = 0
}
}
let {- The Kubernetes resources of a Component
-} KubernetesComponent =
{ Type =
{ Service : Optional Kubernetes.Service.Type
, Deployment : Optional Kubernetes.Deployment.Type
, StatefulSet : Optional Kubernetes.StatefulSet.Type
}
, default =
{ Service = None Kubernetes.Service.Type
, Deployment = None Kubernetes.Deployment.Type
, StatefulSet = None Kubernetes.StatefulSet.Type
}
}
let mkVolumeEmptyDir =
Prelude.List.map
Volume.Type
Kubernetes.Volume.Type
( \(volume : Volume.Type) ->
Kubernetes.Volume::{
, name = volume.name
, emptyDir = Some Kubernetes.EmptyDirVolumeSource::{=}
}
)
let mkVolumeSecret =
Prelude.List.map
Volume.Type
Kubernetes.Volume.Type
( \(volume : Volume.Type) ->
Kubernetes.Volume::{
, name = volume.name
, secret = Some Kubernetes.SecretVolumeSource::{
, secretName = Some volume.name
, defaultMode = Some 256
}
}
)
let mkPodTemplateSpec =
\(component : Component.Type) ->
\(labels : Labels) ->
Kubernetes.PodTemplateSpec::{
, metadata = mkObjectMeta component.name labels
, spec = Some Kubernetes.PodSpec::{
, volumes = Some
( mkVolumeSecret component.volumes
# mkVolumeEmptyDir component.data-dir
# component.extra-volumes
)
, containers = [ component.container ]
, automountServiceAccountToken = Some False
}
}
let mkStatefulSet =
\(app-name : Text) ->
\(component : Component.Type) ->
let labels = mkComponentLabel app-name component.name
let component-name = app-name ++ "-" ++ component.name
let claim =
if Natural/isZero component.claim-size
then [] : List Kubernetes.PersistentVolumeClaim.Type
else [ Kubernetes.PersistentVolumeClaim::{
, apiVersion = ""
, kind = ""
, metadata = Kubernetes.ObjectMeta::{
, name = component-name
}
, spec = Some Kubernetes.PersistentVolumeClaimSpec::{
, accessModes = Some [ "ReadWriteOnce" ]
, resources = Some Kubernetes.ResourceRequirements::{
, requests = Some
( toMap
{ storage =
Natural/show component.claim-size ++ "Gi"
}
)
}
}
}
]
in Kubernetes.StatefulSet::{
, metadata = mkObjectMeta component-name labels
, spec = Some Kubernetes.StatefulSetSpec::{
, serviceName = component.name
, replicas = Some component.count
, selector = mkSelector labels
, template = mkPodTemplateSpec component labels
, volumeClaimTemplates = Some claim
}
}
let mkDeployment =
\(app-name : Text) ->
\(component : Component.Type) ->
let labels = mkComponentLabel app-name component.name
let component-name = app-name ++ "-" ++ component.name
in Kubernetes.Deployment::{
, metadata = mkObjectMeta component-name labels
, spec = Some Kubernetes.DeploymentSpec::{
, replicas = Some component.count
, selector = mkSelector labels
, template = mkPodTemplateSpec component labels
}
}
let mkEnvVarValue =
Prelude.List.map
Label
Kubernetes.EnvVar.Type
( \(env : Label) ->
Kubernetes.EnvVar::{ name = env.mapKey, value = Some env.mapValue }
)
let mkEnvVarSecret =
Prelude.List.map
EnvSecret
Kubernetes.EnvVar.Type
( \(env : EnvSecret) ->
Kubernetes.EnvVar::{
, name = env.name
, valueFrom = Some Kubernetes.EnvVarSource::{
, secretKeyRef = Some Kubernetes.SecretKeySelector::{
, key = env.key
, name = Some env.secret
}
}
}
)
let mkVolumeMount =
Prelude.List.map
Volume.Type
Kubernetes.VolumeMount.Type
( \(volume : Volume.Type) ->
Kubernetes.VolumeMount::{
, name = volume.name
, mountPath = volume.dir
}
)
in { defaultNat
, defaultText
, defaultKey
, newlineSep = Prelude.Text.concatSep "\n"
, mkJobVolume
, mkComponentLabel
, mkObjectMeta
, mkSelector
, mkService
, mkDeployment
, mkStatefulSet
, mkVolumeMount
, mkEnvVarValue
, mkEnvVarSecret
, EnvSecret
, Label
, Labels
, Volume
, Component
, KubernetesComponent
}

@ -1,175 +0,0 @@
{- Zuul CR spec as a dhall schemas
> Note: in dhall, a record with such structure:
> { Type = { foo : Text }, default = { foo = "bar" }}
> is named a `schemas` and it can be used to set default value:
> https://docs.dhall-lang.org/references/Built-in-types.html#id133
The `Schemas` record contains schemas for the CR spec attributes.
The `Input` record is the Zuul CR spec schema.
-}
let JobVolume =
{ context : < trusted | untrusted >
, access : Optional < ro | rw >
, path : Text
, dir : Text
, volume : (../Kubernetes.dhall).Volume.Type
}
let UserSecret = { secretName : Text, key : Optional Text }
let Gerrit =
{ name : Text
, server : Optional Text
, user : Text
, baseurl : Text
, sshkey : UserSecret
}
let GitHub = { name : Text, app_id : Natural, app_key : UserSecret }
let Mqtt =
{ name : Text
, server : Text
, user : Optional Text
, password : Optional UserSecret
}
let Git = { name : Text, baseurl : Text }
let Schemas =
{ Merger =
{ Type =
{ image : Optional Text
, count : Optional Natural
, git_user_email : Optional Text
, git_user_name : Optional Text
}
, default =
{ image = None Text
, count = None Natural
, git_user_email = None Text
, git_user_name = None Text
}
}
, Executor =
{ Type =
{ image : Optional Text
, count : Optional Natural
, ssh_key : UserSecret
}
, default = { image = None Text, count = None Natural }
}
, Web =
{ Type =
{ image : Optional Text
, count : Optional Natural
, status_url : Optional Text
}
, default =
{ image = None Text, count = None Natural, status_url = None Text }
}
, Scheduler =
{ Type =
{ image : Optional Text
, count : Optional Natural
, config : UserSecret
}
, default = { image = None Text, count = None Natural }
}
, Registry =
{ Type =
{ image : Optional Text
, count : Optional Natural
, storage-size : Optional Natural
, public-url : Optional Text
}
, default =
{ image = None Text
, count = None Natural
, storage-size = None Natural
, public-url = None Text
}
}
, Preview =
{ Type = { image : Optional Text, count : Optional Natural }
, default = { image = None Text, count = None Natural }
}
, Launcher =
{ Type = { image : Optional Text, config : UserSecret }
, default.image = None Text
}
, Connections =
{ Type =
{ gerrits : Optional (List Gerrit)
, githubs : Optional (List GitHub)
, mqtts : Optional (List Mqtt)
, gits : Optional (List Git)
}
, default =
{ gerrits = None (List Gerrit)
, githubs = None (List GitHub)
, mqtts = None (List Mqtt)
, gits = None (List Git)
}
}
, ExternalConfigs =
{ Type =
{ openstack : Optional UserSecret
, kubernetes : Optional UserSecret
, amazon : Optional UserSecret
}
, default =
{ openstack = None UserSecret
, kubernetes = None UserSecret
, amazon = None UserSecret
}
}
, JobVolume = { Type = JobVolume, default.access = Some < ro | rw >.ro }
, UserSecret = { Type = UserSecret, default.key = None Text }
, Gerrit.Type = Gerrit
, GitHub.Type = GitHub
, Mqtt.Type = Mqtt
, Git.Type = Git
}
let Input =
{ Type =
{ name : Text
, imagePrefix : Optional Text
, merger : Schemas.Merger.Type
, executor : Schemas.Executor.Type
, web : Schemas.Web.Type
, scheduler : Schemas.Scheduler.Type
, registry : Schemas.Registry.Type
, preview : Schemas.Preview.Type
, launcher : Schemas.Launcher.Type
, database : Optional UserSecret
, zookeeper : Optional UserSecret
, externalConfig : Schemas.ExternalConfigs.Type
, connections : Schemas.Connections.Type
, jobVolumes : Optional (List JobVolume)
, withCertManager : Bool
}
, default =
{ imagePrefix = None Text
, database = None UserSecret
, zookeeper = None UserSecret
, externalConfig = Schemas.ExternalConfigs.default
, merger = Schemas.Merger.default
, web = Schemas.Web.default
, scheduler = Schemas.Scheduler.default
, registry = Schemas.Registry.default
, preview = Schemas.Preview.default
, executor = Schemas.Executor.default
, launcher = Schemas.Launcher.default
, connections = Schemas.Connections.default
, jobVolumes = None (List JobVolume)
, withCertManager = True
}
}
in Schemas // { Input }

@ -1,592 +0,0 @@
{- Zuul CR kubernetes resources
The evaluation of that file is a function that takes the cr inputs as an argument,
and returns the list of kubernetes of objects.
Unless cert-manager usage is enabled, the resources expect those secrets to be available:
* `${name}-gearman-tls` with:
* `ca.crt`
* `tls.crt`
* `tls.key`
* `${name}-registry-tls` with:
* `tls.crt`
* `tls.key`
The resources expect those secrets to be available:
* `${name}-zookeeper-tls` with:
* `ca.crt`
* `tls.crt`
* `tls.key`
* `zk.pem` the keystore
* `${name}-registry-user-rw` with:
* `secret` a password
* `username` the user name with write access
* `password` the user password
Unless the input.database db uri is provided, the resources expect this secret to be available:
* `${name}-database-password` the internal database password.
-}
let Prelude = ../Prelude.dhall
let Kubernetes = ../Kubernetes.dhall
let CertManager = ../CertManager.dhall
let Schemas = ./input.dhall
let F = ./functions.dhall
let Input = Schemas.Input.Type
let JobVolume = Schemas.JobVolume.Type
let UserSecret = Schemas.UserSecret.Type
let Volume = F.Volume
in \(input : Input) ->
let zk-conf =
merge
{ None =
{ ServiceVolumes =
[ Volume::{
, name = "${input.name}-secret-zk"
, dir = "/conf-tls"
, files =
[ { path = "zoo.cfg"
, content = ./files/zoo.cfg.dhall "/conf" "/conf"
}
]
}
]
, ClientVolumes =
[ Volume::{
, name = "${input.name}-zookeeper-tls"
, dir = "/etc/zookeeper-tls"
}
]
, Zuul =
''
hosts=zk:2281
tls_cert=/etc/zookeeper-tls/tls.crt
tls_key=/etc/zookeeper-tls/tls.key
tls_ca=/etc/zookeeper-tls/ca.crt
''
, Nodepool =
''
zookeeper-servers:
- host: zk
port: 2281
zookeeper-tls:
cert: /etc/zookeeper-tls/tls.crt
key: /etc/zookeeper-tls/tls.key
ca: /etc/zookeeper-tls/ca.crt
''
, Env = [] : List Kubernetes.EnvVar.Type
}
, Some =
\(some : UserSecret) ->
let empty = [] : List Volume.Type
in { ServiceVolumes = empty
, ClientVolumes = empty
, Zuul = "hosts=%(ZUUL_ZK_HOSTS)"
, Nodepool =
''
zookeeper-servers:
- hosts: %(ZUUL_ZK_HOSTS)"
''
, Env =
F.mkEnvVarSecret
[ { name = "ZUUL_ZK_HOSTS"
, secret = some.secretName
, key = F.defaultText some.key "hosts"
}
]
}
}
input.zookeeper
let db-internal-password-env =
\(env-name : Text) ->
F.mkEnvVarSecret
[ { name = env-name
, secret = "${input.name}-database-password"
, key = "password"
}
]
let org =
merge
{ None = "docker.io/zuul", Some = \(prefix : Text) -> prefix }
input.imagePrefix
let version = "latest"
let image = \(name : Text) -> "${org}/${name}:${version}"
let set-image =
\(default-name : Text) ->
\(input-name : Optional Text) ->
{ image =
merge
{ None = Some default-name
, Some = \(_ : Text) -> input-name
}
input-name
}
let etc-zuul =
Volume::{
, name = input.name ++ "-secret-zuul"
, dir = "/etc/zuul"
, files =
[ { path = "zuul.conf"
, content = ./files/zuul.conf.dhall input zk-conf.Zuul
}
]
}
let etc-zuul-registry =
Volume::{
, name = input.name ++ "-secret-registry"
, dir = "/etc/zuul"
, files =
[ { path = "registry.yaml"
, content =
let public-url =
F.defaultText
input.registry.public-url
"https://registry:9000"
in ./files/registry.yaml.dhall public-url
}
]
}
let etc-nodepool =
Volume::{
, name = input.name ++ "-secret-nodepool"
, dir = "/etc/nodepool"
, files =
[ { path = "nodepool.yaml"
, content = ./files/nodepool.yaml.dhall zk-conf.Nodepool
}
]
}
let Components =
{ CertManager =
let issuer =
{ kind = "Issuer"
, group = "cert-manager.io"
, name = "${input.name}-ca"
}
let registry-enabled =
Natural/isZero (F.defaultNat input.registry.count 0)
== False
let registry-cert =
if registry-enabled
then [ CertManager.Certificate::{
, metadata =
F.mkObjectMeta
"${input.name}-registry-tls"
( F.mkComponentLabel
input.name
"cert-registry"
)
, spec = CertManager.CertificateSpec::{
, secretName = "${input.name}-registry-tls"
, issuerRef = issuer
, dnsNames = Some [ "registry" ]
, usages = Some [ "server auth", "client auth" ]
}
}
]
else [] : List CertManager.Certificate.Type
in { Issuers =
[ CertManager.Issuer::{
, metadata =
F.mkObjectMeta
"${input.name}-selfsigning"
( F.mkComponentLabel
input.name
"issuer-selfsigning"
)
, spec = CertManager.IssuerSpec::{
, selfSigned = Some {=}
}
}
, CertManager.Issuer::{
, metadata =
F.mkObjectMeta
"${input.name}-ca"
(F.mkComponentLabel input.name "issuer-ca")
, spec = CertManager.IssuerSpec::{
, ca = Some { secretName = "${input.name}-ca" }
}
}
]
, Certificates =
[ CertManager.Certificate::{
, metadata =
F.mkObjectMeta
"${input.name}-ca"
(F.mkComponentLabel input.name "cert-ca")
, spec = CertManager.CertificateSpec::{
, secretName = "${input.name}-ca"
, isCA = Some True
, commonName = Some "selfsigned-root-ca"
, issuerRef =
issuer
// { name = "${input.name}-selfsigning" }
, usages = Some
[ "server auth", "client auth", "cert sign" ]
}
}
, CertManager.Certificate::{
, metadata =
F.mkObjectMeta
"${input.name}-gearman-tls"
(F.mkComponentLabel input.name "cert-gearman")
, spec = CertManager.CertificateSpec::{
, secretName = "${input.name}-gearman-tls"
, issuerRef = issuer
, dnsNames = Some [ "gearman" ]
, usages = Some [ "server auth", "client auth" ]
}