zuul
/
zuul-operator
Code Issues Proposed changes

Use kopf operator framework 

This switches from the ansible/dhall operator framework to kopf,
an operator framework written in pure Python.  This allows us to:

* Build the operator application as a Python app.
* Build the operator image using the opendev python builder images.
* Run the operator as a Python CLI program "zuul-operator".
* Write procedural Python code to handle operator tasks (such as
  creating new nodepool launchers when providers are added).
* Use Jinja for templating config files and k8s resource files
  (direct pythonic manipulation of resources is an option too).

The new CR nearly matches the existing one, with some minor differences.

Some missing features and documentation are added in the commits
immediately following; they should be reviewed and merged as a unit.

Also, fx waiting for scheduler to settle in functional test since
we changed this log line in Zuul.

Change-Id: Ib37b67e3444b7cd44692d48eee77775ee9049e9f

Change-Id: I70ec31ecd8fe264118215944022b2e7b513dced9
changes/39/785039/26
James E. Blair 2021-07-08 15:51:11 -07:00
parent 0366b867bf
commit eff9f360f7
80 changed files with 29349 additions and 3163 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
.gitignore vendored
View File

@ -7,3 +7,4 @@
id_rsa
id_rsa.pub
*.patch
*.egg-info/

1
.zuul.yaml
View File

@ -20,7 +20,6 @@
nodeset: ubuntu-bionic
vars:
namespace: 'default'
withCertManager: true
- job:
description: Image and buildset registry job

75
build/Dockerfile
View File

@ -1,59 +1,26 @@
FROM quay.io/operator-framework/ansible-operator:v1.4.2
# Copyright (c) 2020 Red Hat, Inc.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
# implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# dhall versions and digests
ARG DHALL_VERSION=1.33.1
ARG DHALL_JSON_VERSION=1.7.0
ARG DHALL_JSON_DIGEST=cc9fc70e492d35a3986183b589a435653e782f67cda51d33a935dff1ddd15aec
ARG DHALL_LANG_REF=v17.0.0
ARG DHALL_KUBE_REF=v4.0.0
FROM docker.io/opendevorg/python-builder:3.8 as builder
# kubectl versions and digests
ARG KUBECTL_VERSION=v1.17.0
ARG KUBECTL_DIGEST=a5eb7e2e44d858d96410937a4e4c82f9087c9d120cb2b9e92462878eda59d578
COPY . /tmp/src
RUN assemble
# Install extra requirements
USER root
FROM docker.io/opendevorg/python-base:3.8
# Install gear to connect to the scheduler gearman
RUN pip3 install --upgrade gear
COPY --from=builder /output/ /output
RUN /output/install-from-bindep
# Install collections
RUN ansible-galaxy collection install community.kubernetes && chmod -R ug+rwx ${HOME}/.ansible
# unarchive: bzip2 and tar
# generate zuul ssh-keys or certificate: openssh and openssl
# manage configuration: git
RUN dnf install -y bzip2 tar openssh openssl git
# Install kubectl to mitigate https://github.com/operator-framework/operator-sdk/issues/2204
RUN curl -OL https://dl.k8s.io/$KUBECTL_VERSION/kubernetes-client-linux-amd64.tar.gz \
&& echo "$KUBECTL_DIGEST kubernetes-client-linux-amd64.tar.gz" | sha256sum -c \
&& tar -xf kubernetes-client-linux-amd64.tar.gz --strip-components=3 -z --mode='a+x' -C /usr/bin \
&& rm kubernetes-client-linux-amd64.tar.gz
# Install dhall-to-json
RUN curl -OL https://github.com/dhall-lang/dhall-haskell/releases/download/$DHALL_VERSION/dhall-json-$DHALL_JSON_VERSION-x86_64-linux.tar.bz2 \
&& echo "$DHALL_JSON_DIGEST dhall-json-$DHALL_JSON_VERSION-x86_64-linux.tar.bz2" | sha256sum -c \
&& tar -xf dhall-json-$DHALL_JSON_VERSION-x86_64-linux.tar.bz2 --strip-components=2 -j --mode='a+x' -C /usr/bin \
&& rm dhall-json-$DHALL_JSON_VERSION-x86_64-linux.tar.bz2
# Back to the default operator user
USER 1001
# Install dhall libraries
RUN git clone --branch $DHALL_LANG_REF --depth 1 https://github.com/dhall-lang/dhall-lang /opt/ansible/dhall-lang \
&& git clone --branch $DHALL_KUBE_REF --depth 1 https://github.com/dhall-lang/dhall-kubernetes /opt/ansible/dhall-kubernetes
ENV DHALL_PRELUDE=/opt/ansible/dhall-lang/Prelude/package.dhall
ENV DHALL_KUBERNETES=/opt/ansible/dhall-kubernetes/package.dhall
# Copy configuration
COPY conf/ /opt/ansible/conf/
# Cache dhall objects
RUN echo 'let Prelude = ~/conf/Prelude.dhall let Kubernetes = ~/conf/Kubernetes.dhall in "OK"' | \
env DHALL_PRELUDE=/opt/ansible/dhall-lang/Prelude/package.dhall \
DHALL_KUBERNETES=/opt/ansible/dhall-kubernetes/package.dhall dhall-to-json
# Copy ansible operator requirements
COPY watches.yaml ${HOME}/watches.yaml
COPY roles ${HOME}/roles
ENTRYPOINT ["/usr/local/bin/zuul-operator"]

59
conf/CertManager.dhall
View File

@ -1,59 +0,0 @@
{- A local cert manager package that extends the Kubernetes binding
TODO: Use union combinaison once it is available, see https://github.com/dhall-lang/dhall-lang/issues/175
TODO: Check with the dhall kubernetes community if the new type could be contributed,
though it currently only covers what is needed for zuul.
-}
let Kubernetes = ./Kubernetes.dhall
let IssuerSpec =
{ Type = { selfSigned : Optional {}, ca : Optional { secretName : Text } }
, default = { selfSigned = None {}, ca = None { secretName : Text } }
}
let Issuer =
{ Type =
{ apiVersion : Text
, kind : Text
, metadata : Kubernetes.ObjectMeta.Type
, spec : IssuerSpec.Type
}
, default = { apiVersion = "cert-manager.io/v1alpha2", kind = "Issuer" }
}
let CertificateSpec =
{ Type =
{ secretName : Text
, isCA : Optional Bool
, usages : Optional (List Text)
, commonName : Optional Text
, dnsNames : Optional (List Text)
, issuerRef : { name : Text, kind : Text, group : Text }
}
, default =
{ isCA = None Bool
, usages = None (List Text)
, commonName = None Text
, dnsNames = None (List Text)
}
}
let Certificate =
{ Type =
{ apiVersion : Text
, kind : Text
, metadata : Kubernetes.ObjectMeta.Type
, spec : CertificateSpec.Type
}
, default =
{ apiVersion = "cert-manager.io/v1alpha3", kind = "Certificate" }
}
let Union =
< Kubernetes : Kubernetes.Resource
| Issuer : Issuer.Type
| Certificate : Certificate.Type
>
in { IssuerSpec, Issuer, CertificateSpec, Certificate, Union }

3
conf/Kubernetes.dhall
View File

@ -1,3 +0,0 @@
{- Import the kubernetes types, see the ./Prelude.dhall file for documentation -}
env:DHALL_KUBERNETES
? https://raw.githubusercontent.com/dhall-lang/dhall-kubernetes/v4.0.0/package.dhall sha256:d9eac5668d5ed9cb3364c0a39721d4694e4247dad16d8a82827e4619ee1d6188

28
conf/Prelude.dhall
View File

@ -1,28 +0,0 @@
{- This file provides a central `Prelude` import for the rest of the library to
use so that the integrity check only needs to be updated in one place
whenever upgrading the interpreter.
This allows the user to provide their own Prelude import using the
`DHALL_PRELUDE` environment variable, like this:
```
$ export DHALL_PRELUDE='https://prelude.dhall-lang.org/package.dhall sha256:...'
```
Note that overriding the Prelude in this way only works if this repository
is imported locally. Remote imports do not have access to environment
variables and any attempt to import one will fall back to the next available
import. To learn more, read:
* https://docs.dhall-lang.org/discussions/Safety-guarantees.html#cross-site-scripting-xss
This file also provides an import without the integrity check as a slower
fallback if the user is using a different version of the Dhall interpreter.
This pattern is documented in the dhall-nethack repo:
* https://github.com/dhall-lang/dhall-nethack/blob/master/Prelude.dhall
-}
env:DHALL_PRELUDE
? https://prelude.dhall-lang.org/v17.0.0/package.dhall sha256:10db3c919c25e9046833df897a8ffe2701dc390fa0893d958c3430524be5a43e
? https://prelude.dhall-lang.org/v17.0.0/package.dhall

44
conf/zuul/components/Database.dhall
View File

@ -1,44 +0,0 @@
let Kubernetes = ../../Kubernetes.dhall
let F = ../functions.dhall
let db-volumes = [ F.Volume::{ name = "pg-data", dir = "/var/lib/pg/" } ]
in \(app-name : Text) ->
\ ( db-internal-password-env
: forall (env-name : Text) -> List Kubernetes.EnvVar.Type
) ->
F.KubernetesComponent::{
, Service = Some (F.mkService app-name "db" "pg" 5432)
, StatefulSet = Some
( F.mkStatefulSet
app-name
F.Component::{
, name = "db"
, count = 1
, data-dir = db-volumes
, claim-size = 1
, container = Kubernetes.Container::{
, name = "db"
, image = Some "docker.io/library/postgres:12.1"
, imagePullPolicy = Some "IfNotPresent"
, ports = Some
[ Kubernetes.ContainerPort::{
, name = Some "pg"
, containerPort = 5432
}
]
, env = Some
( F.mkEnvVarValue
( toMap
{ POSTGRES_USER = "zuul"
, PGDATA = "/var/lib/pg/data"
}
)
# db-internal-password-env "POSTGRES_PASSWORD"
)
, volumeMounts = Some (F.mkVolumeMount db-volumes)
}
}
)
}

67
conf/zuul/components/Executor.dhall
View File

@ -1,67 +0,0 @@
let Kubernetes = ../../Kubernetes.dhall
let F = ../functions.dhall
let InputExecutor = (../input.dhall).Executor.Type
let JobVolume = (../input.dhall).JobVolume.Type
in \(app-name : Text) ->
\(input-executor : InputExecutor) ->
\(data-dir : List F.Volume.Type) ->
\(volumes : List F.Volume.Type) ->
\(env : List Kubernetes.EnvVar.Type) ->
\(jobVolumes : Optional (List JobVolume)) ->
F.KubernetesComponent::{
, Service = Some (F.mkService app-name "executor" "finger" 7900)
, StatefulSet = Some
( F.mkStatefulSet
app-name
F.Component::{
, name = "executor"
, count = 1
, data-dir
, volumes
, extra-volumes =
let job-volumes =
F.mkJobVolume
Kubernetes.Volume.Type
(\(job-volume : JobVolume) -> job-volume.volume)
jobVolumes
in job-volumes
, claim-size = 0
, container = Kubernetes.Container::{
, name = "executor"
, image = input-executor.image
, imagePullPolicy = Some "IfNotPresent"
, ports = Some
[ Kubernetes.ContainerPort::{
, name = Some "finger"
, containerPort = 7900
}
]
, env = Some env
, volumeMounts =
let job-volumes-mount =
F.mkJobVolume
F.Volume.Type
( \(job-volume : JobVolume) ->
F.Volume::{
, name = job-volume.volume.name
, dir = job-volume.dir
}
)
jobVolumes
in Some
( F.mkVolumeMount
(data-dir # volumes # job-volumes-mount)
)
, securityContext = Some Kubernetes.SecurityContext::{
, privileged = Some True
}
}
}
)
}

30
conf/zuul/components/Merger.dhall
View File

@ -1,30 +0,0 @@
let Kubernetes = ../../Kubernetes.dhall
let F = ../functions.dhall
let InputMerger = (../input.dhall).Merger.Type
in \(app-name : Text) ->
\(input-merger : InputMerger) ->
\(data-dir : List F.Volume.Type) ->
\(volumes : List F.Volume.Type) ->
\(env : List Kubernetes.EnvVar.Type) ->
F.KubernetesComponent::{
, Deployment = Some
( F.mkDeployment
app-name
F.Component::{
, name = "merger"
, count = 1
, data-dir
, volumes
, container = Kubernetes.Container::{
, name = "merger"
, image = input-merger.image
, imagePullPolicy = Some "IfNotPresent"
, env = Some env
, volumeMounts = Some (F.mkVolumeMount (data-dir # volumes))
}
}
)
}

39
conf/zuul/components/Preview.dhall
View File

@ -1,39 +0,0 @@
let Kubernetes = ../../Kubernetes.dhall
let F = ../functions.dhall
let InputPreview = (../input.dhall).Preview.Type
in \(app-name : Text) ->
\(input-preview : InputPreview) ->
\(data-dir : List F.Volume.Type) ->
F.KubernetesComponent::{
, Service = Some (F.mkService app-name "preview" "preview" 80)
, Deployment = Some
( F.mkDeployment
app-name
F.Component::{
, name = "preview"
, count = F.defaultNat input-preview.count 0
, data-dir
, container = Kubernetes.Container::{
, name = "preview"
, image = input-preview.image
, imagePullPolicy = Some "IfNotPresent"
, ports = Some
[ Kubernetes.ContainerPort::{
, name = Some "preview"
, containerPort = 80
}
]
, env = Some
[ Kubernetes.EnvVar::{
, name = "ZUUL_API_URL"
, value = Some "http://web:9000"
}
]
, volumeMounts = Some (F.mkVolumeMount data-dir)
}
}
)
}

67
conf/zuul/components/Registry.dhall
View File

@ -1,67 +0,0 @@
let Prelude = ../../Prelude.dhall
let Kubernetes = ../../Kubernetes.dhall
let F = ../functions.dhall
let InputRegistry = (../input.dhall).Registry.Type
let registry-volumes =
\(app-name : Text) ->
[ F.Volume::{
, name = app-name ++ "-registry-tls"
, dir = "/etc/zuul-registry"
}
]
let registry-env =
\(app-name : Text) ->
F.mkEnvVarSecret
( Prelude.List.map
Text
F.EnvSecret
( \(key : Text) ->
{ name = "ZUUL_REGISTRY_${key}"
, key
, secret = "${app-name}-registry-user-rw"
}
)
[ "secret", "username", "password" ]
)
in \(app-name : Text) ->
\(input-registry : InputRegistry) ->
\(data-dir : List F.Volume.Type) ->
\(volumes : List F.Volume.Type) ->
F.KubernetesComponent::{
, Service = Some (F.mkService app-name "registry" "registry" 9000)
, StatefulSet = Some
( F.mkStatefulSet
app-name
F.Component::{
, name = "registry"
, count = F.defaultNat input-registry.count 0
, data-dir
, volumes = volumes # registry-volumes app-name
, claim-size = F.defaultNat input-registry.storage-size 20
, container = Kubernetes.Container::{
, name = "registry"
, image = input-registry.image
, args = Some
[ "zuul-registry", "-c", "/etc/zuul/registry.yaml", "serve" ]
, imagePullPolicy = Some "IfNotPresent"
, ports = Some
[ Kubernetes.ContainerPort::{
, name = Some "registry"
, containerPort = 9000
}
]
, env = Some (registry-env app-name)
, volumeMounts = Some
( F.mkVolumeMount
(data-dir # volumes # registry-volumes app-name)
)
}
}
)
}

38
conf/zuul/components/Scheduler.dhall
View File

@ -1,38 +0,0 @@
let Kubernetes = ../../Kubernetes.dhall
let F = ../functions.dhall
let InputScheduler = (../input.dhall).Scheduler.Type
in \(app-name : Text) ->
\(input-scheduler : InputScheduler) ->
\(data-dir : List F.Volume.Type) ->
\(volumes : List F.Volume.Type) ->
\(env : List Kubernetes.EnvVar.Type) ->
F.KubernetesComponent::{
, Service = Some (F.mkService app-name "scheduler" "gearman" 4730)
, StatefulSet = Some
( F.mkStatefulSet
app-name
F.Component::{
, name = "scheduler"
, count = 1
, data-dir
, volumes
, claim-size = 5
, container = Kubernetes.Container::{
, name = "scheduler"
, image = input-scheduler.image
, imagePullPolicy = Some "IfNotPresent"
, ports = Some
[ Kubernetes.ContainerPort::{
, name = Some "gearman"
, containerPort = 4730
}
]
, env = Some env
, volumeMounts = Some (F.mkVolumeMount (data-dir # volumes))
}
}
)
}

37
conf/zuul/components/Web.dhall
View File

@ -1,37 +0,0 @@
let Kubernetes = ../../Kubernetes.dhall
let F = ../functions.dhall
let InputWeb = (../input.dhall).Web.Type
in \(app-name : Text) ->
\(input-web : InputWeb) ->
\(data-dir : List F.Volume.Type) ->
\(volumes : List F.Volume.Type) ->
\(env : List Kubernetes.EnvVar.Type) ->
F.KubernetesComponent::{
, Service = Some (F.mkService app-name "web" "api" 9000)
, Deployment = Some
( F.mkDeployment
app-name
F.Component::{
, name = "web"
, count = 1
, data-dir
, volumes
, container = Kubernetes.Container::{
, name = "web"
, image = input-web.image
, imagePullPolicy = Some "IfNotPresent"
, ports = Some
[ Kubernetes.ContainerPort::{
, name = Some "api"
, containerPort = 9000
}
]
, env = Some env
, volumeMounts = Some (F.mkVolumeMount (data-dir # volumes))
}
}
)
}

50
conf/zuul/components/ZooKeeper.dhall
View File

@ -1,50 +0,0 @@
{- This function returns the ZooKeeper component in case the user doesn't provide it's own service.
The volumes list should contains the zoo
-}
let Kubernetes = ../../Kubernetes.dhall
let F = ../functions.dhall
let data-volumes =
[ F.Volume::{ name = "zk-log", dir = "/var/log/zookeeper/" }
, F.Volume::{ name = "zk-dat", dir = "/var/lib/zookeeper/" }
]
in \(app-name : Text) ->
\(client-conf : List F.Volume.Type) ->
F.KubernetesComponent::{
, Service = Some (F.mkService app-name "zk" "zk" 2281)
, StatefulSet = Some
( F.mkStatefulSet
app-name
F.Component::{
, name = "zk"
, count = 1
, data-dir = data-volumes
, volumes = client-conf
, claim-size = 1
, container = Kubernetes.Container::{
, name = "zk"
, command = Some
[ "sh"
, "-c"
, "cp /conf-tls/zoo.cfg /conf/ && "
++ "cp /etc/zookeeper-tls/zk.pem /conf/zk.pem && "
++ "cp /etc/zookeeper-tls/ca.crt /conf/ca.pem && "
++ "chown zookeeper /conf/zoo.cfg /conf/zk.pem /conf/ca.pem && "
++ "exec /docker-entrypoint.sh zkServer.sh start-foreground"
]
, image = Some "docker.io/library/zookeeper"
, imagePullPolicy = Some "IfNotPresent"
, ports = Some
[ Kubernetes.ContainerPort::{
, name = Some "zk"
, containerPort = 2281
}
]
, volumeMounts = Some
(F.mkVolumeMount (data-volumes # client-conf))
}
}
)
}

11
conf/zuul/files/nodepool.yaml.dhall
View File

@ -1,11 +0,0 @@
{- This function converts a zk-host Text to a nodepool.yaml file content
TODO: replace opaque Text by structured zk host list and tls configuration
-}
\(zk-host : Text) ->
''
${zk-host}
webapp:
port: 5000
''

20
conf/zuul/files/registry.yaml.dhall
View File

@ -1,20 +0,0 @@
{- This function converts a public-url Text to a registry.yaml file content
-}
\(public-url : Text) ->
''
registry:
address: '0.0.0.0'
port: 9000
public-url: ${public-url}
tls-cert: /etc/zuul-registry/tls.crt
tls-key: /etc/zuul-registry/tls.key
secret: "%(ZUUL_REGISTRY_secret)"
storage:
driver: filesystem
root: /var/lib/zuul
users:
- name: "%(ZUUL_REGISTRY_username)"
pass: "%(ZUUL_REGISTRY_password)"
access: write
''

23
conf/zuul/files/zoo.cfg.dhall
View File

@ -1,23 +0,0 @@
{- This function converts a client-dir and server-dir Text to a zoo.cfg file content
-}
\(client-dir : Text) ->
\(server-dir : Text) ->
''
dataDir=/data
dataLogDir=/datalog
tickTime=2000
initLimit=5
syncLimit=2
autopurge.snapRetainCount=3
autopurge.purgeInterval=0
maxClientCnxns=60
standaloneEnabled=true
admin.enableServer=true
server.1=0.0.0.0:2888:3888
# TLS configuration
secureClientPort=2281
serverCnxnFactory=org.apache.zookeeper.server.NettyServerCnxnFactory
ssl.keyStore.location=${server-dir}/zk.pem
ssl.trustStore.location=${client-dir}/ca.pem
''

192
conf/zuul/files/zuul.conf.dhall
View File

@ -1,192 +0,0 @@
{- This method renders the zuul.conf.
TODO: replace input schemas by the required attributes.
-}
\(input : (../input.dhall).Input.Type) ->
\(zk-hosts : Text) ->
let Prelude = ../../Prelude.dhall
let Schemas = ../input.dhall
let F = ../functions.dhall
let {- This is a high level method. It takes:
* a Connection type such as `Schemas.Gerrit.Type`,
* an Optional List of that type
* a function that goes from that type to a zuul.conf text blob
Then it returns a text blob for all the connections
-} mkConns =
\(type : Type) ->
\(list : Optional (List type)) ->
\(f : type -> Text) ->
F.newlineSep
( merge
{ None = [] : List Text, Some = Prelude.List.map type Text f }
list
)
let merger-email =
F.defaultText input.merger.git_user_email "${input.name}@localhost"
let merger-user = F.defaultText input.merger.git_user_name "Zuul"
let executor-key-name = F.defaultText input.executor.ssh_key.key "id_rsa"
let sched-config = F.defaultText input.scheduler.config.key "main.yaml"
let web-url = F.defaultText input.web.status_url "http://web:9000"
let extra-kube-path = "/etc/nodepool-kubernetes/"
let db-uri =
merge
{ None = "postgresql://zuul:%(ZUUL_DB_PASSWORD)s@db/zuul"
, Some = \(some : Schemas.UserSecret.Type) -> "%(ZUUL_DB_URI)s"
}
input.database
let gerrits-conf =
mkConns
Schemas.Gerrit.Type
input.connections.gerrits
( \(gerrit : Schemas.Gerrit.Type) ->
let key = F.defaultText gerrit.sshkey.key "id_rsa"
let server = F.defaultText gerrit.server gerrit.name
in ''
[connection ${gerrit.name}]
driver=gerrit
server=${server}
sshkey=/etc/zuul-gerrit-${gerrit.name}/${key}
user=${gerrit.user}
baseurl=${gerrit.baseurl}
''
)
let githubs-conf =
mkConns
Schemas.GitHub.Type
input.connections.githubs
( \(github : Schemas.GitHub.Type) ->
let key = F.defaultText github.app_key.key "github_rsa"
in ''
[connection ${github.name}]
driver=github
server=github.com
app_id={github.app_id}
app_key=/etc/zuul-github-${github.name}/${key}
''
)
let gits-conf =
mkConns
Schemas.Git.Type
input.connections.gits
( \(git : Schemas.Git.Type) ->
''
[connection ${git.name}]
driver=git
baseurl=${git.baseurl}
''
)
let mqtts-conf =
mkConns
Schemas.Mqtt.Type
input.connections.mqtts
( \(mqtt : Schemas.Mqtt.Type) ->
let user =
merge
{ None = "", Some = \(some : Text) -> "user=${some}" }
mqtt.user
let password =
merge
{ None = ""
, Some =
\(some : Schemas.UserSecret.Type) ->
"password=%(ZUUL_MQTT_PASSWORD)"
}
mqtt.password
in ''
[connection ${mqtt.name}]
driver=mqtt
server=${mqtt.server}
${user}
${password}
''
)
let job-volumes =
F.mkJobVolume
Text
( \(job-volume : Schemas.JobVolume.Type) ->
let {- TODO: add support for abritary lists of path per (context, access)
-} context =
merge
{ trusted = "trusted", untrusted = "untrusted" }
job-volume.context
let access =
merge
{ None = "ro"
, Some =
\(access : < ro | rw >) ->
merge { ro = "ro", rw = "rw" } access
}
job-volume.access
in "${context}_${access}_paths=${job-volume.path}"
)
input.jobVolumes
in ''
[gearman]
server=scheduler
ssl_ca=/etc/zuul-gearman/ca.crt
ssl_cert=/etc/zuul-gearman/tls.crt
ssl_key=/etc/zuul-gearman/tls.key
[gearman_server]
start=true
ssl_ca=/etc/zuul-gearman/ca.crt
ssl_cert=/etc/zuul-gearman/tls.crt
ssl_key=/etc/zuul-gearman/tls.key
[zookeeper]
${zk-hosts}
[merger]
git_user_email=${merger-email}
git_user_name=${merger-user}
[scheduler]
tenant_config=/etc/zuul-scheduler/${sched-config}
[web]
listen_address=0.0.0.0
root=${web-url}
[executor]
private_key_file=/etc/zuul-executor/${executor-key-name}
manage_ansible=false
''
++ Prelude.Text.concatSep "\n" job-volumes
++ ''
[connection "sql"]
driver=sql
dburi=${db-uri}
''
++ gits-conf
++ gerrits-conf
++ githubs-conf
++ mqtts-conf

294
conf/zuul/functions.dhall
View File

@ -1,294 +0,0 @@
{- Common functions -}
let Prelude = ../Prelude.dhall
let Kubernetes = ../Kubernetes.dhall
let Schemas = ./input.dhall
let JobVolume = Schemas.JobVolume.Type
let UserSecret = Schemas.UserSecret.Type
let {- This methods process the optional input.job-volumes list. It takes:
* the desired output type
* a function that goes from JobVolume to the output type
* the input.job-volumes spec attribute
Then it returns a list of the output type
-} mkJobVolume =
\(OutputType : Type) ->
\(f : JobVolume -> OutputType) ->
\(job-volumes : Optional (List JobVolume)) ->
merge
{ None = [] : List OutputType
, Some = Prelude.List.map JobVolume OutputType f
}
job-volumes
let defaultNat =
\(value : Optional Natural) ->
\(default : Natural) ->
merge { None = default, Some = \(some : Natural) -> some } value
let defaultText =
\(value : Optional Text) ->
\(default : Text) ->
merge { None = default, Some = \(some : Text) -> some } value
let defaultKey =
\(secret : Optional UserSecret) ->
\(default : Text) ->
merge
{ None = default
, Some = \(some : UserSecret) -> defaultText some.key default
}
secret
let mkAppLabels =
\(app-name : Text) ->
[ { mapKey = "app.kubernetes.io/name", mapValue = app-name }
, { mapKey = "app.kubernetes.io/instance", mapValue = app-name }
, { mapKey = "app.kubernetes.io/part-of", mapValue = "zuul" }
]
let mkComponentLabel =
\(app-name : Text) ->
\(component-name : Text) ->
mkAppLabels app-name
# [ { mapKey = "app.kubernetes.io/component"
, mapValue = component-name
}
]
let Label = { mapKey : Text, mapValue : Text }
let Labels = List Label
let mkObjectMeta =
\(name : Text) ->
\(labels : Labels) ->
Kubernetes.ObjectMeta::{ name, labels = Some labels }
let mkSelector =
\(labels : Labels) ->
Kubernetes.LabelSelector::{ matchLabels = Some labels }
let mkService =
\(app-name : Text) ->
\(name : Text) ->
\(port-name : Text) ->
\(port : Natural) ->
let labels = mkComponentLabel app-name name
in Kubernetes.Service::{
, metadata = mkObjectMeta name labels
, spec = Some Kubernetes.ServiceSpec::{
, type = Some "ClusterIP"
, selector = Some labels
, ports = Some
[ Kubernetes.ServicePort::{
, name = Some port-name
, protocol = Some "TCP"
, targetPort = Some (Kubernetes.IntOrString.String port-name)
, port
}
]
}
}
let EnvSecret = { name : Text, secret : Text, key : Text }
let File = { path : Text, content : Text }
let Volume =
{ Type = { name : Text, dir : Text, files : List File }
, default.files = [] : List File
}
let {- A high level description of a component such as the scheduler or the launcher
-} Component =
{ Type =
{ name : Text
, count : Natural
, container : Kubernetes.Container.Type
, data-dir : List Volume.Type
, volumes : List Volume.Type
, extra-volumes : List Kubernetes.Volume.Type
, claim-size : Natural
}
, default =
{ data-dir = [] : List Volume.Type
, volumes = [] : List Volume.Type
, extra-volumes = [] : List Kubernetes.Volume.Type
, claim-size = 0
}
}
let {- The Kubernetes resources of a Component
-} KubernetesComponent =
{ Type =
{ Service : Optional Kubernetes.Service.Type
, Deployment : Optional Kubernetes.Deployment.Type
, StatefulSet : Optional Kubernetes.StatefulSet.Type
}
, default =
{ Service = None Kubernetes.Service.Type
, Deployment = None Kubernetes.Deployment.Type
, StatefulSet = None Kubernetes.StatefulSet.Type
}
}
let mkVolumeEmptyDir =
Prelude.List.map
Volume.Type
Kubernetes.Volume.Type
( \(volume : Volume.Type) ->
Kubernetes.Volume::{
, name = volume.name
, emptyDir = Some Kubernetes.EmptyDirVolumeSource::{=}
}
)
let mkVolumeSecret =
Prelude.List.map
Volume.Type
Kubernetes.Volume.Type
( \(volume : Volume.Type) ->
Kubernetes.Volume::{
, name = volume.name
, secret = Some Kubernetes.SecretVolumeSource::{
, secretName = Some volume.name
, defaultMode = Some 256
}
}
)
let mkPodTemplateSpec =
\(component : Component.Type) ->
\(labels : Labels) ->
Kubernetes.PodTemplateSpec::{
, metadata = mkObjectMeta component.name labels
, spec = Some Kubernetes.PodSpec::{
, volumes = Some
( mkVolumeSecret component.volumes
# mkVolumeEmptyDir component.data-dir
# component.extra-volumes
)
, containers = [ component.container ]
, automountServiceAccountToken = Some False
}
}
let mkStatefulSet =
\(app-name : Text) ->
\(component : Component.Type) ->
let labels = mkComponentLabel app-name component.name
let component-name = app-name ++ "-" ++ component.name
let claim =
if Natural/isZero component.claim-size
then [] : List Kubernetes.PersistentVolumeClaim.Type
else [ Kubernetes.PersistentVolumeClaim::{
, apiVersion = ""
, kind = ""
, metadata = Kubernetes.ObjectMeta::{
, name = component-name
}
, spec = Some Kubernetes.PersistentVolumeClaimSpec::{
, accessModes = Some [ "ReadWriteOnce" ]
, resources = Some Kubernetes.ResourceRequirements::{
, requests = Some
( toMap
{ storage =
Natural/show component.claim-size ++ "Gi"
}
)
}
}
}
]
in Kubernetes.StatefulSet::{
, metadata = mkObjectMeta component-name labels
, spec = Some Kubernetes.StatefulSetSpec::{
, serviceName = component.name
, replicas = Some component.count
, selector = mkSelector labels
, template = mkPodTemplateSpec component labels
, volumeClaimTemplates = Some claim
}
}
let mkDeployment =
\(app-name : Text) ->
\(component : Component.Type) ->
let labels = mkComponentLabel app-name component.name
let component-name = app-name ++ "-" ++ component.name
in Kubernetes.Deployment::{
, metadata = mkObjectMeta component-name labels
, spec = Some Kubernetes.DeploymentSpec::{
, replicas = Some component.count
, selector = mkSelector labels
, template = mkPodTemplateSpec component labels
}
}
let mkEnvVarValue =
Prelude.List.map
Label
Kubernetes.EnvVar.Type
( \(env : Label) ->
Kubernetes.EnvVar::{ name = env.mapKey, value = Some env.mapValue }
)
let mkEnvVarSecret =
Prelude.List.map
EnvSecret
Kubernetes.EnvVar.Type
( \(env : EnvSecret) ->
Kubernetes.EnvVar::{
, name = env.name
, valueFrom = Some Kubernetes.EnvVarSource::{
, secretKeyRef = Some Kubernetes.SecretKeySelector::{
, key = env.key
, name = Some env.secret
}
}
}
)
let mkVolumeMount =
Prelude.List.map
Volume.Type
Kubernetes.VolumeMount.Type
( \(volume : Volume.Type) ->
Kubernetes.VolumeMount::{
, name = volume.name
, mountPath = volume.dir
}
)
in { defaultNat
, defaultText
, defaultKey
, newlineSep = Prelude.Text.concatSep "\n"
, mkJobVolume
, mkComponentLabel
, mkObjectMeta
, mkSelector
, mkService
, mkDeployment
, mkStatefulSet
, mkVolumeMount
, mkEnvVarValue
, mkEnvVarSecret
, EnvSecret
, Label
, Labels
, Volume
, Component
, KubernetesComponent
}

175
conf/zuul/input.dhall
View File

@ -1,175 +0,0 @@
{- Zuul CR spec as a dhall schemas
> Note: in dhall, a record with such structure:
> { Type = { foo : Text }, default = { foo = "bar" }}
> is named a `schemas` and it can be used to set default value:
> https://docs.dhall-lang.org/references/Built-in-types.html#id133
The `Schemas` record contains schemas for the CR spec attributes.
The `Input` record is the Zuul CR spec schema.
-}
let JobVolume =
{ context : < trusted | untrusted >
, access : Optional < ro | rw >
, path : Text
, dir : Text
, volume : (../Kubernetes.dhall).Volume.Type
}
let UserSecret = { secretName : Text, key : Optional Text }
let Gerrit =
{ name : Text
, server : Optional Text
, user : Text
, baseurl : Text
, sshkey : UserSecret
}
let GitHub = { name : Text, app_id : Natural, app_key : UserSecret }
let Mqtt =
{ name : Text
, server : Text
, user : Optional Text
, password : Optional UserSecret
}
let Git = { name : Text, baseurl : Text }
let Schemas =
{ Merger =
{ Type =
{ image : Optional Text
, count : Optional Natural
, git_user_email : Optional Text
, git_user_name : Optional Text
}
, default =
{ image = None Text
, count = None Natural
, git_user_email = None Text
, git_user_name = None Text
}
}
, Executor =
{ Type =
{ image : Optional Text
, count : Optional Natural
, ssh_key : UserSecret
}
, default = { image = None Text, count = None Natural }
}
, Web =
{ Type =
{ image : Optional Text
, count : Optional Natural
, status_url : Optional Text
}
, default =
{ image = None Text, count = None Natural, status_url = None Text }
}
, Scheduler =
{ Type =
{ image : Optional Text
, count : Optional Natural
, config : UserSecret
}
, default = { image = None Text, count = None Natural }
}
, Registry =
{ Type =
{ image : Optional Text
, count : Optional Natural
, storage-size : Optional Natural
, public-url : Optional Text
}
, default =
{ image = None Text
, count = None Natural
, storage-size = None Natural
, public-url = None Text
}
}
, Preview =
{ Type = { image : Optional Text, count : Optional Natural }
, default = { image = None Text, count = None Natural }
}
, Launcher =
{ Type = { image : Optional Text, config : UserSecret }
, default.image = None Text
}
, Connections =
{ Type =
{ gerrits : Optional (List Gerrit)
, githubs : Optional (List GitHub)
, mqtts : Optional (List Mqtt)
, gits : Optional (List Git)
}
, default =
{ gerrits = None (List Gerrit)
, githubs = None (List GitHub)
, mqtts = None (List Mqtt)
, gits = None (List Git)
}
}
, ExternalConfigs =
{ Type =
{ openstack : Optional UserSecret
, kubernetes : Optional UserSecret
, amazon : Optional UserSecret
}
, default =
{ openstack = None UserSecret
, kubernetes = None UserSecret
, amazon = None UserSecret
}
}
, JobVolume = { Type = JobVolume, default.access = Some < ro | rw >.ro }
, UserSecret = { Type = UserSecret, default.key = None Text }
, Gerrit.Type = Gerrit
, GitHub.Type = GitHub
, Mqtt.Type = Mqtt
, Git.Type = Git
}
let Input =
{ Type =
{ name : Text
, imagePrefix : Optional Text
, merger : Schemas.Merger.Type
, executor : Schemas.Executor.Type
, web : Schemas.Web.Type
, scheduler : Schemas.Scheduler.Type
, registry : Schemas.Registry.Type
, preview : Schemas.Preview.Type
, launcher : Schemas.Launcher.Type
, database : Optional UserSecret
, zookeeper : Optional UserSecret
, externalConfig : Schemas.ExternalConfigs.Type
, connections : Schemas.Connections.Type
, jobVolumes : Optional (List JobVolume)
, withCertManager : Bool
}
, default =
{ imagePrefix = None Text
, database = None UserSecret
, zookeeper = None UserSecret
, externalConfig = Schemas.ExternalConfigs.default
, merger = Schemas.Merger.default
, web = Schemas.Web.default
, scheduler = Schemas.Scheduler.default
, registry = Schemas.Registry.default
, preview = Schemas.Preview.default
, executor = Schemas.Executor.default
, launcher = Schemas.Launcher.default
, connections = Schemas.Connections.default
, jobVolumes = None (List JobVolume)
, withCertManager = True
}
}
in Schemas // { Input }

592
conf/zuul/resources.dhall
View File

@ -1,592 +0,0 @@
{- Zuul CR kubernetes resources
The evaluation of that file is a function that takes the cr inputs as an argument,
and returns the list of kubernetes of objects.
Unless cert-manager usage is enabled, the resources expect those secrets to be available:
* `${name}-gearman-tls` with:
* `ca.crt`
* `tls.crt`
* `tls.key`
* `${name}-registry-tls` with:
* `tls.crt`
* `tls.key`
The resources expect those secrets to be available:
* `${name}-zookeeper-tls` with:
* `ca.crt`
* `tls.crt`
* `tls.key`
* `zk.pem` the keystore
* `${name}-registry-user-rw` with:
* `secret` a password
* `username` the user name with write access
* `password` the user password
Unless the input.database db uri is provided, the resources expect this secret to be available:
* `${name}-database-password` the internal database password.
-}
let Prelude = ../Prelude.dhall
let Kubernetes = ../Kubernetes.dhall
let CertManager = ../CertManager.dhall
let Schemas = ./input.dhall
let F = ./functions.dhall
let Input = Schemas.Input.Type
let JobVolume = Schemas.JobVolume.Type
let UserSecret = Schemas.UserSecret.Type
let Volume = F.Volume
in \(input : Input) ->
let zk-conf =
merge
{ None =
{ ServiceVolumes =
[ Volume::{
, name = "${input.name}-secret-zk"
, dir = "/conf-tls"
, files =
[ { path = "zoo.cfg"
, content = ./files/zoo.cfg.dhall "/conf" "/conf"
}
]
}
]
, ClientVolumes =
[ Volume::{
, name = "${input.name}-zookeeper-tls"
, dir = "/etc/zookeeper-tls"
}
]
, Zuul =
''
hosts=zk:2281
tls_cert=/etc/zookeeper-tls/tls.crt
tls_key=/etc/zookeeper-tls/tls.key
tls_ca=/etc/zookeeper-tls/ca.crt
''
, Nodepool =
''
zookeeper-servers:
- host: zk
port: 2281
zookeeper-tls:
cert: /etc/zookeeper-tls/tls.crt
key: /etc/zookeeper-tls/tls.key
ca: /etc/zookeeper-tls/ca.crt
''
, Env = [] : List Kubernetes.EnvVar.Type
}
, Some =
\(some : UserSecret) ->
let empty = [] : List Volume.Type
in { ServiceVolumes = empty
, ClientVolumes = empty
, Zuul = "hosts=%(ZUUL_ZK_HOSTS)"
, Nodepool =
''
zookeeper-servers:
- hosts: %(ZUUL_ZK_HOSTS)"
''
, Env =
F.mkEnvVarSecret
[ { name = "ZUUL_ZK_HOSTS"
, secret = some.secretName
, key = F.defaultText some.key "hosts"
}
]
}
}
input.zookeeper
let db-internal-password-env =
\(env-name : Text) ->
F.mkEnvVarSecret
[ { name = env-name
, secret = "${input.name}-database-password"
, key = "password"
}
]
let org =
merge
{ None = "docker.io/zuul", Some = \(prefix : Text) -> prefix }
input.imagePrefix
let version = "latest"
let image = \(name : Text) -> "${org}/${name}:${version}"
let set-image =
\(default-name : Text) ->
\(input-name : Optional Text) ->
{ image =
merge
{ None = Some default-name
, Some = \(_ : Text) -> input-name
}
input-name
}
let etc-zuul =
Volume::{
, name = input.name ++ "-secret-zuul"
, dir = "/etc/zuul"
, files =
[ { path = "zuul.conf"
, content = ./files/zuul.conf.dhall input zk-conf.Zuul
}
]
}
let etc-zuul-registry =
Volume::{
, name = input.name ++ "-secret-registry"
, dir = "/etc/zuul"
, files =
[ { path = "registry.yaml"
, content =
let public-url =
F.defaultText
input.registry.public-url
"https://registry:9000"
in ./files/registry.yaml.dhall public-url
}
]
}
let etc-nodepool =
Volume::{
, name = input.name ++ "-secret-nodepool"
, dir = "/etc/nodepool"
, files =
[ { path = "nodepool.yaml"
, content = ./files/nodepool.yaml.dhall zk-conf.Nodepool
}
]
}
let Components =
{ CertManager =
let issuer =
{ kind = "Issuer"
, group = "cert-manager.io"
, name = "${input.name}-ca"
}
let registry-enabled =
Natural/isZero (F.defaultNat input.registry.count 0)
== False
let registry-cert =
if registry-enabled
then [ CertManager.Certificate::{
, metadata =
F.mkObjectMeta
"${input.name}-registry-tls"
( F.mkComponentLabel
input.name
"cert-registry"
)
, spec = CertManager.CertificateSpec::{
, secretName = "${input.name}-registry-tls"
, issuerRef = issuer
, dnsNames = Some [ "registry" ]
, usages = Some [ "server auth", "client auth" ]
}
}
]
else [] : List CertManager.Certificate.Type
in { Issuers =
[ CertManager.Issuer::{
, metadata =
F.mkObjectMeta
"${input.name}-selfsigning"
( F.mkComponentLabel
input.name
"issuer-selfsigning"
)
, spec = CertManager.IssuerSpec::{
, selfSigned = Some {=}
}
}
, CertManager.Issuer::{
, metadata =
F.mkObjectMeta
"${input.name}-ca"
(F.mkComponentLabel input.name "issuer-ca")
, spec = CertManager.IssuerSpec::{
, ca = Some { secretName = "${input.name}-ca" }
}
}
]
, Certificates =
[ CertManager.Certificate::{
, metadata =
F.mkObjectMeta
"${input.name}-ca"
(F.mkComponentLabel input.name "cert-ca")
, spec = CertManager.CertificateSpec::{
, secretName = "${input.name}-ca"
, isCA = Some True
, commonName = Some "selfsigned-root-ca"
, issuerRef =
issuer
// { name = "${input.name}-selfsigning" }
, usages = Some
[ "server auth", "client auth", "cert sign" ]
}
}
, CertManager.Certificate::{
, metadata =
F.mkObjectMeta
"${input.name}-gearman-tls"
(F.mkComponentLabel input.name "cert-gearman")
, spec = CertManager.CertificateSpec::{
, secretName = "${input.name}-gearman-tls"
, issuerRef = issuer
, dnsNames = Some [ "gearman" ]
, usages = Some [ "server auth", "client auth" ]
}
}
]
# registry-cert
}
, Backend =
{ Database =
merge
{ None =
./components/Database.dhall
input.name
db-internal-password-env
, Some =
\(some : UserSecret) -> F.KubernetesComponent.default
}
input.database
, ZooKeeper =
merge
{ None =
./components/ZooKeeper.dhall
input.name
(zk-conf.ClientVolumes # zk-conf.ServiceVolumes)
, Some =
\(some : UserSecret) -> F.KubernetesComponent.default
}
input.zookeeper
}
, Zuul =
let zuul-image =
\(name : Text) -> set-image (image "zuul-${name}")
let zuul-env =
F.mkEnvVarValue (toMap { HOME = "/var/lib/zuul" })
let db-secret-env =
merge
{ None = db-internal-password-env "ZUUL_DB_PASSWORD"
, Some =
\(some : UserSecret) ->
F.mkEnvVarSecret
[ { name = "ZUUL_DB_URI"
, secret = some.secretName
, key = F.defaultText some.key "db_uri"
}
]
}
input.database
let {- executor and merger do not need database info, but they fail to parse config without the env variable
-} db-nosecret-env =
F.mkEnvVarValue (toMap { ZUUL_DB_PASSWORD = "unused" })
let zuul-data-dir =
[ Volume::{ name = "zuul-data", dir = "/var/lib/zuul" } ]
let sched-config =
Volume::{
, name = input.scheduler.config.secretName
, dir = "/etc/zuul-scheduler"
}
let gearman-config =
Volume::{
, name = input.name ++ "-gearman-tls"
, dir = "/etc/zuul-gearman"
}
let executor-ssh-key =
Volume::{
, name = input.executor.ssh_key.secretName
, dir = "/etc/zuul-executor"
}
let zuul-volumes =
[ etc-zuul, gearman-config ] # zk-conf.ClientVolumes
in { Scheduler =
./components/Scheduler.dhall
input.name
( input.scheduler
// zuul-image "scheduler" input.scheduler.image
)
zuul-data-dir
(zuul-volumes # [ sched-config ])
(zuul-env # db-secret-env # zk-conf.Env)
, Executor =
./components/Executor.dhall
input.name
( input.executor
// zuul-image "executor" input.executor.image
)
zuul-data-dir
(zuul-volumes # [ executor-ssh-key ])
(zuul-env # db-nosecret-env)
input.jobVolumes
, Web =
./components/Web.dhall
input.name
(input.web // zuul-image "web" input.web.image)
zuul-data-dir
zuul-volumes
(zuul-env # db-secret-env # zk-conf.Env)
, Merger =
./components/Merger.dhall
input.name
( input.merger
// zuul-image "merger" input.merger.image
)
zuul-data-dir
zuul-volumes
(zuul-env # db-nosecret-env)
, Registry =
./components/Registry.dhall
input.name
( input.registry
// zuul-image "registry" input.registry.image
)
zuul-data-dir
[ etc-zuul-registry ]
, Preview =
./components/Preview.dhall
input.name
( input.preview
// zuul-image "preview" input.preview.image
)
zuul-data-dir
}
, Nodepool =
let nodepool-image =
\(name : Text) -> Some (image ("nodepool-" ++ name))
let nodepool-data-dir =
[ Volume::{
, name = "nodepool-data"
, dir = "/var/lib/nodepool"
}
]
let nodepool-config =
Volume::{
, name = input.launcher.config.secretName
, dir = "/etc/nodepool-config"
}
let openstack-config =
merge
{ None = [] : List Volume.Type
, Some =
\(some : UserSecret) ->
[ Volume::{
, name = some.secretName
, dir = "/etc/nodepool-openstack"
}
]
}
input.externalConfig.openstack
let kubernetes-config =
merge
{ None = [] : List Volume.Type
, Some =
\(some : UserSecret) ->
[ Volume::{
, name = some.secretName
, dir = "/etc/nodepool-kubernetes"
}
]
}
input.externalConfig.kubernetes
let nodepool-env =
F.mkEnvVarValue