zuul
/
zuul-operator
Code Issues Proposed changes

Add support for zuul-registry 

Change-Id: Ie206f8f7c48da3cfaabb33d16f2a339b35db4e5e
This commit is contained in:
James E. Blair 2021-04-10 18:21:39 -07:00
parent 6e0fb7fe1b
commit d87ea64c06
8 changed files with 260 additions and 27 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
deploy/crds/zuul-ci_v1alpha2_zuul_cr.yaml
View File

@ -13,8 +13,6 @@ spec:
scheduler:
config:
secretName: zuul-yaml-conf
registry:
count: 1
launcher:
config:
secretName: nodepool-yaml-conf

60
doc/source/index.rst
View File

@ -192,6 +192,33 @@ static HTML/Javascript sites). If you enable this, the operator will
configure a ``zuul-preview`` service to which you may route an Ingress
or LoadBalancer.
Zuul Registry
-------------
The operator has optional support for deploying a zuul-registry
service. This is an experimental add-on for Zuul to act as an
intermediate registry for the container image jobs in `zuul-jobs`.
If you enable this, the operator will, by default, configure a
``zuul-registry`` service in a manner appropriate for access from
within the cluster only. If you need to access the registry from
outside the cluster, you will need to additionally add an Ingress or
LoadBalancer, as well as provide TLS certs with the appropriate
hostname. Currently, zuul-registry performs its own TLS termination.
If you usue this, you will also need to provide a ``registry.yaml``
config file in a secret. You only need to provide the ``users`` and,
if you are accessing the registry outside the cluster, the
``public-url`` setting (omit it if you are accessing it from within
the cluster only).
.. code-block:: yaml
registry:
users:
- name: testuser
pass: testpass
access: write
Specification Reference
-----------------------
@ -481,3 +508,36 @@ verbatim):
:default: 0
How many Zuul Preview servers to manage.
.. attr:: registry
.. attr:: count
:default: 0
How many Zuul Registry servers to manage.
.. attr:: volumeSize
:default: 80Gi
The requested size of the registry storage volume.
.. attr:: tls
.. attr:: secretName
The name of a secret containing a TLS client certificate
and key for Zuul Registry. This should be (or the format
should match) a standard Kubernetes TLS secret.
If you omit this, the operator will create a secret for
you.
.. attr:: config
.. attr:: secretName
The name of a secret containing a registry
configuration file. The key in the secret should be
``registry.yaml``. Only provide the ``users`` and, if
exposing the registry outside the cluster, the
``public-url`` entries.

6
playbooks/zuul-operator-functional/run.yaml
View File

@ -37,6 +37,8 @@
secretName: nodepool-kube-config
registry:
count: 1
config:
secretName: zuul-registry-conf
preview:
count: 1
@ -45,7 +47,3 @@
- name: Test the cert-manager
include_tasks: ./tasks/test_cert_manager.yaml
# TODO: implement
# - name: Test the registry
# include_tasks: ./tasks/test_registry.yaml

9
playbooks/zuul-operator-functional/tasks/create_test_secrets.yaml
View File

@ -68,3 +68,12 @@
- name: nodepool-kube-config
data:
kube.config: "{{ _kube_config.stdout }}"
- name: zuul-registry-conf
data:
registry.yaml: |
registry:
users:
- name: testuser
pass: testpass
access: write

44
playbooks/zuul-operator-functional/tasks/test_registry.yaml
View File

@ -1,21 +1,23 @@
- name: Get registry service ip
command: kubectl get svc registry -o "jsonpath={.spec.clusterIP}"
register: _registry_ip
- name: Add registry to /etc/hosts
become: yes
lineinfile:
path: /etc/hosts
regexp: "^.* registry$"
line: "{{ _registry_ip.stdout_lines[0] }} registry"
- name: Get registry password
command: kubectl get secret zuul-registry-user-rw -o "jsonpath={.data.password}"
register: _registry_password
- name: Test registry login
command: >
podman login
--tls-verify=false registry:9000
-u zuul
-p "{{ _registry_password.stdout_lines[0] | b64decode }}"
- k8s:
namespace: default
definition:
apiVersion: batch/v1
kind: Job
metadata:
name: test-registry
spec:
template:
spec:
containers:
- name: test-registry
image: quay.io/containers/podman:latest
command: ['podman', 'login', '--tls-verify=false', 'https://zuul-registry/', '-u', 'testuser', '-p', 'testpass']
securityContext:
privileged: true
restartPolicy: Never
backoffLimit: 4
wait: yes
wait_timeout: 300
wait_condition:
type: Complete
status: "True"

3
playbooks/zuul-operator-functional/test.yaml
View File

@ -156,3 +156,6 @@
- name: Test the preview
include_tasks: ./tasks/test_preview.yaml
- name: Test the registry
include_tasks: ./tasks/test_registry.yaml

107
zuul_operator/templates/zuul-registry.yaml Normal file
View File

@ -0,0 +1,107 @@
{%- if manage_registry_cert %}
---
apiVersion: cert-manager.io/v1alpha2
kind: Certificate
metadata:
name: zuul-registry-tls
labels:
app.kubernetes.io/name: zuul
app.kubernetes.io/instance: {{ instance_name }}
app.kubernetes.io/part-of: zuul
app.kubernetes.io/component: zuul-registry-tls
spec:
keyEncoding: pkcs8
secretName: zuul-registry-tls
commonName: client
usages:
- digital signature
- key encipherment
- server auth
- client auth
issuerRef:
name: ca-issuer
kind: Issuer
{%- endif %}
---
apiVersion: v1
kind: Service
metadata:
name: zuul-registry
labels:
app.kubernetes.io/name: zuul
app.kubernetes.io/instance: {{ instance_name }}
app.kubernetes.io/part-of: zuul
app.kubernetes.io/component: zuul-registry
spec:
type: NodePort
ports:
- name: zuul-registry
port: 443
protocol: TCP
targetPort: registry
selector:
app.kubernetes.io/name: zuul
app.kubernetes.io/instance: {{ instance_name }}
app.kubernetes.io/part-of: zuul
app.kubernetes.io/component: zuul-registry
---
apiVersion: apps/v1
kind: StatefulSet
metadata:
name: zuul-registry
labels:
app.kubernetes.io/name: zuul
app.kubernetes.io/instance: {{ instance_name }}
app.kubernetes.io/part-of: zuul
app.kubernetes.io/component: zuul-registry
spec:
replicas: {{ spec.registry.count }}
serviceName: zuul-registry
selector:
matchLabels:
app.kubernetes.io/name: zuul
app.kubernetes.io/instance: {{ instance_name }}
app.kubernetes.io/part-of: zuul
app.kubernetes.io/component: zuul-registry
template:
metadata:
labels:
app.kubernetes.io/name: zuul
app.kubernetes.io/instance: {{ instance_name }}
app.kubernetes.io/part-of: zuul
app.kubernetes.io/component: zuul-registry
spec:
containers:
- name: registry
image: {{ spec.imagePrefix }}/zuul-registry:{{ spec.zuulImageVersion }}
env:
- name: DEBUG
value: '1'
ports:
- name: registry
containerPort: 9000
volumeMounts:
- name: zuul-registry-config
mountPath: /conf
readOnly: true
- name: zuul-registry-tls
mountPath: /tls
readOnly: true
- name: zuul-registry
mountPath: /storage
volumes:
- name: zuul-registry-config
secret:
secretName: zuul-registry-generated-config
- name: zuul-registry-tls
secret:
secretName: {{ spec.registry.tls.secretName }}
volumeClaimTemplates:
- metadata:
name: zuul-registry
spec:
accessModes:
- ReadWriteOnce
resources:
requests:
storage: {{ spec.registry.volumeSize }} #80Gi

56
zuul_operator/zuul.py
View File

@ -12,6 +12,7 @@
# License for the specific language governing permissions and limitations
# under the License.
import kopf
import copy
import base64
import hashlib
@ -72,10 +73,17 @@ class Zuul:
self.spec.setdefault('web', {}).setdefault('count', 1)
self.spec.setdefault('fingergw', {}).setdefault('count', 1)
self.spec.setdefault('preview', {}).setdefault('count', 0)
registry = self.spec.setdefault('registry', {})
registry.setdefault('count', 0)
registry.setdefault('volumeSize', '80Gi')
registry_tls = registry.setdefault('tls', {})
self.manage_registry_cert = ('secretName' not in registry_tls)
registry_tls.setdefault('secretName', 'zuul-registry-tls')
self.spec.setdefault('imagePrefix', 'docker.io/zuul')
self.spec.setdefault('zuulImageVersion', 'latest')
self.spec.setdefault('zuulPreviewImageVersion', 'latest')
self.spec.setdefault('zuulRegistryImageVersion', 'latest')
self.spec.setdefault('nodepoolImageVersion', 'latest')
self.cert_manager = certmanager.CertManager(
@ -314,7 +322,55 @@ class Zuul:
except pykube.exceptions.ObjectDoesNotExist:
pass
def write_registry_conf(self):
config_secret = self.spec['registry'].get('config', {}).get('secretName')
if not config_secret:
raise kopf.PermanentError("No registry config secret found")
try:
obj = objects.Secret.objects(self.api).\
filter(namespace=self.namespace).\
get(name=config_secret)
except pykube.exceptions.ObjectDoesNotExist:
raise kopf.TemporaryError("Registry config secret not found")
# Shard the config so we can create a deployment + secret for
# each provider.
registry_yaml = yaml.safe_load(base64.b64decode(
obj.obj['data']['registry.yaml']))
reg = registry_yaml['registry']
if 'public-url' not in reg:
reg['public-url'] = 'https://zuul-registry'
reg['address'] = '0.0.0.0'
reg['port'] = 9000
reg['tls-cert'] = '/tls/tls.crt'
reg['tls-key'] = '/tls/tls.key'
reg['secret'] = utils.generate_password(56)
reg['storage'] = {
'driver': 'filesystem',
'root': '/storage',
}
text = yaml.dump(registry_yaml)
utils.update_secret(self.api, self.namespace,
'zuul-registry-generated-config',
string_data={'registry.yaml': text})
def create_registry(self):
self.write_registry_conf()
kw = {
'instance_name': self.name,
'spec': self.spec,
'manage_registry_cert': self.manage_registry_cert,
}
utils.apply_file(self.api, 'zuul-registry.yaml',
namespace=self.namespace, **kw)
def create_zuul(self):
if self.spec['registry']['count']:
self.create_registry()
kw = {
'zuul_conf_sha': self.zuul_conf_sha,
'zuul_tenant_secret': self.tenant_secret,