refactor: change images and zuul

We no longer use underscores in image names, we instead use dashes
because that is a sane pattern.

We also create a tool which updates zuul jobs automatically based on the
images and builds each image in it's own job (only if necessary when we
have Dockerfile changes).

Change-Id: I23437d65f8761dbea1f4ffed31ed87ff73ec9142
This commit is contained in:
Mohammed Naser 2020-04-11 16:44:10 -04:00
parent 38054e2625
commit 7074b159df
17 changed files with 386 additions and 79 deletions

View File

@ -1,74 +0,0 @@
- job:
name: openstack-operator:images:build
parent: vexxhost-build-docker-image
provides: openstack-operator:image:operator
vars: &openstack_operator_images
docker_images:
- context: images/mcrouter
repository: vexxhost/mcrouter
- context: images/mcrouter_exporter
repository: vexxhost/mcrouter_exporter
- context: images/memcached
repository: vexxhost/memcached
- context: images/memcached_exporter
repository: vexxhost/memcached_exporter
- context: images/rabbitmq
repository: vexxhost/rabbitmq
- context: .
repository: vexxhost/openstack-operator
- job:
name: openstack-operator:images:upload
parent: vexxhost-upload-docker-image
provides: openstack-operator:image:operator
vars: *openstack_operator_images
- job:
name: openstack-operator:images:promote
parent: vexxhost-promote-docker-image
vars: *openstack_operator_images
- job:
name: openstack-operator:linters:chart
parent: chart-testing-lint
vars:
zuul_work_dir: "{{ zuul.project.src_dir }}/chart"
- job:
name: openstack-operator:functional
parent: devstack-tempest
requires:
- openstack-operator:images
pre-run: playbooks/functional/pre.yaml
run: playbooks/functional/run.yaml
post-run: playbooks/functional/post.yaml
vars:
devstack_services:
etcd3: false
docker_use_buildset_registry: true
minikube_dns_resolvers: ['1.1.1.1', '8.8.8.8']
- project:
templates:
- publish-opendev-tox-docs
check:
jobs:
- golangci-lint
- golang-go-test
- openstack-operator:linters:chart
- openstack-operator:images:build
- openstack-operator:functional:
dependencies:
- openstack-operator:images:build
gate:
jobs:
- golangci-lint
- golang-go-test
- openstack-operator:linters:chart
- openstack-operator:images:upload
- openstack-operator:functional:
dependencies:
- openstack-operator:images:upload
promote:
jobs:
- openstack-operator:images:promote

View File

@ -61,7 +61,7 @@ generate: controller-gen
# Build the docker image
docker-build: test
docker build . -t ${IMG}
docker build . -f images/openstack-operator/Dockerfile -t ${IMG}
# Push the docker image
docker-push:
@ -87,8 +87,8 @@ endif
# run-time images
images:
docker build images/mcrouter -t vexxhost/mcrouter:latest
docker build images/mcrouter_exporter -t vexxhost/mcrouter_exporter:latest
docker build images/mcrouter-exporter -t vexxhost/mcrouter-exporter:latest
docker build images/memcached -t vexxhost/memcached:latest
docker build images/memcached_exporter -t vexxhost/memcached_exporter:latest
docker build images/memcached-exporter -t vexxhost/memcached-exporter:latest
docker build images/rabbitmq -t vexxhost/rabbitmq:latest

View File

@ -107,7 +107,7 @@ func (r *McrouterReconciler) Reconcile(req ctrl.Request) (ctrl.Result, error) {
RunAsUser(999).
RunAsGroup(999),
),
builders.Container("exporter", "vexxhost/mcrouter_exporter:latest").
builders.Container("exporter", "vexxhost/mcrouter-exporter:latest").
Args("-mcrouter.address", "localhost:11211").
Port("metrics", 9442).HTTPProbe("metrics", "/metrics", 10, 30).
Resources(500, 128, 500, 2).

View File

@ -111,7 +111,7 @@ func (r *MemcachedReconciler) Reconcile(req ctrl.Request) (ctrl.Result, error) {
builders.SecurityContext().
RunAsUser(1001),
),
builders.Container("exporter", "vexxhost/memcached_exporter:latest").
builders.Container("exporter", "vexxhost/memcached-exporter:latest").
Port("metrics", 9150).HTTPProbe("metrics", "/metrics", 10, 30).
Resources(500, 128, 500, 2).
SecurityContext(

108
hack/update-zuul-jobs.py Executable file
View File

@ -0,0 +1,108 @@
#!/usr/bin/env python
import copy
import os
import sys
from ruamel import yaml
yaml = yaml.YAML()
images = os.listdir("images")
build_jobs = []
upload_jobs = []
for image in images:
files = []
if image != 'openstack-operator':
files = ['^images/%s/.*' % image]
deps = ['opendev-buildset-registry']
job_vars = {
'docker_images': [
{
'context': 'images/%s' % image,
'repository': 'vexxhost/%s' % image,
}
]
}
if image == 'openstack-operator':
job_vars['docker_images'][0]['context'] = '.'
job_vars['docker_images'][0]['dockerfile'] = 'images/openstack-operator/Dockerfile'
build_job = {
'job': {
'name': 'openstack-operator:images:build:%s' % image,
'parent': 'vexxhost-build-docker-image',
'provides': 'openstack-operator:image:%s' % image,
'dependencies': deps,
'vars': job_vars,
}
}
upload_job = {
'job': {
'name': 'openstack-operator:images:upload:%s' % image,
'parent': 'vexxhost-upload-docker-image',
'provides': 'openstack-operator:image:%s' % image,
'dependencies': deps,
'vars': job_vars,
}
}
promote_job = {
'job': {
'name': 'openstack-operator:images:promote:%s' % image,
'parent': 'vexxhost-promote-docker-image',
'vars': job_vars,
}
}
if files:
build_job['job']['files'] = files
upload_job['job']['files'] = files
promote_job['job']['files'] = files
project_config = {
'project': {
'check': {'jobs': [build_job['job']['name']]},
'gate': {'jobs': [upload_job['job']['name']]},
'promote': {'jobs': [promote_job['job']['name']]},
}
}
config = [
build_job,
upload_job,
promote_job,
project_config
]
if image == 'openstack-operator':
build_jobs.append(build_job['job']['name'])
upload_jobs.append(upload_job['job']['name'])
else:
build_jobs.append({'name': build_job['job']['name'], 'soft': True})
upload_jobs.append({'name': upload_job['job']['name'], 'soft': True})
with open("zuul.d/%s-jobs.yaml" % image, "w+") as fd:
yaml.dump(config, fd)
with open("zuul.d/functional-jobs.yaml") as fd:
data = yaml.load(fd)
for obj in data:
if 'project' in obj:
for job in obj['project']['check']['jobs']:
if 'openstack-operator:functional' in job:
job['openstack-operator:functional']['dependencies'] = \
build_jobs
for job in obj['project']['gate']['jobs']:
if 'openstack-operator:functional' in job:
job['openstack-operator:functional']['dependencies'] = \
upload_jobs
with open("zuul.d/functional-jobs.yaml", "w+") as fd:
yaml.dump(data, fd)

View File

@ -2,6 +2,12 @@
minversion = 2.0
skipsdist = True
[testenv:update-zuul-jobs]
deps =
ruamel.yaml
commands =
{toxinidir}/hack/update-zuul-jobs.py
[testenv:docs]
deps =
-r{toxinidir}/doc/requirements.txt

View File

@ -0,0 +1,47 @@
- job:
name: openstack-operator:functional
parent: devstack-tempest
requires:
- openstack-operator:images
pre-run: playbooks/functional/pre.yaml
run: playbooks/functional/run.yaml
post-run: playbooks/functional/post.yaml
vars:
devstack_services:
etcd3: false
docker_use_buildset_registry: true
minikube_dns_resolvers: [1.1.1.1, 8.8.8.8]
- project:
check:
jobs:
- opendev-buildset-registry
- openstack-operator:functional:
dependencies:
- name: openstack-operator:images:build:rabbitmq
soft: true
- name: openstack-operator:images:build:memcached
soft: true
- name: openstack-operator:images:build:mcrouter
soft: true
- openstack-operator:images:build:openstack-operator
- name: openstack-operator:images:build:mcrouter-exporter
soft: true
- name: openstack-operator:images:build:memcached-exporter
soft: true
gate:
jobs:
- opendev-buildset-registry
- openstack-operator:functional:
dependencies:
- name: openstack-operator:images:upload:rabbitmq
soft: true
- name: openstack-operator:images:upload:memcached
soft: true
- name: openstack-operator:images:upload:mcrouter
soft: true
- openstack-operator:images:upload:openstack-operator
- name: openstack-operator:images:upload:mcrouter-exporter
soft: true
- name: openstack-operator:images:upload:memcached-exporter
soft: true

View File

@ -0,0 +1,34 @@
- job:
name: openstack-operator:images:build:mcrouter-exporter
parent: vexxhost-build-docker-image
provides: openstack-operator:image:mcrouter-exporter
dependencies: &id001
- opendev-buildset-registry
vars: &id002
docker_images:
- context: images/mcrouter-exporter
repository: vexxhost/mcrouter-exporter
files: &id003
- ^images/mcrouter-exporter/.*
- job:
name: openstack-operator:images:upload:mcrouter-exporter
parent: vexxhost-upload-docker-image
provides: openstack-operator:image:mcrouter-exporter
dependencies: *id001
vars: *id002
files: *id003
- job:
name: openstack-operator:images:promote:mcrouter-exporter
parent: vexxhost-promote-docker-image
vars: *id002
files: *id003
- project:
check:
jobs:
- openstack-operator:images:build:mcrouter-exporter
gate:
jobs:
- openstack-operator:images:upload:mcrouter-exporter
promote:
jobs:
- openstack-operator:images:promote:mcrouter-exporter

34
zuul.d/mcrouter-jobs.yaml Normal file
View File

@ -0,0 +1,34 @@
- job:
name: openstack-operator:images:build:mcrouter
parent: vexxhost-build-docker-image
provides: openstack-operator:image:mcrouter
dependencies: &id001
- opendev-buildset-registry
vars: &id002
docker_images:
- context: images/mcrouter
repository: vexxhost/mcrouter
files: &id003
- ^images/mcrouter/.*
- job:
name: openstack-operator:images:upload:mcrouter
parent: vexxhost-upload-docker-image
provides: openstack-operator:image:mcrouter
dependencies: *id001
vars: *id002
files: *id003
- job:
name: openstack-operator:images:promote:mcrouter
parent: vexxhost-promote-docker-image
vars: *id002
files: *id003
- project:
check:
jobs:
- openstack-operator:images:build:mcrouter
gate:
jobs:
- openstack-operator:images:upload:mcrouter
promote:
jobs:
- openstack-operator:images:promote:mcrouter

View File

@ -0,0 +1,34 @@
- job:
name: openstack-operator:images:build:memcached-exporter
parent: vexxhost-build-docker-image
provides: openstack-operator:image:memcached-exporter
dependencies: &id001
- opendev-buildset-registry
vars: &id002
docker_images:
- context: images/memcached-exporter
repository: vexxhost/memcached-exporter
files: &id003
- ^images/memcached-exporter/.*
- job:
name: openstack-operator:images:upload:memcached-exporter
parent: vexxhost-upload-docker-image
provides: openstack-operator:image:memcached-exporter
dependencies: *id001
vars: *id002
files: *id003
- job:
name: openstack-operator:images:promote:memcached-exporter
parent: vexxhost-promote-docker-image
vars: *id002
files: *id003
- project:
check:
jobs:
- openstack-operator:images:build:memcached-exporter
gate:
jobs:
- openstack-operator:images:upload:memcached-exporter
promote:
jobs:
- openstack-operator:images:promote:memcached-exporter

View File

@ -0,0 +1,34 @@
- job:
name: openstack-operator:images:build:memcached
parent: vexxhost-build-docker-image
provides: openstack-operator:image:memcached
dependencies: &id001
- opendev-buildset-registry
vars: &id002
docker_images:
- context: images/memcached
repository: vexxhost/memcached
files: &id003
- ^images/memcached/.*
- job:
name: openstack-operator:images:upload:memcached
parent: vexxhost-upload-docker-image
provides: openstack-operator:image:memcached
dependencies: *id001
vars: *id002
files: *id003
- job:
name: openstack-operator:images:promote:memcached
parent: vexxhost-promote-docker-image
vars: *id002
files: *id003
- project:
check:
jobs:
- openstack-operator:images:build:memcached
gate:
jobs:
- openstack-operator:images:upload:memcached
promote:
jobs:
- openstack-operator:images:promote:memcached

19
zuul.d/misc-jobs.yaml Normal file
View File

@ -0,0 +1,19 @@
- job:
name: openstack-operator:linters:chart
parent: chart-testing-lint
vars:
zuul_work_dir: "{{ zuul.project.src_dir }}/chart"
- project:
templates:
- publish-opendev-tox-docs
check:
jobs:
- golangci-lint
- golang-go-test
- openstack-operator:linters:chart
gate:
jobs:
- golangci-lint
- golang-go-test
- openstack-operator:linters:chart

View File

@ -0,0 +1,31 @@
- job:
name: openstack-operator:images:build:openstack-operator
parent: vexxhost-build-docker-image
provides: openstack-operator:image:openstack-operator
dependencies: &id001
- opendev-buildset-registry
vars: &id002
docker_images:
- context: .
repository: vexxhost/openstack-operator
dockerfile: images/openstack-operator/Dockerfile
- job:
name: openstack-operator:images:upload:openstack-operator
parent: vexxhost-upload-docker-image
provides: openstack-operator:image:openstack-operator
dependencies: *id001
vars: *id002
- job:
name: openstack-operator:images:promote:openstack-operator
parent: vexxhost-promote-docker-image
vars: *id002
- project:
check:
jobs:
- openstack-operator:images:build:openstack-operator
gate:
jobs:
- openstack-operator:images:upload:openstack-operator
promote:
jobs:
- openstack-operator:images:promote:openstack-operator

34
zuul.d/rabbitmq-jobs.yaml Normal file
View File

@ -0,0 +1,34 @@
- job:
name: openstack-operator:images:build:rabbitmq
parent: vexxhost-build-docker-image
provides: openstack-operator:image:rabbitmq
dependencies: &id001
- opendev-buildset-registry
vars: &id002
docker_images:
- context: images/rabbitmq
repository: vexxhost/rabbitmq
files: &id003
- ^images/rabbitmq/.*
- job:
name: openstack-operator:images:upload:rabbitmq
parent: vexxhost-upload-docker-image
provides: openstack-operator:image:rabbitmq
dependencies: *id001
vars: *id002
files: *id003
- job:
name: openstack-operator:images:promote:rabbitmq
parent: vexxhost-promote-docker-image
vars: *id002
files: *id003
- project:
check:
jobs:
- openstack-operator:images:build:rabbitmq
gate:
jobs:
- openstack-operator:images:upload:rabbitmq
promote:
jobs:
- openstack-operator:images:promote:rabbitmq