zuul/nodepool
Code Issues Proposed changes
nodepool/.zuul.yaml
Clark Boylan ed42e5535b Build arm64 images using arm64 wheels from openstack if available 
We updated python-base and python-builder to include arm64 images in
support of nodepool's arm64 python-builder image. In doing so we have
discovered a number of issues, but the biggest is slowness of building
python packages in an emulated environment.

In order to speed up package builds we consume the OpenDev linaro
cloud arm64 wheel cache. This doesn't have wheels for every package we
need, but for the things that it does have it will speed up our builds.

One of the risks with this setup is that we're relying on wheels built
for openstack on arm64 and those follow openstack's contraints. In order
to mitigate this risk we set pip install's --prefer-binary flag in the
pip.conf. This means that if openstack's constraints lag what is
availale on pypi we should use the existing wheels as long as they are
valid version according to requirements rather than trying to build from
sdist.

Co-Authored-By: James E. Blair <corvus@inaugust.com>
Co-Authored-By: Ian Wienand <iwienand@redhat.com>
Change-Id: I3b358721eebbceafc12daf9d706306634048b196
2020-09-03 14:02:21 -07:00

411 lines
13 KiB
YAML
Raw Blame History

- job:
name: nodepool-zuul-functional
parent: tox
description: |
Test that zuul works with current nodepool sources.
This test runs the zuul project ``nodepool`` environment.
pre-run: playbooks/nodepool-zuul-functional/pre.yaml
post-run: playbooks/nodepool-zuul-functional/post.yaml
required-projects:
- zuul/nodepool
- zuul/zuul
vars:
zuul_work_dir: "{{ zuul.projects['opendev.org/zuul/zuul'].src_dir }}"
tox_envlist: nodepool
- job:
name: nodepool-functional-openstack-base
description: |
Test Nodepool with an OpenStack.
Sets up a working OpenStack system, builds an image, launches a
node, and verifies SSH access to the node.
.. zuul:jobvar:: nodepool_extra_packages
:type: list
A list of extra packages to install at the start of the job
(i.e., anything needed for the DIB build).
.. zuul:jobvar:: nodepool_diskimage
:type: dict
.. zuul:jobvar:: base_element
The base element to use when building the disk image.
.. zuul:jobvar:: elements-dir
:type: string
Path to directory with external elements
.. zuul:jobvar:: extra_elements
:type: list
:default: []
List of extra DIB elements to be added to image
configuration.
.. zuul:jobvar:: release
The release number or name to use with the base element.
.. zuul:jobvar:: mirror
The URL for the package mirror to use when building the
disk image. This is optional.
.. zuul:jobvar:: env-vars
:type: dict
A dictionary of environment variables and values to add
to the DIB environment when building the image.
.. zuul:jobvar:: nodepool_debug
Boolean flag to enable debug logging for the nodepool builder
and launcher services. By default we log at INFO level and
higher. This enables DEBUG and higher. This is optional.
timeout: 5400
required-projects:
- zuul/nodepool
pre-run: playbooks/nodepool-functional-openstack/pre.yaml
run: playbooks/nodepool-functional-openstack/run.yaml
post-run: playbooks/nodepool-functional-openstack/post.yaml
vars:
zuul_copy_output:
/var/log/nodepool: logs
- job:
name: nodepool-functional-openstack
description: Test Nodepool with an OpenStack
parent: nodepool-functional-openstack-base
vars: &nodepool_diskimage_vars
nodepool_extra_packages:
- rpm
- yum-utils
nodepool_diskimage:
base_element: centos-minimal
release: 8
mirror: "http://{{ zuul_site_mirror_fqdn }}/centos"
env-vars:
DIB_SIMPLE_INIT_NETWORKMANAGER: '1'
- job:
name: nodepool-functional-openstack-src-base
description: Test Nodepool and OpenStack, with some projects from source
parent: nodepool-functional-openstack-base
required-projects:
- name: github.com/sqlalchemy/dogpile.cache
- name: opendev/glean
- name: zuul/nodepool
- name: openstack/diskimage-builder
- name: openstack/openstacksdk
- job:
name: nodepool-functional-openstack-src
description: Test Nodepool and OpenStack, with some projects from source
parent: nodepool-functional-openstack-src-base
vars: *nodepool_diskimage_vars
- job:
description: |
Test that nodepool works with kubernetes.
name: nodepool-functional-k8s
pre-run: playbooks/nodepool-functional-k8s/pre.yaml
run: playbooks/nodepool-functional-k8s/run.yaml
post-run: playbooks/nodepool-functional-k8s/post.yaml
nodeset: ubuntu-xenial
required-projects:
- zuul/nodepool
- job:
description: |
Test that nodepool works with openshift.
name: nodepool-functional-openshift
pre-run: playbooks/nodepool-functional-openshift/pre.yaml
run: playbooks/nodepool-functional-openshift/run.yaml
nodeset:
nodes:
- name: cluster
label: centos-7
- name: launcher
label: fedora-30
required-projects:
- zuul/nodepool
- secret:
name: nodepool-dockerhub
data:
username: zuulzuul
password: !encrypted/pkcs1-oaep
- VV5m4XbsYvAz0/7T+WoIypvD3ijRsPYrGo4O5JuOHt6TX14s9B1KddeMwnabEEKapPdbb
cwqbzFKoHRMi8DdXiDKxrgbTSC1jb3BvKq7XE0/xF4Bq2dM+r3yDnXS6a7BuijSqFSiMG
/9bf0jqEAWXWCvbYUAxKOklJIRLjcKywCP5CulmW6EV8f1Iya8Y7UGpjGyL34FVQuc52Y
5MLp+Dra95pvBJMgikbu7LtMPCXOMctUFBpwNArCjWKGhz00UZwyAmsbPPHwQWb+884ZP
v9zi2FuRAL+rn4DRVwyIw28ZdCEr5F5EkZon3lSIhikhdQIyHPtW8VwMJTL4RxBdmgcEt
+zZBbLlOU+0AHPGLlO/yQNvHeSzwY9pe2oIGuqSu3h6zp6Y0NIE/Q0VpiIXWpOAnLOmno
uOd3j9tZ1beol2mg4Yk7rOaXu1Fvtfhd1K3LpLzEbiGzCMIEZuFHU2ZInAbUmtm0BtbDa
/bYvI9dB/uqA2xgse5rVMa++v3AfWJTJhFY/wBU4fHOWlFBUZKFuSoijUcWMfnfMMyU2B
A5CQKhv4Wmtzbvo/oneYkwFOTYaPNj2DS0BcFJFkR8FZDYmTBPnKoBH5wpwwnrzPOpCMa
L21z+Vp1Vce1XCsaOduMrmZDvuQcK1YOKIIUeElAQw8GuwTfcXqekoPvp54SeE=
- job:
name: nodepool-build-image
parent: opendev-build-docker-image
description: Build Docker images.
timeout: 4800
dependencies:
- opendev-buildset-registry
requires:
- python-builder-3.8-container-image
- python-base-3.8-container-image
provides: nodepool-container-image
vars: &nodepool_image_vars
docker_images:
- context: .
repository: zuul/nodepool
target: nodepool
tags:
# If zuul.tag is defined: [ '3', '3.19', '3.19.0' ]. Only works for 3-component tags.
# Otherwise: ['latest']
&imagetag "{{ zuul.tag is defined | ternary([zuul.get('tag', '').split('.')[0], '.'.join(zuul.get('tag', '').split('.')[:2]), zuul.get('tag', '')], ['latest']) }}"
arch:
- linux/amd64
- linux/arm64
- context: .
repository: zuul/nodepool-launcher
target: nodepool-launcher
tags: *imagetag
arch:
- linux/amd64
- linux/arm64
- context: .
repository: zuul/nodepool-builder
target: nodepool-builder
tags: *imagetag
arch:
- linux/amd64
- linux/arm64
- job:
name: nodepool-upload-image
parent: opendev-upload-docker-image
description: Build Docker images and upload to Docker Hub.
timeout: 4800
requires:
- python-builder-3.8-container-image
- python-base-3.8-container-image
provides: nodepool-container-image
vars: *nodepool_image_vars
secrets:
- name: docker_credentials
secret: nodepool-dockerhub
pass-to-parent: true
- job:
name: nodepool-promote-image
parent: opendev-promote-docker-image
description: Promote previously uploaded Docker images.
vars: *nodepool_image_vars
secrets:
- name: docker_credentials
secret: nodepool-dockerhub
pass-to-parent: true
# NOTE(ianw) : kept separate as these aren't intended to be released
- job:
name: nodepool-build-image-siblings
dependencies:
- opendev-buildset-registry
parent: opendev-build-docker-image
description: Build container images, with required projects as sibling from source
required-projects:
- zuul/nodepool
- openstack/diskimage-builder
- openstack/openstacksdk
requires:
- python-builder-3.8-container-image
- python-base-3.8-container-image
provides: nodepool-siblings-container-image
vars:
zuul_work_dir: "{{ zuul.projects['opendev.org/zuul/nodepool'].src_dir }}"
docker_images:
- context: .
repository: zuul/nodepool-launcher
target: nodepool-launcher
tags:
- siblings
siblings:
- opendev.org/openstack/openstacksdk
- context: .
repository: zuul/nodepool-builder
target: nodepool-builder
tags:
- siblings
siblings:
- opendev.org/openstack/openstacksdk
- opendev.org/openstack/diskimage-builder
- job:
name: nodepool-functional-container-openstack-base
abstract: true
description: |
Test Nodepool containers with an OpenStack.
This is similar to nodepool-functional-openstack-base, but
uses containers to deploy nodepool.
Note this is an abstract job and does not define an image to
build and test, and you should not inherit directly from this
job.
If you wish to build the nodepool containers with released
dependencies, you should inherit from
nodepool-functional-container-openstack-release-base and define
nodepool_diskimage for the image to build and test.
If you wish to build the nodepool containers with dependencies
installed from source, you should inherit from
nodepool-functional-container-openstack-siblings-base and define
nodepool_diskimage for the image build and test.
timeout: 5400
required-projects:
- zuul/nodepool
pre-run: playbooks/nodepool-functional-container-openstack/pre.yaml
run: playbooks/nodepool-functional-container-openstack/run.yaml
post-run: playbooks/nodepool-functional-container-openstack/post.yaml
vars:
zuul_copy_output:
/var/log/nodepool: logs
- job:
name: nodepool-functional-container-openstack-release-base
abstract: true
description: |
Test Nodepool containers and OpenStack, against released
dependencies.
Note, this job does not define an image to build. It should be
inherited from and nodepool_diskimage defined.
parent: nodepool-functional-container-openstack-base
dependencies:
- nodepool-build-image
required-projects:
- name: opendev/glean # note, installed by dib
- name: zuul/nodepool
- job:
name: nodepool-functional-container-openstack-release
description: |
Test Nodepool containers and OpenStack, with released projects
This builds, uploads and boots a CentOS 8 image into a
OpenStack cloud.
parent: nodepool-functional-container-openstack-release-base
vars:
nodepool_diskimage:
base_element: centos-minimal
release: 8
mirror: "http://{{ zuul_site_mirror_fqdn }}/centos"
env-vars:
DIB_SIMPLE_INIT_NETWORKMANAGER: '1'
- job:
name: nodepool-functional-container-openstack-siblings-base
abstract: true
description: |
Test Nodepool containers and OpenStack, with some projects from source
Note, this job does not define an image to build. It should be
inherited from and nodepool_diskimage defined.
parent: nodepool-functional-container-openstack-base
dependencies:
- nodepool-build-image-siblings
required-projects:
- name: opendev/glean # note, installed by dib
- name: zuul/nodepool
vars:
nodepool_container_tag: ':siblings'
- job:
name: nodepool-functional-container-openstack-siblings
description: |
Test Nodepool containers and OpenStack, with some projects from source
This builds, uploads and boots a CentOS 8 image into a
OpenStack cloud.
parent: nodepool-functional-container-openstack-siblings-base
vars:
nodepool_diskimage:
base_element: centos-minimal
release: 8
mirror: "http://{{ zuul_site_mirror_fqdn }}/centos"
env-vars:
DIB_SIMPLE_INIT_NETWORKMANAGER: '1'
- project:
vars:
release_python: python3
check:
jobs:
- opendev-buildset-registry
- nodepool-build-image
- nodepool-build-image-siblings
- zuul-tox-docs
- tox-pep8
- tox-py35:
nodeset: ubuntu-xenial
- tox-py38
- nodepool-zuul-functional:
voting: false
- nodepool-functional-openstack:
vars:
nodepool_debug: true
- nodepool-functional-openstack-src:
vars:
nodepool_debug: true
- nodepool-functional-container-openstack-release
- nodepool-functional-container-openstack-siblings
- nodepool-functional-k8s
- nodepool-functional-openshift
- zuul-quick-start:
requires: zuul-container-image
dependencies: nodepool-build-image
- build-python-release
gate:
jobs:
- nodepool-upload-image
- zuul-tox-docs
- tox-pep8
- tox-py35:
nodeset: ubuntu-xenial
- tox-py38
- nodepool-functional-openstack
- nodepool-functional-openstack-src
- nodepool-functional-k8s
- nodepool-functional-openshift
- zuul-quick-start:
requires: zuul-container-image
dependencies: nodepool-upload-image
- build-python-release
promote:
jobs:
- nodepool-promote-image
- zuul-promote-docs
- opendev-promote-python
release:
jobs:
- opendev-release-python
- zuul-publish-tox-docs
- upload-docker-image:
secrets:
name: docker_credentials
secret: nodepool-dockerhub
pass-to-parent: true
vars:
<<: *nodepool_image_vars
upload_docker_image_promote: false
View Git Blame Copy Permalink