Add docker build gate

There is a future requirement for Spyglass to have helm charts. These
charts will need docker images published in a source countrolled
manner.

This change depends on https://review.opendev.org/#/c/657614/ which
adds the airship_spyglass_quay_creds parameters. This patch:
1. Adds the docker-image-build playbook for Spyglass.
2. Uses the playbook in zuul jobs: check, gate and post.
3. Adds the spyglass-single-node nodeset.
4. Adds the vars.yaml file required by docker-image-build playbook.
5. Adds the image_tags.py script required by docker-image-build
   playbook.
6. Adds the docker-systemd.conf file required by docker-image-build
   playbook.

Change-Id: I76981e719a7e42984089fec08e1161a6923c5c52
This commit is contained in:
Alexander Hughes 2019-05-07 12:11:24 -05:00 committed by Alexander Hughes
parent 7f2613bfb6
commit b9f2ce32b3
5 changed files with 336 additions and 0 deletions

View File

@ -13,9 +13,47 @@
check:
jobs:
- openstack-tox-pep8
- spyglass-docker-build-gate
gate:
jobs:
- openstack-tox-pep8
- spyglass-docker-build-gate
post:
jobs:
- spyglass-docker-publish
- nodeset:
name: spyglass-single-node
nodes:
- name: primary
label: ubuntu-xenial
- job:
name: spyglass-docker-build-gate
timeout: 1800
run: tools/gate/playbooks/docker-image-build.yaml
nodeset: spyglass-single-node
vars:
publish: false
tags:
dynamic:
patch_set: true
- job:
name: spyglass-docker-publish
timeout: 1800
run: tools/gate/playbooks/docker-image-build.yaml
nodeset: armada-single-node
secrets:
- airship_spyglass_quay_creds
vars:
publish: true
tags:
dynamic:
branch: true
commit: true
static:
- latest
- secret:
name: airship_spyglass_quay_creds

View File

@ -0,0 +1,130 @@
# Copyright 2019 AT&T Intellectual Property. All other rights reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
- hosts: primary
tasks:
- include_vars: vars.yaml
- name: Debug tag generation inputs
block:
- debug:
var: publish
- debug:
var: tags
- debug:
var: zuul
- debug:
msg: "{{ tags | to_json }}"
- name: Determine tags
shell: echo '{{ tags | to_json }}' | python3 {{ zuul.project.src_dir }}/tools/image_tags.py
environment:
BRANCH: "{{ zuul.branch | default('') }}"
CHANGE: "{{ zuul.change | default('') }}"
COMMIT: "{{ zuul.newrev | default('') }}"
PATCHSET: "{{ zuul.patchset | default('') }}"
register: image_tags
- name: Debug computed tags
debug:
var: image_tags
- name: Install Docker (Debian)
when: ansible_os_family == 'Debian'
block:
- file:
path: "{{ item }}"
state: directory
with_items:
- /etc/docker/
- /etc/systemd/system/docker.service.d/
- /var/lib/docker/
- mount:
path: /var/lib/docker/
src: tmpfs
fstype: tmpfs
opts: size=25g
state: mounted
- copy: "{{ item }}"
with_items:
- content: "{{ docker_daemon | to_json }}"
dest: /etc/docker/daemon.json
- src: files/docker-systemd.conf
dest: /etc/systemd/system/docker.service.d/
- apt_key:
url: https://download.docker.com/linux/ubuntu/gpg
- apt_repository:
repo: deb http://{{ zuul_site_mirror_fqdn }}/deb-docker xenial stable
- apt:
name: "{{ item }}"
allow_unauthenticated: True
with_items:
- docker-ce
- python-pip
- pip:
name: docker
version: 2.7.0
# NOTE(SamYaple): Allow all connections from containers to host so the
# containers can access the http server for git and wheels
- iptables:
action: insert
chain: INPUT
in_interface: docker0
jump: ACCEPT
become: True
- name: Make images
when: not publish
block:
- make:
chdir: "{{ zuul.project.src_dir }}"
target: images
params:
IMAGE_TAG: "{{ item }}"
with_items: "{{ image_tags.stdout_lines }}"
- shell: "docker images"
register: docker_images
- debug:
var: docker_images
become: True
- name: Publish images
block:
- docker_login:
username: "{{ airship_spyglass_quay_creds.username }}"
password: "{{ airship_spyglass_quay_creds.password }}"
registry_url: "https://quay.io/api/v1/"
- make:
chdir: "{{ zuul.project.src_dir }}"
target: images
params:
DOCKER_REGISTRY: "quay.io"
IMAGE_PREFIX: "airshipit"
IMAGE_TAG: "{{ item }}"
COMMIT: "{{ zuul.newrev | default('') }}"
PUSH_IMAGE: "true"
with_items: "{{ image_tags.stdout_lines }}"
- shell: "docker images"
register: docker_images
- debug:
var: docker_images
when: publish
become: True

View File

@ -0,0 +1,22 @@
# Copyright 2019 AT&T Intellectual Property. All other rights reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#
# NOTE(SamYaple): CentOS cannot be build with userns-remap enabled. httpd uses
# cap_set_file capability and there is no way to pass that in at build as of
# docker 17.06.
# TODO(SamYaple): Periodically check to see if this is possible in newer
# versions of Docker
[Service]
ExecStart=
ExecStart=/usr/bin/dockerd

View File

@ -0,0 +1,19 @@
# Copyright 2019 AT&T Intellectual Property. All other rights reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
docker_daemon:
group: zuul
registry-mirrors:
- "http://{{ zuul_site_mirror_fqdn }}:8082/"
storage-driver: overlay2

127
tools/image_tags.py Normal file
View File

@ -0,0 +1,127 @@
#!/usr/bin/python3
# Copyright 2019 AT&T Intellectual Property. All other rights reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
import json
import logging
import os
import sys
LOG = logging.getLogger(__name__)
LOG_FORMAT = '%(asctime)s %(levelname)-8s %(name)s:%(filename)s:' \
'%(lineno)3d:%(funcName)s %(message)s'
class TagGenExeception(Exception):
pass
def read_config(stream, env):
config = {}
try:
config['tags'] = json.load(stream)
except ValueError:
LOG.exception('Failed to decode JSON from input stream')
config['tags'] = {}
LOG.debug('Configuration after reading stream: %s', config)
config['context'] = {
'branch': env.get('BRANCH'),
'change': env.get('CHANGE'),
'commit': env.get('COMMIT'),
'ps': env.get('PATCHSET'),
}
LOG.info('Final configuration: %s', config)
return config
def build_tags(config):
tags = config.get('tags', {}).get('static', [])
LOG.debug('Dynamic tags: %s', tags)
tags.extend(build_dynamic_tags(config))
LOG.info('All tags: %s', tags)
return tags
def build_dynamic_tags(config):
dynamic_tags = []
dynamic_tags.extend(_build_branch_tag(config))
dynamic_tags.extend(_build_commit_tag(config))
dynamic_tags.extend(_build_ps_tag(config))
return dynamic_tags
def _build_branch_tag(config):
if _valid_dg(config, 'branch'):
return [config['context']['branch']]
else:
return []
def _build_commit_tag(config):
if _valid_dg(config, 'commit'):
return [config['context']['commit']]
else:
return []
def _build_ps_tag(config):
if _valid_dg(config, 'patch_set', 'change') and _valid_dg(
config, 'patch_set', 'ps'):
return [
'%s-%s' % (config['context']['change'], config['context']['ps'])
]
else:
return []
def _valid_dg(config, dynamic_tag, context_name=None):
if context_name is None:
context_name = dynamic_tag
if config.get('tags', {}).get('dynamic', {}).get(dynamic_tag):
if config.get('context', {}).get(context_name):
return True
else:
raise TagGenExeception('Dynamic tag "%s" requested, but "%s"'
' not found in context' % (dynamic_tag,
context_name))
else:
return False
def main():
config = read_config(sys.stdin, os.environ)
tags = build_tags(config)
for tag in tags:
print(tag)
if __name__ == '__main__':
logging.basicConfig(format=LOG_FORMAT, level=logging.WARNING)
try:
main()
except TagGenExeception:
LOG.exception('Failed to generate tags')
sys.exit(1)
except Exception:
LOG.exception('Unexpected exception')
sys.exit(2)