From 22483e9a158620fdbae721bcc4599ee87fd2bfd8 Mon Sep 17 00:00:00 2001 From: Scott Hussey Date: Fri, 8 Jun 2018 05:38:15 -0500 Subject: [PATCH] (zuul) Add Docker image jobs - Add check/gate jobs to build images - Add post job to build and publish images Change-Id: I9c09923996f2d6341f3b9fe3cb27eecc8ae79c64 --- .zuul.yaml | 62 +++++++++ Makefile | 8 +- tools/gate/playbooks/docker-image-build.yaml | 96 ++++++++++++++ tools/image_tags.py | 126 +++++++++++++++++++ 4 files changed, 290 insertions(+), 2 deletions(-) create mode 100644 tools/gate/playbooks/docker-image-build.yaml create mode 100644 tools/image_tags.py diff --git a/.zuul.yaml b/.zuul.yaml index 29618d2..6624ea3 100644 --- a/.zuul.yaml +++ b/.zuul.yaml @@ -15,10 +15,15 @@ jobs: - airship-maas-lint-ws - airship-maas-lint-chart + - airship-maas-docker-build-gate gate: jobs: - airship-maas-lint-ws - airship-maas-lint-chart + - airship-maas-docker-build-gate + post: + jobs: + - airship-maas-docker-publish - nodeset: name: airship-maas-single-node @@ -43,3 +48,60 @@ run: tools/gate/playbooks/helm-linter.yaml timeout: 600 nodeset: airship-maas-single-node + +- job: + name: airship-maas-docker-build-gate + timeout: 1800 + run: tools/gate/playbooks/docker-image-build.yaml + nodeset: airship-maas-single-node + files: + - '^images/.*' + vars: + publish: false + tags: + dynamic: + patch_set: true + +- job: + name: airship-maas-docker-publish + timeout: 1800 + run: tools/gate/playbooks/docker-image-build.yaml + nodeset: airship-maas-single-node + secrets: + - airship_maas_quay_creds + irrelevant-files: + - '^images/.*' + vars: + publish: true + tags: + dynamic: + branch: true + commit: true + static: + - latest + +- secret: + name: airship_maas_quay_creds + data: + username: !encrypted/pkcs1-oaep + - C3OlPbC72jwWy2KO224B+JoIThvsAhEvNG2USHtIISHWQPTod3fA2rDut28dopMoBiit4 + qRK3V0e93LsjfiVLYgdxnlkNcKH6iNk6YT6pkCGx5veVHUxJcfl+x0EAFwK0sEMHjLxvN + 3nahnLQG67WUDGdz0lMFnDG8pLU/tOzD9E5rNnbZOYjyVdmWXWVHIQGkwnK7fTkLbBCfu + KQR2DfmbqNcXMAQpsAM7himvYcRO/Fh18bO0ebsNscV2C31KhZgBnmtnaYu7wJvTh2W9+ + rCFpKzBwJlnD97KfMv4ZlWwSH9QvwUC2UO/vcED9yZDwVFHU5okwI0QSwLyoGRs7T+Flc + Rwk8xGghJDgcxRjH7HHS3pDHaB1kZucEN+g10o2IRFLUcS4w2zYPrXfOZDWKymsvdkUW8 + 4XRuqSAuDdTHcjDk7aKpnySb0hDcVf5zNNFA7GqLajkBmaH59fKx1MjNocO3KmpHQ6yb+ + TTIn4aK8xhT/S/Azy0khTTN/w/4A6JZ+eRVtDsnP0X4yJvWiYUP80OKk7/Vr3EyV3s1WN + n7YRW8veKEqi9CqzfZtW1MeQ6Bu8VrIduc/XST6JyYd8AyjmliiSs7YNCbd/Sz+pjl4I8 + rQW7vThr7W9CQCEi+UMoUebArOM9d+LBGtlqZBROI1P1QaSPcxPm3f1/Rl7aqM= + password: !encrypted/pkcs1-oaep + - Aun3qSkZsZT+d7Zk7sLR2QoVEhK95OEJjJ6TdPMeSfuhAyIwQ0b9sBUk1BAsrOjlIdQO1 + AOYhSpunpaita2KP1nt67GQwuQZkcZIE3UMQAUjIkU1fyTGwa3ZYR2Z4/fTzuslzRzOcL + 9lvTaULTeXEoxs413shUK1W6EpMb5GMVUW4DvKwy4ei9ZBNVO+540p741+GgDZEH6UDKc + KGtYbsGDRudAAded82NsaODvoIOiXq9oQxiBDepv6Hyah8gqMuKBpLIQIsBRPeANtEBLN + M5S9SG4PygAveOa2pQya+HfPXy03QCDJ1DKhq7JRQGXfegSFi3RA52EXOpMXwrubM8mB/ + LItJngcTvvKAGhipd7bTkMSSzXEOrnBBFLz6uol2pJNQtAHf//9Q1sI1PiV4Uzq91k4Yb + rFw9aOWKzlR3p6uxI/h+lOBdJhweb3IDkSMTbaWwNXn0RHVwBQJ8lwuXmyLNcl1VA7JFL + 6sHAsxpOY+KFOLee2SfH/5uoNZW9ElxAjdGxzvKo7a+xnwgctzadSzVSoXRODf3RTFqbY + AQfFdvKSs5s1ytan8O2orSm0OtAMSjlL45WO/WXOk8xh9NcHh3XnOdLHm++A0x5vRDx9S + XOd766NTksnmoHHWP27YvRXxxFA+EBjgLq+xCiKS2QlYtOp88aBO+0hVHDTcLE= diff --git a/Makefile b/Makefile index 3463beb..a76d705 100644 --- a/Makefile +++ b/Makefile @@ -20,10 +20,11 @@ RACK_SUFFIX ?= maas-rack RACK_IMG_DIR ?= images/maas-rack-controller CACHE_SUFFIX ?= maas-cache CACHE_IMG_DIR ?= images/sstream-cache -IMAGE_PREFIX ?= attcomdev -IMAGE_TAG ?= latest +IMAGE_PREFIX ?= airshipit +IMAGE_TAG ?= untagged PROXY ?= http://one.proxy.att.com:8080 USE_PROXY ?= false +PUSH_IMAGE ?= false LABEL ?= commit-id IMAGE_NAME := maas-rack-controller maas-region-controller sstream-cache BUILD_DIR := $(shell mktemp -d) @@ -67,6 +68,9 @@ ifeq ($(USE_PROXY), true) else docker build -t $(IMAGE) --label $(LABEL) -f $(IMAGE_DIR)/Dockerfile $(IMAGE_DIR) endif +ifeq ($(PUSH_IMAGE), true) + docker push $(IMAGE) +endif .PHONY: clean clean: diff --git a/tools/gate/playbooks/docker-image-build.yaml b/tools/gate/playbooks/docker-image-build.yaml new file mode 100644 index 0000000..7f4af61 --- /dev/null +++ b/tools/gate/playbooks/docker-image-build.yaml @@ -0,0 +1,96 @@ +# Copyright 2018 AT&T Intellectual Property. All other rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +- hosts: primary + tasks: + - name: Debug tag generation inputs + block: + - debug: + var: publish + - debug: + var: tags + - debug: + var: zuul + - debug: + msg: "{{ tags | to_json }}" + + - name: Determine tags + shell: echo '{{ tags | to_json }}' | python {{ zuul.project.src_dir }}/tools/image_tags.py + environment: + BRANCH: "{{ zuul.branch }}" + CHANGE: "{{ zuul.change }}" + COMMIT: "{{ zuul.newrev }}" + PATCHSET: "{{ zuul.patchset }}" + register: image_tags + + - name: Debug computed tags + debug: + var: image_tags + + - name: Install Docker (Debian) + block: + - apt: + name: "{{ item }}" + with_items: + - docker.io + - python-pip + when: ansible_os_family == 'Debian' + - pip: + name: docker + version: 2.7.0 + become: True + + - name: Make images + when: not publish + block: + - make: + chdir: "{{ zuul.project.src_dir }}" + target: images + params: + IMAGE_TAG: "{{ item }}" + with_items: "{{ image_tags.stdout_lines }}" + + - shell: "docker images" + register: docker_images + + - debug: + var: docker_images + + become: True + + - name: Publish images + block: + - docker_login: + username: "{{ airship_maas_quay_creds.username }}" + password: "{{ airship_maas_quay_creds.password }}" + registry_url: "https://quay.io/api/v1/" + + - make: + chdir: "{{ zuul.project.src_dir }}" + target: images + params: + DOCKER_REGISTRY: "quay.io" + IMAGE_PREFIX: "airshipit" + IMAGE_TAG: "{{ item }}" + PUSH_IMAGE: "true" + with_items: "{{ image_tags.stdout_lines }}" + + - shell: "docker images" + register: docker_images + + - debug: + var: docker_images + + when: publish + become: True diff --git a/tools/image_tags.py b/tools/image_tags.py new file mode 100644 index 0000000..9bdab59 --- /dev/null +++ b/tools/image_tags.py @@ -0,0 +1,126 @@ +#!/bin/python +# Copyright 2018 AT&T Intellectual Property. All other rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import json +import logging +import os +import sys + +LOG = logging.getLogger(__name__) + +LOG_FORMAT = '%(asctime)s %(levelname)-8s %(name)s:%(funcName)s [%(lineno)3d] %(message)s' # noqa + + +class TagGenExeception(Exception): + pass + + +def read_config(stream, env): + config = {} + try: + config['tags'] = json.load(stream) + except ValueError: + LOG.exception('Failed to decode JSON from input stream') + config['tags'] = {} + + LOG.debug('Configuration after reading stream: %s', config) + + config['context'] = { + 'branch': env.get('BRANCH'), + 'change': env.get('CHANGE'), + 'commit': env.get('COMMIT'), + 'ps': env.get('PATCHSET'), + } + + LOG.info('Final configuration: %s', config) + + return config + + +def build_tags(config): + tags = config.get('tags', {}).get('static', []) + LOG.debug('Dynamic tags: %s', tags) + tags.extend(build_dynamic_tags(config)) + LOG.info('All tags: %s', tags) + return tags + + +def build_dynamic_tags(config): + dynamic_tags = [] + + dynamic_tags.extend(_build_branch_tag(config)) + dynamic_tags.extend(_build_commit_tag(config)) + dynamic_tags.extend(_build_ps_tag(config)) + + return dynamic_tags + + +def _build_branch_tag(config): + if _valid_dg(config, 'branch'): + return [config['context']['branch']] + else: + return [] + + +def _build_commit_tag(config): + if _valid_dg(config, 'commit'): + return [config['context']['commit']] + else: + return [] + + +def _build_ps_tag(config): + if _valid_dg(config, 'patch_set', 'change') and _valid_dg( + config, 'patch_set', 'ps'): + return [ + '%s-%s' % (config['context']['change'], config['context']['ps']) + ] + else: + return [] + + +def _valid_dg(config, dynamic_tag, context_name=None): + if context_name is None: + context_name = dynamic_tag + + if config.get('tags', {}).get('dynamic', {}).get(dynamic_tag): + if config.get('context', {}).get(context_name): + return True + else: + raise TagGenExeception('Dynamic tag "%s" requested, but "%s"' + ' not found in context' % (dynamic_tag, + context_name)) + else: + return False + + +def main(): + config = read_config(sys.stdin, os.environ) + tags = build_tags(config) + + for tag in tags: + print(tag) + + +if __name__ == '__main__': + logging.basicConfig(format=LOG_FORMAT, level=logging.WARNING) + try: + main() + except TagGenExeception: + LOG.exception('Failed to generate tags') + sys.exit(1) + except Exception: + LOG.exception('Unexpected exception') + sys.exit(2)