From 1f7138684dd4284139a1dfd618377e7e6c6cd6cd Mon Sep 17 00:00:00 2001 From: Walter Wahlstedt Date: Mon, 4 Oct 2021 13:25:14 -0400 Subject: [PATCH] Reduce number of jobs needed to tag images Change-Id: Ie21cfa4ac909f0a69913f876a8659d9bb495df76 --- image-builder/Makefile | 171 +++++++++++------- image-builder/tools/cut_image.sh | 1 - playbooks/airship-image-builder-build.yaml | 7 +- .../airship-image-builder-publish-latest.yaml | 8 +- zuul.d/jobs.yaml | 53 +++--- zuul.d/projects.yaml | 6 +- 6 files changed, 142 insertions(+), 104 deletions(-) diff --git a/image-builder/Makefile b/image-builder/Makefile index 047ccdb..8e62196 100644 --- a/image-builder/Makefile +++ b/image-builder/Makefile @@ -12,39 +12,43 @@ # See the License for the specific language governing permissions and # limitations under the License. -SHELL := /bin/bash -COMMIT ?= $(shell git rev-parse HEAD) -LABEL ?= org.airshipit.build=community -IMAGE_NAME ?= image-builder -DOCKER_REGISTRY ?= quay.io -IMAGE_PREFIX ?= airshipit -IMAGE_TAG ?= latest -IMAGE_TYPE ?= iso # iso | qcow -PUSH_IMAGE ?= false -DISTRO ?= ubuntu_focal -WORKDIR ?= ./manifests +SHELL := /bin/bash +COMMIT ?= $(shell git rev-parse HEAD) +LABEL ?= org.airshipit.build=community +IMAGE_NAME ?= image-builder +DOCKER_REGISTRY ?= quay.io +IMAGE_PREFIX ?= airshipit +IMAGE_TAG ?= +IMAGE_TAG_FIRST ?= +IMAGE_TYPE ?= iso # iso | qcow +PUSH_IMAGE ?= false +DISTRO ?= ubuntu_focal +WORKDIR ?= ./manifests # Specifiy if you want to use a different profile than the default. # i.e. PROFILE ?= k8s-1.18 -PROFILE ?= -QCOW_BUNDLE ?= ${WORKDIR}/qcow-bundle +PROFILE ?= +QCOW_BUNDLE ?= ${WORKDIR}/qcow-bundle # Specify if you want to only build a certain subset of QCOW bundles -QCOW_BUNDLE_DIRS ?= +QCOW_BUNDLE_DIRS ?= # Set to true to skip multistrap.sh script. Useful for testing -SKIP_MULTISTRAP ?= +SKIP_MULTISTRAP ?= # Set to true to skip multistrap playbook. Useful for testing -SKIP_MULTI_ROLE ?= +SKIP_MULTI_ROLE ?= # Set to true to skip osconfig playbook. Useful for testing -SKIP_OSCONFIG_ROLE ?= +SKIP_OSCONFIG_ROLE ?= # Set to true to skip livecdcontent playbook. Useful for testing -SKIP_LIVECDCONTENT_ROLE ?= -IMAGE ?= ${DOCKER_REGISTRY}/${IMAGE_PREFIX}/${IMAGE_NAME}:${IMAGE_TAG}-${DISTRO} -PROXY ?= -NO_PROXY ?= localhost,127.0.0.1 -BOOT_TIMEOUT ?= 300 # Image boot validation timeout. Set to 0 to disable to make console available for manual debugging. +SKIP_LIVECDCONTENT_ROLE ?= +IMAGE ?= ${DOCKER_REGISTRY}/${IMAGE_PREFIX}/${IMAGE_NAME}:${IMAGE_TAG}-${DISTRO} +TARGET_IMAGE_TAG ?= +PROXY ?= +NO_PROXY ?= localhost,127.0.0.1 +# Image boot validation timeout. Set to 0 to disable to make console available for manual debugging. +BOOT_TIMEOUT ?= 300 #it doesn't matter - we're not going to publish it, because it's useless without jsons -KRM_BASE_IMAGE ?= ${DOCKER_REGISTRY}/${IMAGE_PREFIX}/image-profile-krm:${IMAGE_TAG} +KRM_IMAGE_TAG ?= latest +KRM_BASE_IMAGE ?= ${DOCKER_REGISTRY}/${IMAGE_PREFIX}/image-profile-krm:${KRM_IMAGE_TAG} -.PHONY: help build images cut_image package_qcow krm_base_image run clean +.PHONY: help build images cut_image package_qcow krm_base_image run clean docker_build tag push .ONESHELL: @@ -55,17 +59,21 @@ help: ## This help. images: build generate_iso package_qcow clean build: -ifneq ($(PROFILE), ) set -ex +ifneq ($(PROFILE), ) # Apply any user-defined profiles overrides to playbooks rsync -rc ./../profiles/$(PROFILE)/manifests/ $(WORKDIR)/ endif - set -ex # Apply any user-defined rootfs overrides to playbooks cp $(WORKDIR)/rootfs/multistrap-vars.yaml assets/playbooks/roles/multistrap/vars/main.yaml cp $(WORKDIR)/rootfs/osconfig-vars.yaml assets/playbooks/roles/osconfig/vars/main.yaml cp $(WORKDIR)/rootfs/livecdcontent-vars.yaml assets/playbooks/roles/livecdcontent/vars/main.yaml + export DOCKERFILE=Dockerfile.$(DISTRO) + export DOCKERFILE_FOLDER="." + export IMAGE_NAME=image-builder + export SOURCE_IMAGE_TAG=${DOCKER_REGISTRY}/${IMAGE_PREFIX}/${IMAGE_NAME}:${IMAGE_TAG}-${DISTRO} +ifneq (,$(findstring latest, $(IMAGE_TAG))) ifneq ($(PROXY), ) sudo -E ./tools/docker_proxy.sh $(PROXY) $(NO_PROXY) export http_proxy=$(PROXY) @@ -77,40 +85,34 @@ ifneq ($(PROXY), ) ifneq ($(SKIP_MULTISTRAP), true) sudo -E ./tools/multistrap.sh $(WORKDIR) endif - sudo -E DOCKER_BUILDKIT=1 docker -D -l debug build --tag $(IMAGE) -f Dockerfile.$(DISTRO) . \ - --label $(LABEL) \ - --label "org.opencontainers.image.revision=$(COMMIT)" \ - --label "org.opencontainers.image.created=\ - $(shell date --rfc-3339=seconds --utc)" \ - --label "org.opencontainers.image.title=$(IMAGE_NAME)" \ - --build-arg http_proxy=$(PROXY) \ - --build-arg https_proxy=$(PROXY) \ - --build-arg HTTP_PROXY=$(PROXY) \ - --build-arg HTTPS_PROXY=$(PROXY) \ - --build-arg no_proxy=$(NO_PROXY) \ - --build-arg NO_PROXY=$(NO_PROXY) +# run docker build for w/proxy + sudo -E make docker_build else ifneq ($(SKIP_MULTISTRAP), true) sudo -E ./tools/multistrap.sh $(WORKDIR) endif - sudo -E DOCKER_BUILDKIT=1 docker -D -l debug build --tag $(IMAGE) -f Dockerfile.$(DISTRO) . \ - --label $(LABEL) \ - --label "org.opencontainers.image.revision=$(COMMIT)" \ - --label "org.opencontainers.image.created=\ - $(shell date --rfc-3339=seconds --utc)" \ - --label "org.opencontainers.image.title=$(IMAGE_NAME)" +# run docker build for wo/proxy + sudo -E make docker_build endif imgId=`sudo docker images | grep 'image-builder ' | awk '{print $$3}'` time sudo -E DOCKER_BUILDKIT=1 docker run $$imgId ls -ltra /build/usr/bin/sudo > /tmp/sticky_result time sudo grep '^-rws' /tmp/sticky_result >& /dev/null || \ (echo Could not find sticky bit set on target image sudo binary. Are you using buildkit? && \ sudo cat /tmp/sticky_result && exit 1) +else +# Set SOURCE_IMAGE_TAG and TARGET_IMAGE_TAG for tagging purposes + export SOURCE_IMAGE_TAG="${DOCKER_REGISTRY}/${IMAGE_PREFIX}/${IMAGE_NAME}:${IMAGE_TAG_FIRST}-${DISTRO}" + export TARGET_IMAGE_TAG="${DOCKER_REGISTRY}/${IMAGE_PREFIX}/${IMAGE_NAME}:${IMAGE_TAG}-${DISTRO}" + # Don't repackage image-builder, just tag it. + sudo -E make tag +endif ifeq ($(PUSH_IMAGE), true) - sudo -E DOCKER_BUILDKIT=1 docker push $(IMAGE) + sudo -E make push endif cut_image: krm_base_image set -ex + IMAGE_NAME=image-builder ifneq ($(PROXY), ) sudo -E ./tools/docker_proxy.sh $(PROXY) $(NO_PROXY) export http_proxy=$(PROXY) @@ -127,16 +129,18 @@ else # Trailing / allows proper function with symlinks iterDirs="$$(find $(QCOW_BUNDLE)/ -maxdepth 1 -mindepth 1 -type d -exec basename {} \;)" if [[ -z $$iterDirs ]]; then - echo "Could not find any qcow images defined for bundle - exiting." - exit 1 + echo "Could not find any qcow images defined for bundle - exiting." + exit 1 fi for subdir in $$iterDirs; do - # QCOW configs - export osconfig_params="$(QCOW_BUNDLE)/$$subdir/osconfig-vars.yaml" - export qcow_params="$(QCOW_BUNDLE)/$$subdir/qcow-vars.yaml" - # Image name - export img_name=$$subdir.qcow2 - sudo -E tools/cut_image.sh $(IMAGE_TYPE) $(QCOW_BUNDLE) $(IMAGE) $(BOOT_TIMEOUT) "$(PROXY)" "$(NO_PROXY)" + # QCOW configs + export osconfig_params="$(QCOW_BUNDLE)/$$subdir/osconfig-vars.yaml" + export qcow_params="$(QCOW_BUNDLE)/$$subdir/qcow-vars.yaml" + # Image name + export img_name=$$subdir.qcow2 + echo "Image: $(IMAGE)" + echo "SOURCE_IMAGE_TAG: $(SOURCE_IMAGE_TAG)" + sudo -E tools/cut_image.sh $(IMAGE_TYPE) $(QCOW_BUNDLE) $(IMAGE) $(BOOT_TIMEOUT) "$(PROXY)" "$(NO_PROXY)" # keep profile info [ -e $(QCOW_BUNDLE)/profile_entrypoint.json ] && cat $(QCOW_BUNDLE)/profile_entrypoint.json && mv $(QCOW_BUNDLE)/profile_entrypoint.json $(QCOW_BUNDLE)/profile_qcow_$$subdir.json done @@ -145,6 +149,7 @@ endif generate_iso: set -ex export IMAGE_TYPE=iso + export IMAGE_NAME=image-builder sudo -E make cut_image package_qcow: @@ -162,23 +167,63 @@ endif exit 1 fi for bundledir in $$bundleDirs; do - export QCOW_BUNDLE="$(WORKDIR)/$$bundledir" - sudo -E make cut_image - sudo -E DOCKER_BUILDKIT=1 docker -D -l debug build --tag $(DOCKER_REGISTRY)/$(IMAGE_PREFIX)/$$bundledir:$(IMAGE_TAG)-$(DISTRO) -f Dockerfile-qcow.$(DISTRO) $(WORKDIR)/$$bundledir \ - --label $(LABEL) \ - --label "org.opencontainers.image.revision=$(COMMIT)" \ - --label "org.opencontainers.image.created=\ - $(shell date --rfc-3339=seconds --utc)" \ - --label "org.opencontainers.image.title=$(DOCKER_REGISTRY)/$(IMAGE_PREFIX)/$$bundledir:$(IMAGE_TAG)-$(DISTRO)" \ - --build-arg KRM_BASE_IMAGE=$(KRM_BASE_IMAGE) +# if IMAGE_TAG contains latest then docker_build else its a tag +ifneq (,$(findstring latest, $(IMAGE_TAG))) +# Set vars for docker_build and push + export QCOW_BUNDLE="$(WORKDIR)/$$bundledir" + export IMAGE_NAME=image-builder + sudo -E make cut_image + export DOCKERFILE="Dockerfile-qcow.$(DISTRO)" + export DOCKERFILE_FOLDER="$(WORKDIR)/$$bundledir" + export IMAGE_NAME=$$bundledir + export SOURCE_IMAGE_TAG=${DOCKER_REGISTRY}/${IMAGE_PREFIX}/$$bundledir:${IMAGE_TAG}-${DISTRO} + sudo -E make docker_build +else +# Set SOURCE_IMAGE_TAG & TARGET_IMAGE_TAG for tagging purposes + export SOURCE_IMAGE_TAG=${DOCKER_REGISTRY}/${IMAGE_PREFIX}/$$bundledir:${IMAGE_TAG_FIRST}-${DISTRO} + export TARGET_IMAGE_TAG="${DOCKER_REGISTRY}/${IMAGE_PREFIX}/$$bundledir:${IMAGE_TAG}-${DISTRO}" + # Don't repackage qcow, just tag it. + sudo -E make tag +endif ifeq ($(PUSH_IMAGE), true) - sudo -E DOCKER_BUILDKIT=1 docker push $(DOCKER_REGISTRY)/$(IMAGE_PREFIX)/$$bundledir:$(IMAGE_TAG)-$(DISTRO) + sudo -E make push endif done krm_base_image: sudo -E docker build krm-function -t $(KRM_BASE_IMAGE) +docker_build: + set -ex +ifneq ($(PROXY), ) + sudo -E DOCKER_BUILDKIT=1 docker -D -l debug build --tag $(SOURCE_IMAGE_TAG) -f $(DOCKERFILE) $(DOCKERFILE_FOLDER) \ + --label $(LABEL) \ + --label "org.opencontainers.image.revision=$(COMMIT)" \ + --label "org.opencontainers.image.created=$(shell date --rfc-3339=seconds --utc)" \ + --label "org.opencontainers.image.title=$(IMAGE_NAME)" \ + --build-arg http_proxy=$(PROXY) \ + --build-arg https_proxy=$(PROXY) \ + --build-arg HTTP_PROXY=$(PROXY) \ + --build-arg HTTPS_PROXY=$(PROXY) \ + --build-arg no_proxy=$(NO_PROXY) \ + --build-arg NO_PROXY=$(NO_PROXY) \ + --build-arg KRM_BASE_IMAGE=$(KRM_BASE_IMAGE) +else + sudo -E DOCKER_BUILDKIT=1 docker -D -l debug build --tag $(SOURCE_IMAGE_TAG) -f $(DOCKERFILE) $(DOCKERFILE_FOLDER) \ + --label $(LABEL) \ + --label "org.opencontainers.image.revision=$(COMMIT)" \ + --label "org.opencontainers.image.created=$(shell date --rfc-3339=seconds --utc)" \ + --label "org.opencontainers.image.title=$(IMAGE_NAME)" +endif + +tag: + set -ex + sudo -E DOCKER_BUILDKIT=1 docker tag $(SOURCE_IMAGE_TAG) $(TARGET_IMAGE_TAG) + +push: + set -ex + sudo -E DOCKER_BUILDKIT=1 docker push $(SOURCE_IMAGE_TAG) $(TARGET_IMAGE_TAG) + tests: true diff --git a/image-builder/tools/cut_image.sh b/image-builder/tools/cut_image.sh index e03bd6d..7ddf124 100755 --- a/image-builder/tools/cut_image.sh +++ b/image-builder/tools/cut_image.sh @@ -229,4 +229,3 @@ if [[ $perform_boot_test = "true" ]]; then exit 1 fi fi - diff --git a/playbooks/airship-image-builder-build.yaml b/playbooks/airship-image-builder-build.yaml index b081185..4c0e33f 100644 --- a/playbooks/airship-image-builder-build.yaml +++ b/playbooks/airship-image-builder-build.yaml @@ -38,7 +38,8 @@ when: pre_build_role is defined - name: Publish images block: - - command: make images clean + - name: Publish Images with Latest and Commit Tags + command: make images args: chdir: "{{ makefile_chdir }}" environment: @@ -47,7 +48,8 @@ DOCKER_REGISTRY: "{{ docker_registry }}" GCP_SDK: "{{ gcp_sdk }}" IMAGE_PREFIX: "{{ image_prefix | default('airshipit') }}" - IMAGE_TAG: "{{ image_tag | default('latest') }}" + IMAGE_TAG_FIRST: "{{ image_tag | first | default('latest') }}" + IMAGE_TAG: "{{ item }}" LABEL: "{{ image_label | default('org.airshipit.build=community') }}" NO_PROXY: "{{ proxy.noproxy }}" PUSH_IMAGE: "false" @@ -56,4 +58,5 @@ USE_PROXY: "{{ proxy.enabled | lower }}" WORKDIR: "{{ image_work_dir | default('manifests') }}" PROFILE: "{{ image_profile_dir | default('') }}" + loop: "{{ image_tag|flatten(levels=1) }}" become: True diff --git a/playbooks/airship-image-builder-publish-latest.yaml b/playbooks/airship-image-builder-publish-latest.yaml index 2e6c74c..271f030 100644 --- a/playbooks/airship-image-builder-publish-latest.yaml +++ b/playbooks/airship-image-builder-publish-latest.yaml @@ -43,8 +43,8 @@ username: "{{ image_builder_quay_creds.username }}" password: "{{ image_builder_quay_creds.password }}" registry_url: "{{ docker_registry }}" - - name: Push Images with Latest and Commit Tags - command: make images clean + - name: Publish Images with Latest and Commit Tags + command: make images args: chdir: "{{ makefile_chdir }}" environment: @@ -53,7 +53,8 @@ DOCKER_REGISTRY: "{{ docker_registry }}" GCP_SDK: "{{ gcp_sdk }}" IMAGE_PREFIX: "{{ image_prefix | default('airshipit') }}" - IMAGE_TAG: "{{ image_tag | default('latest') }}" + IMAGE_TAG_FIRST: "{{ image_tag | first | default('latest') }}" + IMAGE_TAG: "{{ item }}" LABEL: "{{ image_label | default('org.airshipit.build=community') }}" NO_PROXY: "{{ proxy.noproxy }}" PUSH_IMAGE: "true" @@ -62,4 +63,5 @@ USE_PROXY: "{{ proxy.enabled | lower }}" WORKDIR: "{{ image_work_dir | default('manifests') }}" PROFILE: "{{ image_profile_dir | default('') }}" + loop: "{{ image_tag|flatten(levels=1) }}" become: True diff --git a/zuul.d/jobs.yaml b/zuul.d/jobs.yaml index e503b48..99f2ac9 100644 --- a/zuul.d/jobs.yaml +++ b/zuul.d/jobs.yaml @@ -9,7 +9,7 @@ secret: airship_image_builder_airshipit_github_key pass-to-parent: true -# Run the default build job 1.18 +# Run the build job 1.18 - job: name: airship-image-builder-build-k8s-1.18 nodeset: airship-image-builder-single-node @@ -20,7 +20,9 @@ post-run: playbooks/airship-image-builder-collect-logs.yaml vars: image_profile_dir: k8s-1.18 - + image_tag: + - "k8s-1.18-latest" + - "k8s-1.18-FAKE-SHA-TEST" # Run the build job 1.19 - job: name: airship-image-builder-build-k8s-1.19 @@ -32,6 +34,9 @@ post-run: playbooks/airship-image-builder-collect-logs.yaml vars: image_profile_dir: k8s-1.19 + image_tag: + - "k8s-1.19-latest" + - "k8s-1.19-FAKE-SHA-TEST" # Run the build job 1.21 - job: @@ -44,8 +49,11 @@ post-run: playbooks/airship-image-builder-collect-logs.yaml vars: image_profile_dir: k8s-1.21 + image_tag: + - "k8s-1.21-latest" + - "k8s-1.21-FAKE-SHA-TEST" -# Tag default job with 1.18 +# Publish Job 1.18 - job: name: airship-image-builder-publish-k8s-1.18 parent: airship-image-builder-build-k8s-1.18 @@ -58,9 +66,11 @@ - name: image_builder_quay_creds secret: image_builder_quay_creds vars: - image_tag: "k8s-1.18-latest" + image_tag: + - "k8s-1.18-latest" + - "k8s-1.18-{{ zuul.newrev }}" -# Tag the 1.19 job with 1.19 +# Publish Job 1.19 - job: name: airship-image-builder-publish-k8s-1.19 parent: airship-image-builder-build-k8s-1.19 @@ -73,9 +83,11 @@ - name: image_builder_quay_creds secret: image_builder_quay_creds vars: - image_tag: "k8s-1.19-latest" + image_tag: + - "k8s-1.19-latest" + - "k8s-1.19-{{ zuul.newrev }}" -# Tag the 1.21 job with 1.21 +# Publish Job 1.21 - job: name: airship-image-builder-publish-k8s-1.21 parent: airship-image-builder-build-k8s-1.21 @@ -88,31 +100,12 @@ - name: image_builder_quay_creds secret: image_builder_quay_creds vars: - image_tag: "k8s-1.21-latest" - -# Tag the 1.18 job with zuul rev tag -- job: - name: airship-image-builder-publish-commit-k8s-1.18 - parent: airship-image-builder-publish-k8s-1.18 - vars: - image_tag: "k8s-1.18-{{ zuul.newrev }}" - -# Tag the 1.19 job with zuul rev tag -- job: - name: airship-image-builder-publish-commit-k8s-1.19 - parent: airship-image-builder-publish-k8s-1.19 - vars: - image_tag: "k8s-1.19-{{ zuul.newrev }}" + image_tag: + - "k8s-1.21-latest" + - "k8s-1.21-{{ zuul.newrev }}" - nodeset: name: airship-image-builder-single-node nodes: - name: primary - label: ubuntu-bionic-32GB - -# Tag the 1.21 job with zuul rev tag -- job: - name: airship-image-builder-publish-commit-k8s-1.21 - parent: airship-image-builder-publish-k8s-1.21 - vars: - image_tag: "k8s-1.21-{{ zuul.newrev }}" + label: ubuntu-bionic-32GB \ No newline at end of file diff --git a/zuul.d/projects.yaml b/zuul.d/projects.yaml index 5a23c7d..d7ef0da 100644 --- a/zuul.d/projects.yaml +++ b/zuul.d/projects.yaml @@ -12,7 +12,7 @@ image_config_dir: manifests image_prefix: airshipit image_label: org.airshipit.build=community - image_tag: latest + image_tag: [latest] check: jobs: - airship-image-builder-build-k8s-1.18 @@ -23,16 +23,12 @@ - airship-image-builder-build-k8s-1.18 - airship-image-builder-build-k8s-1.19 - airship-image-builder-build-k8s-1.21 - post: jobs: - airship-image-builder-upload-git-mirror - airship-image-builder-publish-k8s-1.18 - airship-image-builder-publish-k8s-1.19 - - airship-image-builder-publish-commit-k8s-1.18 - - airship-image-builder-publish-commit-k8s-1.19 - airship-image-builder-publish-k8s-1.21 - - airship-image-builder-publish-commit-k8s-1.21 tag: jobs: - airship-image-builder-upload-git-mirror \ No newline at end of file