Add support for multiple qcow bundles

Add support for multiple qcow bundles to image-builder.

Change-Id: I25b4d1fd47a382ba15587032515e65394f6efb3e

.gitignore

@@ -7,9 +7,9 @@ bootstrap_capz/go.sum
 bootstrap_capo/capo-ephemeral
 
 # image-builder artifacts to ignore
-image-builder/config/*.iso
-image-builder/config/*.qcow2
-image-builder/config/*.md5sum
+**.iso
+**.qcow2
+**.md5sum
 image-builder/assets/playbooks/roles/multistrap/vars/main.yaml
 image-builder/assets/playbooks/roles/livecdcontent/vars/main.yaml
 image-builder/assets/playbooks/roles/osconfig/vars/main.yaml

image-builder/Makefile

@@ -16,7 +16,6 @@ SHELL		:= /bin/bash
 COMMIT		?= $(shell git rev-parse HEAD)
 LABEL		?= org.airshipit.build=community
 IMAGE_NAME	?= image-builder
-QCOW_IMAGE_NAME ?= qcow-bundle
 DOCKER_REGISTRY	?= quay.io
 IMAGE_PREFIX	?= airshipit
 IMAGE_TAG	?= latest
@@ -24,8 +23,9 @@ IMAGE_TYPE	?= iso # iso | qcow
 PUSH_IMAGE	?= false
 DISTRO          ?= ubuntu_focal
 WORKDIR         ?= ./config
-# Specify if you want to specify qcow config directories manually instead of using default naming conventions
-QCOW_CONF_DIRS  ?=
+QCOW_BUNDLE     ?= ${WORKDIR}/qcow-bundle
+# Specify if you want to only build a certain subset of QCOW bundles
+QCOW_BUNDLE_DIRS  ?=
 # Set to true to skip multistrap.sh script. Useful for testing
 SKIP_MULTISTRAP ?=
 # Set to true to skip multistrap playbook. Useful for testing
@@ -35,7 +35,6 @@ SKIP_OSCONFIG_ROLE ?=
 # Set to true to skip livecdcontent playbook. Useful for testing
 SKIP_LIVECDCONTENT_ROLE ?=
 IMAGE           ?= ${DOCKER_REGISTRY}/${IMAGE_PREFIX}/${IMAGE_NAME}:${IMAGE_TAG}-${DISTRO}
-QCOW_IMAGE      ?= ${DOCKER_REGISTRY}/${IMAGE_PREFIX}/${QCOW_IMAGE_NAME}:${IMAGE_TAG}-${DISTRO}
 PROXY           ?=
 NO_PROXY        ?= localhost,127.0.0.1
 
@@ -109,23 +108,23 @@ ifneq ($(PROXY), )
 	export HTTPS_PROXY=$(PROXY)
 	export NO_PROXY=$(NO_PROXY)
 endif
-ifneq ($(EXPLICIT_DIRS), )
-	iterDirs="$(EXPLICIT_DIRS)"
+ifeq ($(IMAGE_TYPE), iso)
+	sudo -E tools/cut_image.sh $(IMAGE_TYPE) $(WORKDIR)/iso $(IMAGE) "$(PROXY)" "$(NO_PROXY)"
 else
-	# Assemble all images based on configs defined in each $(IMAGE_TYPE)* subdirectory
-	iterDirs=`find $(WORKDIR) -maxdepth 1 -name "$(IMAGE_TYPE)*" -type d -exec basename {} \;`
-endif
+	# Assemble all images based on configs defined in each subdirectory
+	iterDirs=`find $(QCOW_BUNDLE) -maxdepth 1 -mindepth 1 -type d -exec basename {} \;`
 	for subdir in $$iterDirs; do
 	    # ISO configs
-	    export user_data=$(WORKDIR)/$$subdir/user_data
-	    export network_config=$(WORKDIR)/$$subdir/network_data.json
+	    export user_data=$(QCOW_BUNDLE)/$$subdir/user_data
+	    export network_config=$(QCOW_BUNDLE)/$$subdir/network_data.json
 	    # QCOW configs
-	    export osconfig_params=$(WORKDIR)/$$subdir/osconfig-*-vars.yaml
-	    export qcow_params=$(WORKDIR)/$$subdir/qcow-*-vars.yaml
+	    export osconfig_params=$(QCOW_BUNDLE)/$$subdir/osconfig-*-vars.yaml
+	    export qcow_params=$(QCOW_BUNDLE)/$$subdir/qcow-*-vars.yaml
 	    # Shared configs
-	    export img_name=$$(cat $(WORKDIR)/$$subdir/img_name)
-	    sudo -E tools/cut_image.sh $(IMAGE_TYPE) $(WORKDIR) $(IMAGE) "$(PROXY)" "$(NO_PROXY)"
+	    export img_name=$$(cat $(QCOW_BUNDLE)/$$subdir/img_name)
+	    sudo -E tools/cut_image.sh $(IMAGE_TYPE) $(QCOW_BUNDLE) $(IMAGE) "$(PROXY)" "$(NO_PROXY)"
 	done
+endif
 
 generate_iso:
 	set -ex
@@ -135,17 +134,25 @@ generate_iso:
 package_qcow:
 	set -ex
 	export IMAGE_TYPE=qcow
-	export EXPLICIT_DIRS=$(QCOW_CONF_DIRS)
-	sudo -E make cut_image
-	sudo -E DOCKER_BUILDKIT=1 docker -D -l debug build --tag $(QCOW_IMAGE) -f Dockerfile-qcow.$(DISTRO) $(WORKDIR) \
-	--label $(LABEL) \
-	--label "org.opencontainers.image.revision=$(COMMIT)" \
-	--label "org.opencontainers.image.created=\
-	$(shell date --rfc-3339=seconds --utc)" \
-	--label "org.opencontainers.image.title=$(QCOW_IMAGE_NAME)"
-ifeq ($(PUSH_IMAGE), true)
-	sudo -E DOCKER_BUILDKIT=1 docker push $(QCOW_IMAGE)
+ifneq ($(QCOW_BUNDLE_DIRS), )
+	bundleDirs="$(QCOW_BUNDLE_DIRS)"
+else
+	# Assemble all images based on configs defined in each $(IMAGE_TYPE)* subdirectory
+	bundleDirs=`find $(WORKDIR) -maxdepth 1 -mindepth 1 -name "qcow-bundle*" -type d -exec basename {} \;`
 endif
+	for bundledir in $$bundleDirs; do
+	    export QCOW_BUNDLE="$(WORKDIR)/$$bundledir"
+	    sudo -E make cut_image
+	    sudo -E DOCKER_BUILDKIT=1 docker -D -l debug build --tag $$bundledir -f Dockerfile-qcow.$(DISTRO) $(WORKDIR)/$$bundledir \
+	    --label $(LABEL) \
+	    --label "org.opencontainers.image.revision=$(COMMIT)" \
+	    --label "org.opencontainers.image.created=\
+	    $(shell date --rfc-3339=seconds --utc)" \
+	    --label "org.opencontainers.image.title=$(DOCKER_REGISTRY)/$(IMAGE_PREFIX)/$$bundledir:$(IMAGE_TAG)-$(DISTRO)"
+ifeq ($(PUSH_IMAGE), true)
+	    sudo -E DOCKER_BUILDKIT=1 docker push $$bundledir
+endif
+	done
 
 tests:
 	true
@@ -153,6 +160,6 @@ tests:
 clean:
 	set -ex
 	sudo -E tools/multistrap.sh clean
-	if ls $(WORKDIR)/*.iso >& /dev/null; then sudo rm $(WORKDIR)/*.iso; fi
-	if ls $(WORKDIR)/*.qcow2 >& /dev/null; then sudo rm $(WORKDIR)/*.qcow2; fi
-	if ls $(WORKDIR)/*.md5sum >& /dev/null; then sudo rm $(WORKDIR)/*.md5sum; fi
+	find $(WORKDIR) -name "*.iso" -exec rm {} \; >& /dev/null
+	find $(WORKDIR) -name "*.qcow2" -exec rm {} \; >& /dev/null
+	find $(WORKDIR) -name "*.md5sum" -exec rm {} \; >& /dev/null

image-builder/assets/functions_v2.sh

@@ -96,7 +96,8 @@ _process_input_data_set_vars_osconfig(){
   OSCONFIG_FILE="${yaml_dir}/${OSCONFIG_FILE}"
   # Optional user-supplied playbook vars
   if [[ -f "${OSCONFIG_FILE}" ]]; then
-    cp "${OSCONFIG_FILE}" /opt/assets/playbooks/roles/osconfig/vars/main.yaml
+    echo "" >> /opt/assets/playbooks/roles/osconfig/vars/main.yaml
+    cat "${OSCONFIG_FILE}" >> /opt/assets/playbooks/roles/osconfig/vars/main.yaml
   fi
 }
 

image-builder/assets/playbooks/roles/multistrap/defaults/main.yaml

@@ -76,6 +76,7 @@ ubuntu_packages:
     - xz-utils
 unapproved_packages: # provide the exact name of the packages that need to be blocked
     - unattended-upgrades
+    - systemd-timesyncd
 repos:
     - register_repo_with_rootfs: true
       name: Ubuntu

image-builder/config/README.md

@@ -1,24 +1,37 @@
+Directory structure:
+
+|-- config
+  |-- iso
+    +-- network_data.json
+    +-- user_data
+  |-- qcow-bundle
+    |-- qcow-control-plane
+      +-- img_name
+      +-- osconfig-control-plane-vars.yaml
+      +-- qcow-control-plane-vars.yaml
+    |-- qcow-data-plane
+      +-- img_name
+      +-- osconfig-control-plane-vars.yaml
+      +-- qcow-control-plane-vars.yaml
+
 The `generate_iso` and `package_qcow` make target can be used to build ISO and
 QCOW artifacts respectively, after the shared `image-builder` container is
 built (built with the `build` target).
 
-By default, one image will be built for each subdirectory that matches the
-corresponding `IMAGE_TYPE` for the build.
+The ISO always builds out of the `config/iso` directory, because this is only
+used for local testing. It is not an artifact that is promoted or published.
 
-In other words, ISOs will be built using files from subdirs with names starting
-with `iso*`, while QCOWs are built from subdirs with names starting with
-`qcow*`. If you want to build QCOWs from an explicit list of dirs, you can
-supply them using the `QCOW_CONF_DIRS` parameter to the makefile.
+QCOWs are grouped into publishable "bundles", i.e. a container image where all
+QCOWs needed for a given deployment are stored. A bundle will be built for each
+`config/qcow-bundle*` directory. Each `config/qcow-bundle*` directory contains
+one subdirectory per QCOW that is part of that bundle, where overrides for
+those images can be placed.
 
-ISOs expect the following files to be present in their directory:
+The following items are expected in the `iso` directory:
 - `user_data` - YAML file containing cloud-init user-data
 - `network_data.json` - JSON file containing cloud-init network data
-- `img_name` - text file containing the desired name for the image
-
-Note that ISO generation here is *only* for testing. It is not published or
-promoted anywhere.
 
 QCOWs expect the following files to be present in their directory:
+- `img_name` - text file containing the desired name for the image
 - `osconfig-*-vars.yaml` - YAML file containing `osconfig` playbook overrides
 - `qcow-*-vars.yaml` - YAML file containing `qcow` playboook overrides
-- `img_name` - text file containing the desired name for the image

image-builder/config/qcow-bundle/README.md

@@ -0,0 +1,2 @@
+This folder represents a "QCOW bundle", i.e. a container image that will
+contain all of the target QCOW images needed for a particular deployment.

image-builder/tools/cut_image.sh

@@ -23,8 +23,8 @@ noproxy="$5"
 workdir="$(realpath ${host_mount_directory})"
 
 # Overrides
-: ${user_data:=$workdir/iso/user_data}
-: ${network_config:=$workdir/iso/network_data.json}
+: ${user_data:=$workdir/user_data}
+: ${network_config:=$workdir/network_data.json}
 : ${osconfig_params:=$workdir/qcow-control-plane/osconfig-control-plane-vars.yaml}
 : ${qcow_params:=$workdir/qcow-control-plane/qcow-control-plane-vars.yaml}
 
@@ -79,7 +79,6 @@ $(cat $network_config | sed 's/^/    /g')
 outputFileName: $img_name" > ${iso_config}
   sudo -E docker run -i --rm \
    --volume $workdir:/config \
-   --env BUILDER_CONFIG=/config/${build_type}.yaml \
    --env IMAGE_TYPE="iso" \
    --env VERSION="v2" \
    --env http_proxy=$proxy \