Add makefile target to publish QCOWs in container

Change-Id: Ife109f83558759c13423b297007a8fbc0c185cc7

.gitignore

@@ -7,6 +7,6 @@ bootstrap_capz/go.sum
 bootstrap_capo/capo-ephemeral
 
 # image-builder artifacts to ignore
-image-builder/examples/output-metadata.yaml
-image-builder/examples/*.iso
-image-builder/examples/*.qcow2
+image-builder/config/*.iso
+image-builder/config/*.qcow2
+image-builder/config/*.md5sum

image-builder/Dockerfile-qcow.ubuntu_focal

@@ -0,0 +1,15 @@
+FROM httpd:2.4
+
+ARG WORKDIR
+
+LABEL org.opencontainers.image.authors='airship-discuss@lists.airshipit.org, irc://#airshipit@freenode' \
+      org.opencontainers.image.url='https://airshipit.org' \
+      org.opencontainers.image.documentation='https://airship-images.readthedocs.org' \
+      org.opencontainers.image.source='https://opendev.org/airship/images' \
+      org.opencontainers.image.vendor='The Airship Authors' \
+      org.opencontainers.image.licenses='Apache-2.0'
+
+SHELL ["bash", "-exc"]
+
+COPY $WORKDIR/*.qcow2 /var/www/
+COPY $WORKDIR/*.md5sum /var/www/

image-builder/Makefile

@@ -16,18 +16,20 @@ SHELL		:= /bin/bash
 COMMIT		?= $(shell git rev-parse HEAD)
 LABEL		?= org.airshipit.build=community
 IMAGE_NAME	?= image-builder
+QCOW_IMAGE_NAME ?= qcow-bundle
 DOCKER_REGISTRY	?= quay.io
 IMAGE_PREFIX	?= airshipit
 IMAGE_TAG	?= latest
 IMAGE_TYPE	?= iso # iso | qcow
 PUSH_IMAGE	?= false
 DISTRO          ?= ubuntu_focal
+WORKDIR         ?= ./config
 IMAGE           ?= ${DOCKER_REGISTRY}/${IMAGE_PREFIX}/${IMAGE_NAME}:${IMAGE_TAG}-${DISTRO}
-IMAGE_ALIAS	?= $(DOCKER_REGISTRY)-$(IMAGE_NAME)-$(IMAGE_TAG)-${DISTRO}-${IMAGE_TYPE}
+QCOW_IMAGE      ?= ${DOCKER_REGISTRY}/${IMAGE_PREFIX}/${QCOW_IMAGE_NAME}:${IMAGE_TAG}-${DISTRO}
 PROXY           ?=
 NO_PROXY        ?= localhost,127.0.0.1
 
-.PHONY: help build images cut_image run clean
+.PHONY: help build images cut_image package_qcow run clean
 
 .ONESHELL:
 
@@ -35,7 +37,7 @@ help: ## This help.
 	@awk 'BEGIN {FS = ":.*##"; printf "\nUsage:\n  make \033[36m<target>\033[0m\n"} /^[a-zA-Z0-9_-]+:.*?##/ { printf "  \033[36m%-15s\033[0m %s\n", $$1, $$2 } /^##@/ { printf "\n\033[1m%s\033[0m\n", substr($$0, 5) } ' $(MAKEFILE_LIST)
 
 # Make target name that zuul expects for each project in this repo
-images: build
+images: build generate_iso package_qcow
 
 build:
 ifneq ($(PROXY), )
@@ -82,10 +84,31 @@ ifneq ($(PROXY), )
 	export HTTPS_PROXY=$(PROXY)
 	export NO_PROXY=$(NO_PROXY)
 endif
-	sudo -E tools/cut_image.sh $(IMAGE_TYPE) ./examples $(IMAGE) $(IMAGE_ALIAS) "$(PROXY)" "$(NO_PROXY)"
+	# Assemble all qcows based on configs defined in each $(IMAGE_TYPE)* subdirectory
+	for subdir in `find $(WORKDIR) -name "$(IMAGE_TYPE)*" -type d -exec basename {} \;`; do
+	    export osconfig_params=$(WORKDIR)/$$subdir/osconfig-*-vars.yaml
+	    export qcow_params=$(WORKDIR)/$$subdir/qcow-*-vars.yaml
+	    export img_name=$$(cat $(WORKDIR)/$$subdir/img_name)
+	    sudo -E tools/cut_image.sh $(IMAGE_TYPE) $(WORKDIR) $(IMAGE) "$(PROXY)" "$(NO_PROXY)" || exit 1
+	done
 
-run: ## Run the iso in kvm for testing
-	virsh start $(IMAGE_ALIAS)
+generate_iso:
+	export IMAGE_TYPE=iso
+	make cut_image
+
+package_qcow:
+	export IMAGE_TYPE=qcow
+	make cut_image
+	sudo -E docker -D -l debug build --tag $(QCOW_IMAGE) -f Dockerfile-qcow.$(DISTRO) . \
+	--label $(LABEL) \
+	--label "org.opencontainers.image.revision=$(COMMIT)" \
+	--label "org.opencontainers.image.created=\
+	$(shell date --rfc-3339=seconds --utc)" \
+	--label "org.opencontainers.image.title=$(QCOW_IMAGE_NAME)" \
+	--build-arg WORKDIR=$(WORKDIR) || exit 1
+ifeq ($(PUSH_IMAGE), true)
+	sudo -E docker push $(QCOW_IMAGE)
+endif
 
 tests:
 	true

image-builder/README.md

@@ -45,7 +45,7 @@ sudo make IMAGE_TYPE=qcow cut_image
 
 In the above example, set ``IMAGE_TYPE`` to ``iso`` or ``qcow`` as appropriate. This will be passed into the container to instruct it which type of image to build. Also include ``DOCKER_REGISTRY`` override if you wish to use a local docker image as described in the previous section.
 
-This makefile target uses config files provided in the images/image-builder/examples directory. Modify these files as needed in order to customize your iso and qcow generation.
+This makefile target uses config files provided in the images/image-builder/config directory. Modify these files as needed in order to customize your iso and qcow generation.
 
 # Building behind a proxy
 

image-builder/assets/entrypoint.sh

@@ -61,5 +61,8 @@ else
   exit 1
 fi
 
+# Write md5sum
+_make_metadata "${IMG_NAME}"
+
 echo "All Ansible plays completed successfully"
 

image-builder/assets/functions_v2.sh

@@ -84,6 +84,14 @@ fi
 
 OUTPUT_FILE_NAME="$(yq r ${yaml_dir}/builder_config outputFileName)"
 
+_make_metadata(){
+  IMG_NAME="$1"
+  # Write and print md5sum
+  md5sum=$(md5sum "${VOLUME}/${IMG_NAME}" | awk '{print $1}')
+  echo "md5sum:"
+  echo "$md5sum" | tee -a "${VOLUME}/${IMG_NAME}.md5sum"
+}
+
 _process_input_data_set_vars_osconfig(){
   OSCONFIG_FILE="${yaml_dir}/${OSCONFIG_FILE}"
   # Optional user-supplied playbook vars

image-builder/config/qcow-control-plane/README.md

@@ -0,0 +1,2 @@
+By default, the QCOW produced from this set of configs will be
+deployed to control-plane nodes in airshipctl deployment.

image-builder/config/qcow-control-plane/img_name

@@ -0,0 +1 @@
+control-plane.qcow2

image-builder/config/qcow-data-plane/README.md

@@ -0,0 +1,5 @@
+By default, the QCOW produced from this set of configs will be
+deployed to data-plane nodes in airshipctl deployment.
+
+Currently these are symlinks since there is no difference, but
+this can be changed if/when worker nodes use a different config.

image-builder/config/qcow-data-plane/img_name

@@ -0,0 +1 @@
+data-plane.qcow2

image-builder/config/qcow-data-plane/osconfig-control-plane-vars.yaml

@@ -0,0 +1 @@
+../qcow-control-plane/osconfig-control-plane-vars.yaml

image-builder/config/qcow-data-plane/qcow-control-plane-vars.yaml

@@ -0,0 +1 @@
+../qcow-control-plane/qcow-control-plane-vars.yaml

image-builder/examples/iso.yaml

@@ -1,15 +0,0 @@
-builder:
-  # Optional. Default imageType = iso
-  imageType: iso
-  # Required for imageType = iso
-  userDataFileName: user_data
-  # Required for imageType = iso
-  networkConfigFileName: network_data.json
-  # Optional. Default outputMetadataFilenName = output-metadata.yaml
-  outputMetadataFileName: output-metadata.yaml
-  # Optional. Default ISO name = ephemeral.iso
-  outputFileName: ephemeral.iso
-container:
-  volume: /host/path/to/examples:/config
-  image: port/image-builder:latest-ubuntu_focal
-  containerRuntime: docker

image-builder/examples/qcow-control-plane-vars-full-partitions.yaml

@@ -1,83 +0,0 @@
-qcow_capacity: 412G
-partitions:
-    # Partition numbering is according to list ordering.
-    # Ironic default cloud-init configdrive injection requires
-    # root partition to be the first numbered partition.
-    - mount: /
-      mount_order: 0
-      part_start: 1284MiB
-      part_end: 32GiB
-      filesystem:
-        type: ext4
-        fstab:
-          options: "defaults,errors=remount-ro,noatime"
-          dump: 0
-          fsck: 1
-    - mount: none
-      mount_order: 99
-      part_start: 1MiB
-      part_end: 5MiB
-      flags:
-        - bios_grub
-    - mount: /boot/efi
-      mount_order: 2
-      part_start: 5MiB
-      part_end: 516MiB
-      flags:
-        - esp
-      filesystem:
-        type: vfat
-        fstab:
-          options: "defaults,errors=remount-ro,noatime"
-          dump: 0
-          fsck: 1
-    - mount: /boot
-      mount_order: 1
-      part_start: 516MiB
-      part_end: 1284MiB
-      filesystem:
-        type: ext4
-        fstab:
-          options: "defaults,errors=remount-ro,noatime"
-          dump: 0
-          fsck: 2
-    - mount: /var
-      mount_order: 3
-      part_start: 32GiB
-      part_end: 232GiB
-      filesystem:
-        type: ext4
-        fstab:
-          options: "defaults,errors=remount-ro,noatime"
-          dump: 0
-          fsck: 1
-    - mount: /var/crash
-      mount_order: 4
-      part_start: 232GiB
-      part_end: 302GiB
-      filesystem:
-        type: ext4
-        fstab:
-          options: "defaults,errors=remount-ro,noatime"
-          dump: 0
-          fsck: 1
-    - mount: /var/lib/openstack-helm
-      mount_order: 5
-      part_start: 302GiB
-      part_end: 312GiB
-      filesystem:
-        type: ext4
-        fstab:
-          options: "defaults,errors=remount-ro,noatime"
-          dump: 0
-          fsck: 1
-    - mount: /var/log
-      mount_order: 6
-      part_start: 312GiB
-      part_end: '100%'
-      filesystem:
-        type: ext4
-        fstab:
-          options: "defaults,errors=remount-ro,noatime"
-          dump: 0
-          fsck: 1

image-builder/examples/qcow.yaml

@@ -1,15 +0,0 @@
-builder:
-  # Required for building 'qcow' over the 'iso' default
-  imageType: qcow
-  # Optional for imageType = qcow. Specify to override 'qcow' playbook default vars.
-  qcowVarsFileName: qcow-control-plane-vars.yaml
-  # Optional for imageType = qcow. Specify to override 'osconfig' playbook default vars.
-  osconfigVarsFileName: osconfig-control-plane-vars.yaml
-  # Optional. Default outputMetadataFileName = output-metadata.yaml
-  outputMetadataFileName: output-metadata.yaml
-  # Optional. When ommitted, defaults to playbook variable img_name
-  outputFileName: control-plane.qcow2
-container:
-  volume: /host/path/to/examples:/config
-  image: port/image-builder:latest-ubuntu_focal
-  containerRuntime: docker

image-builder/tools/cut_image.sh

@@ -12,16 +12,21 @@ BASEDIR="$( cd -P "$( dirname "$SOURCE" )" >/dev/null 2>&1 && pwd )"
 # Whether to build an 'iso' or 'qcow'
 build_type="${1:-qcow}"
 # The host mount to use to exchange data with this container
-host_mount_directory="${2:-$BASEDIR/../examples}"
+host_mount_directory="${2:-$BASEDIR/../config}"
 # Docker image to use when launching this container
 image="${3:-port/image-builder:latest-ubuntu_focal}"
-# Libvirt instance name to use for a new libvirt XML definition that
-# will be created to reference the newly created ISO or QCOW2 image.
-img_alias="${4:-port-image-builder-latest-ubuntu_focal-$build_type}"
 # proxy to use, if applicable
-proxy="$5"
+proxy="$4"
 # noproxy to use, if applicable
-noproxy="$6"
+noproxy="$5"
+
+workdir="$(realpath ${host_mount_directory})"
+
+# Overrides
+: ${user_data:=$workdir/iso/user_data}
+: ${network_config:=$workdir/iso/network_data.json}
+: ${osconfig_params:=$workdir/control-plane/osconfig-control-plane-vars.yaml}
+: ${qcow_params:=$workdir/control-plane/qcow-control-plane-vars.yaml}
 
 if [ -n "$proxy" ]; then
   export http_proxy=$proxy
@@ -54,15 +59,14 @@ if [ -d /sys/firmware/efi ]; then
   uefi_boot_arg='--boot uefi'
 fi
 
-workdir="$(realpath ${host_mount_directory})"
-
 if [[ $build_type = iso ]]; then
-  iso_config=/tmp/iso_config
+  : ${img_name:=ephemeral.iso}
+  iso_config=/tmp/${img_name}_config
   echo "user_data:
-$(cat $host_mount_directory/user_data | sed 's/^/    /g')
+$(cat $user_data | sed 's/^/    /g')
 network_config:
-$(cat $host_mount_directory/network_data.json | sed 's/^/    /g')
-outputFileName: ephemeral.iso" > ${iso_config}
+$(cat $network_config | sed 's/^/    /g')
+outputFileName: $img_name" > ${iso_config}
   sudo -E docker run -i --rm \
    --volume $workdir:/config \
    --env BUILDER_CONFIG=/config/${build_type}.yaml \
@@ -75,15 +79,16 @@ outputFileName: ephemeral.iso" > ${iso_config}
    --env no_proxy=$noproxy \
    --env NO_PROXY=$noproxy \
    ${image} < ${iso_config}
-  disk1="--disk path=${workdir}/ephemeral.iso,device=cdrom"
+  disk1="--disk path=${workdir}/${img_name},device=cdrom"
 elif [[ $build_type == qcow ]]; then
+  : ${img_name:=airship-ubuntu.qcow2}
   sudo -E modprobe nbd
-  qcow_config=/tmp/qcow_config
+  qcow_config=/tmp/${img_name}_config
   echo "osconfig:
-$(cat $host_mount_directory/osconfig-control-plane-vars.yaml | sed 's/^/    /g')
+$(cat $osconfig_params | sed 's/^/    /g')
 qcow:
-$(cat $host_mount_directory/qcow-control-plane-vars.yaml | sed 's/^/    /g')
-outputFileName: control-plane.qcow2" > ${qcow_config}
+$(cat $qcow_params | sed 's/^/    /g')
+outputFileName: $img_name" > ${qcow_config}
   echo "Note: This step can be slow if you don't have an SSD."
   sudo -E docker run -i --rm \
    --privileged \
@@ -106,7 +111,7 @@ outputFileName: control-plane.qcow2" > ${qcow_config}
    ${image} < ${qcow_config}
   cloud_init_config_dir='assets/tests/qcow/cloud-init'
   sudo -E cloud-localds -v --network-config="${cloud_init_config_dir}/network-config" "${workdir}/airship-ubuntu_config.iso" "${cloud_init_config_dir}/user-data" "${cloud_init_config_dir}/meta-data"
-  disk1="--disk path=${workdir}/control-plane.qcow2"
+  disk1="--disk path=${workdir}/${img_name}"
   disk2="--disk path=${workdir}/airship-ubuntu_config.iso,device=cdrom"
 else
   echo Unknown build type: $build_type, exiting.
@@ -116,20 +121,20 @@ fi
 imagePath=$(echo $disk1 | cut -d'=' -f2 | cut -d',' -f1)
 echo Image successfully written to $imagePath
 
-sudo -E virsh destroy ${img_alias} 2> /dev/null || true
-sudo -E virsh undefine ${img_alias} --nvram 2> /dev/null || true
+sudo -E virsh destroy ${img_name} 2> /dev/null || true
+sudo -E virsh undefine ${img_name} --nvram 2> /dev/null || true
 
 cpu_type=''
 kvm-ok >& /dev/null && cpu_type='--cpu host-passthrough' || true
 
-network='--network network=default,mac=52:54:00:6c:99:85'
+network='--network network=default'
 if ! sudo -E virsh net-list | grep default | grep active > /dev/null; then
   network='--network none'
 fi
 
 xml=$(mktemp)
 sudo -E virt-install --connect qemu:///system \
- --name ${img_alias} \
+ --name ${img_name} \
  --memory 1536 \
  ${network} \
  ${cpu_type} \