Read inputs from stdin

Airshipctl implementation will provide container inputs via stdin. This
patchset allows the image-builder container to accept inputs of this
kind. It maintains backwards compatibility with previous verison until
it is no longer used by airshipctl.

Add control to enable or disable qcow compression.

Add wget package to image defaults.

Change-Id: I835c183be7ef6e06f2e3550de2a726df6f9d0e3f

.zuul.yaml

@@ -42,7 +42,7 @@
 
 - job:
     name: airship-images-build
-    nodeset: airship-images-single-node
+    nodeset: airship-images-single-node-32GB
     timeout: 7200
     post-timeout: 7200
     pre-run: playbooks/airship-images-deploy-docker.yaml
@@ -51,7 +51,7 @@
 
 - job:
     name: airship-images-publish
-    nodeset: airship-images-single-node
+    nodeset: airship-images-single-node-32GB
     timeout: 7200
     post-timeout: 7200
     pre-run: playbooks/airship-images-deploy-docker.yaml
@@ -73,6 +73,12 @@
 
 - nodeset:
     name: airship-images-single-node
+    nodes:
+      - name: primary
+        label: ubuntu-bionic
+
+- nodeset:
+    name: airship-images-single-node-32GB
     nodes:
       - name: primary
         label: ubuntu-bionic-32GB

image-builder/assets/entrypoint.sh

@@ -1,5 +1,5 @@
 #!/bin/bash
-set -ex
+set -e
 
 SOURCE="${BASH_SOURCE[0]}"
 while [ -h "$SOURCE" ]; do # resolve $SOURCE until the file is no longer a symlink
@@ -11,7 +11,11 @@ BASEDIR="$( cd -P "$( dirname "$SOURCE" )" >/dev/null 2>&1 && pwd )"
 cd "$BASEDIR"
 
 BASEDIR="$(dirname "$(realpath "$0")")"
-source "${BASEDIR}/functions.sh"
+if [ "${VERSION}" = "v2" ]; then
+  source "${BASEDIR}/functions_v2.sh"
+else
+  source "${BASEDIR}/functions.sh"
+fi
 
 export http_proxy
 export https_proxy
@@ -57,8 +61,5 @@ else
   exit 1
 fi
 
-# Write metadata output file containing host path to image and md5sum
-_make_metadata "${IMG_NAME}"
-
 echo "All Ansible plays completed successfully"
 

image-builder/assets/functions.sh

@@ -1,4 +1,8 @@
 #!/bin/bash
+# NOTE: These functions are deprecated. It is only left here
+# for backwards compatibility until airshipctl is migrated
+# away from using it.
+set -x
 
 # Defaults
 OUTPUT_METADATA_FILE_NAME_DEFAULT='output-metadata.yaml'

image-builder/assets/functions_v2.sh

@@ -0,0 +1,149 @@
+#!/bin/bash
+
+# Defaults
+ISO_NAME_DEFAULT='ephemeral.iso'
+
+_validate_param(){
+  PARAM_VAL="$1"
+  PARAM_NAME="$2"
+  # Validate that a paramter is defined (default) or that
+  # it is defined and represents the path of a file or
+  # directory that is found on the filesystem (VAL_TYPE=file)
+  VAL_TYPE="$3"
+  NO_NULL_EXIT="$4"
+  echo "${PARAM_VAL:?}"
+  # yq will return the 'null' string if a key is either undefined or defined with no value
+  if [[ "${PARAM_VAL}" =~ null$ ]]
+  then
+    echo "variable ${PARAM_NAME} is not present in user-supplied config."
+    if [[ "${NO_NULL_EXIT}" == 'no_null_exit' ]]; then
+      echo "Using defaults"
+    else
+      exit 1
+    fi
+  else
+    if [[ ${VAL_TYPE} == 'file' ]]; then
+      if [[ ! -e "${PARAM_VAL}" ]]
+      then
+        echo "${PARAM_VAL} not exist"
+        exit 1
+      fi
+    fi
+  fi
+}
+
+# Capture stdin
+stdin=$(cat)
+
+yaml_dir=/tmp
+echo "$stdin" > ${yaml_dir}/builder_config
+
+OSCONFIG_FILE=osconfig
+USER_DATA_FILE=user_data
+NET_CONFIG_FILE=network_config
+QCOW_CONFIG_FILE=qcow
+
+file_list="${OSCONFIG_FILE}
+${USER_DATA_FILE}
+${NET_CONFIG_FILE}
+${QCOW_CONFIG_FILE}"
+
+IFS=$'\n'
+for f in $file_list; do
+  found_file=no
+  for l in $stdin; do
+    if [ "${l:0:1}" != " " ]; then
+      found_file=no
+    fi
+    if [ "$found_file" = "yes" ]; then
+      echo "$l" | sed 's/^    //g' >> ${yaml_dir}/${f}
+    fi
+    if [ "$l" = "${f}:" ]; then
+      found_file=yes
+    fi
+  done
+done
+unset IFS
+
+# Turn on -x after stdin is finished
+set -x
+
+# Output images to the first root-level mounted volume
+for f in $(ls / | grep -v 'proc\|sys\|dev'); do mountpoint /$f >& /dev/null && VOLUME=/$f; done
+if [ -z "$VOLUME" ]; then
+  echo "Error: Could not find a root-level volume mount to output images. Exiting."
+  exit 1
+fi
+
+# Read IMAGE_TYPE from the builder config yaml if not supplied as an env var
+if [[ -z "${IMAGE_TYPE}" ]]; then
+  # Make iso builds the default for backwards compatibility
+  echo "NOTE: No IMAGE_TYPE specified. Assuming 'iso'."
+  IMAGE_TYPE='iso'
+fi
+
+OUTPUT_FILE_NAME="$(yq r ${yaml_dir}/builder_config outputFileName)"
+
+_process_input_data_set_vars_osconfig(){
+  OSCONFIG_FILE="${yaml_dir}/${OSCONFIG_FILE}"
+  # Optional user-supplied playbook vars
+  if [[ -f "${OSCONFIG_FILE}" ]]; then
+    cp "${OSCONFIG_FILE}" /opt/assets/playbooks/roles/osconfig/vars/main.yaml
+  fi
+}
+
+_process_input_data_set_vars_iso(){
+  # Required user provided input
+  USER_DATA_FILE="${yaml_dir}/${USER_DATA_FILE}"
+  if [ ! -e $USER_DATA_FILE ]; then
+    echo "No user_data file supplied! Exiting."
+    exit 1
+  fi
+
+  # Required user provided input
+  NET_CONFIG_FILE="${yaml_dir}/${NET_CONFIG_FILE}"
+  if [ ! -e $USER_DATA_FILE ]; then
+    echo "No net_config file supplied! Exiting."
+    exit 1
+  fi
+  # cloud-init expects net confing specifically in json format
+  NET_CONFIG_JSON_FILE=/tmp/network_data.json
+  yq r -j "${NET_CONFIG_FILE}" > "${NET_CONFIG_JSON_FILE}"
+
+  # Optional user provided input
+  if [[ ${OUTPUT_FILE_NAME} != null ]]; then
+    IMG_NAME="${OUTPUT_FILE_NAME}"
+  else
+    IMG_NAME="${ISO_NAME_DEFAULT}"
+  fi
+  cat << EOF > /opt/assets/playbooks/roles/iso/vars/main.yaml
+meta_data_file: ${BASEDIR}/meta_data.json
+user_data_file: ${USER_DATA_FILE}
+network_data_file: ${NET_CONFIG_JSON_FILE}
+EOF
+}
+
+_process_input_data_set_vars_qcow(){
+  IMG_NAME=null
+  QCOW_CONFIG_FILE="${yaml_dir}/${QCOW_CONFIG_FILE}"
+  # Optional user-supplied playbook vars
+  if [[ -f "${QCOW_CONFIG_FILE}" ]]; then
+    cp "${QCOW_CONFIG_FILE}" /opt/assets/playbooks/roles/qcow/vars/main.yaml
+
+    # Extract the image output name in the ansible vars file provided
+    IMG_NAME="$(yq r "${QCOW_CONFIG_FILE}" img_name)"
+  fi
+
+  # Retrieve from playbook defaults if not provided in user input
+  if [[ "${IMG_NAME}" == 'null' ]]; then
+    IMG_NAME="$(yq r /opt/assets/playbooks/roles/qcow/defaults/main.yaml img_name)"
+  fi
+
+  # User-supplied image output name in builder-config takes precedence
+  if [[ ${OUTPUT_FILE_NAME} != null ]]; then
+    IMG_NAME="${OUTPUT_FILE_NAME}"
+  else
+    _validate_param "${IMG_NAME}" img_name
+  fi
+}
+

image-builder/assets/playbooks/roles/multistrap/defaults/main.yaml

@@ -68,6 +68,7 @@ ubuntu_packages:
     - traceroute
     - vim
     - vlan
+    - wget
     - xfsprogs
     - xz-utils
 repos:

image-builder/assets/playbooks/roles/qcow/defaults/main.yaml

@@ -4,6 +4,7 @@ nbd_build_dir: /tmp/nbd_build_dir
 img_output_dir: /config
 img_name: airship-ubuntu.qcow2
 qcow_capacity: 5G
+qcow_compress: true
 partitions:
     # Partition numbering is according to list ordering.
     # Ironic default cloud-init configdrive injection requires

image-builder/assets/playbooks/roles/qcow/tasks/qcow-detach-n-compress.yaml

@@ -6,3 +6,9 @@
 - name: "QCOW | Compressing QCoW and writing out to {{ img_output_dir }}/{{ img_name }}"
   shell: |
         qemu-img convert -p -O qcow2 -c {{ nbd_build_dir }}/{{ img_name }} {{ img_output_dir }}/{{ img_name }}
+  when: qcow_compress
+
+- name: "QCOW | Writing QCoW to {{ img_output_dir }}/{{ img_name }}"
+  shell: |
+        qemu-img convert -p -O qcow2 {{ nbd_build_dir }}/{{ img_name }} {{ img_output_dir }}/{{ img_name }}
+  when: not qcow_compress

image-builder/tools/cut_image.sh

@@ -57,22 +57,35 @@ fi
 workdir="$(realpath ${host_mount_directory})"
 
 if [[ $build_type = iso ]]; then
-  sudo -E docker run -t --rm \
+  iso_config=/tmp/iso_config
+  echo "user_data:
+$(cat $host_mount_directory/user_data | sed 's/^/    /g')
+network_config:
+$(cat $host_mount_directory/network_data.json | sed 's/^/    /g')
+outputFileName: ephemeral.iso" > ${iso_config}
+  sudo -E docker run -i --rm \
    --volume $workdir:/config \
    --env BUILDER_CONFIG=/config/${build_type}.yaml \
    --env IMAGE_TYPE="iso" \
+   --env VERSION="v2" \
    --env http_proxy=$proxy \
    --env https_proxy=$proxy \
    --env HTTP_PROXY=$proxy \
    --env HTTPS_PROXY=$proxy \
    --env no_proxy=$noproxy \
    --env NO_PROXY=$noproxy \
-   ${image}
+   ${image} < ${iso_config}
   disk1="--disk path=${workdir}/ephemeral.iso,device=cdrom"
 elif [[ $build_type == qcow ]]; then
   sudo -E modprobe nbd
+  qcow_config=/tmp/qcow_config
+  echo "osconfig:
+$(cat $host_mount_directory/osconfig-control-plane-vars.yaml | sed 's/^/    /g')
+qcow:
+$(cat $host_mount_directory/qcow-control-plane-vars.yaml | sed 's/^/    /g')
+outputFileName: control-plane.qcow2" > ${qcow_config}
   echo "Note: This step can be slow if you don't have an SSD."
-  sudo -E docker run -t --rm \
+  sudo -E docker run -i --rm \
    --privileged \
    --volume /dev:/dev:rw  \
    --volume /dev/pts:/dev/pts:rw \
@@ -83,13 +96,14 @@ elif [[ $build_type == qcow ]]; then
    ${uefi_mount} \
    --env BUILDER_CONFIG=/config/${build_type}.yaml \
    --env IMAGE_TYPE="qcow" \
+   --env VERSION="v2" \
    --env http_proxy=$proxy \
    --env https_proxy=$proxy \
    --env HTTP_PROXY=$proxy \
    --env HTTPS_PROXY=$proxy \
    --env no_proxy=$noproxy \
    --env NO_PROXY=$noproxy \
-   ${image}
+   ${image} < ${qcow_config}
   cloud_init_config_dir='assets/tests/qcow/cloud-init'
   sudo -E cloud-localds -v --network-config="${cloud_init_config_dir}/network-config" "${workdir}/airship-ubuntu_config.iso" "${cloud_init_config_dir}/user-data" "${cloud_init_config_dir}/meta-data"
   disk1="--disk path=${workdir}/control-plane.qcow2"