Add guest image customization to DevStack plugin.

CI jobs for this plugin use DevStack to install and configure
OpenStack and to upload guest images to be used for tests.

This change allow to customize guest images just before
being sent to image service with the purpose of installing
new packages (like socat), configuring N network devices, etc.

This change has been created because for testing multicast
socat is required, but it was designed with the idea
of being used to allow further guest images customizations.

Change-Id: I88491dbb65031fe3743d1c3d27c38a57b5511794

devstack/customize_image.sh

@@ -0,0 +1,158 @@
+# This script include functions that allow guest image files customization
+# before uploading them to OpenStack image service
+
+# ensure we don't re-source this in the same environment
+[[ -z "$_NEUTRON_TEMPEST_PLUGIN_CUSTOMIZE_IMAGE" ]] || return 0
+declare -r -g _NEUTRON_TEMPEST_PLUGIN_CUSTOMIZE_IMAGE=1
+
+source "${NEUTRON_TEMPEST_PLUGIN_DIR}/functions.sh"
+
+# By default enable guest image customization. It will be automatically skipped
+# for cirros images
+CUSTOMIZE_IMAGE=${CUSTOMIZE_IMAGE:-True}
+
+# Image customization is performed using virt-customize
+# using direct backend by default
+LIBGUESTFS_BACKEND=${LIBGUESTFS_BACKEND:-direct}
+
+# Disable KVM hardware accelleration by default
+LIBGUESTFS_BACKEND_SETTINGS=${LIBGUESTFS_BACKEND_SETTINGS:-force_tcg}
+
+# Install tools required for customizing guest image files
+function install_customize_image_tools {
+    local do_customize=$(trueorfalse True CUSTOMIZE_IMAGE)
+    if [ ${do_customize} == True ]; then
+        # Make sure virt-customize is installed
+        install_package libguestfs-tools
+    fi
+}
+
+# Wraps upload_image function to eventually customize image file before
+# uploading it via "openstack image create" command
+save_function upload_image overridden_upload_image
+function upload_image {
+    local image_url=$1
+
+    # Fork a subshell to have environment restored at the end of this function
+    (
+        # Check user configuration
+        local customize_image=$(trueorfalse True CUSTOMIZE_IMAGE)
+        if [ ${customize_image} == True ]; then
+            # Temporarly wraps openstack command with openstack_image_create
+            # function
+            function openstack {
+                IMAGE_URL=${image_url} upload_custom_image "$@"
+            }
+        fi
+
+        # Execute original upload_image function
+        overridden_upload_image "$@"
+    )
+}
+
+# Wraps "openstack image create" command to customize image file before
+# uploading it to OpenstackImage service.
+# Called only when ${CUSTOMIZE_IMAGE} is True
+function upload_custom_image {
+    # Copy command arguments for later use
+    local args=( "$@" )
+
+    # Look for image create sub-command:
+    # skip any argument before "image" and "create" words
+    local i=0
+    local subcommands=()
+    for subcommand in image create; do
+        for (( ; i < ${#args[@]}; )) {
+            local arg=${args[i]}
+            (( ++i ))
+            if [ "${arg}" == "${subcommand}" ]; then
+                subcommands+=( "${arg}" )
+                break
+            fi
+        }
+    done
+
+    if [ "${subcommands[*]}" == "image create" ]; then
+        # create image subcommand has been detected
+
+        # Put here temporary files to be delete before exiting from this
+        # function
+        local temp_dir=$(mktemp -d)
+        chmod 777 "${temp_dir}"
+
+        # Parse openstack image create subcommand arguments
+        local image_url="${IMAGE_URL}"
+        local image_file=
+        local disk_format=auto
+        local container_format=bare
+
+        for (( ; i < ${#args[@]}; )) {
+            local arg=${args[$i]}
+            (( ++i ))
+
+            if [[ "${arg}" == --* ]]; then
+                # Handle --<option_name>=<option_value> syntax
+                local option_fields=(${arg//=/ })
+                local option_name=${option_fields[0]}
+                local option_value=${option_fields[1]:-}
+
+                case "${option_name}" in
+
+                    --container-format)  # Found container format
+                        container_format=${option_value:-${args[ (( i++ )) ]}}
+                        ;;
+
+                    --disk-format)  # Found disk format
+                        disk_format=${option_value:-${args[ (( i++ )) ]}}
+                        ;;
+
+                   --file)  # Found image file name
+                        image_file=${option_value:-${args[ (( i++ )) ]}}
+                        ;;
+                esac
+            fi
+        }
+
+        if [ "${image_file}" == "" ]; then
+            # Copy image file from stdin to a temporary file
+            image_file=${temp_dir}/$(basename "${image_url}")
+            cat > "${image_file}"
+
+            # Add option to load image from file
+            args+=( --file "${image_file}" )
+        fi
+
+        # Make image file readable and writable by qemu user
+        sudo chmod 666 "${image_file}"
+
+        # Customize image file
+        TEMP_DIR=${temp_dir} \
+            DISK_FORMAT=${disk_format} \
+            customize_image "${image_file}"
+    fi
+
+    # Upload custom image file
+    overridden_openstack "${args[@]}" || local error=$?
+
+    # Finally delete temporary files
+    sudo rm -fR "${temp_dir}" || true
+
+    return ${error:-0}
+}
+
+function overridden_openstack {
+    "$(which openstack)" "$@"
+}
+
+# Execute customization commands on a VM with attached guest image file.
+# Called only when ${CUSTOMIZE_IMAGE} is True
+function customize_image {
+    local image_file=$1
+    local top_dir=$(dirname "${NEUTRON_TEMPEST_PLUGIN_DIR}")
+    (
+        export TEMP_DIR DISK_FORMAT RC_DIR
+        if [[ "$(basename ${image_file})" == ubuntu-* ]]; then
+            "${top_dir}/tools/customize_ubuntu_image" "${image_file}"
+        fi
+    )
+}

devstack/functions.sh

@@ -0,0 +1,32 @@
+# Generic use functions
+
+# ensure we don't re-source this in the same environment
+[[ -z "$_NEUTRON_TEMPEST_PLUGIN_FUNCTIONS" ]] || return 0
+declare -r -g _NEUTRON_TEMPEST_PLUGIN_FUNCTIONS=1
+
+# Create a function copying the code from an existing one
+function save_function {
+    local old_name=$1
+    local new_name=$2
+
+    # Saving the same function again after redefining it could produce a
+    # recorsive function in case for example this plugin is sourced twice
+    if type -t "${new_name}"; then
+        # Prevent copying the same function twice
+        return 0
+    fi
+
+    # Save xtrace setting
+    _XTRACE_FUNCTIONS=$(set +o | grep xtrace)
+    set +o xtrace
+
+    # Get code of the original function
+    local old_code=$(declare -f ${old_name})
+    # Produce code for the new function
+    local new_code="${new_name}${old_code#${old_name}}"
+    # Define the new function
+    eval "${new_code}"
+
+    # Restore xtrace
+    $_XTRACE_FUNCTIONS
+}

devstack/plugin.sh

@@ -1,6 +1,12 @@
+# Directory where this plugin.sh file is
+NEUTRON_TEMPEST_PLUGIN_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)
+
+source "${NEUTRON_TEMPEST_PLUGIN_DIR}/customize_image.sh"
+
 # install_neutron_tempest_plugin
 function install_neutron_tempest_plugin {
     setup_dev_lib "neutron-tempest-plugin"
+    install_customize_image_tools
 }
 
 if [[ "$1" == "stack" ]]; then

tools/customize_ubuntu_image

@@ -31,8 +31,11 @@ function customize_image {
     chown _apt.root -fR "${apt_user_folders[@]}"
 
     # Install desired packages to Ubuntu guest image
-    apt-get update -y
+    (
-    apt-get install -y "${INSTALL_GUEST_PACKAGES[@]}"
+        DEBIAN_FRONTEND=noninteractive
+        apt-get update -y
+        apt-get install -y "${INSTALL_GUEST_PACKAGES[@]}"
+    )
 }
 
 function main {
@@ -64,9 +67,12 @@ function chroot_image {
     bind_dir "${temp_dir}/tmp" "${mount_dir}/var/tmp"
     bind_dir "${temp_dir}/apt" "${mount_dir}/var/lib/apt"
 
-    # Replace /etc/resolv.conf symlink to use the same DNS as this host
+    # Temporarly replace /etc/resolv.conf symlink to use the same DNS as this
-    sudo rm -f "${mount_dir}/etc/resolv.conf"
+    # host
-    sudo cp /etc/resolv.conf "${mount_dir}/etc/resolv.conf"
+    local resolv_file=${mount_dir}/etc/resolv.conf
+    sudo mv -f "${resolv_file}" "${resolv_file}.orig"
+    sudo cp /etc/resolv.conf "${resolv_file}"
+    add_cleanup sudo mv -f "${resolv_file}.orig" "${resolv_file}"
 
     # Makesure /etc/fstab exists and it is readable because it is required by
     # /sbin/dhclient-script