openstack/deb-python-diskimage-builder
Code Issues Proposed changes
Files
adf39c52cf483be06d000e659129c1eebb8533be
deb-python-diskimage-builder/diskimage_builder/lib/disk-image-create
Ian Wienand adf39c52cf Merge tag '1.26.1' into merge-branch 
Release 1.21.1

Change-Id: Ib9eb3dd1d384fc5b6a9846608216e056c57a173a
2017-02-02 20:36:23 +11:00

519 lines
20 KiB
Bash
Raw Blame History

#!/bin/bash
#
# Copyright 2012 Hewlett-Packard Development Company, L.P.
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
set -eE
# Prevent perl from complaining a lot, but also remove any unexpected side-effects
# of $LANG varying between build hosts
export LANG=C
# Store our initial environment and command line args for later
export DIB_ARGS="$@"
export DIB_ENV=$(export | grep ' DIB_.*=')
SCRIPTNAME=$(basename $0)
if [ -z "$_LIB" ]; then
echo "_LIB not set!"
exit 1
fi
_BASE_ELEMENT_DIR=$(python -c '
import diskimage_builder.paths
diskimage_builder.paths.show_path("elements")')
source $_LIB/die
IS_RAMDISK=0
if [ "$SCRIPTNAME" == "ramdisk-image-create" ]; then
IS_RAMDISK=1
fi
function show_options () {
echo "Usage: ${SCRIPTNAME} [OPTION]... [ELEMENT]..."
echo
echo "Options:"
echo " -a i386|amd64|armhf -- set the architecture of the image(default amd64)"
echo " -o imagename -- set the imagename of the output image file(default image)"
echo " -t qcow2,tar,tgz,squashfs,vhd,docker,aci,raw -- set the image types of the output image files (default qcow2)"
echo " File types should be comma separated. VHD outputting requires the vhd-util"
echo " executable be in your PATH. ACI outputting requires the ACI_MANIFEST "
echo " environment variable be a path to a manifest file."
echo " -x -- turn on tracing (use -x -x for very detailed tracing)"
echo " -u -- uncompressed; do not compress the image - larger but faster"
echo " -c -- clear environment before starting work"
echo " --checksum -- generate MD5 and SHA256 checksum files for the created image"
echo " --image-size size -- image size in GB for the created image"
echo " --image-cache directory -- location for cached images(default ~/.cache/image-create)"
echo " --max-online-resize size -- max number of filesystem blocks to support when resizing."
echo " Useful if you want a really large root partition when the image is deployed."
echo " Using a very large value may run into a known bug in resize2fs."
echo " Setting the value to 274877906944 will get you a 1PB root file system."
echo " Making this value unnecessarily large will consume extra disk space "
echo " on the root partition with extra file system inodes."
echo " --min-tmpfs size -- minimum size in GB needed in tmpfs to build the image"
echo " --mkfs-options -- option flags to be passed directly to mkfs."
echo " Options should be passed as a single string value."
echo " --no-tmpfs -- do not use tmpfs to speed image build"
echo " --offline -- do not update cached resources"
echo " --qemu-img-options -- option flags to be passed directly to qemu-img."
echo " Options need to be comma separated, and follow the key=value pattern."
echo " --root-label label -- label for the root filesystem. Defaults to 'cloudimg-rootfs'."
echo " --ramdisk-element -- specify the main element to be used for building ramdisks."
echo " Defaults to 'ramdisk'. Should be set to 'dracut-ramdisk' for platforms such"
echo " as RHEL and CentOS that do not package busybox."
echo " --install-type -- specify the default installation type. Defaults to 'source'. Set to 'package' to use package based installations by default."
echo " --docker-target -- specify the repo and tag to use if the output type is docker. Defaults to the value of output imagename"
if [ "$IS_RAMDISK" == "0" ]; then
echo " -n skip the default inclusion of the 'base' element"
echo " -p package[,package,package] -- list of packages to install in the image. If specified multiple times the packages are appended to the list."
fi
echo " -h|--help -- display this help and exit"
echo " --version -- display version and exit"
echo
echo "ELEMENTS_PATH will allow you to specify multiple locations for the elements."
echo
echo "NOTE: At least one distribution root element must be specified."
echo
echo "NOTE: If using the VHD output format you need to have a patched version of vhd-util installed for the image"
echo " to be bootable. The patch is available here: https://github.com/emonty/vhd-util/blob/master/debian/patches/citrix"
echo " and a PPA with the patched tool is available here: https://launchpad.net/~openstack-ci-core/+archive/ubuntu/vhd-util"
echo
echo "Examples:"
if [ "$IS_RAMDISK" == "0" ]; then
echo " ${SCRIPTNAME} -a amd64 -o ubuntu-amd64 vm ubuntu"
echo " export ELEMENTS_PATH=~/source/tripleo-image-elements/elements"
echo " ${SCRIPTNAME} -a amd64 -o fedora-amd64-heat-cfntools vm fedora heat-cfntools"
else
echo " ${SCRIPTNAME} -a amd64 -o fedora-deploy deploy fedora"
echo " ${SCRIPTNAME} -a amd64 -o ubuntu-ramdisk ramdisk ubuntu"
fi
}
function show_version() {
python -c "from diskimage_builder import version; print(version.version_info.version_string())"
}
INSTALL_PACKAGES=""
IMAGE_TYPES=("qcow2")
COMPRESS_IMAGE="true"
export DIB_ROOT_LABEL=""
DIB_DEFAULT_INSTALLTYPE=${DIB_DEFAULT_INSTALLTYPE:-"source"}
MKFS_OPTS=""
ACI_MANIFEST=${ACI_MANIFEST:-}
DOCKER_TARGET=""
TEMP=`getopt -o a:ho:t:xucnp: -l checksum,no-tmpfs,offline,help,version,min-tmpfs:,image-size:,image-cache:,max-online-resize:,mkfs-options:,qemu-img-options:,ramdisk-element:,root-label:,install-type:,docker-target: -n $SCRIPTNAME -- "$@"`
if [ $? -ne 0 ] ; then echo "Terminating..." >&2 ; exit 1 ; fi
# Note the quotes around `$TEMP': they are essential!
eval set -- "$TEMP"
while true ; do
case "$1" in
-a) export ARCH=$2; shift 2 ;;
-o) export IMAGE_NAME=$2; shift 2 ;;
-t) IFS="," read -a IMAGE_TYPES <<< "$2"; export IMAGE_TYPES ; shift 2 ;;
-h|--help) show_options; exit 0;;
--version) show_version; exit 0;;
-x) shift; export DIB_DEBUG_TRACE=$(( $DIB_DEBUG_TRACE + 1 )); set -x;;
-u) shift; export COMPRESS_IMAGE="";;
-c) shift ; export CLEAR_ENV=1;;
-n) shift; export SKIP_BASE="1";;
-p) IFS="," read -a _INSTALL_PACKAGES <<< "$2"; export INSTALL_PACKAGES=( ${INSTALL_PACKAGES[@]} ${_INSTALL_PACKAGES[@]} ) ; shift 2 ;;
--checksum) shift; export DIB_CHECKSUM=1;;
--image-size) export DIB_IMAGE_SIZE=$2; shift 2;;
--image-cache) export DIB_IMAGE_CACHE=$2; shift 2;;
--max-online-resize) export MAX_ONLINE_RESIZE=$2; shift 2;;
--mkfs-options) MKFS_OPTS=$2; shift 2;;
--min-tmpfs) export DIB_MIN_TMPFS=$2; shift 2;;
--no-tmpfs) shift; export DIB_NO_TMPFS=1;;
--offline) shift; export DIB_OFFLINE=1;;
--qemu-img-options) QEMU_IMG_OPTIONS=$2; shift 2;;
--root-label) export DIB_ROOT_LABEL=$2; shift 2;;
--ramdisk-element) RAMDISK_ELEMENT=$2; shift 2;;
--install-type) DIB_DEFAULT_INSTALLTYPE=$2; shift 2;;
--docker-target) export DOCKER_TARGET=$2; shift 2 ;;
--) shift ; break ;;
*) echo "Internal error!" ; exit 1 ;;
esac
done
export DIB_IMAGE_CACHE=${DIB_IMAGE_CACHE:-~/.cache/image-create}
mkdir -p $DIB_IMAGE_CACHE
if [ "$CLEAR_ENV" = "1" -a "$HOME" != "" ]; then
echo "Re-execing to clear environment."
echo "(note this will prevent much of the local_config element from working)"
exec -c $0 "$@"
fi
# Display the current file/function/line in the debug output
function _ps4 {
IFS=" " called=($(caller 0))
local f=$(readlink -f ${called[2]})
printf "%-80s " "$f:${called[1]}:${called[0]}"
}
export -f _ps4
export PS4='+ $(_ps4): '
source $_LIB/img-defaults
source $_LIB/common-functions
source $_LIB/img-functions
if [ "$IS_RAMDISK" == "1" ]; then
source $_LIB/ramdisk-defaults
source $_LIB/ramdisk-functions
fi
echo "diskimage-builder version $(show_version)"
# If no elements are specified theres no way we can succeed
if [ -z "$*" ]; then
echo "ERROR: At least one distribution root element must be specified"
exit 1
fi
arg_to_elements "$@"
if [ "${#IMAGE_TYPES[@]}" = "1" ]; then
export IMAGE_NAME=${IMAGE_NAME%%\.${IMAGE_TYPES[0]}}
fi
# Check for required tools early on
for X in ${!IMAGE_TYPES[@]}; do
case "${IMAGE_TYPES[$X]}" in
qcow2)
if [ -z "$(which qemu-img)" ]; then
echo "qcow2 output format specified but qemu-img executable not found."
exit 1
fi
;;
tgz)
# Force tar to be created.
IMAGE_TYPES+=('tar')
;;
vhd)
if [ -z "$(which vhd-util)" ]; then
echo "vhd output format specified but no vhd-util executable found."
exit 1
fi
;;
squashfs)
if [ -z "$(which mksquashfs)" ]; then
echo "squashfs output format specified but no mksquashfs executable found."
exit 1
fi
;;
docker)
if [ -z "$(which docker)" ]; then
echo "docker output format specified but no docker executable found."
exit 1
fi
if [ -z "$DOCKER_TARGET" ]; then
echo "Please set --docker-target."
exit 1
fi
;;
esac
done
# NOTE: fstrim is on most all recent systems. It is provided by the util-linux
# package.
if [[ -z "$(which fstrim)" ]]; then
echo "fstrim utility is not found. This is provided by util-linux package"
echo "Please check your PATH variable is set correctly"
exit 1
fi
# NOTE: Tuning the rootfs uuid works only for ext filesystems.
# Rely on the below environment variable only for ext filesystems.
export DIB_IMAGE_ROOT_FS_UUID=$(uuidgen -r)
if echo "$FS_TYPE" | grep -q "^ext" && [ -z "${DIB_IMAGE_ROOT_FS_UUID}" ]; then
echo "ext filesystem detected but no DIB_IMAGE_ROOT_FS_UUID found."
echo "Is the uuidgen utility installed on your system?"
exit 1
fi
# FS_TYPE isn't available until after we source img-defaults
if [ -z "$DIB_ROOT_LABEL" ]; then
# NOTE(bnemec): XFS has a limit of 12 characters for filesystem labels
# Not changing the default for other filesystems to maintain backwards compatibility
if [ "$FS_TYPE" = "xfs" ]; then
DIB_ROOT_LABEL="img-rootfs"
else
DIB_ROOT_LABEL="cloudimg-rootfs"
fi
fi
# xattr support cannot be relied upon with tmpfs builds
# some kernels supoprt it, some don't
if [[ -n "${GENTOO_PROFILE}" ]]; then
if [[ "${GENTOO_PROFILE}" =~ "hardened" ]]; then
echo 'disabling tmpfs for gentoo hardened build'
export DIB_NO_TMPFS=1
fi
fi
mk_build_dir
create_base
# This variable needs to be propagated into the chroot
mkdir -p $TMP_HOOKS_PATH/environment.d
echo "export DIB_DEFAULT_INSTALLTYPE=\${DIB_DEFAULT_INSTALLTYPE:-\"${DIB_DEFAULT_INSTALLTYPE}\"}" > $TMP_HOOKS_PATH/environment.d/11-dib-install-type.bash
run_d extra-data
# Run pre-install scripts. These do things that prepare the chroot for package installs
run_d_in_target pre-install
do_extra_package_install
# Call install scripts to pull in the software users want.
run_d_in_target install
run_d_in_target post-install
# ensure we do not have a lost+found directory in the root folder
# that could cause copy to fail (it will be created again later
# when creating the file system, if it needs such directory)
if [ -e "$TMP_BUILD_DIR/mnt/lost+found" ]; then
sudo rm -rf "$TMP_BUILD_DIR/mnt/lost+found"
fi
# Free up /mnt
unmount_image
mv $TMP_BUILD_DIR/mnt $TMP_BUILD_DIR/built
# save xtrace state, as we want to turn it off to avoid spamming the
# logs with du output below.
xtrace=$(set +o | grep xtrace)
# temp file for holding du output
du_output=${TMP_BUILD_DIR}/du_output.tmp
if [ -n "$DIB_IMAGE_SIZE" ]; then
du_size=$(echo "$DIB_IMAGE_SIZE" | awk '{printf("%d\n",$1 * 1024 *1024)}')
else
set +o xtrace
echo "Calculating image size (this may take a minute)..."
sudo du -a -c -x ${TMP_BUILD_DIR}/built > ${du_output}
# the last line is the total size from "-c".
# scale this by 0.6 to create a slightly bigger image
du_size=$(tail -n1 ${du_output} | cut -f1 | awk '{print int($1 / 0.6)}')
$xtrace
fi
if [[ "${DIB_SHOW_IMAGE_USAGE:-0}" != 0 ]]; then
set +o xtrace
if [ ! -f "$du_output" ]; then
sudo du -a -c -x ${TMP_BUILD_DIR}/built > ${du_output}
fi
du_output_show="sort -nr ${du_output} |
numfmt --to=iec-i --padding=7
--suffix=B --field=1 --from-unit=1024"
# by default show the 10MiB and greater files & directories -- a
# dir with lots of little files will still show up, but this helps
# signal:noise ratio
if [[ ${DIB_SHOW_IMAGE_USAGE_FULL:-0} == 0 ]]; then
# numfmt will start giving a decimal place when < 10MiB
du_output_show+="| egrep 'MiB|GiB|TiB|PiB' | grep -v '\..MiB'"
echo "================================="
echo "Image size report (files > 10MiB)"
echo "================================="
else
echo "================="
echo "Image size report"
echo "================="
fi
eval ${du_output_show}
echo
echo "===== end image size report ====="
echo
$xtrace
fi
rm -f ${du_output}
if [ "$FS_TYPE" = "ext4" ] ; then
# Very conservative to handle images being resized a lot
# We set journal size to 64M so our journal is large enough when we
# perform an FS resize.
MKFS_OPTS="-i 4096 -J size=64 $MKFS_OPTS"
# Grow the image size to account for the journal, only if the user
# has not asked for a specific size.
if [ -z "$DIB_IMAGE_SIZE" ]; then
du_size=$(( $du_size + 65536 ))
fi
fi
# Rounding down size so that is is a multiple of 64, works around a bug in
# qemu-img that may occur when compressing raw images that aren't a multiple
# of 64k. https://bugs.launchpad.net/ubuntu/+source/linux/+bug/1180021
export DIB_IMAGE_SIZE=$(echo "$du_size" | awk ' { if ($1 % 64 != 0) print $1 + 64 - ( $1 % 64); else print $1; } ')
if [ -n "$MAX_ONLINE_RESIZE" ]; then
MKFS_OPTS="-E resize=$MAX_ONLINE_RESIZE $MKFS_OPTS"
fi
export TMP_IMAGE_DIR
# Try the 'old fashioned' way calling the block device
# phase. If this gives no result, use the configuration based approach:
eval_run_d block-device "IMAGE_BLOCK_DEVICE="
# Because there is currently no generic way of passing in variables
# from elements to the main, get here the well known config of a well
# known element.
# (This is only temporary and will go away when the complete block
# device handling including file system handling and mounting is
# implemented using python.)
if [[ $IMAGE_ELEMENT =~ vm ]]; then
for EPATH in $(echo ${ELEMENTS_PATH} | tr ":" " "); do
PART_CFG_PATH=${EPATH}/vm/environment.d/10-partitioning
[ -e ${PART_CFG_PATH} ] && source ${PART_CFG_PATH}
done
fi
# There is the need to get the path of the dib-block-device entry
# point, because when running in a venv, the standard path of
# sudo does not include this.
DIB_BLOCK_DEVICE_SCRIPT=$(which dib-block-device)
if [ -z ${IMAGE_BLOCK_DEVICE} ] ; then
IMAGE_BLOCK_DEVICE=$(${DIB_BLOCK_DEVICE_SCRIPT} \
--phase=create \
--config="${DIB_BLOCK_DEVICE_CONFIG:-}" \
--image-size="${DIB_IMAGE_SIZE}"KiB \
--image-dir="${TMP_IMAGE_DIR}" \
--build-dir="${TMP_BUILD_DIR}" )
fi
export IMAGE_BLOCK_DEVICE
LOOPDEV=${IMAGE_BLOCK_DEVICE}
IMAGE_BLOCK_DEVICE_WITHOUT_PART=$(echo ${IMAGE_BLOCK_DEVICE} \
| sed -e "s|^\(.*loop[0-9]*\)p[0-9]*$|\1|g")
export IMAGE_BLOCK_DEVICE_WITHOUT_PART
export EXTRA_DETACH="detach_loopback ${IMAGE_BLOCK_DEVICE_WITHOUT_PART}"
export EXTRA_UNMOUNT="dib-block-device --phase=cleanup \
--build-dir=\"${TMP_BUILD_DIR}\""
# Create the partitions and make them visible to the system
sudo partprobe $IMAGE_BLOCK_DEVICE_WITHOUT_PART
# To ensure no race conditions exist from calling partprobe
sudo udevadm settle
# If the partition isn't under /dev/loop*p1, create it with kpartx
DM=
if [ ! -e "${IMAGE_BLOCK_DEVICE}" ]; then
DM=${IMAGE_BLOCK_DEVICE/#\/dev/\/dev\/mapper}
# If running inside Docker, make our nodes manually, because udev will not be working.
if [ -f /.dockerenv ]; then
# kpartx cannot run in sync mode in docker.
sudo kpartx -av ${IMAGE_BLOCK_DEVICE_WITHOUT_PART}
sudo dmsetup --noudevsync mknodes
else
sudo kpartx -asv ${IMAGE_BLOCK_DEVICE_WITHOUT_PART}
fi
elif [[ "$ARCH" =~ "ppc" ]]; then
sudo kpartx -asv ${IMAGE_BLOCK_DEVICE_WITHOUT_PART}
fi
# End: Creation of the partitions
sudo mkfs -t $FS_TYPE $MKFS_OPTS -L ${DIB_ROOT_LABEL} ${IMAGE_BLOCK_DEVICE}
# Tuning the rootfs uuid works only for ext filesystems.
if echo "$FS_TYPE" | grep -q "^ext"; then
sudo tune2fs -U ${DIB_IMAGE_ROOT_FS_UUID} ${IMAGE_BLOCK_DEVICE}
fi
mkdir $TMP_BUILD_DIR/mnt
sudo mount ${IMAGE_BLOCK_DEVICE} $TMP_BUILD_DIR/mnt
sudo mv -t $TMP_BUILD_DIR/mnt ${TMP_BUILD_DIR}/built/*
mount_proc_dev_sys
run_d_in_target finalise
finalise_base
for X in ${!IMAGE_TYPES[@]} ; do
if [[ " tar aci " =~ "${IMAGE_TYPES[$X]}" ]]; then
if [ "${IMAGE_TYPES[$X]}" = "aci" ]; then
sudo tar -C ${TMP_BUILD_DIR}/mnt -cf $IMAGE_NAME.aci --exclude ./sys \
--exclude ./proc --xattrs --xattrs-include=\* \
--transform 's,^.,rootfs,S' .
if [ -n "$ACI_MANIFEST" ]; then
cp $ACI_MANIFEST ${TMP_BUILD_DIR}/manifest
sudo tar -C ${TMP_BUILD_DIR} --append -f $IMAGE_NAME.aci manifest
else
echo "No ACI_MANIFEST specified. An ACI_MANIFEST must be specified for"
echo " this image to be usable."
fi
else
sudo tar -C ${TMP_BUILD_DIR}/mnt -cf $IMAGE_NAME.tar --exclude ./sys \
--exclude ./proc --xattrs --xattrs-include=\* .
fi
sudo chown $USER: $IMAGE_NAME.${IMAGE_TYPES[$X]}
unset IMAGE_TYPES[$X]
elif [ "${IMAGE_TYPES[$x]}" == "squashfs" ]; then
sudo mksquashfs ${TMP_BUILD_DIR}/mnt $IMAGE_NAME.squash -comp xz \
-noappend -root-becomes ${TMP_BUILD_DIR}/mnt \
-wildcards -e "proc/*" -e "sys/*" -no-recovery
elif [ "${IMAGE_TYPES[$X]}" == "docker" ]; then
sudo tar -C ${TMP_BUILD_DIR}/mnt -cf - --exclude ./sys \
--exclude ./proc --xattrs --xattrs-include=\* . \
| sudo docker import - $DOCKER_TARGET
unset IMAGE_TYPES[$X]
fi
done
if [[ ! $IMAGE_ELEMENT =~ no-final-image ]]; then
# Prep filesystem by discarding all unused space
fstrim_image
fi
# Unmount and cleanup the /mnt and /build subdirectories, to save
# space before converting the image to some other format.
export EXTRA_UNMOUNT=""
unmount_image
export TMP_IMAGE_PATH=$(${DIB_BLOCK_DEVICE_SCRIPT} \
--phase=umount \
--build-dir="${TMP_BUILD_DIR}" )
${DIB_BLOCK_DEVICE_SCRIPT} \
--phase=cleanup \
--build-dir="${TMP_BUILD_DIR}"
cleanup_build_dir
if [[ (! $IMAGE_ELEMENT =~ no-final-image) && "$IS_RAMDISK" == "0" ]]; then
has_raw_type=
for IMAGE_TYPE in ${IMAGE_TYPES[@]} ; do
# We have to do raw last because it is destructive
if [ "$IMAGE_TYPE" = "raw" ]; then
has_raw_type=1
elif [ "$IMAGE_TYPE" != "squashfs" ]; then
compress_and_save_image $IMAGE_NAME.$IMAGE_TYPE
fi
done
if [ -n "$has_raw_type" ]; then
IMAGE_TYPE="raw"
compress_and_save_image $IMAGE_NAME.$IMAGE_TYPE
fi
fi
# Remove the leftovers, i.e. the temporary image directory.
cleanup_image_dir
# All done!
trap EXIT
View Git Blame Copy Permalink