diff --git a/.gitignore b/.gitignore index 091b3585a..d2238b44b 100644 --- a/.gitignore +++ b/.gitignore @@ -1,3 +1,6 @@ +.coverage +coverage.xml +cover/* *~ .testrepository *.sw? diff --git a/.testr.conf b/.testr.conf index 25d7f8d95..d7b8b32fe 100644 --- a/.testr.conf +++ b/.testr.conf @@ -4,7 +4,7 @@ test_command=OS_STDOUT_CAPTURE=${OS_STDOUT_CAPTURE:-1} \ OS_LOG_CAPTURE=${OS_LOG_CAPTURE:-1} \ OS_TEST_TIMEOUT=${OS_TEST_TIMEOUT:-60} \ OS_DEBUG=${OS_DEBUG:-0} \ - python -m subunit.run discover . $LISTOPT $IDOPTION + ${PYTHON:-python} -m subunit.run discover . $LISTOPT $IDOPTION test_id_option=--load-list $IDFILE test_list_option=--list diff --git a/bin/dib-lint b/bin/dib-lint index 1755270f8..078963d29 100755 --- a/bin/dib-lint +++ b/bin/dib-lint @@ -36,11 +36,15 @@ parse_exclusions() { # ignore = sete setu section="dib-lint" option="ignore" - global_exclusions=$(python -c \ - "import ConfigParser; \ - conf=ConfigParser.ConfigParser(); \ - conf.read('tox.ini'); \ - print conf.get('$section', '$option') if conf.has_option('$section', '$option') else ''" + global_exclusions=$(python - <> /etc/default/grub + echo 'GRUB_DISABLE_LINUX_UUID=true' >> /etc/default/grub echo "GRUB_TIMEOUT=${DIB_GRUB_TIMEOUT:-5}" >>/etc/default/grub echo 'GRUB_TERMINAL="serial console"' >>/etc/default/grub echo 'GRUB_GFXPAYLOAD_LINUX=text' >>/etc/default/grub @@ -175,17 +179,7 @@ function install_grub2 { sed -i "s%search --no.*%%" $GRUB_CFG sed -i "s%set root=.*%set root=(hd0,1)%" $GRUB_CFG fi - # force use of a LABEL: - # NOTE: Updating the grub config by hand once deployed should work, its just - # prepping it in a different environment that needs fiddling. - sed -i "s%$PART_DEV%LABEL=${DIB_ROOT_LABEL}%" $GRUB_CFG - sed -i "s%search --no-floppy --fs-uuid --set=root .*$%search --no-floppy --set=root --label ${DIB_ROOT_LABEL}%" $GRUB_CFG - sed -i "s%root=UUID=[A-Za-z0-9\-]*%root=LABEL=${DIB_ROOT_LABEL}%" $GRUB_CFG - if [ "$DISTRO_NAME" = 'fedora' ] ; then - if [ "$DIB_RELEASE" = '19' ]; then - sed -i "s%UUID=[A-Za-z0-9\-]*%LABEL=${DIB_ROOT_LABEL}%" /etc/fstab - fi - fi + # Fix efi specific instructions in grub config file if [ -d /sys/firmware/efi ]; then sed -i 's%\(initrd\|linux\)efi /boot%\1 /boot%g' $GRUB_CFG diff --git a/diskimage_builder/elements/centos-minimal/README.rst b/diskimage_builder/elements/centos-minimal/README.rst index 3c83eef30..f43b9d919 100644 --- a/diskimage_builder/elements/centos-minimal/README.rst +++ b/diskimage_builder/elements/centos-minimal/README.rst @@ -6,13 +6,9 @@ Create a minimal image based on CentOS 7. Use of this element will require 'yum' and 'yum-utils' to be installed on Ubuntu and Debian. Nothing additional is needed on Fedora or CentOS. -The `DIB_OFFLINE` or more specific `DIB_YUMCHROOT_USE_CACHE` -variables can be set to prefer the use of a pre-cached root filesystem -tarball. - -By default, `DIB_YUM_MINIMAL_CREATE_INTERFACES` is set to enable the -creation of `/etc/sysconfig/network-scripts/ifcfg-eth[0|1]` scripts to -enable DHCP on the `eth0` & `eth1` interfaces. If you do not have +By default, ``DIB_YUM_MINIMAL_CREATE_INTERFACES`` is set to enable the +creation of ``/etc/sysconfig/network-scripts/ifcfg-eth[0|1]`` scripts to +enable DHCP on the ``eth0`` & ``eth1`` interfaces. If you do not have these interfaces, or if you are using something else to setup the network such as cloud-init, glean or network-manager, you would want -to set this to `0`. +to set this to ``0``. diff --git a/diskimage_builder/elements/centos/environment.d/10-centos6-distro-name.bash b/diskimage_builder/elements/centos/environment.d/10-centos6-distro-name.bash index c65178fde..4f38ab2e9 100644 --- a/diskimage_builder/elements/centos/environment.d/10-centos6-distro-name.bash +++ b/diskimage_builder/elements/centos/environment.d/10-centos6-distro-name.bash @@ -3,3 +3,7 @@ export DIB_RELEASE=${DIB_RELEASE:-GenericCloud} # Useful for elements that work with fedora (dnf) & centos export YUM=${YUM:-yum} + +if [ -n "${DIB_CENTOS_DISTRIBUTION_MIRROR:-}" ]; then + export DIB_DISTRIBUTION_MIRROR=$DIB_CENTOS_DISTRIBUTION_MIRROR +fi diff --git a/diskimage_builder/elements/centos/pre-install.d/02-yum-repos b/diskimage_builder/elements/centos/pre-install.d/02-yum-repos index fb5a0b9b0..099a3eb6e 100755 --- a/diskimage_builder/elements/centos/pre-install.d/02-yum-repos +++ b/diskimage_builder/elements/centos/pre-install.d/02-yum-repos @@ -6,10 +6,12 @@ fi set -eu set -o pipefail +DIB_DISTRIBUTION_MIRROR=${DIB_DISTRIBUTION_MIRROR:-http://mirror.centos.org/centos} + cat << EOF > /etc/yum.repos.d/centos6-latest.repo [rhel6] name=centos6 -baseurl=http://mirror.centos.org/centos/6/os/x86_64/ +baseurl=$DIB_DISTRIBUTION_MIRROR/6/os/x86_64/ enabled=1 metadata_expire=7d gpgcheck=0 diff --git a/diskimage_builder/elements/centos7/environment.d/10-centos7-distro-name.bash b/diskimage_builder/elements/centos7/environment.d/10-centos7-distro-name.bash index c9476d071..c0c42a51c 100644 --- a/diskimage_builder/elements/centos7/environment.d/10-centos7-distro-name.bash +++ b/diskimage_builder/elements/centos7/environment.d/10-centos7-distro-name.bash @@ -3,3 +3,7 @@ export DIB_RELEASE=GenericCloud # Useful for elements that work with fedora (dnf) & centos export YUM=${YUM:-yum} + +if [ -n "${DIB_CENTOS_DISTRIBUTION_MIRROR:-}" ]; then + export DIB_DISTRIBUTION_MIRROR=$DIB_CENTOS_DISTRIBUTION_MIRROR +fi diff --git a/diskimage_builder/elements/debian-minimal/environment.d/10-debian-minimal.bash b/diskimage_builder/elements/debian-minimal/environment.d/10-debian-minimal.bash index 13ac22fc0..26f515990 100644 --- a/diskimage_builder/elements/debian-minimal/environment.d/10-debian-minimal.bash +++ b/diskimage_builder/elements/debian-minimal/environment.d/10-debian-minimal.bash @@ -1,6 +1,11 @@ export DISTRO_NAME=debian export DIB_RELEASE=${DIB_RELEASE:-stable} + +if [ -n "${DIB_DEBIAN_DISTRIBUTION_MIRROR:-}" ]; then + DIB_DISTRIBUTION_MIRROR=$DIB_DEBIAN_DISTRIBUTION_MIRROR +fi export DIB_DISTRIBUTION_MIRROR=${DIB_DISTRIBUTION_MIRROR:-http://ftp.us.debian.org/debian} + export DIB_DEBIAN_COMPONENTS=${DIB_DEBIAN_COMPONENTS:-main} export DIB_DEBIAN_COMPONENTS_WS=${DIB_DEBIAN_COMPONENTS//,/ } diff --git a/diskimage_builder/elements/debootstrap/install.d/10-debian-networking b/diskimage_builder/elements/debootstrap/install.d/10-debian-networking index 7e85cbe57..d5cdc1400 100755 --- a/diskimage_builder/elements/debootstrap/install.d/10-debian-networking +++ b/diskimage_builder/elements/debootstrap/install.d/10-debian-networking @@ -26,7 +26,10 @@ echo $DISTRO_NAME > /etc/hostname # cloud images expect eth0 and eth1 to use dhcp. mkdir -p /etc/network/interfaces.d -echo "source /etc/network/interfaces.d/*" >> /etc/network/interfaces +if ! grep -E -q '^source(|-directory) /etc/network/interfaces.d/\*' /etc/network/interfaces; then + echo "source /etc/network/interfaces.d/*" >> /etc/network/interfaces + echo 'Network configuration set to source /etc/network/interfaces.d/*' +fi for interface in eth0 eth1; do cat << EOF | tee /etc/network/interfaces.d/$interface auto $interface diff --git a/diskimage_builder/elements/dhcp-all-interfaces/README.rst b/diskimage_builder/elements/dhcp-all-interfaces/README.rst index dd7a3f8ca..7ecd387c7 100644 --- a/diskimage_builder/elements/dhcp-all-interfaces/README.rst +++ b/diskimage_builder/elements/dhcp-all-interfaces/README.rst @@ -17,3 +17,13 @@ configured properly before networking services are started. On Gentoo based distributions we will install the dhcpcd package and ensure the service starts at boot. This service automatically sets up all interfaces found via dhcp and/or dhcpv6 (or SLAAC). + +Environment Variables +--------------------- + +DIB_DHCP_TIMEOUT + :Required: No + :Default: 30 + :Description: Amount of time in seconds that the systemd service will + wait to get an address. + :Example: DIB_DHCP_TIMEOUT=300 diff --git a/diskimage_builder/elements/dhcp-all-interfaces/install.d/50-dhcp-all-interfaces b/diskimage_builder/elements/dhcp-all-interfaces/install.d/50-dhcp-all-interfaces index 972645a70..fa7e2fd7f 100755 --- a/diskimage_builder/elements/dhcp-all-interfaces/install.d/50-dhcp-all-interfaces +++ b/diskimage_builder/elements/dhcp-all-interfaces/install.d/50-dhcp-all-interfaces @@ -24,6 +24,7 @@ if [ "$DIB_INIT_SYSTEM" == "upstart" ]; then elif [ "$DIB_INIT_SYSTEM" == "systemd" ]; then install -D -g root -o root -m 0644 ${SCRIPTDIR}/dhcp-interface@.service /usr/lib/systemd/system/dhcp-interface@.service install -D -g root -o root -m 0644 ${SCRIPTDIR}/dhcp-all-interfaces-udev.rules /etc/udev/rules.d/99-dhcp-all-interfaces.rules + sed -i "s/TimeoutStartSec=DIB_DHCP_TIMEOUT/TimeoutStartSec=${DIB_DHCP_TIMEOUT:-30}s/" /usr/lib/systemd/system/dhcp-interface@.service elif [ "$DIB_INIT_SYSTEM" == "sysv" ]; then install -D -g root -o root -m 0755 ${SCRIPTDIR}/dhcp-all-interfaces.init /etc/init.d/dhcp-all-interfaces update-rc.d dhcp-all-interfaces defaults diff --git a/diskimage_builder/elements/dhcp-all-interfaces/install.d/dhcp-all-interfaces.sh b/diskimage_builder/elements/dhcp-all-interfaces/install.d/dhcp-all-interfaces.sh index 4884c435a..20a49ab6f 100755 --- a/diskimage_builder/elements/dhcp-all-interfaces/install.d/dhcp-all-interfaces.sh +++ b/diskimage_builder/elements/dhcp-all-interfaces/install.d/dhcp-all-interfaces.sh @@ -38,7 +38,7 @@ function serialize_me() { } function get_if_link() { - cat /sys/class/net/${1}/carrier + cat /sys/class/net/${1}/carrier || echo 0 } function enable_interface() { @@ -87,11 +87,11 @@ function inspect_interface() { elif [ "$mac_addr_type" != "0" ]; then echo "Device has generated MAC, skipping." else - ip link set dev $interface up &>/dev/null - local has_link local tries for ((tries = 0; tries < 20; tries++)); do + # Need to set the link up on each iteration + ip link set dev $interface up &>/dev/null has_link=$(get_if_link $interface) [ "$has_link" == "1" ] && break sleep 1 diff --git a/diskimage_builder/elements/dhcp-all-interfaces/install.d/dhcp-interface@.service b/diskimage_builder/elements/dhcp-all-interfaces/install.d/dhcp-interface@.service index 112765029..b066aed63 100644 --- a/diskimage_builder/elements/dhcp-all-interfaces/install.d/dhcp-interface@.service +++ b/diskimage_builder/elements/dhcp-all-interfaces/install.d/dhcp-interface@.service @@ -1,17 +1,21 @@ [Unit] -Description=DHCP interface %I -Before=network-pre.target -Wants=network-pre.target +Description=DHCP interface %i +# We want to run after network.target so it doesn't try to bring +# up the interfaces a second time, but network-online should not +# be reached until after we've brought up the interfaces. +After=network.target +Before=network-online.target +Wants=network-online.target -ConditionPathExists=!/etc/sysconfig/network-scripts/ifcfg-%I +ConditionPathExists=!/etc/sysconfig/network-scripts/ifcfg-%i [Service] Type=oneshot User=root -ExecStartPre=/usr/local/sbin/dhcp-all-interfaces.sh %I -ExecStart=/sbin/ifup %I +ExecStartPre=/usr/local/sbin/dhcp-all-interfaces.sh %i +ExecStart=/sbin/ifup %i RemainAfterExit=true -TimeoutStartSec=30s +TimeoutStartSec=DIB_DHCP_TIMEOUT [Install] WantedBy=multi-user.target diff --git a/diskimage_builder/elements/dhcp-all-interfaces/package-installs.yaml b/diskimage_builder/elements/dhcp-all-interfaces/package-installs.yaml index 98034e6c1..f73108729 100644 --- a/diskimage_builder/elements/dhcp-all-interfaces/package-installs.yaml +++ b/diskimage_builder/elements/dhcp-all-interfaces/package-installs.yaml @@ -1 +1,2 @@ dhcp-client: +ifupdown: diff --git a/diskimage_builder/elements/dhcp-all-interfaces/pkg-map b/diskimage_builder/elements/dhcp-all-interfaces/pkg-map index 6abf6d80f..080f6cd4e 100644 --- a/diskimage_builder/elements/dhcp-all-interfaces/pkg-map +++ b/diskimage_builder/elements/dhcp-all-interfaces/pkg-map @@ -8,9 +8,13 @@ }, "suse": { "dhcp-client": "dhcp-client" + }, + "debian": { + "ifupdown": "ifupdown" } }, "default": { - "dhcp-client": "isc-dhcp-client" + "dhcp-client": "isc-dhcp-client", + "ifupdown": "" } } diff --git a/diskimage_builder/elements/fedora-minimal/README.rst b/diskimage_builder/elements/fedora-minimal/README.rst index 88c88ecd3..f1c77370a 100644 --- a/diskimage_builder/elements/fedora-minimal/README.rst +++ b/diskimage_builder/elements/fedora-minimal/README.rst @@ -11,10 +11,6 @@ Due to a bug in the released version of urlgrabber, on many systems an installation of urlgrabber from git is required. The git repository can be found here: http://yum.baseurl.org/gitweb?p=urlgrabber.git;a=summary -The `DIB_OFFLINE` or more specific `DIB_YUMCHROOT_USE_CACHE` -variables can be set to prefer the use of a pre-cached root filesystem -tarball. - -This element sets the `DIB_RELEASE` var to 'fedora'. The release of fedora -to be installed can be controlled through the `DIB_RELEASE` variable, which -defaults to '21'. +This element sets the ``DIB_RELEASE`` var to 'fedora'. The release of +fedora to be installed can be controlled through the ``DIB_RELEASE`` +variable, which defaults the latest supported release. diff --git a/diskimage_builder/elements/fedora-minimal/environment.d/10-fedora-distro-name.bash b/diskimage_builder/elements/fedora-minimal/environment.d/10-fedora-distro-name.bash index 3477f1eac..0ffbc4f4e 100644 --- a/diskimage_builder/elements/fedora-minimal/environment.d/10-fedora-distro-name.bash +++ b/diskimage_builder/elements/fedora-minimal/environment.d/10-fedora-distro-name.bash @@ -1,2 +1,2 @@ export DISTRO_NAME=fedora -export DIB_RELEASE=${DIB_RELEASE:-24} +export DIB_RELEASE=${DIB_RELEASE:-25} diff --git a/diskimage_builder/elements/fedora/environment.d/10-fedora-distro-name.bash b/diskimage_builder/elements/fedora/environment.d/10-fedora-distro-name.bash index 3477f1eac..5c83ef2e8 100644 --- a/diskimage_builder/elements/fedora/environment.d/10-fedora-distro-name.bash +++ b/diskimage_builder/elements/fedora/environment.d/10-fedora-distro-name.bash @@ -1,2 +1,5 @@ export DISTRO_NAME=fedora -export DIB_RELEASE=${DIB_RELEASE:-24} +export DIB_RELEASE=${DIB_RELEASE:-25} +if [ -n "${DIB_FEDORA_DISTRIBUTION_MIRROR:-}" ]; then + export DIB_DISTRIBUTION_MIRROR=$DIB_FEDORA_DISTRIBUTION_MIRROR +fi diff --git a/diskimage_builder/elements/gentoo/bin/install-packages b/diskimage_builder/elements/gentoo/bin/install-packages index f0a4bf49a..a059bc78f 100755 --- a/diskimage_builder/elements/gentoo/bin/install-packages +++ b/diskimage_builder/elements/gentoo/bin/install-packages @@ -33,15 +33,25 @@ function show_options { } function fix_shm { + # make /dev/shm dir if it doesn't exist + # mount tmpfs and chown it + # existing programs could be using /dev/shm + # This means it cannot be moved or backed + # up as a copy easily. The only remaining + # option is to move the link if it exists + # as a link. Existing programs will still + # hold the file handle of the original + # location open and new programs can use + # the fixed /dev/shm. if [[ "${RUN_ONCE_SHM}" == '1' ]]; then - if [[ -L /dev/shm.orig ]]; then - rm /dev/shm.orig + if [[ ! -d /dev/shm ]]; then + if [[ ! -e /dev/shm ]]; then + if [[ -L /dev/shm ]]; then + mv /dev/shm /dev/shm.orig + fi + mkdir /dev/shm + fi fi - if [[ -d /dev/shm.orig ]]; then - rm -Rf /dev/shm.orig - fi - mv /dev/shm /dev/shm.orig - mkdir /dev/shm mount -t tmpfs none /dev/shm chmod 1777 /dev/shm RUN_ONCE_SHM='0' @@ -49,10 +59,16 @@ function fix_shm { } function unfix_shm { + # unmount tmpfs + # care about anything still using it if [[ "${RUN_ONCE_SHM}" == '0' ]]; then umount /dev/shm - rmdir /dev/shm - mv /dev/shm.orig /dev/shm + if fuser /dev/shm; then + rmdir /dev/shm + fi + if [[ -e /dev/shm.orig ]]; then + mv /dev/shm.orig /dev/shm + fi fi } diff --git a/diskimage_builder/elements/gentoo/post-install.d/99-cleanup b/diskimage_builder/elements/gentoo/finalise.d/99-cleanup similarity index 100% rename from diskimage_builder/elements/gentoo/post-install.d/99-cleanup rename to diskimage_builder/elements/gentoo/finalise.d/99-cleanup diff --git a/diskimage_builder/elements/ironic-agent/README.rst b/diskimage_builder/elements/ironic-agent/README.rst index a6382a029..112caa3e5 100644 --- a/diskimage_builder/elements/ironic-agent/README.rst +++ b/diskimage_builder/elements/ironic-agent/README.rst @@ -18,6 +18,10 @@ Beyond installing the ironic-python-agent, this element does the following: * Install the certificate if any, which is set to the environment variable ``DIB_IPA_CERT`` for validating the authenticity by ironic-python-agent. The certificate can be self-signed certificate or CA certificate. +* Compresses initramfs with command specified in environment variable + ``DIB_IPA_COMPRESS_CMD``, which is 'gzip' by default. This command should listen + for raw data from stdin and write compressed data to stdout. Command can be + with arguments. This element outputs three files: diff --git a/diskimage_builder/elements/ironic-agent/cleanup.d/99-ramdisk-create b/diskimage_builder/elements/ironic-agent/cleanup.d/99-ramdisk-create index 5e04a18ea..3db28bc76 100755 --- a/diskimage_builder/elements/ironic-agent/cleanup.d/99-ramdisk-create +++ b/diskimage_builder/elements/ironic-agent/cleanup.d/99-ramdisk-create @@ -18,6 +18,8 @@ source $_LIB/img-functions IMAGE_PATH=$(readlink -f $IMAGE_NAME) cd $TARGET_ROOT +DIB_IPA_COMPRESS_CMD="${DIB_IPA_COMPRESS_CMD:-gzip}" + echo "#disabled" > ./tmp/fstab.new sudo mv ./tmp/fstab.new ./etc/fstab sudo ln -s ./sbin/init ./ @@ -42,7 +44,7 @@ sudo find . -xdev \ -path './var/cache/*' -prune -o \ -name '*.pyc' -prune -o \ -name '*.pyo' -prune -o \ - -print | sudo cpio -o -H newc | gzip > ${IMAGE_PATH}.initramfs + -print | sudo cpio -o -H newc | ${DIB_IPA_COMPRESS_CMD} > ${IMAGE_PATH}.initramfs select_boot_kernel_initrd $TARGET_ROOT sudo cp $BOOTDIR/$KERNEL ${IMAGE_PATH}.kernel diff --git a/diskimage_builder/elements/ironic-agent/install.d/ironic-agent-source-install/ironic-python-agent.conf b/diskimage_builder/elements/ironic-agent/install.d/ironic-agent-source-install/ironic-python-agent.conf index cfd134583..6fe38dc9d 100644 --- a/diskimage_builder/elements/ironic-agent/install.d/ironic-agent-source-install/ironic-python-agent.conf +++ b/diskimage_builder/elements/ironic-agent/install.d/ironic-agent-source-install/ironic-python-agent.conf @@ -1,4 +1,4 @@ -# ironic-python-agent - Openstack Ironic Python Agnet +# ironic-python-agent - OpenStack Ironic Python Agent # # The ironic-python-agent helps ironic in deploying instances. diff --git a/diskimage_builder/elements/ironic-agent/install.d/ironic-agent-source-install/ironic-python-agent.service b/diskimage_builder/elements/ironic-agent/install.d/ironic-agent-source-install/ironic-python-agent.service index da9709660..d6e4ca2a3 100644 --- a/diskimage_builder/elements/ironic-agent/install.d/ironic-agent-source-install/ironic-python-agent.service +++ b/diskimage_builder/elements/ironic-agent/install.d/ironic-agent-source-install/ironic-python-agent.service @@ -3,7 +3,7 @@ Description=Ironic Python Agent After=network-online.target [Service] -ExecStartPre=/usr/sbin/modprobe vfat +ExecStartPre=/sbin/modprobe vfat ExecStart=/usr/local/bin/ironic-python-agent Restart=always RestartSec=30s diff --git a/diskimage_builder/elements/package-installs/bin/package-installs-squash b/diskimage_builder/elements/package-installs/bin/package-installs-squash index eafdd9f42..949a8ae85 100755 --- a/diskimage_builder/elements/package-installs/bin/package-installs-squash +++ b/diskimage_builder/elements/package-installs/bin/package-installs-squash @@ -59,7 +59,7 @@ def collect_data(data, filename, element_name): try: objs = json.load(open(filename)) except ValueError: - objs = yaml.load(open(filename)) + objs = yaml.safe_load(open(filename)) for pkg_name, params in objs.items(): if not params: params = {} diff --git a/diskimage_builder/elements/pip-and-virtualenv/pkg-map b/diskimage_builder/elements/pip-and-virtualenv/pkg-map index 09e1529f0..64619ca94 100644 --- a/diskimage_builder/elements/pip-and-virtualenv/pkg-map +++ b/diskimage_builder/elements/pip-and-virtualenv/pkg-map @@ -2,7 +2,10 @@ "family": { "gentoo": { "python-pip": "dev-python/pip", + "python3-pip": "dev-python/pip", "python-virtualenv": "dev-python/virtualenv", + "python3-virtualenv": "dev-python/virtualenv", + "python-dev": "dev-lang/python", "python3-dev": "dev-lang/python" }, "suse": { diff --git a/diskimage_builder/elements/proliant-tools/README.rst b/diskimage_builder/elements/proliant-tools/README.rst index 836560b83..f745afc5d 100644 --- a/diskimage_builder/elements/proliant-tools/README.rst +++ b/diskimage_builder/elements/proliant-tools/README.rst @@ -3,7 +3,7 @@ proliant-tools * This element can be used when building ironic-agent ramdisk. It enables ironic-agent ramdisk to do in-band cleaning operations specific - to HP ProLiant hardware. + to HPE ProLiant hardware. * Works with ubuntu and fedora distributions (on which ironic-agent element is supported). @@ -11,20 +11,24 @@ proliant-tools * Currently the following utilities are installed: + `proliantutils`_ - This module registers an ironic-python-agent hardware - manager for HP ProLiant hardware, which implements in-band cleaning + manager for HPE ProLiant hardware, which implements in-band cleaning steps. The latest version of ``proliantutils`` available is installed. This python module is released with Apache license. - + `HP Smart Storage Administrator (HP SSA) CLI for Linux 64-bit`_ - This + + `HPE Smart Storage Administrator (HPE SSA) CLI for Linux 64-bit`_ - This utility is used by ``proliantutils`` library above for doing in-band RAID - configuration on HP ProLiant hardware. Currently installed version is - 2.30. Newer version of ``hpssacli`` when available, may be installed to - the ramdisk by using the environment variable ``DIB_HPSSACLI_URL``. - ``DIB_HPSSACLI_URL`` should contain the HTTP(S) URL for downloading the - RPM package for ``hpssacli`` utility. Availability of newer versions can - be in the Revision History in the above link. This utility is closed source - and is released with `HP End User License Agreement – Enterprise Version`_. + configuration on HPE ProLiant hardware. Currently installed version is + 2.60. Newer version of ``ssacli`` when available, may be installed to + the ramdisk by using the environment variable ``DIB_SSACLI_URL``. + ``DIB_SSACLI_URL`` should contain the HTTP(S) URL for downloading the + RPM package for ``ssacli`` utility. The old environmental variable + ``DIB_HPSSACLI_URL``,a HTTP(S) URL for downloading the RPM package for + ``hpssacli`` utility, is deprecated. The ``hpssacli`` utility is not + supported anymore, use ``ssacli`` instead for the same functionality. + Availability of newer versions can be in the Revision History + in the above link. This utility is closed source and is released with + `HPE End User License Agreement – Enterprise Version`_. .. _`proliantutils`: https://pypi.python.org/pypi/proliantutils -.. _`HP Smart Storage Administrator (HP SSA) CLI for Linux 64-bit`: http://h20564.www2.hpe.com/hpsc/swd/public/detail?swItemId=MTX_b6a6acb9762443b182280db805 -.. _`HP End User License Agreement – Enterprise Version`: ftp://ftp.hp.com/pub/softlib2/software1/doc/p2057331991/v33194/hpeula-en.html +.. _`HPE Smart Storage Administrator (HPE SSA) CLI for Linux 64-bit`: http://h20564.www2.hpe.com/hpsc/swd/public/detail?swItemId=MTX_3d16386b418a443388c18da82f&swEnvOid=4181 +.. _`HPE End User License Agreement – Enterprise Version`: https://downloads.hpe.com/pub/softlib2/software1/doc/p1796552785/v113125/eula-en.html diff --git a/diskimage_builder/elements/proliant-tools/install.d/65-proliant-tools-install b/diskimage_builder/elements/proliant-tools/install.d/65-proliant-tools-install index 38a9b9c28..2e3de5d8a 100755 --- a/diskimage_builder/elements/proliant-tools/install.d/65-proliant-tools-install +++ b/diskimage_builder/elements/proliant-tools/install.d/65-proliant-tools-install @@ -21,18 +21,22 @@ fi set -eu set -o pipefail -# Set the below variable to allow hpssacli to be installed from custom URLs. -DIB_HPSSACLI_URL=${DIB_HPSSACLI_URL:-https://ftp.hp.com/pub/softlib2/software1/pubsw-linux/p1857046646/v109216/hpssacli-2.30-6.0.x86_64.rpm} -curl -k -o /tmp/hpssacli.rpm $DIB_HPSSACLI_URL - -if [[ $DISTRO_NAME = "ubuntu" || $DISTRO_NAME = "debian" ]]; then - # There is no deb package for hpssacli. Install with alien. - alien -i /tmp/hpssacli.rpm -else - rpm -iv /tmp/hpssacli.rpm +# Set the below variable to allow ssacli to be installed from custom URLs. +if [[ -n "${DIB_HPSSACLI_URL:=}" ]]; then + echo "The environment variable DIB_HPSSACLI_URL is deprecated; use DIB_SSACLI_URL instead." fi -rm -f /tmp/hpssacli.rpm +DIB_SSACLI_URL=${DIB_SSACLI_URL:-${DIB_HPSSACLI_URL:-https://downloads.hpe.com/pub/softlib2/software1/pubsw-linux/p1857046646/v123474/ssacli-2.60-19.0.x86_64.rpm}} +curl -k -o /tmp/ssacli.rpm $DIB_SSACLI_URL + +if [[ $DISTRO_NAME = "ubuntu" || $DISTRO_NAME = "debian" ]]; then + # There is no deb package for ssacli. Install with alien. + alien -i /tmp/ssacli.rpm +else + rpm -iv /tmp/ssacli.rpm +fi + +rm -f /tmp/ssacli.rpm # Install proliantutils python module in the # virtual environment of ironic-python-agent. diff --git a/diskimage_builder/elements/pypi/pre-install.d/00-configure-pypi-mirror b/diskimage_builder/elements/pypi/pre-install.d/00-configure-pypi-mirror index b2124d0b7..fc177a8f3 100755 --- a/diskimage_builder/elements/pypi/pre-install.d/00-configure-pypi-mirror +++ b/diskimage_builder/elements/pypi/pre-install.d/00-configure-pypi-mirror @@ -45,7 +45,7 @@ def main(): if use_pypi_python_org: indices.append('https://pypi.python.org/simple') retries = os.environ.get('DIB_PIP_RETRIES') - with file(home + '/.pip/pip.conf', 'wt') as output: + with open(home + '/.pip/pip.conf', 'wt') as output: output.write('[global]\n') output.write('log = %s/pip.log\n' % (home,)) output.write('index-url = %s\n' % (indices[0],)) @@ -53,7 +53,7 @@ def main(): output.write('retries = %s\n' % retries) for index in indices[1:]: output.write('extra-index-url = %s\n' % (index,)) - with file(home + '/.pydistutils.cfg', 'wt') as output: + with open(home + '/.pydistutils.cfg', 'wt') as output: output.write('[easy_install]\n') output.write('index_url = %s\n' % (easy_index,)) diff --git a/diskimage_builder/elements/svc-map/bin/svc-map b/diskimage_builder/elements/svc-map/bin/svc-map index 034c1fca8..2df6ba912 100755 --- a/diskimage_builder/elements/svc-map/bin/svc-map +++ b/diskimage_builder/elements/svc-map/bin/svc-map @@ -24,7 +24,7 @@ def load_service_mapping(filepath="/usr/share/svc-map/services"): if not os.path.isfile(filepath): return {} with open(filepath, 'r') as data_file: - return yaml.load(data_file.read()) + return yaml.safe_load(data_file.read()) def main(): diff --git a/diskimage_builder/elements/svc-map/extra-data.d/10-merge-svc-map-files b/diskimage_builder/elements/svc-map/extra-data.d/10-merge-svc-map-files index cd026007d..bf36c8aa2 100755 --- a/diskimage_builder/elements/svc-map/extra-data.d/10-merge-svc-map-files +++ b/diskimage_builder/elements/svc-map/extra-data.d/10-merge-svc-map-files @@ -66,7 +66,7 @@ def main(): data_path = os.path.join(path, "svc-map") if os.path.exists(data_path): with open(data_path, 'r') as dataFile: - data = yaml.load(dataFile.read()) + data = yaml.safe_load(dataFile.read()) try: service_names = merge_data( data, diff --git a/diskimage_builder/elements/ubuntu-minimal/README.rst b/diskimage_builder/elements/ubuntu-minimal/README.rst index 23764a45b..49a5e9380 100644 --- a/diskimage_builder/elements/ubuntu-minimal/README.rst +++ b/diskimage_builder/elements/ubuntu-minimal/README.rst @@ -6,7 +6,7 @@ Note: The ubuntu element is likely what you want unless you really know you want this one for some reason. The ubuntu element gets a lot more testing coverage and use. -Create a minimal image based on Ubuntu. We default to trusty but DIB_RELEASE +Create a minimal image based on Ubuntu. We default to xenial but DIB_RELEASE is mapped to any series of Ubuntu. If necessary, a custom apt keyring and debootstrap script can be diff --git a/diskimage_builder/elements/ubuntu-minimal/environment.d/10-ubuntu-distro-name.bash b/diskimage_builder/elements/ubuntu-minimal/environment.d/10-ubuntu-distro-name.bash index e9ae083e3..829952b31 100644 --- a/diskimage_builder/elements/ubuntu-minimal/environment.d/10-ubuntu-distro-name.bash +++ b/diskimage_builder/elements/ubuntu-minimal/environment.d/10-ubuntu-distro-name.bash @@ -1,4 +1,8 @@ export DISTRO_NAME=ubuntu export DIB_RELEASE=${DIB_RELEASE:-xenial} export DIB_DEBIAN_COMPONENTS=${DIB_DEBIAN_COMPONENTS:-main,restricted,universe} + +if [ -n "${DIB_UBUNTU_DISTRIBUTION_MIRROR:-}" ]; then + DIB_DISTRIBUTION_MIRROR=$DIB_UBUNTU_DISTRIBUTION_MIRROR +fi export DIB_DISTRIBUTION_MIRROR=${DIB_DISTRIBUTION_MIRROR:-http://archive.ubuntu.com/ubuntu} diff --git a/diskimage_builder/elements/ubuntu-minimal/pre-install.d/01-ubuntu-apt-update b/diskimage_builder/elements/ubuntu-minimal/pre-install.d/01-ubuntu-apt-update index db487e747..f5e8ca4ce 100755 --- a/diskimage_builder/elements/ubuntu-minimal/pre-install.d/01-ubuntu-apt-update +++ b/diskimage_builder/elements/ubuntu-minimal/pre-install.d/01-ubuntu-apt-update @@ -21,14 +21,12 @@ fi set -eu set -o pipefail -DIB_DISTRIBUTION_MIRROR=${DIB_DISTRIBUTION_MIRROR:-http://archive.ubuntu.com/ubuntu} - # We should manage this in a betterer way cat << EOF >/etc/apt/sources.list -deb $DIB_DISTRIBUTION_MIRROR $DIB_RELEASE main restricted universe -deb $DIB_DISTRIBUTION_MIRROR $DIB_RELEASE-updates main restricted universe -deb $DIB_DISTRIBUTION_MIRROR $DIB_RELEASE-backports main restricted universe -deb $DIB_DISTRIBUTION_MIRROR $DIB_RELEASE-security main restricted universe +deb $DIB_DISTRIBUTION_MIRROR $DIB_RELEASE ${DIB_DEBIAN_COMPONENTS//,/ } +deb $DIB_DISTRIBUTION_MIRROR $DIB_RELEASE-updates ${DIB_DEBIAN_COMPONENTS//,/ } +deb $DIB_DISTRIBUTION_MIRROR $DIB_RELEASE-backports ${DIB_DEBIAN_COMPONENTS//,/ } +deb $DIB_DISTRIBUTION_MIRROR $DIB_RELEASE-security ${DIB_DEBIAN_COMPONENTS//,/ } EOF # Need to update to retrieve the signed Release file diff --git a/diskimage_builder/elements/ubuntu-minimal/root.d/75-ubuntu-minimal-baseinstall b/diskimage_builder/elements/ubuntu-minimal/root.d/75-ubuntu-minimal-baseinstall index 359389957..24c80fa4e 100755 --- a/diskimage_builder/elements/ubuntu-minimal/root.d/75-ubuntu-minimal-baseinstall +++ b/diskimage_builder/elements/ubuntu-minimal/root.d/75-ubuntu-minimal-baseinstall @@ -21,14 +21,12 @@ fi set -eu set -o pipefail -DIB_DISTRIBUTION_MIRROR=${DIB_DISTRIBUTION_MIRROR:-http://archive.ubuntu.com/ubuntu} - # We should manage this in a betterer way sudo bash -c "cat << EOF >$TARGET_ROOT/etc/apt/sources.list -deb $DIB_DISTRIBUTION_MIRROR $DIB_RELEASE main restricted universe -deb $DIB_DISTRIBUTION_MIRROR $DIB_RELEASE-updates main restricted universe -deb $DIB_DISTRIBUTION_MIRROR $DIB_RELEASE-backports main restricted universe -deb $DIB_DISTRIBUTION_MIRROR $DIB_RELEASE-security main restricted universe +deb $DIB_DISTRIBUTION_MIRROR $DIB_RELEASE ${DIB_DEBIAN_COMPONENTS//,/ } +deb $DIB_DISTRIBUTION_MIRROR $DIB_RELEASE-updates ${DIB_DEBIAN_COMPONENTS//,/ } +deb $DIB_DISTRIBUTION_MIRROR $DIB_RELEASE-backports ${DIB_DEBIAN_COMPONENTS//,/ } +deb $DIB_DISTRIBUTION_MIRROR $DIB_RELEASE-security ${DIB_DEBIAN_COMPONENTS//,/ } EOF" sudo mount -t proc none $TARGET_ROOT/proc diff --git a/diskimage_builder/elements/yum-minimal/README.rst b/diskimage_builder/elements/yum-minimal/README.rst index ebe6be3cf..e68d704d0 100644 --- a/diskimage_builder/elements/yum-minimal/README.rst +++ b/diskimage_builder/elements/yum-minimal/README.rst @@ -9,10 +9,6 @@ or fedora-minimal elements to get an actual base image. Use of this element will require 'yum' and 'yum-utils' to be installed on Ubuntu and Debian. Nothing additional is needed on Fedora or CentOS. -The `DIB_OFFLINE` or more specific `DIB_YUMCHROOT_USE_CACHE` -variables can be set to prefer the use of a pre-cached root filesystem -tarball. - If you wish to have DHCP networking setup for eth0 & eth1 via /etc/sysconfig/network-config scripts/ifcfg-eth[0|1], set the environment variable `DIB_YUM_MINIMAL_CREATE_INTERFACES` to `1`. diff --git a/diskimage_builder/elements/yum-minimal/root.d/08-yum-chroot b/diskimage_builder/elements/yum-minimal/root.d/08-yum-chroot index 7760897b6..ee96e552b 100755 --- a/diskimage_builder/elements/yum-minimal/root.d/08-yum-chroot +++ b/diskimage_builder/elements/yum-minimal/root.d/08-yum-chroot @@ -31,8 +31,6 @@ if [ $ARCH = amd64 ]; then ARCH=x86_64 fi # Calling elements will need to set DISTRO_NAME and DIB_RELEASE -DIB_YUMCHROOT_EXTRA_ARGS=${DIB_YUMCHROOT_EXTRA_ARGS:-} -YUMCHROOT_TARBALL=$DIB_IMAGE_CACHE/yumchroot-${DISTRO_NAME}-${DIB_RELEASE}-${ARCH}.tar.gz # TODO Maybe deal with DIB_DISTRIBUTION_MIRROR http_proxy=${http_proxy:-} YUM=${YUM:-yum} @@ -189,107 +187,97 @@ function _install_pkg_manager { $TARGET_ROOT/etc/yum.repos.d/*repo } -if [ -n "$DIB_OFFLINE" -o -n "${DIB_YUMCHROOT_USE_CACHE:-}" ] && [ -f $YUMCHROOT_TARBALL ] ; then - echo $YUMCHROOT_TARBALL found in cache. Using. - sudo tar -C $TARGET_ROOT --numeric-owner -xzf $YUMCHROOT_TARBALL +# Note this is not usually done for root.d elements (see +# lib/common-functions:mount_proc_dev_sys) but it's important that +# we have things like /dev/urandom around inside the chroot for +# the rpm [pre|post]inst scripts within the packages. +sudo mkdir -p $TARGET_ROOT/proc $TARGET_ROOT/dev $TARGET_ROOT/sys +sudo mount -t proc none $TARGET_ROOT/proc +sudo mount --bind /dev $TARGET_ROOT/dev +sudo mount --bind /dev/pts $TARGET_ROOT/dev/pts +sudo mount -t sysfs none $TARGET_ROOT/sys + +# initalize rpmdb +sudo mkdir -p $TARGET_ROOT/var/lib/rpm +sudo $_RPM --root $TARGET_ROOT --initdb + +# this makes sure that running yum/dnf in the chroot it can get +# out to download stuff +sudo mkdir $TARGET_ROOT/etc +sudo cp /etc/resolv.conf $TARGET_ROOT/etc/resolv.conf + +# Bind mount the external yum cache inside the chroot. Same logic +# as in the yum element to provide for yum caching copied here +# because the sequencing is wrong otherwise +sudo mkdir -p $TMP_MOUNT_PATH/tmp/yum +sudo mount --bind $YUM_CACHE $TMP_MOUNT_PATH/tmp/yum + +_install_repos + +# install dnf for >= f22 +if [ $DIB_RELEASE -ge 22 ]; then + _install_pkg_manager dnf dnf-plugins-core yum else - # Note this is not usually done for root.d elements (see - # lib/common-functions:mount_proc_dev_sys) but it's important that - # we have things like /dev/urandom around inside the chroot for - # the rpm [pre|post]inst scripts within the packages. - sudo mkdir -p $TARGET_ROOT/proc $TARGET_ROOT/dev $TARGET_ROOT/sys - sudo mount -t proc none $TARGET_ROOT/proc - sudo mount --bind /dev $TARGET_ROOT/dev - sudo mount --bind /dev/pts $TARGET_ROOT/dev/pts - sudo mount -t sysfs none $TARGET_ROOT/sys - - # initalize rpmdb - sudo mkdir -p $TARGET_ROOT/var/lib/rpm - sudo $_RPM --root $TARGET_ROOT --initdb - - # this makes sure that running yum/dnf in the chroot it can get - # out to download stuff - sudo mkdir $TARGET_ROOT/etc - sudo cp /etc/resolv.conf $TARGET_ROOT/etc/resolv.conf - - # Bind mount the external yum cache inside the chroot. Same logic - # as in the yum element to provide for yum caching copied here - # because the sequencing is wrong otherwise - sudo mkdir -p $TMP_MOUNT_PATH/tmp/yum - sudo mount --bind $YUM_CACHE $TMP_MOUNT_PATH/tmp/yum - - _install_repos - - # install dnf for >= f22 - if [ $DIB_RELEASE -ge 22 ]; then - _install_pkg_manager dnf dnf-plugins-core yum - else - _install_pkg_manager yum - fi - - # we just installed yum/dnf with "outside" tools (yum/rpm) which - # might have created /var/lib/[yum|rpm] (etc) that are slighlty - # incompatible. Refresh everything with the in-chroot tools - sudo -E chroot $TARGET_ROOT rpm --rebuilddb - sudo -E chroot $TARGET_ROOT ${YUM} clean all - - # populate the lang reduction macro in the chroot - echo "%_install_langs C:en_US:en_US.UTF-8" | \ - sudo tee -a $TARGET_ROOT/etc/rpm/macros.langs > /dev/null - - # bootstrap the environment within the chroot; bring in new - # metadata with an update and install some base packages we need. - sudo -E chroot $TARGET_ROOT ${YUM} -y update - sudo -E chroot $TARGET_ROOT ${YUM} -y \ - --setopt=cachedir=/tmp/yum/$ARCH/$DIB_RELEASE \ - install systemd passwd findutils sudo util-linux-ng - - # This package is split out from systemd on >F24, dracut is - # missing the dependency and will fail to make an initrd without - # it; see - # https://bugzilla.redhat.com/show_bug.cgi?id=1398505 - if [ $DISTRO_NAME = "fedora" -a $DIB_RELEASE -ge 24 ]; then - sudo -E chroot $TARGET_ROOT ${YUM} -y \ - --setopt=cachedir=/tmp/yum/$ARCH/$DIB_RELEASE \ - install systemd-udev - fi - - # Put in a dummy /etc/resolv.conf over the temporary one we used - # to bootstrap. systemd has a bug/feature [1] that it will assume - # you want systemd-networkd as the network manager and create a - # broken symlink to /run/... if the base image doesn't have one. - # This broken link confuses things like dhclient. - # [1] https://bugzilla.redhat.com/show_bug.cgi?id=1197204 - echo -e "# This file intentionally left blank\n" | \ - sudo tee $TARGET_ROOT/etc/resolv.conf - - # set the most reliable UTF-8 locale - echo -e 'LANG="en_US.UTF-8"' | \ - sudo tee $TARGET_ROOT/etc/locale.conf - # default to UTC - sudo -E chroot $TARGET_ROOT ln -sf /usr/share/zoneinfo/UTC \ - /etc/localtime - - # cleanup - # TODO : move this into a exit trap; and reconsider how - # this integrates with the global exit cleanup path. - sudo umount $TMP_MOUNT_PATH/tmp/yum - sudo umount $TARGET_ROOT/proc - sudo umount $TARGET_ROOT/dev/pts - sudo umount $TARGET_ROOT/dev - sudo umount $TARGET_ROOT/sys - - # RPM doesn't know whether files have been changed since install - # At this point though, we know for certain that we have changed no - # config files, so anything marked .rpmnew is just a bug. - for newfile in $(sudo find $TARGET_ROOT -type f -name '*rpmnew') ; do - sudo mv $newfile $(echo $newfile | sed 's/.rpmnew$//') - done - - echo Caching result in $YUMCHROOT_TARBALL - sudo tar --numeric-owner \ - -C $TARGET_ROOT \ - -zcf $YUMCHROOT_TARBALL --exclude='./tmp/*' . + _install_pkg_manager yum fi +# we just installed yum/dnf with "outside" tools (yum/rpm) which +# might have created /var/lib/[yum|rpm] (etc) that are slighlty +# incompatible. Refresh everything with the in-chroot tools +sudo -E chroot $TARGET_ROOT rpm --rebuilddb +sudo -E chroot $TARGET_ROOT ${YUM} clean all + +# populate the lang reduction macro in the chroot +echo "%_install_langs C:en_US:en_US.UTF-8" | \ + sudo tee -a $TARGET_ROOT/etc/rpm/macros.langs > /dev/null + +# bootstrap the environment within the chroot; bring in new +# metadata with an update and install some base packages we need. +sudo -E chroot $TARGET_ROOT ${YUM} -y update +sudo -E chroot $TARGET_ROOT ${YUM} -y \ + --setopt=cachedir=/tmp/yum/$ARCH/$DIB_RELEASE \ + install systemd passwd findutils sudo util-linux-ng + +# This package is split out from systemd on >F24, dracut is +# missing the dependency and will fail to make an initrd without +# it; see +# https://bugzilla.redhat.com/show_bug.cgi?id=1398505 +if [ $DISTRO_NAME = "fedora" -a $DIB_RELEASE -ge 24 ]; then + sudo -E chroot $TARGET_ROOT ${YUM} -y \ + --setopt=cachedir=/tmp/yum/$ARCH/$DIB_RELEASE \ + install systemd-udev +fi + +# Put in a dummy /etc/resolv.conf over the temporary one we used +# to bootstrap. systemd has a bug/feature [1] that it will assume +# you want systemd-networkd as the network manager and create a +# broken symlink to /run/... if the base image doesn't have one. +# This broken link confuses things like dhclient. +# [1] https://bugzilla.redhat.com/show_bug.cgi?id=1197204 +echo -e "# This file intentionally left blank\n" | \ + sudo tee $TARGET_ROOT/etc/resolv.conf + +# set the most reliable UTF-8 locale +echo -e 'LANG="en_US.UTF-8"' | \ + sudo tee $TARGET_ROOT/etc/locale.conf + # default to UTC +sudo -E chroot $TARGET_ROOT ln -sf /usr/share/zoneinfo/UTC \ + /etc/localtime + +# cleanup +# TODO : move this into a exit trap; and reconsider how +# this integrates with the global exit cleanup path. +sudo umount $TMP_MOUNT_PATH/tmp/yum +sudo umount $TARGET_ROOT/proc +sudo umount $TARGET_ROOT/dev/pts +sudo umount $TARGET_ROOT/dev +sudo umount $TARGET_ROOT/sys + +# RPM doesn't know whether files have been changed since install +# At this point though, we know for certain that we have changed no +# config files, so anything marked .rpmnew is just a bug. +for newfile in $(sudo find $TARGET_ROOT -type f -name '*rpmnew') ; do + sudo mv $newfile $(echo $newfile | sed 's/.rpmnew$//') +done + sudo rm -f ${TARGET_ROOT}/.extra_settings diff --git a/diskimage_builder/lib/common-functions b/diskimage_builder/lib/common-functions index b5b6b234e..6cdfd7e67 100644 --- a/diskimage_builder/lib/common-functions +++ b/diskimage_builder/lib/common-functions @@ -143,24 +143,44 @@ function eval_run_d () { trap - ERR } +# Get any process that appears to be running in $TMP_BUILD_DIR +function _get_chroot_processes () { + # Deselect kernel threads, and use a python script to avoid + # forking lots and lots of readlink / grep processes on a busy + # system. + ps --ppid 2 -p 2 --deselect -o pid= | xargs python -c ' +import os +import sys + +for pid in sys.argv[2:]: + try: + root = os.readlink("/proc/%s/root" % pid) + except: + continue + if sys.argv[1] in root: + print("%s" % pid) +' $TMP_BUILD_DIR +} + + function kill_chroot_processes () { local xtrace xtrace=$(set +o | grep xtrace) set +o xtrace + local pidname + if [ -z "${1}" ]; then echo "ERROR: no chroot directory specified" exit 1 fi - for piddir in /proc/[0-9]*; do - pid=${piddir##/proc/} - pidname=$(cat $piddir/comm 2>/dev/null || echo "unknown") + + for pid in $(_get_chroot_processes); do # If there are open files from the chroot, just kill the process using - # these files. - if sudo readlink -f $piddir/root | grep -q $TMP_BUILD_DIR; then - echo "Killing chroot process: '${pidname}($pid)'" - sudo kill $pid - fi + # these files. This is racy, but good enough + pidname=$(cat $piddir/comm 2>/dev/null || echo "unknown") + echo "Killing chroot process: '${pidname}($pid)'" + sudo kill $pid done $xtrace diff --git a/diskimage_builder/lib/img-functions b/diskimage_builder/lib/img-functions index 2ab320ec4..38f758a5e 100644 --- a/diskimage_builder/lib/img-functions +++ b/diskimage_builder/lib/img-functions @@ -117,6 +117,10 @@ function finalise_base () { unmount_dir $TMP_MOUNT_PATH/tmp fi find $TMP_MOUNT_PATH/tmp -maxdepth 1 -mindepth 1 | xargs sudo rm -rf --one-file-system + # Truncate /var/log files in preparation for first boot + sudo find ${TMP_MOUNT_PATH}/var/log -type f -exec cp /dev/null '{}' \; + # also /root logs + sudo find ${TMP_MOUNT_PATH}/root -name \*.log -type f -delete } function compress_and_save_image () { diff --git a/doc/source/developer/components.rst b/doc/source/developer/components.rst index 35c9c7225..8720623d4 100644 --- a/doc/source/developer/components.rst +++ b/doc/source/developer/components.rst @@ -22,7 +22,7 @@ Components To generate kernel+ramdisk pair for use with ironic, use:: - ramdisk-image-create -o deploy.ramdisk deploy-ironic + ramdisk-image-create -o deploy.ramdisk ironic-agent `element-info` diff --git a/doc/source/specs/v1/approved/block-device-lvl1-partitioning.rst b/doc/source/specs/v1/approved/block-device-lvl1-partitioning.rst new file mode 100644 index 000000000..ec17e9eb7 --- /dev/null +++ b/doc/source/specs/v1/approved/block-device-lvl1-partitioning.rst @@ -0,0 +1,225 @@ +.. + This work is licensed under a Creative Commons Attribution 3.0 Unported + License. + + http://creativecommons.org/licenses/by/3.0/legalcode + +======================================== +Block Device Setup Level 1: Partitioning +======================================== + +During the creation of a disk image (e.g. for a VM), there is the need +to create, setup, configure and afterwards detach some kind of storage +where the newly installed OS can be copied to or directly installed +in. + +Remark +------ + +The implementation for this proposed changed already exists, was +discussed and is currently waiting for reviews [1]. To have a +complete overview over the block device setup, this document is +provided. +The dependencies are not implemented as they should be, because +* the spec process is currently in the phase of discussion and not + finalized [2], +* the implementation was finished and reviewed before the spec process + was described. [1] + +Problem description +=================== + +When setting up a block device there is the need to partitioning the +block device. + +Use Cases +--------- + +User (Actor: End User) wants to create multiple partitions in multiple +block devices where the new system is installed in. + +The user wants to specify if the image should be optimized for speed +or for size. + +The user wants the same behavior independently of the current host or +target OS. + +Proposed change +=============== + +Move the partitioning functionality from +`elements/vm/block-device.d/10-partition` to a new block_device +python module: `level1/partitioning.py`. + +Instead of using a program or a library, the data is written directly +with the help of python `file.write()` into the disk image. + +Alternatives +------------ + +The existing implementation uses the `parted` program (old versions of +DIB were using `sfdisk`). The first implementations of this change +used the python-parted library. + +All these approaches have a major drawback: they automatically +*optimize* based on information collected on the host system - and not +of the target system. Therefore the resulting partitioning layout may +lead to a degradation of performance on the target system. A change +in these external programs and libraries also lead to errors during a +DIB run [4] or there are general issues [7]. + +Also everything build around GNU parted falls under the GPL2 (not +LGPL2) license - which is incompatible with the currently used Apache +license in diskimage-builder. + +API impact +---------- + +Extends the (optional) environment variable +``DIB_BLOCK_DEVICE_CONFIG``: a JSON structure to configure the +(complete) block device setup. For this proposal the second entry in +the original list will be used (the first part (as described in [5]) +is used by the level 0 modules). + +The name of this module is `partitioning` (element[0]). The value +(element[1]) is a dictionary. + +For each disk that should be partitioned there exists one entry in the +dictionary. The key is the name of the disk (see [5] how to specify +names for block device level 0). The value is a dictionary that +defines the partitioning of each disk. + +There are the following key / value pairs to define one disk: + +label + (mandatory) Possible values: 'mbr' + This uses the Master Boot Record (MBR) layout for the disk. + (Later on this can be extended, e.g. using GPT). + +align + (optional - default value '1MiB') + Set the alignment of the partition. This must be a multiple of the + block size (i.e. 512 bytes). The default of 1MiB (~ 2048 * 512 + bytes blocks) is the default for modern systems and known to + perform well on a wide range of targets [6]. For each partition + there might be some space that is not used - which is `align` - 512 + bytes. For the default of 1MiB exactly 1048064 bytes (= 1 MiB - + 512 byte) are not used in the partition itself. Please note that + if a boot loader should be written to the disk or partition, + there is a need for some space. E.g. grub needs 63 * 512 byte + blocks between the MBR and the start of the partition data; this + means when grub will be installed, the `align` must be set at least + to 64 * 512 byte = 32 KiB. + +partitions + (mandatory) A list of dictionaries. Each dictionary describes one + partition. + +The following key / value pairs can be given for each partition: + +name + (mandatory) The name of the partition. With the help of this name, + the partition can later be referenced, e.g. while creating a + file system. + +flags + (optional) List of flags for the partition. Default: empty. + Possible values: + + boot + Sets the boot flag for the partition + +size + (mandatory) The size of the partition. The size can either be an + absolute number using units like `10GiB` or `1.75TB` or relative + (percentage) numbers: in the later case the size is calculated + based on the remaining free space. + +Example: + +:: + ["partitioning", + {"rootdisk": { + "label": "mbr", + "partitions": + [{"name": "part-01", + "flags": ["boot"], + "size": "100%"}]}}] + +Security impact +--------------- + +None - functionality stays the same. + +Other end user impact +--------------------- + +None. + +Performance Impact +------------------ + +Measurements showed there is a performance degradation for the target +system of the partition table is not correctly aligned: writing takes +about three times longer on an incorrect aligned system vs. one that +is correctly aligned. + +Implementation +============== + +Assignee(s) +----------- + +Primary assignee: + ansreas (andreas@florath.net) + +Work Items +---------- + +None - this is already a small part of a bigger change [1]. + +Dependencies +============ + +None. + +Testing +======= + +The refactoring introduces no new test cases: the functionality is +tested during each existing test building VM images. + +Documentation Impact +==================== + +End user: the additional environment variable is described. + +References +========== + +[1] Refactor: block-device handling (partitioning) + https://review.openstack.org/322671 + +[2] Add specs dir + https://review.openstack.org/336109 + +[3] Old implementation using parted-lib + https://review.openstack.org/#/c/322671/1..7/elements/block-device/pylib/block-device/level1/Partitioning.py + +[4] ERROR: embedding is not possible, but this is required + for cross-disk install + http://lists.openstack.org/pipermail/openstack-dev/2016-June/097789.html + +[5] Refactor: block-device handling (local loop) + https://review.openstack.org/319591 + +[6] Proper alignment of partitions on an Advanced Format HDD using Parted + http://askubuntu.com/questions/201164/proper-alignment-of-partitions-on-an-advanced-format-hdd-using-parted + +[7] Red Hat Enterprise Linux 6 - Creating a 7TB Partition Using + parted Always Shows "The resulting partition is not properly + aligned for best performance" + http://h20564.www2.hpe.com/hpsc/doc/public/display?docId=emr_na-c03479326&DocLang=en&docLocale=en_US&jumpid=reg_r11944_uken_c-001_title_r0001 + +[8] Spec for changing the block device handling + https://review.openstack.org/336946 diff --git a/elements/pip-and-virtualenv/test-elements/source-install-fedora/element-deps b/elements/pip-and-virtualenv/test-elements/source-install-fedora/element-deps new file mode 100644 index 000000000..b767efbb8 --- /dev/null +++ b/elements/pip-and-virtualenv/test-elements/source-install-fedora/element-deps @@ -0,0 +1 @@ +fedora-minimal diff --git a/elements/pip-and-virtualenv/test-elements/source-install-fedora/environment.d/10-source-installtype-pip-and-virtualenv b/elements/pip-and-virtualenv/test-elements/source-install-fedora/environment.d/10-source-installtype-pip-and-virtualenv new file mode 100644 index 000000000..0f27f00cc --- /dev/null +++ b/elements/pip-and-virtualenv/test-elements/source-install-fedora/environment.d/10-source-installtype-pip-and-virtualenv @@ -0,0 +1 @@ +export DIB_INSTALLTYPE_pip_and_virtualenv=source diff --git a/elements/pip-and-virtualenv/test-elements/source-install-ubuntu/element-deps b/elements/pip-and-virtualenv/test-elements/source-install-ubuntu/element-deps new file mode 100644 index 000000000..fe735d046 --- /dev/null +++ b/elements/pip-and-virtualenv/test-elements/source-install-ubuntu/element-deps @@ -0,0 +1 @@ +ubuntu-minimal diff --git a/elements/pip-and-virtualenv/test-elements/source-install-ubuntu/environment.d/10-source-installtype-pip-and-virtualenv b/elements/pip-and-virtualenv/test-elements/source-install-ubuntu/environment.d/10-source-installtype-pip-and-virtualenv new file mode 100644 index 000000000..0f27f00cc --- /dev/null +++ b/elements/pip-and-virtualenv/test-elements/source-install-ubuntu/environment.d/10-source-installtype-pip-and-virtualenv @@ -0,0 +1 @@ +export DIB_INSTALLTYPE_pip_and_virtualenv=source diff --git a/releasenotes/notes/move_tidy_logs_to_main-a8c03427fe1a445c.yaml b/releasenotes/notes/move_tidy_logs_to_main-a8c03427fe1a445c.yaml new file mode 100644 index 000000000..fbd729561 --- /dev/null +++ b/releasenotes/notes/move_tidy_logs_to_main-a8c03427fe1a445c.yaml @@ -0,0 +1,8 @@ +--- +features: + - Cleaning logs was split, some was done in the + img-functions.finalise_base, some was done in the base element. + The version unifies tidy up logs in the lib/img-functions. + Especially when building docker container images the base element + cannot be used. This patch removes about some hundreds KB of + useless logs in cases when the base element is not used. \ No newline at end of file diff --git a/releasenotes/notes/yum-cache-removal-148c33012515e56e.yaml b/releasenotes/notes/yum-cache-removal-148c33012515e56e.yaml new file mode 100644 index 000000000..86727a081 --- /dev/null +++ b/releasenotes/notes/yum-cache-removal-148c33012515e56e.yaml @@ -0,0 +1,9 @@ +--- +deprecations: + - The ``DIB_YUMCHROOT_USE_CACHE`` variable has been removed and the + Fedora and CentOS ``-minimal`` initial chroot will always be + created by the package manager. The default creation of a chroot + tarball is stopped for these elements. This unused option was + unsafe; there is no guarantee that the base system will not change + even between runs. Getting the package manager to reuse the cache + for the initial chroot install is future work. diff --git a/test-requirements.txt b/test-requirements.txt index f9f2a6b81..a30316a7e 100644 --- a/test-requirements.txt +++ b/test-requirements.txt @@ -12,3 +12,5 @@ oslosphinx>=4.7.0 # Apache-2.0 # releasenotes reno>=1.8.0 # Apache-2.0 + +coverage>=4.0 # Apache-2.0 diff --git a/tests/run_functests.sh b/tests/run_functests.sh index 7362e932a..66e83a1d6 100755 --- a/tests/run_functests.sh +++ b/tests/run_functests.sh @@ -102,8 +102,8 @@ function run_disk_element_test() { break_cmd="cp -v \$TMP_MOUNT_PATH/tmp/dib-test-should-fail ${dest_dir} || true" \ DIB_SHOW_IMAGE_USAGE=1 \ ELEMENTS_PATH=$DIB_ELEMENTS/$element/test-elements \ - $DIB_CMD -x -t tar,qcow2 ${use_tmp_flag} -o $dest_dir/image -n $element $test_element 2>&1 \ - | log_with_prefix "${element}/${test_element}"; then + $DIB_CMD -x -t tar,qcow2 ${use_tmp_flag} -o $dest_dir/image -n $element $test_element 2>&1 \ + | log_with_prefix "${element}/${test_element}"; then if ! [ -f "$dest_dir/image.qcow2" ]; then echo "Error: qcow2 build failed for element: $element, test-element: $test_element." diff --git a/tox.ini b/tox.ini index e1b0f8bda..e14ffe52a 100644 --- a/tox.ini +++ b/tox.ini @@ -9,8 +9,7 @@ install_command = pip install -U {opts} {packages} deps= -r{toxinidir}/requirements.txt -r{toxinidir}/test-requirements.txt commands= - python setup.py testr --slowest --testr-args='{posargs}' -passenv = ELEMENTS_PATH DIB_RELEASE DIB_DEBUG_TRACE DIB_DEV_USER_USERNAME DIB_DEV_USER_PWDLESS_SUDO DIB_DEV_USER_PASSWORD USER HOME http_proxy https_proxy + python setup.py test --slowest --testr-args='{posargs}' [testenv:pep8] commands = @@ -25,9 +24,10 @@ envdir = {toxworkdir}/venv commands = {toxinidir}/tests/run_functests.sh {posargs} [testenv:cover] -setenv = PYTHON=coverage run --source diskimage_builder -commands = bash -c 'if [ ! -d ./.testrepository ] ; then testr init ; fi' - bash -c 'testr run --parallel ; RET=$? ; coverage combine ; coverage html -d ./cover $OMIT && exit $RET' +# NOTE: this is "setup.py test" (*not* testr) which is a pbr wrapper +# around testr. This understands --coverage-package-name which we +# need due to underscore issues. +commands = python setup.py test --coverage --coverage-package-name diskimage_builder --testr-args='{posargs}' [testenv:docs] commands = python setup.py build_sphinx