diff --git a/diskimage_builder/elements/debian-minimal/package-installs.yaml b/diskimage_builder/elements/debian-minimal/package-installs.yaml index d86cad09c..9e08e6a5d 100644 --- a/diskimage_builder/elements/debian-minimal/package-installs.yaml +++ b/diskimage_builder/elements/debian-minimal/package-installs.yaml @@ -1,2 +1,6 @@ +linux-image-amd64: + arch: amd64 +linux-image-686: + arch: i386 systemd: systemd-sysv: diff --git a/diskimage_builder/elements/debian-minimal/pre-install.d/02-debian-apt-update b/diskimage_builder/elements/debian-minimal/root.d/75-debian-minimal-baseinstall similarity index 75% rename from diskimage_builder/elements/debian-minimal/pre-install.d/02-debian-apt-update rename to diskimage_builder/elements/debian-minimal/root.d/75-debian-minimal-baseinstall index bacb11293..f12774d15 100755 --- a/diskimage_builder/elements/debian-minimal/pre-install.d/02-debian-apt-update +++ b/diskimage_builder/elements/debian-minimal/root.d/75-debian-minimal-baseinstall @@ -15,10 +15,6 @@ # # See the License for the specific language governing permissions and # limitations under the License. -# -# The filename needs to be 02-...: because the install-package script -# is installed in the dpkg/pre-install/01-dpkg and that has to be executed -# first. if [ ${DIB_DEBUG_TRACE:-0} -gt 0 ]; then set -x @@ -34,20 +30,28 @@ set -o pipefail function apt_sources_write { local APT_SOURCES_CONF="$1" - mkdir -p /etc/apt/sources.list.d + sudo mkdir -p $TARGET_ROOT/etc/apt/sources.list.d echo "${APT_SOURCES_CONF}" \ | while read line; do local name=$(echo ${line} | cut -d ":" -f 1) local value=$(echo ${line} | cut -d ":" -f 2-) - echo "$value" >>/etc/apt/sources.list.d/${name}.list + echo "$value" | sudo tee $TARGET_ROOT/etc/apt/sources.list.d/${name}.list done } +sudo mount -t proc none $TARGET_ROOT/proc +sudo mount -t sysfs none $TARGET_ROOT/sys +trap "sudo umount $TARGET_ROOT/proc; sudo umount $TARGET_ROOT/sys" EXIT + +apt_get="sudo chroot $TARGET_ROOT /usr/bin/apt-get" + apt_sources_write "${DIB_APT_SOURCES_CONF}" # Need to update to retrieve the signed Release file -apt-get update +$apt_get update -apt-get clean -install-packages -u +$apt_get clean +$apt_get dist-upgrade -y + +$apt_get install -y busybox python sudo diff --git a/diskimage_builder/elements/debootstrap/root.d/08-debootstrap b/diskimage_builder/elements/debootstrap/root.d/08-debootstrap index 8af7a91e5..36aa40ac9 100755 --- a/diskimage_builder/elements/debootstrap/root.d/08-debootstrap +++ b/diskimage_builder/elements/debootstrap/root.d/08-debootstrap @@ -28,23 +28,9 @@ fi [ -n "$DIB_RELEASE" ] [ -n "$DIB_DISTRIBUTION_MIRROR" ] -if [ $DISTRO_NAME = 'ubuntu' ] ; then - KERNEL='generic' -else - case $ARCH in - amd64) KERNEL='amd64' ;; - i386) KERNEL='686' ;; - arm7) - [ -n "$DIB_ARM_KERNEL" ] - KERNEL="$DIB_ARM_KERNEL" - ;; - esac -fi -KERNEL_PACKAGE="linux-image-$KERNEL" - DIB_DEBIAN_COMPONENTS=${DIB_DEBIAN_COMPONENTS:-main} DIB_DEBOOTSTRAP_EXTRA_ARGS=${DIB_DEBOOTSTRAP_EXTRA_ARGS:-} -DEBOOTSTRAP_TARBALL=$DIB_IMAGE_CACHE/debootstrap-${DISTRO_NAME}-${DIB_RELEASE}-${ARCH}-${KERNEL}.tar.gz +DEBOOTSTRAP_TARBALL=$DIB_IMAGE_CACHE/debootstrap-${DISTRO_NAME}-${DIB_RELEASE}-${ARCH}.tar.gz http_proxy=${http_proxy:-} no_proxy=${no_proxy:-} @@ -63,7 +49,6 @@ else # Have to --include=busybox because initramfs needs it sudo sh -c "http_proxy=$http_proxy no_proxy=$no_proxy debootstrap --verbose \ --variant=minbase \ - --include=python,sudo,busybox,$KERNEL_PACKAGE \ --components=${DIB_DEBIAN_COMPONENTS} \ --arch=${ARCH} \ $KEYRING_OPT \ @@ -79,6 +64,7 @@ else sudo rm -fr ${TARGET_ROOT}/etc/apt/sources.list \ ${TARGET_ROOT}/etc/apt/sources.list.d + echo Caching debootstrap result in $DEBOOTSTRAP_TARBALL if [ "${DIB_DEBOOTSTRAP_CACHE:-0}" != "0" ]; then sudo tar --numeric-owner -C $TARGET_ROOT -zcf $DEBOOTSTRAP_TARBALL --exclude='./tmp/*' . diff --git a/diskimage_builder/elements/dpkg/root.d/60-block-apt-translations b/diskimage_builder/elements/dpkg/root.d/50-block-apt-translations similarity index 100% rename from diskimage_builder/elements/dpkg/root.d/60-block-apt-translations rename to diskimage_builder/elements/dpkg/root.d/50-block-apt-translations diff --git a/diskimage_builder/elements/dpkg/root.d/99-block-daemons b/diskimage_builder/elements/dpkg/root.d/50-block-daemons similarity index 100% rename from diskimage_builder/elements/dpkg/root.d/99-block-daemons rename to diskimage_builder/elements/dpkg/root.d/50-block-daemons diff --git a/diskimage_builder/elements/dpkg/root.d/99-shared_apt_cache b/diskimage_builder/elements/dpkg/root.d/50-shared-apt-cache similarity index 100% rename from diskimage_builder/elements/dpkg/root.d/99-shared_apt_cache rename to diskimage_builder/elements/dpkg/root.d/50-shared-apt-cache diff --git a/diskimage_builder/elements/dpkg/root.d/99-trim-dpkg b/diskimage_builder/elements/dpkg/root.d/50-trim-dpkg similarity index 100% rename from diskimage_builder/elements/dpkg/root.d/99-trim-dpkg rename to diskimage_builder/elements/dpkg/root.d/50-trim-dpkg diff --git a/diskimage_builder/elements/element-manifest/README.rst b/diskimage_builder/elements/element-manifest/README.rst index 3ed20128a..69e622164 100644 --- a/diskimage_builder/elements/element-manifest/README.rst +++ b/diskimage_builder/elements/element-manifest/README.rst @@ -3,6 +3,6 @@ element-manifest ================ Writes a manifest file that is the full list of elements that were used to -build the image. The file path can be overriden by setting +build the image. The file path can be overridden by setting $DIB\_ELEMENT\_MANIFEST\_PATH, and defaults to /etc/dib-manifests/element-manifest. diff --git a/diskimage_builder/elements/package-installs/bin/package-installs-v2 b/diskimage_builder/elements/package-installs/bin/package-installs-v2 index b4a4d0d80..305e2a666 100755 --- a/diskimage_builder/elements/package-installs/bin/package-installs-v2 +++ b/diskimage_builder/elements/package-installs/bin/package-installs-v2 @@ -20,20 +20,28 @@ import subprocess import sys -def process_output(cmdline): - # Try to execute subprocess.check_output(), which is available - # in Python 2.7+, gracefully falling back to subprocess.Popen - # in older Python versions. - try: - return subprocess.check_output(cmdline).decode(encoding='utf-8') - except AttributeError: - proc = subprocess.Popen(cmdline, stdout=subprocess.PIPE) - out = proc.communicate()[0] - if proc.returncode: - e = subprocess.CalledProcessError(proc.returncode, cmdline) - e.output = out - raise e - return out +# run a command, return output +# if follow is set, output will be echoed to stdout +def process_output(cmdline, follow=False): + proc = subprocess.Popen(cmdline, stdout=subprocess.PIPE) + if follow: + print("Running command: %s" % cmdline) + out = "" + with proc.stdout: + for line in iter(proc.stdout.readline, b''): + out += line.decode('utf-8') + print("> %s" % line.strip()) + proc.wait() + print("> -- done") + else: + out = proc.communicate()[0].decode('utf-8') + + if proc.returncode: + e = subprocess.CalledProcessError(proc.returncode, cmdline) + e.output = out + raise e + + return out def main(): @@ -69,8 +77,7 @@ def main(): pkg_map_args = ['pkg-map', '--missing-ok', '--element', element, pkg] try: - map_output = process_output( - pkg_map_args) + map_output = process_output(pkg_map_args) pkgs.extend(map_output.strip().split('\n')) except subprocess.CalledProcessError as e: if e.returncode == 1: @@ -93,7 +100,7 @@ def main(): print(" ".join(install_args)) else: try: - process_output(install_args) + process_output(install_args, follow=True) except subprocess.CalledProcessError as e: print("install failed with error %s" % e.output) sys.exit(1) diff --git a/diskimage_builder/elements/pip-and-virtualenv/package-installs.yaml b/diskimage_builder/elements/pip-and-virtualenv/package-installs.yaml index 85c7f0ebc..6d38d1605 100644 --- a/diskimage_builder/elements/pip-and-virtualenv/package-installs.yaml +++ b/diskimage_builder/elements/pip-and-virtualenv/package-installs.yaml @@ -1,3 +1,7 @@ +# This package is only valid for SUSE and we need to pull it in +# early enough because pip depends on it. +python-xml: + phase: pre-install.d python-pip: installtype: package python-virtualenv: diff --git a/diskimage_builder/elements/pip-and-virtualenv/pkg-map b/diskimage_builder/elements/pip-and-virtualenv/pkg-map index 668d95174..508fd004a 100644 --- a/diskimage_builder/elements/pip-and-virtualenv/pkg-map +++ b/diskimage_builder/elements/pip-and-virtualenv/pkg-map @@ -3,6 +3,12 @@ "gentoo": { "python-pip": "dev-python/pip", "python-virtualenv": "dev-python/virtualenv" + }, + "suse" : { + "python-xml": "python-xml" } + }, + "default": { + "python-xml": "" } } diff --git a/diskimage_builder/elements/ubuntu-minimal/element-deps b/diskimage_builder/elements/ubuntu-minimal/element-deps index cba86060f..e078fde1f 100644 --- a/diskimage_builder/elements/ubuntu-minimal/element-deps +++ b/diskimage_builder/elements/ubuntu-minimal/element-deps @@ -1 +1,2 @@ debootstrap +package-installs diff --git a/diskimage_builder/elements/ubuntu-minimal/package-installs.yaml b/diskimage_builder/elements/ubuntu-minimal/package-installs.yaml new file mode 100644 index 000000000..83f64fcc5 --- /dev/null +++ b/diskimage_builder/elements/ubuntu-minimal/package-installs.yaml @@ -0,0 +1 @@ +linux-image-generic: diff --git a/diskimage_builder/elements/ubuntu-minimal/root.d/75-ubuntu-minimal-baseinstall b/diskimage_builder/elements/ubuntu-minimal/root.d/75-ubuntu-minimal-baseinstall new file mode 100755 index 000000000..6e6de8a0b --- /dev/null +++ b/diskimage_builder/elements/ubuntu-minimal/root.d/75-ubuntu-minimal-baseinstall @@ -0,0 +1,46 @@ +#!/bin/bash +# Copyright (c) 2014 Hewlett-Packard Development Company, L.P. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +# implied. +# +# See the License for the specific language governing permissions and +# limitations under the License. + +if [ ${DIB_DEBUG_TRACE:-0} -gt 0 ]; then + set -x +fi +set -eu +set -o pipefail + +DIB_DISTRIBUTION_MIRROR=${DIB_DISTRIBUTION_MIRROR:-http://archive.ubuntu.com/ubuntu} + +# We should manage this in a betterer way +sudo bash -c "cat << EOF >$TARGET_ROOT/etc/apt/sources.list +deb $DIB_DISTRIBUTION_MIRROR $DIB_RELEASE main restricted universe +deb $DIB_DISTRIBUTION_MIRROR $DIB_RELEASE-updates main restricted universe +deb $DIB_DISTRIBUTION_MIRROR $DIB_RELEASE-backports main restricted universe +deb $DIB_DISTRIBUTION_MIRROR $DIB_RELEASE-security main restricted universe +EOF" + +sudo mount -t proc none $TARGET_ROOT/proc +sudo mount -t sysfs none $TARGET_ROOT/sys +trap "sudo umount $TARGET_ROOT/proc; sudo umount $TARGET_ROOT/sys" EXIT + +apt_get="sudo chroot $TARGET_ROOT /usr/bin/apt-get" # dib-lint: safe_sudo + +# Need to update to retrieve the signed Release file +$apt_get update + +$apt_get clean +$apt_get dist-upgrade -y + +$apt_get install -y busybox python sudo diff --git a/diskimage_builder/elements/zypper-minimal/package-installs.yaml b/diskimage_builder/elements/zypper-minimal/package-installs.yaml index d2ccc1605..b80baaefa 100644 --- a/diskimage_builder/elements/zypper-minimal/package-installs.yaml +++ b/diskimage_builder/elements/zypper-minimal/package-installs.yaml @@ -4,6 +4,7 @@ linux-image-generic: # as dependencies but that may change so lets # be explicit. bash: +ca-certificates-mozilla: lsb-release: openssl: sed: diff --git a/diskimage_builder/lib/common-functions b/diskimage_builder/lib/common-functions index 7c09977d3..b5b6b234e 100644 --- a/diskimage_builder/lib/common-functions +++ b/diskimage_builder/lib/common-functions @@ -61,8 +61,9 @@ function finish_image () { mv $OUT_IMAGE_PATH $1 if [ "$DIB_CHECKSUM" == "1" ]; then - md5sum $1 > $1.md5 - sha256sum $1 > $1.sha256 + # NOTE(pabelanger): Read image into memory once and generate both checksum + # files. + md5sum $1 > $1.md5 & sha256sum $1 > $1.sha256 & wait fi echo "Image file $1 created..." } diff --git a/releasenotes/notes/package-outside-debootstrap-ac93e9ce991819f1.yaml b/releasenotes/notes/package-outside-debootstrap-ac93e9ce991819f1.yaml new file mode 100644 index 000000000..fc877b856 --- /dev/null +++ b/releasenotes/notes/package-outside-debootstrap-ac93e9ce991819f1.yaml @@ -0,0 +1,5 @@ +--- +fixes: + - The `debian-minimal` and and `ubuntu-minimal` elements now install + directly from the updates repo, avoiding the need to + double-install packages during build.