From fd5fbdd4b5c15c613133c5a18c8e83b486ec1dd0 Mon Sep 17 00:00:00 2001 From: Gregory Haynes Date: Sat, 25 Jun 2016 04:20:04 +0000 Subject: [PATCH 01/28] Make xenial the ubuntu default This is the new LTS. Change-Id: I42a6c0520dde8be21df396e7c2e6fb5ae55d2025 --- .../ubuntu-minimal/environment.d/10-ubuntu-distro-name.bash | 2 +- elements/ubuntu/environment.d/10-ubuntu-distro-name.bash | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/elements/ubuntu-minimal/environment.d/10-ubuntu-distro-name.bash b/elements/ubuntu-minimal/environment.d/10-ubuntu-distro-name.bash index ba6783f78..e9ae083e3 100644 --- a/elements/ubuntu-minimal/environment.d/10-ubuntu-distro-name.bash +++ b/elements/ubuntu-minimal/environment.d/10-ubuntu-distro-name.bash @@ -1,4 +1,4 @@ export DISTRO_NAME=ubuntu -export DIB_RELEASE=${DIB_RELEASE:-trusty} +export DIB_RELEASE=${DIB_RELEASE:-xenial} export DIB_DEBIAN_COMPONENTS=${DIB_DEBIAN_COMPONENTS:-main,restricted,universe} export DIB_DISTRIBUTION_MIRROR=${DIB_DISTRIBUTION_MIRROR:-http://archive.ubuntu.com/ubuntu} diff --git a/elements/ubuntu/environment.d/10-ubuntu-distro-name.bash b/elements/ubuntu/environment.d/10-ubuntu-distro-name.bash index 2860427c8..fa19f207c 100644 --- a/elements/ubuntu/environment.d/10-ubuntu-distro-name.bash +++ b/elements/ubuntu/environment.d/10-ubuntu-distro-name.bash @@ -1,2 +1,2 @@ export DISTRO_NAME=ubuntu -export DIB_RELEASE=${DIB_RELEASE:-trusty} +export DIB_RELEASE=${DIB_RELEASE:-xenial} From fdffa56ff2034f943b13996815a9e216c724e3b5 Mon Sep 17 00:00:00 2001 From: Gregory Haynes Date: Wed, 17 Aug 2016 16:20:36 +0000 Subject: [PATCH 02/28] Add element for setting sysctl values Theres a pretty standard workflow for setting a sysctl value which will be applied on image boot which was written by tripleo. Lets move this in tree as other folks (like Octavia) would like to depend on it. Change-Id: I3c266870d417cdba3196f5fa65c4cd634ab13173 --- elements/sysctl/README.rst | 12 +++++++ elements/sysctl/bin/sysctl-set-value | 49 ++++++++++++++++++++++++++ elements/sysctl/bin/sysctl-write-value | 32 +++++++++++++++++ 3 files changed, 93 insertions(+) create mode 100644 elements/sysctl/README.rst create mode 100755 elements/sysctl/bin/sysctl-set-value create mode 100755 elements/sysctl/bin/sysctl-write-value diff --git a/elements/sysctl/README.rst b/elements/sysctl/README.rst new file mode 100644 index 000000000..d6e37d1b1 --- /dev/null +++ b/elements/sysctl/README.rst @@ -0,0 +1,12 @@ +====== +sysctl +====== + +Add a sysctl-set-value command which can be run from within an element. +Running this command will cause the sysctl value to be set on boot (by +writing the value to /etc/sysctl.d). + +Example usage + +:: + sysctl-set-value net.ipv4.ip_forward 1 diff --git a/elements/sysctl/bin/sysctl-set-value b/elements/sysctl/bin/sysctl-set-value new file mode 100755 index 000000000..cbeb6eff9 --- /dev/null +++ b/elements/sysctl/bin/sysctl-set-value @@ -0,0 +1,49 @@ +#!/bin/bash +# +# Copied from tripleo-image-element's sysctl element +# +# Validate and manage setting sysctl settings. +# +# The script is called with name/value pairs which are stored +# in the system default sysctl.d directory. Before adding new +# settings a validation is done to ensure that conflicting +# sysctl settings have not been requested. Once finished sysctl +# is used to activate the changes. + +if [ ${DIB_DEBUG_TRACE:-0} -gt 0 ]; then + set -x +fi +set -eu +set -o pipefail + +NAME=${1:-} +VALUE=${2:-} +# Optional comment used to describe the setting +COMMENT=${3:-"This file was created by diskimage-builder."} + +if [ -z "$NAME" -o -z "$VALUE" ]; then + echo "NAME and VALUE are required." + exit 1 +fi + +FILENAME="/etc/sysctl.d/${NAME}.conf" + +if [ -f $FILENAME ]; then + # check to make sure the settings match... otherwise fail + if ! grep -q "^$NAME = $VALUE" $FILENAME; then + echo "Conflicting sysctl.conf setting for $NAME == $VALUE. Found:" + grep "^$NAME" $FILENAME + exit 1 + fi +else + + if ! sysctl -a | grep -q "^$NAME"; then + echo "Invalid sysctl key: $NAME" + exit 1 + fi + + sysctl-write-value $NAME "$VALUE" "$COMMENT" + + sysctl -p $FILENAME + +fi diff --git a/elements/sysctl/bin/sysctl-write-value b/elements/sysctl/bin/sysctl-write-value new file mode 100755 index 000000000..e60f55352 --- /dev/null +++ b/elements/sysctl/bin/sysctl-write-value @@ -0,0 +1,32 @@ +#!/bin/bash +# +# Copied from tripleo-image-element's sysctl element +# +# Validate and manage setting sysctl settings. +# +# The script is called with name/value pairs which are stored +# in the system default sysctl.d directory. This script performs +# no checking, just writing out the file. + +if [ ${DIB_DEBUG_TRACE:-0} -gt 0 ]; then + set -x +fi +set -eu +set -o pipefail + +NAME=${1:-} +VALUE=${2:-} +# Optional comment used to describe the setting +COMMENT=${3:-"This file was created by diskimage-builder."} + +if [ -z "$NAME" -o -z "$VALUE" ]; then + echo "Usage: sysctl-write-value [comment]" + exit 1 +fi + +FILENAME="/etc/sysctl.d/${NAME}.conf" + +cat > $FILENAME < Date: Tue, 18 Oct 2016 19:02:41 +0100 Subject: [PATCH 03/28] elements: simple-init: Remove SUSE interfaces Make sure SUSE interfaces are removed as well. Change-Id: If993dc606217f2ec243392ac2fa588ebae1cce86 --- .../simple-init/install.d/60-simple-init-remove-interfaces | 3 +++ 1 file changed, 3 insertions(+) diff --git a/elements/simple-init/install.d/60-simple-init-remove-interfaces b/elements/simple-init/install.d/60-simple-init-remove-interfaces index 21bbb88a8..5e47a854b 100755 --- a/elements/simple-init/install.d/60-simple-init-remove-interfaces +++ b/elements/simple-init/install.d/60-simple-init-remove-interfaces @@ -12,6 +12,9 @@ set -o pipefail # Fedora rm -f /etc/sysconfig/network-scripts/ifcfg-eth* +# SUSE +rm -f /etc/sysconfig/network/ifcfg-eth* + # Ubuntu rm -f /etc/network/interfaces.d/eth* From 18a664dd32a59b03f7869e117688f158a63d534a Mon Sep 17 00:00:00 2001 From: Paul Belanger Date: Wed, 28 Sep 2016 16:03:52 -0400 Subject: [PATCH 04/28] Don't use ssh-keygen -A for init scripts We are running into race conditions with glean, which ssh-keygen -A is not handling properly. So, create a new script to first check if the file exists, then use 'yes' to disable overwriting of existing files. Change-Id: Ie82e1e3f832fcc8f32c7e1335c5f0ee16d36f9a8 Signed-off-by: Paul Belanger --- elements/runtime-ssh-host-keys/element-deps | 1 + .../init-scripts/systemd/ssh-keygen.service | 13 +-------- .../init-scripts/upstart/ssh-keygen.conf | 2 +- .../usr/local/sbin/runtime-ssh-host-keys.sh | 29 +++++++++++++++++++ 4 files changed, 32 insertions(+), 13 deletions(-) create mode 100755 elements/runtime-ssh-host-keys/static/usr/local/sbin/runtime-ssh-host-keys.sh diff --git a/elements/runtime-ssh-host-keys/element-deps b/elements/runtime-ssh-host-keys/element-deps index 3a0277624..69a71fded 100644 --- a/elements/runtime-ssh-host-keys/element-deps +++ b/elements/runtime-ssh-host-keys/element-deps @@ -1 +1,2 @@ dib-init-system +install-static diff --git a/elements/runtime-ssh-host-keys/init-scripts/systemd/ssh-keygen.service b/elements/runtime-ssh-host-keys/init-scripts/systemd/ssh-keygen.service index 90a831362..ef2201bcf 100644 --- a/elements/runtime-ssh-host-keys/init-scripts/systemd/ssh-keygen.service +++ b/elements/runtime-ssh-host-keys/init-scripts/systemd/ssh-keygen.service @@ -2,19 +2,8 @@ Description=OpenSSH Server Key Generation Before=ssh.service -ConditionPathExists=|!/etc/ssh/ssh_host_key -ConditionPathExists=|!/etc/ssh/ssh_host_key.pub -ConditionPathExists=|!/etc/ssh/ssh_host_rsa_key -ConditionPathExists=|!/etc/ssh/ssh_host_rsa_key.pub -ConditionPathExists=|!/etc/ssh/ssh_host_dsa_key -ConditionPathExists=|!/etc/ssh/ssh_host_dsa_key.pub -ConditionPathExists=|!/etc/ssh/ssh_host_ecdsa_key -ConditionPathExists=|!/etc/ssh/ssh_host_ecdsa_key.pub -ConditionPathExists=|!/etc/ssh/ssh_host_ed25519_key -ConditionPathExists=|!/etc/ssh/ssh_host_ed25519_key.pub - [Service] -ExecStart=/usr/bin/ssh-keygen -A +ExecStart=/usr/local/sbin/runtime-ssh-host-keys.sh Type=oneshot RemainAfterExit=yes diff --git a/elements/runtime-ssh-host-keys/init-scripts/upstart/ssh-keygen.conf b/elements/runtime-ssh-host-keys/init-scripts/upstart/ssh-keygen.conf index 3fa2c0126..0f85e22a2 100644 --- a/elements/runtime-ssh-host-keys/init-scripts/upstart/ssh-keygen.conf +++ b/elements/runtime-ssh-host-keys/init-scripts/upstart/ssh-keygen.conf @@ -5,4 +5,4 @@ console output task -exec /usr/bin/ssh-keygen -A +exec /usr/local/sbin/runtime-ssh-host-keys.sh diff --git a/elements/runtime-ssh-host-keys/static/usr/local/sbin/runtime-ssh-host-keys.sh b/elements/runtime-ssh-host-keys/static/usr/local/sbin/runtime-ssh-host-keys.sh new file mode 100755 index 000000000..4fa2374d0 --- /dev/null +++ b/elements/runtime-ssh-host-keys/static/usr/local/sbin/runtime-ssh-host-keys.sh @@ -0,0 +1,29 @@ +#!/bin/bash +# Copyright 2016 Red Hat, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. +# dib-lint: disable=dibdebugtrace + +set -exu +set -o pipefail + +# We are running into race conditions with glean, which ssh-keygen -A is +# not handling properly. So, create a new script to first check if the +# file exists, then use 'yes' to disable overwriting of existing files. + +for key in dsa ecdsa ed25519 rsa; do + FILE=/etc/ssh/ssh_host_${key}_key + if ! [ -e $FILE ]; then + /usr/bin/yes n | /usr/bin/ssh-keygen -f $FILE -N '' -t $key + fi +done From 3d44a08c53cc9280b33e83fcec8766cb513da126 Mon Sep 17 00:00:00 2001 From: Markos Chandras Date: Tue, 18 Oct 2016 21:02:10 +0100 Subject: [PATCH 05/28] elements: runtime-ssh-host-keys: Add openssh-client mapping for SUSE The SUSE 'openssh' package contains the openssh client. Change-Id: Ic1da63b6c62158b128d44ac48a0657d5d7c53f67 --- elements/runtime-ssh-host-keys/pkg-map | 3 +++ 1 file changed, 3 insertions(+) diff --git a/elements/runtime-ssh-host-keys/pkg-map b/elements/runtime-ssh-host-keys/pkg-map index ce9fd939e..793124030 100644 --- a/elements/runtime-ssh-host-keys/pkg-map +++ b/elements/runtime-ssh-host-keys/pkg-map @@ -5,6 +5,9 @@ }, "gentoo": { "openssh-client": "" + }, + "suse": { + "openssh-client": "openssh" } } } From 9e392f56b0b78f60bc2598c3cd99f8d7061cc550 Mon Sep 17 00:00:00 2001 From: Ian Wienand Date: Thu, 20 Oct 2016 13:51:16 +1100 Subject: [PATCH 06/28] Don't set tracing in environment files Because environment files are sourced into the current environment, they shouldn't be setting global settings like tracing else they affect every preceeding import. This is quite confusing when only half your imports are traced in the logs, because it was either turned on, or off, by a preceeding environment import. There is a corresponding dib-run-parts change in I29f7df1514aeb988222d1094e8269eddb485c2a0 that will greatly increase debugability for environment files by deliberately logging what files are sourced and consistently turning on tracing around their import. This isn't strictly necessary (since dib-run-parts with the prior change will just turn tracing off after import anyway) but it's a decent cleanup for consistency. A bare-minimum dib-lint check is added. Documentation is updated. Change-Id: I10f68be0642835a04af7e5a2bc101502f61e5357 --- bin/dib-lint | 6 ++++++ doc/source/developer/developing_elements.rst | 11 ++++++----- elements/centos/environment.d/00-bootloader.bash | 8 -------- elements/manifests/environment.d/14-manifests | 8 -------- 4 files changed, 12 insertions(+), 21 deletions(-) diff --git a/bin/dib-lint b/bin/dib-lint index f75c1ccd6..c58865e5c 100755 --- a/bin/dib-lint +++ b/bin/dib-lint @@ -151,6 +151,12 @@ for i in $(find elements -type f \ fi fi + # check that environment files don't "set -x" + if [[ "$i" =~ (environment.d) ]]; then + if grep -q "set -x" $i; then + error "Environment file $i should not set tracing" + fi + fi # check that sudo calls in phases run outside the chroot look # "safe"; meaning that they seem to operate within the chroot diff --git a/doc/source/developer/developing_elements.rst b/doc/source/developer/developing_elements.rst index 0cee5946e..faadd5278 100644 --- a/doc/source/developer/developing_elements.rst +++ b/doc/source/developer/developing_elements.rst @@ -172,11 +172,12 @@ the image as executable files. Environment Variables ^^^^^^^^^^^^^^^^^^^^^ -To set environment variables for other hooks, add a file to your element -``environment.d``. - -This directory contains bash script snippets that are sourced before running -scripts in each phase. +To set environment variables for other hooks, add a file to your +element ``environment.d``. This directory contains bash script +snippets that are sourced before running scripts in each phase. Note +that because environment includes are sourced together, they should +not set global flags like ``set -x`` because they will affect all +preceeding imports. DIB exposes an internal ``$IMAGE_ELEMENT`` variable which provides elements access to the full set of elements that are included in the image build. This diff --git a/elements/centos/environment.d/00-bootloader.bash b/elements/centos/environment.d/00-bootloader.bash index 6478bc967..d406155d7 100755 --- a/elements/centos/environment.d/00-bootloader.bash +++ b/elements/centos/environment.d/00-bootloader.bash @@ -1,9 +1 @@ -#!/bin/bash - -if [ "${DIB_DEBUG_TRACE:-0}" -gt 0 ]; then - set -x -fi -set -eu -set -o pipefail - export DIB_EXTLINUX=1 diff --git a/elements/manifests/environment.d/14-manifests b/elements/manifests/environment.d/14-manifests index c2ae96dc9..a125a6f2f 100755 --- a/elements/manifests/environment.d/14-manifests +++ b/elements/manifests/environment.d/14-manifests @@ -1,5 +1,3 @@ -#!/bin/bash -# # Copyright 2014 Hewlett-Packard Development Company, L.P. # # Licensed under the Apache License, Version 2.0 (the "License"); you may @@ -15,11 +13,5 @@ # under the License. # -if [ ${DIB_DEBUG_TRACE:-0} -gt 0 ]; then - set -x -fi -set -eu -set -o pipefail - export DIB_MANIFEST_IMAGE_DIR=${DIB_MANIFEST_IMAGE_DIR:-/etc/dib-manifests} export DIB_MANIFEST_SAVE_DIR=${DIB_MANIFEST_SAVE_DIR:-${IMAGE_NAME}.d/} From 6fb658a5f1bbc38baa07bb2f51acc6528b8c017a Mon Sep 17 00:00:00 2001 From: Ian Wienand Date: Mon, 24 Oct 2016 11:17:28 +1100 Subject: [PATCH 07/28] Don't log datestamp by default in functional tests We're getting double time-stamps in the console log of upstream jobs. Move the logging of a prefix datestamp into a "-t" option to retain the status quo prior to Id9ea5131f0026c292ca6453ba2c80fe12c47f808 (we could, of course, do it the other way and turn if off in the jobs, but since we didn't have it before...) While poking, make the time-stamp consistent and always prefixed if -t is turned on. Also, it seems the parallel options got a bit of sync with what got merged. Add "-j" documentation and remove unused "p" option. Change-Id: Ic7c2ebeca3f9d5784cac59505b6e6181151f5805 --- tests/run_functests.sh | 29 +++++++++++++++++++++++++---- 1 file changed, 25 insertions(+), 4 deletions(-) diff --git a/tests/run_functests.sh b/tests/run_functests.sh index 2bbd8c57f..84b6d38e8 100755 --- a/tests/run_functests.sh +++ b/tests/run_functests.sh @@ -27,16 +27,26 @@ DEFAULT_SKIP_TESTS=( function log_with_prefix { local pr=$1 + local log while read a; do - echo $(date +"%Y%m%d-%H%M%S.%N") "[$pr] $a" + log="[$pr] $a" + if [[ ${LOG_DATESTAMP} -ne 0 ]]; then + log="$(date +"%Y%m%d-%H%M%S.%N") ${log}" + fi + echo "${log}" done } # Log job control messages function log_jc { local msg="$1" - printf "[JOB-CONTROL] %s %s\n" "$(date)" "${msg}" + local log="[JOB-CONTROL] ${msg}" + + if [[ ${LOG_DATESTAMP} -ne 0 ]]; then + log="$(date +"%Y%m%d-%H%M%S.%N") ${log}" + fi + echo "${log}" } function job_cnt { @@ -156,15 +166,23 @@ for e in $DIB_ELEMENTS/*/test-elements/*; do TESTS+=("$element/$test_element") done +# +# Default values +# JOB_MAX_CNT=1 +LOG_DATESTAMP=0 -while getopts ":hlpj:" opt; do +# +# Parse args +# +while getopts ":hlj:t" opt; do case $opt in h) echo "run_functests.sh [-h] [-l] ..." echo " -h : show this help" echo " -l : list available tests" - echo " -p : run all tests in parallel" + echo " -j : parallel job count (default to 1)" + echo " -t : prefix log messages with timestamp" echo " : functional test to run" echo " Special test 'all' will run all tests" exit 0 @@ -182,6 +200,9 @@ while getopts ":hlpj:" opt; do JOB_MAX_CNT=${OPTARG} echo "Running parallel - using [${JOB_MAX_CNT}] jobs" ;; + t) + LOG_DATESTAMP=1 + ;; \?) echo "Invalid option: -$OPTARG" exit 1 From e531980a14d55ecf437188fb92fc332a82c2d537 Mon Sep 17 00:00:00 2001 From: Ian Wienand Date: Wed, 19 Oct 2016 09:34:54 +0000 Subject: [PATCH 08/28] Remove RedHat grub workaround install AFAICT this is no longer necessary. I've tested minimal and image builds and they seem to work. The original problem seems to be with installing the package in the chroot, although it was never quite clear it ever affected the Red Hat path. This code is currently broken (see I884cb1e78ad8c31d985f3fc94a58091b993edd7d). This is proposed as an alternative to I74eed074494134334d5e49042bb5214bd0dd7339. Related-Bug: #1627000 Change-Id: Iafe3611f4eec3c6357587a6cae6a30a261686ead --- .../finalise.d/99-cleanup-tmp-grub | 9 ---- elements/redhat-common/package-installs.yaml | 11 ----- .../pre-install.d/15-remove-grub | 47 ------------------- 3 files changed, 67 deletions(-) delete mode 100755 elements/redhat-common/finalise.d/99-cleanup-tmp-grub delete mode 100755 elements/redhat-common/pre-install.d/15-remove-grub diff --git a/elements/redhat-common/finalise.d/99-cleanup-tmp-grub b/elements/redhat-common/finalise.d/99-cleanup-tmp-grub deleted file mode 100755 index 6a7909117..000000000 --- a/elements/redhat-common/finalise.d/99-cleanup-tmp-grub +++ /dev/null @@ -1,9 +0,0 @@ -#!/bin/bash - -if [ ${DIB_DEBUG_TRACE:-1} -gt 0 ]; then - set -x -fi -set -eu -set -o pipefail - -rm -rf /tmp/grub diff --git a/elements/redhat-common/package-installs.yaml b/elements/redhat-common/package-installs.yaml index 025158f15..81e19aa9d 100644 --- a/elements/redhat-common/package-installs.yaml +++ b/elements/redhat-common/package-installs.yaml @@ -6,14 +6,3 @@ traceroute: which: gettext: phase: pre-install.d - -# these are being installed to satisfy the dependencies of grub2. See -# 15-remove-grub for more details -grub2-tools: - phase: pre-install.d -os-prober: - phase: pre-install.d -redhat-lsb-core: - phase: pre-install.d -system-logos: - phase: pre-install.d diff --git a/elements/redhat-common/pre-install.d/15-remove-grub b/elements/redhat-common/pre-install.d/15-remove-grub deleted file mode 100755 index ba1d922a1..000000000 --- a/elements/redhat-common/pre-install.d/15-remove-grub +++ /dev/null @@ -1,47 +0,0 @@ -#!/bin/bash -if [ "${DIB_DEBUG_TRACE:-0}" -gt 0 ]; then - set -x -fi -set -eu -set -o pipefail - -# grub2 isn't available on rhel6/centos6; they are setup to use -# extlinux. skip this -# you would think we could match on $DISTRO or something else; but -# we can't because the rhel/centos elements are a bit mixed up; -# centos-minimal for example sets distro to "centos". so the best -# check is just for the original "grub-install" script -if [ -f /sbin/grub-install ]; then - exit 0 -fi - -# remove grub2 package. As described in -# elements/ubuntu/pre-install.d/00-remove-grub; the grub post-kernel -# install hook will barf if the block device can't be found (as -# happens in a chroot). -# -# XXX : it is not clear this is necessary for fedora/centos7 and it's -# install hooks. Investigation is required. -if rpm -q grub2; then - install-packages -e grub-pc -fi - -# now configure things to re-install grub at the end. We don't want -# to rely on vm/finalise.d/51-bootloader to simply reinstall the -# package via the package-manager, because at that point (during -# finalise) the build-time yum-cache has been unmounted (hence the -# local-cache looks empty) and yum may try to repopulate the -# local-cache with all the grub2 dependencies. This is slow, and -# potentially fills up the disk. -# -# XXX : At this point, keepcache=0 *should* probably be set for -# yum/dnf. We have not standarised/documented that this will be done, -# however. This would *probably* stop dependencies being populated -# into the cache. We could investigate this, and possibly remove this -# all together if we standardise some of these behaviours. - -# So we download the latest grub2 package and setup the install script -# to just install the single-package, which will be called later by -# vm/finalise.d/51-bootloader -install-packages -d /tmp/grub grub-pc -echo "rpm -i /tmp/grub/*.rpm" > /tmp/grub/install From 4095046a25f94a06e44368f4f9726e9347f15e11 Mon Sep 17 00:00:00 2001 From: nizam Date: Fri, 28 Oct 2016 19:46:02 +0530 Subject: [PATCH 09/28] Changed the home-page of diskimage-builder in setup.cfg Instead of pointing to https://git.openstack.org/cgit/openstack/diskimage-builder, the homepage has been changed to point to the homepage of diskimage-builder. Change-Id: I795a5fd343ff7a06bfe43ae4a76dc24136db99a2 --- setup.cfg | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.cfg b/setup.cfg index ec5768179..2345eac78 100644 --- a/setup.cfg +++ b/setup.cfg @@ -6,7 +6,7 @@ description-file = author = HP Cloud Services author_email = openstack-dev@lists.openstack.org license: Apache License (2.0) -home-page = https://git.openstack.org/cgit/openstack/diskimage-builder +home-page = http://docs.openstack.org/developer/diskimage-builder/ classifier = Development Status :: 3 - Alpha License :: OSI Approved :: Apache Software License From 51f982e3765705a619ec59671911163012265bea Mon Sep 17 00:00:00 2001 From: nizam Date: Fri, 28 Oct 2016 19:48:33 +0530 Subject: [PATCH 10/28] Drop MANIFEST.in - it's not needed by pbr diskimage-builder already uses PBR:- setuptools.setup( setup_requires=['pbr>=1.8'], pbr=True) This patch removes `MANIFEST.in` file as pbr generates a sensible manifest from git files and some standard files and it removes the need for an explicit `MANIFEST.in` file. Change-Id: Iad8b724c1d1e0ef358606b13da4569d881ec740a --- MANIFEST.in | 6 ------ 1 file changed, 6 deletions(-) delete mode 100644 MANIFEST.in diff --git a/MANIFEST.in b/MANIFEST.in deleted file mode 100644 index f7f0d66fc..000000000 --- a/MANIFEST.in +++ /dev/null @@ -1,6 +0,0 @@ -include bin/disk-image-create -include bin/element-info -include bin/ramdisk-image-create -graft lib -graft elements -include README.rst From 3b855faf5f5a27e2344f0185954f125e7baf931a Mon Sep 17 00:00:00 2001 From: nizam Date: Fri, 28 Oct 2016 19:51:55 +0530 Subject: [PATCH 11/28] Don't include openstack/common in flake8 exclude list The directory openstack/common was used to keep codes from oslo-incubator, we have retired oslo-incubator, so don't use this directory any more Change-Id: Icd13d32674b117626fbecdfdec2881463a87ad5d --- tox.ini | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tox.ini b/tox.ini index 8205d15bc..03e35fffb 100644 --- a/tox.ini +++ b/tox.ini @@ -36,4 +36,4 @@ commands = sphinx-build -a -W -E -d releasenotes/build/doctrees -b html releasen [flake8] ignore = E125,H202,H302,H803 -exclude = .venv,.git,.tox,dist,doc,*openstack/common*,*lib/python*,*egg,build,conf.py +exclude = .venv,.git,.tox,dist,doc,*lib/python*,*egg,build,conf.py From 2854f4063bd2a6dcdb6fa5fab93aa56857e47b59 Mon Sep 17 00:00:00 2001 From: Bruno Cornec Date: Fri, 28 Oct 2016 19:38:50 +0200 Subject: [PATCH 12/28] Fix ironic-python-agent image not loading vfat mod Closes-Bug: 1589450 Load the vfat driver as a Pre Exec action for systemd before starting ironic-python-agent in order to allow reading of parameters.txt file required for the ironic-python-agent to find its configuration. Change-Id: Ibf74dd1b2678ea76e0676711a7aa5ba6b88d5421 --- .../ironic-agent-source-install/ironic-python-agent.service | 1 + 1 file changed, 1 insertion(+) diff --git a/elements/ironic-agent/install.d/ironic-agent-source-install/ironic-python-agent.service b/elements/ironic-agent/install.d/ironic-agent-source-install/ironic-python-agent.service index 50e9d4aa8..da9709660 100644 --- a/elements/ironic-agent/install.d/ironic-agent-source-install/ironic-python-agent.service +++ b/elements/ironic-agent/install.d/ironic-agent-source-install/ironic-python-agent.service @@ -3,6 +3,7 @@ Description=Ironic Python Agent After=network-online.target [Service] +ExecStartPre=/usr/sbin/modprobe vfat ExecStart=/usr/local/bin/ironic-python-agent Restart=always RestartSec=30s From 348a6b337a18f4666ed99a387d20ed83c1aa86c8 Mon Sep 17 00:00:00 2001 From: Markos Chandras Date: Mon, 31 Oct 2016 23:15:46 +0000 Subject: [PATCH 13/28] elements: opensuse-minimal: Add support for building Tumbleweed images Add DIB_RELEASE=Tumbleweed option in order to build openSUSE Tumbleweed images Change-Id: I44cc04ef5a993c1a7f0078e4161888b52995f247 --- .../environment.d/10-opensuse-distro-name.bash | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/elements/opensuse-minimal/environment.d/10-opensuse-distro-name.bash b/elements/opensuse-minimal/environment.d/10-opensuse-distro-name.bash index 58fff6d2e..c6aafbbc2 100644 --- a/elements/opensuse-minimal/environment.d/10-opensuse-distro-name.bash +++ b/elements/opensuse-minimal/environment.d/10-opensuse-distro-name.bash @@ -1,5 +1,6 @@ export DISTRO_NAME=opensuse -export DIB_RELEASE=${DIB_RELEASE:-42.1} +DIB_RELEASE=${DIB_RELEASE:-42.1} +export DIB_RELEASE=${DIB_RELEASE,,} export DIB_OPENSUSE_MIRROR=${DIB_OPENSUSE_MIRROR:-http://download.opensuse.org} case ${DIB_RELEASE} in # We are using "=>" as the assignment symbol since "@" "=" etc could be used in the URI itself. @@ -15,6 +16,11 @@ case ${DIB_RELEASE} in ZYPPER_REPOS="update=>${DIB_OPENSUSE_MIRROR}/update/leap/${DIB_RELEASE}/oss/ " ZYPPER_REPOS+="oss=>${DIB_OPENSUSE_MIRROR}/distribution/leap/${DIB_RELEASE}/repo/oss/" ;; + # Tumbleweed + tumbleweed) + ZYPPER_REPOS="update=>${DIB_OPENSUSE_MIRROR}/update/${DIB_RELEASE}/ " + ZYPPER_REPOS+="oss=>${DIB_OPENSUSE_MIRROR}/${DIB_RELEASE}/repo/oss/" + ;; *) echo "Unsupported openSUSE release: ${DIB_RELEASE}"; exit 1 ;; esac export ZYPPER_REPOS From 290925bd5e0470f69357a8ff13e423ba628c49bb Mon Sep 17 00:00:00 2001 From: Markos Chandras Date: Mon, 31 Oct 2016 23:18:28 +0000 Subject: [PATCH 14/28] elements: zypper-minimal: Refresh repositories after adding the cache The refresh operation must happen after the cache has been added in order to ensure that whatever is in the cache is still relevant to the current build and we are not using stale packages. Change-Id: Iafd718e9738f85b8c235806c027665730f44d89b --- elements/zypper-minimal/root.d/08-zypper-chroot | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/elements/zypper-minimal/root.d/08-zypper-chroot b/elements/zypper-minimal/root.d/08-zypper-chroot index e6d55c30a..413970d0b 100755 --- a/elements/zypper-minimal/root.d/08-zypper-chroot +++ b/elements/zypper-minimal/root.d/08-zypper-chroot @@ -41,9 +41,6 @@ for repo in ${ZYPPER_REPOS}; do sudo zypper ${ZYPPER_TARGET_OPTS} addrepo --name ${reponame} --keep-packages ${repouri} ${reponame} done -# Refresh it -sudo zypper ${ZYPPER_TARGET_OPTS} refresh - # It appears that zypper will clean up the repo's cache when it (re-)adds the # repo so we need to add the cache now, once the repos are added. This is # similar to what the zypper/50-zypper-cache script does @@ -53,6 +50,10 @@ mkdir -p $ZYPPER_CACHE_DIR sudo mkdir -p $TMP_MOUNT_PATH/var/cache/zypp sudo mount --bind $ZYPPER_CACHE_DIR $TMP_MOUNT_PATH/var/cache/zypp +# Refresh it so we get updated data in cased we switched DIB_RELEASE +# since last run. +sudo zypper ${ZYPPER_TARGET_OPTS} refresh + # Install filesystem, base and useful tools sudo zypper ${ZYPPER_TARGET_OPTS} install ${ZYPPER_INSTALL_OPTS} filesystem # Install basic components in order From d5225055ef2403b461083f5d76c67bc8410539a7 Mon Sep 17 00:00:00 2001 From: Derek Higgins Date: Wed, 2 Nov 2016 11:01:15 +0000 Subject: [PATCH 15/28] dhcp-all-interfaces: support Centos/RHEL 6 On Centos and RHEL 6 the init system is upsart but but networking is using sysv compatabiliy and a code path the handle this situation. We can't use DISTRO_NAME because the centos-minimal element sets it to centos for CentOS 7 but the centos element sets it to centos for CentOS 6. Change-Id: Ib8e33ed78b3d6a5737eb7449bccef2d33f72b131 Closes-Bug: #1638527 --- .../dhcp-all-interfaces/install.d/50-dhcp-all-interfaces | 8 +++++++- .../install.d/dhcp-all-interfaces.init | 1 + 2 files changed, 8 insertions(+), 1 deletion(-) diff --git a/elements/dhcp-all-interfaces/install.d/50-dhcp-all-interfaces b/elements/dhcp-all-interfaces/install.d/50-dhcp-all-interfaces index f17b23a7a..24c109ebd 100755 --- a/elements/dhcp-all-interfaces/install.d/50-dhcp-all-interfaces +++ b/elements/dhcp-all-interfaces/install.d/50-dhcp-all-interfaces @@ -15,7 +15,13 @@ fi DIB_INIT_SYSTEM=$(dib-init-system) if [ "$DIB_INIT_SYSTEM" == "upstart" ]; then - install -D -g root -o root -m 0755 ${SCRIPTDIR}/dhcp-all-interfaces.conf /etc/init/dhcp-all-interfaces.conf + if [ -e "/etc/redhat-release" ] ; then + # the init system is upstart but networking is using sysv compatabiliy (i.e. Centos/RHEL 6) + install -D -g root -o root -m 0755 ${SCRIPTDIR}/dhcp-all-interfaces.init /etc/init.d/dhcp-all-interfaces + chkconfig dhcp-all-interfaces on + else + install -D -g root -o root -m 0755 ${SCRIPTDIR}/dhcp-all-interfaces.conf /etc/init/dhcp-all-interfaces.conf + fi elif [ "$DIB_INIT_SYSTEM" == "systemd" ]; then install -D -g root -o root -m 0755 ${SCRIPTDIR}/dhcp-interface@.service /usr/lib/systemd/system/dhcp-interface@.service install -D -g root -o root -m 0644 ${SCRIPTDIR}/dhcp-all-interfaces-udev.rules /etc/udev/rules.d/99-dhcp-all-interfaces.rules diff --git a/elements/dhcp-all-interfaces/install.d/dhcp-all-interfaces.init b/elements/dhcp-all-interfaces/install.d/dhcp-all-interfaces.init index 8023e154b..89fb79197 100755 --- a/elements/dhcp-all-interfaces/install.d/dhcp-all-interfaces.init +++ b/elements/dhcp-all-interfaces/install.d/dhcp-all-interfaces.init @@ -6,6 +6,7 @@ # Default-Start: S # Default-Stop: 0 6 # X-Start-Before: networking +# chkconfig: 3 9 50 # Short-Description: Autodetect network interfaces # Description: Autodetect network interfaces during boot and configure them for DHCP ### END INIT INFO From bc66298937c55321874eb599aeddde25ce125dd3 Mon Sep 17 00:00:00 2001 From: OpenStack Proposal Bot Date: Wed, 2 Nov 2016 15:31:00 +0000 Subject: [PATCH 16/28] Updated from global requirements Change-Id: Ib41e9cf05743c05fb299f84516d9179ea41e3f46 --- test-requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test-requirements.txt b/test-requirements.txt index 8671a4497..f9f2a6b81 100644 --- a/test-requirements.txt +++ b/test-requirements.txt @@ -11,4 +11,4 @@ sphinx!=1.3b1,<1.4,>=1.2.1 # BSD oslosphinx>=4.7.0 # Apache-2.0 # releasenotes -reno>=1.8.0 # Apache2 +reno>=1.8.0 # Apache-2.0 From 435f52a7d1506b87d6b650adcdde35a36c242157 Mon Sep 17 00:00:00 2001 From: Markos Chandras Date: Wed, 2 Nov 2016 22:05:05 +0000 Subject: [PATCH 17/28] elements: zypper-minimal: Mount common pseudo filesystems Mount all the usual /dev /sys /proc pseudo filesystems during the root.d phase in order to make sure they are available for the rpm post-installation phases. Change-Id: I28221debf1036d9eb5137161757eb30811eafab1 --- elements/zypper-minimal/root.d/08-zypper-chroot | 16 +++++++++++++++- 1 file changed, 15 insertions(+), 1 deletion(-) diff --git a/elements/zypper-minimal/root.d/08-zypper-chroot b/elements/zypper-minimal/root.d/08-zypper-chroot index 413970d0b..7e4b22d08 100755 --- a/elements/zypper-minimal/root.d/08-zypper-chroot +++ b/elements/zypper-minimal/root.d/08-zypper-chroot @@ -27,6 +27,10 @@ set -o pipefail [ -n "${ZYPPER_REPOS}" ] function cleanup() { + sudo umount $TARGET_ROOT/proc + sudo umount $TARGET_ROOT/dev/pts + sudo umount $TARGET_ROOT/dev + sudo umount $TARGET_ROOT/sys sudo umount $TMP_MOUNT_PATH/var/cache/zypp } @@ -54,6 +58,16 @@ sudo mount --bind $ZYPPER_CACHE_DIR $TMP_MOUNT_PATH/var/cache/zypp # since last run. sudo zypper ${ZYPPER_TARGET_OPTS} refresh +# Note this is not usually done for root.d elements (see +# lib/common-functions:mount_proc_dev_sys) but it's important that +# we have things like /dev/urandom around inside the chroot for +# the rpm [pre|post]inst scripts within the packages. +sudo mkdir -p $TARGET_ROOT/proc $TARGET_ROOT/dev $TARGET_ROOT/sys +sudo mount -t proc none $TARGET_ROOT/proc +sudo mount --bind /dev $TARGET_ROOT/dev +sudo mount --bind /dev/pts $TARGET_ROOT/dev/pts +sudo mount -t sysfs none $TARGET_ROOT/sys + # Install filesystem, base and useful tools sudo zypper ${ZYPPER_TARGET_OPTS} install ${ZYPPER_INSTALL_OPTS} filesystem # Install basic components in order @@ -84,5 +98,5 @@ for newfile in $(sudo find $TARGET_ROOT -type f -name '*rpmnew') ; do sudo mv $newfile $(echo $newfile | sed 's/.rpmnew$//') done -# Unmounting of the /var/cache/zypp is handled by the cleanup EXIT +# Unmounting of all the mount points is handled by the cleanup EXIT # handler so there is nothing else to do here From c7219a5a605e68f0ca46f4f1c130b602e25be073 Mon Sep 17 00:00:00 2001 From: Luca Lorenzetto Date: Mon, 10 Oct 2016 16:00:16 +0200 Subject: [PATCH 18/28] Avoid disabling rhel-7-server-rh-common-rpms while using disk-image-builder for building overcloud images for TripleO using RDO, this repository is (in my opinion) wrongly disabled because contains certain dependencies needed by RDO packages. Example: python-cheetah is required for python-nova, but is not available through RDO repository but only from rhel-7-server-rh-common-rpms Closes-Bug: #1638938 Change-Id: I76824c8ec02590397f1ff1d4f177ad061c7bf441 Signed-off-by: Luca Lorenzetto --- elements/rhel-common/pre-install.d/00-rhel-registration | 2 -- 1 file changed, 2 deletions(-) diff --git a/elements/rhel-common/pre-install.d/00-rhel-registration b/elements/rhel-common/pre-install.d/00-rhel-registration index 60c449d85..a21ef041c 100755 --- a/elements/rhel-common/pre-install.d/00-rhel-registration +++ b/elements/rhel-common/pre-install.d/00-rhel-registration @@ -114,8 +114,6 @@ case "${REG_METHOD:-}" in subscription-manager repos --disable=\* echo "Enabling repos: $user_repos" subscription-manager $repos - echo "Disabling satellite repo because it is no longer needed" - subscription-manager repos --disable ${satellite_repo} ;; disable) echo "Disabling RHEL registration" From 0f742ce24a39706545333a2cad983bc0a7a11b9e Mon Sep 17 00:00:00 2001 From: OpenStack Proposal Bot Date: Wed, 9 Nov 2016 04:14:55 +0000 Subject: [PATCH 19/28] Updated from global requirements Change-Id: I4b8e6f6fdf78706dbb735e25195a1f83413ab3fd --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index 16b55619b..21abb5546 100644 --- a/requirements.txt +++ b/requirements.txt @@ -3,7 +3,7 @@ # process, which may cause wedges in the gate later. Babel>=2.3.4 # BSD dib-utils # Apache-2.0 -pbr>=1.6 # Apache-2.0 +pbr>=1.8 # Apache-2.0 PyYAML>=3.10.0 # MIT flake8<2.6.0,>=2.5.4 # MIT six>=1.9.0 # MIT From 7e60540c0d8d02c976f7890831e103584501c27c Mon Sep 17 00:00:00 2001 From: Ian Wienand Date: Thu, 10 Nov 2016 14:22:17 +1100 Subject: [PATCH 20/28] Cleanup yumdownloader repos yumdownloader has to have all the repo XML files, etc, which adds up to a not totally insignificant 150MiB or so. Currently we're leaking this directory for every build, which adds up on regualar builders like nodepool. Isolate the call with a separate TMPDIR so we can clean it up after the initial download. Change-Id: Ic65e8ca837cc76b7a1bb9f83027b4a5bdd270f75 --- elements/yum-minimal/root.d/08-yum-chroot | 15 +++++++++++++-- 1 file changed, 13 insertions(+), 2 deletions(-) diff --git a/elements/yum-minimal/root.d/08-yum-chroot b/elements/yum-minimal/root.d/08-yum-chroot index 50c0a6b2d..10fe1d258 100755 --- a/elements/yum-minimal/root.d/08-yum-chroot +++ b/elements/yum-minimal/root.d/08-yum-chroot @@ -56,6 +56,7 @@ _RPM="rpm --dbpath=/var/lib/rpm" # has yum/yumdownloader function _install_repos { local packages + local rc # pre-install the base system packages via rpm. We previously # just left it up to yum to drag these in when we "yum install @@ -79,11 +80,21 @@ function _install_repos { packages+="${DISTRO_NAME}-repos " fi - yumdownloader \ + # yumdownloader puts repo xml files and such into a directory + # ${TMPDIR}/yum-$USER-random. Since we don't need this once the + # initial download happens, redirect TMPDIR for this call so we + # can clean it up nicely + local temp_tmp + temp_tmp=$(mktemp -d) + TMPDIR=${temp_tmp} yumdownloader \ --releasever=$DIB_RELEASE \ --setopt=reposdir=$TMP_HOOKS_PATH/yum.repos.d \ --destdir=$WORKING \ - ${packages} + ${packages} && rc=$? || rc=$? + rm -rf ${temp_tmp} + if [[ ${rc} != 0 ]]; then + die "Failed to download initial packages: ${packages}" + fi # --nodeps works around these wanting /bin/sh in some fedora # releases, see rhbz#1265873 From ae66b64c34ea4209e09adf804c8cd8446d7ae5db Mon Sep 17 00:00:00 2001 From: Oliver Walsh Date: Fri, 11 Nov 2016 17:03:50 +0000 Subject: [PATCH 21/28] In disk-image-create, append to INSTALL_PACKAGES instead of clobbering. This allows -p to be used multiple times. Change-Id: Iabe43982e1606c7ca963a1dd3b23ba47d148ae38 Closes-Bug: #1641157 --- bin/disk-image-create | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/bin/disk-image-create b/bin/disk-image-create index 45d8024bd..415cbaf92 100755 --- a/bin/disk-image-create +++ b/bin/disk-image-create @@ -133,7 +133,7 @@ function show_options () { echo " --docker-target -- specify the repo and tag to use if the output type is docker. Defaults to the value of output imagename" if [ "$IS_RAMDISK" == "0" ]; then echo " -n skip the default inclusion of the 'base' element" - echo " -p package[,package,package] -- list of packages to install in the image" + echo " -p package[,package,package] -- list of packages to install in the image. If specified multiple times the packages are appended to the list." fi echo " -h|--help -- display this help and exit" echo " --version -- display version and exit" @@ -190,7 +190,7 @@ while true ; do -u) shift; export COMPRESS_IMAGE="";; -c) shift ; export CLEAR_ENV=1;; -n) shift; export SKIP_BASE="1";; - -p) IFS="," read -a INSTALL_PACKAGES <<< "$2"; export INSTALL_PACKAGES ; shift 2 ;; + -p) IFS="," read -a _INSTALL_PACKAGES <<< "$2"; export INSTALL_PACKAGES=( ${INSTALL_PACKAGES[@]} ${_INSTALL_PACKAGES[@]} ) ; shift 2 ;; --checksum) shift; export DIB_CHECKSUM=1;; --image-size) export DIB_IMAGE_SIZE=$2; shift 2;; --image-cache) export DIB_IMAGE_CACHE=$2; shift 2;; From 79c27199f2ce11c579efa0cfce7ee2c3443ead89 Mon Sep 17 00:00:00 2001 From: Ben Nemec Date: Fri, 11 Nov 2016 14:30:21 -0600 Subject: [PATCH 22/28] Install lsb package by map name instead of package name We have a pkg-map entry for lsb_release, but in package-installs.yaml we refer to the actual package name instead. This will happen to work on Red Hat platforms, but it's actually wrong. Change-Id: Idb248f96e75fa1090422fa08e5fbb2385cc1f517 --- elements/yum-minimal/package-installs.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/elements/yum-minimal/package-installs.yaml b/elements/yum-minimal/package-installs.yaml index e81126dfd..b7cf1e36d 100644 --- a/elements/yum-minimal/package-installs.yaml +++ b/elements/yum-minimal/package-installs.yaml @@ -4,7 +4,7 @@ grubby: kernel: initscripts: man-pages: -redhat-lsb-core: +lsb_release: selinux-policy: selinux-policy-targeted: libselinux-python: From 5d9d3d5cf0c7ab79a6cd32d43375fa683ae87609 Mon Sep 17 00:00:00 2001 From: Saverio Proto Date: Mon, 14 Nov 2016 15:32:08 +0100 Subject: [PATCH 23/28] debian: install dialog package Without the dialog package is not possible to properly use an interactive frontend. debconf will print the following errors: debconf: unable to initialize frontend: Dialog debconf: (No usable dialog-like program is installed, so the dialog based frontend cannot be used. at /usr/share/perl5/Debconf/FrontEnd/Dialog.pm line 76, <> line 1.) Change-Id: I0c7142f717cacf7437dbac1e1696f39b00cb4c49 --- elements/debian/package-installs.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/elements/debian/package-installs.yaml b/elements/debian/package-installs.yaml index 862c274f3..92d0131d7 100644 --- a/elements/debian/package-installs.yaml +++ b/elements/debian/package-installs.yaml @@ -10,3 +10,4 @@ net-tools: cloud-init: cloud-utils: cloud-initramfs-growroot: +dialog: From d07d7ed15d54c37448d5be67c4e4707ba19edac0 Mon Sep 17 00:00:00 2001 From: Noam Angel Date: Tue, 1 Nov 2016 16:19:44 +0200 Subject: [PATCH 24/28] simplify ARCH param for rhel/centos param can be x86_64 and amd64 for fedora/rhel/centos the main supported ARCH is x86_64. This patch allow to call diskimage-builder with the above distro's with param ARCH=x86_64, And also retain same behaiver when call with ARCH=amd64 as it translate anyway to x86_64. Doing so wil simplify user expirience. Change-Id: I229e0912434109b1b48a030bd35ad8dc1096a629 --- elements/centos7/root.d/10-centos7-cloud-image | 4 ++-- elements/rhel7/root.d/10-rhel7-cloud-image | 4 ++-- elements/rpm-distro/pre-install.d/01-override-yum-arch | 2 +- 3 files changed, 5 insertions(+), 5 deletions(-) diff --git a/elements/centos7/root.d/10-centos7-cloud-image b/elements/centos7/root.d/10-centos7-cloud-image index cd79a4180..7db1557de 100755 --- a/elements/centos7/root.d/10-centos7-cloud-image +++ b/elements/centos7/root.d/10-centos7-cloud-image @@ -9,10 +9,10 @@ set -o pipefail [ -n "$ARCH" ] [ -n "$TARGET_ROOT" ] -if [ 'amd64' = "$ARCH" ] ; then +if [[ "amd64 x86_64" =~ "$ARCH" ]]; then ARCH="x86_64" else - echo 'centos7 root element only support the amd64 $ARCH value.' + echo 'centos7 root element only support the x86_64 $ARCH value.' exit 1 fi diff --git a/elements/rhel7/root.d/10-rhel7-cloud-image b/elements/rhel7/root.d/10-rhel7-cloud-image index be38caa45..bc54b1010 100755 --- a/elements/rhel7/root.d/10-rhel7-cloud-image +++ b/elements/rhel7/root.d/10-rhel7-cloud-image @@ -9,10 +9,10 @@ set -o pipefail [ -n "$ARCH" ] [ -n "$TARGET_ROOT" ] -if [ 'amd64' = "$ARCH" ] ; then +if [[ "amd64 x86_64" =~ "$ARCH" ]]; then ARCH="x86_64" else - echo 'rhel7 root element only support the amd64 $ARCH value.' + echo 'rhel7 root element only support the x86_64 $ARCH value.' exit 1 fi diff --git a/elements/rpm-distro/pre-install.d/01-override-yum-arch b/elements/rpm-distro/pre-install.d/01-override-yum-arch index ff84375c6..138bf3134 100755 --- a/elements/rpm-distro/pre-install.d/01-override-yum-arch +++ b/elements/rpm-distro/pre-install.d/01-override-yum-arch @@ -9,7 +9,7 @@ set -o pipefail if [ "i386" = "$ARCH" ]; then basearch=i386 arch=i686 -elif [ "amd64" = "$ARCH" ]; then +elif [[ "amd64 x86_64" =~ "$ARCH" ]]; then basearch=x86_64 arch=x86_64 elif [[ "$ARCH" = "ppc64" ]]; then From e88d6b37df5a210ad15da24403274914f737956e Mon Sep 17 00:00:00 2001 From: Noam Angel Date: Mon, 31 Oct 2016 11:54:58 +0200 Subject: [PATCH 25/28] add support for SUSE in dhcp-all-interfaces This patch will add support for SUSE network scripts, network script in SUSE saved under "/etc/sysconfig/network/ifcfg-*" see: https://www.suse.com/documentation/sled11/book_sle_admin/data/sec_basicnet_manconf.html Change-Id: I87ac2e327cee4945c15da9f2e4adc0a8b7650712 --- .../install.d/dhcp-all-interfaces.sh | 17 ++++++++++++----- 1 file changed, 12 insertions(+), 5 deletions(-) diff --git a/elements/dhcp-all-interfaces/install.d/dhcp-all-interfaces.sh b/elements/dhcp-all-interfaces/install.d/dhcp-all-interfaces.sh index cb30ceed3..4884c435a 100755 --- a/elements/dhcp-all-interfaces/install.d/dhcp-all-interfaces.sh +++ b/elements/dhcp-all-interfaces/install.d/dhcp-all-interfaces.sh @@ -14,7 +14,12 @@ PATH=/sbin:$PATH if [ -d "/etc/network" ]; then CONF_TYPE="eni" elif [ -d "/etc/sysconfig/network-scripts/" ]; then - CONF_TYPE="netscripts" + CONF_TYPE="rhel-netscripts" + SCRIPTS_PATH="/etc/sysconfig/network-scripts/" +elif [ -d "/etc/sysconfig/network/" ]; then + # SUSE network scripts location + CONF_TYPE="suse-netscripts" + SCRIPTS_PATH="/etc/sysconfig/network/" else echo "Unsupported network configuration type!" exit 1 @@ -42,8 +47,10 @@ function enable_interface() { serialize_me if [ "$CONF_TYPE" == "eni" ]; then printf "auto $interface\niface $interface inet dhcp\n\n" >>$ENI_FILE - elif [ "$CONF_TYPE" == "netscripts" ]; then - printf "DEVICE=\"$interface\"\nBOOTPROTO=\"dhcp\"\nONBOOT=\"yes\"\nTYPE=\"Ethernet\"" >"/etc/sysconfig/network-scripts/ifcfg-$interface" + elif [ "$CONF_TYPE" == "rhel-netscripts" ]; then + printf "DEVICE=\"$interface\"\nBOOTPROTO=\"dhcp\"\nONBOOT=\"yes\"\nTYPE=\"Ethernet\"" >"${SCRIPTS_PATH}ifcfg-$interface" + elif [ "$CONF_TYPE" == "suse-netscripts" ]; then + printf "BOOTPROTO=\"dhcp\"\nSTARTMODE=\"auto\"" >"${SCRIPTS_PATH}ifcfg-$interface" fi echo "Configured $1" @@ -51,8 +58,8 @@ function enable_interface() { function config_exists() { local interface=$1 - if [ "$CONF_TYPE" == "netscripts" ]; then - if [ -f "/etc/sysconfig/network-scripts/ifcfg-$interface" ]; then + if [[ "$CONF_TYPE" =~ "netscripts" ]]; then + if [ -f "${SCRIPTS_PATH}ifcfg-$interface" ]; then return 0 fi else From a1f57b8cad0717090eb12014d7847a35bd0563a3 Mon Sep 17 00:00:00 2001 From: Markos Chandras Date: Mon, 31 Oct 2016 21:43:47 +0000 Subject: [PATCH 26/28] lib: common-functions: Fix tmpfs umounting It has been observed that some chroot operations spawn additional processes which rely on chroot files. More specifically, zypper, uses gpg-agent to import and validate gpg keys for its repositories. This gpg-agent process may stay alive for longer which prevents unmounting of the tmpfs directory since the gpg-agent process still uses libraries etc which were present in the chroot. We try to solve this by using walking all the pids in /proc to find out the running processes in the chroot and kill them gracefully. If that fails for whatever reason, then we simply keep trying to umount the tmpfs directory before we give up. The gpg-agent process usually terminates soon after its home directory disappears but on fast systems we can reach the 'umount tmpfs' point before gpg-agent terminates by itself. The solution is generic enough so other 'chroot processes' can also be handled appropriately. Change-Id: Iccf332678c79266113e76f062884fc5ee79e515d --- lib/common-functions | 31 +++++++++++++++++++++++++++++-- 1 file changed, 29 insertions(+), 2 deletions(-) diff --git a/lib/common-functions b/lib/common-functions index 254525868..92c42f881 100644 --- a/lib/common-functions +++ b/lib/common-functions @@ -134,21 +134,48 @@ function eval_run_d () { trap - ERR } +function kill_chroot_processes () { + if [ -z "${1}" ]; then + echo "ERROR: no chroot directory specified" + exit 1 + fi + for piddir in /proc/[0-9]*; do + pid=${piddir##/proc/} + pidname=$(cat $piddir/comm 2>/dev/null || echo "unknown") + # If there are open files from the chroot, just kill the process using + # these files. + if sudo readlink -f $piddir/root | grep -q $TMP_BUILD_DIR; then + echo "Killing chroot process: '${pidname}($pid)'" + sudo kill $pid + fi + done +} + function cleanup_build_dir () { if ! timeout 5 sh -c " while ! sudo rm -rf $TMP_BUILD_DIR/built; do sleep 1; done"; then echo "ERROR: unable to cleanly remove $TMP_BUILD_DIR/built" exit 1 fi sudo rm -rf $TMP_BUILD_DIR/mnt + kill_chroot_processes $TMP_BUILD_DIR if tmpfs_check 0; then - sudo umount -f $TMP_BUILD_DIR || true + # If kill_chroot_processes did not succeed then we have to wait for + # init to reap the orphaned chroot processes + if ! timeout 120 sh -c "while ! sudo umount -f $TMP_BUILD_DIR; do sleep 1; done"; then + echo "ERROR: failed to umount the $TMP_BUILD_DIR tmpfs mount point" + exit 1 + fi fi rm -rf --one-file-system $TMP_BUILD_DIR } function cleanup_image_dir () { + kill_chroot_processes $TMP_IMAGE_DIR if tmpfs_check 0; then - sudo umount -f $TMP_IMAGE_DIR || true + if ! timeout 120 sh -c "while ! sudo umount -f $TMP_IMAGE_DIR; do sleep 1; done"; then + echo "ERROR: failed to umount the $TMP_IMAGE_DIR tmpfs mount point" + exit 1 + fi fi rm -rf --one-file-system $TMP_IMAGE_DIR } From fb8cf95b6fb0820867ebfd262c38d2cae5ab43fa Mon Sep 17 00:00:00 2001 From: Ben Nemec Date: Wed, 16 Nov 2016 16:27:24 -0600 Subject: [PATCH 27/28] Disable all repos in os-refresh-config too This change was made for pre-install so it applies during the image build, but wasn't applied to the os-refresh-config script that would run after deployment. The same problems apply there, so we should do the same thing. Change-Id: I4b8534cc9586eeb588b5c358550e76e27d40556a Closes-Bug: 1629922 --- .../os-refresh-config/pre-configure.d/06-rhel-registration | 2 ++ 1 file changed, 2 insertions(+) diff --git a/elements/rhel-common/os-refresh-config/pre-configure.d/06-rhel-registration b/elements/rhel-common/os-refresh-config/pre-configure.d/06-rhel-registration index 21f386f28..851aa08e6 100755 --- a/elements/rhel-common/os-refresh-config/pre-configure.d/06-rhel-registration +++ b/elements/rhel-common/os-refresh-config/pre-configure.d/06-rhel-registration @@ -131,6 +131,8 @@ case "${REG_METHOD:-}" in rpm -Uvh "$REG_SAT_URL/pub/katello-ca-consumer-latest.noarch.rpm" || true echo "Registering with options: $sanitized_opts" subscription-manager register $opts + echo "Disabling all previous repos" + subscription-manager repos --disable=\* echo "Enabling repos: $user_repos" subscription-manager $repos echo "Installing katello-agent" From 8d7362aa9b4a9d2430de7959d4ee4c78429464c0 Mon Sep 17 00:00:00 2001 From: "d.marlin" Date: Thu, 17 Nov 2016 03:36:54 -0500 Subject: [PATCH 28/28] Change path for dnf arch override so basearch is not overwritten. After writing the basearch value to /etc/dnf/vars/basearch the arch value was overwriting the same file. This appears to be incorrect, so changing it to write /etc/dnf/vars/arch, which matches the subsequent 'yum' code paths. Change-Id: I5da54f03224c11f9e286f16b68533936c4174c2a --- elements/rpm-distro/pre-install.d/01-override-yum-arch | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/elements/rpm-distro/pre-install.d/01-override-yum-arch b/elements/rpm-distro/pre-install.d/01-override-yum-arch index 138bf3134..c1347ec95 100755 --- a/elements/rpm-distro/pre-install.d/01-override-yum-arch +++ b/elements/rpm-distro/pre-install.d/01-override-yum-arch @@ -28,7 +28,7 @@ fi if [[ $DISTRO_NAME == "fedora" && $DIB_RELEASE -ge 22 ]]; then mkdir -p /etc/dnf/vars echo $basearch > /etc/dnf/vars/basearch - echo $arch > /etc/dnf/vars/basearch + echo $arch > /etc/dnf/vars/arch else echo $basearch > /etc/yum/vars/basearch echo $arch > /etc/yum/vars/arch