Start at using CI mirrors for fedora/centos

fedora/centos-minimal don't obey DIB_DISTRIBUTION_MIRROR currently.  I
don't really want them too -- we want to be able to separate the
mirrors used during the build process from those embedded into the
final image.  Add DIB_YUM_MINIMAL_BOOTSTRAP_REPOS which is a directory
with repo files to use during the install.

This introduces setup-gate-mirrors.sh which is intended to setup
repo/sources/whatever files in the openstack gate that point to the
local region mirror.  It pulls the info from the mirror_info.sh script
on each CI node.

The openstack-ci-mirrors element is updated to export these variables.
elements are updated to depend on it.  Tests are restored

Change-Id: I7604fc4d41cb1483be16b8d628a24e8fc764f515
This commit is contained in:
Ian Wienand 2017-06-20 10:42:10 +10:00
parent f0b70211c6
commit 649f0b66d9
14 changed files with 172 additions and 12 deletions

81
contrib/setup-gate-mirrors.sh Executable file
View File

@ -0,0 +1,81 @@
#!/bin/bash
set -x
#
# This tool creates repo/sources files that point to the mirrors for
# the host region in the OpenStack CI gate.
#
# This pre-created on CI nodes by slave scripts
source /etc/ci/mirror_info.sh
# Tests should probe for this directory and then use the repos/sources
# files inside it for the gate tests.
BASE_DIR=$WORKSPACE/dib-mirror
mkdir -p $BASE_DIR
## REPOS
# all should start with "dib-mirror-"
# gpg check turned off, because we don't have the keys outside the chroot
# fedora-minimal
FEDORA_MIN_DIR=$BASE_DIR/fedora-minimal/yum.repos.d
mkdir -p $FEDORA_MIN_DIR
cat <<EOF > $FEDORA_MIN_DIR/dib-mirror-fedora.repo
[fedora]
name=Fedora \$releasever - \$basearch
failovermethod=priority
baseurl=$NODEPOOL_FEDORA_MIRROR/releases/\$releasever/Everything/\$basearch/os/
enabled=1
metadata_expire=7d
gpgcheck=0
skip_if_unavailable=False
deltarpm=False
deltarpm_percentage=0
EOF
cat <<EOF > $FEDORA_MIN_DIR/dib-mirror-fedora-updates.repo
[updates]
name=Fedora \$releasever - \$basearch - Updates
failovermethod=priority
baseurl=$NODEPOOL_FEDORA_MIRROR/updates/\$releasever/\$basearch/
enabled=1
gpgcheck=0
metadata_expire=6h
skip_if_unavailable=False
deltarpm=False
deltarpm_percentage=0
EOF
# Centos Minimal
CENTOS_MIN_DIR=$BASE_DIR/centos-minimal/yum.repos.d
mkdir -p $CENTOS_MIN_DIR
cat <<EOF > $CENTOS_MIN_DIR/dib-mirror-base.repo
[base]
name=CentOS-\$releasever - Base
baseurl=$NODEPOOL_CENTOS_MIRROR/\$releasever/os/\$basearch/
gpgcheck=0
EOF
cat <<EOF > $CENTOS_MIN_DIR/dib-mirror-updates.repo
#released updates
[updates]
name=CentOS-\$releasever - Updates
baseurl=$NODEPOOL_CENTOS_MIRROR/\$releasever/updates/\$basearch/
gpgcheck=0
EOF
cat <<EOF > $CENTOS_MIN_DIR/dib-mirror-extras.repo
#additional packages that may be useful
[extras]
name=CentOS-\$releasever - Extras
baseurl=$NODEPOOL_CENTOS_MIRROR/\$releasever/extras/\$basearch/
gpgcheck=0
EOF
## apt sources (todo)

View File

@ -0,0 +1 @@
openstack-ci-mirrors

View File

@ -1 +1,2 @@
base
openstack-ci-mirrors

View File

@ -0,0 +1 @@
openstack-ci-mirrors

View File

@ -1,2 +0,0 @@
# Pin to this mirror because the roundrobin is fairly unreliable
export DIB_DISTRIBUTION_MIRROR=http://dl.fedoraproject.org/pub/fedora/linux

View File

@ -1 +1,2 @@
base
openstack-ci-mirrors

View File

@ -1,2 +0,0 @@
# Pin to this mirror because the roundrobin is fairly unreliable
export DIB_DISTRIBUTION_MIRROR=http://dl.fedoraproject.org/pub/fedora/linux

View File

@ -1,2 +1,7 @@
# Pin to this mirror because the roundrobin is fairly unreliable
# Set DIB_DISTRIBUTION_MIRROR if running in openstack gate
if [ -f /etc/ci/mirror_info.sh ]; then
source /etc/ci/mirror_info.sh
export DIB_DISTRIBUTION_MIRROR=$NODEPOOL_FEDORA_MIRROR
else
export DIB_DISTRIBUTION_MIRROR=http://dl.fedoraproject.org/pub/fedora/linux
fi

View File

@ -24,3 +24,15 @@ if [ -f /etc/ci/mirror_info.sh ]; then
fi
fi
# This is repo files pre-created for the fedora/centos-minimal jobs in
# the gate
if [[ -d ${WORKSPACE:-/not/a/path/}/dib-mirror ]]; then
if [[ "${DISTRO_NAME}" == "fedora" ]]; then
export DIB_YUM_MINIMAL_BOOTSTRAP_REPOS=${WORKSPACE}/dib-mirror/fedora-minimal/yum.repos.d
elif [[ "${DISTRO_NAME}" == "centos" ]]; then
export DIB_YUM_MINIMAL_BOOTSTRAP_REPOS=${WORKSPACE}/dib-mirror/centos-minimal/yum.repos.d
fi
fi

View File

@ -1 +1,2 @@
fedora-minimal
openstack-ci-mirrors

View File

@ -12,3 +12,9 @@ Ubuntu and Debian. Nothing additional is needed on Fedora or CentOS.
If you wish to have DHCP networking setup for eth0 & eth1 via
/etc/sysconfig/network-config scripts/ifcfg-eth[0|1], set the
environment variable `DIB_YUM_MINIMAL_CREATE_INTERFACES` to `1`.
If you wish to build from specific mirrors, set
``DIB_YUM_MINIMAL_BOOTSTRAP_REPOS`` to a directory with the ``.repo``
files to use during bootstrap and build. The repo files should be
named with a prefix ``dib-mirror-`` and will be removed from the final
image.

View File

@ -0,0 +1,35 @@
#!/bin/bash
#
# Copyright 2015 Hewlett-Packard Development Company, L.P.
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
#
# dib-lint: disable=safe_sudo
if [ "${DIB_DEBUG_TRACE:-0}" -gt 0 ]; then
set -x
fi
set -eu
set -o pipefail
# This removes any build-time repos that might be set in 08-yum-chroot
if [[ -n ${DIB_YUM_MINIMAL_BOOTSTRAP_REPOS:-} ]]; then
sudo rm -f $TMP_MOUNT_PATH/etc/yum.repos.d/dib-mirror*.repo
for repo in $TMP_MOUNT_PATH/etc/yum.repos.d/*.USING_MIRROR; do
sudo mv $repo ${repo/.USING_MIRROR/}
done
fi

View File

@ -78,15 +78,23 @@ function _install_repos {
packages+="${DISTRO_NAME}-repos "
fi
# By default, parent elements (fedora-minimal, centos-minimal)
# have a yum.repos.d directory in the element with a default repo;
# this is copied to TMP_HOOK_PATH by the usual hook-copying
# routines. In the gate, environment.d files for the funtional
# tests will set DIB_YUM_MINIMAL_BOOTSTRAP_REPOS -- this contains
# mirrors correct for the region setup by contrib/setup-gate-mirrors.sh
local repo=${DIB_YUM_MINIMAL_BOOTSTRAP_REPOS:-${TMP_HOOKS_PATH}/yum.repos.d}
# yumdownloader puts repo xml files and such into a directory
# ${TMPDIR}/yum-$USER-random. Since we don't need this once the
# initial download happens, redirect TMPDIR for this call so we
# can clean it up nicely
local temp_tmp
temp_tmp=$(mktemp -d)
TMPDIR=${temp_tmp} yumdownloader \
TMPDIR=${temp_tmp} yumdownloader --verbose \
--releasever=$DIB_RELEASE \
--setopt=reposdir=$TMP_HOOKS_PATH/yum.repos.d \
--setopt=reposdir=$repo \
--destdir=$WORKING \
${packages} && rc=$? || rc=$?
rm -rf ${temp_tmp}
@ -97,6 +105,18 @@ function _install_repos {
# --nodeps works around these wanting /bin/sh in some fedora
# releases, see rhbz#1265873
sudo $_RPM --root $TARGET_ROOT --nodeps -ivh $WORKING/*rpm
# install the bootstrap mirror repos over the default ones, if
# set. we will remove this at the end so the final image has
# regular mirrors
if [[ -n ${DIB_YUM_MINIMAL_BOOTSTRAP_REPOS:-} ]]; then
for repo in $TARGET_ROOT/etc/yum.repos.d/*.repo; do
sudo mv $repo $repo.USING_MIRROR
done
sudo cp ${DIB_YUM_MINIMAL_BOOTSTRAP_REPOS}/* \
$TARGET_ROOT/etc/yum.repos.d/
fi
}
# _install_pkg_manager packages...

View File

@ -11,6 +11,10 @@ export DIB_ELEMENTS=$(python -c '
import diskimage_builder.paths
diskimage_builder.paths.show_path("elements")')
# XXX: This will move into the gate scripts, and happen before
# we run this script ... just here for initial bringup
./openstack/diskimage-builder/contrib/setup-gate-mirrors.sh
# Setup sane locale defaults, because this information is leaked into DIB.
export LANG=en_US.utf8
export LC_ALL=
@ -42,10 +46,6 @@ DEFAULT_SKIP_TESTS=(
debian-minimal/stable-build-succeeds
debian-minimal/stable-vm
debian/build-succeeds
# temporary disable until change to use mirrors
pip-and-virtualenv/source-install-fedora
ironic-agent/build-succeeds-fedora
fedora-minimal/build-succeeds
)
# The default output formats (specified to disk-image-create's "-t"