Merge "Refactor mirror configuration script for scaling"

This commit is contained in:
Jenkins 2016-07-25 20:28:28 +00:00 committed by Gerrit Code Review
commit 2c6c35035f

View File

@ -21,6 +21,9 @@ source /etc/nodepool/provider
# Generate the AFS Slug from the host system. # Generate the AFS Slug from the host system.
source /usr/local/jenkins/slave_scripts/afs-slug.sh source /usr/local/jenkins/slave_scripts/afs-slug.sh
LSBDISTID=$(lsb_release -is)
LSBDISTCODENAME=$(lsb_release -cs)
NODEPOOL_MIRROR_HOST=${NODEPOOL_MIRROR_HOST:-mirror.$NODEPOOL_REGION.$NODEPOOL_CLOUD.openstack.org} NODEPOOL_MIRROR_HOST=${NODEPOOL_MIRROR_HOST:-mirror.$NODEPOOL_REGION.$NODEPOOL_CLOUD.openstack.org}
NODEPOOL_MIRROR_HOST=$(echo $NODEPOOL_MIRROR_HOST|tr '[:upper:]' '[:lower:]') NODEPOOL_MIRROR_HOST=$(echo $NODEPOOL_MIRROR_HOST|tr '[:upper:]' '[:lower:]')
NODEPOOL_PYPI_MIRROR=${NODEPOOL_PYPI_MIRROR:-http://$NODEPOOL_MIRROR_HOST/pypi/simple} NODEPOOL_PYPI_MIRROR=${NODEPOOL_PYPI_MIRROR:-http://$NODEPOOL_MIRROR_HOST/pypi/simple}
@ -32,67 +35,39 @@ NODEPOOL_CEPH_MIRROR=${NODEPOOL_CEPH_MIRROR:-http://$NODEPOOL_MIRROR_HOST/ceph-d
NODEPOOL_UCA_MIRROR=${NODEPOOL_UCA_MIRROR:-http://$NODEPOOL_MIRROR_HOST/ubuntu-cloud-archive} NODEPOOL_UCA_MIRROR=${NODEPOOL_UCA_MIRROR:-http://$NODEPOOL_MIRROR_HOST/ubuntu-cloud-archive}
NODEPOOL_NPM_MIRROR=${NODEPOOL_NPM_MIRROR:-http://$NODEPOOL_MIRROR_HOST/npm/} NODEPOOL_NPM_MIRROR=${NODEPOOL_NPM_MIRROR:-http://$NODEPOOL_MIRROR_HOST/npm/}
cat >/tmp/pip.conf <<EOF PIP_CONF="\
[global] [global]
timeout = 60 timeout = 60
index-url = $NODEPOOL_PYPI_MIRROR index-url = $NODEPOOL_PYPI_MIRROR
trusted-host = $NODEPOOL_MIRROR_HOST trusted-host = $NODEPOOL_MIRROR_HOST
extra-index-url = $NODEPOOL_WHEEL_MIRROR extra-index-url = $NODEPOOL_WHEEL_MIRROR"
EOF
sudo mv /tmp/pip.conf /etc/pip.conf
cat >/home/jenkins/.pydistutils.cfg <<EOF PYDISTUTILS_CFG="\
[easy_install] [easy_install]
index_url = $NODEPOOL_PYPI_MIRROR index_url = $NODEPOOL_PYPI_MIRROR
allow_hosts = *.openstack.org allow_hosts = *.openstack.org"
EOF
cat >/home/jenkins/.npmrc <<EOF NPMRC="\
registry = $NODEPOOL_NPM_MIRROR registry = $NODEPOOL_NPM_MIRROR
# Retry settings # Retry settings
fetch-retries=10 # The number of times to retry getting a package. fetch-retries=10 # The number of times to retry getting a package.
fetch-retry-mintimeout=60000 # Minimum fetch timeout: 1 minute (default 10 seconds) fetch-retry-mintimeout=60000 # Minimum fetch timeout: 1 minute (default 10 seconds)
fetch-retry-maxtimeout=300000 # Maximum fetch timeout: 5 minute (default 1 minute) fetch-retry-maxtimeout=300000 # Maximum fetch timeout: 5 minute (default 1 minute)"
EOF
UBUNTU_SOURCES_LIST="\
# Double check that when the node is made ready it is able
# to resolve names against DNS.
host git.openstack.org
host $NODEPOOL_MIRROR_HOST
LSBDISTID=$(lsb_release -is)
LSBDISTCODENAME=$(lsb_release -cs)
# NOTE(pabelanger): We don't actually have mirrors for ubuntu-precise, so skip
# them.
if [ "$LSBDISTID" == "Ubuntu" ] && [ "$LSBDISTCODENAME" != 'precise' ]; then
sudo dd of=/etc/apt/sources.list <<EOF
deb $NODEPOOL_UBUNTU_MIRROR $LSBDISTCODENAME main universe deb $NODEPOOL_UBUNTU_MIRROR $LSBDISTCODENAME main universe
deb $NODEPOOL_UBUNTU_MIRROR $LSBDISTCODENAME-updates main universe deb $NODEPOOL_UBUNTU_MIRROR $LSBDISTCODENAME-updates main universe
deb $NODEPOOL_UBUNTU_MIRROR $LSBDISTCODENAME-backports main universe deb $NODEPOOL_UBUNTU_MIRROR $LSBDISTCODENAME-backports main universe
deb $NODEPOOL_UBUNTU_MIRROR $LSBDISTCODENAME-security main universe deb $NODEPOOL_UBUNTU_MIRROR $LSBDISTCODENAME-security main universe"
EOF
# Opt in repos. Jobs that want to take advantage of them can copy or
# symlink them into /etc/apt/sources.list.d/
sudo mkdir -p /etc/apt/sources.list.available.d
sudo dd of=/etc/apt/sources.list.available.d/ceph-deb-hammer.list <<EOF
deb $NODEPOOL_CEPH_MIRROR $LSBDISTCODENAME main
EOF
sudo dd of=/etc/apt/sources.list.available.d/ubuntu-cloud-archive.list <<EOF
deb $NODEPOOL_UCA_MIRROR $LSBDISTCODENAME-updates main
EOF
if [ "$LSBDISTCODENAME" != 'precise' ] ; then CEPH_SOURCES_LIST="deb $NODEPOOL_CEPH_MIRROR $LSBDISTCODENAME main"
# Turn off multi-arch
sudo dpkg --remove-architecture i386 UCA_SOURCES_LIST="deb $NODEPOOL_UCA_MIRROR $LSBDISTCODENAME-updates main"
fi
# Turn off checking of GPG signatures APT_CONF_UNAUTHENTICATED="APT::Get::AllowUnauthenticated \"true\";"
sudo dd of=/etc/apt/apt.conf.d/99unauthenticated <<EOF
APT::Get::AllowUnauthenticated "true"; DEBIAN_SOURCES_LIST="\
EOF
elif [ "$LSBDISTID" == "Debian" ] ; then
sudo dd of=/etc/apt/sources.list <<EOF
deb http://httpredir.debian.org/debian $LSBDISTCODENAME main deb http://httpredir.debian.org/debian $LSBDISTCODENAME main
deb-src http://httpredir.debian.org/debian $LSBDISTCODENAME main deb-src http://httpredir.debian.org/debian $LSBDISTCODENAME main
@ -103,10 +78,9 @@ deb http://security.debian.org/ $LSBDISTCODENAME/updates main
deb-src http://security.debian.org/ $LSBDISTCODENAME/updates main deb-src http://security.debian.org/ $LSBDISTCODENAME/updates main
deb http://httpredir.debian.org/debian $LSBDISTCODENAME-backports main deb http://httpredir.debian.org/debian $LSBDISTCODENAME-backports main
deb-src http://httpredir.debian.org/debian $LSBDISTCODENAME-backports main deb-src http://httpredir.debian.org/debian $LSBDISTCODENAME-backports main"
EOF
elif [ "$LSBDISTID" == "CentOS" ]; then YUM_REPOS_CENTOS_BASE="\
sudo dd of=/etc/yum.repos.d/CentOS-Base.repo <<EOF
[base] [base]
name=CentOS-\$releasever - Base name=CentOS-\$releasever - Base
baseurl=$NODEPOOL_CENTOS_MIRROR/\$releasever/os/\$basearch/ baseurl=$NODEPOOL_CENTOS_MIRROR/\$releasever/os/\$basearch/
@ -125,16 +99,79 @@ gpgkey=file:///etc/pki/rpm-gpg/RPM-GPG-KEY-CentOS-\$releasever
name=CentOS-\$releasever - Extras name=CentOS-\$releasever - Extras
baseurl=$NODEPOOL_CENTOS_MIRROR/\$releasever/extras/\$basearch/ baseurl=$NODEPOOL_CENTOS_MIRROR/\$releasever/extras/\$basearch/
gpgcheck=1 gpgcheck=1
gpgkey=file:///etc/pki/rpm-gpg/RPM-GPG-KEY-CentOS-\$releasever gpgkey=file:///etc/pki/rpm-gpg/RPM-GPG-KEY-CentOS-\$releasever"
EOF
sudo dd of=/etc/yum.repos.d/epel.repo <<EOF YUM_REPOS_EPEL="\
[epel] [epel]
name=Extra Packages for Enterprise Linux \$releasever - \$basearch name=Extra Packages for Enterprise Linux \$releasever - \$basearch
baseurl=$NODEPOOL_EPEL_MIRROR/\$releasever/\$basearch baseurl=$NODEPOOL_EPEL_MIRROR/\$releasever/\$basearch
failovermethod=priority failovermethod=priority
enabled=1 enabled=1
gpgcheck=1 gpgcheck=1
gpgkey=file:///etc/pki/rpm-gpg/RPM-GPG-KEY-EPEL-\$releasever gpgkey=file:///etc/pki/rpm-gpg/RPM-GPG-KEY-EPEL-\$releasever"
EOF
# Double check that when the node is made ready it is able
# to resolve names against DNS.
host git.openstack.org
host $NODEPOOL_MIRROR_HOST
# Write global pip configuration
echo "$PIP_CONF" >/tmp/pip.conf
sudo mv /tmp/pip.conf /etc/
sudo chown root:root /etc/pip.conf
sudo chmod 0644 /etc/pip.conf
# Write jenkins user distutils/setuptools configuration used by easy_install
echo "$PYDISTUTILS_CFG" >/home/jenkins/.pydistutils.cfg
# Write jenkins user npm configuration
echo "$NPMRC" >/home/jenkins/.npmrc
# NOTE(pabelanger): We don't actually have mirrors for ubuntu-precise, so skip
# them.
if [ "$LSBDISTID" == "Ubuntu" ] && [ "$LSBDISTCODENAME" != 'precise' ]; then
echo "$UBUNTU_SOURCES_LIST" >/tmp/sources.list
sudo mv /tmp/sources.list /etc/apt/
sudo chown root:root /etc/apt/sources.list
sudo chmod 0644 /etc/apt/sources.list
# Opt in repos. Jobs that want to take advantage of them can copy or
# symlink them into /etc/apt/sources.list.d/
sudo mkdir -p /etc/apt/sources.list.available.d
# Ceph
echo "$CEPH_SOURCES_LIST" >/tmp/ceph-deb-hammer.list
sudo mv /tmp/ceph-deb-hammer.list /etc/apt/sources.list.available.d/
# Ubuntu Cloud Archive
echo "$UCA_SOURCES_LIST" >/tmp/ubuntu-cloud-archive.list
sudo mv /tmp/ubuntu-cloud-archive.list /etc/apt/sources.list.available.d/
sudo chown root:root /etc/apt/sources.list.available.d/*
sudo chmod 0644 /etc/apt/sources.list.available.d/*
if [ "$LSBDISTCODENAME" != 'precise' ] ; then
# Turn off multi-arch
sudo dpkg --remove-architecture i386
fi
# Turn off checking of GPG signatures
echo "$APT_CONF_UNAUTHENTICATED" >/tmp/99unauthenticated
sudo mv /tmp/99unauthenticated /etc/apt/apt.conf.d/
sudo chown root:root /etc/apt/apt.conf.d/99unauthenticated
sudo chmod 0644 /etc/apt/apt.conf.d/99unauthenticated
elif [ "$LSBDISTID" == "Debian" ] ; then
echo "$DEBIAN_SOURCES_LIST" >/tmp/sources.list
sudo mv /tmp/sources.list /etc/apt/
sudo chown root:root /etc/apt/sources.list
sudo chmod 0644 /etc/apt/sources.list
elif [ "$LSBDISTID" == "CentOS" ]; then
echo "$YUM_REPOS_CENTOS_BASE" >/tmp/CentOS-Base.repo
sudo mv /tmp/CentOS-Base.repo /etc/yum.repos.d/
echo "$YUM_REPOS_EPEL" >/tmp/epel.repo
sudo mv /tmp/epel.repo /etc/yum.repos.d/
sudo chown root:root /etc/yum.repos.d/*
sudo chmod 0644 /etc/yum.repos.d/*
fi fi