From 0ba568d42e53c4526e5735ac5181c0d036d4998b Mon Sep 17 00:00:00 2001
From: Jeremy Stanley <fungi@yuggoth.org>
Date: Thu, 21 Jul 2016 18:40:55 +0000
Subject: [PATCH] Refactor mirror configuration script for scaling

To make it easier to follow additions to configure_mirror.sh as we
grow more and more mirrors of things, separate the configuration
file contents into separate variable definitions so they don't make
the script logic as hard to follow.

Change-Id: I421da8eefeb040d9daded59b8d62158f0aa1610b
Signed-off-by: Paul Belanger <pabelanger@redhat.com>
---
 nodepool/scripts/configure_mirror.sh | 141 +++++++++++++++++----------
 1 file changed, 89 insertions(+), 52 deletions(-)

diff --git a/nodepool/scripts/configure_mirror.sh b/nodepool/scripts/configure_mirror.sh
index 01c4b433fc..c253f33864 100755
--- a/nodepool/scripts/configure_mirror.sh
+++ b/nodepool/scripts/configure_mirror.sh
@@ -21,6 +21,9 @@ source /etc/nodepool/provider
 # Generate the AFS Slug from the host system.
 source /usr/local/jenkins/slave_scripts/afs-slug.sh
 
+LSBDISTID=$(lsb_release -is)
+LSBDISTCODENAME=$(lsb_release -cs)
+
 NODEPOOL_MIRROR_HOST=${NODEPOOL_MIRROR_HOST:-mirror.$NODEPOOL_REGION.$NODEPOOL_CLOUD.openstack.org}
 NODEPOOL_MIRROR_HOST=$(echo $NODEPOOL_MIRROR_HOST|tr '[:upper:]' '[:lower:]')
 NODEPOOL_PYPI_MIRROR=${NODEPOOL_PYPI_MIRROR:-http://$NODEPOOL_MIRROR_HOST/pypi/simple}
@@ -32,67 +35,39 @@ NODEPOOL_CEPH_MIRROR=${NODEPOOL_CEPH_MIRROR:-http://$NODEPOOL_MIRROR_HOST/ceph-d
 NODEPOOL_UCA_MIRROR=${NODEPOOL_UCA_MIRROR:-http://$NODEPOOL_MIRROR_HOST/ubuntu-cloud-archive}
 NODEPOOL_NPM_MIRROR=${NODEPOOL_NPM_MIRROR:-http://$NODEPOOL_MIRROR_HOST/npm/}
 
-cat >/tmp/pip.conf <<EOF
+PIP_CONF="\
 [global]
 timeout = 60
 index-url = $NODEPOOL_PYPI_MIRROR
 trusted-host = $NODEPOOL_MIRROR_HOST
-extra-index-url = $NODEPOOL_WHEEL_MIRROR
-EOF
-sudo mv /tmp/pip.conf /etc/pip.conf
+extra-index-url = $NODEPOOL_WHEEL_MIRROR"
 
-cat >/home/jenkins/.pydistutils.cfg <<EOF
+PYDISTUTILS_CFG="\
 [easy_install]
 index_url = $NODEPOOL_PYPI_MIRROR
-allow_hosts = *.openstack.org
-EOF
+allow_hosts = *.openstack.org"
 
-cat >/home/jenkins/.npmrc <<EOF
+NPMRC="\
 registry = $NODEPOOL_NPM_MIRROR
 
 # Retry settings
 fetch-retries=10              # The number of times to retry getting a package.
 fetch-retry-mintimeout=60000  # Minimum fetch timeout: 1 minute (default 10 seconds)
-fetch-retry-maxtimeout=300000 # Maximum fetch timeout: 5 minute (default 1 minute)
-EOF
+fetch-retry-maxtimeout=300000 # Maximum fetch timeout: 5 minute (default 1 minute)"
 
-
-# Double check that when the node is made ready it is able
-# to resolve names against DNS.
-host git.openstack.org
-host $NODEPOOL_MIRROR_HOST
-
-LSBDISTID=$(lsb_release -is)
-LSBDISTCODENAME=$(lsb_release -cs)
-# NOTE(pabelanger): We don't actually have mirrors for ubuntu-precise, so skip
-# them.
-if [ "$LSBDISTID" == "Ubuntu" ] && [ "$LSBDISTCODENAME" != 'precise' ]; then
-        sudo dd of=/etc/apt/sources.list <<EOF
+UBUNTU_SOURCES_LIST="\
 deb $NODEPOOL_UBUNTU_MIRROR $LSBDISTCODENAME main universe
 deb $NODEPOOL_UBUNTU_MIRROR $LSBDISTCODENAME-updates main universe
 deb $NODEPOOL_UBUNTU_MIRROR $LSBDISTCODENAME-backports main universe
-deb $NODEPOOL_UBUNTU_MIRROR $LSBDISTCODENAME-security main universe
-EOF
-    # Opt in repos. Jobs that want to take advantage of them can copy or
-    # symlink them into /etc/apt/sources.list.d/
-    sudo mkdir -p /etc/apt/sources.list.available.d
-    sudo dd of=/etc/apt/sources.list.available.d/ceph-deb-hammer.list <<EOF
-deb $NODEPOOL_CEPH_MIRROR $LSBDISTCODENAME main
-EOF
-    sudo dd of=/etc/apt/sources.list.available.d/ubuntu-cloud-archive.list <<EOF
-deb $NODEPOOL_UCA_MIRROR $LSBDISTCODENAME-updates main
-EOF
+deb $NODEPOOL_UBUNTU_MIRROR $LSBDISTCODENAME-security main universe"
 
-    if [ "$LSBDISTCODENAME" != 'precise' ] ; then
-        # Turn off multi-arch
-        sudo dpkg --remove-architecture i386
-    fi
-    # Turn off checking of GPG signatures
-    sudo dd of=/etc/apt/apt.conf.d/99unauthenticated <<EOF
-APT::Get::AllowUnauthenticated "true";
-EOF
-elif [ "$LSBDISTID" == "Debian" ] ; then
-sudo dd of=/etc/apt/sources.list <<EOF
+CEPH_SOURCES_LIST="deb $NODEPOOL_CEPH_MIRROR $LSBDISTCODENAME main"
+
+UCA_SOURCES_LIST="deb $NODEPOOL_UCA_MIRROR $LSBDISTCODENAME-updates main"
+
+APT_CONF_UNAUTHENTICATED="APT::Get::AllowUnauthenticated \"true\";"
+
+DEBIAN_SOURCES_LIST="\
 deb http://httpredir.debian.org/debian $LSBDISTCODENAME main
 deb-src http://httpredir.debian.org/debian $LSBDISTCODENAME main
 
@@ -103,10 +78,9 @@ deb http://security.debian.org/ $LSBDISTCODENAME/updates main
 deb-src http://security.debian.org/ $LSBDISTCODENAME/updates main
 
 deb http://httpredir.debian.org/debian $LSBDISTCODENAME-backports main
-deb-src http://httpredir.debian.org/debian $LSBDISTCODENAME-backports main
-EOF
-elif [ "$LSBDISTID" == "CentOS" ]; then
-    sudo dd of=/etc/yum.repos.d/CentOS-Base.repo <<EOF
+deb-src http://httpredir.debian.org/debian $LSBDISTCODENAME-backports main"
+
+YUM_REPOS_CENTOS_BASE="\
 [base]
 name=CentOS-\$releasever - Base
 baseurl=$NODEPOOL_CENTOS_MIRROR/\$releasever/os/\$basearch/
@@ -125,16 +99,79 @@ gpgkey=file:///etc/pki/rpm-gpg/RPM-GPG-KEY-CentOS-\$releasever
 name=CentOS-\$releasever - Extras
 baseurl=$NODEPOOL_CENTOS_MIRROR/\$releasever/extras/\$basearch/
 gpgcheck=1
-gpgkey=file:///etc/pki/rpm-gpg/RPM-GPG-KEY-CentOS-\$releasever
-EOF
+gpgkey=file:///etc/pki/rpm-gpg/RPM-GPG-KEY-CentOS-\$releasever"
 
-    sudo dd of=/etc/yum.repos.d/epel.repo <<EOF
+YUM_REPOS_EPEL="\
 [epel]
 name=Extra Packages for Enterprise Linux \$releasever - \$basearch
 baseurl=$NODEPOOL_EPEL_MIRROR/\$releasever/\$basearch
 failovermethod=priority
 enabled=1
 gpgcheck=1
-gpgkey=file:///etc/pki/rpm-gpg/RPM-GPG-KEY-EPEL-\$releasever
-EOF
+gpgkey=file:///etc/pki/rpm-gpg/RPM-GPG-KEY-EPEL-\$releasever"
+
+# Double check that when the node is made ready it is able
+# to resolve names against DNS.
+host git.openstack.org
+host $NODEPOOL_MIRROR_HOST
+
+# Write global pip configuration
+echo "$PIP_CONF" >/tmp/pip.conf
+sudo mv /tmp/pip.conf /etc/
+sudo chown root:root /etc/pip.conf
+sudo chmod 0644 /etc/pip.conf
+
+# Write jenkins user distutils/setuptools configuration used by easy_install
+echo "$PYDISTUTILS_CFG" >/home/jenkins/.pydistutils.cfg
+
+# Write jenkins user npm configuration
+echo "$NPMRC" >/home/jenkins/.npmrc
+
+# NOTE(pabelanger): We don't actually have mirrors for ubuntu-precise, so skip
+# them.
+if [ "$LSBDISTID" == "Ubuntu" ] && [ "$LSBDISTCODENAME" != 'precise' ]; then
+    echo "$UBUNTU_SOURCES_LIST" >/tmp/sources.list
+    sudo mv /tmp/sources.list /etc/apt/
+    sudo chown root:root /etc/apt/sources.list
+    sudo chmod 0644 /etc/apt/sources.list
+
+    # Opt in repos. Jobs that want to take advantage of them can copy or
+    # symlink them into /etc/apt/sources.list.d/
+    sudo mkdir -p /etc/apt/sources.list.available.d
+
+    # Ceph
+    echo "$CEPH_SOURCES_LIST" >/tmp/ceph-deb-hammer.list
+    sudo mv /tmp/ceph-deb-hammer.list /etc/apt/sources.list.available.d/
+
+    # Ubuntu Cloud Archive
+    echo "$UCA_SOURCES_LIST" >/tmp/ubuntu-cloud-archive.list
+    sudo mv /tmp/ubuntu-cloud-archive.list /etc/apt/sources.list.available.d/
+
+    sudo chown root:root /etc/apt/sources.list.available.d/*
+    sudo chmod 0644 /etc/apt/sources.list.available.d/*
+
+    if [ "$LSBDISTCODENAME" != 'precise' ] ; then
+        # Turn off multi-arch
+        sudo dpkg --remove-architecture i386
+    fi
+    # Turn off checking of GPG signatures
+    echo "$APT_CONF_UNAUTHENTICATED" >/tmp/99unauthenticated
+    sudo mv /tmp/99unauthenticated /etc/apt/apt.conf.d/
+    sudo chown root:root /etc/apt/apt.conf.d/99unauthenticated
+    sudo chmod 0644 /etc/apt/apt.conf.d/99unauthenticated
+
+elif [ "$LSBDISTID" == "Debian" ] ; then
+    echo "$DEBIAN_SOURCES_LIST" >/tmp/sources.list
+    sudo mv /tmp/sources.list /etc/apt/
+    sudo chown root:root /etc/apt/sources.list
+    sudo chmod 0644 /etc/apt/sources.list
+
+elif [ "$LSBDISTID" == "CentOS" ]; then
+    echo "$YUM_REPOS_CENTOS_BASE" >/tmp/CentOS-Base.repo
+    sudo mv /tmp/CentOS-Base.repo /etc/yum.repos.d/
+    echo "$YUM_REPOS_EPEL" >/tmp/epel.repo
+    sudo mv /tmp/epel.repo /etc/yum.repos.d/
+    sudo chown root:root /etc/yum.repos.d/*
+    sudo chmod 0644 /etc/yum.repos.d/*
+
 fi