From 54b038a76cc0abf6f377366eb2e6d96857e40553 Mon Sep 17 00:00:00 2001 From: Monty Taylor Date: Sun, 25 May 2014 11:01:12 -0400 Subject: [PATCH] Add support for disk-image-builder in nodepool In order to use disk-image-builder to build nodepool images, we need a few things. First, we need additional debs to be installed for disk-image-builder to work. Then we need the image elements themselves to describe the images we want to create. Also, a few helper scripts for locally working with the disk images have been included. Tested with build-image.sh script; devstack-gate-trusty.qcow2 created correctly Change-Id: I539743147341dc5b387d103fecd3e9ff0f01fdfc --- .../files/nodepool/elements/README.rst | 54 +++++ .../elements/cache-devstack/README.rst | 1 + .../elements/cache-devstack/element-deps | 1 + .../extra-data.d/50-early-source-repo | 209 ++++++++++++++++++ .../extra-data.d/55-cache-devstack-repos | 185 ++++++++++++++++ .../extra-data.d/60-rm-early-source-repo | 25 +++ .../cache-devstack/install.d/50-download-pkgs | 30 +++ .../elements/node-devstack/README.rst | 1 + .../elements/node-devstack/element-deps | 2 + .../node-devstack/install.d/20-prepare-node | 28 +++ .../elements/nodepool-base/README.rst | 1 + .../exta-data.d/50-copy-nodepool-scripts | 24 ++ .../nodepool-base/finalise.d/99-unbound | 54 +++++ .../nodepool-base/install.d/05-record-details | 18 ++ .../install.d/50-restrict-memory | 31 +++ .../nodepool-base/install.d/99-install-zuul | 22 ++ .../elements/openstack-repos/README.rst | 1 + .../elements/openstack-repos/element-deps | 3 + .../extra-data.d/50-create-repo-list | 55 +++++ .../install.d/95-chown-jenkins | 20 ++ .../files/nodepool/elements/puppet/README.rst | 1 + .../nodepool/elements/puppet/bin/prepare-node | 55 +++++ .../nodepool/elements/puppet/element-deps | 3 + .../elements/puppet/install.d/05-puppet | 27 +++ .../elements/puppet/install.d/95-clean-repos | 37 ++++ .../elements/puppet/install.d/96-clean-cron | 23 ++ .../elements/puppet/pre-install.d/10-preseed | 39 ++++ .../nodepool/elements/slave-db/README.rst | 1 + 28 files changed, 951 insertions(+) create mode 100644 modules/openstack_project/files/nodepool/elements/README.rst create mode 100644 modules/openstack_project/files/nodepool/elements/cache-devstack/README.rst create mode 100644 modules/openstack_project/files/nodepool/elements/cache-devstack/element-deps create mode 100755 modules/openstack_project/files/nodepool/elements/cache-devstack/extra-data.d/50-early-source-repo create mode 100755 modules/openstack_project/files/nodepool/elements/cache-devstack/extra-data.d/55-cache-devstack-repos create mode 100755 modules/openstack_project/files/nodepool/elements/cache-devstack/extra-data.d/60-rm-early-source-repo create mode 100644 modules/openstack_project/files/nodepool/elements/cache-devstack/install.d/50-download-pkgs create mode 100644 modules/openstack_project/files/nodepool/elements/node-devstack/README.rst create mode 100644 modules/openstack_project/files/nodepool/elements/node-devstack/element-deps create mode 100755 modules/openstack_project/files/nodepool/elements/node-devstack/install.d/20-prepare-node create mode 100644 modules/openstack_project/files/nodepool/elements/nodepool-base/README.rst create mode 100755 modules/openstack_project/files/nodepool/elements/nodepool-base/exta-data.d/50-copy-nodepool-scripts create mode 100755 modules/openstack_project/files/nodepool/elements/nodepool-base/finalise.d/99-unbound create mode 100755 modules/openstack_project/files/nodepool/elements/nodepool-base/install.d/05-record-details create mode 100755 modules/openstack_project/files/nodepool/elements/nodepool-base/install.d/50-restrict-memory create mode 100755 modules/openstack_project/files/nodepool/elements/nodepool-base/install.d/99-install-zuul create mode 100644 modules/openstack_project/files/nodepool/elements/openstack-repos/README.rst create mode 100644 modules/openstack_project/files/nodepool/elements/openstack-repos/element-deps create mode 100755 modules/openstack_project/files/nodepool/elements/openstack-repos/extra-data.d/50-create-repo-list create mode 100755 modules/openstack_project/files/nodepool/elements/openstack-repos/install.d/95-chown-jenkins create mode 100644 modules/openstack_project/files/nodepool/elements/puppet/README.rst create mode 100644 modules/openstack_project/files/nodepool/elements/puppet/bin/prepare-node create mode 100644 modules/openstack_project/files/nodepool/elements/puppet/element-deps create mode 100755 modules/openstack_project/files/nodepool/elements/puppet/install.d/05-puppet create mode 100755 modules/openstack_project/files/nodepool/elements/puppet/install.d/95-clean-repos create mode 100755 modules/openstack_project/files/nodepool/elements/puppet/install.d/96-clean-cron create mode 100755 modules/openstack_project/files/nodepool/elements/puppet/pre-install.d/10-preseed create mode 100644 modules/openstack_project/files/nodepool/elements/slave-db/README.rst diff --git a/modules/openstack_project/files/nodepool/elements/README.rst b/modules/openstack_project/files/nodepool/elements/README.rst new file mode 100644 index 0000000000..cb98263bcc --- /dev/null +++ b/modules/openstack_project/files/nodepool/elements/README.rst @@ -0,0 +1,54 @@ +Using diskimage-builder to build devstack-gate nodes +==================================================== + +In addition to being able to just download and consume images that are the +same as what run devstack-gate, it's easy to make your own for local dev or +testing - or just for fun. + +Install diskimage-builder +------------------------- + +Install the dependencies: + +:: + + sudo apt-get install kpartx qemu-utils curl + +Install diskimage-builder: + +:: + + sudo pip install diskimage-builder + + +Build an image +-------------- + +Building an image is simple, we have a script! + +:: + + bash tools/build-image.sh + +You should be left with a file called devstack-gate-precise.qcow2. + +Mounting the image +------------------ + +If you would like to examine the contents of the image, you can mount it on +a loopback device using qemu-nbd. + +:: + + sudo apt-get install qemu-utils + sudo modprobe nbd max_part=16 + sudo mkdir -p /tmp/newimage + sudo qemu-nbd -c /dev/nbd1 devstack-gate-precise.qcow2 + sudo mount /dev/nbd1 /tmp/newimage + +Other things +------------ + +It's a qcow2 image, so you can do tons of things with it. You can upload it +to glance, you can boot it using kvm, and you can even copy it to a cloud +server, replace the contents of the server with it and kexec the new kernel. diff --git a/modules/openstack_project/files/nodepool/elements/cache-devstack/README.rst b/modules/openstack_project/files/nodepool/elements/cache-devstack/README.rst new file mode 100644 index 0000000000..1a568aceed --- /dev/null +++ b/modules/openstack_project/files/nodepool/elements/cache-devstack/README.rst @@ -0,0 +1 @@ +Pre-cache all of the things devstack might need diff --git a/modules/openstack_project/files/nodepool/elements/cache-devstack/element-deps b/modules/openstack_project/files/nodepool/elements/cache-devstack/element-deps new file mode 100644 index 0000000000..7b2a984ae5 --- /dev/null +++ b/modules/openstack_project/files/nodepool/elements/cache-devstack/element-deps @@ -0,0 +1 @@ +openstack-repos diff --git a/modules/openstack_project/files/nodepool/elements/cache-devstack/extra-data.d/50-early-source-repo b/modules/openstack_project/files/nodepool/elements/cache-devstack/extra-data.d/50-early-source-repo new file mode 100755 index 0000000000..c6ef8c2d12 --- /dev/null +++ b/modules/openstack_project/files/nodepool/elements/cache-devstack/extra-data.d/50-early-source-repo @@ -0,0 +1,209 @@ +#!/bin/bash + +# Copyright (c) 2012-2014 Hewlett-Packard Development Company, L.P. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +# implied. +# +# See the License for the specific language governing permissions and +# limitations under the License. + +# We need a copy of the devstack repo so that we can read it so that +# we can generate the list of repos that we need to get. We'd like to +# use the normal repo fetching and caching routines for that, so use +# a modified version. + +# The bulk of this code is copied from +# elements/source-repositories/extra-data.d/98-source-repositories +# Most of it should be removed when we can source and call the +# functions directly + +set -eu + +# If the old cache exists, move it to the new name +function make_new_cache(){ + local OLD_CACHE_BASE=$1 + local CACHE_BASE=$2 + + # If the old cache name exists, move it to the new cache name + if [ -e "$OLD_CACHE_BASE" ] ; then + if [ ! -e "$CACHE_BASE" ] ; then + mv -n $OLD_CACHE_BASE $CACHE_BASE + else + echo "Not replacing new cache location with old cache" + fi + fi +} + +# Gets repositories or individual files listed in the a repository file +# and places them in the specified destination path. +# The format of the repository file is one or more lines matching +# [] +function get_repos_for_element(){ + local REPO_SOURCES=$1 + local CACHE_URL=$TMP_HOOKS_PATH/bin/cache-url + + local REGEX="^([^ ]+) (git|tar|file|package) ?(/[^ ]+)? ?([^ ]+)? ?([^ ]*)$" + + while read line; do + # expand variables + line=$(eval echo $line) + + # ignore blank lines and lines beginning in '#' + [[ "$line" == \#* ]] || [[ -z "$line" ]] && continue + + if [[ "$line" =~ $REGEX ]] ; then + local REPONAME=${BASH_REMATCH[1]} + local REPOTYPE=${BASH_REMATCH[2]} + local REPOPATH=${BASH_REMATCH[3]} + local REPOLOCATION=${BASH_REMATCH[4]} + local REPO_ORIG_LOCATION=$REPOLOCATION + local REPOREF=${BASH_REMATCH[5]:-master} + + local REPO_DEST=$TMP_MOUNT_PATH$REPOPATH + local REPO_SUB_DIRECTORY=$(dirname $REPO_DEST) + + # REPOTYPE can be overridden with DIB_REPOTYPE_{name} + local REPOTYPE_OVERRIDE=DIB_REPOTYPE_${REPONAME//[^A-Za-z0-9]/_} + REPOTYPE=${!REPOTYPE_OVERRIDE:-$REPOTYPE} + + # REPOLOCATION can be overridden with DIB_REPOLOCATION_{name} + local REPOLOCATION_OVERRIDE=DIB_REPOLOCATION_${REPONAME//[^A-Za-z0-9]/_} + REPOLOCATION=${!REPOLOCATION_OVERRIDE:-$REPOLOCATION} + + # REPOREF can be overridden with DIB_REPOREF_{name} + local REPOREF_OVERRIDE=DIB_REPOREF_${REPONAME//[^A-Za-z0-9]/_} + REPOREF=${!REPOREF_OVERRIDE:-$REPOREF} + + # Determine a unique cache path for this repo + CACHE_NAME=$(echo "${REPOTYPE}_${REPOLOCATION}" | sha1sum | awk '{ print $1 }' ) + OLD_CACHE_PATH=${CACHE_BASE}/${CACHE_NAME} + # Add the repo name to the sha1sum for readability + CACHE_NAME=${REPONAME//[^A-Za-z0-9]/_}_${CACHE_NAME} + CACHE_PATH=${CACHE_BASE}/$CACHE_NAME + make_new_cache $OLD_CACHE_PATH $CACHE_PATH + + # Return if install type is not source + local INSTALL_TYPE_VAR=DIB_INSTALLTYPE_${REPONAME//[^A-Za-z0-9]/_} + local INSTALL_TYPE=${!INSTALL_TYPE_VAR:-source} + if [ ! $INSTALL_TYPE = "source" ]; then + echo "$REPONAME install type not set to source" + continue + fi + + case $REPOTYPE in + git) + if [ -z "${!REPOLOCATION_OVERRIDE:-""}" -a -n "${DIB_GITREPOBASE:-""}" ] ; then + # Transform the current repo base to the new one + local NEW_REPOLOCATION=$(echo $REPOLOCATION |\ + sed "s,^[^:]\+://[^/]\+/\(~[^/]\+\)\?\(.*\)$,${DIB_GITREPOBASE}\2,g") + echo "Transformed ${REPOLOCATION} to ${NEW_REPOLOCATION}" + REPOLOCATION=$NEW_REPOLOCATION + # Also update the cache location + CACHE_NAME=$(echo "${REPOTYPE}_${REPOLOCATION}" | sha1sum | awk '{ print $1 }' ) + CACHE_PATH=~/.cache/image-create/repository-sources/$CACHE_NAME + fi + sudo mkdir -p $REPO_SUB_DIRECTORY + + if [ ! -e "$CACHE_PATH" ] ; then + echo "Caching $REPONAME from $REPOLOCATION in $CACHE_PATH" + git clone $REPOLOCATION $CACHE_PATH.tmp + mv ${CACHE_PATH}{.tmp,} + fi + + HAS_REF=$(git --git-dir=$CACHE_PATH/.git name-rev $REPOREF 2>/dev/null || true) + if [ -z "$DIB_OFFLINE" -o -z "$HAS_REF" ] ; then + echo "Updating cache of $REPOLOCATION in $CACHE_PATH with ref $REPOREF" + git --git-dir=$CACHE_PATH/.git fetch --update-head-ok $REPOLOCATION +refs/heads/*:refs/heads/* + fi + + echo "Cloning from $REPONAME cache and applying ref $REPOREF" + sudo git clone $CACHE_PATH $REPO_DEST + pushd $REPO_DEST + sudo git fetch $CACHE_PATH $REPOREF + sudo git reset --hard FETCH_HEAD + # Get the reference in use + git_ref=$(git rev-parse FETCH_HEAD) + popd + + # Write the reference being used into the source-repositories manifest + echo "$REPONAME git $REPOPATH $REPOLOCATION $git_ref" >> $GIT_MANIFEST + ;; + tar) + # The top level directory of the tarball mightn't have a fixed name i.e. + # it could contain version numbers etc... so we write it to a tmpdir + # the then move the contents into the directory we want it in, this does + # assume the tarball only contains a single top level directory + local tmpdir=$(mktemp --tmpdir=$TMP_MOUNT_PATH/tmp -d) + if [ -n "$CACHE_PATH" ] ; then + echo "Caching $REPONAME tarball from $REPOLOCATION in $CACHE_PATH" + if [ ! -f "$CACHE_PATH" -o -z "$DIB_OFFLINE" ] ; then + $CACHE_URL $REPOLOCATION $CACHE_PATH + fi + tar -C $tmpdir -xzf $CACHE_PATH + else + echo "Fetching $REPONAME tarball from $REPOLOCATION" + curl $REPOLOCATION | tar -C $tmpdir -xzf - + fi + sudo mkdir -p $REPO_DEST + sudo mv $tmpdir/*/* $REPO_DEST + rm -rf $tmpdir + ;; + file) + sudo mkdir -p $REPO_SUB_DIRECTORY + if [ -n "$CACHE_PATH" ] ; then + echo "Caching $REPONAME file from $REPOLOCATION in $CACHE_PATH" + if [ ! -f "$CACHE_PATH" -o -z "$DIB_OFFLINE" ] ; then + $CACHE_URL $REPOLOCATION $CACHE_PATH + fi + sudo cp $CACHE_PATH $REPO_DEST + else + echo "Fetching $REPONAME file from $REPOLOCATION" + sudo curl $REPOLOCATION -o $REPO_DEST + fi + ;; + *) + echo "Unsupported repository type: $REPOTYPE" + return 1 + ;; + esac + + # Capture the in-instance repository path for later review / other + # elements (like a pypi dependency cache). + echo "$REPOPATH" | sudo dd of=$TMP_MOUNT_PATH/etc/dib-source-repositories oflag=append conv=notrunc + + else + echo "Couldn't parse '$line' as a source repository" + return 1 + fi + done < $REPO_SOURCES +} + +CACHE_BASE=~/.cache/image-create/source-repositories +OLD_CACHE_BASE=~/.cache/image-create/repository-sources +make_new_cache $OLD_CACHE_BASE $CACHE_BASE +mkdir -p $CACHE_BASE +# Use the IMAGE_NAME from the calling script, and make it unique with the temporary path +GIT_MANIFEST_NAME=dib-manifest-git-$(basename ${IMAGE_NAME}) +GIT_MANIFEST_CACHE_NAME=${GIT_MANIFEST_NAME}_$(dirname ${TMP_MOUNT_PATH##*.}) +GIT_MANIFEST=$CACHE_BASE/${GIT_MANIFEST_CACHE_NAME} +rm -f $GIT_MANIFEST + +# Get source repositories for the target +echo "devstack git /opt/git/openstack-dev/devstack git://git.openstack.org/openstack-dev/devstack.git" > $TMP_HOOKS_PATH/early-source-repository-config +for _SOURCEREPO in $(find $TMP_HOOKS_PATH -maxdepth 1 -name "early-source-repository-*" -not -name '*~'); do + get_repos_for_element $_SOURCEREPO +done + +# Copy the manifest into the image if it exists (there may be no git repositories used) +if [ -e "$GIT_MANIFEST" ] ; then + sudo cp $GIT_MANIFEST $TMP_MOUNT_PATH/${DIB_MANIFEST_IMAGE_DIR}/$GIT_MANIFEST_NAME +fi diff --git a/modules/openstack_project/files/nodepool/elements/cache-devstack/extra-data.d/55-cache-devstack-repos b/modules/openstack_project/files/nodepool/elements/cache-devstack/extra-data.d/55-cache-devstack-repos new file mode 100755 index 0000000000..bb2afee4de --- /dev/null +++ b/modules/openstack_project/files/nodepool/elements/cache-devstack/extra-data.d/55-cache-devstack-repos @@ -0,0 +1,185 @@ +#!/usr/bin/env python + +# Copyright (C) 2011-2013 OpenStack Foundation +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +# implied. +# +# See the License for the specific language governing permissions and +# limitations under the License. + +import os +import subprocess +import sys + + +RELEASE = os.environ['DIB_RELEASE'] +TMP_MOUNT_PATH = os.environ['TMP_MOUNT_PATH'] +TMP_HOOKS_PATH = os.environ['TMP_HOOKS_PATH'] + +DEVSTACK = os.path.join(TMP_MOUNT_PATH, 'opt/git/openstack-dev/devstack') +CACHEDIR = os.path.join(TMP_MOUNT_PATH, 'tmp') +IMAGES=os.path.join(TMP_HOOKS_PATH, 'source-repository-images') + + +def run_local(cmd, status=False, cwd='.', env={}): + print "Running:", cmd + newenv = os.environ + newenv.update(env) + p = subprocess.Popen(cmd, stdout=subprocess.PIPE, cwd=cwd, + stderr=subprocess.STDOUT, env=newenv) + (out, nothing) = p.communicate() + if status: + return (p.returncode, out.strip()) + return out.strip() + + +def git_branches(): + branches = [] + for branch in run_local(['git', 'branch', '-a'], cwd=DEVSTACK).split("\n"): + branch = branch.strip() + if not branch.startswith('remotes/origin'): + continue + branches.append(branch) + return branches + + +def tokenize(fn, tokens, distribution, comment=None): + for line in open(fn): + if 'dist:' in line and ('dist:%s' % distribution not in line): + continue + if 'qpid' in line: + continue # TODO: explain why this is here + if comment and comment in line: + line = line[:line.rfind(comment)] + line = line.strip() + if line and line not in tokens: + tokens.append(line) + + +def _legacy_find_images(basedir): + """Divine what images we should use based on parsing stackrc.""" + images = [] + for line in open(os.path.join(basedir, 'stackrc')): + line = line.strip() + if line.startswith('IMAGE_URLS'): + if '#' in line: + line = line[:line.rfind('#')] + if line.endswith(';;'): + line = line[:-2] + line = line.split('=', 1)[1].strip() + if line.startswith('${IMAGE_URLS:-'): + line = line[len('${IMAGE_URLS:-'):] + if line.endswith('}'): + line = line[:-1] + if not line: + continue + if line[0] == line[-1] == '"': + line = line[1:-1] + # Add image to the list to be downloaded, but + # skip downloading giant vmware images + images += [x.strip() for x in line.split(',') + if not x.strip().endswith('vmdk')] + return images + + +def _find_images(basedir): + images = [] + try: + image_tool = os.path.join(basedir, 'tools', 'image_list.sh') + if os.path.exists(image_tool): + images = subprocess.check_output(image_tool).split('\n') + except subprocess.CalledProcessError as ce: + print "image_list.sh failed" + print "Exit: %s, Output: %s" % (ce.returncode, ce.output) + # reset images so we'll fall back + images = [] + return images + + +def local_prep(distribution): + branches = [] + for branch in git_branches(): + # Ignore branches of the form 'somestring -> someotherstring' + # as this denotes a symbolic reference and the entire string + # as is cannot be checked out. We can do this safely as the + # reference will refer to one of the other branches returned + # by git_branches. + if ' -> ' in branch: + continue + branch_data = {'name': branch} + print 'Branch: ', branch + run_local(['git', 'checkout', branch], cwd=DEVSTACK) + run_local(['git', 'pull', '--ff-only', 'origin'], cwd=DEVSTACK) + + if os.path.exists(os.path.join(TMP_MOUNT_PATH, 'usr/bin/apt-get')): + debs = [] + debdir = os.path.join(DEVSTACK, 'files', 'apts') + for fn in os.listdir(debdir): + fn = os.path.join(debdir, fn) + tokenize(fn, debs, distribution, comment='#') + branch_data['debs'] = debs + + if os.path.exists(os.path.join(TMP_MOUNT_PATH, 'usr/bin/rpm')): + rpms = [] + rpmdir = os.path.join(DEVSTACK, 'files', 'rpms') + for fn in os.listdir(rpmdir): + fn = os.path.join(rpmdir, fn) + tokenize(fn, rpms, distribution, comment='#') + branch_data['rpms'] = rpms + + images = _find_images(DEVSTACK) + if not images: + images = _legacy_find_images(DEVSTACK) + + branch_data['images'] = images + branches.append(branch_data) + return branches + + +def main(): + + branches = local_prep(RELEASE) + + with open(os.path.join(CACHEDIR, 'pkgs-to-install'), 'w') as pkgs: + for branch_data in branches: + if branch_data.get('debs'): + pkgs.write(" ".join(branch_data['debs']) + "\n") + elif branch_data.get('rpms'): + pkgs.write(" ".join(branch_data['rpms']) + "\n") + else: + sys.exit('No supported package data found.') + + image_filenames = [] + line_template = "%(name)s file %(location)s %(url)s\n" + with open(IMAGES, 'w') as images_list: + for branch_data in branches: + for url in branch_data['images']: + fname = url.split('/')[-1].strip() + if fname == '': + continue + # TODO: There is an image in devstack that starts with + # $HEAT_FETCHED_TEST_IMAGE + if fname.startswith('$'): + continue + if fname in image_filenames: + continue + image_filenames.append(fname) + args = dict( + name=fname, + location=os.path.join('/home/jenkins/cache/files', fname), + url=url) + + images_list.write(line_template % args) + + +if __name__ == '__main__': + main() diff --git a/modules/openstack_project/files/nodepool/elements/cache-devstack/extra-data.d/60-rm-early-source-repo b/modules/openstack_project/files/nodepool/elements/cache-devstack/extra-data.d/60-rm-early-source-repo new file mode 100755 index 0000000000..c757bc3c80 --- /dev/null +++ b/modules/openstack_project/files/nodepool/elements/cache-devstack/extra-data.d/60-rm-early-source-repo @@ -0,0 +1,25 @@ +#!/bin/bash + +# Copyright (c) 2014 Hewlett-Packard Development Company, L.P. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +# implied. +# +# See the License for the specific language governing permissions and +# limitations under the License. + +# Remove the copy of devstack that we cloned in for getting devstack +# cache data so that the later cloning of all of the OpenStack repos +# won't bomb out + +set -eu + +sudo rm -rf $TMP_MOUNT_PATH/opt/git/openstack-dev/devstack diff --git a/modules/openstack_project/files/nodepool/elements/cache-devstack/install.d/50-download-pkgs b/modules/openstack_project/files/nodepool/elements/cache-devstack/install.d/50-download-pkgs new file mode 100644 index 0000000000..b80311d434 --- /dev/null +++ b/modules/openstack_project/files/nodepool/elements/cache-devstack/install.d/50-download-pkgs @@ -0,0 +1,30 @@ +#!/bin/bash +# Copyright (C) 2011-2013 OpenStack Foundation +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +# implied. +# +# See the License for the specific language governing permissions and +# limitations under the License. + +set -e + +while read line ; do + + if [ -f /usr/bin/apt-get ] ; then + apt-get -y -d install $line + else + yum install -y --downloadonly $line + fi + +done < ~/tmp/pkgs-to-install + +rm /tmp/pkgs-to-install diff --git a/modules/openstack_project/files/nodepool/elements/node-devstack/README.rst b/modules/openstack_project/files/nodepool/elements/node-devstack/README.rst new file mode 100644 index 0000000000..455c31e147 --- /dev/null +++ b/modules/openstack_project/files/nodepool/elements/node-devstack/README.rst @@ -0,0 +1 @@ +Prepare a node to be a devstack slave diff --git a/modules/openstack_project/files/nodepool/elements/node-devstack/element-deps b/modules/openstack_project/files/nodepool/elements/node-devstack/element-deps new file mode 100644 index 0000000000..4a446a2f00 --- /dev/null +++ b/modules/openstack_project/files/nodepool/elements/node-devstack/element-deps @@ -0,0 +1,2 @@ +puppet +cache-devstack diff --git a/modules/openstack_project/files/nodepool/elements/node-devstack/install.d/20-prepare-node b/modules/openstack_project/files/nodepool/elements/node-devstack/install.d/20-prepare-node new file mode 100755 index 0000000000..ebf2f32cb2 --- /dev/null +++ b/modules/openstack_project/files/nodepool/elements/node-devstack/install.d/20-prepare-node @@ -0,0 +1,28 @@ +#!/bin/bash +# Copyright (C) 2011-2013 OpenStack Foundation +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +# implied. +# +# See the License for the specific language governing permissions and +# limitations under the License. + +set -e + +export SUDO='true' +export THIN='true' + +prepare-node + +mkdir -p ~jenkins/cache/files +mkdir -p ~jenkins/cache/pip + +install-packages build-essential python-dev diff --git a/modules/openstack_project/files/nodepool/elements/nodepool-base/README.rst b/modules/openstack_project/files/nodepool/elements/nodepool-base/README.rst new file mode 100644 index 0000000000..ec1436dded --- /dev/null +++ b/modules/openstack_project/files/nodepool/elements/nodepool-base/README.rst @@ -0,0 +1 @@ +Tasks to deal with image metadata and other nodepool cloud specific tweaks. diff --git a/modules/openstack_project/files/nodepool/elements/nodepool-base/exta-data.d/50-copy-nodepool-scripts b/modules/openstack_project/files/nodepool/elements/nodepool-base/exta-data.d/50-copy-nodepool-scripts new file mode 100755 index 0000000000..e690500834 --- /dev/null +++ b/modules/openstack_project/files/nodepool/elements/nodepool-base/exta-data.d/50-copy-nodepool-scripts @@ -0,0 +1,24 @@ +#!/bin/bash + +# Copyright (c) 2014 Hewlett-Packard Development Company, L.P. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +# implied. +# +# See the License for the specific language governing permissions and +# limitations under the License. + +# Copy the nodepools scripts into the image + +set -eu + +sudo cp -a $NODEPOOL_SCRIPTDIR $TMP_MOUNT_PATH/opt/nodepool-scripts +sudo chmod -R a+rx $TMP_MOUNT_PATH/opt/nodepool-scripts diff --git a/modules/openstack_project/files/nodepool/elements/nodepool-base/finalise.d/99-unbound b/modules/openstack_project/files/nodepool/elements/nodepool-base/finalise.d/99-unbound new file mode 100755 index 0000000000..ddf28346f1 --- /dev/null +++ b/modules/openstack_project/files/nodepool/elements/nodepool-base/finalise.d/99-unbound @@ -0,0 +1,54 @@ +#!/bin/bash +# Copyright (C) 2011-2013 OpenStack Foundation +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +# implied. +# +# See the License for the specific language governing permissions and +# limitations under the License. + +set -e + +# The puppet modules should install unbound. +dd of=/tmp/forwarding.conf < /etc/resolv.conf + +exit 0 +EOF + +echo 'include: /etc/unbound/forwarding.conf' >> /etc/unbound/unbound.conf diff --git a/modules/openstack_project/files/nodepool/elements/nodepool-base/install.d/05-record-details b/modules/openstack_project/files/nodepool/elements/nodepool-base/install.d/05-record-details new file mode 100755 index 0000000000..534399c29d --- /dev/null +++ b/modules/openstack_project/files/nodepool/elements/nodepool-base/install.d/05-record-details @@ -0,0 +1,18 @@ +#!/bin/bash -xe +# Copyright (C) 2014 Hewlett-Packard Development Company, L.P. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +# implied. +# +# See the License for the specific language governing permissions and +# limitations under the License. + +echo $DIB_IMAGE_NAME > /etc/image-hostname.txt diff --git a/modules/openstack_project/files/nodepool/elements/nodepool-base/install.d/50-restrict-memory b/modules/openstack_project/files/nodepool/elements/nodepool-base/install.d/50-restrict-memory new file mode 100755 index 0000000000..5dec858a53 --- /dev/null +++ b/modules/openstack_project/files/nodepool/elements/nodepool-base/install.d/50-restrict-memory @@ -0,0 +1,31 @@ +#!/bin/bash -xe +# Copyright (C) 2014 Hewlett-Packard Development Company, L.P. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +# implied. +# +# See the License for the specific language governing permissions and +# limitations under the License. + +# This is done in 50 because the vm element has, at 51, an bunch of code +# for properly handling grub. All we need to do here is get our values +# in to the files. + +set -e + +# Limit all test slaves to 8GB of memory so that larger flavors with more +# cpu resources can be used without the risk of becoming dependent on more +# memory. +if [ -f /etc/default/grub ] ; then + sed -i -e 's/^GRUB_TIMEOUT=[0-9]\+/GRUB_TIMEOUT=0/' -e 's/#\?GRUB_CMDLINE_LINUX=""/GRUB_CMDLINE_LINUX="mem=8G"/g' /etc/default/grub +elif [ -f /boot/grub/grub.conf ] ; then + sed -i -e 's/^timeout=[0-9]\+/timeout=0/' -e 's/\(^\s\+kernel.*\)/\1 mem=8G/' /boot/grub/grub.conf +fi diff --git a/modules/openstack_project/files/nodepool/elements/nodepool-base/install.d/99-install-zuul b/modules/openstack_project/files/nodepool/elements/nodepool-base/install.d/99-install-zuul new file mode 100755 index 0000000000..5b94945fbf --- /dev/null +++ b/modules/openstack_project/files/nodepool/elements/nodepool-base/install.d/99-install-zuul @@ -0,0 +1,22 @@ +#!/bin/bash -xe +# Copyright (C) 2014 Hewlett-Packard Development Company, L.P. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +# implied. +# +# Install Zuul into a virtualenv +# This is in /usr instead of /usr/local due to this bug on precise: +# https://bugs.launchpad.net/ubuntu/+source/python2.7/+bug/839588 + +git clone /opt/git/openstack-infra/zuul /tmp/zuul +sudo virtualenv /usr/zuul-env +sudo /usr/zuul-env/bin/pip install /tmp/zuul +sudo rm -fr /tmp/zuul diff --git a/modules/openstack_project/files/nodepool/elements/openstack-repos/README.rst b/modules/openstack_project/files/nodepool/elements/openstack-repos/README.rst new file mode 100644 index 0000000000..82bcfde8cf --- /dev/null +++ b/modules/openstack_project/files/nodepool/elements/openstack-repos/README.rst @@ -0,0 +1 @@ +Download all repos and packages that a devstack run might need. diff --git a/modules/openstack_project/files/nodepool/elements/openstack-repos/element-deps b/modules/openstack_project/files/nodepool/elements/openstack-repos/element-deps new file mode 100644 index 0000000000..b56002654e --- /dev/null +++ b/modules/openstack_project/files/nodepool/elements/openstack-repos/element-deps @@ -0,0 +1,3 @@ +cache-url +puppet +source-repositories diff --git a/modules/openstack_project/files/nodepool/elements/openstack-repos/extra-data.d/50-create-repo-list b/modules/openstack_project/files/nodepool/elements/openstack-repos/extra-data.d/50-create-repo-list new file mode 100755 index 0000000000..ef84ed6174 --- /dev/null +++ b/modules/openstack_project/files/nodepool/elements/openstack-repos/extra-data.d/50-create-repo-list @@ -0,0 +1,55 @@ +#!/usr/bin/env python + +# Copyright (C) 2011-2013 OpenStack Foundation +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +# implied. +# +# See the License for the specific language governing permissions and +# limitations under the License. + +import os +import urllib2 +import yaml + +URL = ('https://git.openstack.org/cgit/openstack-infra/config/plain/' + 'modules/openstack_project/files/review.projects.yaml') + +TMP_HOOKS_PATH=os.environ['TMP_HOOKS_PATH'] +PROJECTS_REPOS=os.path.join(TMP_HOOKS_PATH, 'source-repository-projects-yaml') +GIT_BASE=os.environ.get('GIT_BASE', 'git://git.openstack.org') + + +def main(): + projects = [f['project'] for f in yaml.load(urllib2.urlopen(URL))] + with open(PROJECTS_REPOS, 'w') as projects_list: + for project in projects: + args = dict( + name=os.path.basename(project), + location=os.path.join('/opt/git', project), + url='%s/%s.git' % (GIT_BASE, project)) + + projects_list.write("%(name)s git %(location)s %(url)s\n" % args) + # Clone openstack-infra/config again so that we can use it to build + # the image without interferring with the slave repo cache. + project = 'openstack-infra/config' + args = dict( + name='config_tmp', + location=os.path.join('/opt/build_git', project), + url=os.environ.get('CONFIG_SOURCE', + '%s/%s.git' % (GIT_BASE, project)), + ref=os.environ.get('CONFIG_REF', 'master')) + projects_list.write( + "%(name)s git %(location)s %(url)s %(ref)s\n" % args) + + +if __name__ == '__main__': + main() diff --git a/modules/openstack_project/files/nodepool/elements/openstack-repos/install.d/95-chown-jenkins b/modules/openstack_project/files/nodepool/elements/openstack-repos/install.d/95-chown-jenkins new file mode 100755 index 0000000000..5037b591a2 --- /dev/null +++ b/modules/openstack_project/files/nodepool/elements/openstack-repos/install.d/95-chown-jenkins @@ -0,0 +1,20 @@ +#!/bin/bash +# Copyright (C) 2011-2013 OpenStack Foundation +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +# implied. +# +# See the License for the specific language governing permissions and +# limitations under the License. + +set -e + +chown -R jenkins:jenkins /home/jenkins diff --git a/modules/openstack_project/files/nodepool/elements/puppet/README.rst b/modules/openstack_project/files/nodepool/elements/puppet/README.rst new file mode 100644 index 0000000000..071da1712b --- /dev/null +++ b/modules/openstack_project/files/nodepool/elements/puppet/README.rst @@ -0,0 +1 @@ +Bootstrap puppet on a node diff --git a/modules/openstack_project/files/nodepool/elements/puppet/bin/prepare-node b/modules/openstack_project/files/nodepool/elements/puppet/bin/prepare-node new file mode 100644 index 0000000000..3ad72c35e9 --- /dev/null +++ b/modules/openstack_project/files/nodepool/elements/puppet/bin/prepare-node @@ -0,0 +1,55 @@ +#!/bin/bash +# Copyright (C) 2011-2013 OpenStack Foundation +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +# implied. +# +# See the License for the specific language governing permissions and +# limitations under the License. + +set -e + +SUDO=${SUDO:-true} +THIN=${THIN:-true} +PYTHON3=${PYTHON3:-false} +PYPY=${PYPY:-false} +ALL_MYSQL_PRIVS=${ALL_MYSQL_PRIVS:-false} + +export FACTER_in_chroot=true +cat >/tmp/local.pp < '/bin/true', + stop => '/bin/true', + status => '/bin/true', + restart => '/bin/true' +} +class {'openstack_project::single_use_slave': + sudo => $SUDO, + thin => $THIN, + python3 => $PYTHON3, + include_pypy => $PYPY, + all_mysql_privs => $ALL_MYSQL_PRIVS, + install_resolv_conf => false +} +EOF +# Puppet doesn't return nonzero if some things fail by default. +# Use detailed exit codes to get that info and determine whether +# the return code indicates failure. +set +e +puppet apply --detailed-exitcodes --modulepath=/opt/build_git/openstack-infra/config/modules:/etc/puppet/modules /tmp/local.pp +PUPPET_RETURN=$? +if [ "$PUPPET_RETURN" -eq 4 ] || [ "$PUPPET_RETURN" -eq 6 ] ; then + exit $PUPPET_RETURN +fi +set -e + +# Make sure resolv.conf settings don't break dib +echo 'nameserver 8.8.8.8'> /etc/resolv.conf diff --git a/modules/openstack_project/files/nodepool/elements/puppet/element-deps b/modules/openstack_project/files/nodepool/elements/puppet/element-deps new file mode 100644 index 0000000000..95746ba5ab --- /dev/null +++ b/modules/openstack_project/files/nodepool/elements/puppet/element-deps @@ -0,0 +1,3 @@ +source-repositories +cache-url +openstack-repos diff --git a/modules/openstack_project/files/nodepool/elements/puppet/install.d/05-puppet b/modules/openstack_project/files/nodepool/elements/puppet/install.d/05-puppet new file mode 100755 index 0000000000..faa80370e6 --- /dev/null +++ b/modules/openstack_project/files/nodepool/elements/puppet/install.d/05-puppet @@ -0,0 +1,27 @@ +#!/bin/bash +# Copyright (C) 2011-2013 OpenStack Foundation +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +# implied. +# +# See the License for the specific language governing permissions and +# limitations under the License. + +set -e + +# Unset the download cache for this invocation to prevent bleed from build host +unset PIP_DOWNLOAD_CACHE +export PUPPET_VERSION=${PUPPET_VERSION:-'2'} + +/bin/bash /opt/git/openstack-infra/config/install_puppet.sh +/bin/bash /opt/git/openstack-infra/config/install_modules.sh + +install -m 0755 -o root -g root $(dirname $0)/../bin/prepare-node /usr/local/bin diff --git a/modules/openstack_project/files/nodepool/elements/puppet/install.d/95-clean-repos b/modules/openstack_project/files/nodepool/elements/puppet/install.d/95-clean-repos new file mode 100755 index 0000000000..613e5d1866 --- /dev/null +++ b/modules/openstack_project/files/nodepool/elements/puppet/install.d/95-clean-repos @@ -0,0 +1,37 @@ +#!/bin/bash +# Copyright (C) 2011-2013 OpenStack Foundation +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +# implied. +# +# See the License for the specific language governing permissions and +# limitations under the License. + +# Remove additional sources used to install puppet or special version of pypi. +# We do this because leaving these sources in place causes every test that +# does an apt-get update to hit those servers which may not have the uptime +# of our local mirrors. + +set -e + +OS_FAMILY=$(facter osfamily) +if [ "$OS_FAMILY" == "Debian" ] ; then + rm -f /etc/apt/sources.list.d/* + apt-get update +elif [ "$OS_FAMILY" == "RedHat" ] ; then + # Can't delete * in yum.repos.d since all of the repos are listed there. + # Be specific instead. + if [ -f /etc/yum.repos.d/puppetlabs.repo ] ; then + sudo rm -f /etc/yum.repos.d/puppetlabs.repo + fi +fi + +rm /usr/local/bin/prepare-node diff --git a/modules/openstack_project/files/nodepool/elements/puppet/install.d/96-clean-cron b/modules/openstack_project/files/nodepool/elements/puppet/install.d/96-clean-cron new file mode 100755 index 0000000000..4c8e54f721 --- /dev/null +++ b/modules/openstack_project/files/nodepool/elements/puppet/install.d/96-clean-cron @@ -0,0 +1,23 @@ +#!/bin/bash +# Copyright (C) 2011-2013 OpenStack Foundation +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +# implied. +# +# See the License for the specific language governing permissions and +# limitations under the License. + +# Remove cron jobs +# We create fresh servers for these hosts, and they are used once. They don't +# need to do things like update the locatedb or the mandb or rotate logs +# or really any of those things. We only want code running here that we want +# here. +rm -f /etc/cron.{monthly,weekly,daily,hourly,d}/* diff --git a/modules/openstack_project/files/nodepool/elements/puppet/pre-install.d/10-preseed b/modules/openstack_project/files/nodepool/elements/puppet/pre-install.d/10-preseed new file mode 100755 index 0000000000..4c5dbef515 --- /dev/null +++ b/modules/openstack_project/files/nodepool/elements/puppet/pre-install.d/10-preseed @@ -0,0 +1,39 @@ +#!/bin/bash +# Copyright (C) 2011-2013 OpenStack Foundation +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +# implied. +# +# See the License for the specific language governing permissions and +# limitations under the License. + +# workaround a broken maintainer script in iptables-persistent that +# assumes it can execute things during postinst. Since we're installing +# in a chroot, we're actually not running the right kernel for its assumptions +# to work. + +set -e + +# If lsb_release is missing, just do nothing. +DISTRO=`lsb_release -si` || true + +case $DISTRO in + 'Ubuntu'|'Debian') + PRESEED=`mktemp` + cat > $PRESEED <