opendev/system-config
Code Issues Proposed changes

Merge "Add support for disk-image-builder in nodepool"

Browse Source
This commit is contained in:
Jenkins 2014-09-19 21:35:55 +00:00 committed by Gerrit Code Review
commit 95080d8703
35 changed files with 1076 additions and 0 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

54
modules/openstack_project/files/nodepool/elements/README.rst Normal file
View File

@ -0,0 +1,54 @@
Using diskimage-builder to build devstack-gate nodes
====================================================
In addition to being able to just download and consume images that are the
same as what run devstack-gate, it's easy to make your own for local dev or
testing - or just for fun.
Install diskimage-builder
-------------------------
Install the dependencies:
::
sudo apt-get install kpartx qemu-utils curl
Install diskimage-builder:
::
sudo pip install diskimage-builder
Build an image
--------------
Building an image is simple, we have a script!
::
bash tools/build-image.sh
You should be left with a file called devstack-gate-precise.qcow2.
Mounting the image
------------------
If you would like to examine the contents of the image, you can mount it on
a loopback device using qemu-nbd.
::
sudo apt-get install qemu-utils
sudo modprobe nbd max_part=16
sudo mkdir -p /tmp/newimage
sudo qemu-nbd -c /dev/nbd1 devstack-gate-precise.qcow2
sudo mount /dev/nbd1 /tmp/newimage
Other things
------------
It's a qcow2 image, so you can do tons of things with it. You can upload it
to glance, you can boot it using kvm, and you can even copy it to a cloud
server, replace the contents of the server with it and kexec the new kernel.

1
modules/openstack_project/files/nodepool/elements/cache-devstack/README.rst Normal file
View File

@ -0,0 +1 @@
Pre-cache all of the things devstack might need

1
modules/openstack_project/files/nodepool/elements/cache-devstack/element-deps Normal file
View File

@ -0,0 +1 @@
openstack-repos

209
modules/openstack_project/files/nodepool/elements/cache-devstack/extra-data.d/50-early-source-repo Executable file
View File

@ -0,0 +1,209 @@
#!/bin/bash
# Copyright (c) 2012-2014 Hewlett-Packard Development Company, L.P.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
# implied.
#
# See the License for the specific language governing permissions and
# limitations under the License.
# We need a copy of the devstack repo so that we can read it so that
# we can generate the list of repos that we need to get. We'd like to
# use the normal repo fetching and caching routines for that, so use
# a modified version.
# The bulk of this code is copied from
# elements/source-repositories/extra-data.d/98-source-repositories
# Most of it should be removed when we can source and call the
# functions directly
set -eu
# If the old cache exists, move it to the new name
function make_new_cache(){
local OLD_CACHE_BASE=$1
local CACHE_BASE=$2
# If the old cache name exists, move it to the new cache name
if [ -e "$OLD_CACHE_BASE" ] ; then
if [ ! -e "$CACHE_BASE" ] ; then
mv -n $OLD_CACHE_BASE $CACHE_BASE
else
echo "Not replacing new cache location with old cache"
fi
fi
}
# Gets repositories or individual files listed in the a repository file
# and places them in the specified destination path.
# The format of the repository file is one or more lines matching
# <name> <type> <destination> <location> [<ref>]
function get_repos_for_element(){
local REPO_SOURCES=$1
local CACHE_URL=$TMP_HOOKS_PATH/bin/cache-url
local REGEX="^([^ ]+) (git|tar|file|package) ?(/[^ ]+)? ?([^ ]+)? ?([^ ]*)$"
while read line; do
# expand variables
line=$(eval echo $line)
# ignore blank lines and lines beginning in '#'
[[ "$line" == \#* ]] || [[ -z "$line" ]] && continue
if [[ "$line" =~ $REGEX ]] ; then
local REPONAME=${BASH_REMATCH[1]}
local REPOTYPE=${BASH_REMATCH[2]}
local REPOPATH=${BASH_REMATCH[3]}
local REPOLOCATION=${BASH_REMATCH[4]}
local REPO_ORIG_LOCATION=$REPOLOCATION
local REPOREF=${BASH_REMATCH[5]:-master}
local REPO_DEST=$TMP_MOUNT_PATH$REPOPATH
local REPO_SUB_DIRECTORY=$(dirname $REPO_DEST)
# REPOTYPE can be overridden with DIB_REPOTYPE_{name}
local REPOTYPE_OVERRIDE=DIB_REPOTYPE_${REPONAME//[^A-Za-z0-9]/_}
REPOTYPE=${!REPOTYPE_OVERRIDE:-$REPOTYPE}
# REPOLOCATION can be overridden with DIB_REPOLOCATION_{name}
local REPOLOCATION_OVERRIDE=DIB_REPOLOCATION_${REPONAME//[^A-Za-z0-9]/_}
REPOLOCATION=${!REPOLOCATION_OVERRIDE:-$REPOLOCATION}
# REPOREF can be overridden with DIB_REPOREF_{name}
local REPOREF_OVERRIDE=DIB_REPOREF_${REPONAME//[^A-Za-z0-9]/_}
REPOREF=${!REPOREF_OVERRIDE:-$REPOREF}
# Determine a unique cache path for this repo
CACHE_NAME=$(echo "${REPOTYPE}_${REPOLOCATION}" | sha1sum | awk '{ print $1 }' )
OLD_CACHE_PATH=${CACHE_BASE}/${CACHE_NAME}
# Add the repo name to the sha1sum for readability
CACHE_NAME=${REPONAME//[^A-Za-z0-9]/_}_${CACHE_NAME}
CACHE_PATH=${CACHE_BASE}/$CACHE_NAME
make_new_cache $OLD_CACHE_PATH $CACHE_PATH
# Return if install type is not source
local INSTALL_TYPE_VAR=DIB_INSTALLTYPE_${REPONAME//[^A-Za-z0-9]/_}
local INSTALL_TYPE=${!INSTALL_TYPE_VAR:-source}
if [ ! $INSTALL_TYPE = "source" ]; then
echo "$REPONAME install type not set to source"
continue
fi
case $REPOTYPE in
git)
if [ -z "${!REPOLOCATION_OVERRIDE:-""}" -a -n "${DIB_GITREPOBASE:-""}" ] ; then
# Transform the current repo base to the new one
local NEW_REPOLOCATION=$(echo $REPOLOCATION |\
sed "s,^[^:]\+://[^/]\+/\(~[^/]\+\)\?\(.*\)$,${DIB_GITREPOBASE}\2,g")
echo "Transformed ${REPOLOCATION} to ${NEW_REPOLOCATION}"
REPOLOCATION=$NEW_REPOLOCATION
# Also update the cache location
CACHE_NAME=$(echo "${REPOTYPE}_${REPOLOCATION}" | sha1sum | awk '{ print $1 }' )
CACHE_PATH=~/.cache/image-create/repository-sources/$CACHE_NAME
fi
sudo mkdir -p $REPO_SUB_DIRECTORY
if [ ! -e "$CACHE_PATH" ] ; then
echo "Caching $REPONAME from $REPOLOCATION in $CACHE_PATH"
git clone $REPOLOCATION $CACHE_PATH.tmp
mv ${CACHE_PATH}{.tmp,}
fi
HAS_REF=$(git --git-dir=$CACHE_PATH/.git name-rev $REPOREF 2>/dev/null || true)
if [ -z "$DIB_OFFLINE" -o -z "$HAS_REF" ] ; then
echo "Updating cache of $REPOLOCATION in $CACHE_PATH with ref $REPOREF"
git --git-dir=$CACHE_PATH/.git fetch --update-head-ok $REPOLOCATION +refs/heads/*:refs/heads/*
fi
echo "Cloning from $REPONAME cache and applying ref $REPOREF"
sudo git clone $CACHE_PATH $REPO_DEST
pushd $REPO_DEST
sudo git fetch $CACHE_PATH $REPOREF
sudo git reset --hard FETCH_HEAD
# Get the reference in use
git_ref=$(git rev-parse FETCH_HEAD)
popd
# Write the reference being used into the source-repositories manifest
echo "$REPONAME git $REPOPATH $REPOLOCATION $git_ref" >> $GIT_MANIFEST
;;
tar)
# The top level directory of the tarball mightn't have a fixed name i.e.
# it could contain version numbers etc... so we write it to a tmpdir
# the then move the contents into the directory we want it in, this does
# assume the tarball only contains a single top level directory
local tmpdir=$(mktemp --tmpdir=$TMP_MOUNT_PATH/tmp -d)
if [ -n "$CACHE_PATH" ] ; then
echo "Caching $REPONAME tarball from $REPOLOCATION in $CACHE_PATH"
if [ ! -f "$CACHE_PATH" -o -z "$DIB_OFFLINE" ] ; then
$CACHE_URL $REPOLOCATION $CACHE_PATH
fi
tar -C $tmpdir -xzf $CACHE_PATH
else
echo "Fetching $REPONAME tarball from $REPOLOCATION"
curl $REPOLOCATION | tar -C $tmpdir -xzf -
fi
sudo mkdir -p $REPO_DEST
sudo mv $tmpdir/*/* $REPO_DEST
rm -rf $tmpdir
;;
file)
sudo mkdir -p $REPO_SUB_DIRECTORY
if [ -n "$CACHE_PATH" ] ; then
echo "Caching $REPONAME file from $REPOLOCATION in $CACHE_PATH"
if [ ! -f "$CACHE_PATH" -o -z "$DIB_OFFLINE" ] ; then
$CACHE_URL $REPOLOCATION $CACHE_PATH
fi
sudo cp $CACHE_PATH $REPO_DEST
else
echo "Fetching $REPONAME file from $REPOLOCATION"
sudo curl $REPOLOCATION -o $REPO_DEST
fi
;;
*)
echo "Unsupported repository type: $REPOTYPE"
return 1
;;
esac
# Capture the in-instance repository path for later review / other
# elements (like a pypi dependency cache).
echo "$REPOPATH" | sudo dd of=$TMP_MOUNT_PATH/etc/dib-source-repositories oflag=append conv=notrunc
else
echo "Couldn't parse '$line' as a source repository"
return 1
fi
done < $REPO_SOURCES
}
CACHE_BASE=~/.cache/image-create/source-repositories
OLD_CACHE_BASE=~/.cache/image-create/repository-sources
make_new_cache $OLD_CACHE_BASE $CACHE_BASE
mkdir -p $CACHE_BASE
# Use the IMAGE_NAME from the calling script, and make it unique with the temporary path
GIT_MANIFEST_NAME=dib-manifest-git-$(basename ${IMAGE_NAME})
GIT_MANIFEST_CACHE_NAME=${GIT_MANIFEST_NAME}_$(dirname ${TMP_MOUNT_PATH##*.})
GIT_MANIFEST=$CACHE_BASE/${GIT_MANIFEST_CACHE_NAME}
rm -f $GIT_MANIFEST
# Get source repositories for the target
echo "devstack git /opt/git/openstack-dev/devstack git://git.openstack.org/openstack-dev/devstack.git" > $TMP_HOOKS_PATH/early-source-repository-config
for _SOURCEREPO in $(find $TMP_HOOKS_PATH -maxdepth 1 -name "early-source-repository-*" -not -name '*~'); do
get_repos_for_element $_SOURCEREPO
done
# Copy the manifest into the image if it exists (there may be no git repositories used)
if [ -e "$GIT_MANIFEST" ] ; then
sudo cp $GIT_MANIFEST $TMP_MOUNT_PATH/${DIB_MANIFEST_IMAGE_DIR}/$GIT_MANIFEST_NAME
fi

185
modules/openstack_project/files/nodepool/elements/cache-devstack/extra-data.d/55-cache-devstack-repos Executable file
View File

@ -0,0 +1,185 @@
#!/usr/bin/env python
# Copyright (C) 2011-2013 OpenStack Foundation
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
# implied.
#
# See the License for the specific language governing permissions and
# limitations under the License.
import os
import subprocess
import sys
RELEASE = os.environ['DIB_RELEASE']
TMP_MOUNT_PATH = os.environ['TMP_MOUNT_PATH']
TMP_HOOKS_PATH = os.environ['TMP_HOOKS_PATH']
DEVSTACK = os.path.join(TMP_MOUNT_PATH, 'opt/git/openstack-dev/devstack')
CACHEDIR = os.path.join(TMP_MOUNT_PATH, 'tmp')
IMAGES=os.path.join(TMP_HOOKS_PATH, 'source-repository-images')
def run_local(cmd, status=False, cwd='.', env={}):
print "Running:", cmd
newenv = os.environ
newenv.update(env)
p = subprocess.Popen(cmd, stdout=subprocess.PIPE, cwd=cwd,
stderr=subprocess.STDOUT, env=newenv)
(out, nothing) = p.communicate()
if status:
return (p.returncode, out.strip())
return out.strip()
def git_branches():
branches = []
for branch in run_local(['git', 'branch', '-a'], cwd=DEVSTACK).split("\n"):
branch = branch.strip()
if not branch.startswith('remotes/origin'):
continue
branches.append(branch)
return branches
def tokenize(fn, tokens, distribution, comment=None):
for line in open(fn):
if 'dist:' in line and ('dist:%s' % distribution not in line):
continue
if 'qpid' in line:
continue # TODO: explain why this is here
if comment and comment in line:
line = line[:line.rfind(comment)]
line = line.strip()
if line and line not in tokens:
tokens.append(line)
def _legacy_find_images(basedir):
"""Divine what images we should use based on parsing stackrc."""
images = []
for line in open(os.path.join(basedir, 'stackrc')):
line = line.strip()
if line.startswith('IMAGE_URLS'):
if '#' in line:
line = line[:line.rfind('#')]
if line.endswith(';;'):
line = line[:-2]
line = line.split('=', 1)[1].strip()
if line.startswith('${IMAGE_URLS:-'):
line = line[len('${IMAGE_URLS:-'):]
if line.endswith('}'):
line = line[:-1]
if not line:
continue
if line[0] == line[-1] == '"':
line = line[1:-1]
# Add image to the list to be downloaded, but
# skip downloading giant vmware images
images += [x.strip() for x in line.split(',')
if not x.strip().endswith('vmdk')]
return images
def _find_images(basedir):
images = []
try:
image_tool = os.path.join(basedir, 'tools', 'image_list.sh')
if os.path.exists(image_tool):
images = subprocess.check_output(image_tool).split('\n')
except subprocess.CalledProcessError as ce:
print "image_list.sh failed"
print "Exit: %s, Output: %s" % (ce.returncode, ce.output)
# reset images so we'll fall back
images = []
return images
def local_prep(distribution):
branches = []
for branch in git_branches():
# Ignore branches of the form 'somestring -> someotherstring'
# as this denotes a symbolic reference and the entire string
# as is cannot be checked out. We can do this safely as the
# reference will refer to one of the other branches returned
# by git_branches.
if ' -> ' in branch:
continue
branch_data = {'name': branch}
print 'Branch: ', branch
run_local(['git', 'checkout', branch], cwd=DEVSTACK)
run_local(['git', 'pull', '--ff-only', 'origin'], cwd=DEVSTACK)
if os.path.exists(os.path.join(TMP_MOUNT_PATH, 'usr/bin/apt-get')):
debs = []
debdir = os.path.join(DEVSTACK, 'files', 'apts')
for fn in os.listdir(debdir):
fn = os.path.join(debdir, fn)
tokenize(fn, debs, distribution, comment='#')
branch_data['debs'] = debs
if os.path.exists(os.path.join(TMP_MOUNT_PATH, 'usr/bin/rpm')):
rpms = []
rpmdir = os.path.join(DEVSTACK, 'files', 'rpms')
for fn in os.listdir(rpmdir):
fn = os.path.join(rpmdir, fn)
tokenize(fn, rpms, distribution, comment='#')
branch_data['rpms'] = rpms
images = _find_images(DEVSTACK)
if not images:
images = _legacy_find_images(DEVSTACK)
branch_data['images'] = images
branches.append(branch_data)
return branches
def main():
branches = local_prep(RELEASE)
with open(os.path.join(CACHEDIR, 'pkgs-to-install'), 'w') as pkgs:
for branch_data in branches:
if branch_data.get('debs'):
pkgs.write(" ".join(branch_data['debs']) + "\n")
elif branch_data.get('rpms'):
pkgs.write(" ".join(branch_data['rpms']) + "\n")
else:
sys.exit('No supported package data found.')
image_filenames = []
line_template = "%(name)s file %(location)s %(url)s\n"
with open(IMAGES, 'w') as images_list:
for branch_data in branches:
for url in branch_data['images']:
fname = url.split('/')[-1].strip()
if fname == '':
continue
# TODO: There is an image in devstack that starts with
# $HEAT_FETCHED_TEST_IMAGE
if fname.startswith('$'):
continue
if fname in image_filenames:
continue
image_filenames.append(fname)
args = dict(
name=fname,
location=os.path.join('/home/jenkins/cache/files', fname),
url=url)
images_list.write(line_template % args)
if __name__ == '__main__':
main()

25
modules/openstack_project/files/nodepool/elements/cache-devstack/extra-data.d/60-rm-early-source-repo Executable file
View File

@ -0,0 +1,25 @@
#!/bin/bash
# Copyright (c) 2014 Hewlett-Packard Development Company, L.P.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
# implied.
#
# See the License for the specific language governing permissions and
# limitations under the License.
# Remove the copy of devstack that we cloned in for getting devstack
# cache data so that the later cloning of all of the OpenStack repos
# won't bomb out
set -eu
sudo rm -rf $TMP_MOUNT_PATH/opt/git/openstack-dev/devstack

30
modules/openstack_project/files/nodepool/elements/cache-devstack/install.d/50-download-pkgs Normal file
View File

@ -0,0 +1,30 @@
#!/bin/bash
# Copyright (C) 2011-2013 OpenStack Foundation
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
# implied.
#
# See the License for the specific language governing permissions and
# limitations under the License.
set -e
while read line ; do
if [ -f /usr/bin/apt-get ] ; then
apt-get -y -d install $line
else
yum install -y --downloadonly $line
fi
done < ~/tmp/pkgs-to-install
rm /tmp/pkgs-to-install

1
modules/openstack_project/files/nodepool/elements/node-devstack/README.rst Normal file
View File

@ -0,0 +1 @@
Prepare a node to be a devstack slave

2
modules/openstack_project/files/nodepool/elements/node-devstack/element-deps Normal file
View File

@ -0,0 +1,2 @@
puppet
cache-devstack

28
modules/openstack_project/files/nodepool/elements/node-devstack/install.d/20-prepare-node Executable file
View File

@ -0,0 +1,28 @@
#!/bin/bash
# Copyright (C) 2011-2013 OpenStack Foundation
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
# implied.
#
# See the License for the specific language governing permissions and
# limitations under the License.
set -e
export SUDO='true'
export THIN='true'
prepare-node
mkdir -p ~jenkins/cache/files
mkdir -p ~jenkins/cache/pip
install-packages build-essential python-dev

1
modules/openstack_project/files/nodepool/elements/nodepool-base/README.rst Normal file
View File

@ -0,0 +1 @@
Tasks to deal with image metadata and other nodepool cloud specific tweaks.

24
modules/openstack_project/files/nodepool/elements/nodepool-base/exta-data.d/50-copy-nodepool-scripts Executable file
View File

@ -0,0 +1,24 @@
#!/bin/bash
# Copyright (c) 2014 Hewlett-Packard Development Company, L.P.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
# implied.
#
# See the License for the specific language governing permissions and
# limitations under the License.
# Copy the nodepools scripts into the image
set -eu
sudo cp -a $NODEPOOL_SCRIPTDIR $TMP_MOUNT_PATH/opt/nodepool-scripts
sudo chmod -R a+rx $TMP_MOUNT_PATH/opt/nodepool-scripts

54
modules/openstack_project/files/nodepool/elements/nodepool-base/finalise.d/99-unbound Executable file
View File

@ -0,0 +1,54 @@
#!/bin/bash
# Copyright (C) 2011-2013 OpenStack Foundation
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
# implied.
#
# See the License for the specific language governing permissions and
# limitations under the License.
set -e
# The puppet modules should install unbound.
dd of=/tmp/forwarding.conf <<EOF
forward-zone:
name: "."
forward-addr: 8.8.8.8
EOF
mv /tmp/forwarding.conf /etc/unbound/
chown root:root /etc/unbound/forwarding.conf
chmod a+r /etc/unbound/forwarding.conf
# HPCloud has selinux enabled by default, Rackspace apparently not.
# Regardless, apply the correct context.
if [ -x /sbin/restorecon ] ; then
chcon system_u:object_r:named_conf_t:s0 /etc/unbound/forwarding.conf
fi
# Overwrite /etc/resolv.conf at boot
dd of=/etc/rc.local <<EOF
#!/bin/bash
set -o xtrace
# Some providers inject dynamic network config statically. Work around this
# for DNS nameservers. This is expected to fail on some nodes so remove -e.
set +e
sed -i -e 's/^\(DNS[0-9]*=[.0-9]\+\)/#\1/g' /etc/sysconfig/network-scripts/ifcfg-*
set -e
echo 'nameserver 127.0.0.1' > /etc/resolv.conf
exit 0
EOF
echo 'include: /etc/unbound/forwarding.conf' >> /etc/unbound/unbound.conf

18
modules/openstack_project/files/nodepool/elements/nodepool-base/install.d/05-record-details Executable file
View File

@ -0,0 +1,18 @@
#!/bin/bash -xe
# Copyright (C) 2014 Hewlett-Packard Development Company, L.P.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
# implied.
#
# See the License for the specific language governing permissions and
# limitations under the License.
echo $DIB_IMAGE_NAME > /etc/image-hostname.txt

31
modules/openstack_project/files/nodepool/elements/nodepool-base/install.d/50-restrict-memory Executable file
View File

@ -0,0 +1,31 @@
#!/bin/bash -xe
# Copyright (C) 2014 Hewlett-Packard Development Company, L.P.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
# implied.
#
# See the License for the specific language governing permissions and
# limitations under the License.
# This is done in 50 because the vm element has, at 51, an bunch of code
# for properly handling grub. All we need to do here is get our values
# in to the files.
set -e
# Limit all test slaves to 8GB of memory so that larger flavors with more
# cpu resources can be used without the risk of becoming dependent on more
# memory.
if [ -f /etc/default/grub ] ; then
sed -i -e 's/^GRUB_TIMEOUT=[0-9]\+/GRUB_TIMEOUT=0/' -e 's/#\?GRUB_CMDLINE_LINUX=""/GRUB_CMDLINE_LINUX="mem=8G"/g' /etc/default/grub
elif [ -f /boot/grub/grub.conf ] ; then
sed -i -e 's/^timeout=[0-9]\+/timeout=0/' -e 's/\(^\s\+kernel.*\)/\1 mem=8G/' /boot/grub/grub.conf
fi

22
modules/openstack_project/files/nodepool/elements/nodepool-base/install.d/99-install-zuul Executable file
View File

@ -0,0 +1,22 @@
#!/bin/bash -xe
# Copyright (C) 2014 Hewlett-Packard Development Company, L.P.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
# implied.
#
# Install Zuul into a virtualenv
# This is in /usr instead of /usr/local due to this bug on precise:
# https://bugs.launchpad.net/ubuntu/+source/python2.7/+bug/839588
git clone /opt/git/openstack-infra/zuul /tmp/zuul
sudo virtualenv /usr/zuul-env
sudo /usr/zuul-env/bin/pip install /tmp/zuul
sudo rm -fr /tmp/zuul

1
modules/openstack_project/files/nodepool/elements/openstack-repos/README.rst Normal file
View File

@ -0,0 +1 @@
Download all repos and packages that a devstack run might need.

3
modules/openstack_project/files/nodepool/elements/openstack-repos/element-deps Normal file
View File

@ -0,0 +1,3 @@
cache-url
puppet
source-repositories

55
modules/openstack_project/files/nodepool/elements/openstack-repos/extra-data.d/50-create-repo-list Executable file
View File

@ -0,0 +1,55 @@
#!/usr/bin/env python
# Copyright (C) 2011-2013 OpenStack Foundation
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
# implied.
#
# See the License for the specific language governing permissions and
# limitations under the License.
import os
import urllib2
import yaml
URL = ('https://git.openstack.org/cgit/openstack-infra/config/plain/'
'modules/openstack_project/files/review.projects.yaml')
TMP_HOOKS_PATH=os.environ['TMP_HOOKS_PATH']
PROJECTS_REPOS=os.path.join(TMP_HOOKS_PATH, 'source-repository-projects-yaml')
GIT_BASE=os.environ.get('GIT_BASE', 'git://git.openstack.org')
def main():
projects = [f['project'] for f in yaml.load(urllib2.urlopen(URL))]
with open(PROJECTS_REPOS, 'w') as projects_list:
for project in projects:
args = dict(
name=os.path.basename(project),
location=os.path.join('/opt/git', project),
url='%s/%s.git' % (GIT_BASE, project))
projects_list.write("%(name)s git %(location)s %(url)s\n" % args)
# Clone openstack-infra/config again so that we can use it to build
# the image without interferring with the slave repo cache.
project = 'openstack-infra/config'
args = dict(
name='config_tmp',
location=os.path.join('/opt/build_git', project),
url=os.environ.get('CONFIG_SOURCE',
'%s/%s.git' % (GIT_BASE, project)),
ref=os.environ.get('CONFIG_REF', 'master'))
projects_list.write(
"%(name)s git %(location)s %(url)s %(ref)s\n" % args)
if __name__ == '__main__':
main()

20
modules/openstack_project/files/nodepool/elements/openstack-repos/install.d/95-chown-jenkins Executable file
View File

@ -0,0 +1,20 @@
#!/bin/bash
# Copyright (C) 2011-2013 OpenStack Foundation
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
# implied.
#
# See the License for the specific language governing permissions and
# limitations under the License.
set -e
chown -R jenkins:jenkins /home/jenkins

1
modules/openstack_project/files/nodepool/elements/puppet/README.rst Normal file
View File

@ -0,0 +1 @@
Bootstrap puppet on a node

55
modules/openstack_project/files/nodepool/elements/puppet/bin/prepare-node Normal file
View File

@ -0,0 +1,55 @@
#!/bin/bash
# Copyright (C) 2011-2013 OpenStack Foundation
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
# implied.
#
# See the License for the specific language governing permissions and
# limitations under the License.
set -e
SUDO=${SUDO:-true}
THIN=${THIN:-true}
PYTHON3=${PYTHON3:-false}
PYPY=${PYPY:-false}
ALL_MYSQL_PRIVS=${ALL_MYSQL_PRIVS:-false}
export FACTER_in_chroot=true
cat >/tmp/local.pp <<EOF
Service {
start => '/bin/true',
stop => '/bin/true',
status => '/bin/true',
restart => '/bin/true'
}
class {'openstack_project::single_use_slave':
sudo => $SUDO,
thin => $THIN,
python3 => $PYTHON3,
include_pypy => $PYPY,
all_mysql_privs => $ALL_MYSQL_PRIVS,
install_resolv_conf => false
}
EOF
# Puppet doesn't return nonzero if some things fail by default.
# Use detailed exit codes to get that info and determine whether
# the return code indicates failure.
set +e
puppet apply --detailed-exitcodes --modulepath=/opt/build_git/openstack-infra/config/modules:/etc/puppet/modules /tmp/local.pp
PUPPET_RETURN=$?
if [ "$PUPPET_RETURN" -eq 4 ] || [ "$PUPPET_RETURN" -eq 6 ] ; then
exit $PUPPET_RETURN
fi
set -e
# Make sure resolv.conf settings don't break dib
echo 'nameserver 8.8.8.8'> /etc/resolv.conf

3
modules/openstack_project/files/nodepool/elements/puppet/element-deps Normal file
View File

@ -0,0 +1,3 @@
source-repositories
cache-url
openstack-repos

27
modules/openstack_project/files/nodepool/elements/puppet/install.d/05-puppet Executable file
View File

@ -0,0 +1,27 @@
#!/bin/bash
# Copyright (C) 2011-2013 OpenStack Foundation
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
# implied.
#
# See the License for the specific language governing permissions and
# limitations under the License.
set -e
# Unset the download cache for this invocation to prevent bleed from build host
unset PIP_DOWNLOAD_CACHE
export PUPPET_VERSION=${PUPPET_VERSION:-'2'}
/bin/bash /opt/git/openstack-infra/config/install_puppet.sh
/bin/bash /opt/git/openstack-infra/config/install_modules.sh
install -m 0755 -o root -g root $(dirname $0)/../bin/prepare-node /usr/local/bin

37
modules/openstack_project/files/nodepool/elements/puppet/install.d/95-clean-repos Executable file
View File

@ -0,0 +1,37 @@
#!/bin/bash
# Copyright (C) 2011-2013 OpenStack Foundation
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
# implied.
#
# See the License for the specific language governing permissions and
# limitations under the License.
# Remove additional sources used to install puppet or special version of pypi.
# We do this because leaving these sources in place causes every test that
# does an apt-get update to hit those servers which may not have the uptime
# of our local mirrors.
set -e
OS_FAMILY=$(facter osfamily)
if [ "$OS_FAMILY" == "Debian" ] ; then
rm -f /etc/apt/sources.list.d/*
apt-get update
elif [ "$OS_FAMILY" == "RedHat" ] ; then
# Can't delete * in yum.repos.d since all of the repos are listed there.
# Be specific instead.
if [ -f /etc/yum.repos.d/puppetlabs.repo ] ; then
sudo rm -f /etc/yum.repos.d/puppetlabs.repo
fi
fi
rm /usr/local/bin/prepare-node

23
modules/openstack_project/files/nodepool/elements/puppet/install.d/96-clean-cron Executable file
View File

@ -0,0 +1,23 @@
#!/bin/bash
# Copyright (C) 2011-2013 OpenStack Foundation
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
# implied.
#
# See the License for the specific language governing permissions and
# limitations under the License.
# Remove cron jobs
# We create fresh servers for these hosts, and they are used once. They don't
# need to do things like update the locatedb or the mandb or rotate logs
# or really any of those things. We only want code running here that we want
# here.
rm -f /etc/cron.{monthly,weekly,daily,hourly,d}/*

39
modules/openstack_project/files/nodepool/elements/puppet/pre-install.d/10-preseed Executable file
View File

@ -0,0 +1,39 @@
#!/bin/bash
# Copyright (C) 2011-2013 OpenStack Foundation
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
# implied.
#
# See the License for the specific language governing permissions and
# limitations under the License.
# workaround a broken maintainer script in iptables-persistent that
# assumes it can execute things during postinst. Since we're installing
# in a chroot, we're actually not running the right kernel for its assumptions
# to work.
set -e
# If lsb_release is missing, just do nothing.
DISTRO=`lsb_release -si` || true
case $DISTRO in
'Ubuntu'|'Debian')
PRESEED=`mktemp`
cat > $PRESEED <<EOF
d-i iptables-persistent/autosave_done boolean true
d-i iptables-persistent/autosave_v4 boolean false
d-i iptables-persistent/autosave_v6 boolean false
EOF
debconf-set-selections $PRESEED
rm $PRESEED
;;
esac

1
modules/openstack_project/files/nodepool/elements/slave-db/README.rst Normal file
View File

@ -0,0 +1 @@
Install databases on slave nodes

12
modules/openstack_project/manifests/nodepool_prod.pp
View File

@ -60,4 +60,16 @@ class openstack_project::nodepool_prod(
source => 'puppet:///modules/openstack_project/nodepool/scripts', source => 'puppet:///modules/openstack_project/nodepool/scripts',
} }
file { '/etc/nodepool/elements':
ensure => directory,
owner => 'root',
group => 'root',
mode => '0755',
recurse => true,
purge => true,
force => true,
require => File['/etc/nodepool'],
source => 'puppet:///modules/openstack_project/nodepool/elements',
}
} }

23
modules/openstack_project/manifests/slave_common.pp
View File

@ -92,4 +92,27 @@ class openstack_project::slave_common(
} }
} }
} }
# install linux-headers depending on OS version
case $::osfamily {
'RedHat': {
$header_packages = ['kernel-devel', 'kernel-headers']
}
'Debian': {
if ($::operatingsystem == 'Debian') {
# install depending on kernel release
$header_packages = [ "linux-headers-${::kernelrelease}", ]
}
else {
$header_packages = ['linux-headers-virtual', 'linux-headers-generic']
}
}
default: {
fail("Unsupported osfamily: ${::osfamily}.")
}
}
package { $header_packages:
ensure => present
}
} }

2
modules/openstack_project/templates/nodepool/nodepool.yaml.erb
View File

@ -1,4 +1,6 @@
script-dir: /etc/nodepool/scripts script-dir: /etc/nodepool/scripts
elements-dir: /etc/nodepool/elements
images-dir: /opt/nodepool_dib
dburi: 'mysql://nodepool:<%= mysql_password %>@localhost/nodepool' dburi: 'mysql://nodepool:<%= mysql_password %>@localhost/nodepool'
cron: cron:

16
tools/build-dib-in-docker.sh Normal file
View File

@ -0,0 +1,16 @@
# Copyright (c) 2014 Hewlett-Packard Development Company, L.P.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
# implied.
# See the License for the specific language governing permissions and
# limitations under the License.
docker.io run --privileged=true -v ~/.cache:/.cache -v $(git rev-parse --show-toplevel):/opt/config -w /opt/config -i -t openstack_infra/ubuntu /bin/bash tools/build-image.sh

28
tools/build-image.sh Executable file
View File

@ -0,0 +1,28 @@
#!/bin/bash
# Copyright (c) 2014 Hewlett-Packard Development Company, L.P.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
# implied.
# See the License for the specific language governing permissions and
# limitations under the License.
set -e
export ELEMENTS_PATH=${ELEMENTS_PATH:-modules/openstack_project/files/nodepool/elements}
export DISTRO=${DISTRO:-ubuntu}
export DIB_RELEASE=${DIB_RELEASE:-trusty}
export DIB_IMAGE_NAME=${DIB_IMAGE_NAME:-${DISTRO}_${DIB_RELEASE}}
export DIB_IMAGE_FILENAME=${DIB_IMAGE_FILENAME:-${DIB_IMAGE_NAME}.qcow}
export NODEPOOL_SCRIPTDIR=${NODEPOOL_SCRIPTDIR:-modules/openstack_project/files/nodepool/scripts}
export CONFIG_SOURCE=${CONFIG_SOURCE:-file://$(pwd)}
export CONFIG_REF=${CONFIG_REF:-$(git rev-parse HEAD)}
disk-image-create -x --no-tmpfs -o devstack-gate-$DIB_RELEASE $DISTRO vm openstack-repos puppet nodepool-base node-devstack

25
tools/mount-image.sh Normal file
View File

@ -0,0 +1,25 @@
#!/bin/bash
# Copyright (c) 2014 Hewlett-Packard Development Company, L.P.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
# implied.
# See the License for the specific language governing permissions and
# limitations under the License.
set -e
IMAGE=$1
rm -rf /tmp/newimage
mkdir -p /tmp/newimage
qemu-nbd -c /dev/nbd1 $1
mount /dev/nbd1 /tmp/newimage

19
tools/umount-image.sh Normal file
View File

@ -0,0 +1,19 @@
# Copyright (c) 2014 Hewlett-Packard Development Company, L.P.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
# implied.
# See the License for the specific language governing permissions and
# limitations under the License.
set -e
umount /tmp/newimage
qemu-nbd -d /dev/nbd1