#!/usr/bin/env python
# Copyright (C) 2011-2013 OpenStack Foundation
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
# implied.
#
# See the License for the specific language governing permissions and
# limitations under the License.
import os
import subprocess
import sys
TMP_MOUNT_PATH = os.environ['TMP_MOUNT_PATH']
TMP_HOOKS_PATH = os.environ['TMP_HOOKS_PATH']
# this is a bit weird; we want to get the filter to use on the
# devstack lib/[apt|rpm] files to pre-install them. So we have to
# swizzle things around
RELEASE=None
if 'DIB_RELEASE' in os.environ:
RELEASE=os.environ['DIB_RELEASE']
try:
if open(os.path.join(
TMP_MOUNT_PATH,
'etc/redhat-release')).read().startswith('CentOS release 6'):
# not actually used in devstack, but for consistency and because this
# script will error if run on a platform where RELEASE isn't detected
# as something
RELEASE='rhel6'
except IOError:
pass
if 'DISTRO_NAME' in os.environ:
if os.environ['DISTRO_NAME'] == 'centos7':
# centos7 matches as rhel7 in devstack
RELEASE='rhel7'
if not RELEASE:
print "Can not determine RELEASE"
sys.exit(1)
DEVSTACK = os.path.join(TMP_MOUNT_PATH, 'opt/git/openstack-dev/devstack')
CACHEDIR = os.path.join(TMP_MOUNT_PATH, 'tmp')
IMAGES=os.path.join(TMP_HOOKS_PATH, 'source-repository-images')
def run_local(cmd, status=False, cwd='.', env={}):
print "Running:", cmd
newenv = os.environ
newenv.update(env)
p = subprocess.Popen(cmd, stdout=subprocess.PIPE, cwd=cwd,
stderr=subprocess.STDOUT, env=newenv)
(out, nothing) = p.communicate()
if status:
return (p.returncode, out.strip())
return out.strip()
def git_branches():
branches = []
for branch in run_local(['git', 'branch', '-a'], cwd=DEVSTACK).split("\n"):
branch = branch.strip()
if not branch.startswith('remotes/origin'):
continue
branches.append(branch)
return branches
def tokenize(fn, tokens, distribution, comment=None):
for line in open(fn):
if 'dist:' in line and ('dist:%s' % distribution not in line):
continue
if 'qpid' in line:
continue # TODO: explain why this is here
if comment and comment in line:
line = line[:line.rfind(comment)]
line = line.strip()
if line and line not in tokens:
tokens.append(line)
def _legacy_find_images(basedir):
"""Divine what images we should use based on parsing stackrc."""
images = []
for line in open(os.path.join(basedir, 'stackrc')):
line = line.strip()
if line.startswith('IMAGE_URLS'):
if '#' in line:
line = line[:line.rfind('#')]
if line.endswith(';;'):
line = line[:-2]
line = line.split('=', 1)[1].strip()
if line.startswith('${IMAGE_URLS:-'):
line = line[len('${IMAGE_URLS:-'):]
if line.endswith('}'):
line = line[:-1]
if not line:
continue
if line[0] == line[-1] == '"':
line = line[1:-1]
# Add image to the list to be downloaded, but
# skip downloading giant vmware images
images += [x.strip() for x in line.split(',')
if not x.strip().endswith('vmdk')]
return images
def _find_images(basedir):
images = []
image_tool = os.path.join(basedir, 'tools', 'image_list.sh')
if os.path.exists(image_tool):
returncode, out = run_local(image_tool, status=True)
if returncode:
print "%s failed" % image_tool
print "Exit: %s, Output: %s" % (returncode, out)
# reset images so we'll fall back
images = []
else:
images = out.split('\n')
return images
def local_prep(distribution):
branches = []
for branch in git_branches():
# Ignore branches of the form 'somestring -> someotherstring'
# as this denotes a symbolic reference and the entire string
# as is cannot be checked out. We can do this safely as the
# reference will refer to one of the other branches returned
# by git_branches.
if ' -> ' in branch:
continue
branch_data = {'name': branch}
print 'Branch: ', branch
run_local(['sudo', 'git', 'checkout', branch], cwd=DEVSTACK)
run_local(['sudo', 'git', 'pull', '--ff-only', 'origin'], cwd=DEVSTACK)
if os.path.exists(os.path.join(TMP_MOUNT_PATH, 'usr/bin/apt-get')):
debs = []
debdir = os.path.join(DEVSTACK, 'files', 'debs')
if not os.path.exists(debdir):
debdir = os.path.join(DEVSTACK, 'files', 'apts')
for fn in os.listdir(debdir):
fn = os.path.join(debdir, fn)
tokenize(fn, debs, distribution, comment='#')
branch_data['debs'] = debs
if os.path.exists(os.path.join(TMP_MOUNT_PATH, 'usr/bin/yum')):
rpms = []
rpmdir = os.path.join(DEVSTACK, 'files', 'rpms')
for fn in os.listdir(rpmdir):
fn = os.path.join(rpmdir, fn)
tokenize(fn, rpms, distribution, comment='#')
branch_data['rpms'] = rpms
images = _find_images(DEVSTACK)
if not images:
images = _legacy_find_images(DEVSTACK)
branch_data['images'] = images
branches.append(branch_data)
return branches
def main():
branches = local_prep(RELEASE)
with open(os.path.join(CACHEDIR, 'pkgs-to-install'), 'w') as pkgs:
for branch_data in branches:
if branch_data.get('debs'):
pkgs.write(" ".join(branch_data['debs']) + "\n")
elif branch_data.get('rpms'):
pkgs.write(" ".join(branch_data['rpms']) + "\n")
else:
sys.exit('No supported package data found.')
image_filenames = []
line_template = "%(name)s file %(location)s %(url)s\n"
with open(IMAGES, 'w') as images_list:
for branch_data in branches:
for url in branch_data['images']:
fname = url.split('/')[-1].strip()
if fname == '':
continue
# TODO: There is an image in devstack that starts with
# $HEAT_FETCHED_TEST_IMAGE
if fname.startswith('$'):
continue
if fname in image_filenames:
continue
image_filenames.append(fname)
args = dict(
name=fname,
location=os.path.join('/home/jenkins/cache/files', fname),
url=url)
images_list.write(line_template % args)
if __name__ == '__main__':
main()