3d5b04ed19
Some packages may depend on python-setuptools, which is not installed and cannot be reinstalled on CentOS 6.x once yum has erased them, so use --skip-broken to avoid aborting. Also on this platform --downloadonly causes yum to return nonzero even when it succeeds, so ignore its exit code. Change-Id: Iaada39ae81e1e47fe9d0bedba80fd19e4e0e6f38
208 lines
7.4 KiB
Python
Executable File
208 lines
7.4 KiB
Python
Executable File
#!/usr/bin/env python
|
|
|
|
# Copyright (C) 2011-2013 OpenStack Foundation
|
|
#
|
|
# Licensed under the Apache License, Version 2.0 (the "License");
|
|
# you may not use this file except in compliance with the License.
|
|
# You may obtain a copy of the License at
|
|
#
|
|
# http://www.apache.org/licenses/LICENSE-2.0
|
|
#
|
|
# Unless required by applicable law or agreed to in writing, software
|
|
# distributed under the License is distributed on an "AS IS" BASIS,
|
|
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
|
|
# implied.
|
|
#
|
|
# See the License for the specific language governing permissions and
|
|
# limitations under the License.
|
|
|
|
import os
|
|
import sys
|
|
|
|
from common import run_local
|
|
|
|
DEVSTACK = os.path.expanduser('/opt/git/openstack-dev/devstack')
|
|
CACHEDIR = os.path.expanduser('~/cache/files')
|
|
|
|
# Some jobs require newer distro packages. We pre-cache deb packages from
|
|
# specified Ubuntu Cloud Archive pockets. The icehouse pocket contains
|
|
# Ubuntu 14.04 packages built for 12.04.
|
|
UCA_POCKETS = ['icehouse']
|
|
|
|
|
|
def git_branches():
|
|
branches = []
|
|
for branch in run_local(['git', 'branch', '-a'], cwd=DEVSTACK).split("\n"):
|
|
branch = branch.strip()
|
|
if not branch.startswith('remotes/origin'):
|
|
continue
|
|
branches.append(branch)
|
|
return branches
|
|
|
|
|
|
def tokenize(fn, tokens, distribution, comment=None):
|
|
for line in open(fn):
|
|
if 'dist:' in line and ('dist:%s' % distribution not in line):
|
|
continue
|
|
if 'qpid' in line:
|
|
continue # TODO: explain why this is here
|
|
if comment and comment in line:
|
|
line = line[:line.rfind(comment)]
|
|
line = line.strip()
|
|
if line and line not in tokens:
|
|
tokens.append(line)
|
|
|
|
|
|
def _legacy_find_images(basedir):
|
|
"""Divine what images we should use based on parsing stackrc."""
|
|
images = []
|
|
for line in open(os.path.join(basedir, 'stackrc')):
|
|
line = line.strip()
|
|
if line.startswith('IMAGE_URLS'):
|
|
if '#' in line:
|
|
line = line[:line.rfind('#')]
|
|
if line.endswith(';;'):
|
|
line = line[:-2]
|
|
line = line.split('=', 1)[1].strip()
|
|
if line.startswith('${IMAGE_URLS:-'):
|
|
line = line[len('${IMAGE_URLS:-'):]
|
|
if line.endswith('}'):
|
|
line = line[:-1]
|
|
if not line:
|
|
continue
|
|
if line[0] == line[-1] == '"':
|
|
line = line[1:-1]
|
|
# Add image to the list to be downloaded, but
|
|
# skip downloading giant vmware images
|
|
images += [x.strip() for x in line.split(',')
|
|
if not x.strip().endswith('vmdk')]
|
|
return images
|
|
|
|
|
|
def _find_images(basedir):
|
|
images = []
|
|
image_tool = os.path.join(basedir, 'tools', 'image_list.sh')
|
|
if os.path.exists(image_tool):
|
|
returncode, out = run_local(image_tool, status=True)
|
|
if returncode:
|
|
print "%s failed" % image_tool
|
|
print "Exit: %s, Output: %s" % (returncode, out)
|
|
# reset images so we'll fall back
|
|
images = []
|
|
else:
|
|
images = out.split('\n')
|
|
return images
|
|
|
|
|
|
def local_prep(distribution):
|
|
branches = []
|
|
for branch in git_branches():
|
|
# Ignore branches of the form 'somestring -> someotherstring'
|
|
# as this denotes a symbolic reference and the entire string
|
|
# as is cannot be checked out. We can do this safely as the
|
|
# reference will refer to one of the other branches returned
|
|
# by git_branches.
|
|
if ' -> ' in branch:
|
|
continue
|
|
branch_data = {'name': branch}
|
|
print 'Branch: ', branch
|
|
run_local(['sudo', 'git', 'checkout', branch], cwd=DEVSTACK)
|
|
run_local(['sudo', 'git', 'pull', '--ff-only', 'origin'], cwd=DEVSTACK)
|
|
|
|
if os.path.exists('/usr/bin/apt-get'):
|
|
debs = []
|
|
debdir = os.path.join(DEVSTACK, 'files', 'debs')
|
|
if not os.path.exists(debdir):
|
|
debdir = os.path.join(DEVSTACK, 'files', 'apts')
|
|
for fn in os.listdir(debdir):
|
|
fn = os.path.join(debdir, fn)
|
|
tokenize(fn, debs, distribution, comment='#')
|
|
branch_data['debs'] = debs
|
|
|
|
if os.path.exists('/usr/bin/yum'):
|
|
rpms = []
|
|
rpmdir = os.path.join(DEVSTACK, 'files', 'rpms')
|
|
for fn in os.listdir(rpmdir):
|
|
fn = os.path.join(rpmdir, fn)
|
|
tokenize(fn, rpms, distribution, comment='#')
|
|
branch_data['rpms'] = rpms
|
|
|
|
images = _find_images(DEVSTACK)
|
|
if not images:
|
|
images = _legacy_find_images(DEVSTACK)
|
|
|
|
branch_data['images'] = images
|
|
branches.append(branch_data)
|
|
return branches
|
|
|
|
|
|
def download(url, fname):
|
|
run_local(['wget', '-nv', '-c', url, '-O', os.path.join(CACHEDIR, fname)])
|
|
|
|
|
|
def cache_debs(debs, uca_pocket=None):
|
|
"""Cache a list of deb packages, optionally pulling from an Ubuntu Cloud
|
|
Archive pocket. If a UCA pocket is specified, it is enabled temporarily
|
|
for caching only.
|
|
"""
|
|
if uca_pocket:
|
|
# Note this will install the ubuntu-cloud-keyring package which
|
|
# contains the required GPG key.
|
|
run_local(['sudo', 'add-apt-repository', '-y',
|
|
'cloud-archive:%s' % uca_pocket])
|
|
run_local(['sudo', 'apt-get', 'update'])
|
|
run_local(['sudo', 'apt-get', '-y', '-d', 'install'] + debs)
|
|
if uca_pocket:
|
|
run_local(['sudo', 'rm', '-f',
|
|
'/etc/apt/sources.list.d/cloudarchive-%s.list' % uca_pocket])
|
|
run_local(['sudo', 'apt-get', 'update'])
|
|
|
|
|
|
def main():
|
|
distribution = sys.argv[1]
|
|
|
|
if (os.path.exists('/etc/redhat-release') and
|
|
open('/etc/redhat-release').read().startswith("CentOS release 6")):
|
|
# --downloadonly is provided by the yum-plugin-downloadonly package
|
|
# on CentOS 6.x
|
|
centos6 = True
|
|
run_local(['sudo', 'yum', 'install', '-y', 'yum-plugin-downloadonly'])
|
|
else:
|
|
centos6 = False
|
|
branches = local_prep(distribution)
|
|
image_filenames = []
|
|
for branch_data in branches:
|
|
if branch_data.get('debs'):
|
|
cache_debs(branch_data['debs'])
|
|
for uca in sorted(UCA_POCKETS):
|
|
cache_debs(branch_data['debs'], uca)
|
|
elif branch_data.get('rpms'):
|
|
if centos6:
|
|
# some packages may depend on python-setuptools, which is not
|
|
# installed and cannot be reinstalled on CentOS 6.x once yum
|
|
# has erased them, so use --skip-broken to avoid aborting; also
|
|
# on this platform --downloadonly causes yum to return nonzero
|
|
# even when it succeeds, so ignore its exit code
|
|
run_local(['sudo', 'yum', 'install', '-y', '--downloadonly',
|
|
'--skip-broken'] + branch_data['rpms'])
|
|
else:
|
|
run_local(['sudo', 'yum', 'install', '-y', '--downloadonly'] +
|
|
branch_data['rpms'])
|
|
else:
|
|
sys.exit('No supported package data found.')
|
|
|
|
for url in branch_data['images']:
|
|
fname = url.split('/')[-1]
|
|
if fname in image_filenames:
|
|
continue
|
|
image_filenames.append(fname)
|
|
download(url, fname)
|
|
|
|
# cache get-pip, because upstream network connection fails more
|
|
# often than you might imagine.
|
|
download('https://bootstrap.pypa.io/get-pip.py', 'get-pip.py')
|
|
|
|
|
|
if __name__ == '__main__':
|
|
main()
|