project-config/nodepool/scripts/cache_devstack.py
Andreas Jaeger ca687cd8b4 Remove centos6 handling from cache_devstack
With CentOS6 support gone, we can remove special treatment from
nodepool scripts.

Change-Id: Ia50de3450c0f21c27740d2d6294d0f2451196e21
2015-12-04 21:52:49 +01:00

190 lines
6.4 KiB
Python
Executable File

#!/usr/bin/env python
# Copyright (C) 2011-2013 OpenStack Foundation
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
# implied.
#
# See the License for the specific language governing permissions and
# limitations under the License.
import os
import sys
from common import run_local
DEVSTACK = os.path.expanduser('/opt/git/openstack-dev/devstack')
CACHEDIR = os.path.expanduser('~/cache/files')
# Some jobs might require newer distro packages, so we can pre-cache
# deb packages from specified Ubuntu Cloud Archive pockets.
UCA_POCKETS = []
def git_branches():
branches = []
for branch in run_local(['git', 'branch', '-a'], cwd=DEVSTACK).split("\n"):
branch = branch.strip()
if not branch.startswith('remotes/origin'):
continue
branches.append(branch)
return branches
def tokenize(fn, tokens, distribution, comment=None):
for line in open(fn):
if 'dist:' in line and ('dist:%s' % distribution not in line):
continue
if 'qpid' in line:
continue # TODO: explain why this is here
if comment and comment in line:
line = line[:line.rfind(comment)]
line = line.strip()
if line and line not in tokens:
tokens.append(line)
def _legacy_find_images(basedir):
"""Divine what images we should use based on parsing stackrc."""
images = []
for line in open(os.path.join(basedir, 'stackrc')):
line = line.strip()
if line.startswith('IMAGE_URLS'):
if '#' in line:
line = line[:line.rfind('#')]
if line.endswith(';;'):
line = line[:-2]
line = line.split('=', 1)[1].strip()
if line.startswith('${IMAGE_URLS:-'):
line = line[len('${IMAGE_URLS:-'):]
if line.endswith('}'):
line = line[:-1]
if not line:
continue
if line[0] == line[-1] == '"':
line = line[1:-1]
# Add image to the list to be downloaded, but
# skip downloading giant vmware images
images += [x.strip() for x in line.split(',')
if not x.strip().endswith('vmdk')]
return images
def _find_images(basedir):
images = []
image_tool = os.path.join(basedir, 'tools', 'image_list.sh')
if os.path.exists(image_tool):
returncode, out = run_local(image_tool, status=True)
if returncode:
print "%s failed" % image_tool
print "Exit: %s, Output: %s" % (returncode, out)
# reset images so we'll fall back
images = []
else:
images = out.split('\n')
return images
def local_prep(distribution):
branches = []
for branch in git_branches():
# Ignore branches of the form 'somestring -> someotherstring'
# as this denotes a symbolic reference and the entire string
# as is cannot be checked out. We can do this safely as the
# reference will refer to one of the other branches returned
# by git_branches.
if ' -> ' in branch:
continue
branch_data = {'name': branch}
print 'Branch: ', branch
run_local(['sudo', 'git', 'checkout', branch], cwd=DEVSTACK)
run_local(['sudo', 'git', 'pull', '--ff-only', 'origin'], cwd=DEVSTACK)
if os.path.exists('/usr/bin/apt-get'):
debs = []
debdir = os.path.join(DEVSTACK, 'files', 'debs')
if not os.path.exists(debdir):
debdir = os.path.join(DEVSTACK, 'files', 'apts')
for fn in os.listdir(debdir):
fn = os.path.join(debdir, fn)
tokenize(fn, debs, distribution, comment='#')
branch_data['debs'] = debs
if os.path.exists('/usr/bin/yum'):
rpms = []
rpmdir = os.path.join(DEVSTACK, 'files', 'rpms')
for fn in os.listdir(rpmdir):
fn = os.path.join(rpmdir, fn)
tokenize(fn, rpms, distribution, comment='#')
branch_data['rpms'] = rpms
images = _find_images(DEVSTACK)
if not images:
images = _legacy_find_images(DEVSTACK)
branch_data['images'] = images
branches.append(branch_data)
return branches
def download(url, fname):
run_local(['wget', '-nv', '-c', url, '-O', os.path.join(CACHEDIR, fname)])
def cache_debs(debs, uca_pocket=None):
"""Cache a list of deb packages, optionally pulling from an Ubuntu Cloud
Archive pocket. If a UCA pocket is specified, it is enabled temporarily
for caching only.
"""
if uca_pocket:
# Note this will install the ubuntu-cloud-keyring package which
# contains the required GPG key.
run_local(['sudo', 'add-apt-repository', '-y',
'cloud-archive:%s' % uca_pocket])
run_local(['sudo', 'apt-get', 'update'])
run_local(['sudo', 'apt-get', '-y', '-d', 'install'] + debs)
if uca_pocket:
run_local(['sudo', 'rm', '-f',
'/etc/apt/sources.list.d/cloudarchive-%s.list' % uca_pocket])
run_local(['sudo', 'apt-get', 'update'])
def main():
distribution = sys.argv[1]
branches = local_prep(distribution)
image_filenames = []
for branch_data in branches:
if branch_data.get('debs'):
cache_debs(branch_data['debs'])
for uca in sorted(UCA_POCKETS):
cache_debs(branch_data['debs'], uca)
elif branch_data.get('rpms'):
run_local(['sudo', 'yum', 'install', '-y', '--downloadonly'] +
branch_data['rpms'])
else:
sys.exit('No supported package data found.')
for url in branch_data['images']:
fname = url.split('/')[-1]
if fname in image_filenames:
continue
image_filenames.append(fname)
download(url, fname)
# cache get-pip, because upstream network connection fails more
# often than you might imagine.
download('https://bootstrap.pypa.io/get-pip.py', 'get-pip.py')
if __name__ == '__main__':
main()