#!/usr/bin/env python # Copyright (C) 2011-2013 OpenStack Foundation # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or # implied. # # See the License for the specific language governing permissions and # limitations under the License. import os import subprocess import sys # this is a bit weird; we want to get the filter to use on the # devstack lib/[apt|rpm] files to pre-install them. So we have to # swizzle things around RELEASE=None if 'DIB_RELEASE' in os.environ: RELEASE=os.environ['DIB_RELEASE'] if 'DISTRO_NAME' in os.environ: if os.environ['DISTRO_NAME'] == 'centos7': # centos7 matches as rhel7 in devstack RELEASE='rhel7' if not RELEASE: print "Can not determine RELEASE" sys.exit(1) TMP_MOUNT_PATH = os.environ['TMP_MOUNT_PATH'] TMP_HOOKS_PATH = os.environ['TMP_HOOKS_PATH'] DEVSTACK = os.path.join(TMP_MOUNT_PATH, 'opt/git/openstack-dev/devstack') CACHEDIR = os.path.join(TMP_MOUNT_PATH, 'tmp') IMAGES=os.path.join(TMP_HOOKS_PATH, 'source-repository-images') def run_local(cmd, status=False, cwd='.', env={}): print "Running:", cmd newenv = os.environ newenv.update(env) p = subprocess.Popen(cmd, stdout=subprocess.PIPE, cwd=cwd, stderr=subprocess.STDOUT, env=newenv) (out, nothing) = p.communicate() if status: return (p.returncode, out.strip()) return out.strip() def git_branches(): branches = [] for branch in run_local(['git', 'branch', '-a'], cwd=DEVSTACK).split("\n"): branch = branch.strip() if not branch.startswith('remotes/origin'): continue branches.append(branch) return branches def tokenize(fn, tokens, distribution, comment=None): for line in open(fn): if 'dist:' in line and ('dist:%s' % distribution not in line): continue if 'qpid' in line: continue # TODO: explain why this is here if comment and comment in line: line = line[:line.rfind(comment)] line = line.strip() if line and line not in tokens: tokens.append(line) def _legacy_find_images(basedir): """Divine what images we should use based on parsing stackrc.""" images = [] for line in open(os.path.join(basedir, 'stackrc')): line = line.strip() if line.startswith('IMAGE_URLS'): if '#' in line: line = line[:line.rfind('#')] if line.endswith(';;'): line = line[:-2] line = line.split('=', 1)[1].strip() if line.startswith('${IMAGE_URLS:-'): line = line[len('${IMAGE_URLS:-'):] if line.endswith('}'): line = line[:-1] if not line: continue if line[0] == line[-1] == '"': line = line[1:-1] # Add image to the list to be downloaded, but # skip downloading giant vmware images images += [x.strip() for x in line.split(',') if not x.strip().endswith('vmdk')] return images def _find_images(basedir): images = [] try: image_tool = os.path.join(basedir, 'tools', 'image_list.sh') if os.path.exists(image_tool): images = subprocess.check_output(image_tool).split('\n') except subprocess.CalledProcessError as ce: print "image_list.sh failed" print "Exit: %s, Output: %s" % (ce.returncode, ce.output) # reset images so we'll fall back images = [] return images def local_prep(distribution): branches = [] for branch in git_branches(): # Ignore branches of the form 'somestring -> someotherstring' # as this denotes a symbolic reference and the entire string # as is cannot be checked out. We can do this safely as the # reference will refer to one of the other branches returned # by git_branches. if ' -> ' in branch: continue branch_data = {'name': branch} print 'Branch: ', branch run_local(['sudo', 'git', 'checkout', branch], cwd=DEVSTACK) run_local(['sudo', 'git', 'pull', '--ff-only', 'origin'], cwd=DEVSTACK) if os.path.exists(os.path.join(TMP_MOUNT_PATH, 'usr/bin/apt-get')): debs = [] debdir = os.path.join(DEVSTACK, 'files', 'debs') if not os.path.exists(debdir): debdir = os.path.join(DEVSTACK, 'files', 'apts') for fn in os.listdir(debdir): fn = os.path.join(debdir, fn) tokenize(fn, debs, distribution, comment='#') branch_data['debs'] = debs if os.path.exists(os.path.join(TMP_MOUNT_PATH, 'usr/bin/rpm')): rpms = [] rpmdir = os.path.join(DEVSTACK, 'files', 'rpms') for fn in os.listdir(rpmdir): fn = os.path.join(rpmdir, fn) tokenize(fn, rpms, distribution, comment='#') branch_data['rpms'] = rpms images = _find_images(DEVSTACK) if not images: images = _legacy_find_images(DEVSTACK) branch_data['images'] = images branches.append(branch_data) return branches def main(): branches = local_prep(RELEASE) with open(os.path.join(CACHEDIR, 'pkgs-to-install'), 'w') as pkgs: for branch_data in branches: if branch_data.get('debs'): pkgs.write(" ".join(branch_data['debs']) + "\n") elif branch_data.get('rpms'): pkgs.write(" ".join(branch_data['rpms']) + "\n") else: sys.exit('No supported package data found.') image_filenames = [] line_template = "%(name)s file %(location)s %(url)s\n" with open(IMAGES, 'w') as images_list: for branch_data in branches: for url in branch_data['images']: fname = url.split('/')[-1].strip() if fname == '': continue # TODO: There is an image in devstack that starts with # $HEAT_FETCHED_TEST_IMAGE if fname.startswith('$'): continue if fname in image_filenames: continue image_filenames.append(fname) args = dict( name=fname, location=os.path.join('/home/jenkins/cache/files', fname), url=url) images_list.write(line_template % args) if __name__ == '__main__': main()