project-config/nodepool/elements/cache-devstack/extra-data.d/55-cache-devstack-repos

#!/usr/bin/env python

# Copyright (C) 2011-2013 OpenStack Foundation
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#    http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
# implied.
#
# See the License for the specific language governing permissions and
# limitations under the License.

from __future__ import print_function
from __future__ import unicode_literals

import os
import subprocess
import sys

TMP_MOUNT_PATH = os.environ['TMP_MOUNT_PATH']
TMP_HOOKS_PATH = os.environ['TMP_HOOKS_PATH']

# this is a bit weird; we want to get the filter to use on the
# devstack lib/[apt|rpm] files to pre-install them.  So we have to
# swizzle things around
RELEASE=None
if 'DIB_RELEASE' in os.environ:
    RELEASE=os.environ['DIB_RELEASE']
try:
    if open(os.path.join(
            TMP_MOUNT_PATH,
            'etc/redhat-release')).read().startswith('CentOS release 6'):
        # not actually used in devstack, but for consistency and because this
        # script will error if run on a platform where RELEASE isn't detected
        # as something
        RELEASE='rhel6'
except IOError:
    pass
if 'DISTRO_NAME' in os.environ:
    if os.environ['DISTRO_NAME'] == 'centos7':
        # centos7 matches as rhel7 in devstack
        RELEASE='rhel7'
if not RELEASE:
    print("Can not determine RELEASE")
    sys.exit(1)

DEVSTACK = os.path.join(TMP_MOUNT_PATH, 'opt/git/openstack-dev/devstack')
CACHEDIR = os.path.join(TMP_MOUNT_PATH, 'tmp')
IMAGES=os.path.join(TMP_HOOKS_PATH, 'source-repository-images')


def run_local(cmd, status=False, cwd='.', env={}):
    print("Running:", cmd)
    newenv = os.environ
    newenv.update(env)
    p = subprocess.Popen(cmd, stdout=subprocess.PIPE, cwd=cwd,
                         stderr=subprocess.STDOUT, env=newenv,
                         universal_newlines=True)
    (out, nothing) = p.communicate()
    if status:
        return (p.returncode, out.strip())
    return out.strip()


def git_branches():
    branches = []
    for branch in run_local(['git', 'branch', '-a'], cwd=DEVSTACK).split("\n"):
        branch = branch.strip()
        if not branch.startswith('remotes/origin'):
            continue
        branches.append(branch)
    return branches


def _legacy_find_images(basedir):
    """Divine what images we should use based on parsing stackrc."""
    images = []
    for line in open(os.path.join(basedir, 'stackrc')):
        line = line.strip()
        if line.startswith('IMAGE_URLS'):
            if '#' in line:
                line = line[:line.find('#')]
            if line.endswith(';;'):
                line = line[:-2]
            line = line.split('=', 1)[1].strip()
            if line.startswith('${IMAGE_URLS:-'):
                line = line[len('${IMAGE_URLS:-'):]
            if line.endswith('}'):
                line = line[:-1]
            if not line:
                continue
            if line[0] == line[-1] == '"':
                line = line[1:-1]
            # Add image to the list to be downloaded, but
            # skip downloading giant vmware images
            images += [x.strip() for x in line.split(',')
                       if not x.strip().endswith('vmdk')]
    return images


def _find_images(basedir):
    images = []
    image_tool = os.path.join(basedir, 'tools', 'image_list.sh')
    if os.path.exists(image_tool):
        returncode, out = run_local(image_tool, status=True)
        if returncode:
            print("%s failed" % image_tool)
            print("Exit: %s, Output: %s" % (returncode, out))
            # reset images so we'll fall back
            images = []
        else:
            images = out.split('\n')
    return images


def local_prep(distribution):
    branches = []
    for branch in git_branches():
        # Ignore branches of the form 'somestring -> someotherstring'
        # as this denotes a symbolic reference and the entire string
        # as is cannot be checked out. We can do this safely as the
        # reference will refer to one of the other branches returned
        # by git_branches.
        if ' -> ' in branch:
            continue
        branch_data = {'name': branch}
        print('Branch: ', branch)
        run_local(['sudo', 'git', 'checkout', branch], cwd=DEVSTACK)
        run_local(['sudo', 'git', 'pull', '--ff-only', 'origin'], cwd=DEVSTACK)

        images = _find_images(DEVSTACK)
        if not images:
            images = _legacy_find_images(DEVSTACK)

        branch_data['images'] = images
        branches.append(branch_data)
    return branches


def main():

    branches = local_prep(RELEASE)

    image_filenames = []
    line_template = "%(name)s file %(location)s %(url)s\n"
    with open(IMAGES, 'w') as images_list:
        for branch_data in branches:
            for url in branch_data['images']:
                fname = url.split('/')[-1].strip()
                if fname == '':
                    continue
                # TODO: There is an image in devstack that starts with
                #       $HEAT_FETCHED_TEST_IMAGE
                if fname.startswith('$'):
                    continue
                if fname in image_filenames:
                    continue
                image_filenames.append(fname)
                args = dict(
                    name=fname,
                    location=os.path.join('/opt/cache/files', fname),
                    url=url)

                images_list.write(line_template % args)


if __name__ == '__main__':
    main()