From 249e36dec6198c1dfd8e4f80d1f0a815fe6f36aa Mon Sep 17 00:00:00 2001 From: Steve Baker Date: Thu, 5 Mar 2015 14:01:45 +1300 Subject: [PATCH] Remove lib/dib diskimage-builder is a utility rather than a service, and is already installed in devstack via pip when required. lib/dib was created to allow an image to be created during a devstack run for the heat functional tests, however this approach is no longer being taken and there are no other known uses for lib/dib. This change removes lib/dib and moves the pip mirror building to lib/heat so that snapshot pip packages of the heat agent projects can be made available to servers which the heat functional tests boot. This also removes tripleo-image-elements, which has never been utilised, and since images won't be created during heat functional test runs it is no longer required. Change-Id: Ic77f841437ea23c0645d3a34d9dd6bfd1ee28714 --- doc/source/index.rst | 2 - extras.d/40-dib.sh | 27 ----- files/apache-dib-pip-repo.template | 15 --- files/apache-heat-pip-repo.template | 15 +++ lib/dib | 149 ---------------------------- lib/heat | 65 ++++++++---- stack.sh | 6 +- stackrc | 10 +- 8 files changed, 63 insertions(+), 226 deletions(-) delete mode 100644 extras.d/40-dib.sh delete mode 100644 files/apache-dib-pip-repo.template create mode 100644 files/apache-heat-pip-repo.template delete mode 100644 lib/dib diff --git a/doc/source/index.rst b/doc/source/index.rst index 10f4355c07..cfde99132d 100644 --- a/doc/source/index.rst +++ b/doc/source/index.rst @@ -158,7 +158,6 @@ Scripts * `lib/cinder `__ * `lib/config `__ * `lib/database `__ -* `lib/dib `__ * `lib/dstat `__ * `lib/glance `__ * `lib/heat `__ @@ -181,7 +180,6 @@ Scripts * `clean.sh `__ * `run\_tests.sh `__ -* `extras.d/40-dib.sh `__ * `extras.d/50-ironic.sh `__ * `extras.d/60-ceph.sh `__ * `extras.d/70-sahara.sh `__ diff --git a/extras.d/40-dib.sh b/extras.d/40-dib.sh deleted file mode 100644 index fdae01191f..0000000000 --- a/extras.d/40-dib.sh +++ /dev/null @@ -1,27 +0,0 @@ -# dib.sh - Devstack extras script to install diskimage-builder - -if is_service_enabled dib; then - if [[ "$1" == "source" ]]; then - # Initial source - source $TOP_DIR/lib/dib - elif [[ "$1" == "stack" && "$2" == "install" ]]; then - echo_summary "Installing diskimage-builder" - install_dib - elif [[ "$1" == "stack" && "$2" == "post-config" ]]; then - # no-op - : - elif [[ "$1" == "stack" && "$2" == "extra" ]]; then - # no-op - : - fi - - if [[ "$1" == "unstack" ]]; then - # no-op - : - fi - - if [[ "$1" == "clean" ]]; then - # no-op - : - fi -fi diff --git a/files/apache-dib-pip-repo.template b/files/apache-dib-pip-repo.template deleted file mode 100644 index 5d2379b5bb..0000000000 --- a/files/apache-dib-pip-repo.template +++ /dev/null @@ -1,15 +0,0 @@ -Listen %DIB_PIP_REPO_PORT% - - - DocumentRoot %DIB_PIP_REPO% - - DirectoryIndex index.html - Require all granted - Order allow,deny - allow from all - - - ErrorLog /var/log/%APACHE_NAME%/dib_pip_repo_error.log - LogLevel warn - CustomLog /var/log/%APACHE_NAME%/dib_pip_repo_access.log combined - diff --git a/files/apache-heat-pip-repo.template b/files/apache-heat-pip-repo.template new file mode 100644 index 0000000000..d88ac3e35a --- /dev/null +++ b/files/apache-heat-pip-repo.template @@ -0,0 +1,15 @@ +Listen %HEAT_PIP_REPO_PORT% + + + DocumentRoot %HEAT_PIP_REPO% + + DirectoryIndex index.html + Require all granted + Order allow,deny + allow from all + + + ErrorLog /var/log/%APACHE_NAME%/heat_pip_repo_error.log + LogLevel warn + CustomLog /var/log/%APACHE_NAME%/heat_pip_repo_access.log combined + diff --git a/lib/dib b/lib/dib deleted file mode 100644 index 88d9fd8434..0000000000 --- a/lib/dib +++ /dev/null @@ -1,149 +0,0 @@ -#!/bin/bash -# -# lib/dib -# Install and build images with **diskimage-builder** - -# Dependencies: -# -# - functions -# - DEST, DATA_DIR must be defined - -# stack.sh -# --------- -# - install_dib - -# Save trace setting -XTRACE=$(set +o | grep xtrace) -set +o xtrace - -# Defaults -# -------- - -# set up default directories -DIB_DIR=$DEST/diskimage-builder -TIE_DIR=$DEST/tripleo-image-elements - -# NOTE: Setting DIB_APT_SOURCES assumes you will be building -# Debian/Ubuntu based images. Leave unset for other flavors. -DIB_APT_SOURCES=${DIB_APT_SOURCES:-""} -DIB_BUILD_OFFLINE=$(trueorfalse False DIB_BUILD_OFFLINE) -DIB_IMAGE_CACHE=$DATA_DIR/diskimage-builder/image-create -DIB_PIP_REPO=$DATA_DIR/diskimage-builder/pip-repo -DIB_PIP_REPO_PORT=${DIB_PIP_REPO_PORT:-8899} - -OCC_DIR=$DEST/os-collect-config -ORC_DIR=$DEST/os-refresh-config -OAC_DIR=$DEST/os-apply-config - -# Functions -# --------- - -# install_dib() - Collect source and prepare -function install_dib { - pip_install diskimage-builder - - git_clone $TIE_REPO $TIE_DIR $TIE_BRANCH - git_clone $OCC_REPO $OCC_DIR $OCC_BRANCH - git_clone $ORC_REPO $ORC_DIR $ORC_BRANCH - git_clone $OAC_REPO $OAC_DIR $OAC_BRANCH - mkdir -p $DIB_IMAGE_CACHE -} - -# build_dib_pip_repo() - Builds a local pip repo from local projects -function build_dib_pip_repo { - local project_dirs=$1 - local projpath proj package - - rm -rf $DIB_PIP_REPO - mkdir -p $DIB_PIP_REPO - - echo "" > $DIB_PIP_REPO/index.html - for projpath in $project_dirs; do - proj=$(basename $projpath) - mkdir -p $DIB_PIP_REPO/$proj - pushd $projpath - rm -rf dist - python setup.py sdist - pushd dist - package=$(ls *) - mv $package $DIB_PIP_REPO/$proj/$package - popd - - echo "$package" > $DIB_PIP_REPO/$proj/index.html - echo "$proj
" >> $DIB_PIP_REPO/index.html - - popd - done - - echo "" >> $DIB_PIP_REPO/index.html - - local dib_pip_repo_apache_conf=$(apache_site_config_for dib_pip_repo) - - sudo cp $FILES/apache-dib-pip-repo.template $dib_pip_repo_apache_conf - sudo sed -e " - s|%DIB_PIP_REPO%|$DIB_PIP_REPO|g; - s|%DIB_PIP_REPO_PORT%|$DIB_PIP_REPO_PORT|g; - s|%APACHE_NAME%|$APACHE_NAME|g; - " -i $dib_pip_repo_apache_conf - enable_apache_site dib_pip_repo -} - -# disk_image_create_upload() - Creates and uploads a diskimage-builder built image -function disk_image_create_upload { - - local image_name=$1 - local image_elements=$2 - local elements_path=$3 - - local image_path=$TOP_DIR/files/$image_name.qcow2 - - # Include the apt-sources element in builds if we have an - # alternative sources.list specified. - if [ -n "$DIB_APT_SOURCES" ]; then - if [ ! -e "$DIB_APT_SOURCES" ]; then - die $LINENO "DIB_APT_SOURCES set but not found at $DIB_APT_SOURCES" - fi - local extra_elements="apt-sources" - fi - - # Set the local pip repo as the primary index mirror so the - # image is built with local packages - local pypi_mirror_url=http://$SERVICE_HOST:$DIB_PIP_REPO_PORT/ - local pypi_mirror_url_1 - - if [ -a $HOME/.pip/pip.conf ]; then - # Add the current pip.conf index-url as an extra-index-url - # in the image build - pypi_mirror_url_1=$(iniget $HOME/.pip/pip.conf global index-url) - else - # If no pip.conf, set upstream pypi as an extra mirror - # (this also sets the .pydistutils.cfg index-url) - pypi_mirror_url_1=http://pypi.python.org/simple - fi - - # The disk-image-create command to run - ELEMENTS_PATH=$elements_path \ - DIB_APT_SOURCES=$DIB_APT_SOURCES \ - DIB_OFFLINE=$DIB_BUILD_OFFLINE \ - PYPI_MIRROR_URL=$pypi_mirror_url \ - PYPI_MIRROR_URL_1=$pypi_mirror_url_1 \ - disk-image-create -a amd64 $image_elements ${extra_elements:-} \ - --image-cache $DIB_IMAGE_CACHE \ - -o $image_path - - local token=$(keystone token-get | grep ' id ' | get_field 2) - die_if_not_set $LINENO token "Keystone fail to get token" - - glance --os-auth-token $token --os-image-url http://$GLANCE_HOSTPORT \ - image-create --name $image_name --is-public True \ - --container-format=bare --disk-format qcow2 \ - < $image_path -} - -# Restore xtrace -$XTRACE - -# Tell emacs to use shell-script-mode -## Local variables: -## mode: shell-script -## End: diff --git a/lib/heat b/lib/heat index c1021639b4..a088e82886 100644 --- a/lib/heat +++ b/lib/heat @@ -8,9 +8,7 @@ # ENABLED_SERVICES+=,heat,h-api,h-api-cfn,h-api-cw,h-eng # Dependencies: -# -# - functions -# - dib (if HEAT_CREATE_TEST_IMAGE=True) +# (none) # stack.sh # --------- @@ -37,6 +35,13 @@ GITDIR["python-heatclient"]=$DEST/python-heatclient HEAT_DIR=$DEST/heat HEAT_CFNTOOLS_DIR=$DEST/heat-cfntools HEAT_TEMPLATES_REPO_DIR=$DEST/heat-templates +OCC_DIR=$DEST/os-collect-config +ORC_DIR=$DEST/os-refresh-config +OAC_DIR=$DEST/os-apply-config + +HEAT_PIP_REPO=$DATA_DIR/heat-pip-repo +HEAT_PIP_REPO_PORT=${HEAT_PIP_REPO_PORT:-8899} + HEAT_AUTH_CACHE_DIR=${HEAT_AUTH_CACHE_DIR:-/var/cache/heat} HEAT_STANDALONE=$(trueorfalse False HEAT_STANDALONE) HEAT_ENABLE_ADOPT_ABANDON=$(trueorfalse False HEAT_ENABLE_ADOPT_ABANDON) @@ -47,10 +52,6 @@ HEAT_TEMPLATES_DIR=$HEAT_CONF_DIR/templates HEAT_STACK_DOMAIN=$(trueorfalse True HEAT_STACK_DOMAIN) HEAT_API_HOST=${HEAT_API_HOST:-$HOST_IP} HEAT_API_PORT=${HEAT_API_PORT:-8004} -HEAT_FUNCTIONAL_IMAGE_ELEMENTS=${HEAT_FUNCTIONAL_IMAGE_ELEMENTS:-\ -vm fedora selinux-permissive pypi os-collect-config os-refresh-config \ -os-apply-config heat-cfntools heat-config heat-config-cfn-init \ -heat-config-puppet heat-config-script} # other default options @@ -296,22 +297,44 @@ function create_heat_accounts { fi } -# build_heat_functional_test_image() - Build and upload functional test image -function build_heat_functional_test_image { - if is_service_enabled dib; then - build_dib_pip_repo "$OCC_DIR $OAC_DIR $ORC_DIR $HEAT_CFNTOOLS_DIR" - local image_name=heat-functional-tests-image +# build_heat_pip_mirror() - Build a pip mirror containing heat agent projects +function build_heat_pip_mirror { + local project_dirs="$OCC_DIR $OAC_DIR $ORC_DIR $HEAT_CFNTOOLS_DIR" + local projpath proj package - # Elements path for tripleo-image-elements and heat-templates software-config - local elements_path=$TIE_DIR/elements:$HEAT_TEMPLATES_REPO_DIR/hot/software-config/elements + rm -rf $HEAT_PIP_REPO + mkdir -p $HEAT_PIP_REPO - disk_image_create_upload "$image_name" "$HEAT_FUNCTIONAL_IMAGE_ELEMENTS" "$elements_path" - iniset $TEMPEST_CONFIG orchestration image_ref $image_name - else - echo "Error, HEAT_CREATE_TEST_IMAGE=True requires dib" >&2 - echo "Add \"enable_service dib\" to your localrc" >&2 - exit 1 - fi + echo "" > $HEAT_PIP_REPO/index.html + for projpath in $project_dirs; do + proj=$(basename $projpath) + mkdir -p $HEAT_PIP_REPO/$proj + pushd $projpath + rm -rf dist + python setup.py sdist + pushd dist + package=$(ls *) + mv $package $HEAT_PIP_REPO/$proj/$package + popd + + echo "$package" > $HEAT_PIP_REPO/$proj/index.html + echo "$proj
" >> $HEAT_PIP_REPO/index.html + + popd + done + + echo "" >> $HEAT_PIP_REPO/index.html + + local heat_pip_repo_apache_conf=$(apache_site_config_for heat_pip_repo) + + sudo cp $FILES/apache-heat-pip-repo.template $heat_pip_repo_apache_conf + sudo sed -e " + s|%HEAT_PIP_REPO%|$HEAT_PIP_REPO|g; + s|%HEAT_PIP_REPO_PORT%|$HEAT_PIP_REPO_PORT|g; + s|%APACHE_NAME%|$APACHE_NAME|g; + " -i $heat_pip_repo_apache_conf + enable_apache_site heat_pip_repo + restart_apache_server } # Restore xtrace diff --git a/stack.sh b/stack.sh index 2ac7dfadeb..bf9fc0118e 100755 --- a/stack.sh +++ b/stack.sh @@ -1227,9 +1227,9 @@ if is_service_enabled heat; then init_heat echo_summary "Starting Heat" start_heat - if [ "$HEAT_CREATE_TEST_IMAGE" = "True" ]; then - echo_summary "Building Heat functional test image" - build_heat_functional_test_image + if [ "$HEAT_BUILD_PIP_MIRROR" = "True" ]; then + echo_summary "Building Heat pip mirror" + build_heat_pip_mirror fi fi diff --git a/stackrc b/stackrc index cb044b8fed..efea1e4560 100644 --- a/stackrc +++ b/stackrc @@ -419,14 +419,10 @@ GITBRANCH["ceilometermiddleware"]=${CEILOMETERMIDDLEWARE_BRANCH:-master} ################## # -# TripleO Components +# TripleO / Heat Agent Components # ################## -# diskimage-builder -DIB_REPO=${DIB_REPO:-${GIT_BASE}/openstack/diskimage-builder.git} -DIB_BRANCH=${DIB_BRANCH:-master} - # os-apply-config configuration template tool OAC_REPO=${OAC_REPO:-${GIT_BASE}/openstack/os-apply-config.git} OAC_BRANCH=${OAC_BRANCH:-master} @@ -439,10 +435,6 @@ OCC_BRANCH=${OCC_BRANCH:-master} ORC_REPO=${ORC_REPO:-${GIT_BASE}/openstack/os-refresh-config.git} ORC_BRANCH=${ORC_BRANCH:-master} -# Tripleo elements for diskimage-builder images -TIE_REPO=${TIE_REPO:-${GIT_BASE}/openstack/tripleo-image-elements.git} -TIE_BRANCH=${TIE_BRANCH:-master} - ################# # # 3rd Party Components (non pip installable)