Fix sources-branch-updater and add docs
Problems resolved: - Errors relating to 'etc' folders which did not exist in the target repositories have been eliminated. - Some repositories were not being excluded from evaluation for file/template updates. All appropriate exclusions are now implemented. - If an OSA role repository did not have the required folder to copy rootwrap files into, the copy would fail and the rootwrap files therefore be left out of the patch. - When executed on Mac OS X, python2 is not found. It also adds documentation for how to use the script to handle the periodic SHA bumps. Change-Id: I628cd2c3156d118c17ca2f90d4ccf0bc5a080bbf
This commit is contained in:
Jesse Pretorius
parent
6c7826ea88
commit
ec80a2d797
|
|
@ -209,3 +209,44 @@ OpenStack-CI through the following jobs:
|
|||
|
||||
While this script is primarily developed and maintained for use in
|
||||
OpenStack-CI, it can be used in other environments.
|
||||
|
||||
Dependency Updates
|
||||
^^^^^^^^^^^^^^^^^^
|
||||
|
||||
The dependencies for OpenStack-Ansible are updated approximately every two
|
||||
weeks through the use of ``scripts/sources-branch-updater.sh``. This script
|
||||
updates all pinned SHA's for OpenStack services, OpenStack-Ansible roles
|
||||
and other python dependencies which are not handles by the OpenStack global
|
||||
requirements management process. This script also handles the updating of
|
||||
the statically held templates/files in each role to ensure that they are
|
||||
always up to date. Finally, it also does a minor version increment of the
|
||||
value for ``openstack_release``.
|
||||
|
||||
The update script is used as follows:
|
||||
|
||||
.. parsed-literal::
|
||||
|
||||
# change directory to the openstack-ansible checkout
|
||||
cd ~/code/openstack-ansible
|
||||
|
||||
# create the local branch for the update
|
||||
git checkout -b sha-update
|
||||
|
||||
# execute the script for all openstack services
|
||||
./scripts/sources-branch-updater.sh -b stable/|current_release_branch_name| -o stable/|current_release_branch_name|
|
||||
|
||||
# execute the script for gnocchi
|
||||
./scripts/sources-branch-updater.sh -s playbooks/defaults/repo_packages/gnocchi.yml -b stable/3.1 -o stable/|current_release_branch_name|
|
||||
|
||||
# commit the changes
|
||||
new_version=$(awk '/openstack_release/ {print $2}' playbooks/inventory/group_vars/all.yml | head -n 1)
|
||||
git add --all
|
||||
git commit -a -m "Update all SHAs for ${new_version}" \
|
||||
-m "This patch updates all the roles to the latest available stable
|
||||
SHA's, copies the release notes from the updated roles into the
|
||||
integrated repo, updates all the OpenStack Service SHA's, and
|
||||
updates the appropriate python requirements pins.
|
||||
|
||||
# push the changes up to gerrit
|
||||
git review
|
||||
|
||||
|
|
|
|||
|
|
@ -1,4 +1,4 @@
|
|||
#!/usr/bin/env python2
|
||||
#!/usr/bin/env python2.7
|
||||
#
|
||||
# Copyright 2016, Rackspace US, Inc.
|
||||
#
|
||||
|
|
|
|||
|
|
@ -1,4 +1,4 @@
|
|||
#!/usr/bin/env python2
|
||||
#!/usr/bin/env python2.7
|
||||
#
|
||||
# Copyright 2016, Rackspace US, Inc.
|
||||
#
|
||||
|
|
|
|||
|
|
@ -1,4 +1,4 @@
|
|||
#!/usr/bin/env python2
|
||||
#!/usr/bin/env python2.7
|
||||
#
|
||||
# Copyright 2016, Rackspace US, Inc.
|
||||
#
|
||||
|
|
|
|||
|
|
@ -21,7 +21,7 @@
|
|||
OS_BRANCH=${OS_BRANCH:-"master"}
|
||||
OSA_BRANCH=${OSA_BRANCH:-"$OS_BRANCH"}
|
||||
SERVICE_FILE=${SERVICE_FILE:-"playbooks/defaults/repo_packages/openstack_services.yml"}
|
||||
OPENSTACK_SERVICE_LIST=${OPENSTACK_SERVICE_LIST:-"$(grep 'git_repo\:' ${SERVICE_FILE} | awk -F '/' '{ print $NF }' | egrep -v 'requirements|-' | tr '\n' ' ')"}
|
||||
OPENSTACK_SERVICE_LIST=${OPENSTACK_SERVICE_LIST:-""}
|
||||
PRE_RELEASE=${PRE_RELEASE:-"false"}
|
||||
|
||||
IFS=$'\n'
|
||||
|
|
@ -58,6 +58,47 @@ esac
|
|||
shift
|
||||
done
|
||||
|
||||
# Here we inspect the service file to compile the list of repositories
|
||||
# we're interested in inspecting for the purpose of doing in-repo updates
|
||||
# of static files that we template/copy when doing installs.
|
||||
#
|
||||
# If a predefined list is provided, skip all this.
|
||||
if [[ -z ${OPENSTACK_SERVICE_LIST} ]]; then
|
||||
# Setup an array of all the repositories in the
|
||||
# service file provided.
|
||||
OPENSTACK_REPO_LIST=( $(grep 'git_repo\:' ${SERVICE_FILE} | awk -F '/' '{ print $NF }') )
|
||||
|
||||
# Define the repositories to skip in an array.
|
||||
# These items are removed as they are not service projects
|
||||
# and therefore do not have policy/api-paste/etc files.
|
||||
OPENSTACK_REPO_SKIP_LIST=( requirements dragonflow swift3 )
|
||||
|
||||
# Define the skip regex for any additional items to remove.
|
||||
# Items with a '-' are removed as those repositories are
|
||||
# typically extensions/drivers/dashboards and therefore
|
||||
# do not include policy/api-paste/etc files.
|
||||
OPENSTACK_REPO_SKIP_REGEX='.*-.*'
|
||||
|
||||
# Loop through each item and if it does not match
|
||||
# an item in the SKIP_LIST or match the SKIP_REGEX
|
||||
# then add it to the OPENSTACK_SERVICE_LIST string.
|
||||
for item_to_check in "${OPENSTACK_REPO_LIST[@]}"; do
|
||||
add_item="yes"
|
||||
if [[ ! "${item_to_check}" =~ ${OPENSTACK_REPO_SKIP_REGEX} ]]; then
|
||||
for item_to_delete in "${OPENSTACK_REPO_SKIP_LIST[@]}"; do
|
||||
if [[ "${item_to_delete}" == "${item_to_check}" ]]; then
|
||||
add_item="no"
|
||||
fi
|
||||
done
|
||||
else
|
||||
add_item="no"
|
||||
fi
|
||||
if [[ "${add_item}" == "yes" ]]; then
|
||||
OPENSTACK_SERVICE_LIST="${OPENSTACK_SERVICE_LIST} ${item_to_check}"
|
||||
fi
|
||||
done
|
||||
fi
|
||||
|
||||
# Iterate through the service file
|
||||
for repo in $(grep 'git_repo\:' ${SERVICE_FILE}); do
|
||||
|
||||
|
|
@ -86,6 +127,7 @@ for repo in $(grep 'git_repo\:' ${SERVICE_FILE}); do
|
|||
|
||||
# If the repo is in the specified list, then action the additional updates
|
||||
if [[ "${OPENSTACK_SERVICE_LIST}" =~ "${repo_name}" ]]; then
|
||||
|
||||
os_repo_tmp_path="/tmp/os_${repo_name}"
|
||||
osa_repo_tmp_path="/tmp/osa_${repo_name}"
|
||||
|
||||
|
|
@ -107,50 +149,71 @@ for repo in $(grep 'git_repo\:' ${SERVICE_FILE}); do
|
|||
git checkout --quiet origin/${OSA_BRANCH}
|
||||
popd > /dev/null
|
||||
|
||||
# Update the policy files
|
||||
if [ "${repo_name}" != "gnocchi" ] && [ "${repo_name}" != "ceilometer" ]; then
|
||||
find ${os_repo_tmp_path}/etc -name "policy.json" -exec \
|
||||
cp {} "${osa_repo_tmp_path}/templates/policy.json.j2" \;
|
||||
fi
|
||||
# We have implemented tooling to dynamically fetch the
|
||||
# api-paste and other static/template files from these
|
||||
# repositories, so skip trying to update their templates
|
||||
# and static files.
|
||||
static_file_repo_skip_list=( ceilometer gnocchi )
|
||||
|
||||
# Tweak the paste files for any hmac key entries
|
||||
find ${os_repo_tmp_path}/etc -name "*[_-]paste.ini" -exec \
|
||||
sed -i.bak "s|hmac_keys = SECRET_KEY|hmac_keys = {{ ${repo_name}_profiler_hmac_key }}|" {} \;
|
||||
# Check if this repo is in the static file skip list
|
||||
skip_this_repo="no"
|
||||
for skip_list_item in "${static_file_repo_skip_list[@]}"; do
|
||||
if [[ "${repo_name}" == "${skip_list_item}" ]]; then
|
||||
skip_this_repo="yes"
|
||||
fi
|
||||
done
|
||||
|
||||
# Tweak the barbican paste file to support keystone auth
|
||||
if [ "${repo_name}" = "barbican" ]; then
|
||||
if [[ "${skip_this_repo}" != "yes" ]] && [[ -e "${os_repo_tmp_path}/etc" ]]; then
|
||||
|
||||
# Update the policy files
|
||||
if [ "${repo_name}" != "gnocchi" ] && [ "${repo_name}" != "ceilometer" ]; then
|
||||
find ${os_repo_tmp_path}/etc -name "policy.json" -exec \
|
||||
cp {} "${osa_repo_tmp_path}/templates/policy.json.j2" \;
|
||||
fi
|
||||
|
||||
# Tweak the paste files for any hmac key entries
|
||||
find ${os_repo_tmp_path}/etc -name "*[_-]paste.ini" -exec \
|
||||
sed -i.bak 's|\/v1\: barbican-api-keystone|\/v1\: {{ (barbican_keystone_auth \| bool) \| ternary('barbican-api-keystone', 'barbican_api') }}|'{} \;
|
||||
fi
|
||||
sed -i.bak "s|hmac_keys = SECRET_KEY|hmac_keys = {{ ${repo_name}_profiler_hmac_key }}|" {} \;
|
||||
|
||||
# Tweak the barbican paste file to support keystone auth
|
||||
if [ "${repo_name}" = "barbican" ]; then
|
||||
find ${os_repo_tmp_path}/etc -name "*[_-]paste.ini" -exec \
|
||||
sed -i.bak "s|\/v1\: barbican-api-keystone|\/v1\: {{ (barbican_keystone_auth \| bool) \| ternary('barbican-api-keystone', 'barbican_api') }}|" {} \;
|
||||
fi
|
||||
|
||||
if [ "${repo_name}" != "gnocchi" ] && [ "${repo_name}" != "ceilometer" ]; then
|
||||
# Update the paste files
|
||||
find ${os_repo_tmp_path}/etc -name "*[_-]paste.ini" -exec \
|
||||
bash -c "name=\"{}\"; cp \${name} \"${osa_repo_tmp_path}/templates/\$(basename \${name}).j2\"" \;
|
||||
|
||||
# Update the yaml files for Heat
|
||||
if [ "${repo_name}" = "heat" ]; then
|
||||
find ${os_repo_tmp_path}/etc -name "*.yaml" -exec \
|
||||
bash -c "name=\"{}\"; cp \${name} \"${osa_repo_tmp_path}/templates/\$(echo \${name} | rev | cut -sd / -f -2 | rev).j2\"" \;
|
||||
fi
|
||||
fi
|
||||
|
||||
# Tweak the rootwrap conf filters_path (for neutron only)
|
||||
if [ "${repo_name}" = "neutron" ]; then
|
||||
# We have to check for rootwrap files in *all* service repositories
|
||||
# as we have no dynamic way of fetching them at this stage.
|
||||
if [[ -e "${os_repo_tmp_path}/etc" ]]; then
|
||||
|
||||
# Tweak the rootwrap conf filters_path (for neutron only)
|
||||
if [ "${repo_name}" = "neutron" ]; then
|
||||
find ${os_repo_tmp_path}/etc -name "rootwrap.conf" -exec \
|
||||
sed -i.bak "s|filters_path=/etc/neutron|filters_path={{ ${repo_name}_conf_dir }}|" {} \;
|
||||
fi
|
||||
|
||||
# Tweak the rootwrap conf exec_dirs
|
||||
find ${os_repo_tmp_path}/etc -name "rootwrap.conf" -exec \
|
||||
sed -i.bak "s|filters_path=/etc/neutron|filters_path={{ ${repo_name}_conf_dir }}|" {} \;
|
||||
fi
|
||||
sed -i.bak "s|exec_dirs=|exec_dirs={{ ${repo_name}_bin }},|" {} \;
|
||||
|
||||
# Tweak the rootwrap conf exec_dirs
|
||||
find ${os_repo_tmp_path}/etc -name "rootwrap.conf" -exec \
|
||||
sed -i.bak "s|exec_dirs=|exec_dirs={{ ${repo_name}_bin }},|" {} \;
|
||||
# Update the rootwrap conf files
|
||||
find ${os_repo_tmp_path}/etc -name "rootwrap.conf" -exec \
|
||||
cp {} "${osa_repo_tmp_path}/templates/rootwrap.conf.j2" \;
|
||||
|
||||
# Update the rootwrap conf files
|
||||
find ${os_repo_tmp_path}/etc -name "rootwrap.conf" -exec \
|
||||
cp {} "${osa_repo_tmp_path}/templates/rootwrap.conf.j2" \;
|
||||
|
||||
# Update the rootwrap filters
|
||||
find ${os_repo_tmp_path}/etc -name "*.filters" -exec \
|
||||
bash -c "name=\"{}\"; cp \${name} \"${osa_repo_tmp_path}/files/rootwrap.d/\$(basename \${name})\"" \;
|
||||
|
||||
# Update the yaml files for Heat
|
||||
if [ "${repo_name}" = "heat" ]; then
|
||||
find ${os_repo_tmp_path}/etc -name "*.yaml" -exec \
|
||||
bash -c "name=\"{}\"; cp \${name} \"${osa_repo_tmp_path}/templates/\$(echo \${name} | rev | cut -sd / -f -2 | rev).j2\"" \;
|
||||
# Update the rootwrap filters
|
||||
mkdir -p ${osa_repo_tmp_path}/files/rootwrap.d
|
||||
find ${os_repo_tmp_path}/etc -name "*.filters" -exec \
|
||||
bash -c "name=\"{}\"; cp \${name} \"${osa_repo_tmp_path}/files/rootwrap.d/\$(basename \${name})\"" \;
|
||||
fi
|
||||
|
||||
# Switch into the OSA git directory to work with it
|
||||
|
|
|
|||
Loading…
Reference in New Issue