Fix sources-branch-updater and add docs
Problems resolved: - Errors relating to 'etc' folders which did not exist in the target repositories have been eliminated. - Some repositories were not being excluded from evaluation for file/template updates. All appropriate exclusions are now implemented. - If an OSA role repository did not have the required folder to copy rootwrap files into, the copy would fail and the rootwrap files therefore be left out of the patch. - When executed on Mac OS X, python2 is not found. - Documentation is added to describe how to use the script to handle periodic SHA bumps. Change-Id: I628cd2c3156d118c17ca2f90d4ccf0bc5a080bbf
This commit is contained in:
Jesse Pretorius
parent
a9f5b9f84e
commit
e8abf19647
|
|
@ -323,10 +323,12 @@ if watermark == "":
|
|||
deploy_branch_link_name = "latest"
|
||||
dev_branch_link_name = ""
|
||||
current_release_git_branch_name = "master"
|
||||
current_release_gnocchi_git_branch_name = "master"
|
||||
else:
|
||||
deploy_branch_link_name = current_release_branch_name
|
||||
dev_branch_link_name = "{}/".format(current_release_branch_name)
|
||||
current_release_git_branch_name = 'stable/' + current_release_branch_name
|
||||
current_release_gnocchi_git_branch_name = "4.1"
|
||||
|
||||
previous_release_capital_name = previous_release_branch_name.upper()
|
||||
previous_release_formal_name = previous_release_branch_name.capitalize()
|
||||
|
|
@ -342,6 +344,7 @@ rst_epilog = """
|
|||
.. |previous_release_branch_name| replace:: %s
|
||||
.. |current_release_branch_name| replace:: %s
|
||||
.. |current_release_git_branch_name| replace:: %s
|
||||
.. |current_release_gnocchi_git_branch_name| replace:: %s
|
||||
.. |previous_release_capital_name| replace:: %s
|
||||
.. |previous_release_formal_name| replace:: %s
|
||||
.. |current_release_capital_name| replace:: %s
|
||||
|
|
@ -351,6 +354,7 @@ rst_epilog = """
|
|||
""" % (previous_release_branch_name,
|
||||
current_release_branch_name,
|
||||
current_release_git_branch_name,
|
||||
current_release_gnocchi_git_branch_name,
|
||||
previous_release_capital_name,
|
||||
previous_release_formal_name,
|
||||
current_release_capital_name,
|
||||
|
|
|
|||
|
|
@ -16,7 +16,7 @@ Bootstrapping
|
|||
bootstrap-ansible.sh
|
||||
--------------------
|
||||
|
||||
The ``bootstrap-ansible.sh`` script installs Ansible including `core`_ and
|
||||
The ``bootstrap-ansible.sh`` script installs Ansible, including the `core`_ and
|
||||
`extras`_ module repositories and Galaxy roles.
|
||||
|
||||
While there are several configurable environment variables which this script
|
||||
|
|
@ -189,3 +189,57 @@ OpenStack-CI through the following jobs:
|
|||
|
||||
While this script is primarily developed and maintained for use in
|
||||
OpenStack-CI, it can be used in other environments.
|
||||
|
||||
Dependency Updates
|
||||
^^^^^^^^^^^^^^^^^^
|
||||
|
||||
The dependencies for OpenStack-Ansible are updated approximately every two
|
||||
weeks through the use of ``scripts/sources-branch-updater.sh``. This script
|
||||
updates all pinned SHA's for OpenStack services, OpenStack-Ansible roles,
|
||||
and other python dependencies which are not handled by the OpenStack global
|
||||
requirements management process. This script also updates the statically
|
||||
held templates/files in each role to ensure that they are always up to date.
|
||||
Finally, it also does a minor version increment of the value for
|
||||
``openstack_release``.
|
||||
|
||||
The update script is used as follows:
|
||||
|
||||
.. parsed-literal::
|
||||
|
||||
# change directory to the openstack-ansible checkout
|
||||
cd ~/code/openstack-ansible
|
||||
|
||||
# ensure that the correct branch is checked out
|
||||
git checkout |current_release_git_branch_name|
|
||||
|
||||
# ensure that the branch is up to date
|
||||
git pull
|
||||
|
||||
# create the local branch for the update
|
||||
git checkout -b sha-update
|
||||
|
||||
# execute the script for all openstack services
|
||||
./scripts/sources-branch-updater.sh -b |current_release_git_branch_name| -o |current_release_git_branch_name|
|
||||
|
||||
# execute the script for gnocchi
|
||||
./scripts/sources-branch-updater.sh -s playbooks/defaults/repo_packages/gnocchi.yml -b |current_release_gnocchi_git_branch_name| -o |current_release_git_branch_name|
|
||||
|
||||
# the console code should only be updated when necessary for a security fix, or for the OSA master branch
|
||||
./scripts/sources-branch-updater.sh -s playbooks/defaults/repo_packages/nova_consoles.yml -b master
|
||||
|
||||
# the testing repositories should not be updated for stable branches as the new tests
|
||||
# or other changes introduced may not work for older branches
|
||||
./scripts/sources-branch-updater.sh -s playbooks/defaults/repo_packages/openstack_testing.yml -b master
|
||||
|
||||
# commit the changes
|
||||
new_version=$(awk '/^openstack_release/ {print $2}' inventory/group_vars/all/all.yml)
|
||||
git add --all
|
||||
git commit -a -m "Update all SHAs for ${new_version}" \
|
||||
-m "This patch updates all the roles to the latest available stable
|
||||
SHA's, copies the release notes from the updated roles into the
|
||||
integrated repo, updates all the OpenStack Service SHA's, and
|
||||
updates the appropriate python requirements pins.
|
||||
|
||||
# push the changes up to gerrit
|
||||
git review
|
||||
|
||||
|
|
|
|||
|
|
@ -1,4 +1,4 @@
|
|||
#!/usr/bin/env python2
|
||||
#!/usr/bin/env python2.7
|
||||
#
|
||||
# Copyright 2016, Rackspace US, Inc.
|
||||
#
|
||||
|
|
|
|||
|
|
@ -1,4 +1,4 @@
|
|||
#!/usr/bin/env python2
|
||||
#!/usr/bin/env python2.7
|
||||
#
|
||||
# Copyright 2016, Rackspace US, Inc.
|
||||
#
|
||||
|
|
|
|||
|
|
@ -1,4 +1,4 @@
|
|||
#!/usr/bin/env python2
|
||||
#!/usr/bin/env python2.7
|
||||
#
|
||||
# Copyright 2016, Rackspace US, Inc.
|
||||
#
|
||||
|
|
|
|||
|
|
@ -119,58 +119,75 @@ sync_roles_and_packages() {
|
|||
# pre-sync user hook
|
||||
osa_pre_sync_hook ${repo_name} ${os_branch} ${osa_branch} ${repo_address}
|
||||
|
||||
# Update the policy files
|
||||
find ${os_repo_tmp_path}/etc -name "policy.json" -exec \
|
||||
cp {} "${osa_repo_tmp_path}/templates/policy.json.j2" \;
|
||||
# We have implemented tooling to dynamically fetch the
|
||||
# api-paste and other static/template files from these
|
||||
# repositories, so skip trying to update their templates
|
||||
# and static files.
|
||||
local static_file_repo_skip_list=( ceilometer gnocchi keystone )
|
||||
|
||||
# Tweak the paste files for any hmac key entries
|
||||
find ${os_repo_tmp_path}/etc -name "*[_-]paste.ini" -exec \
|
||||
sed -i.bak "s|hmac_keys = SECRET_KEY|hmac_keys = {{ ${repo_name}_profiler_hmac_key }}|" {} \;
|
||||
# Check if this repo is in the static file skip list
|
||||
local skip_this_repo="no"
|
||||
for skip_list_item in "${static_file_repo_skip_list[@]}"; do
|
||||
if [[ "${repo_name}" == "${skip_list_item}" ]]; then
|
||||
skip_this_repo="yes"
|
||||
fi
|
||||
done
|
||||
|
||||
# Tweak the barbican paste file to support keystone auth
|
||||
if [ "${repo_name}" = "barbican" ]; then
|
||||
if [[ "${skip_this_repo}" != "yes" ]] && [[ -e "${os_repo_tmp_path}/etc" ]]; then
|
||||
# Update the policy files
|
||||
find ${os_repo_tmp_path}/etc -name "policy.json" -exec \
|
||||
cp {} "${osa_repo_tmp_path}/templates/policy.json.j2" \;
|
||||
|
||||
# Tweak the paste files for any hmac key entries
|
||||
find ${os_repo_tmp_path}/etc -name "*[_-]paste.ini" -exec \
|
||||
sed -i.bak "s|\/v1\: barbican-api-keystone|\/v1\: {{ (barbican_keystone_auth \| bool) \| ternary('barbican-api-keystone', 'barbican_api') }}|" {} \;
|
||||
fi
|
||||
sed -i.bak "s|hmac_keys = SECRET_KEY|hmac_keys = {{ ${repo_name}_profiler_hmac_key }}|" {} \;
|
||||
|
||||
# Tweak the gnocchi paste file to support keystone auth
|
||||
if [ "${repo_name}" = "gnocchi" ]; then
|
||||
# Tweak the barbican paste file to support keystone auth
|
||||
if [[ "${repo_name}" == "barbican" ]]; then
|
||||
find ${os_repo_tmp_path}/etc -name "*[_-]paste.ini" -exec \
|
||||
sed -i.bak "s|\/v1\: barbican-api-keystone|\/v1\: {{ (barbican_keystone_auth \| bool) \| ternary('barbican-api-keystone', 'barbican_api') }}|" {} \;
|
||||
fi
|
||||
|
||||
# Tweak the gnocchi paste file to support keystone auth
|
||||
if [[ "${repo_name}" == "gnocchi" ]]; then
|
||||
find ${os_repo_tmp_path}/etc -name "*[_-]paste.ini" -exec \
|
||||
sed -i.bak "s|pipeline = gnocchi+noauth|pipeline = {{ (gnocchi_keystone_auth \| bool) \| ternary('gnocchi+auth', 'gnocchi+noauth') }}|" {} \;
|
||||
fi
|
||||
|
||||
# Update the paste files
|
||||
find ${os_repo_tmp_path}/etc -name "*[_-]paste.ini" -exec \
|
||||
sed -i.bak "s|pipeline = gnocchi+noauth|pipeline = {{ (gnocchi_keystone_auth \| bool) \| ternary('gnocchi+auth', 'gnocchi+noauth') }}|" {} \;
|
||||
fi
|
||||
|
||||
# Update the paste files
|
||||
find ${os_repo_tmp_path}/etc -name "*[_-]paste.ini" -exec \
|
||||
bash -c "name=\"{}\"; cp \${name} \"${osa_repo_tmp_path}/templates/\$(basename \${name}).j2\"" \;
|
||||
|
||||
# Tweak the rootwrap conf filters_path (for neutron only)
|
||||
if [ "${repo_name}" = "neutron" ]; then
|
||||
find ${os_repo_tmp_path}/etc -name "rootwrap.conf" -exec \
|
||||
sed -i.bak "s|filters_path=/etc/neutron|filters_path={{ ${repo_name}_conf_dir }}|" {} \;
|
||||
fi
|
||||
|
||||
# Tweak the rootwrap conf exec_dirs
|
||||
find ${os_repo_tmp_path}/etc -name "rootwrap.conf" -exec \
|
||||
sed -i.bak "s|exec_dirs=|exec_dirs={{ ${repo_name}_bin }},|" {} \;
|
||||
|
||||
# Update the rootwrap conf files
|
||||
find ${os_repo_tmp_path}/etc -name "rootwrap.conf" -exec \
|
||||
cp {} "${osa_repo_tmp_path}/templates/rootwrap.conf.j2" \;
|
||||
|
||||
# Update the rootwrap filters
|
||||
find ${os_repo_tmp_path}/etc -name "*.filters" -exec \
|
||||
bash -c "name=\"{}\"; cp \${name} \"${osa_repo_tmp_path}/files/rootwrap.d/\$(basename \${name})\"" \;
|
||||
|
||||
# Update the yaml files for Ceilometer
|
||||
if [ "${repo_name}" = "ceilometer" ]; then
|
||||
find ${os_repo_tmp_path}/etc -name "*.yaml" -exec \
|
||||
bash -c "name=\"{}\"; cp \${name} \"${osa_repo_tmp_path}/templates/\$(basename \${name}).j2\"" \;
|
||||
|
||||
# Update the yaml files for Heat
|
||||
if [[ "${repo_name}" == "heat" ]]; then
|
||||
find ${os_repo_tmp_path}/etc -name "*.yaml" -exec \
|
||||
bash -c "name=\"{}\"; cp \${name} \"${osa_repo_tmp_path}/templates/\$(echo \${name} | rev | cut -sd / -f -2 | rev).j2\"" \;
|
||||
fi
|
||||
fi
|
||||
|
||||
# Update the yaml files for Heat
|
||||
if [ "${repo_name}" = "heat" ]; then
|
||||
find ${os_repo_tmp_path}/etc -name "*.yaml" -exec \
|
||||
bash -c "name=\"{}\"; cp \${name} \"${osa_repo_tmp_path}/templates/\$(echo \${name} | rev | cut -sd / -f -2 | rev).j2\"" \;
|
||||
# We have to check for rootwrap files in *all* service repositories
|
||||
# as we have no dynamic way of fetching them at this stage.
|
||||
if [[ -e "${os_repo_tmp_path}/etc" ]]; then
|
||||
|
||||
# Tweak the rootwrap conf filters_path (for neutron only)
|
||||
if [[ "${repo_name}" == "neutron" ]]; then
|
||||
find ${os_repo_tmp_path}/etc -name "rootwrap.conf" -exec \
|
||||
sed -i.bak "s|filters_path=/etc/neutron|filters_path={{ ${repo_name}_conf_dir }}|" {} \;
|
||||
fi
|
||||
|
||||
# Tweak the rootwrap conf exec_dirs
|
||||
find ${os_repo_tmp_path}/etc -name "rootwrap.conf" -exec \
|
||||
sed -i.bak "s|exec_dirs=|exec_dirs={{ ${repo_name}_bin }},|" {} \;
|
||||
|
||||
# Update the rootwrap conf files
|
||||
find ${os_repo_tmp_path}/etc -name "rootwrap.conf" -exec \
|
||||
cp {} "${osa_repo_tmp_path}/templates/rootwrap.conf.j2" \;
|
||||
|
||||
# Update the rootwrap filters
|
||||
mkdir -p ${osa_repo_tmp_path}/files/rootwrap.d
|
||||
find ${os_repo_tmp_path}/etc -name "*.filters" -exec \
|
||||
bash -c "name=\"{}\"; cp \${name} \"${osa_repo_tmp_path}/files/rootwrap.d/\$(basename \${name})\"" \;
|
||||
|
||||
fi
|
||||
|
||||
# post-sync user hook
|
||||
|
|
|
|||
|
|
@ -22,10 +22,51 @@
|
|||
OS_BRANCH=${OS_BRANCH:-"master"}
|
||||
OSA_BRANCH=${OSA_BRANCH:-"$OS_BRANCH"}
|
||||
SERVICE_FILE=${SERVICE_FILE:-"playbooks/defaults/repo_packages/openstack_services.yml"}
|
||||
OPENSTACK_SERVICE_LIST=${OPENSTACK_SERVICE_LIST:-"$(grep 'git_repo\:' ${SERVICE_FILE} | awk -F '/' '{ print $NF }' | egrep -v 'requirements|-' | tr '\n' ' ')"}
|
||||
OPENSTACK_SERVICE_LIST=${OPENSTACK_SERVICE_LIST:-""}
|
||||
PRE_RELEASE=${PRE_RELEASE:-"false"}
|
||||
FORCE_MASTER=${FORCE_MASTER:-"false"}
|
||||
|
||||
# Here we inspect the service file to compile the list of repositories
|
||||
# we're interested in inspecting for the purpose of doing in-repo updates
|
||||
# of static files that we template/copy when doing installs.
|
||||
#
|
||||
# If a predefined list is provided, skip all this.
|
||||
if [[ -z ${OPENSTACK_SERVICE_LIST} ]]; then
|
||||
# Setup an array of all the repositories in the
|
||||
# service file provided.
|
||||
OPENSTACK_REPO_LIST=( $(grep 'git_repo\:' ${SERVICE_FILE} | awk -F '/' '{ print $NF }') )
|
||||
|
||||
# Define the repositories to skip in an array.
|
||||
# These items are removed as they are not service projects
|
||||
# and therefore do not have policy/api-paste/etc files.
|
||||
OPENSTACK_REPO_SKIP_LIST=( requirements dragonflow swift3 )
|
||||
|
||||
# Define the skip regex for any additional items to remove.
|
||||
# Items with a '-' are removed as those repositories are
|
||||
# typically extensions/drivers/dashboards and therefore
|
||||
# do not include policy/api-paste/etc files.
|
||||
OPENSTACK_REPO_SKIP_REGEX='.*-.*'
|
||||
|
||||
# Loop through each item and if it does not match
|
||||
# an item in the SKIP_LIST or match the SKIP_REGEX
|
||||
# then add it to the OPENSTACK_SERVICE_LIST string.
|
||||
for item_to_check in "${OPENSTACK_REPO_LIST[@]}"; do
|
||||
add_item="yes"
|
||||
if [[ ! "${item_to_check}" =~ ${OPENSTACK_REPO_SKIP_REGEX} ]]; then
|
||||
for item_to_delete in "${OPENSTACK_REPO_SKIP_LIST[@]}"; do
|
||||
if [[ "${item_to_delete}" == "${item_to_check}" ]]; then
|
||||
add_item="no"
|
||||
fi
|
||||
done
|
||||
else
|
||||
add_item="no"
|
||||
fi
|
||||
if [[ "${add_item}" == "yes" ]]; then
|
||||
OPENSTACK_SERVICE_LIST="${OPENSTACK_SERVICE_LIST} ${item_to_check}"
|
||||
fi
|
||||
done
|
||||
fi
|
||||
|
||||
source scripts/sources-branch-updater-lib.sh || { echo "Failed to source updater library"; exit 1; }
|
||||
|
||||
if echo "$@" | grep -e '-h' -e '--help';then
|
||||
|
|
|
|||
Loading…
Reference in New Issue