f40ecdeba2
This patch updates all the roles to the latest available stable SHA's, copies the release notes from the updated roles into the integrated repo, updates all the OpenStack Service SHA's, and updates the appropriate python requirements pins. Some adjustments are made to the sources-branch-updater script for improved error detection and efficiency. Also, based on https://review.openstack.org/371999 the ternary search/replace is fixed. Tempest and Rally repo sources are co-located in the repo_packages files as neither of them implement stable/* branches. The nova consoles (novnc/spice) are implemented in their own repo_packages file in order to simplify doing the SHA updates. Gnocchi implements a different style of releasing stable branches so it is also implemented in its own repo_packages file. Change-Id: I6fa4afab4203c9902ce70fa6bc49ad1dddb560b3
287 lines
11 KiB
Bash
Executable File
287 lines
11 KiB
Bash
Executable File
#!/usr/bin/env bash
|
|
# Copyright 2015, Rackspace US, Inc.
|
|
#
|
|
# Licensed under the Apache License, Version 2.0 (the "License");
|
|
# you may not use this file except in compliance with the License.
|
|
# You may obtain a copy of the License at
|
|
#
|
|
# http://www.apache.org/licenses/LICENSE-2.0
|
|
#
|
|
# Unless required by applicable law or agreed to in writing, software
|
|
# distributed under the License is distributed on an "AS IS" BASIS,
|
|
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
# See the License for the specific language governing permissions and
|
|
# limitations under the License.
|
|
|
|
# This script was created to rapidly interate through a repo_package file
|
|
# that contains git sources and set the various repositories inside to
|
|
# the head of given branch via the SHA. This makes it possible to update
|
|
# all of the services that we support in an "automated" fashion.
|
|
|
|
OS_BRANCH=${OS_BRANCH:-"master"}
|
|
OSA_BRANCH=${OSA_BRANCH:-"$OS_BRANCH"}
|
|
SERVICE_FILE=${SERVICE_FILE:-"playbooks/defaults/repo_packages/openstack_services.yml"}
|
|
OPENSTACK_SERVICE_LIST=${OPENSTACK_SERVICE_LIST:-"$(grep 'git_repo\:' ${SERVICE_FILE} | awk -F '/' '{ print $NF }' | egrep -v 'requirements|-' | tr '\n' ' ')"}
|
|
PRE_RELEASE=${PRE_RELEASE:-"false"}
|
|
|
|
IFS=$'\n'
|
|
|
|
if echo "$@" | grep -e '-h' -e '--help';then
|
|
echo "
|
|
Options:
|
|
-b|--openstack-branch (name of OpenStack branch, eg: stable/mitaka)
|
|
-o|--osa-branch (name of the OSA branch, eg: stable/mitaka)
|
|
-s|--service-file (path to service file to parse)
|
|
"
|
|
exit 0
|
|
fi
|
|
|
|
# Provide some CLI options
|
|
while [[ $# > 1 ]]; do
|
|
key="$1"
|
|
case $key in
|
|
-b|--openstack-branch)
|
|
OS_BRANCH="$2"
|
|
shift
|
|
;;
|
|
-o|--osa-branch)
|
|
OSA_BRANCH="$2"
|
|
shift
|
|
;;
|
|
-s|--service-file)
|
|
SERVICE_FILE="$2"
|
|
shift
|
|
;;
|
|
*)
|
|
;;
|
|
esac
|
|
shift
|
|
done
|
|
|
|
# Iterate through the service file
|
|
for repo in $(grep 'git_repo\:' ${SERVICE_FILE}); do
|
|
|
|
echo -e "\nInspecting ${repo}..."
|
|
|
|
# Set the repo name
|
|
repo_name=$(echo "${repo}" | sed 's/_git_repo\:.*//g')
|
|
|
|
# Set the repo address
|
|
repo_address=$(echo ${repo} | awk '{print $2}')
|
|
|
|
# Get the branch data
|
|
branch_data=$(git ls-remote ${repo_address} | grep "${OS_BRANCH}$")
|
|
|
|
# If there is branch data continue
|
|
if [ ! -z "${branch_data}" ];then
|
|
|
|
# Set the branch sha for the head of the branch
|
|
branch_sha=$(echo "${branch_data}" | awk '{print $1}')
|
|
|
|
# Set the branch entry
|
|
branch_entry="${branch_sha} # HEAD of \"$OS_BRANCH\" as of $(date +%d.%m.%Y)"
|
|
|
|
# Write the branch entry into the repo_packages file
|
|
sed -i.bak "s|${repo_name}_git_install_branch:.*|${repo_name}_git_install_branch: $branch_entry|" ${SERVICE_FILE}
|
|
|
|
# If the repo is in the specified list, then action the additional updates
|
|
if [[ "${OPENSTACK_SERVICE_LIST}" =~ "${repo_name}" ]]; then
|
|
os_repo_tmp_path="/tmp/os_${repo_name}"
|
|
osa_repo_tmp_path="/tmp/osa_${repo_name}"
|
|
|
|
# Ensure that the temp path doesn't exist
|
|
rm -rf ${os_repo_tmp_path} ${osa_repo_tmp_path}
|
|
|
|
# Do a shallow clone of the OpenStack repo to work with
|
|
if git clone --quiet --depth=10 --branch ${OS_BRANCH} --no-checkout --single-branch ${repo_address} ${os_repo_tmp_path}; then
|
|
pushd ${os_repo_tmp_path} > /dev/null
|
|
git checkout --quiet ${branch_sha}
|
|
popd > /dev/null
|
|
|
|
# Set the OSA address
|
|
osa_repo_address="https://git.openstack.org/openstack/openstack-ansible-os_${repo_name}"
|
|
|
|
# Do a shallow clone of the OSA repo to work with
|
|
if git clone --quiet --depth=10 --branch ${OSA_BRANCH} --single-branch ${osa_repo_address} ${osa_repo_tmp_path}; then
|
|
pushd ${osa_repo_tmp_path} > /dev/null
|
|
git checkout --quiet origin/${OSA_BRANCH}
|
|
popd > /dev/null
|
|
|
|
# Update the policy files
|
|
find ${os_repo_tmp_path}/etc -name "policy.json" -exec \
|
|
cp {} "${osa_repo_tmp_path}/templates/policy.json.j2" \;
|
|
|
|
# Tweak the paste files for any hmac key entries
|
|
find ${os_repo_tmp_path}/etc -name "*[_-]paste.ini" -exec \
|
|
sed -i.bak "s|hmac_keys = SECRET_KEY|hmac_keys = {{ ${repo_name}_profiler_hmac_key }}|" {} \;
|
|
|
|
# Tweak the gnocchi paste file to support keystone auth
|
|
if [ "${repo_name}" = "gnocchi" ]; then
|
|
find ${os_repo_tmp_path}/etc -name "*[_-]paste.ini" -exec \
|
|
sed -i.bak "s|pipeline = gnocchi+noauth|pipeline = {{ (gnocchi_keystone_auth \| bool) \| ternary('gnocchi+auth', 'gnocchi+noauth') }}|" {} \;
|
|
fi
|
|
|
|
# Update the paste files
|
|
find ${os_repo_tmp_path}/etc -name "*[_-]paste.ini" -exec \
|
|
bash -c "name=\"{}\"; cp \${name} \"${osa_repo_tmp_path}/templates/\$(basename \${name}).j2\"" \;
|
|
|
|
# Tweak the rootwrap conf filters_path (for neutron only)
|
|
if [ "${repo_name}" = "neutron" ]; then
|
|
find ${os_repo_tmp_path}/etc -name "rootwrap.conf" -exec \
|
|
sed -i.bak "s|filters_path=/etc/neutron|filters_path={{ ${repo_name}_conf_dir }}|" {} \;
|
|
fi
|
|
|
|
# Tweak the rootwrap conf exec_dirs
|
|
find ${os_repo_tmp_path}/etc -name "rootwrap.conf" -exec \
|
|
sed -i.bak "s|exec_dirs=|exec_dirs={{ ${repo_name}_bin }},|" {} \;
|
|
|
|
# Update the rootwrap conf files
|
|
find ${os_repo_tmp_path}/etc -name "rootwrap.conf" -exec \
|
|
cp {} "${osa_repo_tmp_path}/templates/rootwrap.conf.j2" \;
|
|
|
|
# Update the rootwrap filters
|
|
find ${os_repo_tmp_path}/etc -name "*.filters" -exec \
|
|
bash -c "name=\"{}\"; cp \${name} \"${osa_repo_tmp_path}/files/rootwrap.d/\$(basename \${name})\"" \;
|
|
|
|
# Update the yaml files for Ceilometer
|
|
if [ "${repo_name}" = "ceilometer" ]; then
|
|
find ${os_repo_tmp_path}/etc -name "*.yaml" -exec \
|
|
bash -c "name=\"{}\"; cp \${name} \"${osa_repo_tmp_path}/templates/\$(basename \${name}).j2\"" \;
|
|
fi
|
|
|
|
# Update the yaml files for Heat
|
|
if [ "${repo_name}" = "heat" ]; then
|
|
find ${os_repo_tmp_path}/etc -name "*.yaml" -exec \
|
|
bash -c "name=\"{}\"; cp \${name} \"${osa_repo_tmp_path}/templates/\$(echo \${name} | rev | cut -sd / -f -2 | rev).j2\"" \;
|
|
fi
|
|
|
|
# Switch into the OSA git directory to work with it
|
|
pushd ${osa_repo_tmp_path} > /dev/null
|
|
|
|
# Check for changed files
|
|
git_changed=$(git status --porcelain | wc -l)
|
|
# Check for untracked files
|
|
git_untracked=$(git ls-files --other --exclude-standard --directory | wc -l)
|
|
if [ ${git_untracked} -gt 0 ]; then
|
|
# If there are untracked files, ensure that the commit message includes
|
|
# a WIP prefix so that the patch is revised in more detail.
|
|
git_msg_prefix="[New files - needs update] "
|
|
else
|
|
git_msg_prefix=""
|
|
fi
|
|
|
|
# If any files have changed, submit a patch including the changes
|
|
if [ ${git_changed} -gt 0 ]; then
|
|
git checkout -b sha-update
|
|
git review -s > /dev/null
|
|
git add --all
|
|
git commit -a -m "${git_msg_prefix}Update paste, policy and rootwrap configurations $(date +%Y-%m-%d)" --quiet
|
|
git review > /dev/null
|
|
fi
|
|
popd > /dev/null
|
|
fi
|
|
fi
|
|
|
|
# Clean up the temporary files
|
|
rm -rf ${os_repo_tmp_path} ${osa_repo_tmp_path}
|
|
fi
|
|
fi
|
|
|
|
echo -e "Processed $repo_name @ $branch_entry\n"
|
|
|
|
done
|
|
|
|
unset IFS
|
|
|
|
# Update the PIP_INSTALL_OPTIONS with the current versions of pip, wheel and setuptools
|
|
PIP_CURRENT_OPTIONS=$(./scripts/get-pypi-pkg-version.py -p pip setuptools wheel -l horizontal)
|
|
sed -i.bak "s|^PIP_INSTALL_OPTIONS=.*|PIP_INSTALL_OPTIONS=\$\{PIP_INSTALL_OPTIONS:-'${PIP_CURRENT_OPTIONS}'\}|" scripts/scripts-library.sh
|
|
|
|
for pin in ${PIP_CURRENT_OPTIONS}; do
|
|
sed -i.bak "s|^$(echo ${pin} | cut -f1 -d=).*|${pin}|" global-requirement-pins.txt
|
|
sed -i.bak "s|^ - $(echo ${pin} | cut -f1 -d=).*| - ${pin}|" playbooks/inventory/group_vars/all.yml
|
|
done
|
|
|
|
echo "Updated pip install options/pins"
|
|
|
|
# Update the ansible-role-requirements.yml file
|
|
# We don't want to be doing this for the master branch
|
|
if [ "${OSA_BRANCH}" != "master" ] || [ "${PRE_RELEASE}" == "true" ]; then
|
|
echo "Updating ansible-role-requirements.yml"
|
|
|
|
if [ "${PRE_RELEASE}" == "true" ]; then
|
|
ROLE_GIT_SOURCES=$(awk '/src: .*/ {print $2}' ansible-role-requirements.yml)
|
|
else
|
|
ROLE_GIT_SOURCES=$(awk '/src: .*\/openstack\// {print $2}' ansible-role-requirements.yml)
|
|
fi
|
|
|
|
# Loop through each of the role git sources, only looking for openstack roles
|
|
for role_src in ${ROLE_GIT_SOURCES}; do
|
|
|
|
# Determine the role's name
|
|
role_name=$(sed 's/^[ \t-]*//' ansible-role-requirements.yml | awk '/src: / || /name: / {print $2}' | grep -B1 "${role_src}" | head -n 1)
|
|
echo "... updating ${role_name}"
|
|
|
|
# If the role_src is NOT from git.openstack.org, try to get a tag first
|
|
if [[ ${role_src} != *"git.openstack.org"* ]]; then
|
|
role_version=$(git ls-remote --tags ${role_src} | awk '{print $2}' | grep -v '{}' | cut -d/ -f 3 | sort -n | tail -n 1)
|
|
fi
|
|
|
|
# Grab the latest SHA that matches the specified branch
|
|
if [[ -z "${role_version}" ]]; then
|
|
role_version=$(git ls-remote ${role_src} | grep "${OSA_BRANCH}$" | awk '{print $1}')
|
|
fi
|
|
|
|
# For OSA roles, get the release notes
|
|
if [[ ${role_src} == *"git.openstack.org"* ]]; then
|
|
# Setup a var for tmp space
|
|
osa_repo_tmp_path="/tmp/osa_${role_name}"
|
|
|
|
# Ensure that the temp path doesn't exist
|
|
rm -rf ${osa_repo_tmp_path}
|
|
|
|
# Do a shallow clone of the repo to work with
|
|
git clone --quiet --depth=10 --branch ${OSA_BRANCH} --single-branch ${role_src} ${osa_repo_tmp_path}
|
|
pushd ${osa_repo_tmp_path} > /dev/null
|
|
git checkout --quiet origin/${OSA_BRANCH}
|
|
popd > /dev/null
|
|
|
|
# If there are releasenotes to copy, then copy them
|
|
if $(ls -1 ${osa_repo_tmp_path}/releasenotes/notes/*.yaml > /dev/null 2>&1); then
|
|
rsync -aq ${osa_repo_tmp_path}/releasenotes/notes/*.yaml releasenotes/notes/
|
|
fi
|
|
|
|
# Clean up the temporary files
|
|
rm -rf ${osa_repo_tmp_path}
|
|
fi
|
|
|
|
# Now use the information we have to update the ansible-role-requirements file
|
|
"$(dirname "${0}")/ansible-role-requirements-editor.py" -f ansible-role-requirements.yml -n "${role_name}" -v "${role_version}"
|
|
|
|
unset role_version
|
|
done
|
|
echo "Completed updating ansible-role-requirements.yml"
|
|
else
|
|
echo "Skipping the ansible-role-requirements.yml update as we're working on the master branch"
|
|
fi
|
|
|
|
# Update the release version in playbooks/inventory/group_vars/all.yml
|
|
# We don't want to be doing this for the master branch
|
|
if [[ "${OSA_BRANCH}" != "master" ]]; then
|
|
|
|
echo "Updating the release version..."
|
|
currentversion=$(awk '/openstack_release/ {print $2}' playbooks/inventory/group_vars/all.yml)
|
|
|
|
# Extract the required version info
|
|
major_version=$( echo ${currentversion} | cut -d. -f1 )
|
|
minor_version=$( echo ${currentversion} | cut -d. -f2 )
|
|
patch_version=$( echo ${currentversion} | cut -d. -f3 )
|
|
|
|
# increment the patch version
|
|
patch_version=$(( patch_version + 1 ))
|
|
|
|
sed -i .bak "s/${currentversion}/${major_version}.${minor_version}.${patch_version}/" playbooks/inventory/group_vars/all.yml
|
|
else
|
|
echo "Skipping the release version update as we're working on the master branch"
|
|
fi
|