diff --git a/doc/source/conf.py b/doc/source/conf.py index 6bf957c46d..817b0b79fe 100644 --- a/doc/source/conf.py +++ b/doc/source/conf.py @@ -311,10 +311,12 @@ if watermark == "": deploy_branch_link_name = "latest" dev_branch_link_name = "" current_release_git_branch_name = "master" + current_release_gnocchi_git_branch_name = "master" else: deploy_branch_link_name = current_release_branch_name dev_branch_link_name = "{}/".format(current_release_branch_name) current_release_git_branch_name = 'stable/' + current_release_branch_name + current_release_gnocchi_git_branch_name = "4.0" previous_release_capital_name = previous_release_branch_name.upper() previous_release_formal_name = previous_release_branch_name.capitalize() @@ -330,6 +332,7 @@ rst_epilog = """ .. |previous_release_branch_name| replace:: %s .. |current_release_branch_name| replace:: %s .. |current_release_git_branch_name| replace:: %s +.. |current_release_gnocchi_git_branch_name| replace:: %s .. |previous_release_capital_name| replace:: %s .. |previous_release_formal_name| replace:: %s .. |current_release_capital_name| replace:: %s @@ -339,6 +342,7 @@ rst_epilog = """ """ % (previous_release_branch_name, current_release_branch_name, current_release_git_branch_name, + current_release_gnocchi_git_branch_name, previous_release_capital_name, previous_release_formal_name, current_release_capital_name, diff --git a/doc/source/contributor/scripts.rst b/doc/source/contributor/scripts.rst index b7e332d7fd..843e9035a4 100644 --- a/doc/source/contributor/scripts.rst +++ b/doc/source/contributor/scripts.rst @@ -16,7 +16,7 @@ Bootstrapping bootstrap-ansible.sh -------------------- -The ``bootstrap-ansible.sh`` script installs Ansible including `core`_ and +The ``bootstrap-ansible.sh`` script installs Ansible, including the `core`_ and `extras`_ module repositories and Galaxy roles. While there are several configurable environment variables which this script @@ -189,3 +189,57 @@ OpenStack-CI through the following jobs: While this script is primarily developed and maintained for use in OpenStack-CI, it can be used in other environments. + +Dependency Updates +^^^^^^^^^^^^^^^^^^ + +The dependencies for OpenStack-Ansible are updated approximately every two +weeks through the use of ``scripts/sources-branch-updater.sh``. This script +updates all pinned SHA's for OpenStack services, OpenStack-Ansible roles, +and other python dependencies which are not handled by the OpenStack global +requirements management process. This script also updates the statically +held templates/files in each role to ensure that they are always up to date. +Finally, it also does a minor version increment of the value for +``openstack_release``. + +The update script is used as follows: + +.. parsed-literal:: + + # change directory to the openstack-ansible checkout + cd ~/code/openstack-ansible + + # ensure that the correct branch is checked out + git checkout |current_release_git_branch_name| + + # ensure that the branch is up to date + git pull + + # create the local branch for the update + git checkout -b sha-update + + # execute the script for all openstack services + ./scripts/sources-branch-updater.sh -b |current_release_git_branch_name| -o |current_release_git_branch_name| + + # execute the script for gnocchi + ./scripts/sources-branch-updater.sh -s playbooks/defaults/repo_packages/gnocchi.yml -b |current_release_gnocchi_git_branch_name| -o |current_release_git_branch_name| + + # the console code should only be updated when necessary for a security fix, or for the OSA master branch + ./scripts/sources-branch-updater.sh -s playbooks/defaults/repo_packages/nova_consoles.yml -b master + + # the testing repositories should not be updated for stable branches as the new tests + # or other changes introduced may not work for older branches + ./scripts/sources-branch-updater.sh -s playbooks/defaults/repo_packages/openstack_testing.yml -b master + + # commit the changes + new_version=$(awk '/^openstack_release/ {print $2}' inventory/group_vars/all/all.yml) + git add --all + git commit -a -m "Update all SHAs for ${new_version}" \ + -m "This patch updates all the roles to the latest available stable + SHA's, copies the release notes from the updated roles into the + integrated repo, updates all the OpenStack Service SHA's, and + updates the appropriate python requirements pins. + + # push the changes up to gerrit + git review + diff --git a/scripts/ansible-role-requirements-editor.py b/scripts/ansible-role-requirements-editor.py index 627dd44f1c..7da5fa7370 100755 --- a/scripts/ansible-role-requirements-editor.py +++ b/scripts/ansible-role-requirements-editor.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python2 +#!/usr/bin/env python2.7 # # Copyright 2016, Rackspace US, Inc. # diff --git a/scripts/get-pypi-pkg-version.py b/scripts/get-pypi-pkg-version.py index d75b894532..3f0ea79236 100755 --- a/scripts/get-pypi-pkg-version.py +++ b/scripts/get-pypi-pkg-version.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python2 +#!/usr/bin/env python2.7 # # Copyright 2016, Rackspace US, Inc. # diff --git a/scripts/release-yaml-file-prep.py b/scripts/release-yaml-file-prep.py index 159a266c55..5c4e909a39 100755 --- a/scripts/release-yaml-file-prep.py +++ b/scripts/release-yaml-file-prep.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python2 +#!/usr/bin/env python2.7 # # Copyright 2016, Rackspace US, Inc. # diff --git a/scripts/sources-branch-updater-lib.sh b/scripts/sources-branch-updater-lib.sh index ec500ad26b..0b86372e26 100644 --- a/scripts/sources-branch-updater-lib.sh +++ b/scripts/sources-branch-updater-lib.sh @@ -119,58 +119,75 @@ sync_roles_and_packages() { # pre-sync user hook osa_pre_sync_hook ${repo_name} ${os_branch} ${osa_branch} ${repo_address} - # Update the policy files - find ${os_repo_tmp_path}/etc -name "policy.json" -exec \ - cp {} "${osa_repo_tmp_path}/templates/policy.json.j2" \; + # We have implemented tooling to dynamically fetch the + # api-paste and other static/template files from these + # repositories, so skip trying to update their templates + # and static files. + local static_file_repo_skip_list=( ceilometer gnocchi keystone ) - # Tweak the paste files for any hmac key entries - find ${os_repo_tmp_path}/etc -name "*[_-]paste.ini" -exec \ - sed -i.bak "s|hmac_keys = SECRET_KEY|hmac_keys = {{ ${repo_name}_profiler_hmac_key }}|" {} \; + # Check if this repo is in the static file skip list + local skip_this_repo="no" + for skip_list_item in "${static_file_repo_skip_list[@]}"; do + if [[ "${repo_name}" == "${skip_list_item}" ]]; then + skip_this_repo="yes" + fi + done - # Tweak the barbican paste file to support keystone auth - if [ "${repo_name}" = "barbican" ]; then + if [[ "${skip_this_repo}" != "yes" ]] && [[ -e "${os_repo_tmp_path}/etc" ]]; then + # Update the policy files + find ${os_repo_tmp_path}/etc -name "policy.json" -exec \ + cp {} "${osa_repo_tmp_path}/templates/policy.json.j2" \; + + # Tweak the paste files for any hmac key entries find ${os_repo_tmp_path}/etc -name "*[_-]paste.ini" -exec \ - sed -i.bak 's|\/v1\: barbican-api-keystone|\/v1\: {{ (barbican_keystone_auth \| bool) \| ternary('barbican-api-keystone', 'barbican_api') }}|'{} \; - fi + sed -i.bak "s|hmac_keys = SECRET_KEY|hmac_keys = {{ ${repo_name}_profiler_hmac_key }}|" {} \; - # Tweak the gnocchi paste file to support keystone auth - if [ "${repo_name}" = "gnocchi" ]; then + # Tweak the barbican paste file to support keystone auth + if [[ "${repo_name}" == "barbican" ]]; then + find ${os_repo_tmp_path}/etc -name "*[_-]paste.ini" -exec \ + sed -i.bak "s|\/v1\: barbican-api-keystone|\/v1\: {{ (barbican_keystone_auth \| bool) \| ternary('barbican-api-keystone', 'barbican_api') }}|" {} \; + fi + + # Tweak the gnocchi paste file to support keystone auth + if [[ "${repo_name}" == "gnocchi" ]]; then + find ${os_repo_tmp_path}/etc -name "*[_-]paste.ini" -exec \ + sed -i.bak "s|pipeline = gnocchi+noauth|pipeline = {{ (gnocchi_keystone_auth \| bool) \| ternary('gnocchi+auth', 'gnocchi+noauth') }}|" {} \; + fi + + # Update the paste files find ${os_repo_tmp_path}/etc -name "*[_-]paste.ini" -exec \ - sed -i.bak "s|pipeline = gnocchi+noauth|pipeline = {{ (gnocchi_keystone_auth \| bool) \| ternary('gnocchi+auth', 'gnocchi+noauth') }}|" {} \; - fi - - # Update the paste files - find ${os_repo_tmp_path}/etc -name "*[_-]paste.ini" -exec \ - bash -c "name=\"{}\"; cp \${name} \"${osa_repo_tmp_path}/templates/\$(basename \${name}).j2\"" \; - - # Tweak the rootwrap conf filters_path (for neutron only) - if [ "${repo_name}" = "neutron" ]; then - find ${os_repo_tmp_path}/etc -name "rootwrap.conf" -exec \ - sed -i.bak "s|filters_path=/etc/neutron|filters_path={{ ${repo_name}_conf_dir }}|" {} \; - fi - - # Tweak the rootwrap conf exec_dirs - find ${os_repo_tmp_path}/etc -name "rootwrap.conf" -exec \ - sed -i.bak "s|exec_dirs=|exec_dirs={{ ${repo_name}_bin }},|" {} \; - - # Update the rootwrap conf files - find ${os_repo_tmp_path}/etc -name "rootwrap.conf" -exec \ - cp {} "${osa_repo_tmp_path}/templates/rootwrap.conf.j2" \; - - # Update the rootwrap filters - find ${os_repo_tmp_path}/etc -name "*.filters" -exec \ - bash -c "name=\"{}\"; cp \${name} \"${osa_repo_tmp_path}/files/rootwrap.d/\$(basename \${name})\"" \; - - # Update the yaml files for Ceilometer - if [ "${repo_name}" = "ceilometer" ]; then - find ${os_repo_tmp_path}/etc -name "*.yaml" -exec \ bash -c "name=\"{}\"; cp \${name} \"${osa_repo_tmp_path}/templates/\$(basename \${name}).j2\"" \; + + # Update the yaml files for Heat + if [[ "${repo_name}" == "heat" ]]; then + find ${os_repo_tmp_path}/etc -name "*.yaml" -exec \ + bash -c "name=\"{}\"; cp \${name} \"${osa_repo_tmp_path}/templates/\$(echo \${name} | rev | cut -sd / -f -2 | rev).j2\"" \; + fi fi - # Update the yaml files for Heat - if [ "${repo_name}" = "heat" ]; then - find ${os_repo_tmp_path}/etc -name "*.yaml" -exec \ - bash -c "name=\"{}\"; cp \${name} \"${osa_repo_tmp_path}/templates/\$(echo \${name} | rev | cut -sd / -f -2 | rev).j2\"" \; + # We have to check for rootwrap files in *all* service repositories + # as we have no dynamic way of fetching them at this stage. + if [[ -e "${os_repo_tmp_path}/etc" ]]; then + + # Tweak the rootwrap conf filters_path (for neutron only) + if [[ "${repo_name}" == "neutron" ]]; then + find ${os_repo_tmp_path}/etc -name "rootwrap.conf" -exec \ + sed -i.bak "s|filters_path=/etc/neutron|filters_path={{ ${repo_name}_conf_dir }}|" {} \; + fi + + # Tweak the rootwrap conf exec_dirs + find ${os_repo_tmp_path}/etc -name "rootwrap.conf" -exec \ + sed -i.bak "s|exec_dirs=|exec_dirs={{ ${repo_name}_bin }},|" {} \; + + # Update the rootwrap conf files + find ${os_repo_tmp_path}/etc -name "rootwrap.conf" -exec \ + cp {} "${osa_repo_tmp_path}/templates/rootwrap.conf.j2" \; + + # Update the rootwrap filters + mkdir -p ${osa_repo_tmp_path}/files/rootwrap.d + find ${os_repo_tmp_path}/etc -name "*.filters" -exec \ + bash -c "name=\"{}\"; cp \${name} \"${osa_repo_tmp_path}/files/rootwrap.d/\$(basename \${name})\"" \; + fi # post-sync user hook @@ -229,8 +246,8 @@ update_ansible_role_requirements() { role_name=$(sed 's/^[ \t-]*//' ansible-role-requirements.yml | awk '/src: / || /name: / {print $2}' | grep -B1 "${role_src}" | head -n 1) echo "... updating ${role_name}" - # If the role_src is NOT from git.openstack.org, try to get a tag first - if [[ ${role_src} != *"git.openstack.org"* ]]; then + # If the role_src is NOT from git.openstack.org, try to get a tag first unless we are working on master + if [[ ${role_src} != *"git.openstack.org"* ]] && [[ "${force_master}" != "true" ]]; then role_version=$(git ls-remote --tags ${role_src} | awk '{print $2}' | grep -v '{}' | cut -d/ -f 3 | sort -n | tail -n 1) fi diff --git a/scripts/sources-branch-updater.sh b/scripts/sources-branch-updater.sh index 4c2dcead4c..15b2a481ef 100755 --- a/scripts/sources-branch-updater.sh +++ b/scripts/sources-branch-updater.sh @@ -22,10 +22,51 @@ OS_BRANCH=${OS_BRANCH:-"master"} OSA_BRANCH=${OSA_BRANCH:-"$OS_BRANCH"} SERVICE_FILE=${SERVICE_FILE:-"playbooks/defaults/repo_packages/openstack_services.yml"} -OPENSTACK_SERVICE_LIST=${OPENSTACK_SERVICE_LIST:-"$(grep 'git_repo\:' ${SERVICE_FILE} | awk -F '/' '{ print $NF }' | egrep -v 'requirements|-' | tr '\n' ' ')"} +OPENSTACK_SERVICE_LIST=${OPENSTACK_SERVICE_LIST:-""} PRE_RELEASE=${PRE_RELEASE:-"false"} FORCE_MASTER=${FORCE_MASTER:-"false"} +# Here we inspect the service file to compile the list of repositories +# we're interested in inspecting for the purpose of doing in-repo updates +# of static files that we template/copy when doing installs. +# +# If a predefined list is provided, skip all this. +if [[ -z ${OPENSTACK_SERVICE_LIST} ]]; then + # Setup an array of all the repositories in the + # service file provided. + OPENSTACK_REPO_LIST=( $(grep 'git_repo\:' ${SERVICE_FILE} | awk -F '/' '{ print $NF }') ) + + # Define the repositories to skip in an array. + # These items are removed as they are not service projects + # and therefore do not have policy/api-paste/etc files. + OPENSTACK_REPO_SKIP_LIST=( requirements dragonflow swift3 ) + + # Define the skip regex for any additional items to remove. + # Items with a '-' are removed as those repositories are + # typically extensions/drivers/dashboards and therefore + # do not include policy/api-paste/etc files. + OPENSTACK_REPO_SKIP_REGEX='.*-.*' + + # Loop through each item and if it does not match + # an item in the SKIP_LIST or match the SKIP_REGEX + # then add it to the OPENSTACK_SERVICE_LIST string. + for item_to_check in "${OPENSTACK_REPO_LIST[@]}"; do + add_item="yes" + if [[ ! "${item_to_check}" =~ ${OPENSTACK_REPO_SKIP_REGEX} ]]; then + for item_to_delete in "${OPENSTACK_REPO_SKIP_LIST[@]}"; do + if [[ "${item_to_delete}" == "${item_to_check}" ]]; then + add_item="no" + fi + done + else + add_item="no" + fi + if [[ "${add_item}" == "yes" ]]; then + OPENSTACK_SERVICE_LIST="${OPENSTACK_SERVICE_LIST} ${item_to_check}" + fi + done +fi + source scripts/sources-branch-updater-lib.sh || { echo "Failed to source updater library"; exit 1; } if echo "$@" | grep -e '-h' -e '--help';then