Fix sources-branch-updater and add docs

Problems resolved:

- Errors relating to 'etc' folders which did not exist in the
  target repositories have been eliminated.

- Some repositories were not being excluded from evaluation
  for file/template updates. All appropriate exclusions are
  now implemented.

- If an OSA role repository did not have the required folder
  to copy rootwrap files into, the copy would fail and the
  rootwrap files therefore be left out of the patch.

- When executed on Mac OS X, python2 is not found.

- Documentation is added to describe how to use the script
  to handle periodic SHA bumps.

Change-Id: I628cd2c3156d118c17ca2f90d4ccf0bc5a080bbf

doc/source/conf.py

@@ -323,10 +323,12 @@ if watermark == "":
     deploy_branch_link_name = "latest"
     dev_branch_link_name = ""
     current_release_git_branch_name = "master"
+    current_release_gnocchi_git_branch_name = "master"
 else:
     deploy_branch_link_name = current_release_branch_name
     dev_branch_link_name = "{}/".format(current_release_branch_name)
     current_release_git_branch_name = 'stable/' + current_release_branch_name
+    current_release_gnocchi_git_branch_name = "4.1"
 
 previous_release_capital_name = previous_release_branch_name.upper()
 previous_release_formal_name = previous_release_branch_name.capitalize()
@@ -342,6 +344,7 @@ rst_epilog = """
 .. |previous_release_branch_name| replace:: %s
 .. |current_release_branch_name| replace:: %s
 .. |current_release_git_branch_name| replace:: %s
+.. |current_release_gnocchi_git_branch_name| replace:: %s
 .. |previous_release_capital_name| replace:: %s
 .. |previous_release_formal_name| replace:: %s
 .. |current_release_capital_name| replace:: %s
@@ -351,6 +354,7 @@ rst_epilog = """
 """ % (previous_release_branch_name,
        current_release_branch_name,
        current_release_git_branch_name,
+       current_release_gnocchi_git_branch_name,
        previous_release_capital_name,
        previous_release_formal_name,
        current_release_capital_name,

doc/source/contributor/scripts.rst

@@ -16,7 +16,7 @@ Bootstrapping
 bootstrap-ansible.sh
 --------------------
 
-The ``bootstrap-ansible.sh`` script installs Ansible including `core`_ and
+The ``bootstrap-ansible.sh`` script installs Ansible, including the `core`_ and
 `extras`_ module repositories and Galaxy roles.
 
 While there are several configurable environment variables which this script
@@ -189,3 +189,57 @@ OpenStack-CI through the following jobs:
 
   While this script is primarily developed and maintained for use in
   OpenStack-CI, it can be used in other environments.
+
+Dependency Updates
+^^^^^^^^^^^^^^^^^^
+
+The dependencies for OpenStack-Ansible are updated approximately every two
+weeks through the use of ``scripts/sources-branch-updater.sh``. This script
+updates all pinned SHA's for OpenStack services, OpenStack-Ansible roles,
+and other python dependencies which are not handled by the OpenStack global
+requirements management process. This script also updates the statically
+held templates/files in each role to ensure that they are always up to date.
+Finally, it also does a minor version increment of the value for
+``openstack_release``.
+
+The update script is used as follows:
+
+.. parsed-literal::
+
+   # change directory to the openstack-ansible checkout
+   cd ~/code/openstack-ansible
+
+   # ensure that the correct branch is checked out
+   git checkout |current_release_git_branch_name|
+
+   # ensure that the branch is up to date
+   git pull
+
+   # create the local branch for the update
+   git checkout -b sha-update
+
+   # execute the script for all openstack services
+   ./scripts/sources-branch-updater.sh -b |current_release_git_branch_name| -o |current_release_git_branch_name|
+
+   # execute the script for gnocchi
+   ./scripts/sources-branch-updater.sh -s playbooks/defaults/repo_packages/gnocchi.yml -b |current_release_gnocchi_git_branch_name| -o |current_release_git_branch_name|
+
+   # the console code should only be updated when necessary for a security fix, or for the OSA master branch
+   ./scripts/sources-branch-updater.sh -s playbooks/defaults/repo_packages/nova_consoles.yml -b master
+
+   # the testing repositories should not be updated for stable branches as the new tests
+   # or other changes introduced may not work for older branches
+   ./scripts/sources-branch-updater.sh -s playbooks/defaults/repo_packages/openstack_testing.yml -b master
+
+   # commit the changes
+   new_version=$(awk '/^openstack_release/ {print $2}' inventory/group_vars/all/all.yml)
+   git add --all
+   git commit -a -m "Update all SHAs for ${new_version}" \
+   -m "This patch updates all the roles to the latest available stable
+   SHA's, copies the release notes from the updated roles into the
+   integrated repo, updates all the OpenStack Service SHA's, and
+   updates the appropriate python requirements pins.
+
+   # push the changes up to gerrit
+   git review
+

scripts/ansible-role-requirements-editor.py

@@ -1,4 +1,4 @@
-#!/usr/bin/env python2
+#!/usr/bin/env python2.7
 #
 # Copyright 2016, Rackspace US, Inc.
 #

scripts/get-pypi-pkg-version.py

@@ -1,4 +1,4 @@
-#!/usr/bin/env python2
+#!/usr/bin/env python2.7
 #
 # Copyright 2016, Rackspace US, Inc.
 #

scripts/release-yaml-file-prep.py

@@ -1,4 +1,4 @@
-#!/usr/bin/env python2
+#!/usr/bin/env python2.7
 #
 # Copyright 2016, Rackspace US, Inc.
 #

scripts/sources-branch-updater-lib.sh

@@ -119,6 +119,21 @@ sync_roles_and_packages() {
         # pre-sync user hook
         osa_pre_sync_hook ${repo_name} ${os_branch} ${osa_branch} ${repo_address}
 
+        # We have implemented tooling to dynamically fetch the
+        # api-paste and other static/template files from these
+        # repositories, so skip trying to update their templates
+        # and static files.
+        local static_file_repo_skip_list=( ceilometer gnocchi keystone )
+
+        # Check if this repo is in the static file skip list
+        local skip_this_repo="no"
+        for skip_list_item in "${static_file_repo_skip_list[@]}"; do
+          if [[ "${repo_name}" == "${skip_list_item}" ]]; then
+            skip_this_repo="yes"
+          fi
+        done
+
+        if [[ "${skip_this_repo}" != "yes" ]] && [[ -e "${os_repo_tmp_path}/etc" ]]; then
           # Update the policy files
           find ${os_repo_tmp_path}/etc -name "policy.json" -exec \
             cp {} "${osa_repo_tmp_path}/templates/policy.json.j2" \;
@@ -128,13 +143,13 @@ sync_roles_and_packages() {
             sed -i.bak "s|hmac_keys = SECRET_KEY|hmac_keys = {{ ${repo_name}_profiler_hmac_key }}|" {} \;
 
           # Tweak the barbican paste file to support keystone auth
-        if [ "${repo_name}" = "barbican" ]; then
+          if [[ "${repo_name}" == "barbican" ]]; then
             find ${os_repo_tmp_path}/etc -name "*[_-]paste.ini" -exec \
               sed -i.bak "s|\/v1\: barbican-api-keystone|\/v1\: {{ (barbican_keystone_auth \| bool) \| ternary('barbican-api-keystone', 'barbican_api') }}|" {} \;
           fi
 
           # Tweak the gnocchi paste file to support keystone auth
-        if [ "${repo_name}" = "gnocchi" ]; then
+          if [[ "${repo_name}" == "gnocchi" ]]; then
             find ${os_repo_tmp_path}/etc -name "*[_-]paste.ini" -exec \
               sed -i.bak "s|pipeline = gnocchi+noauth|pipeline = {{ (gnocchi_keystone_auth \| bool) \| ternary('gnocchi+auth', 'gnocchi+noauth') }}|" {} \;
           fi
@@ -143,8 +158,19 @@ sync_roles_and_packages() {
           find ${os_repo_tmp_path}/etc -name "*[_-]paste.ini" -exec \
             bash -c "name=\"{}\"; cp \${name} \"${osa_repo_tmp_path}/templates/\$(basename \${name}).j2\"" \;
 
+          # Update the yaml files for Heat
+          if [[ "${repo_name}" == "heat" ]]; then
+            find ${os_repo_tmp_path}/etc -name "*.yaml" -exec \
+              bash -c "name=\"{}\"; cp \${name} \"${osa_repo_tmp_path}/templates/\$(echo \${name} | rev | cut -sd / -f -2 | rev).j2\"" \;
+          fi
+        fi
+
+        # We have to check for rootwrap files in *all* service repositories
+        # as we have no dynamic way of fetching them at this stage.
+        if [[ -e "${os_repo_tmp_path}/etc" ]]; then
+
           # Tweak the rootwrap conf filters_path (for neutron only)
-        if [ "${repo_name}" = "neutron" ]; then
+          if [[ "${repo_name}" == "neutron" ]]; then
             find ${os_repo_tmp_path}/etc -name "rootwrap.conf" -exec \
               sed -i.bak "s|filters_path=/etc/neutron|filters_path={{ ${repo_name}_conf_dir }}|" {} \;
           fi
@@ -158,19 +184,10 @@ sync_roles_and_packages() {
             cp {} "${osa_repo_tmp_path}/templates/rootwrap.conf.j2" \;
 
           # Update the rootwrap filters
+          mkdir -p ${osa_repo_tmp_path}/files/rootwrap.d
           find ${os_repo_tmp_path}/etc -name "*.filters" -exec \
             bash -c "name=\"{}\"; cp \${name} \"${osa_repo_tmp_path}/files/rootwrap.d/\$(basename \${name})\"" \;
 
-        # Update the yaml files for Ceilometer
-        if [ "${repo_name}" = "ceilometer" ]; then
-          find ${os_repo_tmp_path}/etc -name "*.yaml" -exec \
-            bash -c "name=\"{}\"; cp \${name} \"${osa_repo_tmp_path}/templates/\$(basename \${name}).j2\"" \;
-        fi
-
-        # Update the yaml files for Heat
-        if [ "${repo_name}" = "heat" ]; then
-          find ${os_repo_tmp_path}/etc -name "*.yaml" -exec \
-            bash -c "name=\"{}\"; cp \${name} \"${osa_repo_tmp_path}/templates/\$(echo \${name} | rev | cut -sd / -f -2 | rev).j2\"" \;
         fi
 
         # post-sync user hook

scripts/sources-branch-updater.sh

@@ -22,10 +22,51 @@
 OS_BRANCH=${OS_BRANCH:-"master"}
 OSA_BRANCH=${OSA_BRANCH:-"$OS_BRANCH"}
 SERVICE_FILE=${SERVICE_FILE:-"playbooks/defaults/repo_packages/openstack_services.yml"}
-OPENSTACK_SERVICE_LIST=${OPENSTACK_SERVICE_LIST:-"$(grep 'git_repo\:' ${SERVICE_FILE} | awk -F '/' '{ print $NF }' | egrep -v 'requirements|-' | tr '\n' ' ')"}
+OPENSTACK_SERVICE_LIST=${OPENSTACK_SERVICE_LIST:-""}
 PRE_RELEASE=${PRE_RELEASE:-"false"}
 FORCE_MASTER=${FORCE_MASTER:-"false"}
 
+# Here we inspect the service file to compile the list of repositories
+# we're interested in inspecting for the purpose of doing in-repo updates
+# of static files that we template/copy when doing installs.
+#
+# If a predefined list is provided, skip all this.
+if [[ -z ${OPENSTACK_SERVICE_LIST} ]]; then
+  # Setup an array of all the repositories in the
+  # service file provided.
+  OPENSTACK_REPO_LIST=( $(grep 'git_repo\:' ${SERVICE_FILE} | awk -F '/' '{ print $NF }') )
+
+  # Define the repositories to skip in an array.
+  # These items are removed as they are not service projects
+  # and therefore do not have policy/api-paste/etc files.
+  OPENSTACK_REPO_SKIP_LIST=( requirements dragonflow swift3 )
+
+  # Define the skip regex for any additional items to remove.
+  # Items with a '-' are removed as those repositories are
+  # typically extensions/drivers/dashboards and therefore
+  # do not include policy/api-paste/etc files.
+  OPENSTACK_REPO_SKIP_REGEX='.*-.*'
+
+  # Loop through each item and if it does not match
+  # an item in the SKIP_LIST or match the SKIP_REGEX
+  # then add it to the OPENSTACK_SERVICE_LIST string.
+  for item_to_check in "${OPENSTACK_REPO_LIST[@]}"; do
+    add_item="yes"
+    if [[ ! "${item_to_check}" =~ ${OPENSTACK_REPO_SKIP_REGEX} ]]; then
+      for item_to_delete in "${OPENSTACK_REPO_SKIP_LIST[@]}"; do
+        if [[ "${item_to_delete}" == "${item_to_check}" ]]; then
+          add_item="no"
+        fi
+      done
+    else
+      add_item="no"
+    fi
+    if [[ "${add_item}" == "yes" ]]; then
+      OPENSTACK_SERVICE_LIST="${OPENSTACK_SERVICE_LIST} ${item_to_check}"
+    fi
+  done
+fi
+
 source scripts/sources-branch-updater-lib.sh || { echo "Failed to source updater library"; exit 1; }
 
 if echo "$@" | grep -e '-h' -e '--help';then