Updates all repo SHAs to prepare for Mitaka release

This patch includes updates of all SHA's and pip, wheel and setuptools
pins.

Due to Bug #1561947, the backend glance store has been changed from swift
to file in the AIO gate. This has been done becuase the glance store project
has an issue with session that causes it to be incompatible with swift. Being
that the master integration gate is depending on this change to go it is
needed to temporarily disable the swift image store in glance until this issue
is fixed.

The branch updater sciprt was modified to search for and update roles if they're
found within the default or user provided role path. This change is resolving
bash errors that happen when the script is run in mitaka+.

Related-Bug: 1561947
Change-Id: I2e09e3e0abb61f5e97e2af7b283f0aed6dadd853
Signed-off-by: Kevin Carter <kevin.carter@rackspace.com>

playbooks/defaults/repo_packages/openstack_other.yml

@@ -27,17 +27,17 @@
 
 ## Tempest service
 tempest_git_repo: https://git.openstack.org/openstack/tempest
-tempest_git_install_branch: 5cc7ef78b4233444a4dcea1b1eb8f213c1548491 # HEAD of "master" as of 17.01.2016
+tempest_git_install_branch: fd3088f8992633f6eb005e606d3bb8596fa23d56 # HEAD of "master" as of 23.03.2016
 tempest_git_dest: "/opt/tempest_{{ tempest_git_install_branch | replace('/', '_') }}"
 
 
 ## NOVNC from source
 novncproxy_git_repo: https://github.com/kanaka/novnc
-novncproxy_git_install_branch: 670dbddb54264fd0082d0aca1b3acb0f1814b1d2 # HEAD of "master" as of 17.01.2016
+novncproxy_git_install_branch: b403cb92fb8de82d04f305b4f14fa978003890d7 # HEAD of "master" as of 23.03.2016
 novncproxy_git_dest: "/opt/novnc_{{ novncproxy_git_install_branch | replace('/', '_') }}"
 
 
 ## spice-html5 from source
 spicehtml5_git_repo: https://github.com/SPICE/spice-html5
-spicehtml5_git_install_branch: ab73d009487c8afd4def39b54a422499b4c13c40 # HEAD of "master" as of 17.01.2016
+spicehtml5_git_install_branch: ab73d009487c8afd4def39b54a422499b4c13c40 # HEAD of "master" as of 23.03.2016
 spicehtml5_git_dest: "/opt/spicehtml5_{{ spicehtml5_git_install_branch | replace('/', '_') }}"

playbooks/defaults/repo_packages/openstack_services.yml

@@ -31,77 +31,77 @@
 
 ## Global Requirements
 requirements_git_repo: https://git.openstack.org/openstack/requirements
-requirements_git_install_branch: 0a0352fb3065308647e0fd49e15b1b733135501a # HEAD of "master" as of 01.02.2016
+requirements_git_install_branch: d371aad12f492315122f2a5ab86db4678dbacfd3 # HEAD of "stable/mitaka" as of 28.03.2016
 requirements_git_dest: "/opt/requirements_{{ requirements_git_install_branch | replace('/', '_') }}"
 
 
 ## Aodh service
 aodh_git_repo: https://git.openstack.org/openstack/aodh
-aodh_git_install_branch: 239e1f629b26557ceadb92de3d62edcd87489b9d # HEAD of "master" as of 17.01.2016
+aodh_git_install_branch: 235f79f3cdd9b5fb607a06011ead7a3962c91921 # HEAD of "stable/mitaka" as of 28.03.2016
 aodh_git_dest: "/opt/aodh_{{ aodh_git_install_branch | replace('/', '_') }}"
 
 
 ## Ceilometer service
 ceilometer_git_repo: https://git.openstack.org/openstack/ceilometer
-ceilometer_git_install_branch: 333024b69aa7810e78aef85e5171cfd6dbd6b740 # HEAD of "master" as of 17.01.2016
+ceilometer_git_install_branch: 7ee29aae2835c9eb0d1a747db8ba06b56f74082c # HEAD of "stable/mitaka" as of 28.03.2016
 ceilometer_git_dest: "/opt/ceilometer_{{ceilometer_git_install_branch | replace('/', '_') }}"
 
 
 ## Cinder service
 cinder_git_repo: https://git.openstack.org/openstack/cinder
-cinder_git_install_branch: 94ae8598b96e2f86844fdf0f35a8b83a94c7b4c4 # HEAD of "master" as of 17.01.2016
+cinder_git_install_branch: da28ab66c37fa9c762d7562f68ce5b07f2240a6b # HEAD of "stable/mitaka" as of 28.03.2016
 cinder_git_dest: "/opt/cinder_{{ cinder_git_install_branch | replace('/', '_') }}"
 
 
 ## Glance service
 glance_git_repo: https://git.openstack.org/openstack/glance
-glance_git_install_branch: 7d5c3710ce2739a8ac356208d4e104f2ce3ec9ab # HEAD of "master" as of 17.01.2016
+glance_git_install_branch: 1cb2bdaf613878ee290c43807d1ab3893bedada2 # HEAD of "stable/mitaka" as of 28.03.2016
 glance_git_dest: "/opt/glance_{{ glance_git_install_branch | replace('/', '_') }}"
 
 
 ## Heat service
 heat_git_repo: https://git.openstack.org/openstack/heat
-heat_git_install_branch: 7e3e4087f476a0431d1d278730b1736e02e5fd06 # HEAD of "master" as of 17.01.2016
+heat_git_install_branch: 588e7e385e316d87a21b62be526d7e6ba4bb2ad6 # HEAD of "stable/mitaka" as of 28.03.2016
 heat_git_dest: "/opt/heat_{{ heat_git_install_branch | replace('/', '_') }}"
 
 
 ## Horizon service
 horizon_git_repo: https://git.openstack.org/openstack/horizon
-horizon_git_install_branch: 18f1605bddd428a014d0e43ef52d1af6305e1e03 # HEAD of "master" as of 17.01.2016
+horizon_git_install_branch: 2143bb60e9a84052ebd54a43b3dfb740b95f338a # HEAD of "stable/mitaka" as of 28.03.2016
 horizon_git_dest: "/opt/horizon_{{ horizon_git_install_branch | replace('/', '_') }}"
 
 
 ## Keystone service
 keystone_git_repo: https://git.openstack.org/openstack/keystone
-keystone_git_install_branch: a55128044f763f5cfe2fdc57c738eaca97636448 # HEAD of "master" as of 17.01.2016
+keystone_git_install_branch: aa53ad975fdc1a70091b87400c85ff4f1d658d2c # HEAD of "stable/mitaka" as of 28.03.2016
 keystone_git_dest: "/opt/keystone_{{ keystone_git_install_branch | replace('/', '_') }}"
 
 
 ## Neutron service
 neutron_git_repo: https://git.openstack.org/openstack/neutron
-neutron_git_install_branch: d6d43b32ca825b6c3c2c908f5ff7bc50c736546e # HEAD of "master" as of 17.01.2016
+neutron_git_install_branch: 439740e92cdb86d65b703dac95e7095fdf569619 # HEAD of "stable/mitaka" as of 28.03.2016
 neutron_git_dest: "/opt/neutron_{{ neutron_git_install_branch | replace('/', '_') }}"
 
 neutron_lbaas_git_repo: https://git.openstack.org/openstack/neutron-lbaas
-neutron_lbaas_git_install_branch: b5d4e5c0fe02a897ad2ab0bc548f695915998831 # HEAD of "master" as of 17.01.2016
+neutron_lbaas_git_install_branch: a29a6013ebaf2b9a11ae734f5fa9926d56a4718d # HEAD of "stable/mitaka" as of 28.03.2016
 neutron_lbaas_git_dest: "/opt/neutron_lbaas_{{ neutron_lbaas_git_install_branch | replace('/', '_') }}"
 
 neutron_vpnaas_git_repo: https://git.openstack.org/openstack/neutron-vpnaas
-neutron_vpnaas_git_install_branch: 832b875b79d801e17a5b997054f30c9d88b36914 # HEAD of "master" as of 17.01.2016
+neutron_vpnaas_git_install_branch: 10a56ce65329724a460b133328e8c9e6d16706a4 # HEAD of "stable/mitaka" as of 28.03.2016
 neutron_vpnaas_git_dest: "/opt/neutron_vpnaas_{{ neutron_vpnaas_git_install_branch | replace('/', '_') }}"
 
 neutron_fwaas_git_repo: https://git.openstack.org/openstack/neutron-fwaas
-neutron_fwaas_git_install_branch: cb0093d185a97cafc320bd64d9b45dc737cdfdb2 # HEAD of "master" as of 17.01.2016
+neutron_fwaas_git_install_branch: c213ddc915353207a1b0cc35d6d10370ca7eaade # HEAD of "stable/mitaka" as of 28.03.2016
 neutron_fwaas_git_dest: "/opt/neutron_fwaas_{{ neutron_fwaas_git_install_branch | replace('/', '_') }}"
 
 
 ## Nova service
 nova_git_repo: https://git.openstack.org/openstack/nova
-nova_git_install_branch: 2fc930a84e0e934c3f3d2c24679c698aeb98f8df # HEAD of "master" as of 01.02.2016
+nova_git_install_branch: 803171fe8abc2e466bef11c3bbafdacde87fa1f8 # HEAD of "stable/mitaka" as of 28.03.2016
 nova_git_dest: "/opt/nova_{{ nova_git_install_branch | replace('/', '_') }}"
 
 
 ## Swift service
 swift_git_repo: https://git.openstack.org/openstack/swift
-swift_git_install_branch: 4db7e2e2e4d80757a717485e3b639b16e0a66f68 # HEAD of "master" as of 17.01.2016
+swift_git_install_branch: dc363cf209e4662ed7ecdb7a49eaef00f0d4daca # HEAD of "stable/mitaka" as of 28.03.2016
 swift_git_dest: "/opt/swift_{{ swift_git_install_branch | replace('/', '_') }}"

playbooks/inventory/group_vars/hosts.yml

@@ -74,8 +74,8 @@ pip_links:
 # These are pinned to ensure exactly the same behaviour forever!
 # These pins are updated through the sources-branch-updater script
 pip_packages:
-  - pip==8.0.3
-  - setuptools==20.1.1
+  - pip==8.1.1
+  - setuptools==20.3.1
   - wheel==0.29.0
 
 ## Memcached options

requirements.txt

@@ -8,6 +8,6 @@ PyYAML>=3.1.0           # ansible
 ### These are pinned to ensure exactly the same behaviour forever!   ###
 ### These pins are updated through the sources-branch-updater script ###
 ###
-pip==8.0.3
-setuptools==20.1.1
+pip==8.1.1
+setuptools==20.3.1
 wheel==0.29.0

scripts/scripts-library.sh

@@ -21,7 +21,7 @@ MAX_RETRIES=${MAX_RETRIES:-5}
 REPORT_DATA=${REPORT_DATA:-""}
 ANSIBLE_PARAMETERS=${ANSIBLE_PARAMETERS:-""}
 STARTTIME="${STARTTIME:-$(date +%s)}"
-PIP_INSTALL_OPTIONS=${PIP_INSTALL_OPTIONS:-'pip==8.0.3 setuptools==20.1.1 wheel==0.29.0 '}
+PIP_INSTALL_OPTIONS=${PIP_INSTALL_OPTIONS:-'pip==8.1.1 setuptools==20.3.1 wheel==0.29.0 '}
 
 # The default SSHD configuration has MaxSessions = 10. If a deployer changes
 #  their SSHD config, then the FORKS may be set to a higher number. We set the

scripts/sources-branch-updater.sh

@@ -21,6 +21,9 @@
 ONLINE_BRANCH=${ONLINE_BRANCH:-"master"}
 SERVICE_FILE=${SERVICE_FILE:-"playbooks/defaults/repo_packages/openstack_services.yml"}
 OPENSTACK_SERVICE_LIST=${OPENSTACK_SERVICE_LIST:-"aodh ceilometer cinder glance heat keystone neutron nova"}
+OSA_ROLE_PATH="${OSA_ROLE_PATH:-playbooks/roles}"
+OSA_ROLE_PREFIX="${OSA_ROLE_PREFIX:-os}"
+OSA_ROLE_LOCAL="${OSA_ROLE_PATH}/${OSA_ROLE_PREFIX}"
 
 IFS=$'\n'
 
@@ -28,7 +31,9 @@ if echo "$@" | grep -e '-h' -e '--help';then
     echo "
 Options:
   -b|--branch          (name of branch, eg: stable/liberty)
-  -s|--service-file (path to service file to parse)
+  -s|--service-file    (path to service yaml file to parse)
+  -r|--osa-role-path   (path to stored roles to update, eg: 'playbooks/roles')
+  -p|--osa-role-prefix (the prefix in front of OpenStack service roles, eg: 'os')
 "
 exit 0
 fi
@@ -45,6 +50,14 @@ case $key in
     SERVICE_FILE="$2"
     shift
     ;;
+    -r|--osa-role-path)
+    OSA_ROLE_PATH="$2"
+    shift
+    ;;
+    -p|--osa-role-prefix)
+    OSA_ROLE_PREFIX="$2"
+    shift
+    ;;
     *)
     ;;
 esac
@@ -79,20 +92,21 @@ for repo in $(grep 'git_repo\:' ${SERVICE_FILE}); do
 
     # If the repo is in the specified list, then action the additional updates
     if [[ "${OPENSTACK_SERVICE_LIST}" =~ "${repo_name}" ]]; then
+      if [[ -d "${OSA_ROLE_LOCAL}_${repo_name}" ]]; then
         repo_tmp_path="/tmp/${repo_name}"
 
         # Ensure that the temp path doesn't exist
-      rm -rf ${repo_tmp_path}
+        [[ -d "repo_tmp_path" ]] && rm -rf "${repo_tmp_path}"
 
         # Do a shallow clone of the repo to work with
-      git clone --quiet --depth=10 --branch ${ONLINE_BRANCH} --no-checkout --single-branch ${repo_address} ${repo_tmp_path}
+        git clone --quiet --depth=5 --branch ${ONLINE_BRANCH} --no-checkout --single-branch ${repo_address} "${repo_tmp_path}"
         pushd ${repo_tmp_path} > /dev/null
-        git checkout --quiet ${branch_sha}
+          git checkout --quiet "${branch_sha}"
         popd > /dev/null
 
         # Update the policy files
         find ${repo_tmp_path}/etc -name "policy.json" -exec \
-        cp {} "playbooks/roles/os_${repo_name}/templates/policy.json.j2" \;
+          cp {} "${OSA_ROLE_LOCAL}_${repo_name}/templates/policy.json.j2" \;
 
         # Tweak the paste files
         find ${repo_tmp_path}/etc -name "*[_-]paste.ini" -exec \
@@ -100,7 +114,7 @@ for repo in $(grep 'git_repo\:' ${SERVICE_FILE}); do
 
         # Update the paste files
         find ${repo_tmp_path}/etc -name "*[_-]paste.ini" -exec \
-        bash -c "name=\"{}\"; cp \${name} \"playbooks/roles/os_${repo_name}/templates/\$(basename \${name}).j2\"" \;
+          bash -c "name=\"{}\"; cp \${name} \"${OSA_ROLE_LOCAL}_${repo_name}/templates/\$(basename \${name}).j2\"" \;
 
         # Tweak the rootwrap conf files
         find ${repo_tmp_path}/etc -name "rootwrap.conf" -exec \
@@ -108,26 +122,27 @@ for repo in $(grep 'git_repo\:' ${SERVICE_FILE}); do
 
         # Update the rootwrap conf files
         find ${repo_tmp_path}/etc -name "rootwrap.conf" -exec \
-        cp {} "playbooks/roles/os_${repo_name}/templates/rootwrap.conf.j2" \;
+          cp {} "${OSA_ROLE_LOCAL}_${repo_name}/templates/rootwrap.conf.j2" \;
 
         # Update the rootwrap filters
         find ${repo_tmp_path}/etc -name "*.filters" -exec \
-        bash -c "name=\"{}\"; cp \${name} \"playbooks/roles/os_${repo_name}/files/rootwrap.d/\$(basename \${name})\"" \;
+          bash -c "name=\"{}\"; cp \${name} \"${OSA_ROLE_LOCAL}_${repo_name}/files/rootwrap.d/\$(basename \${name})\"" \;
 
         # Update the yaml files for Ceilometer
         if [ "${repo_name}" = "ceilometer" ]; then
           find ${repo_tmp_path}/etc -name "*.yaml" -exec \
-          bash -c "name=\"{}\"; cp \${name} \"playbooks/roles/os_${repo_name}/templates/\$(basename \${name}).j2\"" \;
+            bash -c "name=\"{}\"; cp \${name} \"${OSA_ROLE_LOCAL}_${repo_name}/templates/\$(basename \${name}).j2\"" \;
         fi
 
         # Update the yaml files for Heat
         if [ "${repo_name}" = "heat" ]; then
           find ${repo_tmp_path}/etc -name "*.yaml" -exec \
-          bash -c "name=\"{}\"; cp \${name} \"playbooks/roles/os_${repo_name}/templates/\$(echo \${name} | rev | cut -sd / -f -2 | rev).j2\"" \;
+            bash -c "name=\"{}\"; cp \${name} \"${OSA_ROLE_LOCAL}_${repo_name}/templates/\$(echo \${name} | rev | cut -sd / -f -2 | rev).j2\"" \;
         fi
 
         # Clean up the temporary files
-      rm -rf ${repo_tmp_path}
+        [[ -d "repo_tmp_path" ]] && rm -rf "${repo_tmp_path}"
+      fi
     fi
   fi
 

tests/roles/bootstrap-host/templates/user_variables.aio.yml.j2

@@ -34,7 +34,11 @@ aodh_db_ip: {{ bootstrap_host_mongodb_address }}
 aodh_db_port: 27017
 
 ## Glance Options
-glance_default_store: swift
+# this has been set to file in the gate because of an upstream bug in glance_store:
+#  * https://bugs.launchpad.net/glance/+bug/1561947
+#  once this issue has been resolved this _needs_ to be reset to "swift" for full
+#  integration testing in a psuedo production environment.
+glance_default_store: file
 
 ## SSL Settings
 ssl_protocol: "ALL -SSLv2 -SSLv3"