From f635e090bd62c202e89d5fd87bd8198cfa68c534 Mon Sep 17 00:00:00 2001 From: Jesse Pretorius Date: Wed, 23 Mar 2016 21:04:07 +0000 Subject: [PATCH] Updates all repo SHAs to prepare for Mitaka release This patch includes updates of all SHA's and pip, wheel and setuptools pins. Due to Bug #1561947, the backend glance store has been changed from swift to file in the AIO gate. This has been done becuase the glance store project has an issue with session that causes it to be incompatible with swift. Being that the master integration gate is depending on this change to go it is needed to temporarily disable the swift image store in glance until this issue is fixed. The branch updater sciprt was modified to search for and update roles if they're found within the default or user provided role path. This change is resolving bash errors that happen when the script is run in mitaka+. Related-Bug: 1561947 Change-Id: I2e09e3e0abb61f5e97e2af7b283f0aed6dadd853 Signed-off-by: Kevin Carter --- .../repo_packages/openstack_other.yml | 6 +- .../repo_packages/openstack_services.yml | 28 +++--- playbooks/inventory/group_vars/hosts.yml | 4 +- requirements.txt | 4 +- scripts/scripts-library.sh | 2 +- scripts/sources-branch-updater.sh | 97 +++++++++++-------- .../templates/user_variables.aio.yml.j2 | 6 +- 7 files changed, 83 insertions(+), 64 deletions(-) diff --git a/playbooks/defaults/repo_packages/openstack_other.yml b/playbooks/defaults/repo_packages/openstack_other.yml index 9f14284955..dae40ba3d8 100644 --- a/playbooks/defaults/repo_packages/openstack_other.yml +++ b/playbooks/defaults/repo_packages/openstack_other.yml @@ -27,17 +27,17 @@ ## Tempest service tempest_git_repo: https://git.openstack.org/openstack/tempest -tempest_git_install_branch: 5cc7ef78b4233444a4dcea1b1eb8f213c1548491 # HEAD of "master" as of 17.01.2016 +tempest_git_install_branch: fd3088f8992633f6eb005e606d3bb8596fa23d56 # HEAD of "master" as of 23.03.2016 tempest_git_dest: "/opt/tempest_{{ tempest_git_install_branch | replace('/', '_') }}" ## NOVNC from source novncproxy_git_repo: https://github.com/kanaka/novnc -novncproxy_git_install_branch: 670dbddb54264fd0082d0aca1b3acb0f1814b1d2 # HEAD of "master" as of 17.01.2016 +novncproxy_git_install_branch: b403cb92fb8de82d04f305b4f14fa978003890d7 # HEAD of "master" as of 23.03.2016 novncproxy_git_dest: "/opt/novnc_{{ novncproxy_git_install_branch | replace('/', '_') }}" ## spice-html5 from source spicehtml5_git_repo: https://github.com/SPICE/spice-html5 -spicehtml5_git_install_branch: ab73d009487c8afd4def39b54a422499b4c13c40 # HEAD of "master" as of 17.01.2016 +spicehtml5_git_install_branch: ab73d009487c8afd4def39b54a422499b4c13c40 # HEAD of "master" as of 23.03.2016 spicehtml5_git_dest: "/opt/spicehtml5_{{ spicehtml5_git_install_branch | replace('/', '_') }}" diff --git a/playbooks/defaults/repo_packages/openstack_services.yml b/playbooks/defaults/repo_packages/openstack_services.yml index 450ae880c7..4d9a75a80a 100644 --- a/playbooks/defaults/repo_packages/openstack_services.yml +++ b/playbooks/defaults/repo_packages/openstack_services.yml @@ -31,77 +31,77 @@ ## Global Requirements requirements_git_repo: https://git.openstack.org/openstack/requirements -requirements_git_install_branch: 0a0352fb3065308647e0fd49e15b1b733135501a # HEAD of "master" as of 01.02.2016 +requirements_git_install_branch: d371aad12f492315122f2a5ab86db4678dbacfd3 # HEAD of "stable/mitaka" as of 28.03.2016 requirements_git_dest: "/opt/requirements_{{ requirements_git_install_branch | replace('/', '_') }}" ## Aodh service aodh_git_repo: https://git.openstack.org/openstack/aodh -aodh_git_install_branch: 239e1f629b26557ceadb92de3d62edcd87489b9d # HEAD of "master" as of 17.01.2016 +aodh_git_install_branch: 235f79f3cdd9b5fb607a06011ead7a3962c91921 # HEAD of "stable/mitaka" as of 28.03.2016 aodh_git_dest: "/opt/aodh_{{ aodh_git_install_branch | replace('/', '_') }}" ## Ceilometer service ceilometer_git_repo: https://git.openstack.org/openstack/ceilometer -ceilometer_git_install_branch: 333024b69aa7810e78aef85e5171cfd6dbd6b740 # HEAD of "master" as of 17.01.2016 +ceilometer_git_install_branch: 7ee29aae2835c9eb0d1a747db8ba06b56f74082c # HEAD of "stable/mitaka" as of 28.03.2016 ceilometer_git_dest: "/opt/ceilometer_{{ceilometer_git_install_branch | replace('/', '_') }}" ## Cinder service cinder_git_repo: https://git.openstack.org/openstack/cinder -cinder_git_install_branch: 94ae8598b96e2f86844fdf0f35a8b83a94c7b4c4 # HEAD of "master" as of 17.01.2016 +cinder_git_install_branch: da28ab66c37fa9c762d7562f68ce5b07f2240a6b # HEAD of "stable/mitaka" as of 28.03.2016 cinder_git_dest: "/opt/cinder_{{ cinder_git_install_branch | replace('/', '_') }}" ## Glance service glance_git_repo: https://git.openstack.org/openstack/glance -glance_git_install_branch: 7d5c3710ce2739a8ac356208d4e104f2ce3ec9ab # HEAD of "master" as of 17.01.2016 +glance_git_install_branch: 1cb2bdaf613878ee290c43807d1ab3893bedada2 # HEAD of "stable/mitaka" as of 28.03.2016 glance_git_dest: "/opt/glance_{{ glance_git_install_branch | replace('/', '_') }}" ## Heat service heat_git_repo: https://git.openstack.org/openstack/heat -heat_git_install_branch: 7e3e4087f476a0431d1d278730b1736e02e5fd06 # HEAD of "master" as of 17.01.2016 +heat_git_install_branch: 588e7e385e316d87a21b62be526d7e6ba4bb2ad6 # HEAD of "stable/mitaka" as of 28.03.2016 heat_git_dest: "/opt/heat_{{ heat_git_install_branch | replace('/', '_') }}" ## Horizon service horizon_git_repo: https://git.openstack.org/openstack/horizon -horizon_git_install_branch: 18f1605bddd428a014d0e43ef52d1af6305e1e03 # HEAD of "master" as of 17.01.2016 +horizon_git_install_branch: 2143bb60e9a84052ebd54a43b3dfb740b95f338a # HEAD of "stable/mitaka" as of 28.03.2016 horizon_git_dest: "/opt/horizon_{{ horizon_git_install_branch | replace('/', '_') }}" ## Keystone service keystone_git_repo: https://git.openstack.org/openstack/keystone -keystone_git_install_branch: a55128044f763f5cfe2fdc57c738eaca97636448 # HEAD of "master" as of 17.01.2016 +keystone_git_install_branch: aa53ad975fdc1a70091b87400c85ff4f1d658d2c # HEAD of "stable/mitaka" as of 28.03.2016 keystone_git_dest: "/opt/keystone_{{ keystone_git_install_branch | replace('/', '_') }}" ## Neutron service neutron_git_repo: https://git.openstack.org/openstack/neutron -neutron_git_install_branch: d6d43b32ca825b6c3c2c908f5ff7bc50c736546e # HEAD of "master" as of 17.01.2016 +neutron_git_install_branch: 439740e92cdb86d65b703dac95e7095fdf569619 # HEAD of "stable/mitaka" as of 28.03.2016 neutron_git_dest: "/opt/neutron_{{ neutron_git_install_branch | replace('/', '_') }}" neutron_lbaas_git_repo: https://git.openstack.org/openstack/neutron-lbaas -neutron_lbaas_git_install_branch: b5d4e5c0fe02a897ad2ab0bc548f695915998831 # HEAD of "master" as of 17.01.2016 +neutron_lbaas_git_install_branch: a29a6013ebaf2b9a11ae734f5fa9926d56a4718d # HEAD of "stable/mitaka" as of 28.03.2016 neutron_lbaas_git_dest: "/opt/neutron_lbaas_{{ neutron_lbaas_git_install_branch | replace('/', '_') }}" neutron_vpnaas_git_repo: https://git.openstack.org/openstack/neutron-vpnaas -neutron_vpnaas_git_install_branch: 832b875b79d801e17a5b997054f30c9d88b36914 # HEAD of "master" as of 17.01.2016 +neutron_vpnaas_git_install_branch: 10a56ce65329724a460b133328e8c9e6d16706a4 # HEAD of "stable/mitaka" as of 28.03.2016 neutron_vpnaas_git_dest: "/opt/neutron_vpnaas_{{ neutron_vpnaas_git_install_branch | replace('/', '_') }}" neutron_fwaas_git_repo: https://git.openstack.org/openstack/neutron-fwaas -neutron_fwaas_git_install_branch: cb0093d185a97cafc320bd64d9b45dc737cdfdb2 # HEAD of "master" as of 17.01.2016 +neutron_fwaas_git_install_branch: c213ddc915353207a1b0cc35d6d10370ca7eaade # HEAD of "stable/mitaka" as of 28.03.2016 neutron_fwaas_git_dest: "/opt/neutron_fwaas_{{ neutron_fwaas_git_install_branch | replace('/', '_') }}" ## Nova service nova_git_repo: https://git.openstack.org/openstack/nova -nova_git_install_branch: 2fc930a84e0e934c3f3d2c24679c698aeb98f8df # HEAD of "master" as of 01.02.2016 +nova_git_install_branch: 803171fe8abc2e466bef11c3bbafdacde87fa1f8 # HEAD of "stable/mitaka" as of 28.03.2016 nova_git_dest: "/opt/nova_{{ nova_git_install_branch | replace('/', '_') }}" ## Swift service swift_git_repo: https://git.openstack.org/openstack/swift -swift_git_install_branch: 4db7e2e2e4d80757a717485e3b639b16e0a66f68 # HEAD of "master" as of 17.01.2016 +swift_git_install_branch: dc363cf209e4662ed7ecdb7a49eaef00f0d4daca # HEAD of "stable/mitaka" as of 28.03.2016 swift_git_dest: "/opt/swift_{{ swift_git_install_branch | replace('/', '_') }}" diff --git a/playbooks/inventory/group_vars/hosts.yml b/playbooks/inventory/group_vars/hosts.yml index 925e5ec6a2..b397fca6fc 100644 --- a/playbooks/inventory/group_vars/hosts.yml +++ b/playbooks/inventory/group_vars/hosts.yml @@ -74,8 +74,8 @@ pip_links: # These are pinned to ensure exactly the same behaviour forever! # These pins are updated through the sources-branch-updater script pip_packages: - - pip==8.0.3 - - setuptools==20.1.1 + - pip==8.1.1 + - setuptools==20.3.1 - wheel==0.29.0 ## Memcached options diff --git a/requirements.txt b/requirements.txt index 0219c4d97c..b3f9ea65c5 100644 --- a/requirements.txt +++ b/requirements.txt @@ -8,6 +8,6 @@ PyYAML>=3.1.0 # ansible ### These are pinned to ensure exactly the same behaviour forever! ### ### These pins are updated through the sources-branch-updater script ### ### -pip==8.0.3 -setuptools==20.1.1 +pip==8.1.1 +setuptools==20.3.1 wheel==0.29.0 diff --git a/scripts/scripts-library.sh b/scripts/scripts-library.sh index 7c0e716846..ba93b80840 100755 --- a/scripts/scripts-library.sh +++ b/scripts/scripts-library.sh @@ -21,7 +21,7 @@ MAX_RETRIES=${MAX_RETRIES:-5} REPORT_DATA=${REPORT_DATA:-""} ANSIBLE_PARAMETERS=${ANSIBLE_PARAMETERS:-""} STARTTIME="${STARTTIME:-$(date +%s)}" -PIP_INSTALL_OPTIONS=${PIP_INSTALL_OPTIONS:-'pip==8.0.3 setuptools==20.1.1 wheel==0.29.0 '} +PIP_INSTALL_OPTIONS=${PIP_INSTALL_OPTIONS:-'pip==8.1.1 setuptools==20.3.1 wheel==0.29.0 '} # The default SSHD configuration has MaxSessions = 10. If a deployer changes # their SSHD config, then the FORKS may be set to a higher number. We set the diff --git a/scripts/sources-branch-updater.sh b/scripts/sources-branch-updater.sh index be8e5ce191..6bb73aad47 100755 --- a/scripts/sources-branch-updater.sh +++ b/scripts/sources-branch-updater.sh @@ -21,14 +21,19 @@ ONLINE_BRANCH=${ONLINE_BRANCH:-"master"} SERVICE_FILE=${SERVICE_FILE:-"playbooks/defaults/repo_packages/openstack_services.yml"} OPENSTACK_SERVICE_LIST=${OPENSTACK_SERVICE_LIST:-"aodh ceilometer cinder glance heat keystone neutron nova"} +OSA_ROLE_PATH="${OSA_ROLE_PATH:-playbooks/roles}" +OSA_ROLE_PREFIX="${OSA_ROLE_PREFIX:-os}" +OSA_ROLE_LOCAL="${OSA_ROLE_PATH}/${OSA_ROLE_PREFIX}" IFS=$'\n' if echo "$@" | grep -e '-h' -e '--help';then echo " Options: - -b|--branch (name of branch, eg: stable/liberty) - -s|--service-file (path to service file to parse) + -b|--branch (name of branch, eg: stable/liberty) + -s|--service-file (path to service yaml file to parse) + -r|--osa-role-path (path to stored roles to update, eg: 'playbooks/roles') + -p|--osa-role-prefix (the prefix in front of OpenStack service roles, eg: 'os') " exit 0 fi @@ -45,6 +50,14 @@ case $key in SERVICE_FILE="$2" shift ;; + -r|--osa-role-path) + OSA_ROLE_PATH="$2" + shift + ;; + -p|--osa-role-prefix) + OSA_ROLE_PREFIX="$2" + shift + ;; *) ;; esac @@ -79,55 +92,57 @@ for repo in $(grep 'git_repo\:' ${SERVICE_FILE}); do # If the repo is in the specified list, then action the additional updates if [[ "${OPENSTACK_SERVICE_LIST}" =~ "${repo_name}" ]]; then - repo_tmp_path="/tmp/${repo_name}" + if [[ -d "${OSA_ROLE_LOCAL}_${repo_name}" ]]; then + repo_tmp_path="/tmp/${repo_name}" - # Ensure that the temp path doesn't exist - rm -rf ${repo_tmp_path} + # Ensure that the temp path doesn't exist + [[ -d "repo_tmp_path" ]] && rm -rf "${repo_tmp_path}" - # Do a shallow clone of the repo to work with - git clone --quiet --depth=10 --branch ${ONLINE_BRANCH} --no-checkout --single-branch ${repo_address} ${repo_tmp_path} - pushd ${repo_tmp_path} > /dev/null - git checkout --quiet ${branch_sha} - popd > /dev/null + # Do a shallow clone of the repo to work with + git clone --quiet --depth=5 --branch ${ONLINE_BRANCH} --no-checkout --single-branch ${repo_address} "${repo_tmp_path}" + pushd ${repo_tmp_path} > /dev/null + git checkout --quiet "${branch_sha}" + popd > /dev/null - # Update the policy files - find ${repo_tmp_path}/etc -name "policy.json" -exec \ - cp {} "playbooks/roles/os_${repo_name}/templates/policy.json.j2" \; + # Update the policy files + find ${repo_tmp_path}/etc -name "policy.json" -exec \ + cp {} "${OSA_ROLE_LOCAL}_${repo_name}/templates/policy.json.j2" \; - # Tweak the paste files - find ${repo_tmp_path}/etc -name "*[_-]paste.ini" -exec \ - sed -i.bak "s|hmac_keys = SECRET_KEY|hmac_keys = {{ ${repo_name}_profiler_hmac_key }}|" {} \; + # Tweak the paste files + find ${repo_tmp_path}/etc -name "*[_-]paste.ini" -exec \ + sed -i.bak "s|hmac_keys = SECRET_KEY|hmac_keys = {{ ${repo_name}_profiler_hmac_key }}|" {} \; - # Update the paste files - find ${repo_tmp_path}/etc -name "*[_-]paste.ini" -exec \ - bash -c "name=\"{}\"; cp \${name} \"playbooks/roles/os_${repo_name}/templates/\$(basename \${name}).j2\"" \; + # Update the paste files + find ${repo_tmp_path}/etc -name "*[_-]paste.ini" -exec \ + bash -c "name=\"{}\"; cp \${name} \"${OSA_ROLE_LOCAL}_${repo_name}/templates/\$(basename \${name}).j2\"" \; - # Tweak the rootwrap conf files - find ${repo_tmp_path}/etc -name "rootwrap.conf" -exec \ - sed -i.bak "s|exec_dirs=|exec_dirs={{ ${repo_name}_bin }},|" {} \; + # Tweak the rootwrap conf files + find ${repo_tmp_path}/etc -name "rootwrap.conf" -exec \ + sed -i.bak "s|exec_dirs=|exec_dirs={{ ${repo_name}_bin }},|" {} \; - # Update the rootwrap conf files - find ${repo_tmp_path}/etc -name "rootwrap.conf" -exec \ - cp {} "playbooks/roles/os_${repo_name}/templates/rootwrap.conf.j2" \; + # Update the rootwrap conf files + find ${repo_tmp_path}/etc -name "rootwrap.conf" -exec \ + cp {} "${OSA_ROLE_LOCAL}_${repo_name}/templates/rootwrap.conf.j2" \; - # Update the rootwrap filters - find ${repo_tmp_path}/etc -name "*.filters" -exec \ - bash -c "name=\"{}\"; cp \${name} \"playbooks/roles/os_${repo_name}/files/rootwrap.d/\$(basename \${name})\"" \; + # Update the rootwrap filters + find ${repo_tmp_path}/etc -name "*.filters" -exec \ + bash -c "name=\"{}\"; cp \${name} \"${OSA_ROLE_LOCAL}_${repo_name}/files/rootwrap.d/\$(basename \${name})\"" \; - # Update the yaml files for Ceilometer - if [ "${repo_name}" = "ceilometer" ]; then - find ${repo_tmp_path}/etc -name "*.yaml" -exec \ - bash -c "name=\"{}\"; cp \${name} \"playbooks/roles/os_${repo_name}/templates/\$(basename \${name}).j2\"" \; + # Update the yaml files for Ceilometer + if [ "${repo_name}" = "ceilometer" ]; then + find ${repo_tmp_path}/etc -name "*.yaml" -exec \ + bash -c "name=\"{}\"; cp \${name} \"${OSA_ROLE_LOCAL}_${repo_name}/templates/\$(basename \${name}).j2\"" \; + fi + + # Update the yaml files for Heat + if [ "${repo_name}" = "heat" ]; then + find ${repo_tmp_path}/etc -name "*.yaml" -exec \ + bash -c "name=\"{}\"; cp \${name} \"${OSA_ROLE_LOCAL}_${repo_name}/templates/\$(echo \${name} | rev | cut -sd / -f -2 | rev).j2\"" \; + fi + + # Clean up the temporary files + [[ -d "repo_tmp_path" ]] && rm -rf "${repo_tmp_path}" fi - - # Update the yaml files for Heat - if [ "${repo_name}" = "heat" ]; then - find ${repo_tmp_path}/etc -name "*.yaml" -exec \ - bash -c "name=\"{}\"; cp \${name} \"playbooks/roles/os_${repo_name}/templates/\$(echo \${name} | rev | cut -sd / -f -2 | rev).j2\"" \; - fi - - # Clean up the temporary files - rm -rf ${repo_tmp_path} fi fi diff --git a/tests/roles/bootstrap-host/templates/user_variables.aio.yml.j2 b/tests/roles/bootstrap-host/templates/user_variables.aio.yml.j2 index 8bb72c1884..9a81f01b94 100644 --- a/tests/roles/bootstrap-host/templates/user_variables.aio.yml.j2 +++ b/tests/roles/bootstrap-host/templates/user_variables.aio.yml.j2 @@ -34,7 +34,11 @@ aodh_db_ip: {{ bootstrap_host_mongodb_address }} aodh_db_port: 27017 ## Glance Options -glance_default_store: swift +# this has been set to file in the gate because of an upstream bug in glance_store: +# * https://bugs.launchpad.net/glance/+bug/1561947 +# once this issue has been resolved this _needs_ to be reset to "swift" for full +# integration testing in a psuedo production environment. +glance_default_store: file ## SSL Settings ssl_protocol: "ALL -SSLv2 -SSLv3"