diff --git a/doc/source/admin/upgrades/major-upgrades-manual-upgrade.rst b/doc/source/admin/upgrades/major-upgrades-manual-upgrade.rst index 684a76b94d..e48e7f63f7 100644 --- a/doc/source/admin/upgrades/major-upgrades-manual-upgrade.rst +++ b/doc/source/admin/upgrades/major-upgrades-manual-upgrade.rst @@ -252,6 +252,26 @@ see :ref:`memcached-flush`. # openstack-ansible "${UPGRADE_PLAYBOOKS}/memcached-flush.yml" +Implement inventory to deploy neutron agents on network_hosts +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +In releases prior to Queens, neutron agents were deployed in a container. This +turned out to be problematic in major upgrades where the LXC container +configuration may have changed, resulting in the containers restarting and +therefore all L3 networking going down for some time. + +To prevent this happening in the future, the neutron agents are now deployed +on the network_hosts directly (not in containers). This ensures that whenever +an upgrade is run, the L3 networks do not go down. + +In order to handle this transition, we need to temporarily implement a +temporary inventory change which adds the network_hosts into each of the +agent groups so that the os-neutron-install playbook installs agents on them. + +.. code-block:: console + + # openstack-ansible "${UPGRADE_PLAYBOOKS}/neutron-tmp-inventory.yml" + Upgrade OpenStack ~~~~~~~~~~~~~~~~~ @@ -279,6 +299,9 @@ changes to the container/service setup: ``ironic_conductor_container`` can be removed. # All nova services are consolidated into the ``nova_api_container`` and the rest of the nova containers can be removed. +# All neutron agents are moved from containers onto the network_hosts. + The previously implemented ``neutron_agents_container`` can therefore + be removed. # All trove services have been consolidated into the ``trove_api_container``. The previously implemented ``trove_conductor_container`` and ``trove_taskmanager_container`` @@ -303,3 +326,4 @@ from the haproxy configuration. # openstack-ansible "${UPGRADE_PLAYBOOKS}/cleanup-nova.yml" -e force_containers_destroy=yes -e force_containers_data_destroy=yes # openstack-ansible "${UPGRADE_PLAYBOOKS}/cleanup-trove.yml" -e force_containers_destroy=yes -e force_containers_data_destroy=yes # openstack-ansible --tags haproxy_server-config haproxy-install.yml + # openstack-ansible "${UPGRADE_PLAYBOOKS}/cleanup-neutron.yml" -e force_containers_destroy=yes -e force_containers_data_destroy=yes diff --git a/releasenotes/notes/neutron-agent-auto-migration-a8b160509d7f83d0.yaml b/releasenotes/notes/neutron-agent-auto-migration-a8b160509d7f83d0.yaml new file mode 100644 index 0000000000..511d165d14 --- /dev/null +++ b/releasenotes/notes/neutron-agent-auto-migration-a8b160509d7f83d0.yaml @@ -0,0 +1,9 @@ +--- +upgrade: + - | + During an upgrade using the run-upgrade script, the neutron agents will + now automatically be migrated from the neutron_agents containers on to + the network_hosts. The neutron_agents containers will be deleted as they + are no longer necessary. Any environments which previously upgraded to + Queens can make use of the same playbooks to handle to migration, or + inspect the playbooks to determine how to do it by hand if preferred. diff --git a/scripts/run-upgrade.sh b/scripts/run-upgrade.sh index 2bc9a8d0b4..7261ec7b1e 100755 --- a/scripts/run-upgrade.sh +++ b/scripts/run-upgrade.sh @@ -196,6 +196,7 @@ function main { RUN_TASKS+=("utility-install.yml") RUN_TASKS+=("rsyslog-install.yml") RUN_TASKS+=("${UPGRADE_PLAYBOOKS}/memcached-flush.yml") + RUN_TASKS+=("${UPGRADE_PLAYBOOKS}/neutron-tmp-inventory.yml") RUN_TASKS+=("setup-openstack.yml") # clean up the containers which are no longer required # now that the services are hyperconverged @@ -204,7 +205,12 @@ function main { RUN_TASKS+=("${UPGRADE_PLAYBOOKS}/cleanup-heat.yml -e force_containers_destroy=yes -e force_containers_data_destroy=yes") RUN_TASKS+=("${UPGRADE_PLAYBOOKS}/cleanup-ironic.yml -e force_containers_destroy=yes -e force_containers_data_destroy=yes") RUN_TASKS+=("${UPGRADE_PLAYBOOKS}/cleanup-trove.yml -e force_containers_destroy=yes -e force_containers_data_destroy=yes") + # reconfigure haproxy to ensure that the old back-ends are removed RUN_TASKS+=("haproxy-install-rerun.yml --tags=haproxy_server-config") + # finalise the migration of the neutron agents to bare + # metal and clean up the neutron agent containers which + # are no longer required + RUN_TASKS+=("${UPGRADE_PLAYBOOKS}/cleanup-neutron.yml -e force_containers_destroy=yes -e force_containers_data_destroy=yes") # Run the tasks in order for item in ${!RUN_TASKS[@]}; do run_lock $item "${RUN_TASKS[$item]}" diff --git a/scripts/upgrade-utilities/files/neutron-migration-inventory.ini b/scripts/upgrade-utilities/files/neutron-migration-inventory.ini new file mode 100644 index 0000000000..9a4d366237 --- /dev/null +++ b/scripts/upgrade-utilities/files/neutron-migration-inventory.ini @@ -0,0 +1,33 @@ +# This is an Ansible inventory to be used for the Pike->Queens +# transition of neutron agents to the bare metal hosts. Once +# the agents have been transitioned, this file should be removed. + +[neutron_agent:children] +network_hosts + +[neutron_bgp_dragent:children] +network_hosts + +[neutron_dhcp_agent:children] +network_hosts + +[neutron_l3_agent:children] +network_hosts + +[neutron_lbaas_agent:children] +network_hosts + +[neutron_linuxbridge_agent:children] +network_hosts + +[neutron_metadata_agent:children] +network_hosts + +[neutron_metering_agent:children] +network_hosts + +[neutron_openvswitch_agent:children] +network_hosts + +[neutron_sriov_nic_agent:children] +network_hosts diff --git a/scripts/upgrade-utilities/playbooks/cleanup-neutron.yml b/scripts/upgrade-utilities/playbooks/cleanup-neutron.yml new file mode 100644 index 0000000000..bad5d18606 --- /dev/null +++ b/scripts/upgrade-utilities/playbooks/cleanup-neutron.yml @@ -0,0 +1,121 @@ +--- +# Copyright 2018, Rackspace US, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +- hosts: neutron_agents_container + become: yes + gather_facts: no + tasks: + - name: Gather the necessary facts + setup: + gather_subset: "!facter,!ohai" + +- hosts: utility[0] + become: yes + gather_facts: no + tasks: + - name: Collect the ID's for the neutron services to delete + command: >- + openstack --os-cloud default network agent list + --format value --column ID + --host {{ service_host }} + register: _neutron_service_to_remove + until: _neutron_service_to_remove is succeeded + retries: 5 + delay: 2 + vars: + service_host: "{{ hostvars[item]['ansible_hostname'] }}" + with_items: "{{ groups['neutron_agents_container'] }}" + + - name: Disable neutron services in containers which will be deleted + command: >- + openstack --os-cloud default network agent set --disable {{ item }} + with_items: "{{ _neutron_service_to_remove.results | json_query('[].stdout_lines[]') }}" + register: _neutron_service_disable + until: _neutron_service_disable is succeeded + retries: 5 + delay: 2 + +- hosts: neutron_agents_container + become: yes + gather_facts: no + tasks: + - name: Discover the list of services to shut down in containers which will be deleted + shell: "systemctl list-unit-files --state=enabled --type=service | awk '/neutron.* enabled$/ {print $1}'" + args: + executable: "/bin/bash" + register: _enabled_services + changed_when: false + + - name: Shut down neutron services in containers which will be deleted + service: + name: "{{ item }}" + enabled: no + state: stopped + with_items: "{{ _enabled_services.stdout_lines }}" + +- include: "{{ playbook_dir }}/../../../playbooks/lxc-containers-destroy.yml" + vars: + container_group: "neutron_agents_container" + +- hosts: utility[0] + become: yes + gather_facts: no + tasks: + - name: Collect the ID's for the neutron services to delete + command: >- + openstack --os-cloud default network agent list + --format value --column ID + --host {{ service_host }} + register: _neutron_service_to_remove + until: _neutron_service_to_remove is succeeded + retries: 5 + delay: 2 + vars: + service_host: "{{ hostvars[item]['ansible_hostname'] }}" + with_items: "{{ groups['neutron_agents_container'] }}" + + - name: Delete the neutron services which were running in the deleted containers + command: >- + openstack --os-cloud default network agent delete {{ item }} + with_items: "{{ _neutron_service_to_remove.results | json_query('[].stdout_lines[]') }}" + register: _neutron_service_delete + until: _neutron_service_delete is succeeded + retries: 5 + delay: 2 + +- hosts: localhost + connection: local + become: yes + gather_facts: no + tasks: + - name: Remove the inventory entries for the deleted containers + command: >- + {{ playbook_dir }}/../../inventory-manage.py -r {{ item }} + with_items: "{{ groups['neutron_agents_container'] }}" + + - name: Remove the transitional user-space inventory file + shell: | + set -e + if [[ -e /etc/openstack_deploy/inventory.ini.org ]]; then + echo "Reverting the inventory.ini in /etc/openstack_deploy to the original backup" + mv /etc/openstack_deploy/inventory.ini.org /etc/openstack_deploy/inventory.ini + exit 2 + fi + args: + executable: /bin/bash + register: _remove_ini + changed_when: _remove_ini.rc == 2 + failed_when: _remove_ini.rc not in [0,2] + diff --git a/scripts/upgrade-utilities/playbooks/neutron-tmp-inventory.yml b/scripts/upgrade-utilities/playbooks/neutron-tmp-inventory.yml new file mode 100644 index 0000000000..df3efd22f4 --- /dev/null +++ b/scripts/upgrade-utilities/playbooks/neutron-tmp-inventory.yml @@ -0,0 +1,40 @@ +--- +# Copyright 2018, Rackspace US, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +- hosts: localhost + connection: local + become: yes + gather_facts: no + tasks: + - name: Implement the user-space transitional inventory + shell: | + set -e + if [[ ! -e /etc/openstack_deploy/inventory.ini.org ]]; then + if [[ -e /etc/openstack_deploy/inventory.ini ]]; then + echo "Backing up the existing inventory.ini in /etc/openstack_deploy" + mv /etc/openstack_deploy/inventory.ini /etc/openstack_deploy/inventory.ini.org + else + echo "Adding a new inventory.ini into /etc/openstack_deploy" + echo '[all]' > /etc/openstack_deploy/inventory.ini.org + fi + echo "Copying the transitional inventory.ini for neutron into /etc/openstack_deploy" + cp {{ playbook_dir }}/../files/neutron-migration-inventory.ini /etc/openstack_deploy/inventory.ini + exit 2 + fi + args: + executable: /bin/bash + register: _add_ini + changed_when: _add_ini.rc == 2 + failed_when: _add_ini.rc not in [0,2]