From 746d269862ace91839a69e2a765a36ebdffde058 Mon Sep 17 00:00:00 2001 From: Lukas Bezdicka Date: Tue, 6 Oct 2020 10:10:50 +0200 Subject: [PATCH] [train-only] Introduce hybrid state also for ovn_controller Turns out OVN Controllers can't talk to DB with different versions. There is issue with docker insecure registries: - When setting insecure registries in /etc/containers docker ignores upate to this file on daemon reload. The registry settings will be ingored unless you restart. - Restarting docker restarts all the containers back up which might contradict with system_upgrade_stop_services. - Setting the insecure registries in /etc/docker/daemon.json would be an option if we didn't also provide the INSECURE_REGISTRY environment option to /etc/sysconfig/docker. This causes docker to complain on reload on duplicate option and it will ignore the /etc/docker/daemon.json settings. - This leaves us no other option than to restart docker. For this we list containers that were running before restart and compare it against containers running after the restart to shut down anything that should not be running. This is a bit racy but should work as one shot solution. - As a bonus we apply the change of image to hybrid state by applying the paunch config. This in any case will try to restart all the containers. This is potential issue for the neutron-sriov hybrid state as well as for the nova-compute hybrid state. If we hit same issue there we will have to apply fix in separate commit. Resolves: rhbz#1885212 Change-Id: I0e775e15b6e5e117e7ad01574a27081f337ecf20 --- .../ovn/ovn-controller-container-puppet.yaml | 84 ++++++++++++++++++- 1 file changed, 82 insertions(+), 2 deletions(-) diff --git a/deployment/ovn/ovn-controller-container-puppet.yaml b/deployment/ovn/ovn-controller-container-puppet.yaml index 83e3e7266e..cbe525b437 100644 --- a/deployment/ovn/ovn-controller-container-puppet.yaml +++ b/deployment/ovn/ovn-controller-container-puppet.yaml @@ -98,10 +98,17 @@ parameters: OpenvSwitch integration bridge, in seconds. type: number default: 60 + DockerInsecureRegistryAddress: + description: Optional. The IP Address and Port of an insecure docker + namespace that will be configured in /etc/sysconfig/docker. + The value can be multiple addresses separated by commas. + type: comma_delimited_list + default: [] conditions: force_config_drive: {equals: [{get_param: OVNMetadataEnabled}, false]} internal_tls_enabled: {equals: [{get_param: EnableInternalTLS}, true]} + insecure_registry_is_empty: {equals : [{get_param: DockerInsecureRegistryAddress}, []]} resources: @@ -295,12 +302,82 @@ outputs: persistent: yes state: yes upgrade_tasks: - - name: Switch ovn remote setting + - name: Gather missing facts + setup: + gather_subset: "distribution" + when: >- + ansible_facts['distribution'] is not defined or + ansible_facts['distribution_major_version'] is not defined tags: - never - nova_hybrid_state - when: step|int == 0 + - name: Switch ovn-controller to hybrid state + vars: + ovn_controller_image: {get_param: ContainerOvnControllerImage} + ovn_interaction_bridge: {get_param: OVNIntegrationBridge} + tags: + - never + - nova_hybrid_state + when: + - step|int == 0 + - ansible_facts['distribution'] == 'RedHat' + - ansible_facts['distribution_major_version'] is version('7', '==') block: + - name: Check if we need to update the ovn_controller paunch config + shell: | + set -o pipefail + jq ."ovn_controller"."image" /var/lib/tripleo-config/docker-container-startup-config-step_4.json + register: ovn_controller_paunch_image + - name: Implement the ovn_controller hybrid state (only if the controller is still Queens) + when: ovn_controller_paunch_image.stdout != ovn_controller_image + block: + - name: Update the ovn_controller paunch image in config + shell: | + set -o pipefail + cat <<< $(jq '.ovn_controller.image = "{{ ovn_controller_image }}"' \ + /var/lib/tripleo-config/docker-container-startup-config-step_4.json) >\ + /var/lib/tripleo-config/docker-container-startup-config-step_4.json + - name: Add new volumes to ovn_controller config + shell: | + set -o pipefail + cat <<< $(jq '.ovn_controller.volumes += ["/var/lib/openvswitch/ovn:/run/ovn:shared", "/var/log/containers/openvswitch:/var/log/ovn"]' \ + /var/lib/tripleo-config/docker-container-startup-config-step_4.json) >\ + /var/lib/tripleo-config/docker-container-startup-config-step_4.json + - name: Make sure the Undercloud hostname is included in /etc/hosts + when: + - undercloud_hosts_entries is defined + lineinfile: + dest: /etc/hosts + line: "{{ undercloud_hosts_entries | join('') }}" + state: present + - name: Set container_registry_insecure_registries fact. + set_fact: + container_registry_insecure_registries: + if: + - insecure_registry_is_empty + - [] + - {get_param: DockerInsecureRegistryAddress} + - name: Set container_registry_insecure registries + when: container_registry_insecure_registries != [] + shell: crudini --set /etc/containers/registries.conf registries.insecure registries "[{{ container_registry_insecure_registries | map('regex_replace', '(.*)', "'\1'") | join(',') }}]" + - name: Restart docker and apply the paunch config + when: container_registry_insecure_registries != [] + shell: | + set -o pipefail + # Get list of running containers + RUNNING="$( docker ps --format '{{ '{{' }}.Names{{ '}}' }}' )" + # Restart docker + systemctl restart docker + # Apply the paunch so if we start even more stuff we start it before shutting down + paunch apply --file /var/lib/tripleo-config/docker-container-startup-config-step_4.json --config-id tripleo_step4 + # Compare running containers now vs before + TO_STOP="$(grep -v -f <(echo "${RUNNING}") <(docker ps --format '{{ '{{' }}.Names{{ '}}' }}'))" + # Check if we need to stop anything and stop it + if [ -n "${TO_STOP}" ]; then + echo "${TO_STOP}" | xargs -r docker stop + fi + args: + executable: /usr/bin/bash - name: Get ovn remote setting shell: | ovs-vsctl get open . external_ids:ovn-remote @@ -312,3 +389,6 @@ outputs: shell: | ovs-vsctl set open . external_ids:ovn-remote="{{ ovn_sb_conn_str }}" when: ovn_sb_conn_str not in ovn_remote.stdout + - name: Update OVNIntegrationBridge protocols to OpenFlow13,OpenFlow15 + shell: | + ovs-vsctl set bridge {{ ovn_interaction_bridge }} protocols="OpenFlow13,OpenFlow15"