Make the openvswitch 2.4->2.5 upgrade more robust

In I9b1f0eaa0d36a28e20b507bec6a4e9b3af1781ae and
I11fcf688982ceda5eef7afc8904afae44300c2d9 we added a manual step
for upgrading openvswitch in order to specify the --nopostun
as discussed in the bug below.

This change adds a minor update to make this workaround more
robust. It removes any existing rpms that may be around from
an earlier run, and also checks that the rpms installed are
at least newer than the version we are on.

This also refactors the code into a common definition in the
pacemaker_common_functions.sh which is included even for the
heredocs generating upgrade scripts during init. Thanks
Sofer Athlan-Guyot and Jirka Stransky for help with that.

Change-Id: Idc863de7b5a8c116c990ee8c1472cfe377836d37
Related-Bug: 1635205

extraconfig/tasks/major_upgrade_block_storage.sh

@@ -5,18 +5,7 @@
 set -eu
 
 # Special-case OVS for https://bugs.launchpad.net/tripleo/+bug/1635205
-if [[ -n $(rpm -q --scripts openvswitch | awk '/postuninstall/,/*/' | grep "systemctl.*try-restart") ]]; then
-    echo "Manual upgrade of openvswitch - restart in postun detected"
-    mkdir OVS_UPGRADE || true
-    pushd OVS_UPGRADE
-    echo "Attempting to downloading latest openvswitch with yumdownloader"
-    yumdownloader --resolve openvswitch
-    echo "Updating openvswitch with nopostun option"
-    rpm -U --replacepkgs --nopostun ./*.rpm
-    popd
-else
-    echo "Skipping manual upgrade of openvswitch - no restart in postun detected"
-fi
+special_case_ovs_upgrade_if_needed
 
 yum -y install python-zaqarclient  # needed for os-collect-config
 yum -y -q update

extraconfig/tasks/major_upgrade_ceph_storage.sh

@@ -8,7 +8,9 @@ set -o pipefail
 
 UPGRADE_SCRIPT=/root/tripleo_upgrade_node.sh
 
-cat > $UPGRADE_SCRIPT << 'ENDOFCAT'
+declare -f special_case_ovs_upgrade_if_needed > $UPGRADE_SCRIPT
+# use >> here so we don't lose the declaration we added above
+cat >> $UPGRADE_SCRIPT << 'ENDOFCAT'
 #!/bin/bash
 ### DO NOT MODIFY THIS FILE
 ### This file is automatically delivered to the ceph-storage nodes as part of the
@@ -49,19 +51,7 @@ timeout 60 bash -c "while kill -0 ${OSD_PIDS} 2> /dev/null; do
   sleep 2;
 done"
 
-# Special-case OVS for https://bugs.launchpad.net/tripleo/+bug/1635205
-if [[ -n $(rpm -q --scripts openvswitch | awk '/postuninstall/,/*/' | grep "systemctl.*try-restart") ]]; then
-    echo "Manual upgrade of openvswitch - restart in postun detected"
-    mkdir OVS_UPGRADE || true
-    pushd OVS_UPGRADE
-    echo "Attempting to downloading latest openvswitch with yumdownloader"
-    yumdownloader --resolve openvswitch
-    echo "Updating openvswitch with nopostun option"
-    rpm -U --replacepkgs --nopostun ./*.rpm
-    popd
-else
-    echo "Skipping manual upgrade of openvswitch - no restart in postun detected"
-fi
+special_case_ovs_upgrade_if_needed
 
 # Update (Ceph to Jewel)
 yum -y install python-zaqarclient  # needed for os-collect-config

extraconfig/tasks/major_upgrade_compute.sh

@@ -18,19 +18,8 @@ set -eu
 
 crudini  --set /etc/nova/nova.conf upgrade_levels compute $upgrade_level_nova_compute
 
-# Special-case OVS for https://bugs.launchpad.net/tripleo/+bug/1635205
-if [[ -n \$(rpm -q --scripts openvswitch | awk '/postuninstall/,/*/' | grep "systemctl.*try-restart") ]]; then
-    echo "Manual upgrade of openvswitch - restart in postun detected"
-    mkdir OVS_UPGRADE || true
-    pushd OVS_UPGRADE
-    echo "Attempting to downloading latest openvswitch with yumdownloader"
-    yumdownloader --resolve openvswitch
-    echo "Updating openvswitch with nopostun option"
-    rpm -U --replacepkgs --nopostun ./*.rpm
-    popd
-else
-    echo "Skipping manual upgrade of openvswitch - no restart in postun detected"
-fi
+$(declare -f special_case_ovs_upgrade_if_needed)
+special_case_ovs_upgrade_if_needed
 
 yum -y install python-zaqarclient  # needed for os-collect-config
 yum -y update

extraconfig/tasks/major_upgrade_controller_pacemaker_2.sh

@@ -100,18 +100,7 @@ if [ $DO_MYSQL_UPGRADE -eq 1 ]; then
 fi
 
 # Special-case OVS for https://bugs.launchpad.net/tripleo/+bug/1635205
-if [[ -n $(rpm -q --scripts openvswitch | awk '/postuninstall/,/*/' | grep "systemctl.*try-restart") ]]; then
-    echo "Manual upgrade of openvswitch - restart in postun detected"
-    mkdir OVS_UPGRADE || true
-    pushd OVS_UPGRADE
-    echo "Attempting to downloading latest openvswitch with yumdownloader"
-    yumdownloader --resolve openvswitch
-    echo "Updating openvswitch with nopostun option"
-    rpm -U --replacepkgs --nopostun ./*.rpm
-    popd
-else
-    echo "Skipping manual upgrade of openvswitch - no restart in postun detected"
-fi
+special_case_ovs_upgrade_if_needed
 
 yum -y install python-zaqarclient  # needed for os-collect-config
 yum -y -q update

extraconfig/tasks/major_upgrade_object_storage.sh

@@ -23,19 +23,8 @@ function systemctl_swift {
     done
 }
 
-# Special-case OVS for https://bugs.launchpad.net/tripleo/+bug/1635205
-if [[ -n \$(rpm -q --scripts openvswitch | awk '/postuninstall/,/*/' | grep "systemctl.*try-restart") ]]; then
-    echo "Manual upgrade of openvswitch - restart in postun detected"
-    mkdir OVS_UPGRADE || true
-    pushd OVS_UPGRADE
-    echo "Attempting to downloading latest openvswitch with yumdownloader"
-    yumdownloader --resolve openvswitch
-    echo "Updating openvswitch with nopostun option"
-    rpm -U --replacepkgs --nopostun ./*.rpm
-    popd
-else
-    echo "Skipping manual upgrade of openvswitch - no restart in postun detected"
-fi
+$(declare -f special_case_ovs_upgrade_if_needed)
+special_case_ovs_upgrade_if_needed
 
 systemctl_swift stop
 

extraconfig/tasks/major_upgrade_pacemaker.yaml

@@ -97,7 +97,11 @@ resources:
     depends_on: ControllerPacemakerUpgradeDeployment_Step1
     properties:
       group: script
-      config: {get_file: major_upgrade_block_storage.sh}
+      config:
+        list_join:
+        - ''
+        - - get_file: pacemaker_common_functions.sh
+          - get_file: major_upgrade_block_storage.sh
 
   BlockStorageUpgradeDeployment:
     type: OS::Heat::SoftwareDeploymentGroup

extraconfig/tasks/major_upgrade_pacemaker_init.j2.yaml

@@ -54,19 +54,28 @@ resources:
                 upgrade_level_nova_compute='UPGRADE_LEVEL_NOVA_COMPUTE'
               params:
                 UPGRADE_LEVEL_NOVA_COMPUTE: {get_param: UpgradeLevelNovaCompute}
+          - get_file: pacemaker_common_functions.sh
           - get_file: major_upgrade_compute.sh
 
   ObjectStorageDeliverUpgradeScriptConfig:
     type: OS::Heat::SoftwareConfig
     properties:
       group: script
-      config: {get_file: major_upgrade_object_storage.sh}
+      config:
+        list_join:
+        - ''
+        - - get_file: pacemaker_common_functions.sh
+          - get_file: major_upgrade_object_storage.sh
 
   CephStorageDeliverUpgradeScriptConfig:
     type: OS::Heat::SoftwareConfig
     properties:
       group: script
-      config: {get_file: major_upgrade_ceph_storage.sh}
+      config:
+        list_join:
+        - ''
+        - - get_file: pacemaker_common_functions.sh
+          - get_file: major_upgrade_ceph_storage.sh
 
 {% for role in roles %}
   UpgradeInit{{role.name}}Deployment:

extraconfig/tasks/pacemaker_common_functions.sh

@@ -297,3 +297,27 @@ function systemctl_swift {
         manage_systemd_service $action $service
     done
 }
+
+# Special-case OVS for https://bugs.launchpad.net/tripleo/+bug/1635205
+function special_case_ovs_upgrade_if_needed {
+    if [[ -n $(rpm -q --scripts openvswitch | awk '/postuninstall/,/*/' | grep "systemctl.*try-restart") ]]; then
+        echo "Manual upgrade of openvswitch - restart in postun detected"
+        rm -rf OVS_UPGRADE
+        mkdir OVS_UPGRADE && pushd OVS_UPGRADE
+        echo "Attempting to downloading latest openvswitch with yumdownloader"
+        yumdownloader --resolve openvswitch
+        for pkg in $(ls -1 *.rpm);  do
+            if rpm -U --test $pkg 2>&1 | grep "already installed" ; then
+                echo "Looks like newer version of $pkg is already installed, skipping"
+            else
+                echo "Updating $pkg with nopostun option"
+                rpm -U --replacepkgs --nopostun $pkg
+            fi
+        done
+        popd
+    else
+        echo "Skipping manual upgrade of openvswitch - no restart in postun detected"
+    fi
+
+}
+

extraconfig/tasks/yum_update.sh

@@ -63,18 +63,7 @@ if [[ "$pacemaker_status" == "active" && \
 fi
 
 # Special-case OVS for https://bugs.launchpad.net/tripleo/+bug/1635205
-if [[ -n $(rpm -q --scripts openvswitch | awk '/postuninstall/,/*/' | grep "systemctl.*try-restart") ]]; then
-    echo "Manual upgrade of openvswitch - restart in postun detected"
-    mkdir OVS_UPGRADE || true
-    pushd OVS_UPGRADE
-    echo "Attempting to downloading latest openvswitch with yumdownloader"
-    yumdownloader --resolve openvswitch
-    echo "Updating openvswitch with nopostun option"
-    rpm -U --replacepkgs --nopostun ./*.rpm
-    popd
-else
-    echo "Skipping manual upgrade of openvswitch - no restart in postun detected"
-fi
+special_case_ovs_upgrade_if_needed
 
 if [[ "$pacemaker_status" == "active" ]] ; then
     echo "Pacemaker running, stopping cluster node and doing full package update"

extraconfig/tasks/yum_update.yaml

@@ -9,7 +9,12 @@ resources:
     type: OS::Heat::SoftwareConfig
     properties:
       group: script
-      config: {get_file: yum_update.sh}
+      config:
+        list_join:
+        - ''
+        - - get_file: pacemaker_common_functions.sh
+          - get_file: yum_update.sh
+
       inputs:
       - name: update_identifier
         description: yum will only run for previously unused values of update_identifier