Support networking pod image updates during simplex upgrade

Networking image pods may change from one release to the next. In a
simplex upgrade the previous images are not downloaded and the
networking deployments must be upgraded to use the new images.

Story: 2008966
Task: 42838
Change-Id: I706a67a0ef8d427faaa4ec94f6e621262b51e84d
Signed-off-by: David Sullivan <david.sullivan@windriver.com>

playbookconfig/src/playbooks/roles/bootstrap/bringup-essential-services/tasks/main.yml

@@ -175,6 +175,7 @@
     - name: Override async parameters
       set_fact:
         async_timeout: 120
+        async_retries: 40
 
     - name: Wait for 30 seconds to ensure deployments have time to scale back up
       wait_for:
@@ -182,6 +183,11 @@
 
     when: mode == 'restore'
 
+  - name: Upgrade k8s networking
+    import_role:
+      name: common/upgrade-k8s-networking
+    when: migrate_platform_data is defined and migrate_platform_data
+
   - name: Start parallel tasks to wait for Kubernetes component and Networking pods to reach ready state
     # Only check for pods on the current host to avoid waiting for pods on downed nodes
     # This speeds up "Get wait tasks results" on multi-node systems
@@ -212,8 +218,8 @@
       jid: "{{ item.ansible_job_id }}"
     register: wait_job_result
     until: wait_job_result.finished
-    # Set the retry to 10 times (60 seconds) but the async jobs above will
-    # complete (success or failure) within 30 seconds
+    # The retry length should be x2 the length of the async_timeout
+    # eg async_retries = async_timeout * 2 / delay
     retries: "{{ async_retries }}"
     delay: 6
     failed_when: false

playbookconfig/src/playbooks/roles/common/upgrade-k8s-networking/tasks/main.yml

@@ -0,0 +1,88 @@
+---
+#
+# Copyright (c) 2021 Wind River Systems, Inc.
+#
+# SPDX-License-Identifier: Apache-2.0
+#
+- block:
+  - name: Backup mode variable
+    set_fact:
+      previous_mode: "{{ mode }}"
+
+  when: mode is defined
+
+- name: Initialize variables
+  set_fact:
+    overrides_file: "/tmp/upgrade_overrides.yaml"
+    mode: "upgrade_k8s_networking"
+
+- name: Fail if kubernetes_version is not defined
+  fail: msg="The mandatory parameter kubernetes_version is not defined."
+  when: kubernetes_version is not defined
+
+- name: Create the upgrade overrides file
+  command: "/usr/bin/sysinv-utils create-host-overrides {{ overrides_file }}"
+
+- name: Read the overrides variables
+  include_vars:
+    file: "{{ overrides_file }}"
+
+- name: Set networking facts
+  set_fact:
+    kubelet_cni_bin_dir: "/usr/libexec/cni"
+    kubelet_vol_plugin_dir: "/usr/libexec/kubernetes/kubelet-plugins/volume/exec/"
+    cluster_network_ipv4: "{{ cluster_pod_subnet | ipv4 }}"
+    cluster_network_ipv6: "{{ cluster_pod_subnet | ipv6 }}"
+    cluster_floating_address: "{{ cluster_host_floating_address }}"
+    controller_0_cluster_host: "{{ cluster_host_node_0_address }}"
+
+- name: Upgrade k8s networking images
+  import_role:
+    name: common/push-docker-images
+
+- name: Create Calico config file
+  template:
+    src: "roles/bootstrap/bringup-essential-services/templates/k8s-{{ kubernetes_version }}/calico-cni.yaml.j2"
+    dest: /etc/kubernetes/update_calico.yaml
+
+- name: Update Calico Networking
+  command: "kubectl --kubeconfig=/etc/kubernetes/admin.conf apply -f /etc/kubernetes/update_calico.yaml"
+
+- name: Create Multus config file
+  template:
+    src: "roles/bootstrap/bringup-essential-services/templates/k8s-{{ kubernetes_version }}/multus-cni.yaml.j2"
+    dest: /etc/kubernetes/update_multus.yaml
+
+- name: Update Multus Networking
+  command: "kubectl --kubeconfig=/etc/kubernetes/admin.conf apply -f /etc/kubernetes/update_multus.yaml"
+
+- name: Create SRIOV Networking config file
+  template:
+    src: "roles/bootstrap/bringup-essential-services/templates/k8s-{{ kubernetes_version }}/sriov-cni.yaml.j2"
+    dest: /etc/kubernetes/update_sriov-cni.yaml
+
+- name: Update SRIOV Networking
+  command: "kubectl --kubeconfig=/etc/kubernetes/admin.conf apply -f /etc/kubernetes/update_sriov-cni.yaml"
+
+- name: Create SRIOV device plugin config file
+  template:
+    src: "roles/bootstrap/bringup-essential-services/templates/k8s-{{ kubernetes_version }}/sriov-plugin.yaml.j2"
+    dest: /etc/kubernetes/update_sriovdp-daemonset.yaml
+
+- name: Update SRIOV device plugin
+  command: "kubectl --kubeconfig=/etc/kubernetes/admin.conf apply -f /etc/kubernetes/update_sriovdp-daemonset.yaml"
+
+- name: Create Coredns config file
+  template:
+    src: "roles/bootstrap/bringup-essential-services/templates/coredns.yaml.j2"
+    dest: /etc/kubernetes/update_coredns.yaml
+
+- name: Update Coredns config map
+  command: "kubectl --kubeconfig=/etc/kubernetes/admin.conf apply -f /etc/kubernetes/update_coredns.yaml"
+
+- block:
+  - name: Restore mode variable
+    set_fact:
+      mode: "{{ previous_mode }}"
+
+  when: previous_mode is defined

playbookconfig/src/playbooks/upgrade-k8s-networking.yml

@@ -7,71 +7,5 @@
 
 - hosts: all
 
-  vars:
-    mode: "upgrade_k8s_networking"
-    overrides_file: "/tmp/upgrade_overrides.yaml"
-
-  tasks:
-    - name: Fail if kubernetes_version is not defined
-      fail: msg="The mandatory parameter kubernetes_version is not defined."
-      when: kubernetes_version is not defined
-
-    - name: Create the upgrade overrides file
-      command: "/usr/bin/sysinv-utils create-host-overrides {{ overrides_file }}"
-
-    - name: Read the overrides variables
-      include_vars:
-        file: "{{ overrides_file }}"
-
-    - name: Set networking facts
-      set_fact:
-        kubelet_cni_bin_dir: "/usr/libexec/cni"
-        kubelet_vol_plugin_dir: "/usr/libexec/kubernetes/kubelet-plugins/volume/exec/"
-        cluster_network_ipv4: "{{ cluster_pod_subnet | ipv4 }}"
-        cluster_network_ipv6: "{{ cluster_pod_subnet | ipv6 }}"
-        cluster_floating_address: "{{ cluster_host_floating_address }}"
-        controller_0_cluster_host: "{{ cluster_host_node_0_address }}"
-
-    - name: Upgrade k8s networking images
-      import_role:
-        name: common/push-docker-images
-
-    - name: Create Calico config file
-      template:
-        src: "roles/bootstrap/bringup-essential-services/templates/k8s-{{ kubernetes_version }}/calico-cni.yaml.j2"
-        dest: /etc/kubernetes/update_calico.yaml
-
-    - name: Update Calico Networking
-      command: "kubectl --kubeconfig=/etc/kubernetes/admin.conf apply -f /etc/kubernetes/update_calico.yaml"
-
-    - name: Create Multus config file
-      template:
-        src: "roles/bootstrap/bringup-essential-services/templates/k8s-{{ kubernetes_version }}/multus-cni.yaml.j2"
-        dest: /etc/kubernetes/update_multus.yaml
-
-    - name: Update Multus Networking
-      command: "kubectl --kubeconfig=/etc/kubernetes/admin.conf apply -f /etc/kubernetes/update_multus.yaml"
-
-    - name: Create SRIOV Networking config file
-      template:
-        src: "roles/bootstrap/bringup-essential-services/templates/k8s-{{ kubernetes_version }}/sriov-cni.yaml.j2"
-        dest: /etc/kubernetes/update_sriov-cni.yaml
-
-    - name: Update SRIOV Networking
-      command: "kubectl --kubeconfig=/etc/kubernetes/admin.conf apply -f /etc/kubernetes/update_sriov-cni.yaml"
-
-    - name: Create SRIOV device plugin config file
-      template:
-        src: "roles/bootstrap/bringup-essential-services/templates/k8s-{{ kubernetes_version }}/sriov-plugin.yaml.j2"
-        dest: /etc/kubernetes/update_sriovdp-daemonset.yaml
-
-    - name: Update SRIOV device plugin
-      command: "kubectl --kubeconfig=/etc/kubernetes/admin.conf apply -f /etc/kubernetes/update_sriovdp-daemonset.yaml"
-
-    - name: Create Coredns config file
-      template:
-        src: "roles/bootstrap/bringup-essential-services/templates/coredns.yaml.j2"
-        dest: /etc/kubernetes/update_coredns.yaml
-
-    - name: Update Coredns config map
-      command: "kubectl --kubeconfig=/etc/kubernetes/admin.conf apply -f /etc/kubernetes/update_coredns.yaml"
+  roles:
+    - common/upgrade-k8s-networking