Merge "Add support for selecting the k8s version on install"

2021-08-17 15:48:37 +00:00
parent ea2373f396 cb48ffd8a3
commit 9c319adb65
10 changed files with 135 additions and 27 deletions
--- a/playbookconfig/src/playbooks/roles/backup/backup-system/tasks/main.yml
+++ b/playbookconfig/src/playbooks/roles/backup/backup-system/tasks/main.yml
@@ -394,6 +394,18 @@
    - name: Create the override backup file
      command: "/usr/bin/sysinv-utils create-host-overrides {{ override_backup_file }}"

+    - block:
+      - name: Get kubernetes_version from the DB
+        shell: echo "select kubeadm_version from kube_cmd_versions" | psql -qAt -d sysinv
+        register: kube_ver_select_result
+        become_user: postgres
+
+      - name: Set kubernetes_version to the value from DB
+        set_fact:
+          kubernetes_version: "{{ kube_ver_select_result.stdout_lines[0] }}"
+
+      when: kubernetes_version is not defined
+
    - name: Get docker registries information
      include_role:
        name: common/push-docker-images
--- a/playbookconfig/src/playbooks/roles/bootstrap/bringup-essential-services/tasks/bringup_kubemaster.yml
+++ b/playbookconfig/src/playbooks/roles/bootstrap/bringup-essential-services/tasks/bringup_kubemaster.yml
@@ -1,6 +1,6 @@
 ---
 #
-# Copyright (c) 2019 Wind River Systems, Inc.
+# Copyright (c) 2021 Wind River Systems, Inc.
 #
 # SPDX-License-Identifier: Apache-2.0
 #
@@ -132,6 +132,15 @@
  set_fact:
    apiserver_cert_list: "{{ [ cluster_floating_address, loopback_ip ] + apiserver_cert_sans + OAM_addresses}}"

+- name: Update Kube admin yaml with kubernetes version
+  command: "{{ item }}"
+  args:
+    warn: false
+  with_items:
+    - "sed -i -e 's|<%= @kubernetes_version %>|'$KUBERNETES_LONG_VERSION'|g' /etc/kubernetes/kubeadm.yaml"
+  environment:
+    KUBERNETES_LONG_VERSION: "{{ kubernetes_long_version }}"
+
 - name: Update Kube admin yaml with network info
  command: "{{ item }}"
  args:
@@ -218,22 +227,22 @@

 - name: Create Calico config file
  template:
-    src: "k8s-{{ kubernetes_version }}/calico-cni.yaml.j2"
+    src: "k8s-{{ kubernetes_long_version }}/calico-cni.yaml.j2"
    dest: /etc/kubernetes/calico.yaml

 - name: Create Multus config file
  template:
-    src: "k8s-{{ kubernetes_version }}/multus-cni.yaml.j2"
+    src: "k8s-{{ kubernetes_long_version }}/multus-cni.yaml.j2"
    dest: /etc/kubernetes/multus.yaml

 - name: Create SRIOV Networking config file
  template:
-    src: "k8s-{{ kubernetes_version }}/sriov-cni.yaml.j2"
+    src: "k8s-{{ kubernetes_long_version }}/sriov-cni.yaml.j2"
    dest: /etc/kubernetes/sriov-cni.yaml

 - name: Create SRIOV device plugin config file
  template:
-    src: "k8s-{{ kubernetes_version }}/sriov-plugin.yaml.j2"
+    src: "k8s-{{ kubernetes_long_version }}/sriov-plugin.yaml.j2"
    dest: /etc/kubernetes/sriovdp-daemonset.yaml

 - name: Create coredns config file
--- a/playbookconfig/src/playbooks/roles/bootstrap/bringup-essential-services/tasks/k8s_bind_mount.yml
+++ b/playbookconfig/src/playbooks/roles/bootstrap/bringup-essential-services/tasks/k8s_bind_mount.yml
@@ -0,0 +1,43 @@
+---
+#
+# Copyright (c) 2021 Wind River Systems, Inc.
+#
+# SPDX-License-Identifier: Apache-2.0
+#
+# ROLE DESCRIPTION:
+#   These tasks do the preparation for kubernetes staged versions.
+#
+
+- block:
+  - name: Set Kubernetes local directory
+    set_fact:
+      kube_local_dir: "/usr/local/kubernetes"
+
+  - name: Create Kubernetes bind mounts directories
+    file:
+      path: "{{ item }}"
+      state: directory
+      recurse: yes
+      owner: root
+      group: root
+      mode: 0755
+    with_items:
+      - "{{ kube_local_dir }}/current/stage1"
+      - "{{ kube_local_dir }}/current/stage2"
+      - "{{ kube_local_dir }}/{{ kubernetes_version }}/stage1"
+      - "{{ kube_local_dir }}/{{ kubernetes_version }}/stage2"
+
+  # When updating kubernetes, kubeadm and kubelet/kubectl need to be updated separately
+  # and we will have "stage1" and "stage2" subdirectories to separate these different stages.
+  - name: Bind Kubernetes stage1 and stage2 directories
+    mount:
+      path: "{{ kube_local_dir }}/current/{{ item }}"
+      src: "{{ kube_local_dir }}/{{ kubernetes_version }}/{{ item }}"
+      opts: bind
+      state: mounted
+      fstype: none
+    with_items:
+      - "stage1"
+      - "stage2"
+
+  when: kubernetes_version is defined
--- a/playbookconfig/src/playbooks/roles/bootstrap/bringup-essential-services/tasks/main.yml
+++ b/playbookconfig/src/playbooks/roles/bootstrap/bringup-essential-services/tasks/main.yml
@@ -37,6 +37,9 @@
    when: last_config_file_exists and reconfigure_endpoints and
          (mgmt_floating_virtual != prev_mgmt_floating_virtual)

+  - name: Mount current kubernetes version
+    import_tasks: k8s_bind_mount.yml
+
  - name: Refresh local DNS (i.e. /etc/hosts)
    import_tasks: refresh_local_dns.yml

--- a/playbookconfig/src/playbooks/roles/bootstrap/persist-config/files/populate_initial_config.py
+++ b/playbookconfig/src/playbooks/roles/bootstrap/persist-config/files/populate_initial_config.py
@@ -1,7 +1,7 @@
 #!/usr/bin/python

 #
-# Copyright (c) 2019 Wind River Systems, Inc.
+# Copyright (c) 2019-2021 Wind River Systems, Inc.
 #
 # SPDX-License-Identifier: Apache-2.0
 #
@@ -251,6 +251,28 @@ def create_network(client, network_data, network_name):
        raise e


+def populate_kube_cmd_version(client):
+    try:
+        kube_cmd_version = CONF.get('BOOTSTRAP_CONFIG', 'KUBERNETES_VERSION')
+        if kube_cmd_version != 'none':
+            values = {
+                'kubeadm_version': kube_cmd_version,
+                'kubelet_version': kube_cmd_version
+            }
+            patch = dict_to_patch(values)
+            client.sysinv.kube_cmd_version.update(patch)
+    except Exception as e:
+        if INCOMPLETE_BOOTSTRAP:
+            if kube_cmd_version != 'none':
+                kube_cmd_version_incomplete = client.sysinv.kube_cmd_version.get()
+                if kube_cmd_version_incomplete.kubeadm_version == kube_cmd_version \
+                        and kube_cmd_version_incomplete.kubelet_version == kube_cmd_version:
+                    return
+            else:
+                return
+        raise e
+
+
 def delete_network_and_addrpool(client, network_name):
    networks = client.sysinv.network.list()
    network_uuid = addrpool_uuid = None
@@ -613,7 +635,7 @@ def populate_dns_config(client):
    print("DNS config completed.")


-def populate_docker_config(client):
+def populate_docker_kube_config(client):
    http_proxy = CONF.get('BOOTSTRAP_CONFIG', 'DOCKER_HTTP_PROXY')
    https_proxy = CONF.get('BOOTSTRAP_CONFIG', 'DOCKER_HTTPS_PROXY')
    no_proxy = CONF.get('BOOTSTRAP_CONFIG', 'DOCKER_NO_PROXY')
@@ -769,7 +791,7 @@ def populate_docker_config(client):

    print("Populating/Updating kubernetes config...")
    client.sysinv.service_parameter.create(**values)
-
+    populate_kube_cmd_version(client)
    print("Kubernetes config completed.")

    parameters = client.sysinv.service_parameter.list()
@@ -841,7 +863,7 @@ def populate_service_parameter_config(client):
    if not INITIAL_POPULATION and not RECONFIGURE_SERVICE:
        return
    populate_platform_config(client)
-    populate_docker_config(client)
+    populate_docker_kube_config(client)


 def get_management_mac_address():
--- a/playbookconfig/src/playbooks/roles/bootstrap/persist-config/tasks/main.yml
+++ b/playbookconfig/src/playbooks/roles/bootstrap/persist-config/tasks/main.yml
@@ -1,6 +1,6 @@
 ---
 #
-# Copyright (c) 2019 Wind River Systems, Inc.
+# Copyright (c) 2019-2021 Wind River Systems, Inc.
 #
 # SPDX-License-Identifier: Apache-2.0
 #
@@ -202,9 +202,25 @@
    - "DOCKER_REGISTRY_SECRET={{ docker_registry_secret | default('none') }}"
    - "ELASTIC_REGISTRY_SECRET={{ elastic_registry_secret | default('none') }}"

+- name: Append config ini file with Kubernetes version
+  lineinfile:
+    path: "{{ config_permdir + '/' + bootstrap_config_file|basename }}"
+    line: "KUBERNETES_VERSION={{ kubernetes_version | default('none') }}"
+
 - include: update_sysinv_database.yml
  when: save_config_to_db

+- block:
+  - name: Get default kubernetes_version from the DB
+    shell: echo "select kubeadm_version from kube_cmd_versions" | psql -qAt -d sysinv
+    register: kube_ver_select_result
+    become_user: postgres
+
+  - name: Set kubernetes_version to the default value from DB
+    set_fact:
+      kubernetes_version: "{{ kube_ver_select_result.stdout_lines[0] }}"
+
+  when: kubernetes_version is not defined

 # Update docker and containerd config files and restart docker and containerd
 # if docker proxy is configured
--- a/playbookconfig/src/playbooks/roles/common/files/kubeadm.yaml.erb
+++ b/playbookconfig/src/playbooks/roles/common/files/kubeadm.yaml.erb
@@ -48,7 +48,7 @@ etcd:
    certFile: /etc/kubernetes/pki/apiserver-etcd-client.crt
    keyFile: /etc/kubernetes/pki/apiserver-etcd-client.key
 imageRepository: "registry.local:9001/k8s.gcr.io"
-kubernetesVersion: v1.18.1
+kubernetesVersion: <%= @kubernetes_version %>
 networking:
  dnsDomain: <%= @service_domain %>
  podSubnet: <%= @pod_network_cidr %>
--- a/playbookconfig/src/playbooks/roles/common/load-images-information/tasks/main.yml
+++ b/playbookconfig/src/playbooks/roles/common/load-images-information/tasks/main.yml
@@ -8,17 +8,15 @@
 #   This role loads variables containing images.
 #

- set_fact:
-    kubernetes_version: "{{ lookup('file', 'roles/common/files/kubeadm.yaml.erb') |
-      regex_search('^kubernetesVersion: (.*)$', multiline=True) | regex_replace('^(.*): (.*)$', '\\2')}}"
-  when: mode == 'bootstrap' or
-        mode == 'restore' or
-        mode == 'trident_install' or
-        mode == 'backup'
+- name: Set kubernetes long version
+  set_fact:
+    kubernetes_long_version: "{{ kubernetes_version if kubernetes_version.startswith('v')
+                              else 'v' + kubernetes_version }}"

 - block:
  - name: Get the list of kubernetes images
-    shell: "kubeadm --kubeconfig=/etc/kubernetes/admin.conf config images list --kubernetes-version {{ kubernetes_version }}"
+    shell: "kubeadm --kubeconfig=/etc/kubernetes/admin.conf config images list
+           --kubernetes-version {{ kubernetes_long_version }}"
    register: kubernetes_images_output

  - set_fact:
@@ -26,7 +24,7 @@

  - name: Read in system images list
    include_vars:
-      dir: "vars/k8s-{{ kubernetes_version }}"
+      dir: "vars/k8s-{{ kubernetes_long_version }}"
      files_matching: "system-images.yml"

  # Optional system images can be addded in {{ additional_system_images_conf_file }}
--- a/playbookconfig/src/playbooks/roles/common/upgrade-k8s-networking/tasks/main.yml
+++ b/playbookconfig/src/playbooks/roles/common/upgrade-k8s-networking/tasks/main.yml
@@ -11,6 +11,11 @@

  when: mode is defined

+- name: Set kubernetes long version
+  set_fact:
+    kubernetes_long_version: "{{ kubernetes_version if kubernetes_version.startswith('v')
+                              else 'v' + kubernetes_version }}"
+
 - name: Initialize variables
  set_fact:
    overrides_file: "/tmp/upgrade_overrides.yaml"
@@ -63,7 +68,7 @@

 - name: Create Calico config file
  template:
-    src: "roles/bootstrap/bringup-essential-services/templates/k8s-{{ kubernetes_version }}/calico-cni.yaml.j2"
+    src: "roles/bootstrap/bringup-essential-services/templates/k8s-{{ kubernetes_long_version }}/calico-cni.yaml.j2"
    dest: /etc/kubernetes/update_calico.yaml

 - name: Update Calico Networking
@@ -71,7 +76,7 @@

 - name: Create Multus config file
  template:
-    src: "roles/bootstrap/bringup-essential-services/templates/k8s-{{ kubernetes_version }}/multus-cni.yaml.j2"
+    src: "roles/bootstrap/bringup-essential-services/templates/k8s-{{ kubernetes_long_version }}/multus-cni.yaml.j2"
    dest: /etc/kubernetes/update_multus.yaml

 - name: Update Multus Networking
@@ -79,7 +84,7 @@

 - name: Create SRIOV Networking config file
  template:
-    src: "roles/bootstrap/bringup-essential-services/templates/k8s-{{ kubernetes_version }}/sriov-cni.yaml.j2"
+    src: "roles/bootstrap/bringup-essential-services/templates/k8s-{{ kubernetes_long_version }}/sriov-cni.yaml.j2"
    dest: /etc/kubernetes/update_sriov-cni.yaml

 - name: Update SRIOV Networking
@@ -87,7 +92,7 @@

 - name: Create SRIOV device plugin config file
  template:
-    src: "roles/bootstrap/bringup-essential-services/templates/k8s-{{ kubernetes_version }}/sriov-plugin.yaml.j2"
+    src: "roles/bootstrap/bringup-essential-services/templates/k8s-{{ kubernetes_long_version }}/sriov-plugin.yaml.j2"
    dest: /etc/kubernetes/update_sriovdp-daemonset.yaml

 - name: Update SRIOV device plugin
--- a/playbookconfig/src/playbooks/roles/k8s-storage-backends/snapshot-controller/tasks/main.yml
+++ b/playbookconfig/src/playbooks/roles/k8s-storage-backends/snapshot-controller/tasks/main.yml
@@ -1,6 +1,6 @@
 ---
 #
-# Copyright (c) 2020 Wind River Systems, Inc.
+# Copyright (c) 2020-2021 Wind River Systems, Inc.
 #
 # SPDX-License-Identifier: Apache-2.0
 #
@@ -25,14 +25,14 @@
    dest: >-
      {{ snapshot_controller_setup_dir }}/volume-snapshot-controller/{{ item | basename | regex_replace('\.j2$', '') }}
  with_fileglob:
-  - ../templates/k8s-{{ kubernetes_version }}/volume-snapshot-controller/*.j2
+  - "{{ role_path }}/templates/k8s-{{ kubernetes_long_version }}/volume-snapshot-controller/*.j2"

 - name: Copy snapshots CRD files
  copy:
    src: "{{ item }}"
    dest: "{{ snapshot_controller_setup_dir }}/crd/{{ item | basename }}"
  with_fileglob:
-  - k8s-{{ kubernetes_version }}/crd/*.yaml
+  - k8s-{{ kubernetes_long_version }}/crd/*.yaml

 - name: Add Snapshot CRDs
  command: "kubectl apply -f {{ snapshot_controller_setup_dir }}/crd"