Revert "Revert "configure kubernetes to use containerd as CRI""

This reverts commit ae1e45e00e.

Depends-On: https://review.opendev.org/703263
Change-Id: Id7defcd9aa0ef7cf23bb6b4e5b43ed60f1bb5c62
Signed-off-by: Shuicheng Lin <shuicheng.lin@intel.com>

playbookconfig/src/playbooks/roles/bootstrap/bringup-essential-services/tasks/bringup_local_registry.yml

@@ -48,6 +48,51 @@
   when: (insecure_registries is defined and
         insecure_registries | length > 0)
 
+- name: Create containerd config file directory
+  file:
+    path: /etc/containerd
+    state: directory
+    mode: 0700
+
+- name: Create config.toml file for containerd configuration
+  copy:
+    src: "{{ containerd_template }}"
+    dest: /etc/containerd/config.toml
+    remote_src: yes
+    mode: 0600
+
+- name: Remove puppet template for insecure registries
+  replace:
+    path: /etc/containerd/config.toml
+    after: '# Begin of insecure registries'
+    regexp: '^(<%- @insecure_registries.+)\n(.+)\n(.+)\n(.+end -%>)'
+    replace: ''
+
+- name: Update config.toml with insecure registries
+  blockinfile:
+    path: /etc/containerd/config.toml
+    insertafter: '# Begin of insecure registries'
+    marker: "        # {{ item }}"
+    block: |2
+              [plugins.cri.registry.mirrors."{{ item }}"]
+                endpoint = ["http://{{ item }}"]
+  loop:
+    "{{ insecure_registries }}"
+  when: (insecure_registries is defined and
+        insecure_registries | length > 0)
+
+- name: Update config.toml with cni bin dir
+  command: "sed -i -e 's|<%= @k8s_cni_bin_dir %>|$CNI_BIN_DIR|g' /etc/containerd/config.toml"
+  args:
+    warn: false
+  environment:
+    CNI_BIN_DIR: "{{ kubelet_cni_bin_dir }}"
+
+- name: Restart containerd
+  systemd:
+    name: containerd
+    state: restarted
+
 - name: Generate local registry runtime config file from template
   copy:
     src: "{{ registry_config_template }}"

playbookconfig/src/playbooks/roles/bootstrap/bringup-essential-services/vars/main.yml

@@ -17,6 +17,7 @@ registry_token_server_template: /usr/share/puppet/modules/platform/templates/reg
 registry_token_server_file: /etc/docker-distribution/registry/token_server.conf
 cert_cnf_template: /usr/share/puppet/modules/platform/templates/registry-cert-extfile.erb
 insecure_docker_registry_template: /usr/share/puppet/modules/platform/templates/insecuredockerregistry.conf.erb
+containerd_template: /usr/share/puppet/modules/platform/templates/config.toml.erb
 cert_cnf_file: /etc/ssl/private/registry-cert-extfile.cnf
 registry_cert_key: /etc/ssl/private/registry-cert.key
 registry_cert_crt: /etc/ssl/private/registry-cert.crt

playbookconfig/src/playbooks/roles/bootstrap/persist-config/tasks/main.yml

@@ -183,54 +183,75 @@
   when: save_config_to_db
 
 
-# Update docker config file and restart docker if docker proxy is
-# configured
+# Update docker and containerd config files and restart docker and containerd
+# if docker proxy is configured
 - block:
-  - name: Ensure docker config directory exists
+  - name: Ensure docker and containerd config directory exist
     file:
-      path: /etc/systemd/system/docker.service.d
+      path: "{{ item }}"
       state: directory
       owner: root
       group: root
       mode: 0755
+    with_items:
+      - /etc/systemd/system/docker.service.d
+      - /etc/systemd/system/containerd.service.d
 
-  - name: Ensure docker proxy config exists
+  - name: Ensure docker and containerd proxy config exist
     copy:
       content: ""
-      dest: "{{ docker_proxy_conf }}"
+      dest: "{{ item }}"
       force: no
       owner: root
       group: root
       mode: 0644
       remote_src: yes
+    with_items:
+      - "{{ docker_proxy_conf }}"
+      - "{{ containerd_proxy_conf }}"
 
-  - name: Write header to docker proxy conf file
+  - name: Write header to docker and containerd proxy conf files
     lineinfile:
-      path: "{{ docker_proxy_conf }}"
+      path: "{{ item }}"
       line: "[Service]"
+    with_items:
+      - "{{ docker_proxy_conf }}"
+      - "{{ containerd_proxy_conf }}"
 
-  - name: Add http proxy URL to docker proxy conf file
+  - name: Add http proxy URL to docker and containerd proxy conf files
     lineinfile:
-      path: "{{ docker_proxy_conf }}"
+      path: "{{ item }}"
       line: "Environment='HTTP_PROXY={{ docker_http_proxy }}'"
+    with_items:
+      - "{{ docker_proxy_conf }}"
+      - "{{ containerd_proxy_conf }}"
     when: docker_http_proxy != 'undef'
 
-  - name: Add https proxy URL to docker proxy conf file
+  - name: Add https proxy URL to docker and containerd proxy conf files
     lineinfile:
-      path: "{{ docker_proxy_conf }}"
+      path: "{{ item }}"
       line: "Environment='HTTPS_PROXY={{ docker_https_proxy }}'"
+    with_items:
+      - "{{ docker_proxy_conf }}"
+      - "{{ containerd_proxy_conf }}"
     when: docker_https_proxy != 'undef'
 
-  - name: Add no proxy address list to docker proxy config file
+  - name: Add no proxy address list to docker and containerd proxy config files
     lineinfile:
-      path: "{{ docker_proxy_conf }}"
+      path: "{{ item }}"
       line: "Environment='NO_PROXY={{ docker_no_proxy_combined | join(',') }}'"
+    with_items:
+      - "{{ docker_proxy_conf }}"
+      - "{{ containerd_proxy_conf }}"
 
-  - name: Restart Docker
+  - name: Restart Docker and containerd
     systemd:
       state: restarted
       daemon_reload: yes
-      name: docker
+      name: "{{ item }}"
+    with_items:
+      - docker
+      - containerd
 
   when: use_docker_proxy
 

playbookconfig/src/playbooks/roles/bootstrap/persist-config/tasks/shutdown_services.yml

@@ -26,7 +26,7 @@
 
 - block:  # Revert changes done by kubeadm init, clear data cache
   - name: Shut down and remove Kubernetes components
-    command: kubeadm reset -f
+    command: kubeadm reset -f --cri-socket /var/run/containerd/containerd.sock
     register: reset_info
 
   - debug: var=reset_info.stdout_lines

playbookconfig/src/playbooks/roles/bootstrap/persist-config/vars/main.yml

@@ -1,6 +1,7 @@
 ---
 keyring_workdir: /tmp/python_keyring
 docker_proxy_conf: /etc/systemd/system/docker.service.d/http-proxy.conf
+containerd_proxy_conf: /etc/systemd/system/containerd.service.d/http-proxy.conf
 minimum_root_disk_size: 240
 default_security_feature: "nopti nospectre_v2"
 temp_ssl_ca: "/tmp/ca-cert.pem"

playbookconfig/src/playbooks/roles/bootstrap/validate-config/tasks/main.yml

@@ -392,6 +392,8 @@
         - localhost
         - 127.0.0.1
         - registry.local
+        - "{{ cluster_service_start_address if cluster_service_start_address != 'derived'
+            else default_cluster_service_start_address }}"
         - "{{ controller_floating_address }}"
         - "{{ derived_network_params.controller_0_address }}"
         - "{{ external_oam_floating_address }}"

playbookconfig/src/playbooks/roles/common/push-docker-images/files/download_images.py

@@ -11,6 +11,8 @@ import sys
 import time
 import os
 import json
+import keyring
+import subprocess
 
 MAX_DOWNLOAD_ATTEMPTS = 3
 MAX_DOWNLOAD_THREAD = 5
@@ -54,14 +56,29 @@ def download_an_image(img):
     local_img = 'registry.local:9001/' + new_img
     err_msg = " Image download failed: %s" % target_img
 
+    password = str(keyring.get_password("CGCS", "admin"))
+    if not password:
+        raise Exception("Local registry password not found.")
+    auth = '{0}:{1}'.format('admin', password)
+
     for i in range(MAX_DOWNLOAD_ATTEMPTS):
         try:
             client = docker.APIClient()
             client.pull(target_img)
+            print("Image download succeeded: %s" % target_img)
             client.tag(target_img, local_img)
             client.push(local_img)
-            print("Image download succeeded: %s" % target_img)
             print("Image push succeeded: %s" % local_img)
+            # due to crictl doesn't support push function, docker client is used
+            # to pull and push image to local registry, then crictl download image
+            # from local registry.
+            subprocess.check_call(["crictl", "pull", "--creds", auth, local_img])
+            print("Image %s download succeeded by containerd" % target_img)
+            # except armada/tiller, other docker images could be removed.
+            # TODO: run armada with containerd.
+            if not ('armada' in target_img or 'tiller' in target_img):
+                client.remove_image(target_img)
+                client.remove_image(local_img)
             return target_img, True
         except docker.errors.NotFound as e:
             print(err_msg + str(e))