From c9db3fa3d36dcdbaaef7f399804c3ad823a72d83 Mon Sep 17 00:00:00 2001 From: Lin Shuicheng Date: Sun, 19 Jan 2020 01:58:34 +0000 Subject: [PATCH] Revert "Revert "configure kubernetes to use containerd as CRI"" This reverts commit ae1e45e00e0d684a18d1a7026e2e88f17a81295f. Depends-On: https://review.opendev.org/703263 Change-Id: Id7defcd9aa0ef7cf23bb6b4e5b43ed60f1bb5c62 Signed-off-by: Shuicheng Lin --- .../tasks/bringup_local_registry.yml | 45 ++++++++++++++++ .../bringup-essential-services/vars/main.yml | 1 + .../bootstrap/persist-config/tasks/main.yml | 53 +++++++++++++------ .../tasks/shutdown_services.yml | 2 +- .../bootstrap/persist-config/vars/main.yml | 1 + .../bootstrap/validate-config/tasks/main.yml | 2 + .../files/download_images.py | 19 ++++++- 7 files changed, 105 insertions(+), 18 deletions(-) diff --git a/playbookconfig/src/playbooks/roles/bootstrap/bringup-essential-services/tasks/bringup_local_registry.yml b/playbookconfig/src/playbooks/roles/bootstrap/bringup-essential-services/tasks/bringup_local_registry.yml index 0fb8d22b6..6c322316b 100644 --- a/playbookconfig/src/playbooks/roles/bootstrap/bringup-essential-services/tasks/bringup_local_registry.yml +++ b/playbookconfig/src/playbooks/roles/bootstrap/bringup-essential-services/tasks/bringup_local_registry.yml @@ -48,6 +48,51 @@ when: (insecure_registries is defined and insecure_registries | length > 0) +- name: Create containerd config file directory + file: + path: /etc/containerd + state: directory + mode: 0700 + +- name: Create config.toml file for containerd configuration + copy: + src: "{{ containerd_template }}" + dest: /etc/containerd/config.toml + remote_src: yes + mode: 0600 + +- name: Remove puppet template for insecure registries + replace: + path: /etc/containerd/config.toml + after: '# Begin of insecure registries' + regexp: '^(<%- @insecure_registries.+)\n(.+)\n(.+)\n(.+end -%>)' + replace: '' + +- name: Update config.toml with insecure registries + blockinfile: + path: /etc/containerd/config.toml + insertafter: '# Begin of insecure registries' + marker: " # {{ item }}" + block: |2 + [plugins.cri.registry.mirrors."{{ item }}"] + endpoint = ["http://{{ item }}"] + loop: + "{{ insecure_registries }}" + when: (insecure_registries is defined and + insecure_registries | length > 0) + +- name: Update config.toml with cni bin dir + command: "sed -i -e 's|<%= @k8s_cni_bin_dir %>|$CNI_BIN_DIR|g' /etc/containerd/config.toml" + args: + warn: false + environment: + CNI_BIN_DIR: "{{ kubelet_cni_bin_dir }}" + +- name: Restart containerd + systemd: + name: containerd + state: restarted + - name: Generate local registry runtime config file from template copy: src: "{{ registry_config_template }}" diff --git a/playbookconfig/src/playbooks/roles/bootstrap/bringup-essential-services/vars/main.yml b/playbookconfig/src/playbooks/roles/bootstrap/bringup-essential-services/vars/main.yml index bbd359224..029fe3a04 100644 --- a/playbookconfig/src/playbooks/roles/bootstrap/bringup-essential-services/vars/main.yml +++ b/playbookconfig/src/playbooks/roles/bootstrap/bringup-essential-services/vars/main.yml @@ -17,6 +17,7 @@ registry_token_server_template: /usr/share/puppet/modules/platform/templates/reg registry_token_server_file: /etc/docker-distribution/registry/token_server.conf cert_cnf_template: /usr/share/puppet/modules/platform/templates/registry-cert-extfile.erb insecure_docker_registry_template: /usr/share/puppet/modules/platform/templates/insecuredockerregistry.conf.erb +containerd_template: /usr/share/puppet/modules/platform/templates/config.toml.erb cert_cnf_file: /etc/ssl/private/registry-cert-extfile.cnf registry_cert_key: /etc/ssl/private/registry-cert.key registry_cert_crt: /etc/ssl/private/registry-cert.crt diff --git a/playbookconfig/src/playbooks/roles/bootstrap/persist-config/tasks/main.yml b/playbookconfig/src/playbooks/roles/bootstrap/persist-config/tasks/main.yml index 5f7e4ed85..8a75888e9 100644 --- a/playbookconfig/src/playbooks/roles/bootstrap/persist-config/tasks/main.yml +++ b/playbookconfig/src/playbooks/roles/bootstrap/persist-config/tasks/main.yml @@ -183,54 +183,75 @@ when: save_config_to_db -# Update docker config file and restart docker if docker proxy is -# configured +# Update docker and containerd config files and restart docker and containerd +# if docker proxy is configured - block: - - name: Ensure docker config directory exists + - name: Ensure docker and containerd config directory exist file: - path: /etc/systemd/system/docker.service.d + path: "{{ item }}" state: directory owner: root group: root mode: 0755 + with_items: + - /etc/systemd/system/docker.service.d + - /etc/systemd/system/containerd.service.d - - name: Ensure docker proxy config exists + - name: Ensure docker and containerd proxy config exist copy: content: "" - dest: "{{ docker_proxy_conf }}" + dest: "{{ item }}" force: no owner: root group: root mode: 0644 remote_src: yes + with_items: + - "{{ docker_proxy_conf }}" + - "{{ containerd_proxy_conf }}" - - name: Write header to docker proxy conf file + - name: Write header to docker and containerd proxy conf files lineinfile: - path: "{{ docker_proxy_conf }}" + path: "{{ item }}" line: "[Service]" + with_items: + - "{{ docker_proxy_conf }}" + - "{{ containerd_proxy_conf }}" - - name: Add http proxy URL to docker proxy conf file + - name: Add http proxy URL to docker and containerd proxy conf files lineinfile: - path: "{{ docker_proxy_conf }}" + path: "{{ item }}" line: "Environment='HTTP_PROXY={{ docker_http_proxy }}'" + with_items: + - "{{ docker_proxy_conf }}" + - "{{ containerd_proxy_conf }}" when: docker_http_proxy != 'undef' - - name: Add https proxy URL to docker proxy conf file + - name: Add https proxy URL to docker and containerd proxy conf files lineinfile: - path: "{{ docker_proxy_conf }}" + path: "{{ item }}" line: "Environment='HTTPS_PROXY={{ docker_https_proxy }}'" + with_items: + - "{{ docker_proxy_conf }}" + - "{{ containerd_proxy_conf }}" when: docker_https_proxy != 'undef' - - name: Add no proxy address list to docker proxy config file + - name: Add no proxy address list to docker and containerd proxy config files lineinfile: - path: "{{ docker_proxy_conf }}" + path: "{{ item }}" line: "Environment='NO_PROXY={{ docker_no_proxy_combined | join(',') }}'" + with_items: + - "{{ docker_proxy_conf }}" + - "{{ containerd_proxy_conf }}" - - name: Restart Docker + - name: Restart Docker and containerd systemd: state: restarted daemon_reload: yes - name: docker + name: "{{ item }}" + with_items: + - docker + - containerd when: use_docker_proxy diff --git a/playbookconfig/src/playbooks/roles/bootstrap/persist-config/tasks/shutdown_services.yml b/playbookconfig/src/playbooks/roles/bootstrap/persist-config/tasks/shutdown_services.yml index 4edd2959e..6937e3a65 100644 --- a/playbookconfig/src/playbooks/roles/bootstrap/persist-config/tasks/shutdown_services.yml +++ b/playbookconfig/src/playbooks/roles/bootstrap/persist-config/tasks/shutdown_services.yml @@ -26,7 +26,7 @@ - block: # Revert changes done by kubeadm init, clear data cache - name: Shut down and remove Kubernetes components - command: kubeadm reset -f + command: kubeadm reset -f --cri-socket /var/run/containerd/containerd.sock register: reset_info - debug: var=reset_info.stdout_lines diff --git a/playbookconfig/src/playbooks/roles/bootstrap/persist-config/vars/main.yml b/playbookconfig/src/playbooks/roles/bootstrap/persist-config/vars/main.yml index 8861d7245..3b570c714 100644 --- a/playbookconfig/src/playbooks/roles/bootstrap/persist-config/vars/main.yml +++ b/playbookconfig/src/playbooks/roles/bootstrap/persist-config/vars/main.yml @@ -1,6 +1,7 @@ --- keyring_workdir: /tmp/python_keyring docker_proxy_conf: /etc/systemd/system/docker.service.d/http-proxy.conf +containerd_proxy_conf: /etc/systemd/system/containerd.service.d/http-proxy.conf minimum_root_disk_size: 240 default_security_feature: "nopti nospectre_v2" temp_ssl_ca: "/tmp/ca-cert.pem" diff --git a/playbookconfig/src/playbooks/roles/bootstrap/validate-config/tasks/main.yml b/playbookconfig/src/playbooks/roles/bootstrap/validate-config/tasks/main.yml index e6d2212f9..0c0582b28 100644 --- a/playbookconfig/src/playbooks/roles/bootstrap/validate-config/tasks/main.yml +++ b/playbookconfig/src/playbooks/roles/bootstrap/validate-config/tasks/main.yml @@ -392,6 +392,8 @@ - localhost - 127.0.0.1 - registry.local + - "{{ cluster_service_start_address if cluster_service_start_address != 'derived' + else default_cluster_service_start_address }}" - "{{ controller_floating_address }}" - "{{ derived_network_params.controller_0_address }}" - "{{ external_oam_floating_address }}" diff --git a/playbookconfig/src/playbooks/roles/common/push-docker-images/files/download_images.py b/playbookconfig/src/playbooks/roles/common/push-docker-images/files/download_images.py index 9cdfb1c04..bbdecef1f 100644 --- a/playbookconfig/src/playbooks/roles/common/push-docker-images/files/download_images.py +++ b/playbookconfig/src/playbooks/roles/common/push-docker-images/files/download_images.py @@ -11,6 +11,8 @@ import sys import time import os import json +import keyring +import subprocess MAX_DOWNLOAD_ATTEMPTS = 3 MAX_DOWNLOAD_THREAD = 5 @@ -54,14 +56,29 @@ def download_an_image(img): local_img = 'registry.local:9001/' + new_img err_msg = " Image download failed: %s" % target_img + password = str(keyring.get_password("CGCS", "admin")) + if not password: + raise Exception("Local registry password not found.") + auth = '{0}:{1}'.format('admin', password) + for i in range(MAX_DOWNLOAD_ATTEMPTS): try: client = docker.APIClient() client.pull(target_img) + print("Image download succeeded: %s" % target_img) client.tag(target_img, local_img) client.push(local_img) - print("Image download succeeded: %s" % target_img) print("Image push succeeded: %s" % local_img) + # due to crictl doesn't support push function, docker client is used + # to pull and push image to local registry, then crictl download image + # from local registry. + subprocess.check_call(["crictl", "pull", "--creds", auth, local_img]) + print("Image %s download succeeded by containerd" % target_img) + # except armada/tiller, other docker images could be removed. + # TODO: run armada with containerd. + if not ('armada' in target_img or 'tiller' in target_img): + client.remove_image(target_img) + client.remove_image(local_img) return target_img, True except docker.errors.NotFound as e: print(err_msg + str(e))