starlingx/ansible-playbooks
Code Issues Proposed changes

Resolve upstream containerd config format deprecation

Browse Source 
Upstream containerd v1.5 has deprecated a portion of the
configuration file format, and now wants each registry to be
configured in a separate file.
Containerd config files have to be reformatted and the
settings such as runtime_version are changed according to
containerd v1.5.

rm mirror defaults; doc registry deprecations:
d4be6aa8fa
Doc configure image registry:
https://github.com/containerd/containerd/blob/main/docs/cri/registry.md
Doc containerd runtimes configuration:
https://github.com/containerd/containerd/blob/main/docs/cri/config.md
Registry Configuration for hosts:
https://github.com/containerd/containerd/blob/main/docs/hosts.md

Test Plan:
PASS: k8s upgrade from 1.27.5 to 1.28.4 in AIO-SX
PASS: Verify containerd service running properly.
PASS: Verify containerd config file format are as
      expected.
PASS: Verify containerd logs look functional without
      new warning/error.
PASS: Add overrides for the insecure registry list and
      verify that config files get updated appropriately
PASS: Platform upgrade from previous release, verify
      that config files get updated appropriately
PASS: Installing on a subcloud in a DC environment

Story: 2010368
Task: 49067

Change-Id: Ie1f70a4a515b3793d894809fd96f3d74b3f32a2d
Signed-off-by: rakshith mr <rakshith.mr@windriver.com>
This commit is contained in:
rakshith mr 2023-11-07 11:27:54 -05:00
parent fc101416c8
commit 156975dbf7
9 changed files with 193 additions and 135 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
playbookconfig/src/playbooks/roles/bootstrap/bringup-essential-services/tasks/main.yml
View File

@ -79,7 +79,8 @@
import_tasks: configure_dockerd.yml
- name: Configure containerd to use local registry
import_tasks: configure_containerd.yml
import_role:
name: common/configure-containerd
- name: Populate image registry data to /var/lib/docker-distribution
shell: pigz -dc {{ bootstrap_registry_filesystem_fqpn }} | tar -C / --overwrite -xpf - var/lib/docker-distribution

117
playbookconfig/src/playbooks/roles/bootstrap/bringup-essential-services/templates/config.toml.j2
View File

@ -1,117 +0,0 @@
root = "/var/lib/docker"
state = "/var/run/containerd"
oom_score = 0
[grpc]
address = "/var/run/containerd/containerd.sock"
uid = 0
gid = 0
max_recv_message_size = 16777216
max_send_message_size = 16777216
[debug]
address = ""
uid = 0
gid = 0
level = ""
[metrics]
address = ""
grpc_histogram = false
[cgroup]
path = ""
[plugins]
[plugins.cgroups]
no_prometheus = false
[plugins.cri]
stream_server_address = "{{ stream_server_address }}"
stream_server_port = "0"
enable_selinux = false
sandbox_image = "{{ local_registry }}/k8s.gcr.io/pause:3.4.1"
stats_collect_period = 10
systemd_cgroup = false
enable_tls_streaming = false
max_container_log_line_size = 16384
[plugins.cri.containerd]
snapshotter = "overlayfs"
no_pivot = false
default_runtime_name = "runc"
[plugins.cri.containerd.runtimes]
[plugins.cri.containerd.runtimes.runc]
runtime_type = "io.containerd.runc.v1"
[plugins.cri.containerd.runtimes.runc.options]
NoPivotRoot = false
NoNewKeyring = false
ShimCgroup = ""
IoUid = 0
IoGid = 0
BinaryName = "runc"
Root = ""
CriuPath = ""
SystemdCgroup = false
[plugins.cri.containerd.runtimes.kata]
runtime_type = "io.containerd.kata.v2"
[plugins.cri.containerd.runtimes.katacli]
runtime_type = "io.containerd.runc.v1"
[plugins.cri.containerd.runtimes.katacli.options]
NoPivotRoot = false
NoNewKeyring = false
ShimCgroup = ""
IoUid = 0
IoGid = 0
BinaryName = "/usr/bin/kata-runtime"
Root = ""
CriuPath = ""
SystemdCgroup = false
[plugins.cri.containerd.runtimes.untrusted]
runtime_type = "io.containerd.kata.v2"
runtime_engine = ""
runtime_root = ""
[plugins.cri.cni]
# conf_dir is the directory in which the admin places a CNI conf.
conf_dir = "/etc/cni/net.d"
bin_dir = "{{ kubelet_cni_bin_dir }}"
max_conf_num = 1
conf_template = ""
[plugins.cri.registry]
[plugins.cri.registry.mirrors]
# Begin of insecure registries
<%- @insecure_registries.each do |insecure_registry| -%>
[plugins.cri.registry.mirrors."<%= insecure_registry %>"]
endpoint = ["http://<%= insecure_registry %>"]
<%- end -%>
# End of insecure registries
[plugins.cri.registry.configs."registry.local:9001".tls]
ca_file = "/etc/docker/certs.d/registry.local:9001/registry-cert.crt"
[plugins.cri.registry.configs."registry.local:9001".auth]
auth = "{{ registry_auth }}"
# Central cloud registry config for distributed cloud subclouds
{% if "{{ distributed_cloud_role }}" == 'subcloud' %}
[plugins.cri.registry.configs."registry.central:9001".tls]
ca_file = "/etc/docker/certs.d/registry.central:9001/registry-cert.crt"
{% endif -%}
# End central cloud registry config for distributed cloud subclouds
[plugins.cri.x509_key_pair_streaming]
tls_cert_file = ""
tls_key_file = ""
[plugins.diff-service]
default = ["walking"]
[plugins.linux]
shim = "containerd-shim"
runtime = "runc"
runtime_root = ""
no_shim = false
shim_debug = false
[plugins.opt]
path = "/opt/containerd"
[plugins.restart]
interval = "10s"
[plugins.scheduler]
pause_threshold = 0.02
deletion_threshold = 0
mutation_threshold = 100
schedule_delay = "0s"
startup_delay = "100ms"

67
playbookconfig/src/playbooks/roles/bootstrap/bringup-essential-services/tasks/configure_containerd.yml → playbookconfig/src/playbooks/roles/common/configure-containerd/tasks/main.yml
View File

@ -1,12 +1,12 @@
---
#
# Copyright (c) 2022 Wind River Systems, Inc.
# Copyright (c) 2024 Wind River Systems, Inc.
#
# SPDX-License-Identifier: Apache-2.0
#
# SUB-TASKS DESCRIPTION:
# ROLE DESCRIPTION:
# Configure containerd to use local registry
#
- name: Create containerd config file directory
file:
path: /etc/containerd
@ -51,19 +51,6 @@
regexp: '^(<%- @insecure_registries.+)\n(.+)\n(.+)\n(.+end -%>)'
replace: ''
- name: Update config.toml with insecure registries
blockinfile:
path: /etc/containerd/config.toml
insertafter: '# Begin of insecure registries'
marker: " # {{ item }}"
block: |2
[plugins.cri.registry.mirrors."{{ item }}"]
endpoint = ["http://{{ item }}"]
loop:
"{{ insecure_registries }}"
when: (insecure_registries is defined and
insecure_registries | length > 0)
- name: Get local registry credentials
vars:
script_content: |
@ -80,6 +67,54 @@
- set_fact:
local_registry_credentials: "{{ local_registry_credentials_output.stdout }}"
- name: Create containerd certs.d directory
file:
path: /etc/containerd/certs.d
state: directory
mode: 0700
- block:
- name: Default mirror directory for all registries
file:
path: /etc/containerd/certs.d/default
state: directory
mode: 0700
- name: Create hosts.toml file for containerd default mirrors configuration
template:
src: "mirror-hosts.toml.j2"
dest: /etc/containerd/certs.d/default/hosts.toml
mode: 0600
when: insecure_registries is defined
- name: Create containerd local registry directory
file:
path: "/etc/containerd/certs.d/{{ registry_local }}"
state: directory
mode: 0700
- name: Create hosts.toml file for containerd host configuration
template:
src: "local-hosts.toml.j2"
dest: "/etc/containerd/certs.d/{{ registry_local }}/hosts.toml"
mode: 0600
- block:
- name: Create containerd central registry directory
file:
path: "/etc/containerd/certs.d/{{ registry_central }}"
state: directory
mode: 0700
- name: Create hosts.toml file for containerd host configuration
template:
src: "central-hosts.toml.j2"
dest: "/etc/containerd/certs.d/{{ registry_central }}/hosts.toml"
mode: 0600
when: distributed_cloud_role == 'subcloud'
- name: Restart containerd
systemd:
name: containerd

9
playbookconfig/src/playbooks/roles/common/configure-containerd/templates/central-hosts.toml.j2 Normal file
View File

@ -0,0 +1,9 @@
# explicitly use v2 config format for identification
version = 2
server = "{{ registry_central }}"
[host."{{ registry_central }}"]
capabilities = ["pull", "resolve", "push"]
ca = "/etc/docker/certs.d/{{ registry_central }}/registry-cert.crt"
[host."{{ registry_central }}".header]
authorization = "Basic {{ registry_auth }}"

103
playbookconfig/src/playbooks/roles/common/configure-containerd/templates/config.toml.j2 Normal file
View File

@ -0,0 +1,103 @@
# explicitly use v2 config format for identification
version = 2
root = "/var/lib/docker"
state = "/var/run/containerd"
oom_score = 0
[grpc]
address = "/var/run/containerd/containerd.sock"
uid = 0
gid = 0
max_recv_message_size = 16777216
max_send_message_size = 16777216
[debug]
address = ""
uid = 0
gid = 0
level = ""
[metrics]
address = ""
grpc_histogram = false
[cgroup]
path = ""
[plugins]
[plugins."io.containerd.monitor.v1.cgroups"]
no_prometheus = false
[plugins."io.containerd.grpc.v1.cri"]
stream_server_address = "{{ stream_server_address }}"
stream_server_port = "0"
enable_selinux = false
sandbox_image = "{{ local_registry }}/k8s.gcr.io/pause:3.4.1"
stats_collect_period = 10
systemd_cgroup = false
enable_tls_streaming = false
max_container_log_line_size = 16384
[plugins."io.containerd.grpc.v1.cri".containerd]
snapshotter = "overlayfs"
no_pivot = false
default_runtime_name = "runc"
[plugins."io.containerd.grpc.v1.cri".containerd.runtimes]
[plugins."io.containerd.grpc.v1.cri".containerd.runtimes.runc]
runtime_type = "io.containerd.runc.v1"
[plugins."io.containerd.grpc.v1.cri".containerd.runtimes.runc.options]
NoPivotRoot = false
NoNewKeyring = false
ShimCgroup = ""
IoUid = 0
IoGid = 0
BinaryName = "runc"
Root = ""
CriuPath = ""
SystemdCgroup = false
[plugins."io.containerd.grpc.v1.cri".containerd.runtimes.kata]
runtime_type = "io.containerd.kata.v2"
[plugins."io.containerd.grpc.v1.cri".containerd.runtimes.katacli]
runtime_type = "io.containerd.runc.v1"
[plugins."io.containerd.grpc.v1.cri".containerd.runtimes.katacli.options]
NoPivotRoot = false
NoNewKeyring = false
ShimCgroup = ""
IoUid = 0
IoGid = 0
BinaryName = "/usr/bin/kata-runtime"
Root = ""
CriuPath = ""
SystemdCgroup = false
[plugins."io.containerd.grpc.v1.cri".containerd.runtimes.untrusted]
runtime_type = "io.containerd.kata.v2"
runtime_engine = ""
runtime_root = ""
[plugins."io.containerd.grpc.v1.cri".cni]
# conf_dir is the directory in which the admin places a CNI conf.
conf_dir = "/etc/cni/net.d"
bin_dir = "{{ kubelet_cni_bin_dir }}"
max_conf_num = 1
conf_template = ""
[plugins."io.containerd.grpc.v1.cri".registry]
config_path = "/etc/containerd/certs.d"
[plugins."io.containerd.grpc.v1.cri".x509_key_pair_streaming]
tls_cert_file = ""
tls_key_file = ""
[plugins."io.containerd.service.v1.diff-service"]
default = ["walking"]
[plugins."io.containerd.runtime.v1.linux"]
shim = "containerd-shim"
runtime = "runc"
runtime_root = ""
no_shim = false
shim_debug = false
[plugins."io.containerd.internal.v1.opt"]
path = "/opt/containerd"
[plugins."io.containerd.internal.v1.restart"]
interval = "10s"
[plugins."io.containerd.gc.v1.scheduler"]
pause_threshold = 0.02
deletion_threshold = 0
mutation_threshold = 100
schedule_delay = "0s"
startup_delay = "100ms"

9
playbookconfig/src/playbooks/roles/common/configure-containerd/templates/local-hosts.toml.j2 Normal file
View File

@ -0,0 +1,9 @@
# explicitly use v2 config format for identification
version = 2
server = "{{ registry_local }}"
[host."{{ registry_local }}"]
capabilities = ["pull", "resolve", "push"]
ca = "/etc/docker/certs.d/{{ registry_local }}/registry-cert.crt"
[host."{{ registry_local }}".header]
authorization = "Basic {{ registry_auth }}"

12
playbookconfig/src/playbooks/roles/common/configure-containerd/templates/mirror-hosts.toml.j2 Normal file
View File

@ -0,0 +1,12 @@
# explicitly use v2 config format for identification
# Updated the file to the new format
version = 2
# Begin of insecure registries
{% for insecure_registry in insecure_registries %}
server = "{{ insecure_registry }}"
[host."{{ insecure_registry }}"]
capabilities = ["pull", "resolve"]
{% endfor %}
# End of insecure registries

4
playbookconfig/src/playbooks/roles/common/configure-containerd/vars/main.yml Normal file
View File

@ -0,0 +1,4 @@
---
kubelet_cni_bin_dir: /var/opt/cni/bin
registry_central: registry.central:9001
registry_local: registry.local:9001

4
playbookconfig/src/playbooks/upgrade_platform_optimized.yml
View File

@ -1,6 +1,6 @@
---
#
# Copyright (c) 2023 Wind River Systems, Inc.
# Copyright (c) 2023-2024 Wind River Systems, Inc.
#
# SPDX-License-Identifier: Apache-2.0
#
@ -44,6 +44,8 @@
become: yes
- role: common/install-platform-certificates
become: yes
- role: common/configure-containerd
become: yes
- role: optimized-restore/complete-restore
become: yes
- role: common/host-unlock