From 63cbad8f42dbfd4769d93cd86d099b3fac057737 Mon Sep 17 00:00:00 2001 From: Vladimir Kozhukalov Date: Wed, 13 Sep 2023 21:59:42 +0300 Subject: [PATCH] Fix deploy-env role to work on Ubuntu Jammy Change-Id: Idfddfa48a7078ca63efa085191a5a07cdb8c2e61 --- roles/deploy-env/files/containerd_config.toml | 257 +++++++++++++++++- roles/deploy-env/files/hosts | 2 + roles/deploy-env/tasks/common_k8s.yaml | 5 + roles/deploy-env/tasks/control-plane.yaml | 2 + roles/deploy-env/tasks/main.yaml | 6 +- 5 files changed, 263 insertions(+), 9 deletions(-) create mode 100644 roles/deploy-env/files/hosts diff --git a/roles/deploy-env/files/containerd_config.toml b/roles/deploy-env/files/containerd_config.toml index cc6ab0bc4..0f2c22e38 100644 --- a/roles/deploy-env/files/containerd_config.toml +++ b/roles/deploy-env/files/containerd_config.toml @@ -1,11 +1,256 @@ -version = 2 disabled_plugins = [] -[plugins."io.containerd.grpc.v1.cri".registry] -config_path = "/etc/containerd/certs.d" +imports = [] +oom_score = 0 +plugin_dir = "" +required_plugins = [] +root = "/var/lib/containerd" +state = "/run/containerd" +temp = "" +version = 2 +[cgroup] + path = "" + +[debug] + address = "" + format = "" + gid = 0 + level = "" + uid = 0 + +[grpc] + address = "/run/containerd/containerd.sock" + gid = 0 + max_recv_message_size = 16777216 + max_send_message_size = 16777216 + tcp_address = "" + tcp_tls_ca = "" + tcp_tls_cert = "" + tcp_tls_key = "" + uid = 0 + +[metrics] + address = "" + grpc_histogram = false + +[plugins] + + [plugins."io.containerd.gc.v1.scheduler"] + deletion_threshold = 0 + mutation_threshold = 100 + pause_threshold = 0.02 + schedule_delay = "0s" + startup_delay = "100ms" + + [plugins."io.containerd.grpc.v1.cri"] + device_ownership_from_security_context = false + disable_apparmor = false + disable_cgroup = false + disable_hugetlb_controller = true + disable_proc_mount = false + disable_tcp_service = true + enable_selinux = false + enable_tls_streaming = false + enable_unprivileged_icmp = false + enable_unprivileged_ports = false + ignore_image_defined_volumes = false + max_concurrent_downloads = 3 + max_container_log_line_size = 16384 + netns_mounts_under_state_dir = false + restrict_oom_score_adj = false + sandbox_image = "registry.k8s.io/pause:3.6" + selinux_category_range = 1024 + stats_collect_period = 10 + stream_idle_timeout = "4h0m0s" + stream_server_address = "127.0.0.1" + stream_server_port = "0" + systemd_cgroup = false + tolerate_missing_hugetlb_controller = true + unset_seccomp_profile = "" + + [plugins."io.containerd.grpc.v1.cri".cni] + bin_dir = "/opt/cni/bin" + conf_dir = "/etc/cni/net.d" + conf_template = "" + ip_pref = "" + max_conf_num = 1 + + [plugins."io.containerd.grpc.v1.cri".containerd] + default_runtime_name = "runc" + disable_snapshot_annotations = true + discard_unpacked_layers = false + ignore_rdt_not_enabled_errors = false + no_pivot = false + snapshotter = "overlayfs" + + [plugins."io.containerd.grpc.v1.cri".containerd.default_runtime] + base_runtime_spec = "" + cni_conf_dir = "" + cni_max_conf_num = 0 + container_annotations = [] + pod_annotations = [] + privileged_without_host_devices = false + runtime_engine = "" + runtime_path = "" + runtime_root = "" + runtime_type = "" + + [plugins."io.containerd.grpc.v1.cri".containerd.default_runtime.options] + + [plugins."io.containerd.grpc.v1.cri".containerd.runtimes] + + [plugins."io.containerd.grpc.v1.cri".containerd.runtimes.runc] + base_runtime_spec = "" + cni_conf_dir = "" + cni_max_conf_num = 0 + container_annotations = [] + pod_annotations = [] + privileged_without_host_devices = false + runtime_engine = "" + runtime_path = "" + runtime_root = "" + runtime_type = "io.containerd.runc.v2" + + [plugins."io.containerd.grpc.v1.cri".containerd.runtimes.runc.options] + BinaryName = "" + CriuImagePath = "" + CriuPath = "" + CriuWorkPath = "" + IoGid = 0 + IoUid = 0 + NoNewKeyring = false + NoPivotRoot = false + Root = "" + ShimCgroup = "" + SystemdCgroup = true + + [plugins."io.containerd.grpc.v1.cri".containerd.untrusted_workload_runtime] + base_runtime_spec = "" + cni_conf_dir = "" + cni_max_conf_num = 0 + container_annotations = [] + pod_annotations = [] + privileged_without_host_devices = false + runtime_engine = "" + runtime_path = "" + runtime_root = "" + runtime_type = "" + + [plugins."io.containerd.grpc.v1.cri".containerd.untrusted_workload_runtime.options] + + [plugins."io.containerd.grpc.v1.cri".image_decryption] + key_model = "node" + + [plugins."io.containerd.grpc.v1.cri".registry] + config_path = "/etc/containerd/certs.d" + + [plugins."io.containerd.grpc.v1.cri".registry.auths] + + [plugins."io.containerd.grpc.v1.cri".registry.configs] {% for item in registry_namespaces %} {% if item.auth is defined %} -[plugins."io.containerd.grpc.v1.cri".registry.configs."{{ item.namespace }}".auth] -auth = "{{ item.auth }}" + [plugins."io.containerd.grpc.v1.cri".registry.configs."{{ item.namespace }}".auth] + auth = "{{ item.auth }}" {% endif %} -{% endfor %} \ No newline at end of file +{% endfor %} + + [plugins."io.containerd.grpc.v1.cri".registry.headers] + + [plugins."io.containerd.grpc.v1.cri".registry.mirrors] + + [plugins."io.containerd.grpc.v1.cri".x509_key_pair_streaming] + tls_cert_file = "" + tls_key_file = "" + + [plugins."io.containerd.internal.v1.opt"] + path = "/opt/containerd" + + [plugins."io.containerd.internal.v1.restart"] + interval = "10s" + + [plugins."io.containerd.internal.v1.tracing"] + sampling_ratio = 1.0 + service_name = "containerd" + + [plugins."io.containerd.metadata.v1.bolt"] + content_sharing_policy = "shared" + + [plugins."io.containerd.monitor.v1.cgroups"] + no_prometheus = false + + [plugins."io.containerd.runtime.v1.linux"] + no_shim = false + runtime = "runc" + runtime_root = "" + shim = "containerd-shim" + shim_debug = false + + [plugins."io.containerd.runtime.v2.task"] + platforms = ["linux/amd64"] + sched_core = false + + [plugins."io.containerd.service.v1.diff-service"] + default = ["walking"] + + [plugins."io.containerd.service.v1.tasks-service"] + rdt_config_file = "" + + [plugins."io.containerd.snapshotter.v1.aufs"] + root_path = "" + + [plugins."io.containerd.snapshotter.v1.btrfs"] + root_path = "" + + [plugins."io.containerd.snapshotter.v1.devmapper"] + async_remove = false + base_image_size = "" + discard_blocks = false + fs_options = "" + fs_type = "" + pool_name = "" + root_path = "" + + [plugins."io.containerd.snapshotter.v1.native"] + root_path = "" + + [plugins."io.containerd.snapshotter.v1.overlayfs"] + root_path = "" + upperdir_label = false + + [plugins."io.containerd.snapshotter.v1.zfs"] + root_path = "" + + [plugins."io.containerd.tracing.processor.v1.otlp"] + endpoint = "" + insecure = false + protocol = "" + +[proxy_plugins] + +[stream_processors] + + [stream_processors."io.containerd.ocicrypt.decoder.v1.tar"] + accepts = ["application/vnd.oci.image.layer.v1.tar+encrypted"] + args = ["--decryption-keys-path", "/etc/containerd/ocicrypt/keys"] + env = ["OCICRYPT_KEYPROVIDER_CONFIG=/etc/containerd/ocicrypt/ocicrypt_keyprovider.conf"] + path = "ctd-decoder" + returns = "application/vnd.oci.image.layer.v1.tar" + + [stream_processors."io.containerd.ocicrypt.decoder.v1.tar.gzip"] + accepts = ["application/vnd.oci.image.layer.v1.tar+gzip+encrypted"] + args = ["--decryption-keys-path", "/etc/containerd/ocicrypt/keys"] + env = ["OCICRYPT_KEYPROVIDER_CONFIG=/etc/containerd/ocicrypt/ocicrypt_keyprovider.conf"] + path = "ctd-decoder" + returns = "application/vnd.oci.image.layer.v1.tar+gzip" + +[timeouts] + "io.containerd.timeout.bolt.open" = "0s" + "io.containerd.timeout.shim.cleanup" = "5s" + "io.containerd.timeout.shim.load" = "5s" + "io.containerd.timeout.shim.shutdown" = "3s" + "io.containerd.timeout.task.state" = "2s" + +[ttrpc] + address = "" + gid = 0 + uid = 0 diff --git a/roles/deploy-env/files/hosts b/roles/deploy-env/files/hosts new file mode 100644 index 000000000..daf6251a4 --- /dev/null +++ b/roles/deploy-env/files/hosts @@ -0,0 +1,2 @@ +127.0.0.1 localhost +{{ ansible_default_ipv4['address'] }} {{ ansible_hostname }} diff --git a/roles/deploy-env/tasks/common_k8s.yaml b/roles/deploy-env/tasks/common_k8s.yaml index ad222dfde..2d942ec04 100644 --- a/roles/deploy-env/tasks/common_k8s.yaml +++ b/roles/deploy-env/tasks/common_k8s.yaml @@ -85,6 +85,11 @@ - /etc/resolv.conf - /run/systemd/resolve/resolv.conf +- name: Configure /etc/hosts + template: + src: files/hosts + dest: /etc/hosts + # We download Calico manifest on all nodes because we then want to download # Calico images BEFORE deploying it - name: Download Calico manifest diff --git a/roles/deploy-env/tasks/control-plane.yaml b/roles/deploy-env/tasks/control-plane.yaml index 1063aebfc..8c2f9997c 100644 --- a/roles/deploy-env/tasks/control-plane.yaml +++ b/roles/deploy-env/tasks/control-plane.yaml @@ -64,6 +64,8 @@ kubectl patch configmap coredns -n kube-system --patch-file "${PATCH}" kubectl set image deployment coredns -n kube-system "coredns=registry.k8s.io/coredns/coredns:v1.9.4" kubectl rollout restart -n kube-system deployment/coredns + sleep 10 + kubectl -n kube-system wait --timeout=240s --for=condition=Ready pods -l k8s-app=kube-dns rm -f "${PATCH}" args: executable: /bin/bash diff --git a/roles/deploy-env/tasks/main.yaml b/roles/deploy-env/tasks/main.yaml index 99e7925cb..7ba7fec0f 100644 --- a/roles/deploy-env/tasks/main.yaml +++ b/roles/deploy-env/tasks/main.yaml @@ -23,11 +23,11 @@ - name: Wait for cluster is ready become: false block: + - name: Sleep 10 before checking calico nodes + pause: + seconds: 10 - name: Wait for Calico pods ready command: kubectl -n kube-system wait --timeout=240s --for=condition=Ready pods -l k8s-app=calico-node - - - name: Wait for Coredns pods ready - command: kubectl -n kube-system wait --timeout=240s --for=condition=Ready pods -l k8s-app=kube-dns when: inventory_hostname == 'primary' - name: Add coredns to /etc/resolv.conf