K8s deploy script changes

This PS delivers the following changes ot deploy-k8s.sh script: - helm upgrade to 3.11.1 - kubernetes upgrade to 1.27.3 - coredns upgrade to 1.10.1 - calico 3.26.1 - minikube upgrade to 1.30.1 - switch from cri-dockers to containerd - deploy CNI plugins v0.8.5 - fixed dns resolvers issues Also this PS: - bumps up version of postgresql image to 14.8 - removes tiller chart from manifests Change-Id: I35d490a2bf2526c801da21e50ca065ed0b45980a
2023-06-26 18:13:17 +00:00 · 2023-06-26 18:13:17 +00:00 · 6d9dbbd7b2
commit 6d9dbbd7b2
parent 56036e4d04
6 changed files with 106 additions and 145 deletions
--- a/global/software/charts/ucp/armada/chart-group.yaml
+++ b/global/software/charts/ucp/armada/chart-group.yaml
@ -13,5 +13,4 @@ data:
  description: Armada
  sequenced: true
  chart_group:
-    - ucp-tiller
    - ucp-armada
--- a/global/software/charts/ucp/armada/tiller.yaml
+++ b/global/software/charts/ucp/armada/tiller.yaml
@ -1,77 +0,0 @@
---
-schema: armada/Chart/v1
-metadata:
-  schema: metadata/Document/v1
-  name: ucp-tiller
-  labels:
-    name: ucp-tiller-global
-  layeringDefinition:
-    abstract: false
-    layer: global
-  storagePolicy: cleartext
-  substitutions:
-    # Chart source
-    - src:
-        schema: pegleg/SoftwareVersions/v1
-        name: software-versions
-        path: .charts.ucp.tiller
-      dest:
-        path: .source
-
-    # Images
-    - src:
-        schema: pegleg/SoftwareVersions/v1
-        name: software-versions
-        path: .images.ucp.armada.tiller
-      dest:
-        path: .values.images.tags.tiller
-
-data:
-  chart_name: tiller
-  release: ucp-tiller
-  namespace: kube-system
-  wait:
-    timeout: 100
-    labels:
-      release_group: airship-ucp-tiller
-    native:
-      # Allows tiller to update its own release's status to DEPLOYED before it
-      # goes away during an upgrade, otherwise it can get stuck in
-      # PENDING_UPGRADE status.
-      enabled: false
-  install:
-    no_hooks: false
-  upgrade:
-    no_hooks: false
-    pre:
-      delete:
-        - type: job
-          labels:
-            release_group: airship-ucp-tiller
-  values: {}
-  dependencies:
-    - tiller-htk
-...
---
-schema: armada/Chart/v1
-metadata:
-  schema: metadata/Document/v1
-  name: tiller-htk
-  layeringDefinition:
-    abstract: false
-    layer: global
-  storagePolicy: cleartext
-  substitutions:
-    - src:
-        schema: pegleg/SoftwareVersions/v1
-        name: software-versions
-        path: .charts.ucp.tiller-htk
-      dest:
-        path: .source
-data:
-  chart_name: tiller-htk
-  release: tiller-htk
-  namespace: tiller-htk
-  values: {}
-  dependencies: []
-...
--- a/global/software/charts/ucp/shipyard/shipyard.yaml
+++ b/global/software/charts/ucp/shipyard/shipyard.yaml
@ -276,4 +276,6 @@ data:
      shipyard:
        keystone_authtoken:
          memcache_security_strategy: None
+    logrotate:
+      percent_max_log_fs_usage: 90
 ...
--- a/global/software/config/versions.yaml
+++ b/global/software/config/versions.yaml
@ -427,16 +427,6 @@ data:
        location: http://control-plane.minikube.internal:8282/ceph-rgw.tgz
        subpath: ceph-rgw
        type: tar
-      tiller:
-        location: https://opendev.org/airship/armada
-        reference: 50384e47c762438b9e39abe4677f3c29f3c09184
-        subpath: charts/tiller
-        type: git
-      tiller-htk:
-        location: https://opendev.org/openstack/openstack-helm-infra
-        reference: 97ce6d7d8e9a090c748800d69a57bbd9af698b60
-        subpath: helm-toolkit
-        type: git
    utility:
      calicoctl-utility:
        location: https://opendev.org/airship/porthole
@ -913,7 +903,7 @@ data:
        rabbit_init: docker.io/library/rabbitmq:3.9.0-management
        image_repo_sync: docker.io/docker:17.07.0
      deckhand:
-        db_init: docker.io/library/postgres:14.5
+        db_init: docker.io/library/postgres:14.8
        db_sync: quay.io/airshipit/deckhand:latest-ubuntu_focal
        deckhand: quay.io/airshipit/deckhand:latest-ubuntu_focal
        dep_check: quay.io/airshipit/kubernetes-entrypoint:v1.0.0
@ -928,7 +918,7 @@ data:
        ks_user: docker.io/openstackhelm/heat:newton
        ks_service: docker.io/openstackhelm/heat:newton
        ks_endpoints: docker.io/openstackhelm/heat:newton
-        drydock_db_init: docker.io/postgres:14.5
+        drydock_db_init: docker.io/library/postgres:14.8
        drydock_db_cleanup: quay.io/airshipit/drydock:master
        drydock_db_sync: quay.io/airshipit/drydock:master
      ingress:
@ -958,7 +948,7 @@ data:
        dep_check: quay.io/airshipit/kubernetes-entrypoint:v1.0.0
        image_repo_sync: docker.io/docker:17.07.0
      maas:
-        db_init: docker.io/library/postgres:14.5
+        db_init: docker.io/library/postgres:14.8
        db_sync: quay.io/airshipit/maas-region-controller:latest
        maas_rack: quay.io/airshipit/maas-rack-controller:latest
        maas_region: quay.io/airshipit/maas-region-controller:latest
@ -990,12 +980,12 @@ data:
      pegleg:
        pegleg: quay.io/airshipit/pegleg:latest-ubuntu_focal
      postgresql:
-        postgresql: docker.io/library/postgres:14.5
+        postgresql: docker.io/library/postgres:14.8
        dep_check: quay.io/airshipit/kubernetes-entrypoint:v1.0.0
        image_repo_sync: docker.io/library/docker:17.07.0
        ks_user: docker.io/openstackhelm/heat:stein-ubuntu_bionic
        prometheus_postgresql_exporter: docker.io/wrouesnel/postgres_exporter:v0.4.6
-        prometheus_postgresql_exporter_create_user: docker.io/library/postgres:14.5
+        prometheus_postgresql_exporter_create_user: docker.io/library/postgres:14.8
        postgresql_backup: "quay.io/airshipit/porthole-postgresql-utility:latest-ubuntu_focal"
      promenade:
        monitoring_image: busybox:1.28.3
@ -1017,10 +1007,10 @@ data:
        airflow: quay.io/airshipit/airflow:latest-ubuntu_focal
        shipyard: quay.io/airshipit/shipyard:latest-ubuntu_focal
        dep_check: quay.io/stackanetes/kubernetes-entrypoint:v0.3.1
-        shipyard_db_init: docker.io/postgres:14.5
-        shipyard_db_auxiliary: docker.io/postgres:14.5
+        shipyard_db_init: docker.io/library/postgres:14.8
+        shipyard_db_auxiliary: docker.io/library/postgres:14.8
        shipyard_db_sync: quay.io/airshipit/shipyard:latest-ubuntu_focal
-        airflow_db_init: docker.io/postgres:14.5
+        airflow_db_init: docker.io/library/postgres:14.8
        rabbit_init: docker.io/library/rabbitmq:3.9.0-management
        airflow_db_sync: quay.io/airshipit/airflow:latest-ubuntu_focal
        ks_user: docker.io/openstackhelm/heat:ocata
--- a/tools/deployment/airskiff/developer/010-deploy-k8s.sh
+++ b/tools/deployment/airskiff/developer/010-deploy-k8s.sh
@ -25,14 +25,12 @@ if [ -n "${PROXY}" ]; then
 fi

 # Deploy K8s with Minikube
-: "${HELM_VERSION:="v3.6.3"}"
-: "${KUBE_VERSION:="v1.26.3"}"
-: "${CRICTL_VERSION:="v1.26.0"}"
-: "${CRI_DOCKERD_VERSION:="v0.3.1"}"
-: "${CRI_DOCKERD_PACKAGE_VERSION:="0.3.1.3-0.ubuntu-focal"}"
-: "${MINIKUBE_VERSION:="v1.29.0"}"
-: "${CALICO_VERSION:="v3.25"}"
-: "${CORE_DNS_VERSION:="v1.9.4"}"
+: "${HELM_VERSION:="v3.11.1"}"
+: "${KUBE_VERSION:="v1.27.3"}"
+: "${MINIKUBE_VERSION:="v1.30.1"}"
+: "${CRICTL_VERSION:="v1.27.0"}"
+: "${CALICO_VERSION:="v3.26.1"}"
+: "${CORE_DNS_VERSION:="v1.10.1"}"
 : "${YQ_VERSION:="v4.6.0"}"
 : "${KUBE_DNS_IP="10.96.0.10"}"

@ -52,8 +50,7 @@ function configure_resolvconf {

  kube_dns_ip="${KUBE_DNS_IP}"
  # keep all nameservers from both resolv.conf excluding local addresses
-  old_ns=$(grep -P --no-filename "^nameserver\s+(?!127\.0\.0\.|${kube_dns_ip})" \
-           /etc/resolv.conf /run/systemd/resolve/resolv.conf | sort | uniq)
+  old_ns=$(cat /run/systemd/resolve/resolv.conf /etc/resolv.conf /run/systemd/resolve/resolv.conf | sort | uniq)

  if [[ -f "/run/systemd/resolve/resolv.conf" ]]; then
    sudo cp --remove-destination /run/systemd/resolve/resolv.conf /etc/resolv.conf
@ -66,8 +63,7 @@ function configure_resolvconf {
  sudo sed -i  "/^nameserver\s\+127.*/d" /etc/resolv.conf

  # Insert kube DNS as first nameserver instead of entirely overwriting /etc/resolv.conf
-  grep -q "nameserver ${kube_dns_ip}" /etc/resolv.conf || \
-    sudo sed -i -e "1inameserver ${kube_dns_ip}" /etc/resolv.conf
+  grep -q "nameserver ${kube_dns_ip}" /etc/resolv.conf || sudo sed -i -e "1inameserver ${kube_dns_ip}" /etc/resolv.conf

  local dns_servers
  if [ -z "${HTTP_PROXY}" ]; then
@ -76,25 +72,19 @@ function configure_resolvconf {
    dns_servers="${old_ns}"
  fi

-  grep -q "${dns_servers}" /etc/resolv.conf || \
-    echo -e ${dns_servers} | sudo tee -a /etc/resolv.conf
+  grep -q "${dns_servers}" /etc/resolv.conf || echo -e ${dns_servers} | sudo tee -a /etc/resolv.conf

-  grep -q "${dns_servers}" /run/systemd/resolve/resolv.conf || \
-    echo -e ${dns_servers} | sudo tee /run/systemd/resolve/resolv.conf
+  grep -q "${dns_servers}" /run/systemd/resolve/resolv.conf || echo -e ${dns_servers} | sudo tee /run/systemd/resolve/resolv.conf

  local search_options='search svc.cluster.local cluster.local'
-  grep -q "${search_options}" /etc/resolv.conf || \
-    echo "${search_options}" | sudo tee -a /etc/resolv.conf
+  grep -q "${search_options}" /etc/resolv.conf ||  echo "${search_options}" | sudo tee -a /etc/resolv.conf

-  grep -q "${search_options}" /run/systemd/resolve/resolv.conf || \
-    echo "${search_options}" | sudo tee -a /run/systemd/resolve/resolv.conf
+  grep -q "${search_options}" /run/systemd/resolve/resolv.conf ||  echo "${search_options}" | sudo tee -a /run/systemd/resolve/resolv.conf

  local dns_options='options ndots:5 timeout:1 attempts:1'
-  grep -q "${dns_options}" /etc/resolv.conf || \
-    echo ${dns_options} | sudo tee -a /etc/resolv.conf
+  grep -q "${dns_options}" /etc/resolv.conf ||  echo ${dns_options} | sudo tee -a /etc/resolv.conf

-  grep -q "${dns_options}" /run/systemd/resolve/resolv.conf || \
-    echo ${dns_options} | sudo tee -a /run/systemd/resolve/resolv.conf
+  grep -q "${dns_options}" /run/systemd/resolve/resolv.conf || echo ${dns_options} | sudo tee -a /run/systemd/resolve/resolv.conf
 }

 # NOTE: Clean Up hosts file
@ -108,6 +98,9 @@ configure_resolvconf
 # shellcheck disable=SC1091
 . /etc/os-release

+# uninstalling conflicting packages
+for pkg in docker.io docker-doc docker-compose podman-docker containerd runc; do sudo apt remove $pkg || true; done
+
 # NOTE: Add docker repo
 curl -fsSL https://download.docker.com/linux/ubuntu/gpg | sudo apt-key add -
 sudo apt-key fingerprint 0EBFCD88
@ -117,7 +110,7 @@ sudo add-apt-repository \
  stable"

 # NOTE: Configure docker
-docker_resolv="/run/systemd/resolve/resolv.conf"
+docker_resolv="/etc/resolv.conf"
 docker_dns_list="$(awk '/^nameserver/ { printf "%s%s",sep,"\"" $NF "\""; sep=", "} END{print ""}' "${docker_resolv}")"

 sudo -E mkdir -p /etc/docker
@ -134,6 +127,8 @@ sudo -E tee /etc/docker/daemon.json <<EOF
 }
 EOF

+cat /etc/docker/daemon.json
+
 if [ -n "${HTTP_PROXY}" ]; then
  sudo mkdir -p /etc/systemd/system/docker.service.d
  cat <<EOF | sudo -E tee /etc/systemd/system/docker.service.d/http-proxy.conf
@ -147,7 +142,7 @@ fi
 # Install required packages for K8s on host
 wget -q -O- 'https://download.ceph.com/keys/release.asc' | sudo apt-key add -
 RELEASE_NAME=$(grep 'CODENAME' /etc/lsb-release | awk -F= '{print $2}')
-sudo add-apt-repository "deb https://download.ceph.com/debian-nautilus/
+sudo add-apt-repository "deb https://download.ceph.com/debian-quincy/
 ${RELEASE_NAME} main"

 sudo -E apt-get update
@ -155,6 +150,8 @@ sudo -E apt-get install -y \
  docker-ce \
  docker-ce-cli \
  containerd.io \
+  docker-buildx-plugin \
+  docker-compose-plugin \
  socat \
  jq \
  util-linux \
@ -176,6 +173,7 @@ sudo -E tee /etc/modprobe.d/rbd.conf << EOF
 install rbd /bin/true
 EOF

+
 # Prepare tmpfs for etcd when running on CI
 # CI VMs can have slow I/O causing issues for etcd
 # Only do this on CI (when user is zuul), so that local development can have a kubernetes
@ -195,25 +193,70 @@ sudo -E curl -sSLo /usr/local/bin/kubectl "${URL}"/kubernetes-release/release/"$
 sudo -E chmod +x /usr/local/bin/minikube
 sudo -E chmod +x /usr/local/bin/kubectl

-
-# Install cri-dockerd
-# from https://github.com/Mirantis/cri-dockerd/releases
-CRI_TEMP_DIR=$(mktemp -d)
-pushd "${CRI_TEMP_DIR}"
-wget https://github.com/Mirantis/cri-dockerd/releases/download/${CRI_DOCKERD_VERSION}/cri-dockerd_${CRI_DOCKERD_PACKAGE_VERSION}_amd64.deb
-sudo dpkg -i "cri-dockerd_${CRI_DOCKERD_PACKAGE_VERSION}_amd64.deb"
-sudo dpkg --configure -a
-popd
-if [ -d "${CRI_TEMP_DIR}" ]; then
-  rm -rf mkdir "${CRI_TEMP_DIR}"
-fi
-
 # Install cri-tools
 wget https://github.com/kubernetes-sigs/cri-tools/releases/download/${CRICTL_VERSION}/crictl-${CRICTL_VERSION}-linux-amd64.tar.gz
 sudo tar zxvf "crictl-${CRICTL_VERSION}-linux-amd64.tar.g"z -C /usr/local/bin
 rm -f "crictl-${CRICTL_VERSION}-linux-amd64.tar.gz"


+#Forwarding IPv4 and letting iptables see bridged traffic
+cat <<EOF | sudo tee /etc/modules-load.d/k8s.conf
+overlay
+br_netfilter
+EOF
+
+sudo modprobe overlay
+sudo modprobe br_netfilter
+
+# sysctl params required by setup, params persist across reboots
+cat <<EOF | sudo tee /etc/sysctl.d/k8s.conf
+net.bridge.bridge-nf-call-iptables  = 1
+net.bridge.bridge-nf-call-ip6tables = 1
+net.ipv4.ip_forward                 = 1
+EOF
+
+# Apply sysctl params without reboot
+sudo sysctl --system
+sudo sysctl net.bridge.bridge-nf-call-iptables net.bridge.bridge-nf-call-ip6tables net.ipv4.ip_forward
+
+lsmod | grep br_netfilter
+lsmod | grep overlay
+
+
+
+cat << EOF | sudo tee /etc/containerd/config.toml
+version = 2
+
+[debug]
+  level = "warn"
+
+[plugins."io.containerd.grpc.v1.cri".containerd.runtimes.runc]
+  runtime_type = "io.containerd.runc.v2"
+
+  [plugins."io.containerd.grpc.v1.cri".containerd.runtimes.runc.options]
+    SystemdCgroup = true
+EOF
+sudo systemctl restart containerd
+
+
+# Install CNI Plugins
+# from https://github.com/containernetworking/plugins.git
+CNI_TEMP_DIR=$(mktemp -d)
+pushd "${CNI_TEMP_DIR}"
+git clone https://github.com/containernetworking/plugins.git
+pushd plugins
+git checkout v0.8.5
+popd
+docker run --rm  -v ./plugins:/usr/local/src -w /usr/local/src golang:1.13.8  bash -c './build_linux.sh'
+sudo mkdir -p /opt/cni
+sudo cp -a plugins/bin /opt/cni/
+popd
+if [ -d "${CNI_TEMP_DIR}" ]; then
+  sudo rm -rf mkdir "${CNI_TEMP_DIR}"
+fi
+sudo systemctl restart containerd
+sudo systemctl restart docker
+
 # Install Helm
 TMP_DIR=$(mktemp -d)
 sudo -E bash -c \
@ -226,10 +269,6 @@ rm -rf "${TMP_DIR}"
 sudo -E minikube config set kubernetes-version "${KUBE_VERSION}"
 sudo -E minikube config set vm-driver none

-# NOTE: set RemoveSelfLink to false, to enable it as it is required by the ceph-rbd-provisioner.
-# SelfLinks were deprecated in k8s v1.16, and in k8s v1.20, they are
-# disabled by default.
-# https://github.com/kubernetes/enhancements/issues/1164
 export CHANGE_MINIKUBE_NONE_USER=true
 export MINIKUBE_IN_STYLE=false

@ -243,20 +282,24 @@ if [[ "${api_server_status}" != "Running" ]]; then
    --docker-env HTTPS_PROXY="${HTTPS_PROXY}" \
    --docker-env NO_PROXY="${NO_PROXY},10.96.0.0/12" \
    --network-plugin=cni \
+    --cni=calico \
    --wait=apiserver,system_pods \
    --apiserver-names="$(hostname -f)" \
    --extra-config=controller-manager.allocate-node-cidrs=true \
    --extra-config=controller-manager.cluster-cidr=192.168.0.0/16 \
    --extra-config=kube-proxy.mode=ipvs \
    --extra-config=apiserver.service-node-port-range=1-65535 \
-    --embed-certs
+    --extra-config=kubelet.cgroup-driver=systemd \
+    --extra-config=kubelet.resolv-conf=/run/systemd/resolve/resolv.conf \
+    --embed-certs \
+    --container-runtime=containerd
 fi

 sudo -E systemctl enable --now kubelet

 sudo -E minikube addons list

-curl -LSs https://docs.projectcalico.org/archive/"${CALICO_VERSION}"/manifests/calico.yaml -o /tmp/calico.yaml
+curl -LSs https://raw.githubusercontent.com/projectcalico/calico/"${CALICO_VERSION}"/manifests/calico.yaml -o /tmp/calico.yaml

 sed -i -e 's#docker.io/calico/#quay.io/calico/#g' /tmp/calico.yaml

@ -354,6 +397,14 @@ host -v control-plane.minikube.internal

 kubectl label nodes --all --overwrite ucp-control-plane=enabled

+
+kubectl run multitool --image=praqma/network-multitool
+kubectl wait --for=condition=ready pod multitool --timeout=300s
+kubectl exec -it multitool -- nslookup control-plane.minikube.internal
+kubectl exec -it multitool -- ping -c 4 8.8.8.8
+kubectl exec -it multitool -- nslookup google.com
+
+
 # # Add user to Docker group
 # # NOTE: This requires re-authentication. Restart your shell.
 # sudo adduser "$(whoami)" docker
--- a/tools/gate/playbooks/airskiff-deploy-gate.yaml
+++ b/tools/gate/playbooks/airskiff-deploy-gate.yaml
@ -14,11 +14,7 @@

 - hosts: all

-  # roles:
-  #   - disable-systemd-resolved
-
  tasks:
-
    - name: Clone dependencies
      shell: |
        set -ex