Disk space issue on some zuul nodes

This PS moves containerd's and docker's runtime folders to /opt/.

Zuul nodepool now may have nodes with 40GB / and 80 GB /opt that is
different from previouly available 80Gb on / partition. Containerd
increases disk space usage on zuul nodes and cannot fit into 40Gb /.

Change-Id: I5079c83de5cb099890b674c6259965db880fad5f

tools/deployment/airskiff/developer/010-deploy-k8s.sh

@@ -39,6 +39,18 @@ export DEBIAN_FRONTEND=noninteractive
 
 sudo swapoff -a
 
+sudo mkdir -p /opt/ext_vol
+sudo mkdir -p /opt/ext_vol/docker
+sudo mkdir -p /opt/ext_vol/containerd
+
+sudo mkdir -p /opt/ext_vol/run_containerd
+sudo rsync -a /run/containerd/ /opt/ext_vol/run_containerd/
+sudo mount --bind /run/containerd /opt/ext_vol/run_containerd
+sudo systemctl restart containerd
+mount
+sudo fdisk --list
+df -h
+
 echo "DefaultLimitMEMLOCK=16384" | sudo tee -a /etc/systemd/system.conf
 sudo systemctl daemon-reexec
 
@@ -109,25 +121,7 @@ sudo add-apt-repository \
   $(lsb_release -cs) \
   stable"
 
-# NOTE: Configure docker
-docker_resolv="/etc/resolv.conf"
-docker_dns_list="$(awk '/^nameserver/ { printf "%s%s",sep,"\"" $NF "\""; sep=", "} END{print ""}' "${docker_resolv}")"
 
-sudo -E mkdir -p /etc/docker
-sudo -E tee /etc/docker/daemon.json <<EOF
-{
-  "exec-opts": ["native.cgroupdriver=systemd"],
-  "log-driver": "json-file",
-  "log-opts": {
-    "max-size": "100m"
-  },
-  "storage-driver": "overlay2",
-  "live-restore": true,
-  "dns": [${docker_dns_list}]
-}
-EOF
-
-cat /etc/docker/daemon.json
 
 if [ -n "${HTTP_PROXY}" ]; then
   sudo mkdir -p /etc/systemd/system/docker.service.d
@@ -174,6 +168,46 @@ install rbd /bin/true
 EOF
 
 
+
+cat << EOF | sudo tee /etc/containerd/config.toml
+version = 2
+
+# persistent data location
+root = "/opt/ext_vol/containerd"
+
+[debug]
+  level = "warn"
+
+[plugins."io.containerd.grpc.v1.cri".containerd.runtimes.runc]
+  runtime_type = "io.containerd.runc.v2"
+
+  [plugins."io.containerd.grpc.v1.cri".containerd.runtimes.runc.options]
+    SystemdCgroup = true
+EOF
+sudo systemctl restart containerd
+
+# NOTE: Configure docker
+docker_resolv="/etc/resolv.conf"
+docker_dns_list="$(awk '/^nameserver/ { printf "%s%s",sep,"\"" $NF "\""; sep=", "} END{print ""}' "${docker_resolv}")"
+
+sudo -E mkdir -p /etc/docker
+sudo -E tee /etc/docker/daemon.json <<EOF
+{
+  "data-root": "/opt/ext_vol/docker",
+  "exec-opts": ["native.cgroupdriver=systemd"],
+  "log-driver": "json-file",
+  "log-opts": {
+    "max-size": "100m"
+  },
+  "storage-driver": "overlay2",
+  "live-restore": true,
+  "dns": [${docker_dns_list}]
+}
+EOF
+
+sudo systemctl restart docker
+
+
 # Prepare tmpfs for etcd when running on CI
 # CI VMs can have slow I/O causing issues for etcd
 # Only do this on CI (when user is zuul), so that local development can have a kubernetes
@@ -224,20 +258,6 @@ lsmod | grep overlay
 
 
 
-cat << EOF | sudo tee /etc/containerd/config.toml
-version = 2
-
-[debug]
-  level = "warn"
-
-[plugins."io.containerd.grpc.v1.cri".containerd.runtimes.runc]
-  runtime_type = "io.containerd.runc.v2"
-
-  [plugins."io.containerd.grpc.v1.cri".containerd.runtimes.runc.options]
-    SystemdCgroup = true
-EOF
-sudo systemctl restart containerd
-
 
 # Install CNI Plugins
 # from https://github.com/containernetworking/plugins.git
@@ -420,3 +440,4 @@ kubectl exec -it multitool -- nslookup google.com
 #                  w /dev/stdout' /etc/resolv.conf
 
 cd "${CURRENT_DIR}"
+df -h

tools/deployment/airskiff/developer/015-make-all-charts.sh

@@ -111,3 +111,4 @@ if [[ ${MAKE_CHARTS_PROMENADE} = true ]] ; then
 fi
 
 cd "${CURRENT_DIR}"
+df -h

tools/deployment/airskiff/developer/030-armada-bootstrap.sh

@@ -44,3 +44,5 @@ docker run --rm --net host -p 8000:8000 --name armada \
 # # Set back permissions of the files
 sudo chmod "${AIRSKIFF_PERMISSIONS}" airskiff.yaml
 # sudo chmod "${KUBE_CONFIG_PERMISSIONS}" ~/.kube/config
+
+df -h

tools/deployment/airskiff/developer/100-deploy-osh.sh

@@ -43,3 +43,5 @@ ${SHIPYARD} create configdocs airskiff-design \
              --directory=${PL_OUTPUT}
 ${SHIPYARD} commit configdocs
 ${SHIPYARD} create action update_software --allow-intermediate-commits
+
+df -h

tools/gate/playbooks/airskiff-deploy-gate.yaml

@@ -32,6 +32,16 @@
     - name: Deploy Kubernetes with Minikube
       shell: |
         set -ex
+        sudo fdisk --list
+        df -h
+        sudo mkdir -p /opt/ext_vol
+        BIG_VOLUME=$(sudo fdisk -l 2>&1 | grep  -E 80G | grep Linux | awk '{print $1}')
+        if ! mount | grep "${BIG_VOLUME}"
+        then
+          sudo mkfs.ext4 "${BIG_VOLUME}"
+          sudo mount "${BIG_VOLUME}" /opt/ext_vol
+          df -h
+        fi
         ./tools/deployment/airskiff/developer/010-deploy-k8s.sh
       args:
         chdir: "{{ zuul.project.src_dir }}"