Disk space issue on some zuul nodes
This PS moves containerd's and docker's runtime folders to /opt/. Zuul nodepool now may have nodes with 40GB / and 80 GB /opt that is different from previouly available 80Gb on / partition. Containerd increases disk space usage on zuul nodes and cannot fit into 40Gb /. Change-Id: I5079c83de5cb099890b674c6259965db880fad5f
This commit is contained in:
parent
840f697fd1
commit
497ba1d007
|
@ -39,6 +39,26 @@ export DEBIAN_FRONTEND=noninteractive
|
|||
|
||||
sudo swapoff -a
|
||||
|
||||
sudo mkdir -p /opt/ext_vol
|
||||
BIG_VOLUME=$(sudo fdisk -l 2>&1 | grep -E 80G | grep Linux | awk '{print $1}')
|
||||
if ! mount | grep "${BIG_VOLUME}"
|
||||
then
|
||||
sudo mkfs.ext4 "${BIG_VOLUME}"
|
||||
sudo mount "${BIG_VOLUME}" /opt/ext_vol
|
||||
df -h
|
||||
fi
|
||||
|
||||
sudo mkdir -p /opt/ext_vol/docker
|
||||
sudo mkdir -p /opt/ext_vol/containerd
|
||||
|
||||
sudo mkdir -p /opt/ext_vol/run_containerd
|
||||
sudo rsync -a /run/containerd/ /opt/ext_vol/run_containerd/
|
||||
sudo mount --bind /run/containerd /opt/ext_vol/run_containerd
|
||||
sudo systemctl restart containerd
|
||||
mount
|
||||
sudo fdisk --list
|
||||
df -h
|
||||
|
||||
echo "DefaultLimitMEMLOCK=16384" | sudo tee -a /etc/systemd/system.conf
|
||||
sudo systemctl daemon-reexec
|
||||
|
||||
|
@ -109,25 +129,7 @@ sudo add-apt-repository \
|
|||
$(lsb_release -cs) \
|
||||
stable"
|
||||
|
||||
# NOTE: Configure docker
|
||||
docker_resolv="/etc/resolv.conf"
|
||||
docker_dns_list="$(awk '/^nameserver/ { printf "%s%s",sep,"\"" $NF "\""; sep=", "} END{print ""}' "${docker_resolv}")"
|
||||
|
||||
sudo -E mkdir -p /etc/docker
|
||||
sudo -E tee /etc/docker/daemon.json <<EOF
|
||||
{
|
||||
"exec-opts": ["native.cgroupdriver=systemd"],
|
||||
"log-driver": "json-file",
|
||||
"log-opts": {
|
||||
"max-size": "100m"
|
||||
},
|
||||
"storage-driver": "overlay2",
|
||||
"live-restore": true,
|
||||
"dns": [${docker_dns_list}]
|
||||
}
|
||||
EOF
|
||||
|
||||
cat /etc/docker/daemon.json
|
||||
|
||||
if [ -n "${HTTP_PROXY}" ]; then
|
||||
sudo mkdir -p /etc/systemd/system/docker.service.d
|
||||
|
@ -174,6 +176,46 @@ install rbd /bin/true
|
|||
EOF
|
||||
|
||||
|
||||
|
||||
cat << EOF | sudo tee /etc/containerd/config.toml
|
||||
version = 2
|
||||
|
||||
# persistent data location
|
||||
root = "/opt/ext_vol/containerd"
|
||||
|
||||
[debug]
|
||||
level = "warn"
|
||||
|
||||
[plugins."io.containerd.grpc.v1.cri".containerd.runtimes.runc]
|
||||
runtime_type = "io.containerd.runc.v2"
|
||||
|
||||
[plugins."io.containerd.grpc.v1.cri".containerd.runtimes.runc.options]
|
||||
SystemdCgroup = true
|
||||
EOF
|
||||
sudo systemctl restart containerd
|
||||
|
||||
# NOTE: Configure docker
|
||||
docker_resolv="/etc/resolv.conf"
|
||||
docker_dns_list="$(awk '/^nameserver/ { printf "%s%s",sep,"\"" $NF "\""; sep=", "} END{print ""}' "${docker_resolv}")"
|
||||
|
||||
sudo -E mkdir -p /etc/docker
|
||||
sudo -E tee /etc/docker/daemon.json <<EOF
|
||||
{
|
||||
"data-root": "/opt/ext_vol/docker",
|
||||
"exec-opts": ["native.cgroupdriver=systemd"],
|
||||
"log-driver": "json-file",
|
||||
"log-opts": {
|
||||
"max-size": "100m"
|
||||
},
|
||||
"storage-driver": "overlay2",
|
||||
"live-restore": true,
|
||||
"dns": [${docker_dns_list}]
|
||||
}
|
||||
EOF
|
||||
|
||||
sudo systemctl restart docker
|
||||
|
||||
|
||||
# Prepare tmpfs for etcd when running on CI
|
||||
# CI VMs can have slow I/O causing issues for etcd
|
||||
# Only do this on CI (when user is zuul), so that local development can have a kubernetes
|
||||
|
@ -224,20 +266,6 @@ lsmod | grep overlay
|
|||
|
||||
|
||||
|
||||
cat << EOF | sudo tee /etc/containerd/config.toml
|
||||
version = 2
|
||||
|
||||
[debug]
|
||||
level = "warn"
|
||||
|
||||
[plugins."io.containerd.grpc.v1.cri".containerd.runtimes.runc]
|
||||
runtime_type = "io.containerd.runc.v2"
|
||||
|
||||
[plugins."io.containerd.grpc.v1.cri".containerd.runtimes.runc.options]
|
||||
SystemdCgroup = true
|
||||
EOF
|
||||
sudo systemctl restart containerd
|
||||
|
||||
|
||||
# Install CNI Plugins
|
||||
# from https://github.com/containernetworking/plugins.git
|
||||
|
@ -420,3 +448,4 @@ kubectl exec -it multitool -- nslookup google.com
|
|||
# w /dev/stdout' /etc/resolv.conf
|
||||
|
||||
cd "${CURRENT_DIR}"
|
||||
df -h
|
|
@ -111,3 +111,5 @@ if [[ ${MAKE_CHARTS_PROMENADE} = true ]] ; then
|
|||
fi
|
||||
|
||||
cd "${CURRENT_DIR}"
|
||||
|
||||
df -h
|
|
@ -44,3 +44,5 @@ docker run --rm --net host -p 8000:8000 --name armada \
|
|||
# # Set back permissions of the files
|
||||
sudo chmod "${AIRSKIFF_PERMISSIONS}" airskiff.yaml
|
||||
# sudo chmod "${KUBE_CONFIG_PERMISSIONS}" ~/.kube/config
|
||||
|
||||
df -h
|
|
@ -43,3 +43,5 @@ ${SHIPYARD} create configdocs airskiff-design \
|
|||
--directory=${PL_OUTPUT}
|
||||
${SHIPYARD} commit configdocs
|
||||
${SHIPYARD} create action update_software --allow-intermediate-commits
|
||||
|
||||
df -h
|
||||
|
|
|
@ -32,6 +32,8 @@
|
|||
- name: Deploy Kubernetes with Minikube
|
||||
shell: |
|
||||
set -ex
|
||||
sudo fdisk --list
|
||||
df -h
|
||||
./tools/deployment/airskiff/developer/010-deploy-k8s.sh
|
||||
args:
|
||||
chdir: "{{ zuul.project.src_dir }}"
|
||||
|
|
Loading…
Reference in New Issue