Libvirt daemonset for the VINO operator

Change-Id: I47273f6f8cf7325764d1786b87e032e9bd1aa046
2020-12-17 00:02:59 +00:00 · 2020-12-17 00:02:59 +00:00 · d4686cebe5
commit d4686cebe5
parent feb1aaabb7
4 changed files with 261 additions and 0 deletions
--- a/config/libvirt-sushy/libvirt-bin-configmap.yaml
+++ b/config/libvirt-sushy/libvirt-bin-configmap.yaml
@ -0,0 +1,148 @@
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: libvirt-bin
+data:
+  libvirt.sh: |
+    #!/bin/bash
+    set -ex
+
+    if [ -n "$(cat /proc/*/comm 2>/dev/null | grep -w libvirtd)" ]; then
+      set +x
+      for proc in $(ls /proc/*/comm 2>/dev/null); do
+        if [ "x$(cat $proc 2>/dev/null | grep -w libvirtd)" == "xlibvirtd" ]; then
+          set -x
+          libvirtpid=$(echo $proc | cut -f 3 -d '/')
+          echo "WARNING: libvirtd daemon already running on host" 1>&2
+          echo "$(cat "/proc/${libvirtpid}/status" 2>/dev/null | grep State)" 1>&2
+          kill -9 "$libvirtpid" || true
+          set +x
+        fi
+      done
+      set -x
+    fi
+
+    rm -f /var/run/libvirtd.pid
+
+    if [[ -c /dev/kvm ]]; then
+        chmod 660 /dev/kvm
+        chown root:kvm /dev/kvm
+    fi
+
+    CGROUPS=""
+    for CGROUP in cpu rdma hugetlb; do
+      if [ -d /sys/fs/cgroup/${CGROUP} ]; then
+        CGROUPS+="${CGROUP},"
+      fi
+    done
+    cgcreate -g ${CGROUPS%,}:/osh-libvirt
+
+    hp_count="$(cat /proc/meminfo | grep HugePages_Total | tr -cd '[:digit:]')"
+    if [ 0"$hp_count" -gt 0 ]; then
+
+      echo "INFO: Detected hugepage count of '$hp_count'. Enabling hugepage settings for libvirt/qemu."
+
+      if [ -n "$(grep KVM_HUGEPAGES=0 /etc/default/qemu-kvm)" ]; then
+        sed -i 's/.*KVM_HUGEPAGES=0.*/KVM_HUGEPAGES=1/g' /etc/default/qemu-kvm
+      else
+        echo KVM_HUGEPAGES=1 >> /etc/default/qemu-kvm
+      fi
+
+      if [ ! -d /dev/hugepages ]; then
+        echo "ERROR: Hugepages configured in kernel, but libvirtd container cannot access /dev/hugepages"
+        exit 1
+      fi
+
+      if [ -d /sys/fs/cgroup/hugetlb ]; then
+        limits="$(ls /sys/fs/cgroup/hugetlb/{{ .Values.conf.kubernetes.cgroup }}/hugetlb.*.limit_in_bytes)" || \
+          (echo "ERROR: Failed to locate any hugetable limits. Did you set the correct cgroup in your values used for this chart?"
+           exit 1)
+        for limit in $limits; do
+          target="/sys/fs/cgroup/hugetlb/$(dirname $(awk -F: '($2~/hugetlb/){print $3}' /proc/self/cgroup))/$(basename $limit)"
+          if [ ! -f "$target" ]; then
+            echo "ERROR: Could not find write target for hugepage limit: $target"
+          fi
+
+          echo "$(cat $limit)" > "$target"
+        done
+      fi
+
+      default_hp_kb="$(cat /proc/meminfo | grep Hugepagesize | tr -cd '[:digit:]')"
+
+      num_free_pages="$(cat /sys/kernel/mm/hugepages/hugepages-${default_hp_kb}kB/free_hugepages | tr -cd '[:digit:]')"
+      echo "INFO: '$num_free_pages' free hugepages of size ${default_hp_kb}kB"
+      if [ 0"$num_free_pages" -gt 0 ]; then
+        (fallocate -o0 -l "$default_hp_kb" /dev/hugepages/foo && rm /dev/hugepages/foo) || \
+          (echo "ERROR: fallocate failed test at /dev/hugepages with size ${default_hp_kb}kB"
+           rm /dev/hugepages/foo
+           exit 1)
+      fi
+    fi
+
+    if [ -n "${LIBVIRT_CEPH_CINDER_SECRET_UUID}" ] ; then
+      cgexec -g ${CGROUPS%,}:/osh-libvirt systemd-run --scope --slice=system libvirtd --listen &
+
+      tmpsecret=$(mktemp --suffix .xml)
+      if [ -n "${LIBVIRT_EXTERNAL_CEPH_CINDER_SECRET_UUID}" ] ; then
+        tmpsecret2=$(mktemp --suffix .xml)
+      fi
+      function cleanup {
+        rm -f "${tmpsecret}"
+        if [ -n "${LIBVIRT_EXTERNAL_CEPH_CINDER_SECRET_UUID}" ] ; then
+          rm -f "${tmpsecret2}"
+        fi
+      }
+      trap cleanup EXIT
+
+      TIMEOUT=60
+      while [[ ! -f /var/run/libvirtd.pid ]]; do
+        if [[ ${TIMEOUT} -gt 0 ]]; then
+          let TIMEOUT-=1
+          sleep 1
+        else
+          echo "ERROR: libvirt did not start in time (pid file missing)"
+          exit 1
+        fi
+      done
+
+      TIMEOUT=10
+      while [[ ! -e /var/run/libvirt/libvirt-sock ]]; do
+        if [[ ${TIMEOUT} -gt 0 ]]; then
+          let TIMEOUT-=1
+          sleep 1
+        else
+          echo "ERROR: libvirt did not start in time (socket missing)"
+          exit 1
+        fi
+      done
+
+      function create_virsh_libvirt_secret {
+        sec_user=$1
+        sec_uuid=$2
+        sec_ceph_keyring=$3
+        cat > ${tmpsecret} <<EOF
+    <secret ephemeral='no' private='no'>
+      <uuid>${sec_uuid}</uuid>
+      <usage type='ceph'>
+        <name>client.${sec_user}. secret</name>
+      </usage>
+    </secret>
+    EOF
+        virsh secret-define --file ${tmpsecret}
+        virsh secret-set-value --secret "${sec_uuid}" --base64 "${sec_ceph_keyring}"
+      }
+
+      if [ -z "${CEPH_CINDER_KEYRING}" ] ; then
+        CEPH_CINDER_KEYRING=$(awk '/key/{print $3}' /etc/ceph/ceph.client.${CEPH_CINDER_USER}.keyring)
+      fi
+      create_virsh_libvirt_secret ${CEPH_CINDER_USER} ${LIBVIRT_CEPH_CINDER_SECRET_UUID} ${CEPH_CINDER_KEYRING}
+
+      if [ -n "${LIBVIRT_EXTERNAL_CEPH_CINDER_SECRET_UUID}" ] ; then
+        EXTERNAL_CEPH_CINDER_KEYRING=$(cat /tmp/external-ceph-client-keyring)
+        create_virsh_libvirt_secret ${EXTERNAL_CEPH_CINDER_USER} ${LIBVIRT_EXTERNAL_CEPH_CINDER_SECRET_UUID} ${EXTERNAL_CEPH_CINDER_KEYRING}
+      fi
+
+      wait
+    else
+      exec cgexec -g ${CGROUPS%,}:/osh-libvirt systemd-run --scope --slice=system libvirtd
+    fi
--- a/config/libvirt-sushy/libvirt.yaml
+++ b/config/libvirt-sushy/libvirt.yaml
@ -0,0 +1,93 @@
+apiVersion: apps/v1
+kind: DaemonSet
+metadata:
+  name: libvirt
+spec:
+  selector:
+    matchLabels:
+      kubernetes.io/os: linux
+  template:
+    metadata:
+      labels:
+        kubernetes.io/os: linux
+    spec:
+      tolerations:
+      - key: node-role.kubernetes.io/master
+        effect: NoSchedule
+      hostNetwork: true
+      hostPID: true
+      hostIPC: true
+      containers:
+        - name: libvirt
+          command:
+            - /tmp/libvirt.sh
+          image: docker.io/openstackhelm/libvirt:ubuntu_xenial-20190903
+          securityContext:
+            privileged: true
+            runAsUser: 0
+            readOnlyRootFilesystem: false
+          volumeMounts:
+            - name: pod-tmp
+              mountPath: /tmp
+            - name: libvirt-bin
+              mountPath: /tmp/libvirt.sh
+              subPath: libvirt.sh
+              readOnly: true
+            - name: libvirt-etc
+              mountPath: /etc/libvirt/libvirtd.conf
+              subPath: libvirtd.conf
+            - name: libvirt-qemu
+              mountPath: /etc/libvirt/qemu.conf
+              subPath: qemu.conf
+            - mountPath: /lib/modules
+              name: libmodules
+              readOnly: true
+            - name: var-lib-libvirt
+              mountPath: /var/lib/libvirt
+              mountPropagation: Bidirectional
+            - name: var-lib-libvirt-images
+              mountPath: /var/lib/libvirt/images
+            - name: run
+              mountPath: /run
+            - name: dev
+              mountPath: /dev
+            - name: cgroup
+              mountPath: /sys/fs/cgroup
+            - name: logs
+              mountPath: /var/log/libvirt
+      volumes:
+        - name: pod-tmp
+          emptyDir: {}
+        - name: libvirt-bin
+          configMap:
+            name: libvirt-bin
+            defaultMode: 0555
+        - name: libvirt-etc
+          configMap:
+            name: libvirt-etc
+            defaultMode: 0444
+        - name: libvirt-qemu
+          configMap:
+            name: libvirt-qemu
+            defaultMode: 0444
+        - name: libmodules
+          hostPath:
+            path: /lib/modules
+        - name: var-lib-libvirt
+          hostPath:
+            path: /var/lib/libvirt
+        - name: var-lib-libvirt-images
+          hostPath:
+            path: /var/lib/libvirt/images
+        - name: run
+          hostPath:
+            path: /run
+        - name: dev
+          hostPath:
+            path: /dev
+        - name: logs
+          hostPath:
+            path: /var/log/libvirt
+        - name: cgroup
+          hostPath:
+            path: /sys/fs/cgroup
--- a/config/libvirt-sushy/libvirt_conf_configmap.yaml
+++ b/config/libvirt-sushy/libvirt_conf_configmap.yaml
@ -0,0 +1,10 @@
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: libvirt-etc
+data:
+  libvirtd.conf: |
+    unix_sock_group = "libvirtd"
+    unix_sock_ro_perms = "0777"
+    unix_sock_rw_perms = "0770"
+    log_outputs = "1:file:/var/log/libvirt/libvirtd.log"
--- a/config/libvirt-sushy/qemu_conf_configmap.yaml
+++ b/config/libvirt-sushy/qemu_conf_configmap.yaml
@ -0,0 +1,10 @@
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: libvirt-qemu
+data:
+  qemu.conf: |
+    stdio_handler = "file"
+    user = "libvirt-qemu"
+    group = "kvm"
+    security_driver = "none"