apiVersion: v1
kind: ConfigMap
metadata:
  name: libvirt-bin
data:
  libvirt.sh: |
    #!/bin/bash
    set -ex

    if [ -n "$(cat /proc/*/comm 2>/dev/null | grep -w libvirtd)" ]; then
      set +x
      for proc in $(ls /proc/*/comm 2>/dev/null); do
        if [ "x$(cat $proc 2>/dev/null | grep -w libvirtd)" == "xlibvirtd" ]; then
          set -x
          libvirtpid=$(echo $proc | cut -f 3 -d '/')
          echo "WARNING: libvirtd daemon already running on host" 1>&2
          echo "$(cat "/proc/${libvirtpid}/status" 2>/dev/null | grep State)" 1>&2
          kill -9 "$libvirtpid" || true
          set +x
        fi
      done
      set -x
    fi

    rm -f /var/run/libvirtd.pid

    if [[ -c /dev/kvm ]]; then
        chmod 660 /dev/kvm
        chown root:kvm /dev/kvm
    fi

    CGROUPS=""
    for CGROUP in cpu rdma hugetlb; do
      if [ -d /sys/fs/cgroup/${CGROUP} ]; then
        CGROUPS+="${CGROUP},"
      fi
    done
    cgcreate -g ${CGROUPS%,}:/osh-libvirt

    hp_count="$(cat /proc/meminfo | grep HugePages_Total | tr -cd '[:digit:]')"
    if [ 0"$hp_count" -gt 0 ]; then

      echo "INFO: Detected hugepage count of '$hp_count'. Enabling hugepage settings for libvirt/qemu."

      if [ -n "$(grep KVM_HUGEPAGES=0 /etc/default/qemu-kvm)" ]; then
        sed -i 's/.*KVM_HUGEPAGES=0.*/KVM_HUGEPAGES=1/g' /etc/default/qemu-kvm
      else
        echo KVM_HUGEPAGES=1 >> /etc/default/qemu-kvm
      fi

      if [ ! -d /dev/hugepages ]; then
        echo "ERROR: Hugepages configured in kernel, but libvirtd container cannot access /dev/hugepages"
        exit 1
      fi

      if [ -d /sys/fs/cgroup/hugetlb ]; then
        limits="$(ls /sys/fs/cgroup/hugetlb/{{ .Values.conf.kubernetes.cgroup }}/hugetlb.*.limit_in_bytes)" || \
          (echo "ERROR: Failed to locate any hugetable limits. Did you set the correct cgroup in your values used for this chart?"
           exit 1)
        for limit in $limits; do
          target="/sys/fs/cgroup/hugetlb/$(dirname $(awk -F: '($2~/hugetlb/){print $3}' /proc/self/cgroup))/$(basename $limit)"
          if [ ! -f "$target" ]; then
            echo "ERROR: Could not find write target for hugepage limit: $target"
          fi

          echo "$(cat $limit)" > "$target"
        done
      fi

      default_hp_kb="$(cat /proc/meminfo | grep Hugepagesize | tr -cd '[:digit:]')"

      num_free_pages="$(cat /sys/kernel/mm/hugepages/hugepages-${default_hp_kb}kB/free_hugepages | tr -cd '[:digit:]')"
      echo "INFO: '$num_free_pages' free hugepages of size ${default_hp_kb}kB"
      if [ 0"$num_free_pages" -gt 0 ]; then
        (fallocate -o0 -l "$default_hp_kb" /dev/hugepages/foo && rm /dev/hugepages/foo) || \
          (echo "ERROR: fallocate failed test at /dev/hugepages with size ${default_hp_kb}kB"
           rm /dev/hugepages/foo
           exit 1)
      fi
    fi

    if [ -n "${LIBVIRT_CEPH_CINDER_SECRET_UUID}" ] ; then
      cgexec -g ${CGROUPS%,}:/osh-libvirt systemd-run --scope --slice=system libvirtd --listen &

      tmpsecret=$(mktemp --suffix .xml)
      if [ -n "${LIBVIRT_EXTERNAL_CEPH_CINDER_SECRET_UUID}" ] ; then
        tmpsecret2=$(mktemp --suffix .xml)
      fi
      function cleanup {
        rm -f "${tmpsecret}"
        if [ -n "${LIBVIRT_EXTERNAL_CEPH_CINDER_SECRET_UUID}" ] ; then
          rm -f "${tmpsecret2}"
        fi
      }
      trap cleanup EXIT

      TIMEOUT=60
      while [[ ! -f /var/run/libvirtd.pid ]]; do
        if [[ ${TIMEOUT} -gt 0 ]]; then
          let TIMEOUT-=1
          sleep 1
        else
          echo "ERROR: libvirt did not start in time (pid file missing)"
          exit 1
        fi
      done

      TIMEOUT=10
      while [[ ! -e /var/run/libvirt/libvirt-sock ]]; do
        if [[ ${TIMEOUT} -gt 0 ]]; then
          let TIMEOUT-=1
          sleep 1
        else
          echo "ERROR: libvirt did not start in time (socket missing)"
          exit 1
        fi
      done

      function create_virsh_libvirt_secret {
        sec_user=$1
        sec_uuid=$2
        sec_ceph_keyring=$3
        cat > ${tmpsecret} <<EOF
    <secret ephemeral='no' private='no'>
      <uuid>${sec_uuid}</uuid>
      <usage type='ceph'>
        <name>client.${sec_user}. secret</name>
      </usage>
    </secret>
    EOF
        virsh secret-define --file ${tmpsecret}
        virsh secret-set-value --secret "${sec_uuid}" --base64 "${sec_ceph_keyring}"
      }

      if [ -z "${CEPH_CINDER_KEYRING}" ] ; then
        CEPH_CINDER_KEYRING=$(awk '/key/{print $3}' /etc/ceph/ceph.client.${CEPH_CINDER_USER}.keyring)
      fi
      create_virsh_libvirt_secret ${CEPH_CINDER_USER} ${LIBVIRT_CEPH_CINDER_SECRET_UUID} ${CEPH_CINDER_KEYRING}

      if [ -n "${LIBVIRT_EXTERNAL_CEPH_CINDER_SECRET_UUID}" ] ; then
        EXTERNAL_CEPH_CINDER_KEYRING=$(cat /tmp/external-ceph-client-keyring)
        create_virsh_libvirt_secret ${EXTERNAL_CEPH_CINDER_USER} ${LIBVIRT_EXTERNAL_CEPH_CINDER_SECRET_UUID} ${EXTERNAL_CEPH_CINDER_KEYRING}
      fi

      wait
    else
      exec cgexec -g ${CGROUPS%,}:/osh-libvirt systemd-run --scope --slice=system libvirtd
    fi