Libvirt daemonset for the VINO operator
Change-Id: I47273f6f8cf7325764d1786b87e032e9bd1aa046
This commit is contained in:
parent
feb1aaabb7
commit
d4686cebe5
148
config/libvirt-sushy/libvirt-bin-configmap.yaml
Normal file
148
config/libvirt-sushy/libvirt-bin-configmap.yaml
Normal file
@ -0,0 +1,148 @@
|
|||||||
|
apiVersion: v1
|
||||||
|
kind: ConfigMap
|
||||||
|
metadata:
|
||||||
|
name: libvirt-bin
|
||||||
|
data:
|
||||||
|
libvirt.sh: |
|
||||||
|
#!/bin/bash
|
||||||
|
set -ex
|
||||||
|
|
||||||
|
if [ -n "$(cat /proc/*/comm 2>/dev/null | grep -w libvirtd)" ]; then
|
||||||
|
set +x
|
||||||
|
for proc in $(ls /proc/*/comm 2>/dev/null); do
|
||||||
|
if [ "x$(cat $proc 2>/dev/null | grep -w libvirtd)" == "xlibvirtd" ]; then
|
||||||
|
set -x
|
||||||
|
libvirtpid=$(echo $proc | cut -f 3 -d '/')
|
||||||
|
echo "WARNING: libvirtd daemon already running on host" 1>&2
|
||||||
|
echo "$(cat "/proc/${libvirtpid}/status" 2>/dev/null | grep State)" 1>&2
|
||||||
|
kill -9 "$libvirtpid" || true
|
||||||
|
set +x
|
||||||
|
fi
|
||||||
|
done
|
||||||
|
set -x
|
||||||
|
fi
|
||||||
|
|
||||||
|
rm -f /var/run/libvirtd.pid
|
||||||
|
|
||||||
|
if [[ -c /dev/kvm ]]; then
|
||||||
|
chmod 660 /dev/kvm
|
||||||
|
chown root:kvm /dev/kvm
|
||||||
|
fi
|
||||||
|
|
||||||
|
CGROUPS=""
|
||||||
|
for CGROUP in cpu rdma hugetlb; do
|
||||||
|
if [ -d /sys/fs/cgroup/${CGROUP} ]; then
|
||||||
|
CGROUPS+="${CGROUP},"
|
||||||
|
fi
|
||||||
|
done
|
||||||
|
cgcreate -g ${CGROUPS%,}:/osh-libvirt
|
||||||
|
|
||||||
|
hp_count="$(cat /proc/meminfo | grep HugePages_Total | tr -cd '[:digit:]')"
|
||||||
|
if [ 0"$hp_count" -gt 0 ]; then
|
||||||
|
|
||||||
|
echo "INFO: Detected hugepage count of '$hp_count'. Enabling hugepage settings for libvirt/qemu."
|
||||||
|
|
||||||
|
if [ -n "$(grep KVM_HUGEPAGES=0 /etc/default/qemu-kvm)" ]; then
|
||||||
|
sed -i 's/.*KVM_HUGEPAGES=0.*/KVM_HUGEPAGES=1/g' /etc/default/qemu-kvm
|
||||||
|
else
|
||||||
|
echo KVM_HUGEPAGES=1 >> /etc/default/qemu-kvm
|
||||||
|
fi
|
||||||
|
|
||||||
|
if [ ! -d /dev/hugepages ]; then
|
||||||
|
echo "ERROR: Hugepages configured in kernel, but libvirtd container cannot access /dev/hugepages"
|
||||||
|
exit 1
|
||||||
|
fi
|
||||||
|
|
||||||
|
if [ -d /sys/fs/cgroup/hugetlb ]; then
|
||||||
|
limits="$(ls /sys/fs/cgroup/hugetlb/{{ .Values.conf.kubernetes.cgroup }}/hugetlb.*.limit_in_bytes)" || \
|
||||||
|
(echo "ERROR: Failed to locate any hugetable limits. Did you set the correct cgroup in your values used for this chart?"
|
||||||
|
exit 1)
|
||||||
|
for limit in $limits; do
|
||||||
|
target="/sys/fs/cgroup/hugetlb/$(dirname $(awk -F: '($2~/hugetlb/){print $3}' /proc/self/cgroup))/$(basename $limit)"
|
||||||
|
if [ ! -f "$target" ]; then
|
||||||
|
echo "ERROR: Could not find write target for hugepage limit: $target"
|
||||||
|
fi
|
||||||
|
|
||||||
|
echo "$(cat $limit)" > "$target"
|
||||||
|
done
|
||||||
|
fi
|
||||||
|
|
||||||
|
default_hp_kb="$(cat /proc/meminfo | grep Hugepagesize | tr -cd '[:digit:]')"
|
||||||
|
|
||||||
|
num_free_pages="$(cat /sys/kernel/mm/hugepages/hugepages-${default_hp_kb}kB/free_hugepages | tr -cd '[:digit:]')"
|
||||||
|
echo "INFO: '$num_free_pages' free hugepages of size ${default_hp_kb}kB"
|
||||||
|
if [ 0"$num_free_pages" -gt 0 ]; then
|
||||||
|
(fallocate -o0 -l "$default_hp_kb" /dev/hugepages/foo && rm /dev/hugepages/foo) || \
|
||||||
|
(echo "ERROR: fallocate failed test at /dev/hugepages with size ${default_hp_kb}kB"
|
||||||
|
rm /dev/hugepages/foo
|
||||||
|
exit 1)
|
||||||
|
fi
|
||||||
|
fi
|
||||||
|
|
||||||
|
if [ -n "${LIBVIRT_CEPH_CINDER_SECRET_UUID}" ] ; then
|
||||||
|
cgexec -g ${CGROUPS%,}:/osh-libvirt systemd-run --scope --slice=system libvirtd --listen &
|
||||||
|
|
||||||
|
tmpsecret=$(mktemp --suffix .xml)
|
||||||
|
if [ -n "${LIBVIRT_EXTERNAL_CEPH_CINDER_SECRET_UUID}" ] ; then
|
||||||
|
tmpsecret2=$(mktemp --suffix .xml)
|
||||||
|
fi
|
||||||
|
function cleanup {
|
||||||
|
rm -f "${tmpsecret}"
|
||||||
|
if [ -n "${LIBVIRT_EXTERNAL_CEPH_CINDER_SECRET_UUID}" ] ; then
|
||||||
|
rm -f "${tmpsecret2}"
|
||||||
|
fi
|
||||||
|
}
|
||||||
|
trap cleanup EXIT
|
||||||
|
|
||||||
|
TIMEOUT=60
|
||||||
|
while [[ ! -f /var/run/libvirtd.pid ]]; do
|
||||||
|
if [[ ${TIMEOUT} -gt 0 ]]; then
|
||||||
|
let TIMEOUT-=1
|
||||||
|
sleep 1
|
||||||
|
else
|
||||||
|
echo "ERROR: libvirt did not start in time (pid file missing)"
|
||||||
|
exit 1
|
||||||
|
fi
|
||||||
|
done
|
||||||
|
|
||||||
|
TIMEOUT=10
|
||||||
|
while [[ ! -e /var/run/libvirt/libvirt-sock ]]; do
|
||||||
|
if [[ ${TIMEOUT} -gt 0 ]]; then
|
||||||
|
let TIMEOUT-=1
|
||||||
|
sleep 1
|
||||||
|
else
|
||||||
|
echo "ERROR: libvirt did not start in time (socket missing)"
|
||||||
|
exit 1
|
||||||
|
fi
|
||||||
|
done
|
||||||
|
|
||||||
|
function create_virsh_libvirt_secret {
|
||||||
|
sec_user=$1
|
||||||
|
sec_uuid=$2
|
||||||
|
sec_ceph_keyring=$3
|
||||||
|
cat > ${tmpsecret} <<EOF
|
||||||
|
<secret ephemeral='no' private='no'>
|
||||||
|
<uuid>${sec_uuid}</uuid>
|
||||||
|
<usage type='ceph'>
|
||||||
|
<name>client.${sec_user}. secret</name>
|
||||||
|
</usage>
|
||||||
|
</secret>
|
||||||
|
EOF
|
||||||
|
virsh secret-define --file ${tmpsecret}
|
||||||
|
virsh secret-set-value --secret "${sec_uuid}" --base64 "${sec_ceph_keyring}"
|
||||||
|
}
|
||||||
|
|
||||||
|
if [ -z "${CEPH_CINDER_KEYRING}" ] ; then
|
||||||
|
CEPH_CINDER_KEYRING=$(awk '/key/{print $3}' /etc/ceph/ceph.client.${CEPH_CINDER_USER}.keyring)
|
||||||
|
fi
|
||||||
|
create_virsh_libvirt_secret ${CEPH_CINDER_USER} ${LIBVIRT_CEPH_CINDER_SECRET_UUID} ${CEPH_CINDER_KEYRING}
|
||||||
|
|
||||||
|
if [ -n "${LIBVIRT_EXTERNAL_CEPH_CINDER_SECRET_UUID}" ] ; then
|
||||||
|
EXTERNAL_CEPH_CINDER_KEYRING=$(cat /tmp/external-ceph-client-keyring)
|
||||||
|
create_virsh_libvirt_secret ${EXTERNAL_CEPH_CINDER_USER} ${LIBVIRT_EXTERNAL_CEPH_CINDER_SECRET_UUID} ${EXTERNAL_CEPH_CINDER_KEYRING}
|
||||||
|
fi
|
||||||
|
|
||||||
|
wait
|
||||||
|
else
|
||||||
|
exec cgexec -g ${CGROUPS%,}:/osh-libvirt systemd-run --scope --slice=system libvirtd
|
||||||
|
fi
|
93
config/libvirt-sushy/libvirt.yaml
Normal file
93
config/libvirt-sushy/libvirt.yaml
Normal file
@ -0,0 +1,93 @@
|
|||||||
|
apiVersion: apps/v1
|
||||||
|
kind: DaemonSet
|
||||||
|
metadata:
|
||||||
|
name: libvirt
|
||||||
|
spec:
|
||||||
|
selector:
|
||||||
|
matchLabels:
|
||||||
|
kubernetes.io/os: linux
|
||||||
|
template:
|
||||||
|
metadata:
|
||||||
|
labels:
|
||||||
|
kubernetes.io/os: linux
|
||||||
|
spec:
|
||||||
|
tolerations:
|
||||||
|
- key: node-role.kubernetes.io/master
|
||||||
|
effect: NoSchedule
|
||||||
|
hostNetwork: true
|
||||||
|
hostPID: true
|
||||||
|
hostIPC: true
|
||||||
|
containers:
|
||||||
|
- name: libvirt
|
||||||
|
command:
|
||||||
|
- /tmp/libvirt.sh
|
||||||
|
image: docker.io/openstackhelm/libvirt:ubuntu_xenial-20190903
|
||||||
|
securityContext:
|
||||||
|
privileged: true
|
||||||
|
runAsUser: 0
|
||||||
|
readOnlyRootFilesystem: false
|
||||||
|
volumeMounts:
|
||||||
|
- name: pod-tmp
|
||||||
|
mountPath: /tmp
|
||||||
|
- name: libvirt-bin
|
||||||
|
mountPath: /tmp/libvirt.sh
|
||||||
|
subPath: libvirt.sh
|
||||||
|
readOnly: true
|
||||||
|
- name: libvirt-etc
|
||||||
|
mountPath: /etc/libvirt/libvirtd.conf
|
||||||
|
subPath: libvirtd.conf
|
||||||
|
- name: libvirt-qemu
|
||||||
|
mountPath: /etc/libvirt/qemu.conf
|
||||||
|
subPath: qemu.conf
|
||||||
|
- mountPath: /lib/modules
|
||||||
|
name: libmodules
|
||||||
|
readOnly: true
|
||||||
|
- name: var-lib-libvirt
|
||||||
|
mountPath: /var/lib/libvirt
|
||||||
|
mountPropagation: Bidirectional
|
||||||
|
- name: var-lib-libvirt-images
|
||||||
|
mountPath: /var/lib/libvirt/images
|
||||||
|
- name: run
|
||||||
|
mountPath: /run
|
||||||
|
- name: dev
|
||||||
|
mountPath: /dev
|
||||||
|
- name: cgroup
|
||||||
|
mountPath: /sys/fs/cgroup
|
||||||
|
- name: logs
|
||||||
|
mountPath: /var/log/libvirt
|
||||||
|
volumes:
|
||||||
|
- name: pod-tmp
|
||||||
|
emptyDir: {}
|
||||||
|
- name: libvirt-bin
|
||||||
|
configMap:
|
||||||
|
name: libvirt-bin
|
||||||
|
defaultMode: 0555
|
||||||
|
- name: libvirt-etc
|
||||||
|
configMap:
|
||||||
|
name: libvirt-etc
|
||||||
|
defaultMode: 0444
|
||||||
|
- name: libvirt-qemu
|
||||||
|
configMap:
|
||||||
|
name: libvirt-qemu
|
||||||
|
defaultMode: 0444
|
||||||
|
- name: libmodules
|
||||||
|
hostPath:
|
||||||
|
path: /lib/modules
|
||||||
|
- name: var-lib-libvirt
|
||||||
|
hostPath:
|
||||||
|
path: /var/lib/libvirt
|
||||||
|
- name: var-lib-libvirt-images
|
||||||
|
hostPath:
|
||||||
|
path: /var/lib/libvirt/images
|
||||||
|
- name: run
|
||||||
|
hostPath:
|
||||||
|
path: /run
|
||||||
|
- name: dev
|
||||||
|
hostPath:
|
||||||
|
path: /dev
|
||||||
|
- name: logs
|
||||||
|
hostPath:
|
||||||
|
path: /var/log/libvirt
|
||||||
|
- name: cgroup
|
||||||
|
hostPath:
|
||||||
|
path: /sys/fs/cgroup
|
10
config/libvirt-sushy/libvirt_conf_configmap.yaml
Normal file
10
config/libvirt-sushy/libvirt_conf_configmap.yaml
Normal file
@ -0,0 +1,10 @@
|
|||||||
|
apiVersion: v1
|
||||||
|
kind: ConfigMap
|
||||||
|
metadata:
|
||||||
|
name: libvirt-etc
|
||||||
|
data:
|
||||||
|
libvirtd.conf: |
|
||||||
|
unix_sock_group = "libvirtd"
|
||||||
|
unix_sock_ro_perms = "0777"
|
||||||
|
unix_sock_rw_perms = "0770"
|
||||||
|
log_outputs = "1:file:/var/log/libvirt/libvirtd.log"
|
10
config/libvirt-sushy/qemu_conf_configmap.yaml
Normal file
10
config/libvirt-sushy/qemu_conf_configmap.yaml
Normal file
@ -0,0 +1,10 @@
|
|||||||
|
apiVersion: v1
|
||||||
|
kind: ConfigMap
|
||||||
|
metadata:
|
||||||
|
name: libvirt-qemu
|
||||||
|
data:
|
||||||
|
qemu.conf: |
|
||||||
|
stdio_handler = "file"
|
||||||
|
user = "libvirt-qemu"
|
||||||
|
group = "kvm"
|
||||||
|
security_driver = "none"
|
Loading…
Reference in New Issue
Block a user