treasuremap/tools/deployment/seaworthy-virt/airship_gate/lib/virsh.sh

664 lines
17 KiB
Bash

#!/bin/bash
img_base_declare() {
log Validating base image exists
if ! virsh vol-key --pool "${VIRSH_POOL}" --vol airship-gate-base.img > /dev/null; then
log Installing base image from "${BASE_IMAGE_URL}"
cd "${TEMP_DIR}"
curl -q -L -o base.img "${BASE_IMAGE_URL}"
{
virsh vol-create-as \
--pool "${VIRSH_POOL}" \
--name airship-gate-base.img \
--format qcow2 \
--capacity "${BASE_IMAGE_SIZE}" \
--prealloc-metadata
virsh vol-upload \
--vol airship-gate-base.img \
--file base.img \
--pool "${VIRSH_POOL}"
} &>> "${LOG_FILE}"
fi
}
netconfig_gen_mtu() {
MTU="$1"
set +e
IFS= read -r -d '' MTU_TMP <<'EOF'
mtu: ${MTU}
EOF
set -e
MTU="$MTU" envsubst <<< "$MTU_TMP" >> network-config
}
netconfig_gen_physical() {
IFACE_NAME="$1"
MTU="$2"
set +e
IFS= read -r -d '' PHYS_TMP <<'EOF'
- type: physical
name: ${IFACE_NAME}
EOF
set -e
IFACE_NAME="$IFACE_NAME" envsubst <<< "$PHYS_TMP" >> network-config
if [ ! -z "$MTU" ]
then
netconfig_gen_mtu "$MTU"
fi
}
netconfig_gen_subnet() {
IP_ADDR="$1"
IP_MASK="$2"
GW_ADDR="$3"
set +e
IFS= read -r -d '' SUBNET_TMP <<'EOF'
subnets:
- type: static
address: ${IP_ADDR}
netmask: ${IP_MASK}
EOF
IFS= read -r -d '' SUBNET_GW_TMP <<'EOF'
gateway: ${GW_ADDR}
EOF
set -e
IP_ADDR="$IP_ADDR" IP_MASK="$IP_MASK" envsubst <<< "$SUBNET_TMP" >> network-config
if [ ! -z "$GW_ADDR" ]
then
GW_ADDR="$GW_ADDR" envsubst <<< "$SUBNET_GW_TMP" >> network-config
fi
}
netconfig_gen_vlan() {
IFACE_NAME="$1"
VLAN_TAG="$2"
MTU="$3"
set +e
IFS= read -r -d '' VLAN_TMP <<'EOF'
- type: vlan
name: ${IFACE_NAME}.${VLAN_TAG}
vlan_link: ${IFACE_NAME}
vlan_id: ${VLAN_TAG}
EOF
set -e
IFACE_NAME="$IFACE_NAME" VLAN_TAG="$VLAN_TAG" envsubst <<< "$VLAN_TMP" >> network-config
if [ ! -z "$MTU" ]
then
netconfig_gen_mtu "$MTU"
fi
}
netconfig_gen_nameservers() {
NAMESERVERS="$1"
set +e
IFS= read -r -d '' NS_TMP <<'EOF'
- type: nameserver
address: [${DNS_SERVERS}]
EOF
set -e
DNS_SERVERS="$NAMESERVERS" envsubst <<< "$NS_TMP" >> network-config
}
netconfig_gen() {
NAME="$1"
IFS= cat << 'EOF' > network-config
version: 1
config:
EOF
# Generate physical interfaces
for iface in $(config_vm_iface_list "$NAME")
do
iface_network=$(config_vm_iface_network "$NAME" "$iface")
netconfig_gen_physical "$iface" "$(config_net_mtu "$iface_network")"
if [ "$(config_net_is_layer3 "$iface_network")" == "true" ]
then
iface_ip="$(config_vm_net_ip "$NAME" "$iface_network")"
netmask="$(cidr_to_netmask "$(config_net_cidr "$iface_network")")"
net_gw="$(config_net_gateway "$iface_network")"
netconfig_gen_subnet "$iface_ip" "$netmask" "$net_gw"
else
if [ ! -z "$(config_vm_iface_vlans "$NAME" "$iface")" ]
then
for vlan in $(config_vm_iface_vlans "$NAME" "$iface")
do
netconfig_gen_vlan "$iface" "$vlan" "$(config_net_mtu "$iface_network" "$vlan")"
if [ "$(config_net_is_layer3 "$iface_network" "$vlan")" == "true" ]
then
iface_ip="$(config_vm_net_ip "$NAME" "$iface_network" "$vlan")"
netmask="$(cidr_to_netmask "$(config_net_cidr "$iface_network" "$vlan")")"
net_gw="$(config_net_gateway "$iface_network" "$vlan")"
netconfig_gen_subnet "$iface_ip" "$netmask" "$net_gw"
fi
done
fi
fi
done
DNS_SERVERS=$(echo "$UPSTREAM_DNS" | tr ' ' ',')
netconfig_gen_nameservers "$DNS_SERVERS"
sed -i -e '/^$/d' network-config
}
iso_gen() {
NAME=${1}
ADDL_USERDATA="${2}"
disk_layout="$(config_vm_disk_layout "$NAME")"
vm_disks="$(config_disk_list "$disk_layout")"
if virsh vol-key --pool "${VIRSH_POOL}" --vol "cloud-init-${NAME}.iso" &> /dev/null; then
log Removing existing cloud-init ISO for "${NAME}"
virsh vol-delete \
--pool "${VIRSH_POOL}" \
--vol "cloud-init-${NAME}.iso" &>> "${LOG_FILE}"
fi
log "Creating cloud-init ISO for ${NAME}"
ISO_DIR=${TEMP_DIR}/iso/${NAME}
mkdir -p "${ISO_DIR}"
cd "${ISO_DIR}"
netconfig_gen "$NAME"
SSH_PUBLIC_KEY=$(ssh_load_pubkey)
export NAME
export SSH_PUBLIC_KEY
NTP_POOLS="$(join_array ',' "$NTP_POOLS")"
export NTP_POOLS
NTP_SERVERS="$(join_array ',' "$NTP_SERVERS")"
export NTP_SERVERS
envsubst < "${TEMPLATE_DIR}/user-data.sub" > user-data
fs_header="false"
for disk in $vm_disks
do
disk_format="$(config_disk_format "$disk_layout" "$disk")"
if [[ ! -z "$disk_format" ]]
then
if [[ "$fs_header" = "false" ]]
then
echo "fs_header:" >> user-data
fs_header="true"
fi
FS_TYPE="$(config_format_type "$disk_format")" DISK_DEVICE="$disk" envsubst < "${TEMPLATE_DIR}/disk-data.sub" >> user-data
fi
done
echo >> user-data
mount_header="false"
for disk in $vm_disks
do
disk_format="$(config_disk_format "$disk_layout" "$disk")"
if [[ ! -z "$disk_format" ]]
then
if [[ "$mount_header" = "false" ]]
then
echo "mounts:" >> user-data
mount_header="true"
fi
MOUNTPOINT="$(config_format_mount "$disk_format")" DISK_DEVICE="$disk" envsubst < "${TEMPLATE_DIR}/mount-data.sub" >> user-data
fi
done
echo >> user-data
if [[ ! -z "${ADDL_USERDATA}" ]]
then
echo -e "${ADDL_USERDATA}" >> user-data
fi
envsubst < "${TEMPLATE_DIR}/meta-data.sub" > meta-data
{
genisoimage \
-V cidata \
-input-charset utf-8 \
-joliet \
-rock \
-o cidata.iso \
meta-data \
network-config \
user-data
virsh vol-create-as \
--pool "${VIRSH_POOL}" \
--name "cloud-init-${NAME}.iso" \
--capacity "$(stat -c %s "${ISO_DIR}/cidata.iso")" \
--format raw
virsh vol-upload \
--pool "${VIRSH_POOL}" \
--vol "cloud-init-${NAME}.iso" \
--file "${ISO_DIR}/cidata.iso"
} &>> "${LOG_FILE}"
}
iso_path() {
NAME=${1}
echo "${TEMP_DIR}/iso/${NAME}/cidata.iso"
}
net_list() {
namekey="$1"
if [[ -z "$namekey" ]]
then
grepargs=("-v" '^$')
else
grepargs=("^${namekey}")
fi
virsh net-list --name | grep "${grepargs[@]}"
}
nets_clean() {
namekey=$(get_namekey)
for netname in $(net_list "$namekey")
do
log Destroying Airship gate "$netname"
for iface in $(ip -oneline l show type vlan | grep "$netname" | awk -F ' ' '{print $2}' | tr -d ':' | awk -F '@' '{print $1}')
do
# shellcheck disable=SC2024
sudo ip l del dev "$iface" &>> "$LOG_FILE"
done
virsh net-destroy "$netname" &>> "${LOG_FILE}"
virsh net-undefine "$netname" &>> "${LOG_FILE}"
done
}
net_create() {
netname="$1"
namekey=$(get_namekey)
virsh_netname="${namekey}"_"${netname}"
if [[ $(config_net_is_layer3 "$net") == "true" ]]; then
net_template="${TEMPLATE_DIR}/l3network-definition.sub"
NETNAME="${virsh_netname}" NETIP="$(config_net_selfip "$netname")" NETMASK="$(cidr_to_netmask "$(config_net_cidr "$netname")")" NETMAC="$(config_net_mac "$netname")" envsubst < "$net_template" > "${TEMP_DIR}/net-${netname}.xml"
else
net_template="${TEMPLATE_DIR}/l2network-definition.sub"
NETNAME="${virsh_netname}" envsubst < "$net_template" > "${TEMP_DIR}/net-${netname}.xml"
fi
log Creating network "${namekey}"_"${netname}"
virsh net-define "${TEMP_DIR}/net-${netname}.xml" &>> "${LOG_FILE}"
virsh net-start "${virsh_netname}"
virsh net-autostart "${virsh_netname}"
for vlan in $(config_net_vlan_list "$netname")
do
if [[ $(config_net_is_layer3 "$netname") == "true" ]]
then
iface_name="${virsh_netname}-${vlan}"
iface_mtu=$(confg_net_mtu "$netname" "$vlan")
if [[ -z "$iface_mtu" ]]
then
iface_mtu="1500"
fi
iface_mac="$(config_net_mac "$netname" "$vlan")"
sudo ip link add link "${virsh_netname}" name "${iface_name}" type vlan id "${vlan}" mtu $iface_mtu address "$iface_mac"
sudo ip addr add "$(config_net_selfip_cidr "$netname" "$vlan")" dev "${iface_name}"
sudo ip link set dev "${iface_name}" up
fi
done
}
nets_declare() {
nets_clean
for net in $(config_net_list); do
net_create "$net"
done
}
pool_declare() {
log Validating virsh pool setup
if ! virsh pool-uuid "${VIRSH_POOL}" &> /dev/null; then
log Creating pool "${VIRSH_POOL}"
virsh pool-define-as --name "${VIRSH_POOL}" --type dir --target "${VIRSH_POOL_PATH}" &>> "${LOG_FILE}"
virsh pool-start "${VIRSH_POOL}"
virsh pool-autostart "${VIRSH_POOL}"
fi
}
vm_clean() {
NAME=${1}
if virsh list --name | grep "${NAME}" &> /dev/null; then
virsh destroy "${NAME}" &>> "${LOG_FILE}"
fi
if virsh list --name --all | grep "${NAME}" &> /dev/null; then
log Removing VM "${NAME}"
virsh undefine --remove-all-storage --domain "${NAME}" &>> "${LOG_FILE}"
fi
}
vm_clean_all() {
log Removing all VMs
VM_NAMES=($(config_vm_names))
for NAME in ${VM_NAMES[*]}
do
vm_clean "${NAME}"
done
wait
}
# TODO(sh8121att) - Sort out how to ensure the proper NIC
# is used for PXE boot
vm_render_interface() {
vm="$1"
iface="$2"
namekey="$(get_namekey)"
mac="$(config_vm_iface_mac "$vm" "$iface")"
network="$(config_vm_iface_network "$vm" "$iface")"
network="${namekey}_${network}"
slot="$(config_vm_iface_slot "$vm" "$iface")"
port="$(config_vm_iface_port "$vm" "$iface")"
config_string="model=virtio,network=${network}"
if [[ ! -z "$mac" ]]
then
config_string="${config_string},mac=${mac}"
fi
if [[ ! -z "$slot" ]]
then
config_string="${config_string},address.type=pci,address.slot=${slot}"
if [[ ! -z "$port" ]]
then
config_string="${config_string},address.function=${port}"
fi
fi
echo -n "$config_string"
}
vm_create_interfaces() {
vm="$1"
network_opts=""
for interface in $(config_vm_iface_list "$vm")
do
nic_opts="$(vm_render_interface "$vm" "$interface")"
network_opts="$network_opts --network ${nic_opts}"
done
echo "$network_opts"
}
vm_create_vols(){
NAME="$1"
disk_layout="$(config_vm_disk_layout "$NAME")"
vm_disks="$(config_disk_list "$disk_layout")"
bs_disk="$(config_layout_bootstrap "$disk_layout")"
bs_vm="$(config_vm_bootstrap "${NAME}")"
vols=()
for disk in $vm_disks
do
io_prof=$(config_disk_ioprofile "${disk_layout}" "${disk}")
size=$(config_disk_size "${disk_layout}" "${disk}")
if [[ "$bs_vm" = "true" && "$bs_disk" = "$disk" ]]
then
vol_create_disk "$NAME" "$disk" "$size" "true"
else
vol_create_disk "$NAME" "$disk" "$size"
fi
if [[ "$io_prof" == "fast" ]]
then
DISK_OPTS="bus=virtio,cache=none,format=qcow2,io=native"
elif [[ "$io_prof" == "safe" ]]
then
DISK_OPTS="bus=virtio,cache=directsync,discard=unmap,format=qcow2,io=native"
else
DISK_OPTS="bus=virtio,format=qcow2"
fi
vol_cmd="--disk vol=${VIRSH_POOL}/airship-gate-${NAME}-${disk}.img,target=${disk},size=${size},${DISK_OPTS}"
vols+=($vol_cmd)
done
echo "${vols[@]}"
}
vol_create_disk() {
NAME=${1}
DISK=${2}
SIZE=${3}
BS=${4}
if virsh vol-list --pool "${VIRSH_POOL}" | grep "airship-gate-${NAME}-${DISK}.img" &> /dev/null; then
log Deleting previous volume "airship-gate-${NAME}-${DISK}.img"
virsh vol-delete --pool "${VIRSH_POOL}" "airship-gate-${NAME}-${DISK}.img" &>> "${LOG_FILE}"
fi
log Creating volume "${DISK}" for "${NAME}"
if [[ "$BS" == "true" ]]; then
virsh vol-create-as \
--pool "${VIRSH_POOL}" \
--name "airship-gate-${NAME}-${DISK}.img" \
--capacity "${SIZE}"G \
--format qcow2 \
--backing-vol 'airship-gate-base.img' \
--backing-vol-format qcow2 &>> "${LOG_FILE}"
else
virsh vol-create-as \
--pool "${VIRSH_POOL}" \
--name "airship-gate-${NAME}-${DISK}.img" \
--capacity "${SIZE}"G \
--format qcow2 &>> "${LOG_FILE}"
fi
}
vm_create() {
NAME=${1}
DISK_OPTS="$(vm_create_vols "${NAME}")"
NETWORK_OPTS="$(vm_create_interfaces "${NAME}")"
if [[ "$(config_vm_bootstrap "${NAME}")" == "true" ]]; then
iso_gen "${NAME}" "$(config_vm_userdata "${NAME}")"
wait
log Creating VM "${NAME}" and bootstrapping the boot drive
# shellcheck disable=SC2086
virt-install \
--name "${NAME}" \
--os-variant ubuntu16.04 \
--virt-type kvm \
--cpu "${VIRSH_CPU_OPTS}" \
--serial "file,path=${TEMP_DIR}/console/${NAME}.log" \
--graphics vnc,listen=0.0.0.0 \
--noautoconsole \
$NETWORK_OPTS \
--vcpus "$(config_vm_vcpus "${NAME}")" \
--memory "$(config_vm_memory "${NAME}")" \
--import \
$DISK_OPTS \
--disk "vol=${VIRSH_POOL}/cloud-init-${NAME}.iso,device=cdrom" &>> "${LOG_FILE}"
ssh_wait "${NAME}"
ssh_cmd "${NAME}" cloud-init status --wait
ssh_cmd "${NAME}" sync
else
log Creating VM "${NAME}"
# shellcheck disable=SC2086
virt-install \
--name "${NAME}" \
--os-variant ubuntu16.04 \
--virt-type kvm \
--cpu "${VIRSH_CPU_OPTS}" \
--graphics vnc,listen=0.0.0.0 \
--serial file,path="${TEMP_DIR}/console/${NAME}.log" \
--noautoconsole \
$NETWORK_OPTS \
--vcpus "$(config_vm_vcpus "${NAME}")" \
--memory "$(config_vm_memory "${NAME}")" \
--import \
$DISK_OPTS &>> "${LOG_FILE}"
fi
virsh autostart "${NAME}"
}
vm_create_validate() {
NAME=${1}
vm_create "${name}"
if [[ "$(config_vm_bootstrap "${name}")" == "true" ]]
then
vm_validate "${name}"
fi
}
vm_create_all() {
log Starting all VMs
VM_NAMES=($(config_vm_names))
for name in ${VM_NAMES[*]}
do
vm_create_validate "${name}" &
done
wait
}
vm_start() {
NAME=${1}
log Starting VM "${NAME}"
virsh start "${NAME}" &>> "${LOG_FILE}"
ssh_wait "${NAME}"
}
vm_stop() {
NAME=${1}
log Stopping VM "${NAME}"
virsh destroy "${NAME}" &>> "${LOG_FILE}"
}
vm_stop_non_genesis() {
log Stopping all non-genesis VMs in parallel
for NAME in $(config_non_genesis_vms); do
vm_stop "${NAME}" &
done
wait
}
vm_restart_all() {
for NAME in $(config_vm_names); do
vm_stop "${NAME}" &
done
wait
for NAME in $(config_vm_names); do
vm_start "${NAME}" &
done
wait
}
vm_validate() {
NAME=${1}
if ! virsh list --name | grep "${NAME}" &> /dev/null; then
log VM "${NAME}" did not start correctly.
exit 1
fi
}
#Find the correct group name for libvirt access
get_libvirt_group() {
grep -oE '^libvirtd?:' /etc/group | tr -d ':'
}
# Make a user 'virtmgr' if it does not exist and add it to the libvirt group
make_virtmgr_account() {
for libvirt_group in $(get_libvirt_group)
do
if ! grep -qE '^virtmgr:' /etc/passwd
then
sudo useradd -m -s /bin/sh -g "${libvirt_group}" virtmgr
else
sudo usermod -g "${libvirt_group}" virtmgr
fi
done
}
# Generate a new keypair
gen_libvirt_key() {
log Removing any existing virtmgr SSH keys
sudo rm -rf ~virtmgr/.ssh
sudo mkdir -p ~virtmgr/.ssh
if [[ "${GATE_SSH_KEY}" ]]; then
log "Using existing SSH keys for virtmgr"
sudo cp "${GATE_SSH_KEY}" ~virtmgr/.ssh/airship_gate
sudo cp "${GATE_SSH_KEY}.pub" ~virtmgr/.ssh/airship_gate.pub
else
log "Generating new SSH keypair for virtmgr"
#shellcheck disable=SC2024
sudo ssh-keygen -N '' -b 2048 -t rsa -f ~virtmgr/.ssh/airship_gate &>> "${LOG_FILE}"
fi
}
# Install private key into site definition
install_libvirt_key() {
PUB_KEY=$(sudo cat ~virtmgr/.ssh/airship_gate.pub)
export PUB_KEY
mkdir -p "${TEMP_DIR}/tmp"
envsubst < "${TEMPLATE_DIR}/authorized_keys.sub" > "${TEMP_DIR}/tmp/virtmgr.authorized_keys"
sudo cp "${TEMP_DIR}/tmp/virtmgr.authorized_keys" ~virtmgr/.ssh/authorized_keys
sudo chown -R virtmgr ~virtmgr/.ssh
sudo chmod 700 ~virtmgr/.ssh
sudo chmod 600 ~virtmgr/.ssh/authorized_keys
if [[ -n "${USE_EXISTING_SECRETS}" ]]; then
log "Using existing manifests for secrets"
return 0
fi
mkdir -p "${GATE_DEPOT}"
cat << EOF > "${GATE_DEPOT}/airship_drydock_kvm_ssh_key.yaml"
---
schema: deckhand/CertificateKey/v1
metadata:
schema: metadata/Document/v1
name: airship_drydock_kvm_ssh_key
layeringDefinition:
layer: site
abstract: false
storagePolicy: cleartext
data: |-
EOF
sudo cat ~virtmgr/.ssh/airship_gate | sed -e 's/^/ /' >> "${GATE_DEPOT}/airship_drydock_kvm_ssh_key.yaml"
}