openstack
/
fuel-library
Code Issues Proposed changes
fuel-library/files/fuel-migrate/fuel-migrate

545 lines
18 KiB
Bash
Executable File
Raw Blame History

#!/bin/bash
set -e
SSHO="-o PasswordAuthentication=no -o ConnectTimeout=5"
config_bundle="fvm_disk_size fvm_name fvm_ram fvm_cpu migrate_log fuel_astute dkvm_folder \
admin_net_br admin_net_b_type other_net_bridges os_swap os_root os_var os_varlibdocker \
os_varlog fm_reboot del_vm max_worktime help"
work_bundle="dkvm_ip admin_net_ip_bs dkvm_uri admin_net_ip admin_net_if admin_net_nm \
src_disk dst_disk s_base start_time fvm_admin_mac admin_net_mac nailgun_setting"
descr_fm_reboot="Would you like to automatically reboot the machine and begin data sync \
process? 'yes' or 'no'. Default is 'yes'"
descr_fvm_disk_size="Disk size for the destination VM. Default is '100g'"
descr_fvm_name="Name for the destination VM. Default is 'fuel_master'"
descr_fvm_ram="Amount of RAM (in MB) for the destination VM. Default is '2048'"
descr_fvm_cpu="Number of CPU cores for the destination VM. Default is '2'"
descr_migrate_log="Path for log file. Default is '/var/log/fuel-migrate.log'"
descr_fuel_astute="Location of Fuel configuration file. Default is '/etc/fuel/astute.yaml'"
descr_dkvm_folder="Path to folder on kvm host where disk will be created. \
Default is '/var/lib/nova/'"
descr_admin_net_br="Bridge which will be used for connecting VM admin interface. \
Default is 'br-fw-admin'"
descr_admin_net_b_type="Additional data to put inside network interface block definition. \
For example \"<virtualport type='openvswitch'/>\" By default empty"
descr_other_net_bridges="Comma separated set of other net interfaces which will be created.\
Contains 3*X parameters name_of_nic,bridge_on_kvm,additional_data. Where X is number \
of additional nics, name_of_nic looks like eth1, second two are similar to \
admin_net_br and admin_net_b_type"
descr_os_swap="Size of swap volume. By default 2*RAM size"
descr_os_root="Size of root volume. By default the same as source"
descr_os_var="Size for the var volume. By default 20% of the remaining free space. \
(Will be applied in order prompted here)"
descr_os_varlibdocker="Size of varlibdocker volume. By default 30% of the remaining free space."
descr_os_varlog="Size of varlog volume. By default 100% of the remaining free space."
descr_del_vm="Remove destination VM if it already exists? 'yes' or 'no'. Default is 'no'"
descr_max_worktime="Timeout for entire migration process in seconds. Default is 7200"
descr_help="Show this help message and exit"
indent="\t\t\t\t"
wtw="$(($(stty size | awk '{print $2}') - 32 ))"
lockfile_dir=/var/lock
fd=200
banner_file="c-msg.sh"
prog_name="fuel-migrate"
fuel_migrate_vars=/root/fuel-migrate.var
udev_rule=/etc/udev/rules.d/70-persistent-net.rules
file_lock() {
# create lock file
eval "exec $fd>$lockfile_dir/$prog_name.lock"
# acquire the lock
if flock -n $fd ; then
return 0
else
echo "Only one instance of $prog_name can run at a time."
return 1
fi
}
usage(){
[[ ! -z "$1" ]] && echo "$1"
echo -e "
Usage:
\t$prog_name DESTINATION_COMPUTE [ OPTIONS ]
\tDESTINATION_COMPUTE:\r${indent}Name or IP of compute node where VM will be created
\tOPTIONS:
$(var_describe $config_bundle)
"
exit 1
}
var_describe(){
local dn
until [[ -z "$1" ]] ; do
dn="descr_$1"
echo -n -e "\t--$1\r"
# no there ""
echo_indent ${!dn}
shift
done
}
# Formatted out with indent for current screen size
echo_indent(){
[[ -z "$1" ]] && echo ""
local cline=""
until [[ -z "$1" ]] ; do
if [[ "$(( ${#cline} + ${#1} + 1 ))" -gt "$wtw" ]] ; then
echo -e "${indent}${cline}"
cline=" $1"
else
cline="${cline} $1"
fi
shift
done
[[ ! -z "$cline" ]] && echo -e "${indent}${cline}"
}
# Permanent console output
cecho(){
echo "$@" >&6
}
mytrim(){
# no ""
echo $1
}
# Obtain parameter from yaml file
yaml_var(){
local fn="$1"
local py_var=""
shift
until [[ -z "$1" ]] ; do
py_var="$py_var['$1']"
shift
done
if [[ ! -z "py_cmd" ]] ; then
python -c "import yaml; a = yaml.load(open('$fn', 'r')); print a$py_var"
fi
}
# save set vars for usage later
save_vars(){
echo "# Saved at $(date) ">"$fuel_migrate_vars"
until [[ -z "$1" ]] ; do
[[ ! -z "${!1}" ]] && echo "$1=\"$(mytrim ${!1})\"" >>"$fuel_migrate_vars"
shift
done
}
# create network definition template
ifaces(){
until [[ -z "$1" ]] ; do
echo " <interface type='bridge'>
<source bridge='$2'/>
<target dev='vfm_$1'/>"
[[ ! -z "$3" ]] && echo "$3"
echo " <model type='virtio'/>
</interface>"
shift
shift
shift
done
}
# Create destination VM
create_vm(){
cecho "Create VM ${fvm_name} on ${dkvm_ip}"
# check is kvm present
virsh -c "${dkvm_uri}" list
# check is vm is present
vm_state="$(virsh -c "${dkvm_uri}" list --all | awk "/ ${fvm_name} / {print \$3}")"
if [[ "${del_vm}" == "yes" ]] ; then
# destroy vm if it exist
[[ "$vm_state" == "running" ]] && virsh -c "${dkvm_uri}" destroy "${fvm_name}"
[[ ! -z "$vm_state" ]] && virsh -c "${dkvm_uri}" undefine "${fvm_name}"
else
if [[ ! -z "$vm_state" ]] ; then
echo "${fvm_name} already exist on ${dkvm_ip} "
exit 1
fi
fi
# create disk
ssh "${dkvm_ip}" qemu-img create -f raw "${dkvm_folder}/${fvm_name}.img" "${fvm_disk_size}"
# make template for virsh
cat >/tmp/vm.xml <<EOF
<domain type='kvm'>
<name>${fvm_name}</name>
<memory unit='MiB'>${fvm_ram}</memory>
<vcpu >${fvm_cpu}</vcpu>
<os>
<type arch='x86_64' >hvm</type>
<boot dev='hd'/>
<boot dev='network'/>
</os>
<features>
<acpi/>
<apic/>
</features>
<clock offset='utc'/>
<on_poweroff>destroy</on_poweroff>
<on_reboot>restart</on_reboot>
<on_crash>restart</on_crash>
<devices>
<disk type='file' device='disk'>
<driver name='qemu' type='raw' cache='writeback'/>
<source file='${dkvm_folder}/${fvm_name}.img'/>
<target dev='vda' bus='virtio'/>
</disk>
<controller type='usb' index='0'>
</controller>
<controller type='ide' index='0'>
</controller>
<interface type='bridge'>
<source bridge='${admin_net_br}'/>
<target dev='vfm_${admin_net_if}'/>"
${admin_net_b_type}
<model type='virtio'/>
</interface>
EOF
IFS=","
ifaces ${other_net_bridges} >>/tmp/vm.xml
IFS=" "
cat >>/tmp/vm.xml <<EOF
<input type='mouse' bus='ps2'/>
<graphics type='vnc' autoport='yes' listen='0.0.0.0'/>
<video>
<model type='cirrus' />
</video>
<memballoon model='virtio'>
</memballoon>
</devices>
</domain>
EOF
virsh -c "${dkvm_uri}" define /tmp/vm.xml
virsh -c "${dkvm_uri}" autostart ${fvm_name}
virsh -c "${dkvm_uri}" start ${fvm_name}
fvm_admin_mac="$(virsh -c "${dkvm_uri}" domiflist "${fvm_name}" | awk "/vfm_${admin_net_if}/ { print \$5}")"
#wait until vm up
while [[ -z "${admin_net_ip_bs}" ]] ; do
timeout
sleep 10
admin_net_ip_bs="$(mytrim $(fuel node | awk -F'|' "/${fvm_admin_mac}/ {print \$5}"))"
done
}
# make cmd string for sgdisk. Size of last will be truncated (extended) to fit
# current disk size
clone_part_str(){
lastsec="$(( $3 * ${src_bs} / ${dst_bs}))"
[[ $1 == "$pcount" ]] && lastsec=""
echo " -n$1:$(( $2 * ${src_bs} / ${dst_bs})):${lastsec} -t $1:$6 -c $1:$7"
}
# partitioning source disk
mark_dst_disk(){
cecho "Create partition table"
# clear partition table
ssh "${admin_net_ip_bs}" sgdisk -o $dst_disk
src_bs="$(sgdisk -p $src_disk | awk '/Logical sector size: / {print $4}')"
dst_bs="$(ssh ${admin_net_ip_bs} sgdisk -p $dst_disk | awk '/Logical sector size: / {print $4}')"
pcount="$(sgdisk -p $src_disk | tail -n 1 | awk '{print $1}')"
tstr="-a 1 "
# get current config and create cmd line for sgdisk
sgdisk -p $src_disk | sed -n '/^ /p' >/tmp/fuel_migrate.tmp
while read part ; do
tstr="$tstr $(clone_part_str $part)"
export tstr
done </tmp/fuel_migrate.tmp
ssh "${admin_net_ip_bs}" "sgdisk $dst_disk $tstr"
}
mk_lvs_fs(){
# to-do get current fuel data
cecho "Create lvm volumes and file-systems"
ssh "${admin_net_ip_bs}" vgcreate os ${dst_disk}5
ssh "${admin_net_ip_bs}" lvcreate os -n root -L ${os_root}
ssh "${admin_net_ip_bs}" lvcreate os -n swap -L ${os_swap}
ssh "${admin_net_ip_bs}" lvcreate os -n var -l${os_var}free
ssh "${admin_net_ip_bs}" lvcreate os -n varlog -l${os_varlog}free
ssh "${admin_net_ip_bs}" "mkfs.ext4 /dev/os/root; mkfs.ext4 /dev/os/var; mkfs.ext4 /dev/os/varlog; mkswap /dev/os/swap"
}
rsync_vol(){
cecho "Start syncing $2"
ssh "${admin_net_ip_bs}" "mkdir -p ${s_base}${1};"
ssh "${admin_net_ip_bs}" "mount -t ext4 -o user_xattr,acl ${2} ${s_base}${1}"
# Why -X doesn't work ?!!
time rsync -z -axAHS --numeric-ids ${1}/ $admin_net_ip_bs:${s_base}${1}/
}
rsync_data(){
ddopt="bs=1048576"
# use dd to create small 1:1 partitions to avoid FS issues
dd if="${src_disk}1" ${ddopt} | ssh "${admin_net_ip_bs}" "dd of=${dst_disk}1 ${ddopt}"
dd if="${src_disk}2" ${ddopt} | ssh "${admin_net_ip_bs}" "dd of=${dst_disk}2 ${ddopt}"
dd if="${src_disk}3" ${ddopt} | ssh "${admin_net_ip_bs}" "dd of=${dst_disk}3 ${ddopt}"
dd if="${src_disk}4" ${ddopt} | ssh "${admin_net_ip_bs}" "dd of=${dst_disk}4 ${ddopt}"
rsync_vol / /dev/os/root
rsync_vol /var /dev/os/var
rsync_vol /var/log /dev/os/varlog
}
fix_config(){
cecho "Make grub-loader and fix config files"
# make grub loader
ssh "${admin_net_ip_bs}" "sed -i 's#${src_disk}#${dst_disk}#' ${s_base}/etc/mtab"
ssh "${admin_net_ip_bs}" "mount -o bind /dev/ ${s_base}/dev/"
ssh "${admin_net_ip_bs}" "mount -o bind /proc/ ${s_base}/proc/"
ssh "${admin_net_ip_bs}" "mount ${dst_disk}3 ${s_base}/boot/"
ssh "${admin_net_ip_bs}" "chroot ${s_base} grub2-install ${dst_disk}"
# fix configs
ssh "${admin_net_ip_bs}" "ln -fs /proc/self/mounts ${s_base}/etc/mtab"
ssh "${admin_net_ip_bs}" "sed -i '/HWADDR=/d' ${s_base}/etc/sysconfig/network-scripts/ifcfg-*"
ssh "${admin_net_ip_bs}" "sed -i 's/${admin_net_mac}/${fvm_admin_mac}/' ${s_base}${fuel_astute}"
ssh "${admin_net_ip_bs}" "sed -i 's/${admin_net_mac}/${fvm_admin_mac}/' ${s_base}${nailgun_setting}"
ssh "${admin_net_ip_bs}" "cat /root/.ssh/authorized_keys >>${s_base}/root/.ssh/authorized_keys"
ssh "${admin_net_ip_bs}" "umount ${s_base}/var/log ${s_base}/var ${s_base}/boot ${s_base}/dev ${s_base}/proc ${s_base}" || true
}
new_udev_line(){
echo 'SUBSYSTEM=="net", ACTION=="add", DRIVERS=="?*", ATTR{address}=="'${1}'", ATTR{type}=="1", KERNEL=="eth*", NAME="'${2}'"'
}
get_if_list(){
virsh -c "${dkvm_uri}" domiflist "${fvm_name}"
}
new_udev_rule(){
# make udev on destination vm use the same network interface name as on source
fvm_mac=$(get_if_list | awk -F " " "/vfm_${admin_net_if}/ { print \$5}")
echo "# Generated by fuel-migrate"
new_udev_line "${fvm_mac}" "${admin_net_if}"
until [[ -z $1 ]] ; do
vfm_mac=$(get_if_list | awk -F " " "/vfm_${1}/ {print \$5}")
new_udev_line "${vfm_mac}" ${1}
shift || true
shift || true
shift || true
done
}
wait_connection(){
te="$(( $(date +%s) + ${1:-300}))"
until ssh $SSHO "${2:-${admin_net_ip_bs}}" exit 0 ; do
sleep 2
[[ "${te}" -lt "$(date +%s)" ]] && return 1
done
return 0
}
set_log(){
# keep current console output on file descriptor &6 and redirect stderr and stdout into file
if [[ ! -z "$migrate_log" ]] ; then
exec 6>&1
exec 1>>$migrate_log
exec 2>&1
fi
set -x
}
timeout(){
if [[ "$(( $(date +%s) - ${start_time} ))" -ge "$max_worktime" ]] ; then
echo "Timeout !"
return 1
fi
return 0
}
# Stub for required umm functionality
umm(){
if [[ "$1" == "on" ]] ; then
/usr/bin/systemctl set-default umm-mg.target
reboot
fi
}
# Stop script execution if script is sourced
[[ "${BASH_SOURCE[0]}" != "${0}" ]] && return
# Show usage if run without parameters.
[[ -z $1 ]] && usage
# Show usage if help is chosen
[[ "$1" == "--help" ]] && usage
# Check if already running
file_lock
export LC_ALL=C
if [[ "$1" == "start" ]] && [[ -f "$fuel_migrate_vars" ]] ; then
source "$fuel_migrate_vars"
set_log
cecho "Start data sync phase"
mv -f "$fuel_migrate_vars" "${fuel_migrate_vars}.old"
wait_connection 600
wait_connection 60 "${dkvm_ip}"
time rsync_data
ssh "${admin_net_ip_bs}" "echo 'Not ready'>${s_base}/notready"
fvm_admin_mac=$(get_if_list | awk "/vfm_${admin_net_if}/ { print \$5}")
IFS=","
new_udev_rule ${other_net_bridges} | ssh "${admin_net_ip_bs}" "cat >${s_base}${udev_rule}"
IFS=" "
cat >/tmp/${banner_file} <<EOF
#!/bin/bash
echo "
Congratulation! You are on cloned Fuel now!
The migration tasks have completed. The clone should be up and functioning
correctly. You should now check the new system and inspect the log stored in
${migrate_log}.
After the reboot of the Fuel master, it may require additional time before
it becomes fully operational. Please allow at least 10 minutes for the system
to become active.
The source Fuel master is still up in maintenance mode. Once you have
verified that the new Fuel master is functioning correctly, you can shut down
or wipe the original Fuel master. The source Fuel master can be reached by
using ssh to ${admin_net_ip_bs}. Should you want to wipe the disk for bootstraping,
you may do so by running the following:
# ssh ${admin_net_ip_bs} dd if=/dev/zero of=/dev/null count=1024 bs=1024 conv=fdatasync
# ssh ${admin_net_ip_bs} reboot -f
To disable this message, run:
# rm /etc/profile.d/${banner_file}
If the new Fuel master is not functioning correctly, you may return the
source Fuel master back into an operational state by running:
# ssh ${admin_net_ip_bs} reboot ; ssh -tt ${dkvm_ip} virsh destroy ${fvm_name}
This command will reboot the source Fuel master and remove the new
Fuel master clone. To prevent IP conflict, this command should be as indicated.
If you are performing this command over ssh, your session may hang and you may
need to reestablish your ssh connection.
"
EOF
chmod +x /tmp/${banner_file}
rsync /tmp/${banner_file} ${admin_net_ip_bs}:${s_base}/etc/profile.d/${banner_file}
fix_config
cecho "Reboot destination VM"
ssh -tt "${admin_net_ip_bs}" "reboot"
cecho "Stop network and up with new settings"
service network stop
cecho "Source fuelmaster can be obtained by ssh ${admin_net_ip_bs}"
# remove restriction provided by https://review.openstack.org/#/c/255734/
/usr/bin/systemctl set-environment OPTIONS="-f /tmp/mg-ss"
/usr/bin/sed -i /ListenAddress/d /etc/ssh/sshd_config >/tmp/mg-ss
/usr/bin/systemctl restart sshd.service
ifconfig "$admin_net_if" "${admin_net_ip_bs}" netmask "$admin_net_nm"
until ssh -tt ${admin_net_ip} fuel node; do
timeout || reboot
sleep 20
done
# refresh log on destination node
rsync ${migrate_log} ${admin_net_ip}:${migrate_log}
ssh "${admin_net_ip}" "rm -f /notready; touch /tmp/migration-done"
while true ; do sleep 3600 ; done
else
dkvm_ip="$(ssh $SSHO $1 echo \$SSH_CONNECTION)"
dkvm_ip="$(echo $dkvm_ip | awk '{print $3}')"
shift
for i in "$@" ; do
if [[ "${i:0:2}" == "--" ]] ; then
vname="${i:2}"
vvalue="${vname#*=}"
vname="${vname%=*}"
[[ "$vvalue" == "$vname" ]] && vvalue=yes
if echo " $config_bundle " | grep -q " $vname " ; then
eval "${vname%=*}=${vvalue}"
else
usage "Invalid options $i"
fi
else
usage "Invalid options $i"
fi
done
migrate_log="${migrate_log:-/var/log/fuel-migrate.log}"
set_log
date
start_time=$(date +%s)
# trap unexpected exit
trap "{ set +x ; flock -u $fd ; cecho \"ERROR: look at ${migrate_log}\"; tail ${migrate_log} >&6; exit ; }" EXIT
rm -f /tmp/migration-done
max_worktime="${max_worktime:-7200}"
fuel_astute="${fuel_astute:-/etc/fuel/astute.yaml}"
nailgun_setting="${nailgun_setting:-/etc/nailgun/settings.yaml}"
dkvm_ip="${dkvm_ip:?Destination compute not set}"
dkvm_folder="${dkvm_folder:-/var/lib/nova/}"
fvm_disk_size="${fvm_disk_size:-100g}"
fvm_name="${fvm_name:-fuel_master}"
fvm_ram="${fvm_ram:-2048}"
fvm_cpu="${fvm_cpu:-2}"
dkvm_uri="qemu+ssh://root@${dkvm_ip}/system"
admin_net_ip="$(yaml_var $fuel_astute ADMIN_NETWORK ipaddress)"
admin_net_if="$(yaml_var $fuel_astute ADMIN_NETWORK interface)"
admin_net_nm="$(yaml_var $fuel_astute ADMIN_NETWORK netmask)"
admin_net_mac="$(yaml_var $fuel_astute ADMIN_NETWORK mac)"
admin_net_br="${admin_net_br:-br-fw-admin}"
s_base=/tmp/root
fm_reboot="${fm_reboot:-yes}"
src_disk="$(mount | awk '/\/boot / { gsub (substr($1,length($1)),"",$1); print $1 } \')"
dst_disk="${dst_disk:-/dev/vda}"
[[ -z "$os_swap" ]] && os_swap="$((${fvm_ram}*2))"
[[ -z "$os_root" ]] && os_root="$(lvdisplay --units g -C --noheadings -o lv_size /dev/os/root)"
os_var="${os_var:-20%}"
os_varlibdocker="${os_varlibdocker:-30%}"
os_varlog="${os_varlog:-100%}"
del_vm=${del_vm:-no}
create_vm
mark_dst_disk
mk_lvs_fs
save_vars $config_bundle $work_bundle
flock -u $fd
if [[ "$fm_reboot" == "yes" ]] ; then
cecho "Rebooting to begin the data sync process"
umm on -c $0 start
else
/usr/bin/systemctl set-default umm-mg.target
cecho " Destination VM has been created. Please perform reboot to begin data sync process."
fi
trap - EXIT
fi
View Git Blame Copy Permalink