x/timmy
Code Issues Proposed changes

Fix: UnicodeDecodeError, broken spec; add SPT

Browse Source 
- Only convert outputs/errors to unicode if printing required,
  use 'replace' error handling to avoid errors. Write original
  byte str to output files to avoid double conversion.
- Fix spec broken with 1.26.6.
- Add simplified-performance-testing config & rq files, scripts.
  Can be used after deployment to get a few benchmark values.
  SPT_parser.sh can be used to parse results.

Change-Id: I2aacfdbd9574ff737a6b88a29d87ae56abd03e46
This commit is contained in:
Dmitry Sutyagin 2017-02-23 15:00:24 -08:00
parent ffb5092290
commit c8727d1431
15 changed files with 840 additions and 26 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

9
specs/python-timmy.spec
View File

@ -4,7 +4,7 @@
%global pypi_name timmy
Name: python-%{pypi_name}
Version: 1.26.6
Version: 1.26.7
Release: 1%{?dist}~mos0
Summary: Log collector tool for OpenStack Fuel
@ -107,7 +107,12 @@ popd
%changelog
* Thu Jan 19 2016 Dmitry Sutyagin <dsutyagin@mirantis.com> - 1.26.6
* Thu Feb 23 2017 Dmitry Sutyagin <dsutyagin@mirantis.com> - 1.26.7
- Fix: UnicodeDecodeError, broken spec; add SPT
- Update documentation to match timmy 1.26.6 code
- Fix: CLI docs
* Thu Jan 19 2017 Dmitry Sutyagin <dsutyagin@mirantis.com> - 1.26.6
- Add yum.repos.d directory to snapshot
- Change: print only summary for analysis module if all nodes are ok
- Replace yaml.load() with yaml.safe_load()

2
timmy/env.py
View File

@ -16,7 +16,7 @@
# under the License.
project_name = 'timmy'
version = '1.26.6'
version = '1.26.7'
if __name__ == '__main__':
import sys

17
timmy/nodes.py
View File

@ -252,7 +252,7 @@ class Node(object):
c[cmd], errs, ok_codes)
try:
with open(dfile, 'w') as df:
df.write(outs.encode('utf-8'))
df.write(outs)
except IOError:
self.logger.error("can't write to file %s" %
dfile)
@ -260,7 +260,7 @@ class Node(object):
try:
with open(errf, 'w') as ef:
ef.write('exitcode: %s\n' % code)
ef.write(errs.encode('utf-8'))
ef.write(errs)
except IOError:
self.logger.error("can't write to file %s" %
errf)
@ -281,7 +281,7 @@ class Node(object):
errs, ok_codes)
try:
with open(param['output_path'], 'w') as df:
df.write(outs.encode('utf-8'))
df.write(outs)
except IOError:
self.logger.error("can't write to file %s" %
param['output_path'])
@ -289,14 +289,14 @@ class Node(object):
try:
with open(param['stderr_path'], 'w') as ef:
ef.write('exitcode: %s\n' % code)
ef.write(errs.encode('utf-8'))
ef.write(errs)
except IOError:
self.logger.error("can't write to file %s" %
param['stderr_path'])
return mapcmds, self.mapscr
def exec_simple_cmd(self, cmd, timeout=15, infile=None, outfile=None,
fake=False, ok_codes=None, input=None, decode=True):
fake=False, ok_codes=None, input=None):
self.logger.info('%s, exec: %s' % (self.repr, cmd))
if not fake:
outs, errs, code = tools.ssh_node(ip=self.ip,
@ -306,7 +306,6 @@ class Node(object):
timeout=timeout,
outputfile=outfile,
ok_codes=ok_codes,
decode=decode,
input=input,
prefix=self.prefix)
self.check_code(code, 'exec_simple_cmd', cmd, errs, ok_codes)
@ -515,9 +514,10 @@ class Node(object):
def check_code(self, code, func_name, cmd, err, ok_codes=None):
if code:
if not ok_codes or code not in ok_codes:
p_err = unicode(err, 'utf-8', 'replace').rstrip('\n')
self.logger.warning("%s: func: %s: "
"cmd: '%s' exited %d, error: %s" %
(self.repr, func_name, cmd, code, err))
(self.repr, func_name, cmd, code, p_err))
return False
return True
@ -907,8 +907,7 @@ class NodeManager(object):
'timeout': timeout,
'outfile': node.archivelogsfile,
'input': input,
'ok_codes': [0, 1],
'decode': False}
'ok_codes': [0, 1]}
run_items.append(tools.RunItem(target=node.exec_simple_cmd,
args=args))
tools.run_batch(run_items, self.logs_maxthreads)

27
timmy/tools.py
View File

@ -282,7 +282,7 @@ def mdir(directory):
sys.exit(110)
def launch_cmd(cmd, timeout, input=None, ok_codes=None, decode=True):
def launch_cmd(cmd, timeout, input=None, ok_codes=None):
def _timeout_terminate(pid):
try:
os.kill(pid, 15)
@ -303,26 +303,25 @@ def launch_cmd(cmd, timeout, input=None, ok_codes=None, decode=True):
timeout_killer = threading.Timer(timeout, _timeout_terminate, [p.pid])
timeout_killer.start()
outs, errs = p.communicate(input=input)
errs = errs.rstrip('\n')
if decode:
outs = outs.decode('utf-8')
errs = errs.decode('utf-8')
finally:
if timeout_killer:
timeout_killer.cancel()
input = input.decode('utf-8') if input else None
logger.debug(('___command: %s\n'
'_______pid: %s\n'
'_exit_code: %s\n'
'_____stdin: %s\n'
'____stderr: %s') % (cmd, p.pid, p.returncode, input,
errs))
if logger.isEnabledFor(logging.DEBUG):
# p_out = unicode(outs, 'utf-8', 'replace')
p_err = unicode(errs, 'utf-8', 'replace').rstrip('\n')
p_inp = unicode(input, 'utf-8', 'replace') if input else None
logger.debug(('___command: %s\n'
'_______pid: %s\n'
'_exit_code: %s\n'
'_____stdin: %s\n'
'____stderr: %s') % (cmd, p.pid, p.returncode, p_inp,
p_err))
return outs, errs, p.returncode
def ssh_node(ip, command='', ssh_opts=None, env_vars=None, timeout=15,
filename=None, inputfile=None, outputfile=None,
ok_codes=None, input=None, prefix=None, decode=True):
ok_codes=None, input=None, prefix=None):
if not ssh_opts:
ssh_opts = ''
if not env_vars:
@ -354,7 +353,7 @@ def ssh_node(ip, command='', ssh_opts=None, env_vars=None, timeout=15,
"trap 'kill $pid' 2; echo -n \"$input\" | xxd -r -p | " + cmd +
' &:; pid=$!; wait $!')
return launch_cmd(cmd, timeout, input=input,
ok_codes=ok_codes, decode=decode)
ok_codes=ok_codes)
def get_files_rsync(ip, data, ssh_opts, rsync_opts, dpath, timeout=15):

13
timmy_data/simplified-performance-testing/SPT.yaml Normal file
View File

@ -0,0 +1,13 @@
scripts:
once_by_roles:
controller:
- 'glance-1-image-create.sh'
- 'glance-2-image-download.sh'
- {'cinder-VM-volume-write.sh': 'SPT_FLOATING_NET="admin_floating_net" SPT_FLAVOR="m1.small" SPT_IMAGE="xenial" SPT_VM_USER="ubuntu" DD_OPTIONS="oflag=direct" SPT_VM_COOLDOWN="120" DD_TIMEOUT="10" VM_BOOT_TIMEOUT_MINUTES="5"'}
- {'network-VM-to-VM-iperf-tests.sh': 'SPT_FLOATING_NET="admin_floating_net" SPT_FLAVOR="m1.small" SPT_IMAGE="xenial" SPT_VM_USER="ubuntu" SPT_VM_COOLDOWN="120" VM_BOOT_TIMEOUT_MINUTES="5" SPT_AVAILABILITY_ZONE="nova"'}
scripts_all_pairs:
__default:
- server_start: {'iperf-server-start.sh': 'SPT_IPERF_PORT=65432'}
server_stop: 'iperf-server-stop.sh'
client: {'iperf-client.sh': 'SPT_IPERF_PORT=65432'}
network: 'management'

154
timmy_data/simplified-performance-testing/SPT_parser.sh Executable file
View File

@ -0,0 +1,154 @@
#!/bin/bash
results=${1:-"/tmp/timmy/info"}
glance_create="glance-1-image-create.sh"
glance_download="glance-2-image-download.sh"
cinder_volume="cinder-VM-volume-write.sh"
iperf_vm="network-VM-to-VM-iperf-tests.sh"
iperf_host="iperf-client.sh"
res_glance_create=`find $results -name $glance_create`
res_glance_download=`find $results -name $glance_download`
res_cinder=`find $results -name $cinder_volume`
res_iperf_vm=`find $results -name $iperf_vm`
res_iperf_host="$(find $results -name "${iperf_host}*")"
res_iperf_node_dir="$(find $results -name "client" -type d)"
[ -n "$res_iperf_node_dir" ] && res_iperf_node="$(ls $res_iperf_node_dir)" || res_iperf_node=""
function print_result() {
[ -n "$2" ] && printf "$2" || printf "n/a"
echo -e "\t<-- $1"
}
function a() {
[ -n "$1" ] && printf "$1\t" || printf "n/a\t"
}
function b() {
word=`echo $1 | grep -o '[a-zA-Z/]\+'`
number=`echo $1 | grep -o '[0-9.]\+'`
python -c '
import sys
word = sys.argv[1]
number = sys.argv[2]
if word == "Gbits/sec":
sys.stdout.write(str(int(float(number)*1000000000)))
elif word == "Mbits/sec":
sys.stdout.write(str(int(float(number)*1000000)))
elif word == "Kbits/sec":
sys.stdout.write(str(int(float(number)*1000)))
elif word == "bits/sec":
sys.stdout.write(str(int(number)))
' $word $number 2> /dev/null
}
function c() {
echo $1 | python -c 'import sys; print(float(sys.stdin.read())/1000000000)' 2> /dev/null
}
function d() {
word=`echo $1 | grep -o '[a-zA-Z/]\+'`
number=`echo $1 | grep -o '[0-9.]\+'`
python -c '
import sys
word = sys.argv[1]
number = sys.argv[2]
if word == "GB/s":
sys.stdout.write(str(int(float(number)*1000)))
elif word == "MB/s":
sys.stdout.write(str(float(number)))
elif word == "kB/s":
sys.stdout.write(str(int(float(number)/1000)))
' $word $number 2> /dev/null
}
if [ -n "$res_glance_create" ]
then
res_gc=`head -n 4 $res_glance_create | grep '^[0-9.]\+$' | python -c 'import sys; print("%.2f" % (4000/float(sys.stdin.read(),)))'`
res_gc=${res_gc}'MB/s'
fi
print_result "Glance upload" $res_gc
if [ -n "$res_glance_download" ]
then
res_gd=`head -n 1 $res_glance_download | grep '^[0-9.]\+$' | python -c 'import sys; print("%.2f" % (4000/float(sys.stdin.read(),)))'`
res_gd=${res_gd}'MB/s'
fi
print_result "Glance download" $res_gd
if [ -n "$res_cinder" ]
then
res_4k=`grep 'DD_TEST_1' $res_cinder | grep copied | rev | awk '{print $1$2}' | rev`
res_1m=`grep 'DD_TEST_2' $res_cinder | grep copied | rev | awk '{print $1$2}' | rev`
res_1g=`grep 'DD_TEST_3' $res_cinder | grep copied | rev | awk '{print $1$2}' | rev`
fi
print_result "Block Storage Write 4k" $res_4k
print_result "Block Storage Write 1M" $res_1m
print_result "Block Storage Write 1G" $res_1g
if [ -n "$res_iperf_host" ]
then
res_t1=""
res_t10=""
for i in $res_iperf_host
do
res_t1="$(echo "$res_t1"; b $(head -n 7 $i | tail -n 1 | grep '^\[.*sec' | rev | awk '{print $1$2}' | rev | sed 's/ //g'))"
res_t1="$(echo "$res_t1" | grep -v '^$' | sort -n | tail -n 1)"
res_t10="$(echo "$res_t10"; b $(head -n 33 $i | tail -n 1 | grep '^\[SUM.*sec' | rev | awk '{print $1$2}' | rev | sed 's/ //g'))"
res_t10="$(echo "$res_t10" | grep -v '^$' | sort -n | tail -n 1)"
done
fi
res_t1=`echo "$res_t1" | python -c 'import sys; print(float(sys.stdin.read())/1000000000)' 2> /dev/null`
res_t10=`echo "$res_t10" | python -c 'import sys; print(float(sys.stdin.read())/1000000000)' 2> /dev/null`
print_result "HW to HW (best)" $res_t1
print_result "HW to HW (best) - 10 Threads" $res_t10
if [ -n "$res_iperf_vm" ]
then
res1=`head -n 8 $res_iperf_vm | tail -n 1 | grep '^\[.*sec' | rev | awk '{print $1$2}' | rev | sed 's/ //g'`
res2=`head -n 17 $res_iperf_vm | tail -n 1 | grep '^\[.*sec' | rev | awk '{print $1$2}' | rev | sed 's/ //g'`
res3=`head -n 44 $res_iperf_vm | tail -n 1 | grep '^\[SUM.*sec' | rev | awk '{print $1$2}' | rev | sed 's/ //g'`
res4=`head -n 52 $res_iperf_vm | tail -n 1 | grep '^\[.*sec' | rev | awk '{print $1$2}' | rev | sed 's/ //g'`
res5=`head -n 61 $res_iperf_vm | tail -n 1 | grep '^\[.*sec' | rev | awk '{print $1$2}' | rev | sed 's/ //g'`
fi
print_result "VM to VM - VMs on same node - via Private IP - 1 thread" $res1
print_result "VM to VM - VMs on different HW nodes - via Private IP - 1 thread" $res2
print_result "VM to VM - VMs on different HW nodes - via Private IP - MILTI 10 thread" $res3
print_result "VM to VM - via Floating IP and VMs are on different nodes - 1 thread" $res4
print_result "VM to VM - diff nodes, VMs connected to separate networks connected by vRouter - via Private IP - 1 thread" $res5
[ -n "$res_iperf_node" ] && nnum="$(wc -l <<< "$res_iperf_node")" && print_result "Number of nodes (during iperf HW to HW tests)" $nnum
if [ -n "$res_iperf_host" ]
then
res_mt1=""
res_mt10=""
for i in $res_iperf_host
do
res_mt1="$(echo "$res_mt1"; b $(head -n 7 $i | tail -n 1 | grep '^\[.*sec' | rev | awk '{print $1$2}' | rev | sed 's/ //g'))"
res_mt1="$(echo "$res_mt1" | grep -v '^$' | sort -n | head -n 1)"
res_mt10="$(echo "$res_mt10"; b $(head -n 33 $i | tail -n 1 | grep '^\[SUM.*sec' | rev | awk '{print $1$2}' | rev | sed 's/ //g'))"
res_mt10="$(echo "$res_mt10" | grep -v '^$' | sort -n | head -n 1)"
done
fi
res_mt1=`echo "$res_mt1" | python -c 'import sys; print(float(sys.stdin.read())/1000000000)' 2> /dev/null`
res_mt10=`echo "$res_mt10" | python -c 'import sys; print(float(sys.stdin.read())/1000000000)' 2> /dev/null`
print_result "HW to HW (worst)" $res_mt1
print_result "HW to HW (worst) - 10 Threads" $res_mt10
echo "--------------------------"
a `d $res_gc`
a `d $res_gd`
a `d $res_4k`
a `d $res_1m`
a `d $res_1g`
a $res_t1 # HW to HW
a $res_t10 # HW to HW
a $(c `b $res1`)
a $(c `b $res2`)
a $(c `b $res3`)
a $(c `b $res4`)
a $(c `b $res5`)
a $nnum
a $res_mt1 # HW to HW max concurrency
a $res_mt10 # HW to HW max concurrency
echo

7
timmy_data/simplified-performance-testing/config.yaml Normal file
View File

@ -0,0 +1,7 @@
rqfile:
- file: simplified-performance-testing/SPT.yaml
default: false
rqdir: 'simplified-performance-testing'
timeout: 3600
soft_filter:
no_roles: ['fuel']

2
timmy_data/simplified-performance-testing/readme.txt Normal file
View File

@ -0,0 +1,2 @@
Copy or symlink this folder without renaming to your current working directory, and execute the bundle like so:
# timmy -c simplified-performance-testing/config.yaml

197
timmy_data/simplified-performance-testing/scripts/cinder-VM-volume-write.sh Normal file
View File

@ -0,0 +1,197 @@
#!/bin/bash
set -x
# this test requires fully functional dd in VM, cirros (busybox) version will not work
# this test requires sudo and pkill in VM
SPT_FLOATING_NET=${SPT_FLOATING_NET:-"admin_floating_net"}
SPT_FLAVOR=${SPT_FLAVOR:-"m1.small"}
SPT_IMAGE=${SPT_IMAGE:-"xenial"}
SPT_VM_USER=${SPT_VM_USER:-"ubuntu"}
SPT_VM_COOLDOWN=${SPT_VM_COOLDOWN:-"120"}
DD_OPTIONS=${DD_OPTIONS:-"oflag=direct"}
DD_TIMEOUT=${DD_TIMEOUT:-"10"}
SSH_OPTS="-q -n -oStrictHostKeyChecking=no -oUserKnownHostsFile=/dev/null -l $SPT_VM_USER"
VM_BOOT_TIMEOUT_MINUTES=${VM_BOOT_TIMEOUT_MINUTES:-"5"}
vm_boot_check_tries=$(($VM_BOOT_TIMEOUT_MINUTES*2))
vm_boot_check_delay=30
vm_active_check_tries=10
vm_active_check_delay=10
vm_volume_available_check_tries=10
vm_volume_available_check_delay=30
vm_volume_attached_check_tries=10
vm_volume_attached_check_delay=10
function cleanup {
echo "cleaning up..."
[ -n "$instance_id" ] && [ -n "$floatingip" ] && nova floating-ip-disassociate $instance_id $floatingip
[ -n "$floatingip_id" ] && neutron floatingip-delete $floatingip_id
[ -n "$instance_id" ] && [ -n "$secgroup_id" ] && nova remove-secgroup $instance_id $secgroup_id
[ -n "$instance_id" ] && nova delete $instance_id
[ -n "$secgroup_id" ] && neutron security-group-delete $secgroup_id
[ -n "$router_id" ] && [ -n "$subnet_id" ] && neutron router-interface-delete $router_id $subnet_id
[ -n "$router_id" ] && neutron router-delete $router_id
[ -n "$subnet_id" ] && neutron subnet-delete $subnet_id
[ -n "$net_id" ] && neutron net-delete $net_id
[ -n "$keypair_name" ] && nova keypair-delete $keypair_name
[ -f "spt-temporary-keypair" ] && rm "spt-temporary-keypair"
[ -n "$volume_id" ] && cinder delete $volume_id
}
function check_code {
# arguments:
# $1 - exit code
# $2 - error message
# $3 - command output to print
if [ "$1" -ne "0" ]
then
echo "$2:"
echo "$3"
cleanup
exit 1
fi
}
. openrc
# create keypair
result="$(nova keypair-add "spt-temporary-keypair" >"spt-temporary-keypair" 2>&1)"
code=$?
[ "$code" -eq "0" ] && keypair_name="spt-temporary-keypair"
check_code $? "failed to create keypair" "$result"
chmod 600 "spt-temporary-keypair"
# create network
result="$(neutron net-create "spt-temporary-net" 2>&1)"
check_code $? "failed to create network" "$result"
net_id=$(echo "$result" | grep "^| id " | awk '{print $4}')
# create subnet
result="$(neutron subnet-create --name "spt-temporary-subnet" $net_id 10.20.30.0/24 2>&1)"
check_code $? "failed to create subnet" "$result"
subnet_id=$(echo "$result" | grep "^| id " | awk '{print $4}')
# create router
result="$(neutron router-create "spt-temporary-router" 2>&1)"
check_code $? "failed to create router" "$result"
router_id=$(echo "$result" | grep "^| id " | awk '{print $4}')
# add router interface
result="$(neutron router-interface-add $router_id $subnet_id 2>&1)"
check_code $? "failed to add router interface to subnet" "$result"
# set router gateway
result="$(neutron router-gateway-set $router_id $SPT_FLOATING_NET 2>&1)"
check_code $? "failed to set router gateway" "$result"
# create security group
result="$(neutron security-group-create "spt-temporary-security-group" 2>&1)"
check_code $? "failed to create security group" "$result"
secgroup_id=$(echo "$result" | grep "^| id " | awk '{print $4}')
# create security group rule
result="$(neutron security-group-rule-create $secgroup_id --protocol TCP 2>&1)"
check_code $? "failed to create security group rule" "$result"
# create floating ip
result="$(neutron floatingip-create $SPT_FLOATING_NET 2>&1)"
check_code $? "failed to create floatingip" "$result"
floatingip=$(echo "$result" | grep "^| floating_ip_address " | awk '{print $4}')
floatingip_id=$(echo "$result" | grep "^| id " | awk '{print $4}')
# boot VM
result="$(nova boot --image $SPT_IMAGE --flavor $SPT_FLAVOR --nic net-id=$net_id --key-name "spt-temporary-keypair" --security-groups $secgroup_id "spt-temporary-vm" 2>&1)"
check_code $? "failed to boot VM" "$result"
instance_id=$(echo "$result" | grep "^| id " | awk '{print $4}')
# wait for instance to become active
for i in $(seq 1 $vm_active_check_tries)
do
result="$(nova show $instance_id 2>&1)"
vm_status=$(echo "$result" | grep "^| *status" | awk '{printf $4}')
[ "$vm_status" == "ACTIVE" ] && break
[ $i -lt $vm_active_check_tries ] && sleep $vm_active_check_delay
done
! [ "$vm_status" == "ACTIVE" ] && check_code 1 "timeout waiting for VM to become active" "$result"
# associate floatingip
result="$(nova floating-ip-associate $instance_id $floatingip 2>&1)"
check_code $? "failed to associate floatingip" "$result"
# create volume
result="$(cinder create --name "spt-temporary-volume" 4 2>&1)"
check_code $? "failed to create volume" "$result"
volume_id=$(echo "$result" | grep "^| *id " | awk '{print $4}')
# wait for volume to become available
for i in $(seq 1 $vm_volume_available_check_tries)
do
result="$(cinder show $volume_id 2>&1)"
volume_status=$(echo "$result" | grep "^| *status" | awk '{printf $4}')
[ "$volume_status" == "available" ] && break
[ $i -lt $vm_volume_available_check_tries ] && sleep $vm_volume_available_check_delay
done
! [ "$volume_status" == "available" ] && check_code 1 "timeout waiting for volume to become available" "$result"
# test connection to VM
for i in $(seq 1 $vm_boot_check_tries)
do
result="$(ssh $SSH_OPTS -i spt-temporary-keypair $floatingip "uptime" 2>&1)"
code=$?
[ $code -eq 0 ] && break
[ $i -lt $vm_boot_check_tries ] && sleep $vm_boot_check_delay
done
check_code $code "failed to connect to VM" "$result"
# due to unreliable cinder attachment procedure, first get the list of current drives
vm_drives="$(ssh $SSH_OPTS -i spt-temporary-keypair $floatingip "find /dev/vd*" 2>&1)"
check_code $? "failed to find /dev/vd* in VM" "$result"
# attach volume
result="$(nova volume-attach $instance_id $volume_id 2>&1)"
check_code $? "failed to attach volume" "$result"
# wait for volume to appear in VM
for i in $(seq 1 $vm_volume_attached_check_tries)
do
vm_drives_2="$(ssh $SSH_OPTS -i spt-temporary-keypair $floatingip "find /dev/vd*" 2>&1)"
new_drive="$((echo "$vm_drives"; echo "$vm_drives_2") | sort | uniq -u)"
[ -n "$new_drive" ] && break
if [ $i -lt $vm_volume_attached_check_tries ]
then
nova volume-attach $instance_id $volume_id &> /dev/null # retry attaching since it is known to fail silently sometimes
sleep $vm_volume_attached_check_delay
fi
done
! [ -n "$new_drive" ] && check_code 1 "timeout waiting for volume to appear in VM" "$vm_drives_2"
# mkfs in VM
result="$(ssh $SSH_OPTS -i spt-temporary-keypair $floatingip "sudo -i mkfs.ext4 -m 0 $new_drive" 2>&1)"
check_code $? "failed to create ext4 filesystem on VM" "$result"
# mount volume in VM
result="$(ssh $SSH_OPTS -i spt-temporary-keypair $floatingip "sudo -i mount $new_drive /mnt && sudo -i chown $SPT_VM_USER /mnt" 2>&1)"
check_code $? "failed to mount ext4 filesystem on VM" "$result"
# cooldown to allow VM to finish init activities
sleep $SPT_VM_COOLDOWN
# give 10 seconds to dd to run, then getting intermediate stats, waiting 1 second and killing. Need to wait before killing, otherwise intermediate stats may not be printed
timelimit_cmd="pid=\$!; sleep $DD_TIMEOUT; kill -USR1 \$pid; sleep 1; kill \$pid"
# run dd write tests
echo "running dd write test - writing 1GB with bs=4k"
result="$(ssh $SSH_OPTS -i spt-temporary-keypair $floatingip "dd if=/dev/zero of=/mnt/spt-write-test $DD_OPTIONS bs=4k count=262144 &:; $timelimit_cmd" 2>&1)"
echo "$result" | xargs -I@ echo "DD_TEST_1: @"
echo "running dd write test - writing 1GB with bs=1M"
result="$(ssh $SSH_OPTS -i spt-temporary-keypair $floatingip "dd if=/dev/zero of=/mnt/spt-write-test-2 $DD_OPTIONS bs=1M count=1024 &:; $timelimit_cmd" 2>&1)"
echo "$result" | xargs -I@ echo "DD_TEST_2: @"
echo "running dd write test - writing 1GB with bs=1G"
result="$(ssh $SSH_OPTS -i spt-temporary-keypair $floatingip "dd if=/dev/zero of=/mnt/spt-write-test-3 $DD_OPTIONS bs=1G count=1" 2>&1)" # no time limit here because it's a single block operation, no intermediate results available
echo "$result" | xargs -I@ echo "DD_TEST_3: @"
cleanup

11
timmy_data/simplified-performance-testing/scripts/glance-1-image-create.sh Normal file
View File

@ -0,0 +1,11 @@
#!/bin/bash
set -x
. openrc
log='glance-image-upload.log'
rm $log
result="$(dd if=/dev/zero bs=1M count=4000 2>>$log | /usr/bin/time -f%e glance image-create --name "spt-test-image" --container-format bare --disk-format raw 2>>$log)"
cat $log
echo "$result"
rm $log

8
timmy_data/simplified-performance-testing/scripts/glance-1-image-download.sh Normal file
View File

@ -0,0 +1,8 @@
#!/bin/bash
set -x
. openrc
id=$(glance image-list | grep "spt-test-image" | cut -d' ' -f2)
/usr/bin/time -f%e glance image-download $id 2>&1 > /dev/null
glance image-delete $id

28
timmy_data/simplified-performance-testing/scripts/iperf-client.sh Normal file
View File

@ -0,0 +1,28 @@
#!/bin/bash
set -x
function timeout_kill() {
if [ "$1" -gt 0 ]
then
sleep 10.5
[ "$(pgrep iperf | grep -c "$1")" -gt 0 ] && sleep 5
[ "$(pgrep iperf | grep -c "$1")" -gt 0 ] && kill -9 $1 &>/dev/null
fi
}
SPT_IPERF_PORT=${SPT_IPERF_PORT:-"65432"}
[ -z "$SERVER_IP" ] && echo '$SERVER_IP not provided, exiting' && exit 1
# install iperf
which iperf &>/dev/null
if [ "$?" -ne "0" ]
then
result="$(DEBIAN_FRONTEND=noninteractive apt-get -y install iperf 2>&1)"
[ "$?" -ne "0" ] && echo -e "failed to install iperf:\n$result" && exit 1
fi
iperf -c $SERVER_IP -p $SPT_IPERF_PORT &
timeout_kill $!
iperf -c $SERVER_IP -p $SPT_IPERF_PORT -P10 &
timeout_kill $!

23
timmy_data/simplified-performance-testing/scripts/iperf-server-start.sh Normal file
View File

@ -0,0 +1,23 @@
#!/bin/bash
SPT_IPERF_PORT=${SPT_IPERF_PORT:-"65432"}
# install iperf
which iperf &>/dev/null
if [ "$?" -ne "0" ]
then
result="$(DEBIAN_FRONTEND=noninteractive apt-get -y install iperf 2>&1)"
[ "$?" -ne "0" ] && echo -e "failed to install iperf:\n$result" && exit 1
fi
# add firewall rule
if [ "$(iptables -S | grep -c spt-temporary-rule-tcp-${SPT_IPERF_PORT})" -eq 0 ]
then
result="$(iptables -I INPUT 1 -p tcp --dport ${SPT_IPERF_PORT} -j ACCEPT -m comment --comment "spt-temporary-rule-tcp-${SPT_IPERF_PORT}" 2>&1)"
[ "$?" -ne "0" ] && echo -e "failed to add iptables rule:\n$result" && exit 1
fi
# start iperf server
outfile=$(mktemp)
iperf -s -p $SPT_IPERF_PORT &> $outfile &
printf $outfile

18
timmy_data/simplified-performance-testing/scripts/iperf-server-stop.sh Normal file
View File

@ -0,0 +1,18 @@
#!/bin/bash
SPT_IPERF_PORT=${SPT_IPERF_PORT:-"65432"}
killall iperf
while ["$(iptables -L --line-numbers | grep -c 'spt-temporary-rule')" -ge 0 ]
do
rulenum=`iptables -L --line-numbers | grep 'spt-temporary-rule' | head -n 1 | awk '{print $1}'`
[ -n "$sulenum" ] && [ "$rulenum" -ge 0 ] && iptables -D INPUT $rulenum
done
if [ -n $SERVER_OUTPUT ]
then
echo $SERVER_OUTPUT
cat $SERVER_OUTPUT
rm -f $SERVER_OUTPUT
else
echo '$SERVER_OUTPUT not provided'
fi

350
timmy_data/simplified-performance-testing/scripts/network-VM-to-VM-iperf-tests.sh Normal file
View File

@ -0,0 +1,350 @@
#!/bin/bash
set -x
# this test requires sudo in VM
# this test requires iperf in VM (or available for installation)
# this test expects iptables in VM to accept connections, it does not manipulate iptables inside VM
SPT_DNS=${SPT_DNS:-"8.8.8.8"}
SPT_FLOATING_NET=${SPT_FLOATING_NET:-"admin_floating_net"}
SPT_FLAVOR=${SPT_FLAVOR:-"m1.small"}
SPT_IMAGE=${SPT_IMAGE:-"xenial"}
SPT_VM_USER=${SPT_VM_USER:-"ubuntu"}
SPT_VM_COOLDOWN=${SPT_VM_COOLDOWN:-"120"}
SPT_VM_INSTALL_COMMAND=${SPT_VM_INSTALL_COMMAND:-"apt-get"}
SPT_IPERF_PORT=${SPT_IPERF_PORT:-"65432"}
SPT_AVAILABILITY_ZONE=${SPT_AVAILABILITY_ZONE:-"nova"}
SSH_OPTS="-q -n -oStrictHostKeyChecking=no -oUserKnownHostsFile=/dev/null -l $SPT_VM_USER"
VM_BOOT_TIMEOUT_MINUTES=${VM_BOOT_TIMEOUT_MINUTES:-"5"}
vm_boot_check_tries=$(($VM_BOOT_TIMEOUT_MINUTES*2))
vm_boot_check_delay=30
vm_active_check_tries=10
vm_active_check_delay=10
function partial_cleanup_2 {
[ -n "$instance_2_id" ] && [ -n "$floatingip_2" ] && nova floating-ip-disassociate $instance_2_id $floatingip_2
[ -n "$instance_2_id" ] && [ -n "$secgroup_id" ] && nova remove-secgroup $instance_2_id $secgroup_id
[ -n "$instance_2_id" ] && nova delete $instance_2_id
}
function cleanup {
echo "cleaning up..."
partial_cleanup_2
[ -n "$floatingip_2_id" ] && neutron floatingip-delete $floatingip_2_id
[ -n "$instance_id" ] && [ -n "$floatingip" ] && nova floating-ip-disassociate $instance_id $floatingip
[ -n "$floatingip_id" ] && neutron floatingip-delete $floatingip_id
[ -n "$instance_id" ] && [ -n "$secgroup_id" ] && nova remove-secgroup $instance_id $secgroup_id
[ -n "$instance_id" ] && nova delete $instance_id
[ -n "$secgroup_id" ] && neutron security-group-delete $secgroup_id
[ -n "$router_id" ] && [ -n "$subnet_id" ] && neutron router-interface-delete $router_id $subnet_id
[ -n "$router_id" ] && [ -n "$subnet_2_id" ] && neutron router-interface-delete $router_id $subnet_2_id
[ -n "$router_id" ] && neutron router-delete $router_id
[ -n "$subnet_id" ] && neutron subnet-delete $subnet_id
[ -n "$subnet_2_id" ] && neutron subnet-delete $subnet_2_id
[ -n "$net_id" ] && neutron net-delete $net_id
[ -n "$net_2_id" ] && neutron net-delete $net_2_id
[ -n "$keypair_name" ] && nova keypair-delete $keypair_name
[ -f "spt-temporary-keypair" ] && rm "spt-temporary-keypair"
}
function check_code {
# arguments:
# $1 - exit code
# $2 - error message
# $3 - command output to print
if [ "$1" -ne "0" ]
then
echo "$2:"
echo "$3"
cleanup
exit 1
fi
}
. openrc
# create keypair
result="$(nova keypair-add "spt-temporary-keypair" >"spt-temporary-keypair" 2>&1)"
code=$?
[ "$code" -eq "0" ] && keypair_name="spt-temporary-keypair"
check_code $? "failed to create keypair" "$result"
chmod 600 "spt-temporary-keypair"
# create network
result="$(neutron net-create "spt-temporary-net" 2>&1)"
check_code $? "failed to create network" "$result"
net_id=$(echo "$result" | grep "^| id " | awk '{print $4}')
# create subnet
result="$(neutron subnet-create --name "spt-temporary-subnet" --dns-nameserver $SPT_DNS $net_id 10.20.30.0/24 2>&1)"
check_code $? "failed to create subnet" "$result"
subnet_id=$(echo "$result" | grep "^| id " | awk '{print $4}')
# create router
result="$(neutron router-create "spt-temporary-router" 2>&1)"
check_code $? "failed to create router" "$result"
router_id=$(echo "$result" | grep "^| id " | awk '{print $4}')
# add router interface
result="$(neutron router-interface-add $router_id $subnet_id 2>&1)"
check_code $? "failed to add router interface to subnet" "$result"
# set router gateway
result="$(neutron router-gateway-set $router_id $SPT_FLOATING_NET 2>&1)"
check_code $? "failed to set router gateway" "$result"
# create security group
result="$(neutron security-group-create "spt-temporary-security-group" 2>&1)"
check_code $? "failed to create security group" "$result"
secgroup_id=$(echo "$result" | grep "^| id " | awk '{print $4}')
# create security group rule
result="$(neutron security-group-rule-create $secgroup_id --protocol TCP 2>&1)"
check_code $? "failed to create security group rule" "$result"
# create floating ip
result="$(neutron floatingip-create $SPT_FLOATING_NET 2>&1)"
check_code $? "failed to create floatingip" "$result"
floatingip=$(echo "$result" | grep "^| floating_ip_address " | awk '{print $4}')
floatingip_id=$(echo "$result" | grep "^| id " | awk '{print $4}')
# boot VM
result="$(nova boot --image $SPT_IMAGE --flavor $SPT_FLAVOR --nic net-id=$net_id --key-name "spt-temporary-keypair" --security-groups $secgroup_id --availability-zone $SPT_AVAILABILITY_ZONE "spt-temporary-vm" 2>&1)"
check_code $? "failed to boot VM" "$result"
instance_id=$(echo "$result" | grep "^| id " | awk '{print $4}')
# wait for instance to become active
for i in $(seq 1 $vm_active_check_tries)
do
result="$(nova show $instance_id 2>&1)"
vm_status=$(echo "$result" | grep "^| *status" | awk '{printf $4}')
[ "$vm_status" == "ACTIVE" ] && break
[ $i -lt $vm_active_check_tries ] && sleep $vm_active_check_delay
done
! [ "$vm_status" == "ACTIVE" ] && check_code 1 "timeout waiting for VM to become active" "$result"
# associate floatingip
result="$(nova floating-ip-associate $instance_id $floatingip 2>&1)"
check_code $? "failed to associate floatingip" "$result"
# get fixed ip
result="$(neutron floatingip-show $floatingip_id 2>&1)"
check_code $? "failed to show floatingip info" "$result"
fixedip=$(echo "$result" | grep "^| fixed_ip_address " | awk '{print $4}')
# test connection to VM
for i in $(seq 1 $vm_boot_check_tries)
do
result="$(ssh $SSH_OPTS -i spt-temporary-keypair $floatingip "uptime" 2>&1)"
code=$?
[ $code -eq 0 ] && break
[ $i -lt $vm_boot_check_tries ] && sleep $vm_boot_check_delay
done
check_code $code "failed to connect to VM" "$result"
# install iperf
result="$(ssh $SSH_OPTS -i spt-temporary-keypair $floatingip "sudo $SPT_VM_INSTALL_COMMAND install iperf" 2>&1)"
check_code $? "failed to install iperf" "$result"
# start server
result="$(ssh $SSH_OPTS -i spt-temporary-keypair $floatingip '
outfile=$(mktemp)
iperf -s -p '$SPT_IPERF_PORT'&> $outfile &
pid=$!
if ! [ $(pgrep iperf) ]
then
echo "iperf did not start"
cat $outfile
rm -f $outfile
exit 1
fi
echo $pid
echo $outfile' 2>&1)"
check_code $? "failed to start iperf server" "$result"
read -d'\n' server_pid server_output <<< "$result"
##########################################################
### stage 1 - test with another VM on the same compute ###
##########################################################
# find location of the first VM
host=$(mysql -s -N -D nova -e "select host from instances where uuid='$instance_id'" 2>&1)
check_code $? "failed to get host of VM $instance_id" "$host"
# allocate second floating IP
result="$(neutron floatingip-create $SPT_FLOATING_NET 2>&1)"
check_code $? "failed to create second floatingip" "$result"
floatingip_2=$(echo "$result" | grep "^| floating_ip_address " | awk '{print $4}')
floatingip_2_id=$(echo "$result" | grep "^| id " | awk '{print $4}')
# boot second VM on the same host
result="$(nova boot --image $SPT_IMAGE --flavor $SPT_FLAVOR --nic net-id=$net_id --key-name "spt-temporary-keypair" --security-groups $secgroup_id --availability-zone $SPT_AVAILABILITY_ZONE:$host "spt-temporary-vm-2" 2>&1)"
check_code $? "failed to boot second VM on the same host" "$result"
instance_2_id=$(echo "$result" | grep "^| id " | awk '{print $4}')
# wait for instance to become active
for i in $(seq 1 $vm_active_check_tries)
do
result="$(nova show $instance_2_id 2>&1)"
vm_status=$(echo "$result" | grep "^| *status" | awk '{printf $4}')
[ "$vm_status" == "ACTIVE" ] && break
[ $i -lt $vm_active_check_tries ] && sleep $vm_active_check_delay
done
! [ "$vm_status" == "ACTIVE" ] && check_code 1 "timeout waiting for second VM to become active" "$result"
# associate floatingip
result="$(nova floating-ip-associate $instance_2_id $floatingip_2 2>&1)"
check_code $? "failed to associate floatingip_2" "$result"
# test connection to VM
for i in $(seq 1 $vm_boot_check_tries)
do
result="$(ssh $SSH_OPTS -i spt-temporary-keypair $floatingip_2 "uptime" 2>&1)"
code=$?
[ $code -eq 0 ] && break
[ $i -lt $vm_boot_check_tries ] && sleep $vm_boot_check_delay
done
check_code $code "failed to connect to second VM" "$result"
# install iperf on second VM
result="$(ssh $SSH_OPTS -i spt-temporary-keypair $floatingip_2 "sudo $SPT_VM_INSTALL_COMMAND install iperf" 2>&1)"
check_code $? "failed to install iperf on second VM" "$result"
# run VM to VM test via internal IPs
result="$(ssh $SSH_OPTS -i spt-temporary-keypair $floatingip_2 "iperf -c $fixedip -p $SPT_IPERF_PORT" 2>&1)"
echo "VM to VM, same host, internal IP, 1 connection:"
echo "$result"
# removing second VM
partial_cleanup_2
#####################################################
### stage 2 - test with VMs on different computes ###
#####################################################
#find current AZ id
az_id=$(mysql -s -N -D nova -e "select aggregate_id from aggregate_hosts where host='$host' and deleted = 0" 2>&1)
# find another host
if [ -z "$az_id" ]
then
# no AZ defined - all hosts in the default "nova" AZ
host_2=$(mysql -s -N -D nova -e "select host from compute_nodes where host <> '$host' and deleted = 0 limit 1" 2>&1)
else
host_2=$(mysql -s -N -D nova -e "select host from aggregate_hosts where aggregate_id = $az_id and host <> '$host' and deleted = 0 limit 1" 2>&1)
fi
check_code $? "failed to get a different host for a new VM" "$host"
# boot second VM on a different host
result="$(nova boot --image $SPT_IMAGE --flavor $SPT_FLAVOR --nic net-id=$net_id --key-name "spt-temporary-keypair" --security-groups $secgroup_id --availability-zone $SPT_AVAILABILITY_ZONE:$host_2 "spt-temporary-vm-2" 2>&1)"
check_code $? "failed to boot second VM on a different host" "$result"
instance_2_id=$(echo "$result" | grep "^| id " | awk '{print $4}')
# wait for instance to become active
for i in $(seq 1 $vm_active_check_tries)
do
result="$(nova show $instance_2_id 2>&1)"
vm_status=$(echo "$result" | grep "^| *status" | awk '{printf $4}')
[ "$vm_status" == "ACTIVE" ] && break
[ $i -lt $vm_active_check_tries ] && sleep $vm_active_check_delay
done
! [ "$vm_status" == "ACTIVE" ] && check_code 1 "timeout waiting for second VM to become active" "$result"
# associate floatingip
result="$(nova floating-ip-associate $instance_2_id $floatingip_2 2>&1)"
check_code $? "failed to associate floatingip_2" "$result"
# test connection to VM
for i in $(seq 1 $vm_boot_check_tries)
do
result="$(ssh $SSH_OPTS -i spt-temporary-keypair $floatingip_2 "uptime" 2>&1)"
code=$?
[ $code -eq 0 ] && break
[ $i -lt $vm_boot_check_tries ] && sleep $vm_boot_check_delay
done
check_code $code "failed to connect to second VM" "$result"
# install iperf on second VM
result="$(ssh $SSH_OPTS -i spt-temporary-keypair $floatingip_2 "sudo $SPT_VM_INSTALL_COMMAND install iperf" 2>&1)"
check_code $? "failed to install iperf on second VM" "$result"
# run VM to VM test via internal IPs
result="$(ssh $SSH_OPTS -i spt-temporary-keypair $floatingip_2 "iperf -c $fixedip -p $SPT_IPERF_PORT" 2>&1)"
echo "VM to VM, different hosts, internal IP, 1 connection:"
echo "$result"
# run VM to VM test via internal IPs, 10 connections
result="$(ssh $SSH_OPTS -i spt-temporary-keypair $floatingip_2 "iperf -c $fixedip -p $SPT_IPERF_PORT -P10" 2>&1)"
echo "VM to VM, different hosts, internal IP, 10 connections:"
echo "$result"
# run VM to VM test via floating IPs
result="$(ssh $SSH_OPTS -i spt-temporary-keypair $floatingip_2 "iperf -c $floatingip -p $SPT_IPERF_PORT" 2>&1)"
echo "VM to VM, different hosts, floating IP, 1 connection:"
echo "$result"
# removing second VM
partial_cleanup_2
#####################################################################################################################
### stage 3 - test with VMs on different hosts and connected to different networks, networks connected via router ###
#####################################################################################################################
# create second network
result="$(neutron net-create "spt-temporary-net-2" 2>&1)"
check_code $? "failed to create second network" "$result"
net_2_id=$(echo "$result" | grep "^| id " | awk '{print $4}')
# create second subnet
result="$(neutron subnet-create --name "spt-temporary-subnet-2" --dns-nameserver $SPT_DNS $net_2_id 10.20.40.0/24 2>&1)"
check_code $? "failed to create second subnet" "$result"
subnet_2_id=$(echo "$result" | grep "^| id " | awk '{print $4}')
# join subnets via router
result="$(neutron router-interface-add $router_id $subnet_2_id 2>&1)"
check_code $? "failed to add router interface to second subnet" "$result"
# boot second VM in second network
result="$(nova boot --image $SPT_IMAGE --flavor $SPT_FLAVOR --nic net-id=$net_2_id --key-name "spt-temporary-keypair" --security-groups $secgroup_id --availability-zone $SPT_AVAILABILITY_ZONE:$host_2 "spt-temporary-vm-2" 2>&1)"
check_code $? "failed to boot second VM" "$result"
instance_2_id=$(echo "$result" | grep "^| id " | awk '{print $4}')
# wait for instance to become active
for i in $(seq 1 $vm_active_check_tries)
do
result="$(nova show $instance_2_id 2>&1)"
vm_status=$(echo "$result" | grep "^| *status" | awk '{printf $4}')
[ "$vm_status" == "ACTIVE" ] && break
[ $i -lt $vm_active_check_tries ] && sleep $vm_active_check_delay
done
! [ "$vm_status" == "ACTIVE" ] && check_code 1 "timeout waiting for second VM to become active" "$result"
# associate floatingip
result="$(nova floating-ip-associate $instance_2_id $floatingip_2 2>&1)"
check_code $? "failed to associate floatingip_2" "$result"
# test connection to VM
for i in $(seq 1 $vm_boot_check_tries)
do
result="$(ssh $SSH_OPTS -i spt-temporary-keypair $floatingip_2 "uptime" 2>&1)"
code=$?
[ $code -eq 0 ] && break
[ $i -lt $vm_boot_check_tries ] && sleep $vm_boot_check_delay
done
check_code $code "failed to connect to second VM" "$result"
# install iperf on second VM
result="$(ssh $SSH_OPTS -i spt-temporary-keypair $floatingip_2 "sudo $SPT_VM_INSTALL_COMMAND install iperf" 2>&1)"
check_code $? "failed to install iperf on second VM" "$result"
# run VM to VM test via internal IPs
result="$(ssh $SSH_OPTS -i spt-temporary-keypair $floatingip_2 "iperf -c $fixedip -p $SPT_IPERF_PORT" 2>&1)"
echo "VM to VM, different hosts, different networks, internal IP, 1 connection:"
echo "$result"
cleanup