Browse Source

Move non-nested perfload shell commands to script

The script was embedded in the playbook, which leads to some
pain with regard to editing and reviewing as well as manual
testing.

The disadvantage of doing this is that it can make jobs
somewhat less portable between projects, but in this case
that's not really an issue.

There are further improvements that can made to remove duplication
between the nested and non-nested versions of these jobs. This
change will make it easier for those changes to be made as
people have time.

Change-Id: Ia6795ef15a03429c19e66ed6d297f62da72cc052
changes/44/666544/1
Chris Dent 3 years ago
parent
commit
3b040f58f7
  1. 109
      gate/perfload-runner.sh
  2. 137
      playbooks/perfload.yaml

109
gate/perfload-runner.sh

@ -0,0 +1,109 @@
#!/bin/bash -x
WORK_DIR=$1
# Do some performance related information gathering for placement.
EXPLANATION="
This output combines output from placeload with timing information
gathered via curl. The placeload output is the current maximum
microversion of placement followed by an encoded representation of
what it has done. Lowercase 'r', 'i', 'a', and 't' indicate successful
creation of a resource provider and setting inventory, aggregates, and
traits on that resource provider.
If there are upper case versions of any of those letters, a failure
happened for a single request. The letter will be followed by the
HTTP status code and the resource provider uuid. These can be used
to find the relevant entry in logs/screen-placement-api.txt.gz.
Note that placeload does not exit with an error code when this
happens. It merely reports and moves on. Under correct circumstances
the right output is a long string of 4000 characters containing
'r', 'i', 'a', 't' in random order (because async).
After that are three aggregate uuids, timing information for the
placeload run, and then timing information for two identical curl
requests for allocation candidates.
If no timed requests are present it means that the expected number
of resource providers were not created. At this time, only resource
providers are counted, not whether they have the correct inventory,
aggregates, or traits.
"
# This aggregate uuid is a static value in placeload.
AGGREGATE="14a5c8a3-5a99-4e8f-88be-00d85fcb1c17"
TRAIT="HW_CPU_X86_AVX2"
PLACEMENT_QUERY="resources=VCPU:1,DISK_GB:10,MEMORY_MB:256&member_of=${AGGREGATE}&required=${TRAIT}"
PLACEMENT_URL="http://localhost:8000"
LOG=placement-perf.txt
LOG_DEST=${WORK_DIR}/logs
COUNT=1000
trap "sudo cp -p $LOG $LOG_DEST" EXIT
function time_candidates {
(
echo "##### TIMING GET /allocation_candidates?${PLACEMENT_QUERY} twice"
time curl -s -H 'x-auth-token: admin' -H 'openstack-api-version: placement latest' "${PLACEMENT_URL}/allocation_candidates?${PLACEMENT_QUERY}" > /dev/null
time curl -s -H 'x-auth-token: admin' -H 'openstack-api-version: placement latest' "${PLACEMENT_URL}/allocation_candidates?${PLACEMENT_QUERY}" > /dev/null
) 2>&1 | tee -a $LOG
}
function write_allocation {
# Take the first allocation request and send it back as a well-formed allocation
curl -s -H 'x-auth-token: admin' -H 'openstack-api-version: placement latest' "${PLACEMENT_URL}/allocation_candidates?${PLACEMENT_QUERY}&limit=5" \
| jq --arg proj $(uuidgen) --arg user $(uuidgen) '.allocation_requests[0] + {consumer_generation: null, project_id: $proj, user_id: $user}' \
| curl -s -H 'x-auth-token: admin' -H 'content-type: application/json' -H 'openstack-api-version: placement latest' \
-X PUT -d @- "${PLACEMENT_URL}/allocations/$(uuidgen)"
}
function load_candidates {
time_candidates
for iter in {1..99}; do
echo "##### Writing allocation ${iter}" | tee -a $LOG
write_allocation
time_candidates
done
}
function check_placement {
local rp_count
local code
code=0
python -m virtualenv -p python3 .placeload
. .placeload/bin/activate
# install placeload
pip install 'placeload==0.3.0'
set +x
# load with placeload
(
echo "$EXPLANATION"
# preheat the aggregates to avoid https://bugs.launchpad.net/nova/+bug/1804453
placeload $PLACEMENT_URL 10
echo "##### TIMING placeload creating $COUNT resource providers with inventory, aggregates and traits."
time placeload $PLACEMENT_URL $COUNT
) 2>&1 | tee -a $LOG
rp_count=$(curl -H 'x-auth-token: admin' ${PLACEMENT_URL}/resource_providers |json_pp|grep -c '"name"')
# Skip curl and note if we failed to create the required number of rps
if [[ $rp_count -ge $COUNT ]]; then
load_candidates
else
(
echo "Unable to create expected number of resource providers. Expected: ${COUNT}, Got: $rp_count"
echo "See job-output.txt.gz and logs/screen-placement-api.txt.gz for additional detail."
) | tee -a $LOG
code=1
fi
set -x
deactivate
exit $code
}
check_placement

137
playbooks/perfload.yaml

@ -1,6 +1,5 @@
- hosts: all
tasks:
# Borrowed from devstack tasks but we want it early.
- name: Ensure {{ ansible_user_dir }}/logs exists
become: true
file:
@ -12,142 +11,10 @@
chdir: "{{ ansible_user_dir }}/src/opendev.org/openstack/placement"
shell:
executable: /bin/bash
cmd: |
set -x
# TODO(cdent): Presumably ansible can do this, perhaps with 'package'.
# create database
sudo debconf-set-selections <<MYSQL_PRESEED
mysql-server mysql-server/root_password password secret
mysql-server mysql-server/root_password_again password secret
mysql-server mysql-server/start_on_boot boolean true
MYSQL_PRESEED
sudo apt-get install -y mysql-server mysql-client libmysqlclient-dev jq
sudo mysql -uroot -psecret -e "DROP DATABASE IF EXISTS placement;"
sudo mysql -uroot -psecret -e "CREATE DATABASE placement CHARACTER SET utf8;"
sudo mysql -uroot -psecret -e "GRANT ALL PRIVILEGES ON placement.* TO 'root'@'%' identified by 'secret';"
python -m virtualenv -p python3 .placement
. .placement/bin/activate
pip install . PyMySQL uwsgi
# set config via environment
export OS_PLACEMENT_DATABASE__CONNECTION=mysql+pymysql://root:secret@127.0.0.1/placement?charset=utf8
export OS_PLACEMENT_DATABASE__MAX_POOL_SIZE=25
export OS_PLACEMENT_DATABASE__MAX_OVERFLOW=100
export OS_PLACEMENT_DATABASE__SYNC_ON_STARTUP=True
# Increase our chances of allocating to different providers.
export OS_PLACEMENT_PLACEMENT__RANDOMIZE_ALLOCATION_CANDIDATES=True
export OS_DEFAULT__DEBUG=True
export OS_API__AUTH_STRATEGY=noauth2
uwsgi --http :8000 --wsgi-file .placement/bin/placement-api --daemonize {{ ansible_user_dir }}/logs/placement-api.log --processes 5 --threads 25
cmd: gate/perfload-server.sh {{ ansible_user_dir }}
- name: placement performance
args:
chdir: "{{ ansible_user_dir }}/src/opendev.org/openstack/placement"
shell:
executable: /bin/bash
# TODO(cdent): Change this task to a role?
cmd: |
set -x
# Do some performance related information gathering for placement.
EXPLANATION="
This output combines output from placeload with timing information
gathered via curl. The placeload output is the current maximum
microversion of placement followed by an encoded representation of
what it has done. Lowercase 'r', 'i', 'a', and 't' indicate successful
creation of a resource provider and setting inventory, aggregates, and
traits on that resource provider.
If there are upper case versions of any of those letters, a failure
happened for a single request. The letter will be followed by the
HTTP status code and the resource provider uuid. These can be used
to find the relevant entry in logs/screen-placement-api.txt.gz.
Note that placeload does not exit with an error code when this
happens. It merely reports and moves on. Under correct circumstances
the right output is a long string of 4000 characters containing
'r', 'i', 'a', 't' in random order (because async).
After that are three aggregate uuids, timing information for the
placeload run, and then timing information for two identical curl
requests for allocation candidates.
If no timed requests are present it means that the expected number
of resource providers were not created. At this time, only resource
providers are counted, not whether they have the correct inventory,
aggregates, or traits.
"
# This aggregate uuid is a static value in placeload.
AGGREGATE="14a5c8a3-5a99-4e8f-88be-00d85fcb1c17"
TRAIT="HW_CPU_X86_AVX2"
PLACEMENT_QUERY="resources=VCPU:1,DISK_GB:10,MEMORY_MB:256&member_of=${AGGREGATE}&required=${TRAIT}"
PLACEMENT_URL="http://localhost:8000"
LOG=placement-perf.txt
LOG_DEST={{ ansible_user_dir }}/logs
COUNT=1000
trap "sudo cp -p $LOG $LOG_DEST" EXIT
function time_candidates {
(
echo "##### TIMING GET /allocation_candidates?${PLACEMENT_QUERY} twice"
time curl -s -H 'x-auth-token: admin' -H 'openstack-api-version: placement latest' "${PLACEMENT_URL}/allocation_candidates?${PLACEMENT_QUERY}" > /dev/null
time curl -s -H 'x-auth-token: admin' -H 'openstack-api-version: placement latest' "${PLACEMENT_URL}/allocation_candidates?${PLACEMENT_QUERY}" > /dev/null
) 2>&1 | tee -a $LOG
}
function write_allocation {
# Take the first allocation request and send it back as a well-formed allocation
curl -s -H 'x-auth-token: admin' -H 'openstack-api-version: placement latest' "${PLACEMENT_URL}/allocation_candidates?${PLACEMENT_QUERY}&limit=5" \
| jq --arg proj $(uuidgen) --arg user $(uuidgen) '.allocation_requests[0] + {consumer_generation: null, project_id: $proj, user_id: $user}' \
| curl -s -H 'x-auth-token: admin' -H 'content-type: application/json' -H 'openstack-api-version: placement latest' \
-X PUT -d @- "${PLACEMENT_URL}/allocations/$(uuidgen)"
}
function load_candidates {
time_candidates
for iter in {1..99}; do
echo "##### Writing allocation ${iter}" | tee -a $LOG
write_allocation
time_candidates
done
}
function check_placement {
local rp_count
local code
code=0
python -m virtualenv -p python3 .placeload
. .placeload/bin/activate
# install placeload
pip install 'placeload==0.3.0'
set +x
# load with placeload
(
echo "$EXPLANATION"
# preheat the aggregates to avoid https://bugs.launchpad.net/nova/+bug/1804453
placeload $PLACEMENT_URL 10
echo "##### TIMING placeload creating $COUNT resource providers with inventory, aggregates and traits."
time placeload $PLACEMENT_URL $COUNT
) 2>&1 | tee -a $LOG
rp_count=$(curl -H 'x-auth-token: admin' ${PLACEMENT_URL}/resource_providers |json_pp|grep -c '"name"')
# Skip curl and note if we failed to create the required number of rps
if [[ $rp_count -ge $COUNT ]]; then
load_candidates
else
(
echo "Unable to create expected number of resource providers. Expected: ${COUNT}, Got: $rp_count"
echo "See job-output.txt.gz and logs/screen-placement-api.txt.gz for additional detail."
) | tee -a $LOG
code=1
fi
set -x
deactivate
exit $code
}
check_placement
cmd: gate/perfload-runner.sh {{ ansible_user_dir }}

Loading…
Cancel
Save