x/browbeat
Code Issues Proposed changes

BuhBye YODA

Browse Source 
YODA is no longer a used or maintainer workload. This will help remove
any confusion.

Change-Id: Ifef26f1419be9429bd9ac72a1d25f90ad8241bfc
This commit is contained in:
Sai Sindhur Malleni 2019-03-08 15:09:56 -05:00
parent 3423e3c6a3
commit 86bfc78e25
36 changed files with 3 additions and 1221 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
ansible/oooq/roles/template-configs/templates/browbeat-api-ci.yaml.j2
View File

@ -52,9 +52,6 @@ shaker:
sleep_after: 0 sleep_after: 0
shaker_region: regionOne shaker_region: regionOne
external_host: 2.2.2.2 external_host: 2.2.2.2
yoda:
instackenv: "/home/stack/instackenv.json"
stackrc: "/home/stack/stackrc"
workloads: workloads:
- name: authenticate-keystone - name: authenticate-keystone

3
ansible/oooq/roles/template-configs/templates/browbeat-basic.yaml.j2
View File

@ -54,9 +54,6 @@ shaker:
sleep_after: 0 sleep_after: 0
shaker_region: regionOne shaker_region: regionOne
external_host: 2.2.2.2 external_host: 2.2.2.2
yoda:
instackenv: "/home/stack/instackenv.json"
stackrc: "/home/stack/stackrc"
workloads: workloads:
- name: authenticate - name: authenticate

3
ansible/oooq/roles/template-configs/templates/browbeat-minimal-ci.yaml.j2
View File

@ -54,9 +54,6 @@ shaker:
sleep_after: 0 sleep_after: 0
shaker_region: regionOne shaker_region: regionOne
external_host: 2.2.2.2 external_host: 2.2.2.2
yoda:
instackenv: "/home/stack/instackenv.json"
stackrc: "/home/stack/stackrc"
workloads: workloads:
- name: authenticate - name: authenticate

94
ansible/oooq/roles/template-configs/templates/browbeat-yoda-ci.yaml.j2
View File

@ -1,94 +0,0 @@
# Tests to be completed for the install-and-check.sh script minimal and short workloads are performed
# to confirm functionality.
browbeat:
cloud_name: {{ browbeat_cloud_name }}
rerun: 1
rerun_type: iteration
ansible:
hosts: ansible/hosts
metadata_playbook: ansible/gather/site.yml
ssh_config: ansible/ssh-config
elasticsearch:
enabled: {{ elastic_enabled }}
host: {{ elastic_host }}
port: 9200
regather: false
metadata_files:
- name: hardware-metadata
file: metadata/hardware-metadata.json
- name: environment-metadata
file: metadata/environment-metadata.json
- name: software-metadata
file: metadata/software-metadata.json
- name: version
file: metadata/version.json
grafana:
enabled: {{ grafana_enabled }}
host: {{ grafana_host }}
port: 3000
dashboards:
- openstack-general-system-performance
perfkit:
sleep_before: 0
sleep_after: 0
default:
image: centos7
machine_type: m1.small
os_type: rhel
openstack_image_username: centos
openstack_floating_ip_pool: browbeat_public
openstack_network: browbeat_private
timing_measurements: runtimes
ignore_package_requirements: true
rally:
sleep_before: 0
sleep_after: 0
plugins:
- netcreate-boot: rally/rally-plugins/netcreate-boot
shaker:
server: 1.1.1.1
port: 5555
flavor: m1.small
join_timeout: 600
sleep_before: 0
sleep_after: 0
shaker_region: regionOne
external_host: 2.2.2.2
yoda:
instackenv: "/home/stack/instackenv.json"
stackrc: "/home/stack/stackrc"
workloads:
- name: introspect-{{ overcloud_size }}-batch-5
enabled: true
type: yoda
yoda_type: introspection
method: individual
times: 3
timeout: 1800
batch_size: 5
- name: introspect-{{ overcloud_size }}-bulk
enabled: true
type: yoda
yoda_type: introspection
method: bulk
times: 3
timeout: 1800
- name: HA-Max-Compute-{{ overcloud_size }}-stack-update
enabled: true
type: yoda
yoda_type: overcloud
ntp_server: pool.ntp.org
timeout: 600
templates:
- ""
step: 1
keep_stack: true
times: 1
cloud:
- node: "compute"
start_scale: {{ overcloud_size | int - 3 }}
end_scale: {{ overcloud_size | int - 3 }}
- node: "control"
start_scale: 3
end_scale: 3

206
ansible/oooq/roles/template-configs/templates/browbeat-yoda-scale.yaml.j2
View File

@ -1,206 +0,0 @@
browbeat:
cloud_name: {{ browbeat_cloud_name }}
rerun: 1
rerun_type: iteration
ansible:
hosts: ansible/hosts
metadata_playbook: ansible/gather/site.yml
ssh_config: ansible/ssh-config
elasticsearch:
enabled: {{ elastic_enabled }}
host: {{ elastic_host }}
port: 9200
regather: true
metadata_files:
- name: hardware-metadata
file: metadata/hardware-metadata.json
- name: environment-metadata
file: metadata/environment-metadata.json
- name: software-metadata
file: metadata/software-metadata.json
- name: version
file: metadata/version.json
grafana:
enabled: {{ grafana_enabled }}
host: {{ grafana_host }}
port: 3000
dashboards:
- openstack-general-system-performance
perfkit:
sleep_before: 0
sleep_after: 0
default:
image: centos7
machine_type: m1.small
os_type: rhel
openstack_image_username: centos
openstack_floating_ip_pool: browbeat_public
openstack_network: browbeat_private
timing_measurements: runtimes
ignore_package_requirements: true
rally:
sleep_before: 0
sleep_after: 0
plugins:
- netcreate-boot: rally/rally-plugins/netcreate-boot
shaker:
server: 1.1.1.1
port: 5555
flavor: m1.small
join_timeout: 600
sleep_before: 0
sleep_after: 0
shaker_region: regionOne
external_host: 2.2.2.2
yoda:
instackenv: "/home/stack/instackenv.json"
stackrc: "/home/stack/stackrc"
workloads:
- name: introspect-{{ overcloud_size }}-10-individual-batch-2
type: yoda
yoda_type: introspection
enabled: true
method: individual
times: 10
timeout: 900
batch_size: 2
- name: introspect-{{ overcloud_size }}-10-individual-batch-4
type: yoda
yoda_type: introspection
enabled: true
method: individual
times: 10
timeout: 900
batch_size: 4
- name: introspect-{{ overcloud_size }}-10-individual-batch-8
type: yoda
yoda_type: introspection
enabled: true
method: individual
times: 10
timeout: 900
batch_size: 8
- name: introspect-{{ overcloud_size }}-10-individual-batch-16
type: yoda
yoda_type: introspection
enabled: true
method: individual
times: 10
timeout: 900
batch_size: 16
- name: introspect-{{ overcloud_size }}-10-individual-batch-32
type: yoda
yoda_type: introspection
enabled: true
method: individual
times: 10
timeout: 900
batch_size: 32
- name: introspect-{{ overcloud_size }}-10-individual-batch-{{ overcloud_size }}
type: yoda
yoda_type: introspection
enabled: true
method: individual
times: 10
timeout: 900
batch_size: {{ overcloud_size }}
- name: introspect-{{ overcloud_size }}-50-bulk
type: yoda
yoda_type: introspection
enabled: true
method: bulk
times: 50
timeout: 900
- name: No-HA-Max-Compute-{{ overcloud_size }}-full-deploy
type: yoda
yoda_type: overcloud
ntp_server: clock01.util.phx2.redhat.com
timeout: 600
templates:
- ""
enabled: true
step: 5
keep_stack: false
times: 2
cloud:
- node: "compute"
start_scale: 1
end_scale: {{ overcloud_size | int - 1 }}
- node: "control"
start_scale: 1
end_scale: 1
- name: No-HA-Max-Compute-{{ overcloud_size }}-stack-update
type: yoda
yoda_type: overcloud
ntp_server: clock01.util.phx2.redhat.com
timeout: 600
templates:
- ""
instackenv: "/home/stack/instackenv.json"
enabled: true
step: 5
keep_stack: true
times: 2
cloud:
- node: "compute"
start_scale: 1
end_scale: {{ overcloud_size | int - 1 }}
- node: "control"
start_scale: 1
end_scale: 1
- name: HA-Max-Compute-{{ overcloud_size }}-full-deploy
type: yoda
yoda_type: overcloud
ntp_server: clock01.util.phx2.redhat.com
timeout: 600
templates:
- ""
enabled: true
step: 5
keep_stack: false
times: 2
cloud:
- node: "compute"
start_scale: 1
end_scale: {{ overcloud_size | int - 3 }}
- node: "control"
start_scale: 3
end_scale: 3
- name: HA-Max-Compute-{{ overcloud_size }}-stack-update
type: yoda
yoda_type: overcloud
ntp_server: clock01.util.phx2.redhat.com
timeout: 600
templates:
- ""
enabled: true
step: 5
keep_stack: true
times: 2
cloud:
- node: "compute"
start_scale: 1
end_scale: {{ overcloud_size | int - 3 }}
- node: "control"
start_scale: 3
end_scale: 3
- name: HA-Max-Compute-{{ overcloud_size }}-stack-update
type: yoda
yoda_type: overcloud
ntp_server: clock01.util.phx2.redhat.com
timeout: 600
templates:
- ""
enabled: true
step: 5
keep_stack: true
times: 2
cloud:
- node: "compute"
start_scale: 1
end_scale: {{ overcloud_size | int - 3 }}
- node: "control"
start_scale: 3
end_scale: 3

31
browbeat-complete.yaml
View File

@ -56,9 +56,6 @@ shaker:
sleep_after: 5 sleep_after: 5
shaker_region: regionOne shaker_region: regionOne
external_host: 2.2.2.2 external_host: 2.2.2.2
yoda:
instackenv: "/home/stack/instackenv.json"
stackrc: "/home/stack/stackrc"
workloads: workloads:
# PerfKitBenchmarker # PerfKitBenchmarker
@ -374,31 +371,3 @@ workloads:
time: 60 time: 60
placement: single_room placement: single_room
file: lib/python2.7/site-packages/shaker/scenarios/openstack/dense_l3_east_west.yaml file: lib/python2.7/site-packages/shaker/scenarios/openstack/dense_l3_east_west.yaml
# Yoda scenarios WILL redeploy your overcloud
- name: scale-deploy
enabled: true
type: yoda
yoda_type: overcloud
ntp_server: pool.ntp.org
templates:
- ""
timeout: 600 #deploy timeout in minutes
step: 1
keep_stack: false
times: 3
cloud:
- node: "compute"
start_scale: 1
end_scale: 1
- node: "control"
start_scale: 1
end_scale: 3
- name: introspect-batch
enabled: true
type: yoda
yoda_type: introspection
method: individual #other option is bulk
times: 3
timeout: 900 #introspection timeout in seconds
batch_size: 2

3
browbeat-config.yaml
View File

@ -60,9 +60,6 @@ shaker:
sleep_after: 0 sleep_after: 0
shaker_region: regionOne shaker_region: regionOne
external_host: 2.2.2.2 external_host: 2.2.2.2
yoda:
instackenv: "/home/stack/instackenv.json"
stackrc: "/home/stack/stackrc"
workloads: workloads:
# PerfKitBenchmarker # PerfKitBenchmarker

2
browbeat.py
View File

@ -24,7 +24,7 @@ import browbeat.workloadbase
from browbeat.config import load_browbeat_config from browbeat.config import load_browbeat_config
from browbeat.path import results_path from browbeat.path import results_path
_workload_opts = ['perfkit', 'rally', 'shaker', 'yoda'] _workload_opts = ['perfkit', 'rally', 'shaker']
_config_file = 'browbeat-config.yaml' _config_file = 'browbeat-config.yaml'
debug_log_file = 'log/debug.log' debug_log_file = 'log/debug.log'

12
browbeat/schema/browbeat.yml
View File

@ -164,16 +164,6 @@ mapping:
type: str type: str
required: False required: False
pattern: ^([a-zA-Z0-9]|[a-zA-Z0-9][a-zA-Z0-9\-]{0,61}[a-zA-Z0-9])(\.([a-zA-Z0-9]|[a-zA-Z0-9][a-zA-Z0-9\-]{0,61}[a-zA-Z0-9]))*$ pattern: ^([a-zA-Z0-9]|[a-zA-Z0-9][a-zA-Z0-9\-]{0,61}[a-zA-Z0-9])(\.([a-zA-Z0-9]|[a-zA-Z0-9][a-zA-Z0-9\-]{0,61}[a-zA-Z0-9]))*$
yoda:
type: map
required: True
mapping:
instackenv:
type: str
required: True
stackrc:
type: str
required: True
workloads: workloads:
required: True required: True
type: seq type: seq
@ -191,4 +181,4 @@ mapping:
type: type:
type: str type: str
required: True required: True
enum: ['perfkit', 'rally', 'shaker', 'yoda'] enum: ['perfkit', 'rally', 'shaker']

71
browbeat/schema/yoda.yml
View File

@ -1,71 +0,0 @@
# This schema defines how a Yoda workload is formated
name: Yoda workload schema
type: map
mapping:
# Required items to be a Yoda workload
enabled:
required: True
type: bool
name:
type: str
required: True
timeout:
type: int
required: True
times:
type: int
required: True
type:
type: str
required: True
enum: ["yoda"]
yoda_type:
type: str
required: True
enum: ['introspection', 'overcloud']
# Optional/type-specific items for Yoda workload:
batch_size:
type: int
required: False
cloud:
type: seq
required: False
sequence:
- type: map
allowempty: True
mapping:
node:
type: str
required: True
start_scale:
type: int
required: True
end_scale:
type: int
required: True
instackenv:
type: str
required: False
keep_stack:
type: bool
required: False
max_fail_amnt:
type: float
required: False
method:
type: str
required: False
node_pinning:
type: bool
required: False
ntp_server:
type: str
required: False
step:
type: int
required: False
templates:
type: seq
required: False
sequence:
- type: str

3
browbeat/tools.py
View File

@ -19,7 +19,6 @@ import subprocess
from browbeat import perfkit from browbeat import perfkit
from browbeat import rally from browbeat import rally
from browbeat import shaker from browbeat import shaker
from browbeat import yoda
class Tools(object): class Tools(object):
@ -85,8 +84,6 @@ class Tools(object):
workloads = rally.Rally(self.config, result_dir_ts) workloads = rally.Rally(self.config, result_dir_ts)
elif workload["type"] == "shaker": elif workload["type"] == "shaker":
workloads = shaker.Shaker(self.config, result_dir_ts) workloads = shaker.Shaker(self.config, result_dir_ts)
elif workload["type"] == "yoda":
workloads = yoda.Yoda(self.config, result_dir_ts)
else: else:
self.logger.error("Unknown workload provider: {}".format(workload["type"])) self.logger.error("Unknown workload provider: {}".format(workload["type"]))
workloads.run_workload(copy.deepcopy(workload), run_iteration) workloads.run_workload(copy.deepcopy(workload), run_iteration)

688
browbeat/yoda.py
View File

@ -1,688 +0,0 @@
#!/usr/bin/env python
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# Yet another cloud deployment tool
from collections import deque
import datetime
import json
import logging
import os
import time
from openstack import connection
from openstack import exceptions
try:
from ostag import ostag
except ImportError:
ostag = None
import requests
import browbeat.tools
from browbeat import elastic
from browbeat import grafana
from browbeat import workloadbase
from browbeat.path import results_path
class Yoda(workloadbase.WorkloadBase):
def __init__(self, config, result_dir_ts):
self.logger = logging.getLogger('browbeat.yoda')
self.config = config
self.result_dir_ts = result_dir_ts
self.tools = browbeat.tools.Tools(self.config)
self.grafana = grafana.Grafana(self.config)
self.elastic = elastic.Elastic(self.config, self.__class__.__name__.lower())
self.error_count = 0
self.pass_count = 0
self.test_count = 0
self.scenario_count = 0
def get_stats(self):
self.logger.info(
"Current number of YODA tests executed: {}".format(
self.test_count))
self.logger.info(
"Current number of YODA tests passed: {}".format(
self.pass_count))
self.logger.info(
"Current number of YODA tests failed: {}".format(
self.error_count))
def update_tests(self):
self.test_count += 1
self.update_total_tests()
def update_pass_tests(self):
self.pass_count += 1
self.update_total_pass_tests()
def update_fail_tests(self):
self.error_count += 1
self.update_total_fail_tests()
def update_scenarios(self):
self.scenario_count += 1
self.update_total_scenarios()
def state_tracker_extend(self, state, state_list):
if state is None:
return state_list
elif state_list is None:
return [state]
elif state in state_list[-1]:
return state_list
else:
state_list.append(state)
return state_list
def node_is_cleaning(self, provision_state):
ret = provision_state is not None
ret = ret and 'clean' in provision_state
ret = ret and 'fail' not in provision_state
return ret
def is_cleaning(self, conn):
for node in conn.bare_metal.nodes():
if self.node_is_cleaning(node.provision_state):
return True
return False
def failed_cleaning_count(self, conn):
count = 0
for node in conn.bare_metal.nodes():
if self.node_is_cleaning(node.provision_state):
count += 1
return count
def wait_for_clean(self, env_setup, conn):
wait_time = 1
# 15 minute timeout
timeout = (60 * 15)
while self.is_cleaning(conn):
# Cleans can fail, so we just try again
if wait_time % 1000 == 0:
self.set_ironic_node_state("manage", env_setup, conn)
time.sleep(30)
self.set_ironic_node_state("provide", env_setup, conn)
time.sleep(1)
wait_time += 1
if wait_time > timeout:
self.logger.error("Node Cleaning failed")
exit(1)
# Required to use console commands because of this bug
# https://bugs.launchpad.net/python-openstacksdk/+bug/1668767
def set_ironic_node_state(self, state, env_setup, conn, node_uuid=""):
if node_uuid != "":
nodes = [node_uuid]
else:
nodes = deque(map(lambda node: node.id, conn.bare_metal.nodes()))
if state == "manage":
cmd_base = "{} openstack baremetal node manage {}"
for _ in range(len(nodes)):
node = nodes.pop()
node_obj = conn.bare_metal.get_node(node)
if "manage" not in node_obj.provision_state:
nodes.append(node)
elif state == "provide":
cmd_base = "{} openstack baremetal node provide {}"
for _ in range(len(nodes)):
node = nodes.pop()
node_obj = conn.bare_metal.get_node(node)
prov_state = node_obj.provision_state
if prov_state is not None and "available" not in prov_state:
nodes.append(node)
elif state == "inspect":
cmd_base = "{} openstack baremetal introspection start {}"
elif state == "off":
cmd_base = "{} openstack baremetal node power off {}"
for _ in range(len(nodes)):
node = nodes.pop()
node_obj = conn.bare_metal.get_node(node)
if "off" not in node_obj.power_state:
nodes.append(node)
elif state == "on":
cmd_base = "{} openstack baremetal node power on {}"
for _ in range(len(nodes)):
node = nodes.pop()
node_obj = conn.bare_metal.get_node(node)
if "on" not in node_obj.power_state:
nodes.append(node)
elif state == "delete":
cmd_base = "{} openstack baremetal node delete {}"
else:
self.logger.error(
"set_ironic_node_state() called with invalid state")
exit(1)
for node in nodes:
cmd = cmd_base.format(env_setup, node)
self.tools.run_async_cmd(cmd + "\"")
time.sleep(.5)
# Gathers metrics on the instack env import
def import_instackenv(self, filepath, env_setup, conn):
results = {}
filepath = os.path.abspath(os.path.expandvars(filepath))
cmd = "{} openstack overcloud node import {}".format(
env_setup, filepath)
start_time = datetime.datetime.utcnow()
out = self.tools.run_cmd(cmd + "\"")
nodes = conn.bare_metal.nodes()
for node in nodes:
while 'enroll' in node.provision_state:
node = conn.bare_metal.get_node(node)
time.sleep(1)
end_time = datetime.datetime.utcnow()
results['import_time'] = (end_time - start_time).total_seconds()
if out['stderr'] == '' or 'Error' not in out['stderr']:
results['import_status'] = "success"
else:
results['import_status'] = "failure"
self.logger.error("Instackenv import returned 1, printing stderr")
self.logger.error(out['stderr'])
return results
# Introspection with exactly the documented workflow
def introspection_bulk(self, timeout, env_setup, conn):
results = {}
nodes = deque(map(lambda node: node.id, conn.bare_metal.nodes()))
cmd = "{} openstack overcloud node introspect --all-manageable".format(
env_setup)
results['nodes'] = {}
for node in conn.bare_metal.nodes(details=True):
results['nodes'][node.id] = {}
results['nodes'][node.id]["last_error"] = node.last_error
results['nodes'][node.id]["driver"] = node.driver
results['nodes'][node.id]["driver_info"] = node.driver_info
results['nodes'][node.id]["properties"] = node.properties
results['nodes'][node.id]["failures"] = 0
results['nodes'][node.id]["state_list"] = None
self.tools.run_async_cmd(cmd + "\"")
out = self.watch_introspecting_nodes(nodes, timeout, conn, results)
failed = out[0]
results['raw'] = out[1]
results["failure_count"] = len(failed)
return results
def watch_introspecting_nodes(self, nodes, timeout, conn, results):
start_time = datetime.datetime.utcnow()
times = []
timeout = datetime.timedelta(seconds=timeout)
while len(nodes):
node = nodes.pop()
# rate limit
time.sleep(10)
try:
node_obj = conn.bare_metal.get_node(node)
except exceptions.SDKException:
self.logger.error(
"Ironic endpoint is down, retrying in 10 seconds")
time.sleep(10)
continue
if node_obj is None:
self.logger.error(
"Can't find node %s which existed at the start of "
"introspection did you delete it manually?", node)
continue
# == works here for string comparison because they are in fact
# the same object if not changed
stored_properties = str(
results['nodes'][node_obj.id]["properties"])
node_properties = str(node_obj.properties)
changed = not stored_properties == node_properties
powered_off = 'off' in node_obj.power_state
not_cleaning = 'clean' not in node_obj.provision_state
if changed and powered_off and not_cleaning:
results['nodes'][node_obj.id]["properties"] = node_obj.properties
results['nodes'][node_obj.id]["state_list"] = self.state_tracker_extend(
node_obj.provision_state, results['nodes'][node_obj.id]["state_list"])
times.append(
(datetime.datetime.utcnow() - start_time).total_seconds())
elif (datetime.datetime.utcnow() - start_time) > timeout:
# return currently active node to the deque to be failed
nodes.appendleft(node)
for node in nodes:
node_obj = conn.bare_metal.get_node(node)
results['nodes'][node_obj.id]['failures'] += 1
if results['nodes'][node_obj.id]['failures'] > 10:
self.logger.error(
"Node " + node_obj.id + "has failed more than 10 introspections")
self.logger.error(
"This probably means it's misconfigured, exiting")
exit(1)
break
else:
results['nodes'][node_obj.id]["state_list"] = self.state_tracker_extend(
node_obj.provision_state, results['nodes'][node_obj.id]["state_list"])
nodes.appendleft(node)
return (nodes, times)
# Introspection with robust failure handling
def introspection_individual(self, batch_size, timeout, env_setup, conn):
nodes = deque(map(lambda node: node.id, conn.bare_metal.nodes()))
failure_count = 0
batch = deque()
results = {}
results['raw'] = []
results['nodes'] = {}
for node in conn.bare_metal.nodes(details=True):
results['nodes'][node.id] = {}
results['nodes'][node.id]["last_error"] = node.last_error
results['nodes'][node.id]["driver"] = node.driver
results['nodes'][node.id]["driver_info"] = node.driver_info
results['nodes'][node.id]["properties"] = node.properties
results['nodes'][node.id]["failures"] = 0
results['nodes'][node.id]["state_list"] = None
while len(nodes):
node = nodes.pop()
self.set_ironic_node_state("inspect", env_setup, conn, node)
batch.append(node)
if len(batch) >= batch_size or (
len(nodes) == 0 and len(batch) != 0):
out = self.watch_introspecting_nodes(
batch, timeout, conn, results)
failed = out[0]
results['raw'].extend(out[1])
failure_count = failure_count + len(failed)
nodes.extend(failed)
batch.clear()
results["failure_count"] = failure_count
return results
def delete_stack(self, conn):
wait_time = 0
# 30 minute timeout
timeout = (60 * 30)
try:
while conn.orchestration.find_stack("overcloud") is not None:
# Deletes can fail, so we just try again
if wait_time % 2000 == 0:
conn.orchestration.delete_stack("overcloud")
time.sleep(10)
wait_time += 10
if wait_time > timeout:
self.logger.error("Overcloud stack delete failed")
exit(1)
except exceptions.SDKException:
# Recursion is probably the wrong way to handle this
self.logger.error("Heat failure during overcloud delete, retrying")
time.sleep(10)
self.delete_stack(conn)
def setup_nodes_dict(self, benchmark):
nodes = {}
for service in benchmark['cloud']:
nodes[service['node']] = service['start_scale']
nodes["previous_" + service['node']] = -1
return nodes
def update_nodes_dict(self, benchmark, nodes, changed):
# update settings for next round, note if changes are made
step = benchmark['step']
nodes_added = 0
for service in benchmark['cloud']:
node_type = service['node']
end_scale = service['end_scale']
nodes["previous_" + node_type] = nodes[node_type]
if nodes[node_type] < end_scale:
difference = end_scale - nodes[node_type]
allowed_difference = step - nodes_added
add = min(difference, allowed_difference)
nodes[node_type] += add
nodes_added += add
changed = True
# edge cases, we must round up otherwise we get
# stuck forever if step is 1, this also means we must
# violate the step rules to both ensure a valid deployment
# and progression
if 'control' in nodes and nodes['control'] == 2:
nodes['control'] = 3
if 'ceph' in nodes and nodes['ceph'] > 0 and nodes['ceph'] < 3:
nodes['ceph'] = 3
return (nodes, changed)
def deploy_overcloud(self, start_time, results,
ntp_server, conn, env_setup, benchmark):
if not isinstance(ntp_server, str):
self.logger.error("Please configure an NTP server!")
exit(1)
cmd = env_setup + "openstack overcloud deploy --templates "
for template in benchmark['templates']:
cmd = cmd + " " + template + " "
for service in benchmark['cloud']:
cmd = cmd + " --" + service['node'] + \
"-scale " + str(results[service['node']])
cmd = cmd + " --timeout=" + \
str(benchmark['timeout']) + " --ntp-server=" + str(ntp_server)
self.logger.debug("Openstack deployment command is " + cmd)
results["overcloud_deploy_command"] = cmd
deploy_process = self.tools.run_async_cmd(cmd + "\"")
results['cleaning_failures'] = self.failed_cleaning_count(conn)
results['nodes'] = {}
while deploy_process.poll() is None:
time.sleep(5)
try:
for node in conn.compute.servers():
time.sleep(1)
# look for new instances to add to our metadata
if node.name not in results['nodes']:
results['nodes'][node.name] = {}
create_time = datetime.datetime.strptime(
node.created_at, "%Y-%m-%dT%H:%M:%SZ")
results['nodes'][node.name]['created_at'] = \
(create_time - start_time).total_seconds()
results['nodes'][node.name]['scheduler_hints'] = \
node.scheduler_hints
results['nodes'][node.name]['state_list'] = None
# try and figure out which baremetal node this
# instance is scheduled on
if 'bm_node' not in results['nodes'][node.name]:
try:
bm_node = next(
conn.bare_metal.nodes(
details=True, instance_id=node.id))
results['nodes'][node.name]['bm_node'] = \
bm_node.id
results['nodes'][node.name]['bm_node_properties'] = \
bm_node.properties
results['nodes'][node.name]['bm_node_driver'] = \
bm_node.driver
results['nodes'][node.name]['bm_last_error'] = \
bm_node.last_error
except StopIteration:
continue
update_time = datetime.datetime.strptime(
node.updated_at, "%Y-%m-%dT%H:%M:%SZ")
results['nodes'][node.name]['last_updated_at'] = \
(update_time - start_time).total_seconds()
results['nodes'][node.name]['final_status'] = node.status
bm_node = next(conn.bare_metal.nodes(details=True,
instance_id=node.id))
state_list = results['nodes'][node.name]['state_list']
state_list = \
self.state_tracker_extend(bm_node.provision_state,
state_list)
rentry = results['nodes'][node.name]
# Populate this field so it gets indexed every time
# even if nodes are never pingable
rentry['ping_time'] = -1
condition = 'private' in node.addresses
if condition:
ping = self.tools.is_pingable(
node.addresses['private'])
else:
ping = False
condition = condition and 'pingable_at' not in rentry
condition = condition and ping
if condition:
ping_time = datetime.datetime.utcnow()
rentry['ping_time'] = (
ping_time - start_time).total_seconds()
except exceptions.HttpException:
self.logger.error("OpenStack bare_metal API is returning NULL")
self.logger.error(
"This sometimes happens during stack creates")
return results
def elastic_insert(self, results, run, start_time, benchmark, results_dir):
scenario_name = benchmark['name']
results['action'] = scenario_name.strip()
results['browbeat_rerun'] = run
results['timestamp'] = str(start_time).replace(" ", "T")
results['grafana_url'] = self.grafana.grafana_urls()
results['scenario'] = benchmark['name']
results['scenario_config'] = benchmark
# Create list of objects for Elastic insertion rather than
# dict of dicts. Insert key to not lose name data
nodes_data = []
for key in results['nodes']:
results['nodes'][key]['name'] = key
nodes_data.append(results['nodes'][key])
results['nodes'] = nodes_data
results = self.elastic.combine_metadata(results)
if not self.elastic.index_result(results, scenario_name, results_dir):
self.update_index_failures()
def dump_scenario_json(self, results_dir, json, time):
with open(results_dir + "/" + str(time).strip() + ".json", 'w') as outfile:
outfile.write(json)
def setup_scenario(self, benchmark_name, dir_ts):
results_dir = self.tools.create_results_dir(
results_path, dir_ts, benchmark_name, benchmark_name)
if isinstance(results_dir, bool):
self.logger.error(
"Malformed Config, benchmark names must be unique!")
exit(1)
self.logger.debug("Created result directory: {}".format(results_dir))
self.workload_logger(self.__class__.__name__)
return results_dir
def introspection_workload(
self, benchmark, run, results_dir, env_setup, conn):
self.delete_stack(conn)
self.wait_for_clean(env_setup, conn)
test_start = datetime.datetime.utcnow()
self.wait_for_clean(env_setup, conn)
self.set_ironic_node_state("delete", env_setup, conn)
while len(list(conn.bare_metal.nodes())) > 0:
time.sleep(5)
import_results = self.import_instackenv(
benchmark['instackenv'], env_setup, conn)
self.set_ironic_node_state("manage", env_setup, conn)
self.set_ironic_node_state("off", env_setup, conn)
if benchmark['method'] == "individual":
introspection_results = self.introspection_individual(
benchmark['batch_size'], benchmark['timeout'], env_setup, conn)
elif benchmark['method'] == "bulk":
introspection_results = self.introspection_bulk(
benchmark['timeout'], env_setup, conn)
else:
self.logger.error(
"Malformed YODA configuration for " + benchmark['name'])
exit(1)
self.get_stats()
# Combines dicts but mutates import_results rather than
# returning a new value
import_results.update(introspection_results)
results = import_results
results['total_nodes'] = len(
list(map(lambda node: node.id, conn.bare_metal.nodes())))
# If maximum failure precentage is not set, we set it to 10%
if 'max_fail_amnt' not in benchmark:
benchmark['max_fail_amnt'] = .10
if results['failure_count'] >= results['total_nodes'] * \
benchmark['max_fail_amnt']:
self.update_fail_tests()
else:
self.update_pass_tests()
self.update_tests()
self.dump_scenario_json(results_dir, json.dumps(results), test_start)
if self.config['elasticsearch']['enabled']:
self.elastic_insert(
results,
run,
test_start,
benchmark,
results_dir)
def overcloud_workload(self, benchmark, run, results_dir, env_setup, conn):
if conn.orchestration.find_stack("overcloud") is None:
self.set_ironic_node_state("provide", env_setup, conn)
self.wait_for_clean(env_setup, conn)
keep_stack = benchmark['keep_stack']
results = self.setup_nodes_dict(benchmark)
changed = True
while changed:
changed = False
# Can't scale from HA to non HA or back
control_change = results['control'] != results['previous_control']
if keep_stack and not control_change:
results['method'] = "update"
else:
self.delete_stack(conn)
self.wait_for_clean(env_setup, conn)
results['method'] = "new"
start_time = datetime.datetime.utcnow()
if 'node_pinning' in benchmark:
if ostag is None:
self.logger.error("ostag is not installed please run")
self.logger.error(
" pip install git+https://github.com/jkilpatr/ostag")
self.logger.error("Pinning not used in this test!")
elif benchmark['node_pinning']:
ostag.clear_tags(conn)
for node in benchmark['cloud']:
ostag.mark_nodes(
"", node['node'], conn, False, "", node['end_scale'])
else:
ostag.clear_tags(conn)
results = self.deploy_overcloud(start_time, results,
benchmark['ntp_server'],
conn, env_setup,
benchmark)
results['total_time'] = (
datetime.datetime.utcnow() - start_time).total_seconds()
try:
stack_status = conn.orchestration.find_stack("overcloud")
except exceptions.SDKException:
self.logger.error(
"Heat endpoint failed to respond, waiting 10 seconds")
time.sleep(10)
continue
if stack_status is None:
continue
results['result'] = str(stack_status.status)
results['result_reason'] = str(stack_status.status_reason)
results['total_nodes'] = len(
list(map(lambda node: node.id, conn.bare_metal.nodes())))
if "COMPLETE" in results['result']:
self.update_pass_tests()
else:
self.update_fail_tests()
self.update_tests
self.get_stats()
self.tools.gather_metadata()
self.dump_scenario_json(
results_dir, json.dumps(results), start_time)
if self.config['elasticsearch']['enabled']:
self.elastic_insert(
results, run, start_time, benchmark, results_dir)
out = self.update_nodes_dict(benchmark, results, changed)
results = out[0]
changed = out[1]
def run_workload(self, workload, run_iteration):
"""Iterates through all yoda scenarios in browbeat yaml config file"""
self.logger.info("Starting YODA workloads")
es_ts = datetime.datetime.utcnow()
dir_ts = es_ts.strftime("%Y%m%d-%H%M%S")
self.logger.debug("Time Stamp (Prefix): {}".format(dir_ts))
stackrc = self.config.get('yoda')['stackrc']
env_setup = "env -i bash -c \"source {}; ".format(stackrc)
auth_vars = self.tools.load_stackrc(stackrc)
if 'OS_AUTH_URL' not in auth_vars:
self.logger.error(
"Please make sure your stackrc is configured correctly")
exit(1)
auth_args = {
'auth_url': auth_vars['OS_AUTH_URL'],
'project_name': 'admin',
'username': auth_vars['OS_USERNAME'],
'password': auth_vars['OS_PASSWORD'],
'verify': False
}
requests.packages.urllib3.disable_warnings()
conn = connection.Connection(**auth_args)
instackenv = self.config.get('yoda')['instackenv']
results_dir = self.setup_scenario(workload['name'], dir_ts)
times = workload['times']
if 'instackenv' not in workload:
workload['instackenv'] = instackenv
# Correct iteration/rerun
rerun_range = range(self.config["browbeat"]["rerun"])
if self.config["browbeat"]["rerun_type"] == "complete":
rerun_range = range(run_iteration, run_iteration + 1)
for run in rerun_range:
for run in range(times):
if workload['yoda_type'] == "overcloud":
self.overcloud_workload(workload, run, results_dir, env_setup, conn)
elif workload['yoda_type'] == "introspection":
self.introspection_workload(workload, run, results_dir, env_setup, conn)
else:
self.logger.error("Could not identify YODA workload!")
exit(1)
self.update_scenarios()

3
conf/browbeat-glance-complete.yaml
View File

@ -53,9 +53,6 @@ shaker:
sleep_after: 0 sleep_after: 0
shaker_region: regionOne shaker_region: regionOne
external_host: 2.2.2.2 external_host: 2.2.2.2
yoda:
instackenv: "/home/stack/instackenv.json"
stackrc: "/home/stack/stackrc"
workloads: workloads:
- name: glance - name: glance

3
conf/browbeat-keystone-complete.yaml
View File

@ -53,9 +53,6 @@ shaker:
sleep_after: 0 sleep_after: 0
shaker_region: regionOne shaker_region: regionOne
external_host: 2.2.2.2 external_host: 2.2.2.2
yoda:
instackenv: "/home/stack/instackenv.json"
stackrc: "/home/stack/stackrc"
workloads: workloads:
- name: authenticate - name: authenticate

3
conf/browbeat-keystone-minimal.yaml
View File

@ -53,9 +53,6 @@ shaker:
sleep_after: 0 sleep_after: 0
shaker_region: regionOne shaker_region: regionOne
external_host: 2.2.2.2 external_host: 2.2.2.2
yoda:
instackenv: "/home/stack/instackenv.json"
stackrc: "/home/stack/stackrc"
workloads: workloads:
- name: authenticate - name: authenticate

3
conf/browbeat-workloads.yml
View File

@ -53,9 +53,6 @@ shaker:
sleep_after: 0 sleep_after: 0
shaker_region: regionOne shaker_region: regionOne
external_host: 2.2.2.2 external_host: 2.2.2.2
yoda:
instackenv: "/home/stack/instackenv.json"
stackrc: "/home/stack/stackrc"
workloads: workloads:
- name: workloads - name: workloads

3
conf/ceilometer-minimal.yaml
View File

@ -52,9 +52,6 @@ shaker:
sleep_after: 0 sleep_after: 0
shaker_region: regionOne shaker_region: regionOne
external_host: 2.2.2.2 external_host: 2.2.2.2
yoda:
instackenv: "/home/stack/instackenv.json"
stackrc: "/home/stack/stackrc"
workloads: workloads:
- name: CeilometerAlarms - name: CeilometerAlarms

3
conf/ceilometer-persist-response-timing.yaml
View File

@ -57,9 +57,6 @@ shaker:
sleep_after: 0 sleep_after: 0
shaker_region: regionOne shaker_region: regionOne
external_host: 2.2.2.2 external_host: 2.2.2.2
yoda:
instackenv: "/home/stack/instackenv.json"
stackrc: "/home/stack/stackrc"
workloads: workloads:
# Measure Ceilometer Response Timings # Measure Ceilometer Response Timings

3
conf/gnocchi-minimal.yaml
View File

@ -54,9 +54,6 @@ shaker:
sleep_after: 0 sleep_after: 0
shaker_region: regionOne shaker_region: regionOne
external_host: 2.2.2.2 external_host: 2.2.2.2
yoda:
instackenv: "/home/stack/instackenv.json"
stackrc: "/home/stack/stackrc"
workloads: workloads:
- name: Gnocchi - name: Gnocchi

3
conf/perfkit-benchmarks.yaml
View File

@ -61,9 +61,6 @@ shaker:
sleep_after: 0 sleep_after: 0
shaker_region: regionOne shaker_region: regionOne
external_host: 2.2.2.2 external_host: 2.2.2.2
yoda:
instackenv: "/home/stack/instackenv.json"
stackrc: "/home/stack/stackrc"
workloads: workloads:
- name: aerospike-centos-m1-small - name: aerospike-centos-m1-small

3
conf/quickstart.yml
View File

@ -52,9 +52,6 @@ shaker:
sleep_after: 0 sleep_after: 0
shaker_region: regionOne shaker_region: regionOne
external_host: 2.2.2.2 external_host: 2.2.2.2
yoda:
instackenv: "/home/stack/instackenv.json"
stackrc: "/home/stack/stackrc"
workloads: workloads:
- name: ping-m1-tiny-centos - name: ping-m1-tiny-centos

3
conf/telemetry-nova-neutron-gnocchi-10k-1000-per-30m.yaml
View File

@ -63,9 +63,6 @@ shaker:
sleep_after: 0 sleep_after: 0
shaker_region: regionOne shaker_region: regionOne
external_host: 2.2.2.2 external_host: 2.2.2.2
yoda:
instackenv: "/home/stack/instackenv.json"
stackrc: "/home/stack/stackrc"
workloads: workloads:

3
conf/telemetry-nova-neutron-gnocchi-1k-100-per-30m.yaml
View File

@ -63,9 +63,6 @@ shaker:
sleep_after: 0 sleep_after: 0
shaker_region: regionOne shaker_region: regionOne
external_host: 2.2.2.2 external_host: 2.2.2.2
yoda:
instackenv: "/home/stack/instackenv.json"
stackrc: "/home/stack/stackrc"
workloads: workloads:

3
conf/telemetry-nova-neutron-gnocchi-1k-500-per-30m.yaml
View File

@ -63,9 +63,6 @@ shaker:
sleep_after: 0 sleep_after: 0
shaker_region: regionOne shaker_region: regionOne
external_host: 2.2.2.2 external_host: 2.2.2.2
yoda:
instackenv: "/home/stack/instackenv.json"
stackrc: "/home/stack/stackrc"
workloads: workloads:

3
conf/telemetry-nova-neutron-gnocchi-5k-500-per-30m.yaml
View File

@ -63,9 +63,6 @@ shaker:
sleep_after: 0 sleep_after: 0
shaker_region: regionOne shaker_region: regionOne
external_host: 2.2.2.2 external_host: 2.2.2.2
yoda:
instackenv: "/home/stack/instackenv.json"
stackrc: "/home/stack/stackrc"
workloads: workloads:

3
conf/telemetry-nova-persist-instances-with-network-fip.yaml
View File

@ -59,9 +59,6 @@ shaker:
sleep_after: 0 sleep_after: 0
shaker_region: regionOne shaker_region: regionOne
external_host: 2.2.2.2 external_host: 2.2.2.2
yoda:
instackenv: "/home/stack/instackenv.json"
stackrc: "/home/stack/stackrc"
workloads: workloads:
# Boots a total of 1000 instances, 100 at a time with 30minutes between booting storms # Boots a total of 1000 instances, 100 at a time with 30minutes between booting storms

3
conf/telemetry-nova-persist-instances-with-network-volume-fip.yaml
View File

@ -59,9 +59,6 @@ shaker:
sleep_after: 0 sleep_after: 0
shaker_region: regionOne shaker_region: regionOne
external_host: 2.2.2.2 external_host: 2.2.2.2
yoda:
instackenv: "/home/stack/instackenv.json"
stackrc: "/home/stack/stackrc"
workloads: workloads:
# Boots a total of 1000 instances, 100 at a time with 30minutes between booting storms # Boots a total of 1000 instances, 100 at a time with 30minutes between booting storms

3
conf/telemetry-nova-persist-instances-with-network-volume.yaml
View File

@ -58,9 +58,6 @@ shaker:
sleep_after: 0 sleep_after: 0
shaker_region: regionOne shaker_region: regionOne
external_host: 2.2.2.2 external_host: 2.2.2.2
yoda:
instackenv: "/home/stack/instackenv.json"
stackrc: "/home/stack/stackrc"
workloads: workloads:
# Boots a total of 1000 instances, 100 at a time with 30minutes between booting storms # Boots a total of 1000 instances, 100 at a time with 30minutes between booting storms

3
conf/telemetry-nova-persist-instances-with-network.yaml
View File

@ -60,9 +60,6 @@ shaker:
sleep_after: 0 sleep_after: 0
shaker_region: regionOne shaker_region: regionOne
external_host: 2.2.2.2 external_host: 2.2.2.2
yoda:
instackenv: "/home/stack/instackenv.json"
stackrc: "/home/stack/stackrc"
workloads: workloads:
# Boots a total of 1000 instances, 100 at a time with 30minutes between booting storms # Boots a total of 1000 instances, 100 at a time with 30minutes between booting storms

3
conf/telemetry-nova-persist-instances-with-volume.yaml
View File

@ -58,9 +58,6 @@ shaker:
sleep_after: 0 sleep_after: 0
shaker_region: regionOne shaker_region: regionOne
external_host: 2.2.2.2 external_host: 2.2.2.2
yoda:
instackenv: "/home/stack/instackenv.json"
stackrc: "/home/stack/stackrc"
workloads: workloads:
# Boots a total of 1000 instances, 100 at a time with 30minutes between booting storms # Boots a total of 1000 instances, 100 at a time with 30minutes between booting storms

3
conf/telemetry-nova-persist-instances.yaml
View File

@ -58,9 +58,6 @@ shaker:
sleep_after: 0 sleep_after: 0
shaker_region: regionOne shaker_region: regionOne
external_host: 2.2.2.2 external_host: 2.2.2.2
yoda:
instackenv: "/home/stack/instackenv.json"
stackrc: "/home/stack/stackrc"
workloads: workloads:

11
tests/data/invalid_browbeat.yml
View File

@ -52,9 +52,6 @@ shaker:
sleep_after: 0 sleep_after: 0
shaker_region: regionOne shaker_region: regionOne
external_host: 2.2.2.2 external_host: 2.2.2.2
yoda:
instackenv: "/home/stack/instackenv.json"
stackrc: "/home/stack/stackrc"
workloads: workloads:
- name: browbeat-test-perfkit-ping - name: browbeat-test-perfkit-ping
@ -79,11 +76,3 @@ workloads:
progression: linear progression: linear
time: 60 time: 60
file: lib/python2.7/site-packages/shaker/scenarios/openstack/dense_l2.yaml file: lib/python2.7/site-packages/shaker/scenarios/openstack/dense_l2.yaml
- name: browbeat-test-introspect-batch
enabled: false
type: yoda
yoda_type: introspection
method: individual
times: 3
timeout: 900
batch_size: 2

3
tests/data/invalid_browbeat_workload.yml
View File

@ -51,9 +51,6 @@ shaker:
sleep_after: 0 sleep_after: 0
shaker_region: regionOne shaker_region: regionOne
external_host: 2.2.2.2 external_host: 2.2.2.2
yoda:
instackenv: "/home/stack/instackenv.json"
stackrc: "/home/stack/stackrc"
workloads: workloads:
# Missing name # Missing name

11
tests/data/valid_browbeat.yml
View File

@ -51,9 +51,6 @@ shaker:
sleep_after: 0 sleep_after: 0
shaker_region: regionOne shaker_region: regionOne
external_host: 2.2.2.2 external_host: 2.2.2.2
yoda:
instackenv: "/home/stack/instackenv.json"
stackrc: "/home/stack/stackrc"
workloads: workloads:
- name: browbeat-test-perfkit-ping - name: browbeat-test-perfkit-ping
@ -78,11 +75,3 @@ workloads:
progression: linear progression: linear
time: 60 time: 60
file: lib/python2.7/site-packages/shaker/scenarios/openstack/dense_l2.yaml file: lib/python2.7/site-packages/shaker/scenarios/openstack/dense_l2.yaml
- name: browbeat-test-introspect-batch
enabled: false
type: yoda
yoda_type: introspection
method: individual
times: 3
timeout: 900
batch_size: 2

21
tests/data/workloads.yml
View File

@ -64,24 +64,3 @@ shaker:
progression: linear progression: linear
time: 60 time: 60
opps_file: lib/python2.7/site-packages/shaker/scenarios/openstack/dense_l2.yaml opps_file: lib/python2.7/site-packages/shaker/scenarios/openstack/dense_l2.yaml
yoda:
- valid: true
data:
name: valid-test-yoda-introspection
enabled: false
type: yoda
yoda_type: introspection
method: individual
times: 3
timeout: 900
batch_size: 2
- valid: false
data:
name: invalid-test-yoda-introspection
enabled: false
type: yoda
fake_yoda_type: introspection
method: individual
times: 3
timeout: 900
batch_size: 2

2
tests/test_config.py
View File

@ -37,7 +37,7 @@ def test_load_browbeat_config(config):
assert "SchemaError" in str(exception_data) assert "SchemaError" in str(exception_data)
@pytest.mark.parametrize("schema", ["perfkit", "rally", "shaker", "yoda"]) @pytest.mark.parametrize("schema", ["perfkit", "rally", "shaker"])
def test__validate_yaml(schema): def test__validate_yaml(schema):
"""Tests valid and invalid Browbeat workload configurations.""" """Tests valid and invalid Browbeat workload configurations."""
with open("tests/data/workloads.yml", "r") as config_file: with open("tests/data/workloads.yml", "r") as config_file: