diff --git a/ansible/oooq/baremetal-virt-undercloud-int-browbeat.yml b/ansible/oooq/baremetal-virt-undercloud-int-browbeat.yml new file mode 100644 index 000000000..7a6b0affd --- /dev/null +++ b/ansible/oooq/baremetal-virt-undercloud-int-browbeat.yml @@ -0,0 +1,24 @@ +--- +# Browbeat integration test + +- include: baremetal-prep-virthost.yml + +- include: configure-browbeat.yml + +- include: undercloud-collectd.yml + +- include: baremetal-quickstart-extras.yml + +- include: install-browbeat.yml + +- include: disable-ssh-dns.yml + +- include: overcloud-collectd.yml + +- include: gather-metadata.yml + +- name: Run Browbeat + hosts: undercloud + roles: + - browbeat/browbeat-run + - browbeat/browbeat-classify diff --git a/ansible/oooq/baremetal-virt-undercloud-yoda-browbeat.yml b/ansible/oooq/baremetal-virt-undercloud-yoda-browbeat.yml new file mode 100644 index 000000000..761243ed1 --- /dev/null +++ b/ansible/oooq/baremetal-virt-undercloud-yoda-browbeat.yml @@ -0,0 +1,88 @@ +--- +- include: baremetal-prep-virthost.yml + +- name: Validate IPMI and instackenv.json + hosts: undercloud + gather_facts: yes + roles: + - validate-ipmi + +- name: Deploy the undercloud + hosts: undercloud + gather_facts: no + roles: + - undercloud-deploy + +- name: copy over config files + hosts: undercloud + gather_facts: no + roles: + - overcloud-prep-config + +- name: Perpare the baremetal overcloud + hosts: undercloud + gather_facts: yes + roles: + - baremetal-prep-overcloud + +- name: Prepare overcloud containers + hosts: undercloud + gather_facts: no + roles: + - { role: overcloud-prep-containers, when: containerized_overcloud|bool } + +- name: Prepare the overcloud images for deployment + hosts: undercloud + gather_facts: no + roles: + - overcloud-prep-images + +- name: Prepare overcloud flavors + hosts: undercloud + gather_facts: no + roles: + - overcloud-prep-flavors + +- name: Prepare the undercloud networks for the overcloud deployment + hosts: undercloud + gather_facts: no + roles: + - overcloud-prep-network + +- name: Prepare the SSL Configuration for the overcloud deployment + hosts: undercloud + gather_facts: no + roles: + - { role: overcloud-ssl, when: ssl_overcloud|bool } + +- name: Configure Browbeat + hosts: undercloud + roles: + - browbeat/pre-install-setup + - browbeat/oooq-metadata + +- name: Setup Undercloud CollectD + hosts: undercloud + vars: + config_type: "{{group_names[0]}}" + roles: + - browbeat/common + - browbeat/epel + - browbeat/collectd-openstack + +- name: Install Browbeat + hosts: undercloud + vars: + results_in_httpd: false + statsd_host: "{{ graphite_host }}" + roles: + - browbeat/common + - browbeat/browbeat + - browbeat/yoda + - browbeat/template-configs + - browbeat/statsd-ironic + +- name: Run Browbeat + hosts: undercloud + roles: + - browbeat/browbeat-run diff --git a/ansible/oooq/roles/oooq-metadata/tasks/main.yml b/ansible/oooq/roles/oooq-metadata/tasks/main.yml index 33eb7a630..591b82b3e 100644 --- a/ansible/oooq/roles/oooq-metadata/tasks/main.yml +++ b/ansible/oooq/roles/oooq-metadata/tasks/main.yml @@ -10,6 +10,7 @@ shell: docker ps | wc -l register: docker_ps delegate_to: overcloud-controller-0 + when: "'overcloud' in group_names" ignore_errors: True - name: Set var for container deployment @@ -24,6 +25,10 @@ when: docker_ps.stdout|int < 2 ignore_errors: True +- name: Count nodes in Instackenv.json + shell: "grep pm_addr {{instackenv}} | wc -l" + register: num_nodes + - name: Make sure the results directory exists file: "path={{ ansible_env.HOME }}/browbeat/metadata state=directory" diff --git a/ansible/oooq/roles/oooq-metadata/vars/main.yml b/ansible/oooq/roles/oooq-metadata/vars/main.yml index 7e58fa274..4a2a238be 100644 --- a/ansible/oooq/roles/oooq-metadata/vars/main.yml +++ b/ansible/oooq/roles/oooq-metadata/vars/main.yml @@ -1,3 +1,4 @@ dlrn_hash: "Not a pipeline build" rhos_puddle: "Not a pipeline build" logs_link: "https://thirdparty.logs.rdoproject.org/jenkins-{{ lookup('env','JOB_NAME') }}-{{ lookup('env','BUILD_NUMBER') }}/" +instackenv: "/home/stack/instackenv.json" diff --git a/ansible/oooq/roles/template-configs/templates/browbeat-yoda-ci.yaml.j2 b/ansible/oooq/roles/template-configs/templates/browbeat-yoda-ci.yaml.j2 new file mode 100644 index 000000000..4e1456ec4 --- /dev/null +++ b/ansible/oooq/roles/template-configs/templates/browbeat-yoda-ci.yaml.j2 @@ -0,0 +1,147 @@ +# Tests to be compleated for the install-and-check.sh script minimal and short workloads are performed +# to confirm functionality. +browbeat: + results : results/ + rerun: 1 + cloud_name: {{ browbeat_cloud_name }} +elasticsearch: + enabled: {{ elastic_enabled }} + host: {{ elastic_host }} + port: 9200 + regather: false + metadata_files: + - name: hardware-metadata + file: metadata/hardware-metadata.json + - name: environment-metadata + file: metadata/environment-metadata.json + - name: software-metadata + file: metadata/software-metadata.json + - name: version + file: metadata/version.json +ansible: + ssh_config: ansible/ssh-config + hosts: ansible/hosts + adjust: + keystone_token: ansible/browbeat/adjustment-keystone-token.yml + neutron_l3: ansible/browbeat/adjustment-l3.yml + nova_db: ansible/browbeat/adjustment-db.yml + workers: ansible/browbeat/adjustment-workers.yml + grafana_snapshot: ansible/browbeat/snapshot-general-performance-dashboard.yml + metadata: ansible/gather/site.yml +connmon: + enabled: {{ connmon_enabled }} + sudo: true +grafana: + enabled: {{ grafana_enabled }} + grafana_ip: {{ grafana_host }} + grafana_port: 3000 + dashboards: + - openstack-general-system-performance + snapshot: + enabled: false + snapshot_compute: false +yoda: + enabled: true + instackenv: "/home/stack/instackenv.json" + stackrc: "/home/stack/stackrc" + venv: "/home/stack/yoda-venv/bin/activate" + benchmarks: + - name: introspect-{{ overcloud_size }}-batch-5 + type: introspection + enabled: true + method: individual + times: 10 + timeout: 1800 + batch_size: 5 + - name: introspect-{{ overcloud_size }}-bulk + type: introspection + enabled: true + method: bulk + times: 10 + timeout: 1800 + - name: No-HA-Max-Compute-{{ overcloud_size }}-full-deploy + type: overcloud + ntp_server: pool.ntp.org + timeout: 600 + templates: + - "" + enabled: true + step: 5 + keep_stack: false + times: 2 + cloud: + - node: "compute" + start_scale: 1 + end_scale: {{ overcloud_size | int - 1 }} + - node: "control" + start_scale: 1 + end_scale: 1 + - name: No-HA-Max-Compute-{{ overcloud_size }}-stack-update + type: overcloud + ntp_server: pool.ntp.org + timeout: 600 + templates: + - "" + instackenv: "/home/stack/instackenv.json" + enabled: true + step: 5 + keep_stack: true + times: 2 + cloud: + - node: "compute" + start_scale: 1 + end_scale: {{ overcloud_size | int - 1 }} + - node: "control" + start_scale: 1 + end_scale: 1 + - name: HA-Max-Compute-{{ overcloud_size }}-full-deploy + type: overcloud + ntp_server: pool.ntp.org + timeout: 600 + templates: + - "" + enabled: true + step: 5 + keep_stack: false + times: 1 + cloud: + - node: "compute" + start_scale: 1 + end_scale: {{ overcloud_size | int - 3 }} + - node: "control" + start_scale: 3 + end_scale: 3 + - name: HA-Max-Compute-{{ overcloud_size }}-stack-update + type: overcloud + ntp_server: pool.ntp.org + timeout: 600 + templates: + - "" + enabled: true + step: 5 + keep_stack: true + times: 1 + cloud: + - node: "compute" + start_scale: 1 + end_scale: {{ overcloud_size | int - 3 }} + - node: "control" + start_scale: 3 + end_scale: 3 + - name: HA-Max-Compute-{{ overcloud_size }}-stack-update + type: overcloud + ntp_server: pool.ntp.org + timeout: 600 + templates: + - "" + enabled: true + step: 5 + keep_stack: true + times: 1 + cloud: + - node: "compute" + start_scale: 1 + end_scale: {{ overcloud_size | int - 3 }} + - node: "control" + start_scale: 3 + end_scale: 3 diff --git a/ansible/oooq/roles/template-configs/vars/main.yml b/ansible/oooq/roles/template-configs/vars/main.yml index 73929f4bc..b9fcd6010 100644 --- a/ansible/oooq/roles/template-configs/vars/main.yml +++ b/ansible/oooq/roles/template-configs/vars/main.yml @@ -5,5 +5,5 @@ grafana_enabled: false grafana_host: "1.2.3.4.5" browbeat_config_file: "browbeat-basic.yaml.j2" browbeat_cloud_name: "browbeat_ci" -overcloud_size: "{{ groups['overcloud'] | length }}" +overcloud_size: "{{num_nodes.stdout|int}}" ntp_server: "pool.ntp.org" diff --git a/ci-scripts/tripleo/microbrow.sh b/ci-scripts/tripleo/microbrow.sh index 8fa015b1e..51ab66af9 100755 --- a/ci-scripts/tripleo/microbrow.sh +++ b/ci-scripts/tripleo/microbrow.sh @@ -10,7 +10,6 @@ export OPT_DEBUG_ANSIBLE=0 export HW_ENV_DIR=$WORKSPACE/tripleo-environments/hardware_environments/$HW_ENV export NETWORK_ISOLATION=no_vlan export REQS=quickstart-extras-requirements.txt -export PLAYBOOK=baremetal-virt-undercloud-tripleo-browbeat.yml export VARS="elastic_enabled=true \ --extra-vars grafana_enabled=true \ --extra-vars elastic_host=$ELASTIC_HOST \ diff --git a/lib/Yoda.py b/lib/Yoda.py index 75b7913ed..0954fe6c5 100644 --- a/lib/Yoda.py +++ b/lib/Yoda.py @@ -227,8 +227,13 @@ class Yoda(WorkloadBase.WorkloadBase): while len(nodes): node = nodes.pop() # rate limit - time.sleep(1) - node_obj = conn.bare_metal.get_node(node) + time.sleep(10) + try: + node_obj = conn.bare_metal.get_node(node) + except exceptions.SDKException: + self.logger.error("Ironic endpoint is down, retrying in 10 seconds") + time.sleep(10) + continue if node_obj is None: self.logger.error("Can't find node " + node + " Which existed at the start of introspection \ @@ -311,15 +316,21 @@ class Yoda(WorkloadBase.WorkloadBase): wait_time = 0 # 30 minute timeout timeout = (60 * 30) - while conn.orchestration.find_stack("overcloud") is not None: - # Deletes can fail, so we just try again - if wait_time % 2000 == 0: - conn.orchestration.delete_stack("overcloud") - time.sleep(5) - wait_time += 5 - if wait_time > timeout: - self.logger.error("Overcloud stack delete failed") + try: + while conn.orchestration.find_stack("overcloud") is not None: + # Deletes can fail, so we just try again + if wait_time % 2000 == 0: + conn.orchestration.delete_stack("overcloud") + time.sleep(10) + wait_time += 10 + if wait_time > timeout: + self.logger.error("Overcloud stack delete failed") exit(1) + except exceptions.SDKException: + # Recursion is probably the wrong way to handle this + self.logger.error("Heat failure during overcloud delete, retrying") + time.sleep(10) + self.delete_stack(conn) def setup_nodes_dict(self, benchmark): nodes = {} @@ -561,9 +572,17 @@ class Yoda(WorkloadBase.WorkloadBase): benchmark) results['total_time'] = (datetime.datetime.utcnow() - start_time).total_seconds() + try: + stack_status = conn.orchestration.find_stack("overcloud") + except exceptions.SDKException: + self.logger.error("Heat endpoint failed to respond, waiting 10 seconds") + time.sleep(10) + continue + if stack_status is None: + continue + results['result'] = str(stack_status.status) + results['result_reason'] = str(stack_status.status_reason) - results['result'] = str(conn.orchestration.find_stack("overcloud").status) - results['result_reason'] = str(conn.orchestration.find_stack("overcloud").status_reason) results['total_nodes'] = len(list(map(lambda node: node.id, conn.bare_metal.nodes()))) if "COMPLETE" in results['result']: self.update_pass_tests() @@ -589,8 +608,8 @@ class Yoda(WorkloadBase.WorkloadBase): self.logger.debug("Time Stamp (Prefix): {}".format(dir_ts)) stackrc = self.config.get('yoda')['stackrc'] - venv = self.config.get('yoda')['venv'] - env_setup = "source {}; source {};".format(stackrc,venv) + # venv = self.config.get('yoda')['venv'] + env_setup = "source {};".format(stackrc) auth_vars = self.tools.load_stackrc(stackrc) if 'OS_AUTH_URL' not in auth_vars: @@ -619,7 +638,6 @@ class Yoda(WorkloadBase.WorkloadBase): benchmark['instackenv'] = instackenv for rerun in range(self.config['browbeat']['rerun']): for run in range(times): - self.update_tests() if benchmark['type'] == "overcloud": self.overcloud_workload(benchmark, run,