Start and stop collectd containers while running workloads

This patch introduces the following changes.
1. Playbooks have been created to start collectd on different hosts.
2. A feature has been added that allows a user to start collectd containers
before running workloads, and stop the collectd containers after running
the workloads. This will help us in minimising the space used for storing
collectd data.

Change-Id: I7926884f461e97bc67453f46eef0121c46c7f19e
This commit is contained in:
Sanjay Chari 2021-11-18 15:23:43 +05:30
parent 9a00692cfa
commit 8e873e678a
30 changed files with 384 additions and 0 deletions

View File

@ -0,0 +1,8 @@
---
- hosts: localhost
tasks:
- name: Check if graphite_host is empty in group_vars/all.yml
include_vars:
file: "group_vars/all.yml"
name: group_vars
failed_when: group_vars['graphite_host'] is none

View File

@ -0,0 +1,80 @@
---
#
# Playbook to start collectd on undercloud/overcloud
#
# Start on all undercloud/overcloud machines:
# ansible-playbook -i hosts.yml install/start-collectd-baremetal.yml
#
# Or use tags:
# ansible-playbook -i hosts.yml install/start-collectd-baremetal.yml --tag "undercloud"
# ansible-playbook -i hosts.yml install/start-collectd-baremetal.yml --tag "controller"
# ansible-playbook -i hosts.yml install/start-collectd-baremetal.yml --tag "networker"
# ansible-playbook -i hosts.yml install/start-collectd-baremetal.yml --tag "blockstorage"
# ansible-playbook -i hosts.yml install/start-collectd-baremetal.yml --tag "objectstorage"
# ansible-playbook -i hosts.yml install/start-collectd-baremetal.yml --tag "cephstorage"
# ansible-playbook -i hosts.yml install/start-collectd-baremetal.yml --tag "compute"
#
- hosts: Undercloud
remote_user: "{{ local_remote_user }}"
tasks:
- name: Start Collectd on undercloud
service: name=collectd state=started enabled=false
become: true
ignore_errors: true
tags: undercloud
- hosts: Controller
remote_user: "{{ host_remote_user }}"
tasks:
- name: Start Collectd on controller
service: name=collectd state=started enabled=false
become: true
ignore_errors: true
tags: controller
- hosts: Networker
remote_user: "{{ host_remote_user }}"
tasks:
- name: Start Collectd on networker
service: name=collectd state=started enabled=false
become: true
ignore_errors: true
tags: networker
- hosts: BlockStorage
remote_user: "{{ host_remote_user }}"
tasks:
- name: Start Collectd on blockstorage
service: name=collectd state=started enabled=false
become: true
ignore_errors: true
tags: blockstorage
- hosts: ObjectStorage
remote_user: "{{ host_remote_user }}"
tasks:
- name: Start Collectd on objectstorage
service: name=collectd state=started enabled=false
become: true
ignore_errors: true
tags: objectstorage
- hosts: CephStorage
remote_user: "{{ host_remote_user }}"
tasks:
- name: Start Collectd on cephstorage
service: name=collectd state=started enabled=false
become: true
ignore_errors: true
tags: cephstorage
- hosts: Compute
remote_user: "{{ host_remote_user }}"
tasks:
- name: Start Collectd on compute
service: name=collectd state=started enabled=false
become: true
ignore_errors: true
tags: compute

View File

@ -0,0 +1,122 @@
#
# Playbook to start collectd on undercloud/overcloud
#
# Start collectd on all undercloud/overcloud machines:
# ansible-playbook -i hosts.yaml install/start-collectd-container.yml
#
# Or use tags:
# ansible-playbook -i hosts.yml install/start-collectd-container.yml --tag "undercloud"
# ansible-playbook -i hosts.yml install/start-collectd-container.yml --tag "controller"
# ansible-playbook -i hosts.yml install/start-collectd-container.yml --tag "networker"
# ansible-playbook -i hosts.yml install/start-collectd-container.yml --tag "blockstorage"
# ansible-playbook -i hosts.yml install/start-collectd-container.yml --tag "objectstorage"
# ansible-playbook -i hosts.yml install/start-collectd-container.yml --tag "cephstorage"
# ansible-playbook -i hosts.yml install/start-collectd-container.yml --tag "compute"
#
- hosts: Undercloud
remote_user: "{{ local_remote_user }}"
vars:
config_type: undercloud
roles:
- { role: osp_version }
- { role: containers }
tasks:
- name: Start Collectd on {{ config_type }}
shell: |
{{ container_cli }} start collectd-{{ config_type }}
become: yes
ignore_errors: true
tags: undercloud
- hosts: Controller
remote_user: "{{ host_remote_user }}"
vars:
config_type: controller
roles:
- { role: osp_version }
- { role: containers }
tasks:
- name: Start Collectd on {{ config_type }}
shell: |
{{ container_cli }} start collectd-{{ config_type }}
become: yes
ignore_errors: true
tags: controller
- hosts: Networker
remote_user: "{{ host_remote_user }}"
vars:
config_type: networker
roles:
- { role: osp_version }
- { role: containers }
tasks:
- name: Start Collectd on {{ config_type }}
shell: |
{{ container_cli }} start collectd-{{ config_type }}
become: yes
ignore_errors: true
tags: networker
- hosts: BlockStorage
remote_user: "{{ host_remote_user }}"
vars:
config_type: blockstorage
roles:
- { role: osp_version }
- { role: containers }
tasks:
- name: Start Collectd on {{ config_type }}
shell: |
{{ container_cli }} start collectd-{{ config_type }}
become: yes
ignore_errors: true
tags: blockstorage
- hosts: ObjectStorage
remote_user: "{{ host_remote_user }}"
vars:
config_type: objectstorage
roles:
- { role: osp_version }
- { role: containers }
tasks:
- name: Start Collectd on {{ config_type }}
shell: |
{{ container_cli }} start collectd-{{ config_type }}
become: yes
ignore_errors: true
tags: objectstorage
- hosts: CephStorage
remote_user: "{{ host_remote_user }}"
vars:
config_type: cephstorage
roles:
- { role: osp_version }
- { role: containers }
tasks:
- name: Start Collectd on {{ config_type }}
shell: |
{{ container_cli }} start collectd-{{ config_type }}
become: yes
ignore_errors: true
tags: cephstorage
- hosts: Compute
remote_user: "{{ host_remote_user }}"
vars:
config_type: compute
roles:
- { role: osp_version }
- { role: containers }
tasks:
- name: Start Collectd on {{ config_type }}
shell: |
{{ container_cli }} start collectd-{{ config_type }}
become: yes
ignore_errors: true
tags: compute

View File

@ -0,0 +1,11 @@
---
- name: set fact collectd_container
import_playbook: pre-collectd.yml
- name: Start containerized collectd (Stein and greater recommended)
import_playbook: start-collectd-container.yml
when: hostvars['undercloud']['collectd_container']
- name: Start collectd installed through RPMs
import_playbook: start-collectd-baremetal.yml
when: not hostvars['undercloud']['collectd_container']

View File

@ -3,10 +3,20 @@ browbeat:
cloud_name: openstack
rerun: 3
rerun_type: iteration
# This option enables starting collectd before running workloads,
# and stopping collectd after running workloads. It should be enabled
# when it is required to store collectd data only when workloads
# are running. Please install collectd by running the command
# "cd ansible;ansible-playbook -i hosts.yml -vvv install/collectd.yml" before
# setting this option to true.
start_stop_collectd: false
ansible:
hosts: ansible/hosts
metadata_playbook: ansible/gather/stockpile.yml
logging_playbook: ansible/common_logging/browbeat_logging.yml
start_collectd_playbook: ansible/install/start-collectd.yml
stop_collectd_playbook: ansible/install/stop-collectd.yml
check_collectd_config_playbook: ansible/install/check-collectd-config.yml
ssh_config: ansible/ssh-config
elasticsearch:
enabled: true

View File

@ -7,10 +7,20 @@ browbeat:
# complete reruns after all workloads complete
# rerun_type: complete
rerun_type: iteration
# This option enables starting collectd before running workloads,
# and stopping collectd after running workloads. It should be enabled
# when it is required to store collectd data only when workloads
# are running. Please install collectd by running the command
# "cd ansible;ansible-playbook -i hosts.yml -vvv install/collectd.yml" before
# setting this option to true.
start_stop_collectd: false
ansible:
hosts: ansible/hosts.yml
metadata_playbook: ansible/gather/stockpile.yml
logging_playbook: ansible/common_logging/browbeat_logging.yml
start_collectd_playbook: ansible/install/start-collectd.yml
stop_collectd_playbook: ansible/install/stop-collectd.yml
check_collectd_config_playbook: ansible/install/check-collectd-config.yml
ssh_config: ansible/ssh-config
elasticsearch:
enabled: false

View File

@ -182,6 +182,10 @@ def main():
" with browbeat_uuid {}".format(browbeat_uuid))
tools.common_logging(browbeat_uuid, logging_status=True)
if _config['browbeat']['start_stop_collectd'] and tools.check_collectd_config():
_logger.info("Starting collectd")
tools.start_collectd()
_logger.info("Running workload(s): {}".format(','.join(_cli_args.workloads)))
# Iteration rerun_type pushes rerun logic down to the workload itself. This allows the workload
@ -191,6 +195,9 @@ def main():
elif _config["browbeat"]["rerun_type"] == "complete":
# Complete rerun_type, reruns after all workloads have been run.
run_complete(_config, _cli_args, result_dir_ts, _logger, tools)
if _config["browbeat"]["start_stop_collectd"] and tools.check_collectd_config():
_logger.info("Stopping collectd")
tools.stop_collectd()
if terminate:
_logger.info("Browbeat execution halting due to user intervention")
sys.exit(1)

View File

@ -19,6 +19,9 @@ mapping:
type: str
enum: ['iteration', 'complete']
required: True
start_stop_collectd:
type: bool
required: True
ansible:
required: True
type: map
@ -32,6 +35,15 @@ mapping:
logging_playbook:
type: str
required: True
start_collectd_playbook:
type: str
required: True
stop_collectd_playbook:
type: str
required: True
check_collectd_config_playbook:
type: str
required: True
ssh_config:
type: str
required: True

View File

@ -92,6 +92,46 @@ class Tools(object):
self.logger.info("Metadata about cloud has been gathered")
return True
def check_collectd_config(self):
ansible_cmd = \
'ansible-playbook {}' \
.format(self.config['ansible']['check_collectd_config_playbook'])
returncode = self.run_cmd(ansible_cmd)['rc']
if returncode > 0:
self.logger.warning("""graphite_host is empty in all.yml. Please fill it and run the command
(cd ansible;ansible-playbook -i hosts.yml -vvv install/collectd.yml)
in order to install collectd.""")
return False
else:
self.logger.info("Collectd config in all.yml validated")
return True
def start_collectd(self):
ansible_cmd = \
'ansible-playbook -i {} {}' \
.format(self.config['ansible']['hosts'],
self.config['ansible']['start_collectd_playbook'])
returncode = self.run_cmd(ansible_cmd)['rc']
if returncode > 0:
self.logger.warning("Collectd could not be started")
return False
else:
self.logger.info("Collectd started successfully")
return True
def stop_collectd(self):
ansible_cmd = \
'ansible-playbook -i {} {}' \
.format(self.config['ansible']['hosts'],
self.config['ansible']['stop_collectd_playbook'])
returncode = self.run_cmd(ansible_cmd)['rc']
if returncode > 0:
self.logger.warning("Collectd could not be stopped")
return False
else:
self.logger.info("Collectd stopped successfully")
return True
def common_logging(self, browbeat_uuid, logging_status):
os.putenv("ANSIBLE_SSH_ARGS", " -F {}".format(self.config['ansible']['ssh_config']))
ansible_cmd = \

View File

@ -3,10 +3,14 @@ browbeat:
cloud_name: openstack
rerun: 1
rerun_type: iteration
start_stop_collectd: false
ansible:
hosts: ansible/hosts
metadata_playbook: ansible/gather/stockpile.yml
logging_playbook: ansible/common_logging/browbeat_logging.yml
start_collectd_playbook: ansible/install/start-collectd.yml
stop_collectd_playbook: ansible/install/stop-collectd.yml
check_collectd_config_playbook: ansible/install/check-collectd-config.yml
ssh_config: ansible/ssh-config
elasticsearch:
enabled: false

View File

@ -4,10 +4,14 @@ browbeat:
cloud_name: openstack
rerun: 3
rerun_type: iteration
start_stop_collectd: false
ansible:
hosts: ansible/hosts
metadata_playbook: ansible/gather/stockpile.yml
logging_playbook: ansible/common_logging/browbeat_logging.yml
start_collectd_playbook: ansible/install/start-collectd.yml
stop_collectd_playbook: ansible/install/stop-collectd.yml
check_collectd_config_playbook: ansible/install/check-collectd-config.yml
ssh_config: ansible/ssh-config
elasticsearch:
enabled: false

View File

@ -4,10 +4,14 @@ browbeat:
cloud_name: openstack
rerun: 1
rerun_type: iteration
start_stop_collectd: false
ansible:
hosts: ansible/hosts
metadata_playbook: ansible/gather/stockpile.yml
logging_playbook: ansible/common_logging/browbeat_logging.yml
start_collectd_playbook: ansible/install/start-collectd.yml
stop_collectd_playbook: ansible/install/stop-collectd.yml
check_collectd_config_playbook: ansible/install/check-collectd-config.yml
ssh_config: ansible/ssh-config
elasticsearch:
enabled: false

View File

@ -3,10 +3,14 @@ browbeat:
cloud_name: openstack
rerun: 1
rerun_type: iteration
start_stop_collectd: false
ansible:
hosts: ansible/hosts
metadata_playbook: ansible/gather/stockpile.yml
logging_playbook: ansible/common_logging/browbeat_logging.yml
start_collectd_playbook: ansible/install/collectd.yml
stop_collectd_playbook: ansible/install/stop-collectd.yml
check_collectd_config_playbook: ansible/install/check-collectd-config.yml
ssh_config: ansible/ssh-config
elasticsearch:
enabled: false

View File

@ -3,10 +3,14 @@ browbeat:
cloud_name: openstack
rerun: 1
rerun_type: iteration
start_stop_collectd: false
ansible:
hosts: ansible/hosts
metadata_playbook: ansible/gather/stockpile.yml
logging_playbook: ansible/common_logging/browbeat_logging.yml
start_collectd_playbook: ansible/install/start-collectd.yml
stop_collectd_playbook: ansible/install/stop-collectd.yml
check_collectd_config_playbook: ansible/install/check-collectd-config.yml
ssh_config: ansible/ssh-config
elasticsearch:
enabled: false

View File

@ -6,10 +6,14 @@ browbeat:
cloud_name: openstack
rerun: 10
rerun_type: complete
start_stop_collectd: false
ansible:
hosts: ansible/hosts
metadata_playbook: ansible/gather/stockpile.yml
logging_playbook: ansible/common_logging/browbeat_logging.yml
start_collectd_playbook: ansible/install/start-collectd.yml
stop_collectd_playbook: ansible/install/stop-collectd.yml
check_collectd_config_playbook: ansible/install/check-collectd-config.yml
ssh_config: ansible/ssh-config
elasticsearch:
enabled: false

View File

@ -4,10 +4,14 @@ browbeat:
cloud_name: openstack
rerun: 1
rerun_type: iteration
start_stop_collectd: false
ansible:
hosts: ansible/hosts
metadata_playbook: ansible/gather/stockpile.yml
logging_playbook: ansible/common_logging/browbeat_logging.yml
start_collectd_playbook: ansible/install/start-collectd.yml
stop_collectd_playbook: ansible/install/stop-collectd.yml
check_collectd_config_playbook: ansible/install/check-collectd-config.yml
ssh_config: ansible/ssh-config
elasticsearch:
enabled: false

View File

@ -3,10 +3,14 @@ browbeat:
cloud_name: quickstart
rerun: 1
rerun_type: iteration
start_stop_collectd: false
ansible:
hosts: ansible/hosts
metadata_playbook: ansible/gather/stockpile.yml
logging_playbook: ansible/common_logging/browbeat_logging.yml
start_collectd_playbook: ansible/install/collectd.yml
stop_collectd_playbook: ansible/install/stop-collectd.yml
check_collectd_config_playbook: ansible/install/check-collectd-config.yml
ssh_config: ansible/ssh-config
elasticsearch:
enabled: false

View File

@ -10,10 +10,14 @@ browbeat:
cloud_name: openstack
rerun: 10
rerun_type: complete
start_stop_collectd: false
ansible:
hosts: ansible/hosts
metadata_playbook: ansible/gather/stockpile.yml
logging_playbook: ansible/common_logging/browbeat_logging.yml
start_collectd_playbook: ansible/install/start-collectd.yml
stop_collectd_playbook: ansible/install/stop-collectd.yml
check_collectd_config_playbook: ansible/install/check-collectd-config.yml
ssh_config: ansible/ssh-config
elasticsearch:
enabled: false

View File

@ -10,10 +10,14 @@ browbeat:
cloud_name: openstack
rerun: 10
rerun_type: complete
start_stop_collectd: false
ansible:
hosts: ansible/hosts
metadata_playbook: ansible/gather/stockpile.yml
logging_playbook: ansible/common_logging/browbeat_logging.yml
start_collectd_playbook: ansible/install/start-collectd.yml
stop_collectd_playbook: ansible/install/stop-collectd.yml
check_collectd_config_playbook: ansible/install/check-collectd-config.yml
ssh_config: ansible/ssh-config
elasticsearch:
enabled: false

View File

@ -10,10 +10,14 @@ browbeat:
cloud_name: openstack
rerun: 2
rerun_type: complete
start_stop_collectd: false
ansible:
hosts: ansible/hosts
metadata_playbook: ansible/gather/stockpile.yml
logging_playbook: ansible/common_logging/browbeat_logging.yml
start_collectd_playbook: ansible/install/start-collectd.yml
stop_collectd_playbook: ansible/install/stop-collectd.yml
check_collectd_config_playbook: ansible/install/check-collectd-config.yml
ssh_config: ansible/ssh-config
elasticsearch:
enabled: false

View File

@ -10,10 +10,14 @@ browbeat:
cloud_name: openstack
rerun: 10
rerun_type: complete
start_stop_collectd: false
ansible:
hosts: ansible/hosts
metadata_playbook: ansible/gather/stockpile.yml
logging_playbook: ansible/common_logging/browbeat_logging.yml
start_collectd_playbook: ansible/install/start-collectd.yml
stop_collectd_playbook: ansible/install/stop-collectd.yml
check_collectd_config_playbook: ansible/install/check-collectd-config.yml
ssh_config: ansible/ssh-config
elasticsearch:
enabled: false

View File

@ -7,10 +7,14 @@ browbeat:
cloud_name: openstack
rerun: 1
rerun_type: iteration
start_stop_collectd: false
ansible:
hosts: ansible/hosts
metadata_playbook: ansible/gather/stockpile.yml
logging_playbook: ansible/common_logging/browbeat_logging.yml
start_collectd_playbook: ansible/install/start-collectd.yml
stop_collectd_playbook: ansible/install/stop-collectd.yml
check_collectd_config_playbook: ansible/install/check-collectd-config.yml
ssh_config: ansible/ssh-config
elasticsearch:
enabled: false

View File

@ -7,10 +7,14 @@ browbeat:
cloud_name: openstack
rerun: 1
rerun_type: iteration
start_stop_collectd: false
ansible:
hosts: ansible/hosts
metadata_playbook: ansible/gather/stockpile.yml
logging_playbook: ansible/common_logging/browbeat_logging.yml
start_collectd_playbook: ansible/install/start-collectd.yml
stop_collectd_playbook: ansible/install/stop-collectd.yml
check_collectd_config_playbook: ansible/install/check-collectd-config.yml
ssh_config: ansible/ssh-config
elasticsearch:
enabled: false

View File

@ -6,10 +6,14 @@ browbeat:
cloud_name: openstack
rerun: 1
rerun_type: iteration
start_stop_collectd: false
ansible:
hosts: ansible/hosts
metadata_playbook: ansible/gather/stockpile.yml
logging_playbook: ansible/common_logging/browbeat_logging.yml
start_collectd_playbook: ansible/install/start-collectd.yml
stop_collectd_playbook: ansible/install/stop-collectd.yml
check_collectd_config_playbook: ansible/install/check-collectd-config.yml
ssh_config: ansible/ssh-config
elasticsearch:
enabled: false

View File

@ -8,10 +8,14 @@ browbeat:
cloud_name: openstack
rerun: 1
rerun_type: iteration
start_stop_collectd: false
ansible:
hosts: ansible/hosts
metadata_playbook: ansible/gather/stockpile.yml
logging_playbook: ansible/common_logging/browbeat_logging.yml
start_collectd_playbook: ansible/install/start-collectd.yml
stop_collectd_playbook: ansible/install/stop-collectd.yml
check_collectd_config_playbook: ansible/install/check-collectd-config.yml
ssh_config: ansible/ssh-config
elasticsearch:
enabled: false

View File

@ -6,10 +6,14 @@ browbeat:
cloud_name: openstack
rerun: 1
rerun_type: iteration
start_stop_collectd: false
ansible:
hosts: ansible/hosts
metadata_playbook: ansible/gather/stockpile.yml
logging_playbook: ansible/common_logging/browbeat_logging.yml
start_collectd_playbook: ansible/install/start-collectd.yml
stop_collectd_playbook: ansible/install/stop-collectd.yml
check_collectd_config_playbook: ansible/install/check-collectd-config.yml
ssh_config: ansible/ssh-config
elasticsearch:
enabled: false

View File

@ -7,10 +7,14 @@ browbeat:
cloud_name: openstack
rerun: 1
rerun_type: iteration
start_stop_collectd: false
ansible:
hosts: ansible/hosts
metadata_playbook: ansible/gather/stockpile.yml
logging_playbook: ansible/common_logging/browbeat_logging.yml
start_collectd_playbook: ansible/install/start-collectd.yml
stop_collectd_playbook: ansible/install/stop-collectd.yml
check_collectd_config_playbook: ansible/install/check-collectd-config.yml
ssh_config: ansible/ssh-config
elasticsearch:
enabled: false

View File

@ -4,10 +4,14 @@ browbeat:
rerun: 1
rerun_type: complete
invalid_flag: invalid
start_stop_collectd: false
ansible:
hosts: ansible/hosts
metadata_playbook: ansible/gather/site.yml
logging_playbook: ansible/common_logging/browbeat_logging.yml
start_collectd_playbook: ansible/install/start-collectd.yml
stop_collectd_playbook: ansible/install/stop-collectd.yml
check_collectd_config_playbook: ansible/install/check-collectd-config.yml
ssh_config: ansible/ssh-config
elasticsearch:
enabled: false

View File

@ -3,10 +3,14 @@ browbeat:
cloud_name: browbeat-test
rerun: 1
rerun_type: complete
start_stop_collectd: false
ansible:
hosts: ansible/hosts
metadata_playbook: ansible/gather/site.yml
logging_playbook: ansible/common_logging/browbeat_logging.yml
start_collectd_playbook: ansible/install/start-collectd.yml
stop_collectd_playbook: ansible/install/stop-collectd.yml
check_collectd_config_playbook: ansible/install/check-collectd-config.yml
ssh_config: ansible/ssh-config
elasticsearch:
enabled: false

View File

@ -3,10 +3,14 @@ browbeat:
cloud_name: browbeat-test
rerun: 1
rerun_type: complete
start_stop_collectd: false
ansible:
hosts: ansible/hosts
metadata_playbook: ansible/gather/site.yml
logging_playbook: ansible/common_logging/browbeat_logging.yml
start_collectd_playbook: ansible/install/start-collectd.yml
stop_collectd_playbook: ansible/install/stop-collectd.yml
check_collectd_config_playbook: ansible/install/check-collectd-config.yml
ssh_config: ansible/ssh-config
elasticsearch:
enabled: false