From 77612f7eed0ca3176327337b71cdb0a0613a2608 Mon Sep 17 00:00:00 2001 From: Jeffrey Zhang Date: Fri, 15 Dec 2017 16:17:27 +0800 Subject: [PATCH] Add collectd in ci to collect system information Change-Id: Ic5cbdfd4ebca1d907ada5c39f27dc4a7889daa21 --- roles/collect-collectd/files/rrdtool_graph.py | 171 ++++++++++++++++++ roles/collect-collectd/tasks/main.yml | 68 +++++++ .../collect-collectd/templates/graph.html.j2 | 16 ++ roles/collectd/defaults/main.yml | 14 ++ roles/collectd/handlers/main.yml | 5 + roles/collectd/tasks/main.yml | 46 +++++ roles/collectd/templates/collectd.conf.j2 | 27 +++ tests/playbooks/post.yml | 2 + tests/playbooks/pre.yml | 3 +- 9 files changed, 351 insertions(+), 1 deletion(-) create mode 100644 roles/collect-collectd/files/rrdtool_graph.py create mode 100644 roles/collect-collectd/tasks/main.yml create mode 100644 roles/collect-collectd/templates/graph.html.j2 create mode 100644 roles/collectd/defaults/main.yml create mode 100644 roles/collectd/handlers/main.yml create mode 100644 roles/collectd/tasks/main.yml create mode 100644 roles/collectd/templates/collectd.conf.j2 diff --git a/roles/collect-collectd/files/rrdtool_graph.py b/roles/collect-collectd/files/rrdtool_graph.py new file mode 100644 index 0000000000..c5166a66b0 --- /dev/null +++ b/roles/collect-collectd/files/rrdtool_graph.py @@ -0,0 +1,171 @@ +#!/usr/bin/python2 +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +import argparse +import glob +import logging +import os +import re +import sys + +import rrdtool + + +LOG = logging.getLogger('root') + +COLORS = ( + '#0000ff', + '#00ff00', + '#ff0000', + '#00ffff', + '#ffff00', + '#ff00ff' +) + + +def colorgen(): + for color in COLORS: + yield color + + +def get_uptime_seconds(): + with open('/proc/uptime') as f: + return f.read().split(' ')[0] + + +class LoadGraph(object): + + DEF = 'DEF:%(vname)s=%(filename)s:%(ds)s:%(CF)s' + LINE = 'LINE1:%(vname)s%(color)s:%(vname)s' + AREA = 'AREA:%(vname)s%(color)s:%(vname)s' + + def __init__(self, conf, file, color): + self.conf = conf + self.color = colorgen() + self.file = file + self.info = RRDInfo(file) + self.color = color + self.filename = os.path.basename(self.file) + + def _gen_line(self, vname, ds, color): + params = dict(vname=vname, + ds=ds, + filename=self.file, + color=color, + CF=self.conf.cf) + ret = [] + ret.append(self.DEF % params) + draw = self.LINE + if self.conf.area: + draw = self.AREA + if self.conf.stack: + draw += ':STACK' + ret.append(draw % params) + LOG.info('Get params for ds: %s', ret) + return ret + + def gen_params(self): + + ret = [] + for DS in self.info.data_sources(): + filename, ext = os.path.splitext(self.filename) + vname = '%s-%s' % (filename, DS) + ret.extend(self._gen_line(vname, DS, next(self.color))) + return ret + + def graph(self): + params = self.common_params() + params += self.get_period() + params.extend(self.gen_params()) + return rrdtool.graphv(*params) + + +class RRDInfo(object): + + def __init__(self, file): + self.info = rrdtool.info(file) + + @property + def last_update(self): + return self.info['last_update'] + + def data_sources(self): + dss = set() + for key in self.info: + ds = re.findall('ds\[(\w*)', key) + if ds: + dss.add(ds[0]) + return dss + + +def main(): + logging.basicConfig(level=logging.DEBUG) + + parser = argparse.ArgumentParser() + parser.add_argument('--file', nargs='+') + parser.add_argument('--type') + parser.add_argument('--height', default='256') + parser.add_argument('--width', default='1024') + parser.add_argument('--output', default='a.png') + + parser.add_argument('--stack', action='store_true') + parser.add_argument('--area', action='store_true') + parser.add_argument('--cf', default='MAX') + + parser.add_argument('--start', default=None) + parser.add_argument('--end', default=None) + + conf = parser.parse_args(sys.argv[1:]) + + last_update = None + + rrd_definitions = [] + + color = colorgen() + + rrd_files = [] + for f in conf.file: + f = os.path.expanduser(f) + rrd_files.extend(glob.glob(f)) + LOG.info('Get rrdfiles: %s', rrd_files) + + for file in rrd_files: + rrd_definitions.extend(LoadGraph(conf, file, color=color).gen_params()) + if not last_update: + last_update = RRDInfo(file).last_update + else: + last_update = min(RRDInfo(file).last_update, last_update) + + start = conf.start + end = conf.end + if not start: + start = 'end-' + get_uptime_seconds() + if not end: + end = str(last_update + 60) + + params = [ + conf.output, + '--zoom', '4', + '--width', conf.width, + '--height', conf.height, + '--start', start, + '--end', end + ] + params.extend(rrd_definitions) + LOG.debug('params: %s', params) + + rrdtool.graph(*params) + + +if __name__ == "__main__": + main() diff --git a/roles/collect-collectd/tasks/main.yml b/roles/collect-collectd/tasks/main.yml new file mode 100644 index 0000000000..779034b451 --- /dev/null +++ b/roles/collect-collectd/tasks/main.yml @@ -0,0 +1,68 @@ +--- +- name: copy rrdtool_graph file + copy: + src: rrdtool_graph.py + dest: /usr/local/bin/rrdtool_graph + mode: 0755 + become: true + +- name: stop collectd service + service: + name: collectd + state: stopped + become: true + +- name: generate image folder + file: + path: "/tmp/rrd_graph" + state: directory + become: true + +- name: generate load graph + command: "/usr/local/bin/rrdtool_graph --file /var/lib/collectd/rrd/{{ ansible_fqdn }}/load/load.rrd --output /tmp/rrd_graph/load.png" + ignore_errors: true + become: true + +- name: generate memory graph + command: "/usr/local/bin/rrdtool_graph --file /var/lib/collectd/rrd/{{ ansible_fqdn }}/memory/*.rrd --stack --area --output /tmp/rrd_graph/memory.png" + ignore_errors: true + become: true + +- name: generate interface graph + command: "/usr/local/bin/rrdtool_graph --file /var/lib/collectd/rrd/{{ ansible_fqdn }}/interface-{{ ansible_default_ipv4.interface }}/if_packets.rrd --output /tmp/rrd_graph/interface-{{ ansible_default_ipv4.interface }}.png" + ignore_errors: true + become: true + +- name: generate graph.html + vars: + graphs: + - load.png + - memory.png + - "interface-{{ ansible_default_ipv4.interface }}.png" + template: + src: graph.html.j2 + dest: /tmp/rrd_graph/graph.html + ignore_errors: true + become: true + +- name: copy graph files + synchronize: + src: "/tmp/rrd_graph" + dest: "{{ zuul.executor.log_root }}" + mode: pull + copy_links: true + verify_host: true + +- name: copy collectd rrd files + synchronize: + src: "/var/lib/collectd/rrd" + dest: "{{ zuul.executor.log_root }}" + mode: pull + copy_links: true + verify_host: true + +- name: start collectd service + service: + name: collectd + state: started + become: true diff --git a/roles/collect-collectd/templates/graph.html.j2 b/roles/collect-collectd/templates/graph.html.j2 new file mode 100644 index 0000000000..007af26db4 --- /dev/null +++ b/roles/collect-collectd/templates/graph.html.j2 @@ -0,0 +1,16 @@ + + + + + + Graph.html + + + {% for graph in graphs %} +
+

Grap for {{ graph }}

+ {{ graph }} +
+ {% endfor %} + + diff --git a/roles/collectd/defaults/main.yml b/roles/collectd/defaults/main.yml new file mode 100644 index 0000000000..5f5bceafc1 --- /dev/null +++ b/roles/collectd/defaults/main.yml @@ -0,0 +1,14 @@ +collectd_packages: + RedHat: + - collectd + - collectd-disk + - collectd-rrdtool + - rrdtool-python + + Debian: + - collectd + - python-rrdtool + +collectd_config_path: + RedHat: /etc/collectd.conf + Debian: /etc/collectd/collectd.conf diff --git a/roles/collectd/handlers/main.yml b/roles/collectd/handlers/main.yml new file mode 100644 index 0000000000..4ca4e94ce6 --- /dev/null +++ b/roles/collectd/handlers/main.yml @@ -0,0 +1,5 @@ +- name: restart collectd service + service: + name: collectd + state: restarted + become: true diff --git a/roles/collectd/tasks/main.yml b/roles/collectd/tasks/main.yml new file mode 100644 index 0000000000..bf58f73d82 --- /dev/null +++ b/roles/collectd/tasks/main.yml @@ -0,0 +1,46 @@ +--- +- name: install centos-release-opstools repo + package: "name={{ item }}" + become: true + with_items: + - centos-release-opstools + when: + - ansible_os_family == 'RedHat' + +- name: update yum cache + command: yum makecache + become: true + when: + - ansible_os_family == 'RedHat' + +- name: install collectd + package: + name: "{{ item }}" + become: true + with_items: "{{ collectd_packages[ansible_os_family] }}" + notify: + - restart collectd service + +- name: Ensure rrd folder + file: + path: "/var/lib/collectd/rrd" + state: "directory" + become: true + +- name: copy collectd.conf file + template: + src: collectd.conf.j2 + dest: "{{ collectd_config_path[ansible_os_family] }}" + become: true + notify: + - restart collectd service + +- name: Flush handlers + meta: flush_handlers + +- name: enable collectd service + become: true + service: + name: collectd + enabled: yes + state: started diff --git a/roles/collectd/templates/collectd.conf.j2 b/roles/collectd/templates/collectd.conf.j2 new file mode 100644 index 0000000000..5f306f983b --- /dev/null +++ b/roles/collectd/templates/collectd.conf.j2 @@ -0,0 +1,27 @@ +Interval 5 + +LoadPlugin syslog +LoadPlugin cpu +LoadPlugin interface +LoadPlugin load +LoadPlugin memory +LoadPlugin rrdtool + + DataDir "/var/lib/collectd/rrd" + CreateFilesAsync false + CacheTimeout 120 + CacheFlush 900 + WritesPerSecond 50 + + +LoadPlugin disk + + IgnoreSelected false + + +LoadPlugin df + + IgnoreSelected true + + +LoadPlugin processes diff --git a/tests/playbooks/post.yml b/tests/playbooks/post.yml index 345ccab041..8f37351114 100644 --- a/tests/playbooks/post.yml +++ b/tests/playbooks/post.yml @@ -1,4 +1,6 @@ - hosts: all + roles: + - collect-collectd tasks: - shell: cmd: | diff --git a/tests/playbooks/pre.yml b/tests/playbooks/pre.yml index 4be5f1375e..40f3f1213e 100644 --- a/tests/playbooks/pre.yml +++ b/tests/playbooks/pre.yml @@ -1,6 +1,7 @@ - hosts: all become: true - + roles: + - collectd tasks: - name: Install dbus for debian system apt: name=dbus