diff --git a/modules/openstack_project/files/git/haproxy-statsd.default b/modules/openstack_project/files/git/haproxy-statsd.default new file mode 100644 index 0000000000..d057e7ccf8 --- /dev/null +++ b/modules/openstack_project/files/git/haproxy-statsd.default @@ -0,0 +1,2 @@ +STATSD_HOST=graphite.openstack.org +STATSD_PORT=8125 diff --git a/modules/openstack_project/files/git/haproxy-statsd.py b/modules/openstack_project/files/git/haproxy-statsd.py new file mode 100644 index 0000000000..514b618c31 --- /dev/null +++ b/modules/openstack_project/files/git/haproxy-statsd.py @@ -0,0 +1,181 @@ +#!/usr/bin/env python + +# Copyright (C) 2015 Hewlett-Packard Development Company, L.P. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +# implied. +# +# See the License for the specific language governing permissions and +# limitations under the License. + +import logging +import re +import socket +import time + +from statsd.defaults.env import statsd + +INTERVAL = 10 +GAUGES = [ + 'qcur', + # 2. qcur [..BS]: current queued requests. For the backend this + # reports the number queued without a server assigned. + 'scur', + # 4. scur [LFBS]: current sessions + 'act', + # 19. act [..BS]: number of active servers (backend), server is + # active (server) + 'bck', + # 20. bck [..BS]: number of backup servers (backend), server is + # backup (server) + 'qtime', + # 58. qtime [..BS]: the average queue time in ms over the 1024 + # last requests + 'ctime', + # 59. ctime [..BS]: the average connect time in ms over the 1024 + # last requests + 'rtime', + # 60. rtime [..BS]: the average response time in ms over the 1024 + # last requests (0 for TCP) + 'ttime', + # 61. ttime [..BS]: the average total session time in ms over the + # 1024 last requests +] + +COUNTERS = [ + 'stot', + # 7. stot [LFBS]: cumulative number of connections + 'bin', + # 8. bin [LFBS]: bytes in + 'bout', + # 9. bout [LFBS]: bytes out + 'ereq', + # 12. ereq [LF..]: request errors. Some of the possible causes + # are: + # - early termination from the client, before the request has + # been sent. + # - read error from the client + # - client timeout + # - client closed connection + # - various bad requests from the client. + # - request was tarpitted. + 'econ', + # 13. econ [..BS]: number of requests that encountered an error + # trying to connect to a backend server. The backend stat is the + # sum of the stat for all servers of that backend, plus any + # connection errors not associated with a particular server (such + # as the backend having no active servers). + 'eresp', + # 14. eresp [..BS]: response errors. srv_abrt will be counted here + # also. + # Some other errors are: + # - write error on the client socket (won't be counted for the + # server stat) + # - failure applying filters to the response. + 'wretr', + # 15. wretr [..BS]: number of times a connection to a server was + # retried. + 'wredis', + # 16. wredis [..BS]: number of times a request was redispatched to + # another server. The server value counts the number of times that + # server was switched away from. +] + + +class Socket(object): + def __init__(self, path): + self.path = path + self.socket = None + + def open(self): + s = socket.socket(socket.AF_UNIX) + s.settimeout(5) + s.connect(self.path) + self.socket = s + + def __enter__(self): + self.open() + return self.socket + + def __exit__(self, etype, value, tb): + self.socket.close() + self.socket = None + + +class HAProxy(object): + COMMENT_RE = re.compile('^#\s+(\S.*)') + + def __init__(self, path): + self.socket = Socket(path) + self.log = logging.getLogger("HAProxy") + self.prevdata = {} + + def command(self, command): + with self.socket as socket: + socket.send(command + '\n') + data = '' + while True: + r = socket.recv(4096) + data += r + if not r: + break + return data + + def getStats(self): + data = self.command('show stat') + lines = data.split('\n') + m = self.COMMENT_RE.match(lines[0]) + header = m.group(1) + cols = header.split(',')[:-1] + ret = [] + for line in lines[1:]: + if not line: + continue + row = line.split(',')[:-1] + row = dict(zip(cols, row)) + ret.append(row) + return ret + + def reportStats(self, stats): + for row in stats: + base = 'haproxy.%s.%s.' % (row['pxname'], row['svname']) + for key in GAUGES: + value = row[key] + if value != '': + statsd.gauge(base + key, int(value)) + for key in COUNTERS: + metric = base + key + newvalue = row[key] + if newvalue == '': + continue + newvalue = int(newvalue) + oldvalue = self.prevdata.get(metric) + if oldvalue is not None: + value = newvalue - oldvalue + statsd.incr(metric, value) + self.prevdata[metric] = newvalue + + def run(self): + while True: + try: + self._run() + except Exception: + self.log.exception("Exception in main loop:") + + def _run(self): + time.sleep(INTERVAL) + stats = self.getStats() + self.reportStats(stats) + + +logging.basicConfig(level=logging.DEBUG) +p = HAProxy('/var/lib/haproxy/stats') +p.run() diff --git a/modules/openstack_project/files/git/haproxy-statsd.service b/modules/openstack_project/files/git/haproxy-statsd.service new file mode 100644 index 0000000000..270a5a7135 --- /dev/null +++ b/modules/openstack_project/files/git/haproxy-statsd.service @@ -0,0 +1,10 @@ +[Unit] +Description=haproxy-statsd +After=haproxy.service + +[Service] +ExecStart=/usr/local/bin/haproxy-statsd.py +EnvironmentFile=/etc/default/haproxy-statsd + +[Install] +WantedBy=multi-user.target diff --git a/modules/openstack_project/manifests/git.pp b/modules/openstack_project/manifests/git.pp index 23eb3479e6..521363e458 100644 --- a/modules/openstack_project/manifests/git.pp +++ b/modules/openstack_project/manifests/git.pp @@ -143,4 +143,39 @@ class openstack_project::git ( source => 'puppet:///modules/openstack_project/git/rsyslog.haproxy.conf', notify => Service['rsyslog'], } + + file { '/usr/local/bin/haproxy-statsd.py': + ensure => present, + owner => 'root', + group => 'root', + mode => '0755', + source => 'puppet:///modules/openstack_project/git/haproxy-statsd.py', + notify => Service['haproxy-statsd'], + } + + file { '/etc/default/haproxy-statsd': + ensure => present, + owner => 'root', + group => 'root', + mode => '0755', + source => 'puppet:///modules/openstack_project/git/haproxy-statsd.default', + require => File['/usr/local/bin/haproxy-statsd.py'], + notify => Service['haproxy-statsd'], + } + + file { '/etc/systemd/system/haproxy-statsd.service': + ensure => present, + owner => 'root', + group => 'root', + mode => '0644', + source => 'puppet:///modules/openstack_project/git/haproxy-statsd.service', + require => File['/etc/default/haproxy-statsd'], + notify => Service['haproxy-statsd'], + } + + service { 'haproxy-statsd': + provider => systemd, + enable => true, + require => File['/etc/systemd/system/haproxy-statsd.service'], + } }