You can not select more than 25 topics Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.

183 lines
5.8 KiB

# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
Helper to create a statsd client from environment variables
import os
import logging
import statsd
from nodepool.zk import zookeeper as zk
log = logging.getLogger("nodepool.stats")
def get_client():
"""Return a statsd client object setup from environment variables; or
None if they are not set
# note we're just being careful to let the default values fall
# through to StatsClient()
statsd_args = {}
if os.getenv('STATSD_HOST', None):
statsd_args['host'] = os.environ['STATSD_HOST']
if os.getenv('STATSD_PORT', None):
statsd_args['port'] = os.environ['STATSD_PORT']
if statsd_args:
return statsd.StatsClient(**statsd_args)
return None
def normalize_statsd_name(name):
return name.replace('.', '_').replace(':', '_')
class StatsReporter(object):
Class adding statsd reporting functionality.
def __init__(self):
super(StatsReporter, self).__init__()
self._statsd = get_client()
def recordLaunchStats(self, subkey, dt):
Record node launch statistics.
:param str subkey: statsd key
:param int dt: Time delta in milliseconds
if not self._statsd:
keys = [
'nodepool.launch.provider.%s.%s' % (, subkey),
'nodepool.launch.%s' % (subkey,),
keys.append('nodepool.launch.provider.%s.%s.%s' %
(,, subkey))
if self.handler.request.requestor:
# Replace '.' which is a graphite hierarchy, and ':' which is
# a statsd delimeter.
requestor = normalize_statsd_name(self.handler.request.requestor)
keys.append('nodepool.launch.requestor.%s.%s' %
(requestor, subkey))
pipeline = self._statsd.pipeline()
for key in keys:
pipeline.timing(key, dt)
def updateNodeStats(self, zk_conn):
Refresh statistics for all known nodes.
:param ZooKeeper zk_conn: A ZooKeeper connection object.
if not self._statsd:
states = {}
launcher_pools = zk_conn.getRegisteredPools()
labels = set()
for launcher_pool in launcher_pools:
providers = set()
for launcher_pool in launcher_pools:
# Initialize things we know about to zero
for state in zk.Node.VALID_STATES:
key = 'nodepool.nodes.%s' % state
states[key] = 0
for provider in providers:
key = 'nodepool.provider.%s.nodes.%s' % (provider, state)
states[key] = 0
# Initialize label stats to 0
for label in labels:
for state in zk.Node.VALID_STATES:
key = 'nodepool.label.%s.nodes.%s' % (label, state)
states[key] = 0
for node in zk_conn.nodeIterator():
# nodepool.nodes.STATE
key = 'nodepool.nodes.%s' % node.state
states[key] += 1
# nodepool.label.LABEL.nodes.STATE
# nodes can have several labels
for label in node.type:
key = 'nodepool.label.%s.nodes.%s' % (label, node.state)
# It's possible we could see node types that aren't in our
# config
if key in states:
states[key] += 1
states[key] = 1
# nodepool.provider.PROVIDER.nodes.STATE
key = 'nodepool.provider.%s.nodes.%s' % (node.provider, node.state)
# It's possible we could see providers that aren't in our config
if key in states:
states[key] += 1
states[key] = 1
pipeline = self._statsd.pipeline()
for key, count in states.items():
pipeline.gauge(key, count)
def updateProviderLimits(self, provider):
if not self._statsd:
pipeline = self._statsd.pipeline()
# nodepool.provider.PROVIDER.max_servers
key = 'nodepool.provider.%s.max_servers' %
max_servers = sum([p.max_servers for p in provider.pools.values()
if p.max_servers])
pipeline.gauge(key, max_servers)
def updateTenantLimits(self, tenant_limits):
if not self._statsd:
pipeline = self._statsd.pipeline()
# nodepool.tenant_limits.TENANT.[cores,ram,instances]
key_template = 'nodepool.tenant_limits.%s.%s'
for tenant, limits in tenant_limits.items():
for k, lim in limits.items():
# normalize for statsd name, as parts come from arbitrary
# user config
tenant = normalize_statsd_name(tenant)
k = normalize_statsd_name(k)
key = key_template % (tenant, k)
pipeline.gauge(key, lim)