From f920b45956137a98e81fae57a7f444b7057100c5 Mon Sep 17 00:00:00 2001 From: Clint Byrum Date: Thu, 5 Nov 2015 23:56:26 -0800 Subject: [PATCH] Make a subunit2sql.target extension Just having this installed should make subunit2sql shove any counters.json it finds through to statsd. --- openstack_qa_tools/counters2statsd.py | 90 ++++++++++++++----- .../tests/test_counters2statsd.py | 10 ++- requirements.txt | 1 + setup.cfg | 2 + test-requirements.txt | 1 - 5 files changed, 77 insertions(+), 27 deletions(-) diff --git a/openstack_qa_tools/counters2statsd.py b/openstack_qa_tools/counters2statsd.py index fa89f10..56199ad 100644 --- a/openstack_qa_tools/counters2statsd.py +++ b/openstack_qa_tools/counters2statsd.py @@ -10,16 +10,24 @@ # See the License for the specific language governing permissions and # limitations under the License. +import tempfile + import json from oslo_config import cfg -import statsd +try: + import statsd +except ImportError: + statsd = None +import testtools OPTS_GROUP = cfg.OptGroup(name='counters2statsd', title='Counters2Statsd') OPTS = [ - cfg.StrOpt('host', help='Statsd host to connect to', default=None), - cfg.IntOpt('port', help='Port on statsd host to connect to', default=None), + cfg.StrOpt('host', help='Statsd host to connect to', default='localhost'), + cfg.IntOpt('port', help='Port on statsd host to connect to', default=8125), cfg.StrOpt('prefix', help='Prefix to add to stats', default=None), + cfg.BoolOpt('enabled', help='Set to false to disable this plugin', + default=True) ] _statsd_client = None @@ -28,32 +36,70 @@ _statsd_client = None def get_statsd_client(): global _statsd_client if _statsd_client is None: - cfg.CONF.register_group(OPTS_GROUP) - cfg.CONF.register_opts(OPTS, group=OPTS_GROUP) _statsd_client = statsd.StatsClient(cfg.CONF.counters2statsd.host, cfg.CONF.counters2statsd.port, cfg.CONF.counters2statsd.prefix) return _statsd_client -def add_test_run_attachments(attachments, test_run_id, session): - for attachment in attachments: - try: - counters = json.loads(attachment) - except ValueError: - continue - if not isinstance(counters, dict): - continue - if '__counters_meta__' not in counters: - continue +class AttachmentResult(testtools.StreamResult): + """Keeps track of top level results with StreamToDict drops. + + We use a SpooledTemporaryFile to keep it performant with smaller files + but to ensure we don't use up tons of RAM. Anything over 1MB will be + spooled out to disk. + """ + @classmethod + def enabled(cls): + cfg.CONF.register_group(OPTS_GROUP) + cfg.CONF.register_opts(OPTS, group=OPTS_GROUP) + cfg.CONF.register_cli_opts(OPTS, group=OPTS_GROUP) + return bool(statsd) + + def __init__(self): + super(AttachmentResult, self).__init__() + self.attachments = {} + + def status(self, test_id=None, test_status=None, test_tags=None, + runnable=True, file_name=None, file_bytes=None, eof=False, + mime_type=None, route_code=None, timestamp=None): + if not cfg.CONF.counters2statsd.enabled: + return + if test_id is not None: + return + if not file_name: + return + if file_name not in self.attachments: + self.attachments[file_name] = tempfile.SpooledTemporaryFile( + max_size=2 ** 30) + self.attachments[file_name].write(file_bytes) + if eof: + self.attachments[file_name].seek(0) + + def stopTestRun(self): + if not cfg.CONF.counters2statsd.enabled: + return client = get_statsd_client() - for groupname, values in counters.items(): - if not isinstance(values, dict): + for file_name, attachment in self.attachments.items(): + if file_name != 'counters.json': continue - for k, v in values.items(): - k = '{}.{}'.format(groupname, k) + try: try: - v = int(v) - except ValueError: + attachment.seek(0) + counters = json.loads(attachment.read().decode('utf-8')) + except AttributeError: + counters = json.loads(attachment) + except ValueError: + continue + if not isinstance(counters, dict): + continue + for groupname, values in counters.items(): + if not isinstance(values, dict): continue - client.incr(k, v) + for k, v in values.items(): + k = '{}.{}'.format(groupname, k) + try: + v = int(v) + except ValueError: + continue + client.incr(k, v) diff --git a/openstack_qa_tools/tests/test_counters2statsd.py b/openstack_qa_tools/tests/test_counters2statsd.py index 9aa754e..496a3bc 100644 --- a/openstack_qa_tools/tests/test_counters2statsd.py +++ b/openstack_qa_tools/tests/test_counters2statsd.py @@ -34,8 +34,10 @@ class TestOpenStackQaTols(base.TestCase): mock_client.incr = mock.MagicMock('statds_incr') statsd_mock.return_value = mock_client fake_counters = {'mysql': {'Queries': 10}} - fake_counters['__counters_meta__'] = {} - fake_counters = json.dumps(fake_counters) - counters2statsd.add_test_run_attachments([fake_counters], 'foo', None) - statsd_mock.assert_called_with(None, None, None) + fake_counters = json.dumps(fake_counters).encode('utf-8') + self.assertTrue(counters2statsd.AttachmentResult.enabled()) + result = counters2statsd.AttachmentResult() + result.status(file_name='counters.json', file_bytes=fake_counters) + result.stopTestRun() + statsd_mock.assert_called_with('localhost', 8125, None) mock_client.incr.assert_called_with('mysql.Queries', 10) diff --git a/requirements.txt b/requirements.txt index 2797be4..31b3a86 100644 --- a/requirements.txt +++ b/requirements.txt @@ -6,3 +6,4 @@ pbr>=1.6 PyMySQL>=0.6.2 # MIT License statsd>=1.0.0,<3.0 oslo.config>=1.4.0.0a3 +testtools>=1.4.0 diff --git a/setup.cfg b/setup.cfg index 46a5b79..3e20175 100644 --- a/setup.cfg +++ b/setup.cfg @@ -22,6 +22,8 @@ classifier = [entry_points] console_scripts = os-qa-counters = openstack_qa_tools.collect:main +subunit2sql.target = + openstack_qa_statsd = openstack_qa_tools.counters2statsd:AttachmentResult [files] packages = diff --git a/test-requirements.txt b/test-requirements.txt index 227668a..ab77cb9 100644 --- a/test-requirements.txt +++ b/test-requirements.txt @@ -12,5 +12,4 @@ oslosphinx>=2.5.0 # Apache-2.0 oslotest>=1.10.0 # Apache-2.0 testrepository>=0.0.18 testscenarios>=0.4 -testtools>=1.4.0 mock>=1.2