Add testable colector modules and tests

openstack_qa_tools/collectors/mysql.py

@@ -0,0 +1,57 @@
+### Author: HIROSE Masaaki <hirose31 _at_ gmail.com>
+
+import os
+
+import pymysql
+from six.moves import configparser
+
+from openstack_qa_tools import error
+
+COLLECT_COUNTERS = (
+    'Com_delete',
+    'Com_insert',
+    'Com_select',
+    'Com_update',
+    'Connections',
+    'Innodb_buffer_pool_read_requests',
+    'Innodb_data_reads',
+    'Innodb_data_writes',
+    'Innodb_log_writes',
+    'Innodb_rows_deleted',
+    'Innodb_rows_inserted',
+    'Innodb_rows_read',
+    'Innodb_rows_updated',
+    'Queries',
+    'Slow_queries',
+    'Threads_connected',
+    'Threads_running',
+)
+
+
+def _get_config():
+    args = {}
+    try:
+        with open(os.path.expanduser("~/.my.cnf")) as dfile:
+            parser = configparser.ConfigParser()
+            parser.readfp(dfile)
+            for k,v in parser.items('client'):
+                args[k] = v
+    except IOError as e:
+        raise error.CollectionError(str(e))
+    return args
+
+
+def collect():
+    args = _get_config()
+    conn = pymysql.connect(**args)
+    cursor = conn.cursor()
+    counters = {}
+    while True:
+        cursor.execute('show global status')
+        result = cursor.fetchone()
+        if result is None:
+            break
+        k, v = result
+        if k in COLLECT_COUNTERS:
+            counters[k] = int(v)
+    return counters

openstack_qa_tools/collectors/queues.py

@@ -0,0 +1,57 @@
+# This program is free software; you can redistribute it and/or
+# modify it under the terms of the GNU General Public License
+# as published by the Free Software Foundation; either version 2
+# of the License, or (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, write to the Free Software
+# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301, USA.
+
+import base64
+import httplib
+import json
+import os
+import socket
+import urllib2
+
+from openstack_qa_tools import error
+
+DSTAT_RABBITMQ_API = os.environ.get('DSTAT_RABBITMQ_API',
+                                    '127.0.0.1:15672')
+DSTAT_RABBITMQ_API_USER = os.environ.get('DSTAT_RABBITMQ_USER',
+                                    'guest')
+DSTAT_RABBITMQ_API_PASS = os.environ.get('DSTAT_RABBITMQ_PASS',
+                                    'guest')
+
+def collect():
+    conn = httplib.HTTPConnection(DSTAT_RABBITMQ_API)
+    auth = base64.encodestring('%s:%s' % (DSTAT_RABBITMQ_API_USER,
+                                               DSTAT_RABBITMQ_API_PASS))
+    auth = auth.replace('\n', '')
+    auth = {'Authorization': 'Basic %s' % auth}
+    try:
+        conn.request('GET', '/api/queues', headers=auth)
+        content = conn.getresponse().read()
+    except (socket.error, httplib.HTTPException) as e:
+        raise error.CollectionError(str(e))
+
+    content = json.loads(content)
+    if not isinstance(content, list):
+        raise error.CollectionError('Unexpected format encountered. %s' % content)
+    collected = {}
+    for q in content:
+        if not isinstance(q, dict):
+            continue
+        if "name" not in q or "messages" not in q:
+            continue
+        qname = q["name"]
+        if "message_stats" in q and "publish" in q["message_stats"]:
+            newpub = q["message_stats"]["publish"]
+            target = '%s_message_stats' % (qname)
+            collected[target] = q["message_stats"]["publish"]
+    return collected

openstack_qa_tools/error.py

@@ -0,0 +1,2 @@
+class CollectionError(Exception):
+    pass

openstack_qa_tools/tests/test_mysql.py

@@ -0,0 +1,44 @@
+# -*- coding: utf-8 -*-
+
+# Licensed under the Apache License, Version 2.0 (the "License"); you may
+# not use this file except in compliance with the License. You may obtain
+# a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+# License for the specific language governing permissions and limitations
+# under the License.
+
+"""
+test_collectors
+----------------------------------
+
+Tests for `openstack_qa_tools.collectors`
+"""
+
+import mock
+
+from openstack_qa_tools.collectors import mysql
+from openstack_qa_tools import error
+from openstack_qa_tools.tests import base
+
+
+class TestOpenStackQaTols(base.TestCase):
+
+    @mock.patch('openstack_qa_tools.collectors.mysql._get_config')
+    @mock.patch('pymysql.connect')
+    def test_mysql(self, pymysql_mock, get_config_mock):
+        connection = mock.MagicMock()
+        curs = mock.MagicMock()
+        side_effect = [ (k, 0) for k in mysql.COLLECT_COUNTERS ]
+        side_effect.append(None) # Instead of StopIteration pymsql uses None
+        curs.fetchone.side_effect = side_effect
+        connection.cursor.return_value = curs
+        pymysql_mock.return_value = connection
+        result = mysql.collect()
+        self.assertEquals(sorted(mysql.COLLECT_COUNTERS),
+                          sorted(result.keys()))
+        self.assertTrue(all([val == 0 for val in result.values()]))

openstack_qa_tools/tests/test_queues.py

@@ -13,16 +13,26 @@
 # under the License.
 
 """
-test_openstack_qa_tools
+test_collectors
 ----------------------------------
 
-Tests for `openstack_qa_tools` module.
+Tests for `openstack_qa_tools.collectors`
 """
 
+import mock
+
+from openstack_qa_tools.collectors import queues
+from openstack_qa_tools import error
 from openstack_qa_tools.tests import base
 
 
-class TestOpenstack_qa_tools(base.TestCase):
+class TestOpenStackQaTols(base.TestCase):
 
-    def test_something(self):
+    @mock.patch('httplib.HTTPConnection')
-        pass
+    def test_queues(self, httplib_mock):
+        reader = mock.MagicMock()
+        reader.read.return_value = '[]'
+        conn = httplib_mock.return_value
+        conn.getresponse.return_value = reader
+        data = queues.collect()
+        self.assertEquals({}, data)

requirements.txt

@@ -3,4 +3,4 @@
 # process, which may cause wedges in the gate later.
 
 pbr>=1.6
-Babel>=1.3
+PyMySQL>=0.6.2  # MIT License

setup.cfg

@@ -30,17 +30,3 @@ all_files = 1
 
 [upload_sphinx]
 upload-dir = doc/build/html
-
-[compile_catalog]
-directory = openstack_qa_tools/locale
-domain = openstack-qa-tools
-
-[update_catalog]
-domain = openstack-qa-tools
-output_dir = openstack_qa_tools/locale
-input_file = openstack_qa_tools/locale/openstack-qa-tools.pot
-
-[extract_messages]
-keywords = _ gettext ngettext l_ lazy_gettext
-mapping_file = babel.cfg
-output_file = openstack_qa_tools/locale/openstack-qa-tools.pot

test-requirements.txt

@@ -13,3 +13,4 @@ oslotest>=1.10.0 # Apache-2.0
 testrepository>=0.0.18
 testscenarios>=0.4
 testtools>=1.4.0
+mock>=1.2