[HBase] Improves speed of unit tests on real HBase backend
Currently, unittests at real HBase are not usually used. Jenkins jobs and a predominant amount of developers use HBase mock. One of the reasons for this is that unittests at real HBase are too slow. It's due to processing of "disable_table" and "delete_table" commands. In HBase these command take up to 1-2 seconds. Now we create all table-set for each test that's why at real HBase unit tests may be executed several hours. So, at real HBase backend unit tests may be executed several hours. My CR speeds up this case. To solve this problem it was decided to keep all test data in one table. To provide a distinguishability of data from different tests unique row-prefix is used for each one. Creating and deleting required table are implements at setup-test-env.sh. Separating data is implemented with mock.patchs which transforms row value in happybase.Table methods. Change-Id: I1883d6e0619b0b2f223a4e58bdc0fc0656636e1f Closes-bug: #1372912
This commit is contained in:
Ilya Tyaptin
parent
637095c236
commit
125d950dd4
|
|
@ -23,6 +23,7 @@ import os
|
|||
import uuid
|
||||
import warnings
|
||||
|
||||
import mock
|
||||
from oslo.config import fixture as fixture_config
|
||||
from oslotest import mockpatch
|
||||
import six
|
||||
|
|
@ -32,6 +33,7 @@ from testtools import testcase
|
|||
|
||||
from ceilometer import storage
|
||||
from ceilometer.tests import base as test_base
|
||||
from ceilometer.tests import mocks
|
||||
|
||||
|
||||
class MongoDbManager(fixtures.Fixture):
|
||||
|
|
@ -71,12 +73,31 @@ class HBaseManager(fixtures.Fixture):
|
|||
self.url, 'ceilometer.metering.storage')
|
||||
self.alarm_connection = storage.get_connection(
|
||||
self.url, 'ceilometer.alarm.storage')
|
||||
# Unique prefix for each test to keep data is distinguished because
|
||||
# all test data is stored in one table
|
||||
data_prefix = str(uuid.uuid4().hex)
|
||||
|
||||
def table(conn, name):
|
||||
return mocks.MockHBaseTable(name, conn, data_prefix)
|
||||
|
||||
# Mock only real HBase connection, MConnection "table" method
|
||||
# stays origin.
|
||||
mock.patch('happybase.Connection.table', new=table).start()
|
||||
# We shouldn't delete data and tables after each test,
|
||||
# because it last for too long.
|
||||
# All tests tables will be deleted in setup-test-env.sh
|
||||
mock.patch("happybase.Connection.disable_table",
|
||||
new=mock.MagicMock()).start()
|
||||
mock.patch("happybase.Connection.delete_table",
|
||||
new=mock.MagicMock()).start()
|
||||
mock.patch("happybase.Connection.create_table",
|
||||
new=mock.MagicMock()).start()
|
||||
|
||||
@property
|
||||
def url(self):
|
||||
return '%s?table_prefix=%s' % (
|
||||
self._url,
|
||||
uuid.uuid4().hex
|
||||
os.getenv("CEILOMETER_TEST_HBASE_TABLE_PREFIX", "test")
|
||||
)
|
||||
|
||||
|
||||
|
|
@ -106,7 +127,6 @@ class TestBase(testscenarios.testcase.WithScenarios, test_base.BaseTestCase):
|
|||
|
||||
def setUp(self):
|
||||
super(TestBase, self).setUp()
|
||||
|
||||
engine = urlparse.urlparse(self.db_url).scheme
|
||||
|
||||
# NOTE(Alexei_987) Shortcut to skip expensive db setUp
|
||||
|
|
|
|||
|
|
@ -0,0 +1,79 @@
|
|||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License"); you may
|
||||
# not use this file except in compliance with the License. You may obtain
|
||||
# a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
|
||||
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
|
||||
# License for the specific language governing permissions and limitations
|
||||
# under the License.
|
||||
|
||||
import os
|
||||
|
||||
import happybase
|
||||
|
||||
|
||||
class MockHBaseTable(happybase.Table):
|
||||
|
||||
def __init__(self, name, connection, data_prefix):
|
||||
# data_prefix is added to all rows which are written
|
||||
# in this test. It allows to divide data from different tests
|
||||
self.data_prefix = data_prefix
|
||||
# We create happybase Table with prefix from
|
||||
# CEILOMETER_TEST_HBASE_TABLE_PREFIX
|
||||
prefix = os.getenv("CEILOMETER_TEST_HBASE_TABLE_PREFIX", 'test')
|
||||
super(MockHBaseTable, self).__init__(
|
||||
"%s_%s" % (prefix, name),
|
||||
connection)
|
||||
|
||||
def put(self, row, *args, **kwargs):
|
||||
row = self.data_prefix + row
|
||||
return super(MockHBaseTable, self).put(row, *args,
|
||||
**kwargs)
|
||||
|
||||
def scan(self, row_start=None, row_stop=None, row_prefix=None,
|
||||
columns=None, filter=None, timestamp=None,
|
||||
include_timestamp=False, batch_size=10, scan_batching=None,
|
||||
limit=None, sorted_columns=False):
|
||||
# Add data prefix for row parameters
|
||||
# row_prefix could not be combined with row_start or row_stop
|
||||
if not row_start and not row_stop:
|
||||
row_prefix = self.data_prefix + (row_prefix or "")
|
||||
row_start = None
|
||||
row_stop = None
|
||||
elif row_start and not row_stop:
|
||||
# Adding data_prefix to row_start and row_stop does not work
|
||||
# if it looks like row_start = %data_prefix%foo,
|
||||
# row_stop = %data_prefix, because row_start > row_stop
|
||||
filter = self._update_filter_row(filter)
|
||||
row_start = self.data_prefix + row_start
|
||||
else:
|
||||
row_start = self.data_prefix + (row_start or "")
|
||||
row_stop = self.data_prefix + (row_stop or "")
|
||||
gen = super(MockHBaseTable, self).scan(row_start, row_stop,
|
||||
row_prefix, columns,
|
||||
filter, timestamp,
|
||||
include_timestamp, batch_size,
|
||||
scan_batching, limit,
|
||||
sorted_columns)
|
||||
data_prefix_len = len(self.data_prefix)
|
||||
# Restore original row format
|
||||
for row, data in gen:
|
||||
yield (row[data_prefix_len:], data)
|
||||
|
||||
def row(self, row, *args, **kwargs):
|
||||
row = self.data_prefix + row
|
||||
return super(MockHBaseTable, self).row(row, *args, **kwargs)
|
||||
|
||||
def delete(self, row, *args, **kwargs):
|
||||
row = self.data_prefix + row
|
||||
return super(MockHBaseTable, self).delete(row, *args, **kwargs)
|
||||
|
||||
def _update_filter_row(self, filter):
|
||||
if filter:
|
||||
return "PrefixFilter(%s) AND %s" % (self.data_prefix, filter)
|
||||
else:
|
||||
return "PrefixFilter(%s)" % self.data_prefix
|
||||
|
|
@ -26,7 +26,6 @@
|
|||
import mock
|
||||
|
||||
from ceilometer.alarm.storage import impl_hbase as hbase_alarm
|
||||
from ceilometer.storage.hbase import inmemory as hbase_inmemory
|
||||
from ceilometer.storage import impl_hbase as hbase
|
||||
from ceilometer.tests import base as test_base
|
||||
from ceilometer.tests import db as tests_db
|
||||
|
|
@ -37,9 +36,6 @@ class ConnectionTest(tests_db.TestBase,
|
|||
|
||||
@tests_db.run_with('hbase')
|
||||
def test_hbase_connection(self):
|
||||
conn = hbase.Connection(self.db_manager.url)
|
||||
self.assertIsInstance(conn.conn_pool.connection(),
|
||||
hbase_inmemory.MConnection)
|
||||
|
||||
class TestConn(object):
|
||||
def __init__(self, host, port):
|
||||
|
|
|
|||
|
|
@ -4,6 +4,10 @@ set -e
|
|||
function clean_exit(){
|
||||
local error_code="$?"
|
||||
rm -rf ${MONGO_DATA}
|
||||
if test -n "$CEILOMETER_TEST_HBASE_URL"
|
||||
then
|
||||
python tools/test_hbase_table_utils.py --clear
|
||||
fi
|
||||
kill $(jobs -p)
|
||||
return $error_code
|
||||
}
|
||||
|
|
@ -28,6 +32,11 @@ done < ${MONGO_DATA}/out
|
|||
# Read the fifo for ever otherwise mongod would block
|
||||
cat ${MONGO_DATA}/out > /dev/null &
|
||||
export CEILOMETER_TEST_MONGODB_URL="mongodb://localhost:${MONGO_PORT}/ceilometer"
|
||||
if test -n "$CEILOMETER_TEST_HBASE_URL"
|
||||
then
|
||||
export CEILOMETER_TEST_HBASE_TABLE_PREFIX=$(hexdump -n 16 -v -e '/1 "%02X"' /dev/urandom)
|
||||
python tools/test_hbase_table_utils.py --upgrade
|
||||
fi
|
||||
|
||||
# Yield execution to venv command
|
||||
$*
|
||||
|
|
|
|||
|
|
@ -0,0 +1,38 @@
|
|||
# Licensed under the Apache License, Version 2.0 (the "License"); you may
|
||||
# not use this file except in compliance with the License. You may obtain
|
||||
# a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
|
||||
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
|
||||
# License for the specific language governing permissions and limitations
|
||||
# under the License.
|
||||
|
||||
import os
|
||||
import sys
|
||||
|
||||
from oslo.config import cfg
|
||||
|
||||
from ceilometer import storage
|
||||
|
||||
def main(argv):
|
||||
cfg.CONF([], project='ceilometer')
|
||||
if os.getenv("CEILOMETER_TEST_HBASE_URL"):
|
||||
url = ("%s?table_prefix=%s" %
|
||||
(os.getenv("CEILOMETER_TEST_HBASE_URL"),
|
||||
os.getenv("CEILOMETER_TEST_HBASE_TABLE_PREFIX", "test")))
|
||||
conn = storage.get_connection(url, 'ceilometer.metering.storage')
|
||||
alarm_conn = storage.get_connection(url, 'ceilometer.alarm.storage')
|
||||
for arg in argv:
|
||||
if arg == "--upgrade":
|
||||
conn.upgrade()
|
||||
alarm_conn.upgrade()
|
||||
if arg == "--clear":
|
||||
conn.clear()
|
||||
alarm_conn.clear()
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
main(sys.argv[1:])
|
||||
Loading…
Reference in New Issue