openstack/deb-ceilometer
Code Issues Proposed changes

[HBase] Add migration script for new row separate design

Browse Source 
Add migration for HBase data for compatibility with
new row format which presented in CR
https://review.openstack.org/#/c/106376/

Change-Id: I5232c5dad17730b76a1b3ee864d8d973ba1b8749
This commit is contained in:
Ilya Tyaptin 2014-08-20 16:13:03 +04:00
parent 919769480d
commit 3643ca5b3b
5 changed files with 164 additions and 2 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
ceilometer/alarm/storage/impl_hbase.py
View File

@ -25,6 +25,7 @@ from ceilometer.alarm.storage import models
from ceilometer.openstack.common.gettextutils import _
from ceilometer.openstack.common import log
from ceilometer.storage.hbase import inmemory as hbase_inmemory
from ceilometer.storage.hbase import migration as hbase_migration
from ceilometer.storage.hbase import utils as hbase_utils
from ceilometer import utils
@ -102,6 +103,7 @@ class Connection(base.Connection):
column_families = {'f': dict()}
with self.conn_pool.connection() as conn:
hbase_utils.create_tables(conn, tables, column_families)
hbase_migration.migrate_tables(conn, tables)
def clear(self):
LOG.debug(_('Dropping HBase schema...'))

7
ceilometer/storage/hbase/inmemory.py
View File

@ -214,8 +214,12 @@ class MTable(object):
"""
op = args[0]
value = args[1]
is_regex = False
if value.startswith('binaryprefix:'):
value = value[len('binaryprefix:'):]
if value.startswith('regexstring:'):
value = value[len('regexstring:'):]
is_regex = True
column = 'f:' + value
r = {}
for row in rows:
@ -226,7 +230,8 @@ class MTable(object):
(op == '>=' and key >= column) or
(op == '<=' and key <= column) or
(op == '>' and key > column) or
(op == '<' and key < column)):
(op == '<' and key < column) or
(is_regex and re.search(value, key))):
r_data[key] = data[key]
else:
raise ceilometer.NotImplementedError(

119
ceilometer/storage/hbase/migration.py Normal file
View File

@ -0,0 +1,119 @@
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
"""HBase storage backend migrations
"""
import re
from ceilometer.storage.hbase import utils as hbase_utils
def migrate_resource_table(conn, table):
"""Migrate table 'resource' in HBase.
Change qualifiers format from "%s+%s+%s!%s!%s" %
(rts, source, counter_name, counter_type,counter_unit)
in columns with meters f:m_*
to new separator format "%s:%s:%s:%s:%s" %
(rts, source, counter_name, counter_type,counter_unit)
"""
resource_table = conn.table(table)
resource_filter = ("QualifierFilter(=, "
"'regexstring:m_\\d{19}\\+"
"[\\w-\\._]*\\+[\\w-\\._!]')")
gen = resource_table.scan(filter=resource_filter)
for row, data in gen:
columns = []
updated_columns = dict()
column_prefix = "f:"
for column, value in data.items():
if column.startswith('f:m_'):
columns.append(column)
parts = column[2:].split("+", 2)
parts.extend(parts.pop(2).split("!"))
column = hbase_utils.prepare_key(*parts)
updated_columns[column_prefix + column] = value
resource_table.put(row, updated_columns)
resource_table.delete(row, columns)
def migrate_meter_table(conn, table):
"""Migrate table 'meter' in HBase.
Change row format from "%s_%d_%s" % (counter_name, rts, message_signature)
to new separator format "%s:%s:%s" % (counter_name, rts, message_signature)
"""
meter_table = conn.table(table)
meter_filter = ("RowFilter(=, "
"'regexstring:[\\w\\._-]*_\\d{19}_\\w*')")
gen = meter_table.scan(filter=meter_filter)
for row, data in gen:
parts = row.rsplit('_', 2)
new_row = hbase_utils.prepare_key(*parts)
meter_table.put(new_row, data)
meter_table.delete(row)
def migrate_event_table(conn, table):
"""Migrate table 'event' in HBase.
Change row format from ""%d_%s" % timestamp, event_id,
to new separator format "%s:%s" % timestamp, event_id
Also change trait columns from %s+%s % trait.name, trait.dtype
to %s:%s % trait.name, trait.dtype
"""
event_table = conn.table(table)
event_filter = "RowFilter(=, 'regexstring:\\d*_\\w*')"
gen = event_table.scan(filter=event_filter)
trait_pattern = re.compile("f:[\w\-_]*\+\w")
column_prefix = "f:"
for row, data in gen:
row_parts = row.split("_", 1)
update_data = {}
for column, value in data.items():
if trait_pattern.match(column):
trait_parts = column[2:].rsplit('+', 1)
column = hbase_utils.prepare_key(*trait_parts)
update_data[column_prefix + column] = value
new_row = hbase_utils.prepare_key(*row_parts)
event_table.put(new_row, update_data)
event_table.delete(row)
def migrate_alarm_history_table(conn, table):
"""Migrate table 'alarm_h' in HBase.
Change row format from ""%s_%s" % alarm_id, rts,
to new separator format "%s:%s" % alarm_id, rts
"""
alarm_h_table = conn.table(table)
alarm_h_filter = "RowFilter(=, 'regexstring:\\w*_\\d{19}')"
gen = alarm_h_table.scan(filter=alarm_h_filter)
for row, data in gen:
row_parts = row.rsplit('_', 1)
alarm_h_table.put(hbase_utils.prepare_key(*row_parts), data)
alarm_h_table.delete(row)
TABLE_MIGRATION_FUNCS = {'resource': migrate_resource_table,
'alarm_h': migrate_alarm_history_table,
'meter': migrate_meter_table,
'event': migrate_event_table}
def migrate_tables(conn, tables):
if type(tables) is not list:
tables = [tables]
for table in tables:
if table in TABLE_MIGRATION_FUNCS:
TABLE_MIGRATION_FUNCS.get(table)(conn, table)

36
ceilometer/storage/hbase/utils.py
View File

@ -18,6 +18,7 @@ import json
import bson.json_util
from happybase.hbase import ttypes
import six
from ceilometer.openstack.common.gettextutils import _
from ceilometer.openstack.common import log
@ -319,6 +320,20 @@ def format_meter_reference(c_name, c_type, c_unit, rts, source):
return "%s+%s+%s!%s!%s" % (rts, source, c_name, c_type, c_unit)
def prepare_key(*args):
"""Prepares names for rows and columns with correct separator.
:param args: strings or numbers that we want our key construct of
:return: key with quoted args that are separated with character ":"
"""
key_quote = []
for key in args:
if isinstance(key, int):
key = str(key)
key_quote.append(quote(key))
return ":".join(key_quote)
def timestamp_from_record_tuple(record):
"""Extract timestamp from HBase tuple record."""
return record[0]['timestamp']
@ -438,4 +453,23 @@ def create_tables(conn, tables, column_families):
LOG.warn(_("Cannot create table %(table_name)s "
"it already exists. Ignoring error")
% {'table_name': table})
% {'table_name': table})
def quote(s, *args):
"""Return quoted string even if it is unicode one.
:param s: string that should be quoted
:param args: any symbol we want to stay unquoted
"""
s_en = s.encode('utf8')
return six.moves.urllib.parse.quote(s_en, *args)
def unquote(s):
"""Return unquoted and decoded string.
:param s: string that should be unquoted
"""
s_de = six.moves.urllib.parse.unquote(s)
return s_de.decode('utf8')

2
ceilometer/storage/impl_hbase.py
View File

@ -28,6 +28,7 @@ from ceilometer.openstack.common.gettextutils import _
from ceilometer.openstack.common import log
from ceilometer.storage import base
from ceilometer.storage.hbase import inmemory as hbase_inmemory
from ceilometer.storage.hbase import migration as hbase_migration
from ceilometer.storage.hbase import utils as hbase_utils
from ceilometer.storage import models
from ceilometer import utils
@ -166,6 +167,7 @@ class Connection(base.Connection):
column_families = {'f': dict(max_versions=1)}
with self.conn_pool.connection() as conn:
hbase_utils.create_tables(conn, tables, column_families)
hbase_migration.migrate_tables(conn, tables)
def clear(self):
LOG.debug(_('Dropping HBase schema...'))