From 7a6e72d79177ed3ae8a551bca488dacdf098abb9 Mon Sep 17 00:00:00 2001 From: Stefania Alborghetti Date: Fri, 20 Mar 2015 11:35:13 +0800 Subject: [PATCH 1/4] CASSANDRA-7814: Added indexes to table and ks metadata and export_as_string() to IndexMetadata --- cassandra/metadata.py | 43 +++++++++++++++++++++++++++++++++++++++---- 1 file changed, 39 insertions(+), 4 deletions(-) diff --git a/cassandra/metadata.py b/cassandra/metadata.py index 1f88d8f9..9b01a18c 100644 --- a/cassandra/metadata.py +++ b/cassandra/metadata.py @@ -122,10 +122,9 @@ class Metadata(object): keyspace_col_rows = col_def_rows.get(keyspace_meta.name, {}) keyspace_trigger_rows = trigger_rows.get(keyspace_meta.name, {}) for table_row in cf_def_rows.get(keyspace_meta.name, []): - table_meta = self._build_table_metadata( + self._add_table_metadata_to_ks( keyspace_meta, table_row, keyspace_col_rows, keyspace_trigger_rows) - keyspace_meta.tables[table_meta.name] = table_meta for usertype_row in usertype_rows.get(keyspace_meta.name, []): usertype = self._build_usertype(keyspace_meta.name, usertype_row) @@ -184,10 +183,12 @@ class Metadata(object): if not cf_results: # the table was removed - keyspace_meta.tables.pop(table, None) + table_meta = keyspace_meta.tables.pop(table, None) + if table_meta: + self._clear_table_indexes_in_ks(keyspace_meta, table_meta.name) else: assert len(cf_results) == 1 - keyspace_meta.tables[table] = self._build_table_metadata( + self._add_table_metadata_to_ks( keyspace_meta, cf_results[0], {table: col_results}, {table: triggers_result}) @@ -215,6 +216,20 @@ class Metadata(object): return UserType(usertype_row['keyspace_name'], usertype_row['type_name'], usertype_row['field_names'], type_classes) + def _add_table_metadata_to_ks(self, keyspace_metadata, row, col_rows, trigger_rows): + self._clear_table_indexes_in_ks(keyspace_metadata, row["columnfamily_name"]) + + table_metadata = self._build_table_metadata(keyspace_metadata, row, col_rows, trigger_rows) + keyspace_metadata.tables[table_metadata.name] = table_metadata + for index_name, index_metadata in table_metadata.indexes.iteritems(): + keyspace_metadata.indexes[index_name] = index_metadata + + def _clear_table_indexes_in_ks(self, keyspace_metadata, table_name): + if table_name in keyspace_metadata.tables: + table_meta = keyspace_metadata.tables[table_name] + for index_name in table_meta.indexes: + keyspace_metadata.indexes.pop(index_name, None) + def _build_table_metadata(self, keyspace_metadata, row, col_rows, trigger_rows): cfname = row["columnfamily_name"] cf_col_rows = col_rows.get(cfname, []) @@ -385,6 +400,8 @@ class Metadata(object): column_meta = ColumnMetadata(table_metadata, name, data_type, is_static=is_static) index_meta = self._build_index_metadata(column_meta, row) column_meta.index = index_meta + if index_meta: + table_metadata.indexes[index_meta.name] = index_meta return column_meta def _build_index_metadata(self, column_metadata, row): @@ -733,6 +750,11 @@ class KeyspaceMetadata(object): A map from table names to instances of :class:`~.TableMetadata`. """ + indexes = None + """ + A dict mapping index names to :class:`.IndexMetadata` instances. + """ + user_types = None """ A map from user-defined type names to instances of :class:`~cassandra.metadata..UserType`. @@ -745,6 +767,7 @@ class KeyspaceMetadata(object): self.durable_writes = durable_writes self.replication_strategy = ReplicationStrategy.create(strategy_class, strategy_options) self.tables = {} + self.indexes = {} self.user_types = {} def export_as_string(self): @@ -884,6 +907,11 @@ class TableMetadata(object): A dict mapping column names to :class:`.ColumnMetadata` instances. """ + indexes = None + """ + A dict mapping index names to :class:`.IndexMetadata` instances. + """ + is_compact_storage = False options = None @@ -945,6 +973,7 @@ class TableMetadata(object): self.partition_key = [] if partition_key is None else partition_key self.clustering_key = [] if clustering_key is None else clustering_key self.columns = OrderedDict() if columns is None else columns + self.indexes = {} self.options = options self.comparator = None self.triggers = OrderedDict() if triggers is None else triggers @@ -1242,6 +1271,12 @@ class IndexMetadata(object): protect_name(self.column.name), self.index_options["class_name"]) + def export_as_string(self): + """ + Returns a CQL query string that can be used to recreate this index. + """ + return self.as_cql_query() + ';' + class TokenMap(object): """ From 7b15fae5c80093ae479c3a04aaa8db2da3d45f1d Mon Sep 17 00:00:00 2001 From: Stefania Alborghetti Date: Tue, 21 Apr 2015 09:46:52 +0800 Subject: [PATCH 2/4] CASSANDRA-7814: code review comments --- cassandra/metadata.py | 38 +++++++++++++++++--------------------- 1 file changed, 17 insertions(+), 21 deletions(-) diff --git a/cassandra/metadata.py b/cassandra/metadata.py index 9b01a18c..8a11a645 100644 --- a/cassandra/metadata.py +++ b/cassandra/metadata.py @@ -122,9 +122,8 @@ class Metadata(object): keyspace_col_rows = col_def_rows.get(keyspace_meta.name, {}) keyspace_trigger_rows = trigger_rows.get(keyspace_meta.name, {}) for table_row in cf_def_rows.get(keyspace_meta.name, []): - self._add_table_metadata_to_ks( - keyspace_meta, table_row, keyspace_col_rows, - keyspace_trigger_rows) + table_meta = self._build_table_metadata(keyspace_meta, table_row, keyspace_col_rows, keyspace_trigger_rows) + keyspace_meta._add_table_metadata(table_meta) for usertype_row in usertype_rows.get(keyspace_meta.name, []): usertype = self._build_usertype(keyspace_meta.name, usertype_row) @@ -185,12 +184,11 @@ class Metadata(object): # the table was removed table_meta = keyspace_meta.tables.pop(table, None) if table_meta: - self._clear_table_indexes_in_ks(keyspace_meta, table_meta.name) + keyspace_meta._clear_table_indexes(table_meta.name) else: assert len(cf_results) == 1 - self._add_table_metadata_to_ks( - keyspace_meta, cf_results[0], {table: col_results}, - {table: triggers_result}) + table_meta = self._build_table_metadata(keyspace_meta, cf_results[0], {table: col_results}, {table: triggers_result}) + keyspace_meta._add_table_metadata(ctable_meta) def _keyspace_added(self, ksname): if self.token_map: @@ -216,20 +214,6 @@ class Metadata(object): return UserType(usertype_row['keyspace_name'], usertype_row['type_name'], usertype_row['field_names'], type_classes) - def _add_table_metadata_to_ks(self, keyspace_metadata, row, col_rows, trigger_rows): - self._clear_table_indexes_in_ks(keyspace_metadata, row["columnfamily_name"]) - - table_metadata = self._build_table_metadata(keyspace_metadata, row, col_rows, trigger_rows) - keyspace_metadata.tables[table_metadata.name] = table_metadata - for index_name, index_metadata in table_metadata.indexes.iteritems(): - keyspace_metadata.indexes[index_name] = index_metadata - - def _clear_table_indexes_in_ks(self, keyspace_metadata, table_name): - if table_name in keyspace_metadata.tables: - table_meta = keyspace_metadata.tables[table_name] - for index_name in table_meta.indexes: - keyspace_metadata.indexes.pop(index_name, None) - def _build_table_metadata(self, keyspace_metadata, row, col_rows, trigger_rows): cfname = row["columnfamily_name"] cf_col_rows = col_rows.get(cfname, []) @@ -803,6 +787,18 @@ class KeyspaceMetadata(object): self.resolve_user_types(field_type.typename, types, user_type_strings) user_type_strings.append(user_type.as_cql_query(formatted=True)) + def _add_table_metadata(self, table_metadata): + self._clear_table_indexes(table_metadata.name) + + self.tables[table_metadata.name] = table_metadata + for index_name, index_metadata in table_metadata.indexes.iteritems(): + self.indexes[index_name] = index_metadata + + def _clear_table_indexes(self, table_name): + if table_name in self.tables: + table_meta = self.tables[table_name] + for index_name in table_meta.indexes: + self.indexes.pop(index_name, None) class UserType(object): """ From 86944d70868522803508343c52ebc45412e9dd07 Mon Sep 17 00:00:00 2001 From: Adam Holmberg Date: Tue, 21 Apr 2015 13:30:17 -0500 Subject: [PATCH 3/4] Updates to index metadata indexes. Make indexes follow table alter Make keyspace meta clear indexes on table drop Add tests for this feature PYTHON-241 --- cassandra/metadata.py | 15 ++-- tests/integration/standard/test_metadata.py | 97 ++++++++++++++++++++- 2 files changed, 103 insertions(+), 9 deletions(-) diff --git a/cassandra/metadata.py b/cassandra/metadata.py index 8a11a645..dfc69e31 100644 --- a/cassandra/metadata.py +++ b/cassandra/metadata.py @@ -158,6 +158,7 @@ class Metadata(object): if old_keyspace_meta: keyspace_meta.tables = old_keyspace_meta.tables keyspace_meta.user_types = old_keyspace_meta.user_types + keyspace_meta.indexes = old_keyspace_meta.indexes if (keyspace_meta.replication_strategy != old_keyspace_meta.replication_strategy): self._keyspace_updated(keyspace) else: @@ -182,13 +183,11 @@ class Metadata(object): if not cf_results: # the table was removed - table_meta = keyspace_meta.tables.pop(table, None) - if table_meta: - keyspace_meta._clear_table_indexes(table_meta.name) + keyspace_meta._drop_table_metadata(table) else: assert len(cf_results) == 1 table_meta = self._build_table_metadata(keyspace_meta, cf_results[0], {table: col_results}, {table: triggers_result}) - keyspace_meta._add_table_metadata(ctable_meta) + keyspace_meta._add_table_metadata(table_meta) def _keyspace_added(self, ksname): if self.token_map: @@ -788,15 +787,15 @@ class KeyspaceMetadata(object): user_type_strings.append(user_type.as_cql_query(formatted=True)) def _add_table_metadata(self, table_metadata): - self._clear_table_indexes(table_metadata.name) + self._drop_table_metadata(table_metadata.name) self.tables[table_metadata.name] = table_metadata for index_name, index_metadata in table_metadata.indexes.iteritems(): self.indexes[index_name] = index_metadata - def _clear_table_indexes(self, table_name): - if table_name in self.tables: - table_meta = self.tables[table_name] + def _drop_table_metadata(self, table_name): + table_meta = self.tables.pop(table_name, None) + if table_meta: for index_name in table_meta.indexes: self.indexes.pop(index_name, None) diff --git a/tests/integration/standard/test_metadata.py b/tests/integration/standard/test_metadata.py index e5b2eab3..a3f1db71 100644 --- a/tests/integration/standard/test_metadata.py +++ b/tests/integration/standard/test_metadata.py @@ -25,7 +25,7 @@ import sys from cassandra import AlreadyExists from cassandra.cluster import Cluster -from cassandra.metadata import (Metadata, KeyspaceMetadata, TableMetadata, +from cassandra.metadata import (Metadata, KeyspaceMetadata, TableMetadata, IndexMetadata, Token, MD5Token, TokenMap, murmur3) from cassandra.policies import SimpleConvictionPolicy from cassandra.pool import Host @@ -928,3 +928,98 @@ class KeyspaceAlterMetadata(unittest.TestCase): new_keyspace_meta = self.cluster.metadata.keyspaces[name] self.assertNotEqual(original_keyspace_meta, new_keyspace_meta) self.assertEqual(new_keyspace_meta.durable_writes, False) + + +class IndexMapTests(unittest.TestCase): + + keyspace_name = 'index_map_tests' + + @property + def table_name(self): + return self._testMethodName.lower() + + @classmethod + def setup_class(cls): + cls.cluster = Cluster(protocol_version=PROTOCOL_VERSION) + cls.session = cls.cluster.connect() + try: + if cls.keyspace_name in cls.cluster.metadata.keyspaces: + cls.session.execute("DROP KEYSPACE %s" % cls.keyspace_name) + + cls.session.execute( + """ + CREATE KEYSPACE %s + WITH replication = {'class': 'SimpleStrategy', 'replication_factor': '1'}; + """ % cls.keyspace_name) + cls.session.set_keyspace(cls.keyspace_name) + except Exception: + cls.cluster.shutdown() + raise + + @classmethod + def teardown_class(cls): + try: + cls.session.execute("DROP KEYSPACE %s" % cls.keyspace_name) + finally: + cls.cluster.shutdown() + + def create_basic_table(self): + self.session.execute("CREATE TABLE %s (k int PRIMARY KEY, a int)" % self.table_name) + + def drop_basic_table(self): + self.session.execute("DROP TABLE %s" % self.table_name) + + def test_index_updates(self): + self.create_basic_table() + + ks_meta = self.cluster.metadata.keyspaces[self.keyspace_name] + table_meta = ks_meta.tables[self.table_name] + self.assertNotIn('a_idx', ks_meta.indexes) + self.assertNotIn('b_idx', ks_meta.indexes) + self.assertNotIn('a_idx', table_meta.indexes) + self.assertNotIn('b_idx', table_meta.indexes) + + self.session.execute("CREATE INDEX a_idx ON %s (a)" % self.table_name) + self.session.execute("ALTER TABLE %s ADD b int" % self.table_name) + self.session.execute("CREATE INDEX b_idx ON %s (b)" % self.table_name) + + ks_meta = self.cluster.metadata.keyspaces[self.keyspace_name] + table_meta = ks_meta.tables[self.table_name] + self.assertIsInstance(ks_meta.indexes['a_idx'], IndexMetadata) + self.assertIsInstance(ks_meta.indexes['b_idx'], IndexMetadata) + self.assertIsInstance(table_meta.indexes['a_idx'], IndexMetadata) + self.assertIsInstance(table_meta.indexes['b_idx'], IndexMetadata) + + # both indexes updated when index dropped + self.session.execute("DROP INDEX a_idx") + ks_meta = self.cluster.metadata.keyspaces[self.keyspace_name] + table_meta = ks_meta.tables[self.table_name] + self.assertNotIn('a_idx', ks_meta.indexes) + self.assertIsInstance(ks_meta.indexes['b_idx'], IndexMetadata) + self.assertNotIn('a_idx', table_meta.indexes) + self.assertIsInstance(table_meta.indexes['b_idx'], IndexMetadata) + + # keyspace index updated when table dropped + self.drop_basic_table() + ks_meta = self.cluster.metadata.keyspaces[self.keyspace_name] + self.assertNotIn(self.table_name, ks_meta.tables) + self.assertNotIn('a_idx', ks_meta.indexes) + self.assertNotIn('b_idx', ks_meta.indexes) + + def test_index_follows_alter(self): + self.create_basic_table() + + idx = self.table_name + '_idx' + self.session.execute("CREATE INDEX %s ON %s (a)" % (idx, self.table_name)) + ks_meta = self.cluster.metadata.keyspaces[self.keyspace_name] + table_meta = ks_meta.tables[self.table_name] + self.assertIsInstance(ks_meta.indexes[idx], IndexMetadata) + self.assertIsInstance(table_meta.indexes[idx], IndexMetadata) + self.session.execute('ALTER KEYSPACE %s WITH durable_writes = false' % self.keyspace_name) + old_meta = ks_meta + ks_meta = self.cluster.metadata.keyspaces[self.keyspace_name] + self.assertIsNot(ks_meta, old_meta) + table_meta = ks_meta.tables[self.table_name] + self.assertIsInstance(ks_meta.indexes[idx], IndexMetadata) + self.assertIsInstance(table_meta.indexes[idx], IndexMetadata) + self.drop_basic_table() From a84d4c0796da4550aa37c006cf1a66efcf62c428 Mon Sep 17 00:00:00 2001 From: Adam Holmberg Date: Wed, 22 Apr 2015 13:28:25 -0500 Subject: [PATCH 4/4] six.iteritems for newly added index dict --- cassandra/metadata.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cassandra/metadata.py b/cassandra/metadata.py index dfc69e31..a3f2bcc3 100644 --- a/cassandra/metadata.py +++ b/cassandra/metadata.py @@ -790,7 +790,7 @@ class KeyspaceMetadata(object): self._drop_table_metadata(table_metadata.name) self.tables[table_metadata.name] = table_metadata - for index_name, index_metadata in table_metadata.indexes.iteritems(): + for index_name, index_metadata in six.iteritems(table_metadata.indexes): self.indexes[index_name] = index_metadata def _drop_table_metadata(self, table_name):