From 2c21d660b34e7e14081ed72fb040b53ac3400ed3 Mon Sep 17 00:00:00 2001 From: Sergey Kraynev <skraynev@mirantis.com> Date: Wed, 11 Mar 2015 13:28:23 +1000 Subject: [PATCH] Add extra columns for resource table This patch adds following columns for resource table: - `needed_by` (a list of Resource keys) - `requires` (a list of Resource keys) - `replaces` (a single Resource key, Null by default) - `replaced_by` (a single Resource key, Null by default) - `current_template_id` (a single RawTemplate key) Co-Authored-by: Angus Salkeld <asalkeld@mirantis.com> Co-Authored-By: Qiming Teng <tengqim@cn.ibm.com> Change-Id: I65e1032e84b40cb7ae3126fa6b63c914988cc970 Implements: blueprint convergence-resource-table --- .../versions/060_resource_convg_data.py | 89 +++++++++++++++++++ heat/db/sqlalchemy/models.py | 11 +++ heat/db/sqlalchemy/types.py | 55 ++++++++++++ heat/engine/resource.py | 20 +++++ heat/objects/fields.py | 4 + heat/objects/resource.py | 5 ++ heat/tests/db/test_migrations.py | 6 ++ heat/tests/db/test_sqlalchemy_types.py | 58 ++++++++++++ 8 files changed, 248 insertions(+) create mode 100644 heat/db/sqlalchemy/migrate_repo/versions/060_resource_convg_data.py create mode 100644 heat/tests/db/test_sqlalchemy_types.py diff --git a/heat/db/sqlalchemy/migrate_repo/versions/060_resource_convg_data.py b/heat/db/sqlalchemy/migrate_repo/versions/060_resource_convg_data.py new file mode 100644 index 0000000000..5e64831adf --- /dev/null +++ b/heat/db/sqlalchemy/migrate_repo/versions/060_resource_convg_data.py @@ -0,0 +1,89 @@ +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +from migrate.changeset import constraint +import sqlalchemy + +from heat.db.sqlalchemy import types +from heat.db.sqlalchemy import utils as migrate_utils + + +def upgrade(migrate_engine): + meta = sqlalchemy.MetaData(bind=migrate_engine) + + resource = sqlalchemy.Table('resource', meta, autoload=True) + raw_template = sqlalchemy.Table('raw_template', meta, autoload=True) + + needed_by = sqlalchemy.Column('needed_by', types.List) + requires = sqlalchemy.Column('requires', types.List) + replaces = sqlalchemy.Column('replaces', sqlalchemy.Integer) + replaced_by = sqlalchemy.Column('replaced_by', sqlalchemy.Integer) + current_template_id = sqlalchemy.Column('current_template_id', + sqlalchemy.Integer) + needed_by.create(resource) + requires.create(resource) + replaces.create(resource) + replaced_by.create(resource) + current_template_id.create(resource) + + fkey = constraint.ForeignKeyConstraint( + columns=[resource.c.current_template_id], + refcolumns=[raw_template.c.id], + name='current_template_fkey_ref') + fkey.create() + + +def downgrade(migrate_engine): + if migrate_engine.name == 'sqlite': + _downgrade_sqlite(migrate_engine) + return + + meta = sqlalchemy.MetaData(bind=migrate_engine) + + resource = sqlalchemy.Table('resource', meta, autoload=True) + raw_template = sqlalchemy.Table('raw_template', meta, autoload=True) + + fkey = constraint.ForeignKeyConstraint( + columns=[resource.c.current_template_id], + refcolumns=[raw_template.c.id], + name='current_template_fkey_ref') + fkey.drop() + resource.c.current_template_id.drop() + resource.c.needed_by.drop() + resource.c.requires.drop() + resource.c.replaces.drop() + resource.c.replaced_by.drop() + + +def _downgrade_sqlite(migrate_engine): + meta = sqlalchemy.MetaData() + meta.bind = migrate_engine + + resource_table = sqlalchemy.Table('resource', meta, autoload=True) + + ignorecons = ['current_template_fkey_ref'] + ignorecols = [resource_table.c.current_template_id.name, + resource_table.c.needed_by.name, + resource_table.c.requires.name, + resource_table.c.replaces.name, + resource_table.c.replaced_by.name] + new_resource = migrate_utils.clone_table('new_resource', + resource_table, + meta, ignorecols=ignorecols, + ignorecons=ignorecons) + + # migrate resources to new table + migrate_utils.migrate_data(migrate_engine, + resource_table, + new_resource, + skip_columns=ignorecols) diff --git a/heat/db/sqlalchemy/models.py b/heat/db/sqlalchemy/models.py index 037bbf9efe..37d81a7063 100644 --- a/heat/db/sqlalchemy/models.py +++ b/heat/db/sqlalchemy/models.py @@ -310,6 +310,17 @@ class Resource(BASE, HeatBase, StateAware): engine_id = sqlalchemy.Column(sqlalchemy.String(36)) atomic_key = sqlalchemy.Column(sqlalchemy.Integer) + needed_by = sqlalchemy.Column('needed_by', types.List) + requires = sqlalchemy.Column('requires', types.List) + replaces = sqlalchemy.Column('replaces', sqlalchemy.Integer, + default=None) + replaced_by = sqlalchemy.Column('replaced_by', sqlalchemy.Integer, + default=None) + current_template_id = sqlalchemy.Column( + 'current_template_id', + sqlalchemy.Integer, + sqlalchemy.ForeignKey('raw_template.id')) + class WatchRule(BASE, HeatBase): """Represents a watch_rule created by the heat engine.""" diff --git a/heat/db/sqlalchemy/types.py b/heat/db/sqlalchemy/types.py index 79cc04b483..5d6ff17eca 100644 --- a/heat/db/sqlalchemy/types.py +++ b/heat/db/sqlalchemy/types.py @@ -42,5 +42,60 @@ class Json(LongText): return loads(value) +class MutableList(mutable.Mutable, list): + @classmethod + def coerce(cls, key, value): + if not isinstance(value, cls): + if isinstance(value, list): + return cls(value) + return mutable.Mutable.coerce(key, value) + else: + return value + + def __delitem__(self, key): + list.__delitem__(self, key) + self.changed() + + def __setitem__(self, key, value): + list.__setitem__(self, key, value) + self.changed() + + def __getstate__(self): + return list(self) + + def __setstate__(self, state): + len = list.__len__(self) + list.__delslice__(self, 0, len) + list.__add__(self, state) + self.changed() + + def append(self, value): + list.append(self, value) + self.changed() + + def remove(self, value): + list.remove(self, value) + self.changed() + + +class List(types.TypeDecorator): + impl = types.Text + + def load_dialect_impl(self, dialect): + if dialect.name == 'mysql': + return dialect.type_descriptor(mysql.LONGTEXT()) + else: + return self.impl + + def process_bind_param(self, value, dialect): + return dumps(value) + + def process_result_value(self, value, dialect): + if value is None: + return None + return loads(value) + + +MutableList.associate_with(List) mutable.MutableDict.associate_with(LongText) mutable.MutableDict.associate_with(Json) diff --git a/heat/engine/resource.py b/heat/engine/resource.py index 0b52f97f3d..2f4ff67757 100644 --- a/heat/engine/resource.py +++ b/heat/engine/resource.py @@ -171,6 +171,11 @@ class Resource(object): self.created_time = None self.updated_time = None self._rpc_client = None + self.needed_by = None + self.requires = None + self.replaces = None + self.replaced_by = None + self.current_template_id = stack.t.id resource = stack.db_resource_get(name) if resource: @@ -199,6 +204,11 @@ class Resource(object): self._stored_properties_data = resource.properties_data self.created_time = resource.created_at self.updated_time = resource.updated_at + self.needed_by = resource.needed_by + self.requires = resource.requires + self.replaces = resource.replaces + self.replaced_by = resource.replaced_by + self.current_template_id = resource.current_template_id def reparse(self): self.properties = self.t.properties(self.properties_schema, @@ -906,6 +916,11 @@ class Resource(object): 'name': self.name, 'rsrc_metadata': metadata, 'properties_data': self._stored_properties_data, + 'needed_by': self.needed_by, + 'requires': self.requires, + 'replaces': self.replaces, + 'replaced_by': self.replaced_by, + 'current_template_id': self.current_template_id, 'stack_name': self.stack.name} new_rs = resource_objects.Resource.create(self.context, rs) @@ -939,6 +954,11 @@ class Resource(object): 'stack_id': self.stack.id, 'updated_at': self.updated_time, 'properties_data': self._stored_properties_data, + 'needed_by': self.needed_by, + 'requires': self.requires, + 'replaces': self.replaces, + 'replaced_by': self.replaced_by, + 'current_template_id': self.current_template_id, 'nova_instance': self.resource_id}) except Exception as ex: LOG.error(_LE('DB error %s'), ex) diff --git a/heat/objects/fields.py b/heat/objects/fields.py index 28a5dc02f3..82fab1d5ba 100644 --- a/heat/objects/fields.py +++ b/heat/objects/fields.py @@ -34,3 +34,7 @@ class Json(fields.FieldType): class JsonField(fields.AutoTypedField): pass + + +class ListField(fields.AutoTypedField): + pass diff --git a/heat/objects/resource.py b/heat/objects/resource.py index 3dc15e52b8..b5e9cf2b0b 100755 --- a/heat/objects/resource.py +++ b/heat/objects/resource.py @@ -52,6 +52,11 @@ class Resource( 'stack': fields.ObjectField(stack.Stack, nullable=False), 'engine_id': fields.StringField(nullable=True), 'atomic_key': fields.IntegerField(nullable=True), + 'current_template_id': fields.IntegerField(), + 'needed_by': heat_fields.ListField(nullable=True, default=None), + 'requires': heat_fields.ListField(nullable=True, default=None), + 'replaces': fields.IntegerField(nullable=True), + 'replaced_by': fields.IntegerField(nullable=True), } @staticmethod diff --git a/heat/tests/db/test_migrations.py b/heat/tests/db/test_migrations.py index e769239e50..23b3f7d4ff 100644 --- a/heat/tests/db/test_migrations.py +++ b/heat/tests/db/test_migrations.py @@ -584,6 +584,12 @@ class HeatMigrationsCheckers(test_migrations.WalkVersionsMixin, else: self.assertColumnIsNullable(engine, 'sync_point', column[0]) + def _check_060(self, engine, data): + column_list = ['needed_by', 'requires', 'replaces', 'replaced_by', + 'current_template_id'] + for column in column_list: + self.assertColumnExists(engine, 'resource', column) + class TestHeatMigrationsMySQL(HeatMigrationsCheckers, test_base.MySQLOpportunisticTestCase): diff --git a/heat/tests/db/test_sqlalchemy_types.py b/heat/tests/db/test_sqlalchemy_types.py new file mode 100644 index 0000000000..f85317d78c --- /dev/null +++ b/heat/tests/db/test_sqlalchemy_types.py @@ -0,0 +1,58 @@ +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +from sqlalchemy.dialects.mysql import base as mysql_base +from sqlalchemy.dialects.sqlite import base as sqlite_base +from sqlalchemy import types +import testtools + +from heat.db.sqlalchemy import types as db_types + + +class ListTest(testtools.TestCase): + + def setUp(self): + super(ListTest, self).setUp() + self.sqltype = db_types.List() + + def test_load_dialect_impl(self): + dialect = mysql_base.MySQLDialect() + impl = self.sqltype.load_dialect_impl(dialect) + self.assertNotEqual(types.Text, type(impl)) + dialect = sqlite_base.SQLiteDialect() + impl = self.sqltype.load_dialect_impl(dialect) + self.assertEqual(types.Text, type(impl)) + + def test_process_bind_param(self): + dialect = None + value = ['foo', 'bar'] + result = self.sqltype.process_bind_param(value, dialect) + self.assertEqual('["foo", "bar"]', result) + + def test_process_bind_param_null(self): + dialect = None + value = None + result = self.sqltype.process_bind_param(value, dialect) + self.assertEqual('null', result) + + def test_process_result_value(self): + dialect = None + value = '["foo", "bar"]' + result = self.sqltype.process_result_value(value, dialect) + self.assertEqual(['foo', 'bar'], result) + + def test_process_result_value_null(self): + dialect = None + value = None + result = self.sqltype.process_result_value(value, dialect) + self.assertIsNone(result)