db schema improvements: blob to json text

Tacker does consider auth, placement_attr and similar
Objects as python dict but its stored as blob object
in db, which increases the storage space as well while
Debugging, these values can't be understandable as
its in binary format. So this patch stores all these
Objects in json string format, which enables readability
and reduces the size required to stored the object.

Closes-bug: #1587579

Change-Id: Idaee81028d40f60b70db12c5ae6ba78cb19f1992

tacker/db/migration/alembic_migrations/versions/6e56d4474b2a_blob_to_json_text.py

@@ -0,0 +1,55 @@
+# Copyright 2016 OpenStack Foundation
+#
+#    Licensed under the Apache License, Version 2.0 (the "License"); you may
+#    not use this file except in compliance with the License. You may obtain
+#    a copy of the License at
+#
+#         http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+#    License for the specific language governing permissions and limitations
+#    under the License.
+#
+
+"""blob-to-json-text
+
+Revision ID: 6e56d4474b2a
+Revises: f958f58e5daa
+Create Date: 2016-06-01 09:50:46.296206
+
+"""
+
+import json
+import pickle
+
+from alembic import op
+import sqlalchemy as sa
+
+from tacker.db import types
+
+# revision identifiers, used by Alembic.
+revision = '6e56d4474b2a'
+down_revision = 'f958f58e5daa'
+
+
+def _migrate_data(table, column_name):
+    meta = sa.MetaData(bind=op.get_bind())
+    t = sa.Table(table, meta, autoload=True)
+
+    for r in t.select().execute():
+        stmt = t.update().where(t.c.id == r.id).values(
+            {column_name: json.dumps(pickle.loads(getattr(r, column_name)))})
+        op.execute(stmt)
+
+    op.alter_column(table,
+                    column_name,
+                    type_=types.Json)
+
+
+def upgrade(active_plugins=None, options=None):
+    _migrate_data('vims', 'placement_attr')
+    _migrate_data('vimauths', 'vim_project')
+    _migrate_data('vimauths', 'auth_cred')
+    _migrate_data('devices', 'placement_attr')

tacker/db/migration/alembic_migrations/versions/HEAD

@@ -1 +1 @@
-f958f58e5daa
+6e56d4474b2a

tacker/db/nfvo/nfvo_db.py

@@ -40,7 +40,7 @@ class Vim(model_base.BASE, models_v1.HasId, models_v1.HasTenant):
     type = sa.Column(sa.String(255), nullable=False)
     name = sa.Column(sa.String(255), nullable=True)
     description = sa.Column(sa.String(255), nullable=True)
-    placement_attr = sa.Column(sa.PickleType, nullable=True)
+    placement_attr = sa.Column(types.Json, nullable=True)
     shared = sa.Column(sa.Boolean, default=True, server_default=sql.true(
     ), nullable=False)
     vim_auth = orm.relationship('VimAuth')
@@ -51,8 +51,8 @@ class VimAuth(model_base.BASE, models_v1.HasId):
                        nullable=False)
     password = sa.Column(sa.String(128), nullable=False)
     auth_url = sa.Column(sa.String(255), nullable=False)
-    vim_project = sa.Column(sa.PickleType, nullable=False)
-    auth_cred = sa.Column(sa.PickleType, nullable=False)
+    vim_project = sa.Column(types.Json, nullable=False)
+    auth_cred = sa.Column(types.Json, nullable=False)
     __table_args__ = (sa.UniqueConstraint('auth_url'), {})
 
 

tacker/db/types.py

@@ -12,10 +12,11 @@
 # License for the specific language governing permissions and limitations
 # under the License.
 
-
+import json
 import uuid
 
 from sqlalchemy.types import String
+from sqlalchemy.types import Text
 from sqlalchemy.types import TypeDecorator
 
 
@@ -34,3 +35,15 @@ class Uuid(TypeDecorator):
 
     def process_result_value(self, value, dialect):
         return value
+
+
+class Json(TypeDecorator):
+    impl = Text
+
+    def process_bind_param(self, value, dialect):
+        return json.dumps(value)
+
+    def process_result_value(self, value, dialect):
+        if value is None:
+            return None
+        return json.loads(value)

tacker/db/vm/vm_db.py

@@ -113,7 +113,7 @@ class Device(model_base.BASE, models_v1.HasId, models_v1.HasTenant):
 
     status = sa.Column(sa.String(255), nullable=False)
     vim_id = sa.Column(types.Uuid, sa.ForeignKey('vims.id'), nullable=False)
-    placement_attr = sa.Column(sa.PickleType, nullable=True)
+    placement_attr = sa.Column(types.Json, nullable=True)
     vim = orm.relationship('Vim')
     error_reason = sa.Column(sa.Text, nullable=True)