Merge "Add expired_at_int column to trusts"

keystone/common/sql/contract_repo/versions/032_contract_add_expired_at_int_to_trust.py

@@ -0,0 +1,51 @@
+#    Licensed under the Apache License, Version 2.0 (the "License"); you may
+#    not use this file except in compliance with the License. You may obtain
+#    a copy of the License at
+#
+#         http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+#    License for the specific language governing permissions and limitations
+#    under the License.
+
+import datetime
+
+from migrate import UniqueConstraint
+import pytz
+import sqlalchemy as sql
+from sqlalchemy.orm import sessionmaker
+
+
+_epoch = datetime.datetime.fromtimestamp(0, tz=pytz.UTC)
+
+
+def _convert_value_datetime_to_int(dt):
+    dt = dt.replace(tzinfo=pytz.utc)
+    return int((dt - _epoch).total_seconds() * 1000000)
+
+
+def upgrade(migrate_engine):
+    meta = sql.MetaData()
+    meta.bind = migrate_engine
+    maker = sessionmaker(bind=migrate_engine)
+    session = maker()
+
+    trust_table = sql.Table('trust', meta, autoload=True)
+    trusts = list(trust_table.select().execute())
+
+    for trust in trusts:
+        values = {}
+        if trust.expires_at is not None:
+            values['expires_at_int'] = _convert_value_datetime_to_int(
+                trust.expires_at)
+
+            update = trust_table.update().where(
+                trust_table.c.id == trust.id).values(values)
+            session.execute(update)
+            session.commit()
+
+    UniqueConstraint(table=trust_table,
+                     name='duplicate_trust_constraint').drop()
+    session.close()

keystone/common/sql/data_migration_repo/versions/032_migrate_add_expired_at_int_to_trust.py

@@ -0,0 +1,22 @@
+#    Licensed under the Apache License, Version 2.0 (the "License"); you may
+#    not use this file except in compliance with the License. You may obtain
+#    a copy of the License at
+#
+#         http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+#    License for the specific language governing permissions and limitations
+#    under the License.
+
+
+def upgrade(migrate_engine):
+    # A migration here is not needed because the actual marshalling of data
+    # from the old column to the new column is done in the contract phase. This
+    # is because using triggers to convert datetime objects to integers is
+    # complex and error-prone. Instead, we'll migrate the data once all
+    # keystone nodes are on the Queens code-base. From an operator perspective,
+    # this shouldn't affect operability of a rolling upgrade since all nodes
+    # must be running Queens before the contract takes place.
+    pass

keystone/common/sql/expand_repo/versions/032_expand_add_expired_at_int_to_trust.py

@@ -0,0 +1,35 @@
+#    Licensed under the Apache License, Version 2.0 (the "License"); you may
+#    not use this file except in compliance with the License. You may obtain
+#    a copy of the License at
+#
+#         http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+#    License for the specific language governing permissions and limitations
+#    under the License.
+
+
+from migrate import UniqueConstraint
+import sqlalchemy as sql
+
+from keystone.common import sql as ks_sql
+
+
+def upgrade(migrate_engine):
+    meta = sql.MetaData()
+    meta.bind = migrate_engine
+
+    # NOTE(morgan): column is nullable here for migration purposes
+    # it is set to not-nullable in the contract phase to ensure we can handle
+    # rolling upgrades in a sane way. This differs from the model in
+    # keystone.identity.backends.sql_model by design.
+    expires_at = sql.Column('expires_at_int', ks_sql.DateTimeInt())
+    trust_table = sql.Table('trust', meta, autoload=True)
+    trust_table.create_column(expires_at)
+
+    UniqueConstraint('trustor_user_id', 'trustee_user_id', 'project_id',
+                     'impersonation', 'expires_at', 'expires_at_int',
+                     table=trust_table,
+                     name='duplicate_trust_constraint_expanded').create()

keystone/tests/unit/test_backend_sql.py

@@ -44,6 +44,7 @@ from keystone.tests.unit.resource import test_backends as resource_tests
 from keystone.tests.unit.token import test_backends as token_tests
 from keystone.tests.unit.trust import test_backends as trust_tests
 from keystone.token.persistence.backends import sql as token_sql
+from keystone.trust.backends import sql as trust_sql
 
 
 CONF = keystone.conf.CONF
@@ -710,7 +711,15 @@ class SqlIdentity(SqlTests,
 
 
 class SqlTrust(SqlTests, trust_tests.TrustTests):
-    pass
+
+    def test_trust_expires_at_int_matches_expires_at(self):
+        with sql.session_for_write() as session:
+            new_id = uuid.uuid4().hex
+            self.create_sample_trust(new_id)
+            trust_ref = session.query(trust_sql.TrustModel).get(new_id)
+            self.assertIsNotNone(trust_ref._expires_at)
+            self.assertEqual(trust_ref._expires_at, trust_ref.expires_at_int)
+            self.assertEqual(trust_ref.expires_at, trust_ref.expires_at_int)
 
 
 class SqlToken(SqlTests, token_tests.TokenTests):

keystone/tests/unit/test_sql_upgrade.py

@@ -2531,6 +2531,64 @@ class FullMigration(SqlMigrateBase, unit.TestCase):
         }
         system_assignment_table.insert().values(system_group).execute()
 
+    def test_migration_032_add_expires_at_int_column_trust(self):
+
+        self.expand(31)
+        self.migrate(31)
+        self.contract(31)
+
+        trust_table_name = 'trust'
+
+        self.assertTableColumns(
+            trust_table_name,
+            ['id', 'trustor_user_id', 'trustee_user_id', 'project_id',
+             'impersonation', 'deleted_at', 'expires_at', 'remaining_uses',
+             'extra'],
+        )
+
+        self.expand(32)
+
+        self.assertTableColumns(
+            trust_table_name,
+            ['id', 'trustor_user_id', 'trustee_user_id', 'project_id',
+             'impersonation', 'deleted_at', 'expires_at', 'expires_at_int',
+             'remaining_uses', 'extra'],
+        )
+
+        # Create Trust
+        trust_table = sqlalchemy.Table('trust', self.metadata,
+                                       autoload=True)
+        trust_1_data = {
+            'id': uuid.uuid4().hex,
+            'trustor_user_id': uuid.uuid4().hex,
+            'trustee_user_id': uuid.uuid4().hex,
+            'project_id': uuid.uuid4().hex,
+            'impersonation': False,
+            'expires_at': datetime.datetime.utcnow()
+        }
+        trust_2_data = {
+            'id': uuid.uuid4().hex,
+            'trustor_user_id': uuid.uuid4().hex,
+            'trustee_user_id': uuid.uuid4().hex,
+            'project_id': uuid.uuid4().hex,
+            'impersonation': False,
+            'expires_at': None
+        }
+        trust_table.insert().values(trust_1_data).execute()
+        trust_table.insert().values(trust_2_data).execute()
+
+        self.migrate(32)
+        self.contract(32)
+        trusts = list(trust_table.select().execute())
+
+        epoch = datetime.datetime.fromtimestamp(0, tz=pytz.UTC)
+
+        for t in trusts:
+            if t.expires_at:
+                e = t.expires_at.replace(tzinfo=pytz.UTC) - epoch
+                e = e.total_seconds()
+                self.assertEqual(t.expires_at_int, int(e * 1000000))
+
 
 class MySQLOpportunisticFullMigration(FullMigration):
     FIXTURE = test_base.MySQLOpportunisticFixture

keystone/trust/backends/sql.py

@@ -14,6 +14,7 @@
 
 from oslo_utils import timeutils
 from six.moves import range
+from sqlalchemy.ext.hybrid import hybrid_property
 
 from keystone.common import sql
 from keystone import exception
@@ -38,7 +39,8 @@ class TrustModel(sql.ModelBase, sql.ModelDictMixinWithExtras):
     project_id = sql.Column(sql.String(64))
     impersonation = sql.Column(sql.Boolean, nullable=False)
     deleted_at = sql.Column(sql.DateTime)
-    expires_at = sql.Column(sql.DateTime)
+    _expires_at = sql.Column('expires_at', sql.DateTime)
+    expires_at_int = sql.Column(sql.DateTimeInt(), nullable=True)
     remaining_uses = sql.Column(sql.Integer, nullable=True)
     extra = sql.Column(sql.JsonBlob())
     __table_args__ = (sql.UniqueConstraint(
@@ -46,6 +48,15 @@ class TrustModel(sql.ModelBase, sql.ModelDictMixinWithExtras):
                       'impersonation', 'expires_at',
                       name='duplicate_trust_constraint'),)
 
+    @hybrid_property
+    def expires_at(self):
+        return self.expires_at_int or self._expires_at
+
+    @expires_at.setter
+    def expires_at(self, value):
+        self._expires_at = value
+        self.expires_at_int = value
+
 
 class TrustRole(sql.ModelBase):
     __tablename__ = 'trust_role'

releasenotes/notes/add-expires-at-int-to-trusts-60ae3c5d0c00808a.yaml

@@ -0,0 +1,8 @@
+---
+upgrade:
+  - |
+    The trusts table now has an expires_at_int column that represents the
+    expiration time as an integer instead of a datetime object. This will
+    prevent rounding errors related to the way date objects are stored in some
+    versions of MySQL. The expires_at column remains, but will be dropped in
+    Rocky.