Merge "sql: Squash pike migrations"

This commit is contained in:
Zuul 2022-02-08 13:34:59 +00:00 committed by Gerrit Code Review
commit 9daad37b2f
29 changed files with 15 additions and 560 deletions

View File

@ -1,18 +0,0 @@
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
# This is a placeholder for Ocata backports. Do not use this number for new
# Pike work. New Pike work starts after all the placeholders.
def upgrade(migrate_engine):
pass

View File

@ -1,18 +0,0 @@
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
# This is a placeholder for Ocata backports. Do not use this number for new
# Pike work. New Pike work starts after all the placeholders.
def upgrade(migrate_engine):
pass

View File

@ -1,18 +0,0 @@
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
# This is a placeholder for Ocata backports. Do not use this number for new
# Pike work. New Pike work starts after all the placeholders.
def upgrade(migrate_engine):
pass

View File

@ -1,18 +0,0 @@
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
# This is a placeholder for Ocata backports. Do not use this number for new
# Pike work. New Pike work starts after all the placeholders.
def upgrade(migrate_engine):
pass

View File

@ -1,18 +0,0 @@
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
# This is a placeholder for Ocata backports. Do not use this number for new
# Pike work. New Pike work starts after all the placeholders.
def upgrade(migrate_engine):
pass

View File

@ -1,15 +0,0 @@
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
def upgrade(migrate_engine):
pass

View File

@ -1,15 +0,0 @@
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
def upgrade(migrate_engine):
pass

View File

@ -1,61 +0,0 @@
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
import datetime
import pytz
import sqlalchemy as sql
from sqlalchemy.orm import sessionmaker
_epoch = datetime.datetime.fromtimestamp(0, tz=pytz.UTC)
def _convert_value_datetime_to_int(dt):
dt = dt.replace(tzinfo=pytz.utc)
return int((dt - _epoch).total_seconds() * 1000000)
def upgrade(migrate_engine):
meta = sql.MetaData()
meta.bind = migrate_engine
maker = sessionmaker(bind=migrate_engine)
session = maker()
password_table = sql.Table('password', meta, autoload=True)
passwords = list(password_table.select().execute())
for passwd in passwords:
values = {
'created_at_int': _convert_value_datetime_to_int(passwd.created_at)
}
if passwd.expires_at is not None:
values['expires_at_int'] = _convert_value_datetime_to_int(
passwd.expires_at)
update = password_table.update().where(
password_table.c.id == passwd.id).values(values)
session.execute(update)
session.commit()
password_table = sql.Table('password', meta, autoload=True)
# The created_at_int data cannot really be nullable long term. This
# corrects the data to be not nullable, but must be done in the contract
# phase for two reasons. The first is due to "additive only" requirements.
# The second is because we need to ensure all nodes in the deployment are
# running the Pike code-base before we migrate all password entries. This
# avoids locking the password table or having a partial outage while doing
# the migration.
password_table.c.created_at_int.alter(nullable=False, default=0,
server_default='0')
session.close()

View File

@ -1,18 +0,0 @@
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
# This is a placeholder for Ocata backports. Do not use this number for new
# Pike work. New Pike work starts after all the placeholders.
def upgrade(migrate_engine):
pass

View File

@ -1,18 +0,0 @@
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
# This is a placeholder for Ocata backports. Do not use this number for new
# Pike work. New Pike work starts after all the placeholders.
def upgrade(migrate_engine):
pass

View File

@ -1,18 +0,0 @@
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
# This is a placeholder for Ocata backports. Do not use this number for new
# Pike work. New Pike work starts after all the placeholders.
def upgrade(migrate_engine):
pass

View File

@ -1,18 +0,0 @@
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
# This is a placeholder for Ocata backports. Do not use this number for new
# Pike work. New Pike work starts after all the placeholders.
def upgrade(migrate_engine):
pass

View File

@ -1,18 +0,0 @@
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
# This is a placeholder for Ocata backports. Do not use this number for new
# Pike work. New Pike work starts after all the placeholders.
def upgrade(migrate_engine):
pass

View File

@ -1,15 +0,0 @@
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
def upgrade(migrate_engine):
pass

View File

@ -1,15 +0,0 @@
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
def upgrade(migrate_engine):
pass

View File

@ -1,22 +0,0 @@
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
def upgrade(migrate_engine):
# A migration here is not needed because the actual marshalling of data
# from the old column to the new column is done in the contract phase. This
# is because using triggers to convert datetime objects to integers is
# complex and error-prone. Instead, we'll migrate the data once all
# keystone nodes are on the Pike code-base. From an operator perspective,
# this shouldn't affect operability of a rolling upgrade since all nodes
# must be running Pike before the contract takes place.
pass

View File

@ -1,18 +0,0 @@
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
# This is a placeholder for Ocata backports. Do not use this number for new
# Pike work. New Pike work starts after all the placeholders.
def upgrade(migrate_engine):
pass

View File

@ -1,18 +0,0 @@
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
# This is a placeholder for Ocata backports. Do not use this number for new
# Pike work. New Pike work starts after all the placeholders.
def upgrade(migrate_engine):
pass

View File

@ -1,18 +0,0 @@
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
# This is a placeholder for Ocata backports. Do not use this number for new
# Pike work. New Pike work starts after all the placeholders.
def upgrade(migrate_engine):
pass

View File

@ -1,18 +0,0 @@
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
# This is a placeholder for Ocata backports. Do not use this number for new
# Pike work. New Pike work starts after all the placeholders.
def upgrade(migrate_engine):
pass

View File

@ -1,18 +0,0 @@
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
# This is a placeholder for Ocata backports. Do not use this number for new
# Pike work. New Pike work starts after all the placeholders.
def upgrade(migrate_engine):
pass

View File

@ -1,21 +0,0 @@
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
import sqlalchemy as sql
def upgrade(migrate_engine):
meta = sql.MetaData()
meta.bind = migrate_engine
user = sql.Table('user', meta, autoload=True)
sql.Index('ix_default_project_id', user.c.default_project_id).create()

View File

@ -1,25 +0,0 @@
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
import sqlalchemy as sql
def upgrade(migrate_engine):
meta = sql.MetaData()
meta.bind = migrate_engine
# NOTE(notmorgan): To support the full range of scrypt and pbkfd password
# hash lengths, this should be closer to varchar(1500) instead of
# varchar(255).
password_hash = sql.Column('password_hash', sql.String(255), nullable=True)
password_table = sql.Table('password', meta, autoload=True)
password_table.create_column(password_hash)

View File

@ -1,33 +0,0 @@
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
import sqlalchemy as sql
from keystone.common import sql as ks_sql
def upgrade(migrate_engine):
meta = sql.MetaData()
meta.bind = migrate_engine
# NOTE(morgan): column is nullable here for migration purposes
# it is set to not-nullable in the contract phase to ensure we can handle
# rolling upgrades in a sane way. This differs from the model in
# keystone.identity.backends.sql_model by design.
created_at = sql.Column('created_at_int', ks_sql.DateTimeInt(),
nullable=True)
expires_at = sql.Column('expires_at_int', ks_sql.DateTimeInt(),
nullable=True)
password_table = sql.Table('password', meta, autoload=True)
password_table.create_column(created_at)
password_table.create_column(expires_at)

View File

@ -264,6 +264,18 @@ def upgrade(migrate_engine):
server_default='0',
default=False,
),
# NOTE(notmorgan): To support the full range of scrypt and pbkfd
# password hash lengths, this should be closer to varchar(1500) instead
# of varchar(255).
sql.Column('password_hash', sql.String(255), nullable=True),
sql.Column(
'created_at_int',
ks_sql.DateTimeInt(),
nullable=False,
default=0,
server_default='0',
),
sql.Column('expires_at_int', ks_sql.DateTimeInt(), nullable=True),
sql.Column(
'created_at',
sql.DateTime(),
@ -545,6 +557,7 @@ def upgrade(migrate_engine):
nullable=False,
),
sql.UniqueConstraint('id', 'domain_id', name='ixu_user_id_domain_id'),
sql.Index('ix_default_project_id', 'default_project_id'),
mysql_engine='InnoDB',
mysql_charset='utf8',
)

View File

@ -29,7 +29,7 @@ from keystone.i18n import _
USE_TRIGGERS = True
INITIAL_VERSION = 15
INITIAL_VERSION = 23
EXPAND_REPO = 'expand_repo'
DATA_MIGRATION_REPO = 'data_migration_repo'
CONTRACT_REPO = 'contract_repo'

View File

@ -187,7 +187,7 @@ INITIAL_TABLE_STRUCTURE = {
],
'password': [
'id', 'local_user_id', 'password', 'created_at', 'expires_at',
'self_service',
'self_service', 'password_hash', 'created_at_int', 'expires_at_int',
],
'federated_user': [
'id', 'user_id', 'idp_id', 'protocol_id', 'unique_id', 'display_name',
@ -605,72 +605,6 @@ class FullMigration(MigrateBase, unit.TestCase):
upgrades.INITIAL_VERSION + 2,
)
def test_migration_024_add_created_expires_at_int_columns_password(self):
self.expand(23)
self.migrate(23)
self.contract(23)
password_table_name = 'password'
self.assertTableColumns(
password_table_name,
['id', 'local_user_id', 'password', 'password_hash', 'created_at',
'expires_at', 'self_service']
)
self.expand(24)
self.assertTableColumns(
password_table_name,
['id', 'local_user_id', 'password', 'password_hash', 'created_at',
'expires_at', 'created_at_int', 'expires_at_int', 'self_service']
)
# Create User and Local User
project_table = sqlalchemy.Table('project', self.metadata,
autoload=True)
domain_data = {'id': '_domain', 'domain_id': '_domain',
'enabled': True, 'name': '_domain', 'is_domain': True}
project_table.insert().values(domain_data).execute()
user_table = sqlalchemy.Table('user', self.metadata, autoload=True)
user_id = uuid.uuid4().hex
user = {'id': user_id, 'enabled': True, 'domain_id': domain_data['id']}
user_table.insert().values(user).execute()
local_user_table = sqlalchemy.Table('local_user', self.metadata,
autoload=True)
local_user = {
'id': 1, 'user_id': user_id, 'domain_id': user['domain_id'],
'name': 'name'}
local_user_table.insert().values(local_user).execute()
password_table = sqlalchemy.Table('password',
self.metadata, autoload=True)
password_data = {
'local_user_id': local_user['id'],
'created_at': datetime.datetime.utcnow(),
'expires_at': datetime.datetime.utcnow()}
password_table.insert().values(password_data).execute()
self.migrate(24)
self.contract(24)
passwords = list(password_table.select().execute())
epoch = datetime.datetime.fromtimestamp(0, tz=pytz.UTC)
for p in passwords:
c = (p.created_at.replace(tzinfo=pytz.UTC) - epoch).total_seconds()
e = (p.expires_at.replace(tzinfo=pytz.UTC) - epoch).total_seconds()
self.assertEqual(p.created_at_int, int(c * 1000000))
self.assertEqual(p.expires_at_int, int(e * 1000000))
# Test contract phase and ensure data can not be null
self.contract(24)
meta = sqlalchemy.MetaData(self.engine)
pw_table = sqlalchemy.Table('password', meta, autoload=True)
self.assertFalse(pw_table.c.created_at_int.nullable)
def test_migration_030_expand_add_project_tags_table(self):
self.expand(29)
self.migrate(29)