Merge "Remove the extensions repos"

This commit is contained in:
Jenkins 2016-07-29 16:19:53 +00:00 committed by Gerrit Code Review
commit d46b80d7b7
34 changed files with 34 additions and 627 deletions

View File

@ -378,6 +378,15 @@ class Doctor(BaseApp):
raise SystemExit(doctor.diagnose())
def assert_not_extension(extension):
if extension:
print(_("All extensions have been moved into keystone core and as "
"such its migrations are maintained by the main keystone "
"database control. Use the command: keystone-manage "
"db_sync"))
raise RuntimeError
class DbSync(BaseApp):
"""Sync the database."""
@ -393,17 +402,19 @@ class DbSync(BaseApp):
'version. Schema downgrades are not '
'supported.'))
parser.add_argument('--extension', default=None,
help=('Migrate the database for the specified '
'extension. If not provided, db_sync will '
'migrate the common repository.'))
help=('This is a deprecated option to migrate a '
'specified extension. Since extensions are '
'now part of the main repository, '
'specifying db_sync without this option '
'will cause all extensions to be migrated.'))
return parser
@staticmethod
def main():
assert_not_extension(CONF.command.extension)
version = CONF.command.version
extension = CONF.command.extension
migration_helpers.sync_database_to_version(extension, version)
migration_helpers.sync_database_to_version(version)
class DbVersion(BaseApp):
@ -415,15 +426,17 @@ class DbVersion(BaseApp):
def add_argument_parser(cls, subparsers):
parser = super(DbVersion, cls).add_argument_parser(subparsers)
parser.add_argument('--extension', default=None,
help=('Print the migration version of the '
'database for the specified extension. If '
'not provided, print it for the common '
help=('This is a deprecated option to print the '
'version of a specified extension. Since '
'extensions are now part of the main '
'repository, the version of an extension is '
'implicit in the version of the main '
'repository.'))
@staticmethod
def main():
extension = CONF.command.extension
migration_helpers.print_db_version(extension)
assert_not_extension(CONF.command.extension)
migration_helpers.print_db_version()
class BasePermissionsSetup(BaseApp):

View File

@ -15,31 +15,20 @@
# under the License.
import os
import sys
import migrate
from migrate import exceptions
from oslo_db.sqlalchemy import migration
from oslo_utils import importutils
import six
import sqlalchemy
from keystone.common import sql
import keystone.conf
from keystone import contrib
from keystone import exception
from keystone.i18n import _
CONF = keystone.conf.CONF
DEFAULT_EXTENSIONS = []
MIGRATED_EXTENSIONS = ['endpoint_policy',
'federation',
'oauth1',
'revoke',
'endpoint_filter'
]
# Different RDBMSs use different schemes for naming the Foreign Key
@ -158,10 +147,10 @@ def get_init_version(abs_path=None):
return oldest - 1
def _assert_not_schema_downgrade(extension=None, version=None):
def _assert_not_schema_downgrade(version=None):
if version is not None:
try:
current_ver = int(six.text_type(get_db_version(extension)))
current_ver = int(six.text_type(get_db_version()))
if int(version) < current_ver:
raise migration.exception.DbMigrationError(
_("Unable to downgrade schema"))
@ -171,75 +160,16 @@ def _assert_not_schema_downgrade(extension=None, version=None):
pass
def _sync_extension_repo(extension, version):
if extension in MIGRATED_EXTENSIONS:
raise exception.MigrationMovedFailure(extension=extension)
def sync_database_to_version(version=None):
_sync_common_repo(version)
def get_db_version():
with sql.session_for_write() as session:
engine = session.get_bind()
try:
package_name = '.'.join((contrib.__name__, extension))
package = importutils.import_module(package_name)
except ImportError:
raise ImportError(_("%s extension does not exist.")
% package_name)
try:
abs_path = find_migrate_repo(package)
try:
migration.db_version_control(engine, abs_path)
# Register the repo with the version control API
# If it already knows about the repo, it will throw
# an exception that we can safely ignore
except exceptions.DatabaseAlreadyControlledError: # nosec
pass
except exception.MigrationNotProvided as e:
print(e)
sys.exit(1)
_assert_not_schema_downgrade(extension=extension, version=version)
init_version = get_init_version(abs_path=abs_path)
migration.db_sync(engine, abs_path, version=version,
init_version=init_version, sanity_check=False)
return migration.db_version(session.get_bind(),
find_migrate_repo(),
get_init_version())
def sync_database_to_version(extension=None, version=None):
if not extension:
_sync_common_repo(version)
# If version is greater than 0, it is for the common
# repository only, and only that will be synchronized.
if version is None:
for default_extension in DEFAULT_EXTENSIONS:
_sync_extension_repo(default_extension, version)
else:
_sync_extension_repo(extension, version)
def get_db_version(extension=None):
if not extension:
with sql.session_for_write() as session:
return migration.db_version(session.get_bind(),
find_migrate_repo(),
get_init_version())
try:
package_name = '.'.join((contrib.__name__, extension))
package = importutils.import_module(package_name)
except ImportError:
raise ImportError(_("%s extension does not exist.")
% package_name)
with sql.session_for_write() as session:
return migration.db_version(
session.get_bind(), find_migrate_repo(package), 0)
def print_db_version(extension=None):
try:
db_version = get_db_version(extension=extension)
print(db_version)
except exception.MigrationNotProvided as e:
print(e)
sys.exit(1)
def print_db_version():
print(get_db_version())

View File

@ -1,25 +0,0 @@
[db_settings]
# Used to identify which repository this database is versioned under.
# You can use the name of your project.
repository_id=endpoint_filter
# The name of the database table used to track the schema version.
# This name shouldn't already be used by your project.
# If this is changed once a database is under version control, you'll need to
# change the table name in each database too.
version_table=migrate_version
# When committing a change script, Migrate will attempt to generate the
# sql for all supported databases; normally, if one of them fails - probably
# because you don't have that database installed - it is ignored and the
# commit continues, perhaps ending successfully.
# Databases in this list MUST compile successfully during a commit, or the
# entire commit will fail. List the databases your application will actually
# be using to ensure your updates to that database work properly.
# This must be a list; example: ['postgres','sqlite']
required_dbs=[]
# When creating new change scripts, Migrate will stamp the new script with
# a version number. By default this is latest_version + 1. You can set this
# to 'true' to tell Migrate to use the UTC timestamp instead.
use_timestamp_numbering=False

View File

@ -1,19 +0,0 @@
# Copyright 2013 OpenStack Foundation
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
from keystone import exception
def upgrade(migrate_engine):
raise exception.MigrationMovedFailure(extension='endpoint_filter')

View File

@ -1,19 +0,0 @@
# Copyright 2014 Hewlett-Packard Company
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
from keystone import exception
def upgrade(migrate_engine):
raise exception.MigrationMovedFailure(extension='endpoint_filter')

View File

@ -1,25 +0,0 @@
[db_settings]
# Used to identify which repository this database is versioned under.
# You can use the name of your project.
repository_id=federation
# The name of the database table used to track the schema version.
# This name shouldn't already be used by your project.
# If this is changed once a database is under version control, you'll need to
# change the table name in each database too.
version_table=migrate_version
# When committing a change script, Migrate will attempt to generate the
# sql for all supported databases; normally, if one of them fails - probably
# because you don't have that database installed - it is ignored and the
# commit continues, perhaps ending successfully.
# Databases in this list MUST compile successfully during a commit, or the
# entire commit will fail. List the databases your application will actually
# be using to ensure your updates to that database work properly.
# This must be a list; example: ['postgres','sqlite']
required_dbs=[]
# When creating new change scripts, Migrate will stamp the new script with
# a version number. By default this is latest_version + 1. You can set this
# to 'true' to tell Migrate to use the UTC timestamp instead.
use_timestamp_numbering=False

View File

@ -1,17 +0,0 @@
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
from keystone import exception
def upgrade(migrate_engine):
raise exception.MigrationMovedFailure(extension='federation')

View File

@ -1,17 +0,0 @@
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
from keystone import exception
def upgrade(migrate_engine):
raise exception.MigrationMovedFailure(extension='federation')

View File

@ -1,20 +0,0 @@
# Copyright 2014 Mirantis.inc
# All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
from keystone import exception
def upgrade(migrate_engine):
raise exception.MigrationMovedFailure(extension='federation')

View File

@ -1,17 +0,0 @@
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
from keystone import exception
def upgrade(migrate_engine):
raise exception.MigrationMovedFailure(extension='federation')

View File

@ -1,17 +0,0 @@
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
from keystone import exception
def upgrade(migrate_engine):
raise exception.MigrationMovedFailure(extension='federation')

View File

@ -1,17 +0,0 @@
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
from keystone import exception
def upgrade(migrate_engine):
raise exception.MigrationMovedFailure(extension='federation')

View File

@ -1,17 +0,0 @@
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
from keystone import exception
def upgrade(migrate_engine):
raise exception.MigrationMovedFailure(extension='federation')

View File

@ -1,17 +0,0 @@
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
from keystone import exception
def upgrade(migrate_engine):
raise exception.MigrationMovedFailure(extension='federation')

View File

@ -1,25 +0,0 @@
[db_settings]
# Used to identify which repository this database is versioned under.
# You can use the name of your project.
repository_id=oauth1
# The name of the database table used to track the schema version.
# This name shouldn't already be used by your project.
# If this is changed once a database is under version control, you'll need to
# change the table name in each database too.
version_table=migrate_version
# When committing a change script, Migrate will attempt to generate the
# sql for all supported databases; normally, if one of them fails - probably
# because you don't have that database installed - it is ignored and the
# commit continues, perhaps ending successfully.
# Databases in this list MUST compile successfully during a commit, or the
# entire commit will fail. List the databases your application will actually
# be using to ensure your updates to that database work properly.
# This must be a list; example: ['postgres','sqlite']
required_dbs=[]
# When creating new change scripts, Migrate will stamp the new script with
# a version number. By default this is latest_version + 1. You can set this
# to 'true' to tell Migrate to use the UTC timestamp instead.
use_timestamp_numbering=False

View File

@ -1,19 +0,0 @@
# Copyright 2013 OpenStack Foundation
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
from keystone import exception
def upgrade(migrate_engine):
raise exception.MigrationMovedFailure(extension='oauth1')

View File

@ -1,19 +0,0 @@
# Copyright 2013 OpenStack Foundation
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
from keystone import exception
def upgrade(migrate_engine):
raise exception.MigrationMovedFailure(extension='oauth1')

View File

@ -1,19 +0,0 @@
# Copyright 2013 OpenStack Foundation
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
from keystone import exception
def upgrade(migrate_engine):
raise exception.MigrationMovedFailure(extension='oauth1')

View File

@ -1,19 +0,0 @@
# Copyright 2013 OpenStack Foundation
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
from keystone import exception
def upgrade(migrate_engine):
raise exception.MigrationMovedFailure(extension='oauth1')

View File

@ -1,20 +0,0 @@
# Copyright 2014 Mirantis.inc
# All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
from keystone import exception
def upgrade(migrate_engine):
raise exception.MigrationMovedFailure(extension='oauth1')

View File

@ -1,25 +0,0 @@
[db_settings]
# Used to identify which repository this database is versioned under.
# You can use the name of your project.
repository_id=revoke
# The name of the database table used to track the schema version.
# This name shouldn't already be used by your project.
# If this is changed once a database is under version control, you'll need to
# change the table name in each database too.
version_table=migrate_version
# When committing a change script, Migrate will attempt to generate the
# sql for all supported databases; normally, if one of them fails - probably
# because you don't have that database installed - it is ignored and the
# commit continues, perhaps ending successfully.
# Databases in this list MUST compile successfully during a commit, or the
# entire commit will fail. List the databases your application will actually
# be using to ensure your updates to that database work properly.
# This must be a list; example: ['postgres','sqlite']
required_dbs=[]
# When creating new change scripts, Migrate will stamp the new script with
# a version number. By default this is latest_version + 1. You can set this
# to 'true' to tell Migrate to use the UTC timestamp instead.
use_timestamp_numbering=False

View File

@ -1,17 +0,0 @@
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
from keystone import exception
def upgrade(migrate_engine):
raise exception.MigrationMovedFailure(extension='revoke')

View File

@ -1,17 +0,0 @@
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
from keystone import exception
def upgrade(migrate_engine):
raise exception.MigrationMovedFailure(extension='revoke')

View File

@ -538,15 +538,5 @@ class TokenlessAuthConfigError(ValidationError):
'was not found in the request environment.')
class MigrationMovedFailure(RuntimeError):
def __init__(self, extension):
self.extension = extension
msg = _("The %s extension has been moved into keystone core and as "
"such its migrations are maintained by the main keystone "
"database control. Use the command: keystone-manage "
"db_sync") % self.extension
super(MigrationMovedFailure, self).__init__(msg)
class UnsupportedDriverVersion(UnexpectedError):
debug_message_format = _('%(driver)s is not supported driver version')

View File

@ -1,99 +0,0 @@
# Copyright 2012 OpenStack Foundation
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
"""
Test SQL migration extensions.
To run these tests against a live database:
1. Modify the file `keystone/tests/unit/config_files/backend_sql.conf` to use
the connection for your live database.
2. Set up a blank, live database.
3. Run the tests using::
tox -e py27 -- keystone.tests.unit.test_sql_migrate_extensions
WARNING::
Your database will be wiped.
Do not do this against a Database with valuable data as
all data will be lost.
"""
from keystone.contrib import endpoint_filter
from keystone.contrib import federation
from keystone.contrib import oauth1
from keystone.contrib import revoke
from keystone import exception
from keystone.tests.unit import test_sql_upgrade
class SqlUpgradeOAuth1Extension(test_sql_upgrade.SqlMigrateBase):
OAUTH1_MIGRATIONS = 5
def repo_package(self):
return oauth1
def test_upgrade(self):
for version in range(self.OAUTH1_MIGRATIONS):
v = version + 1
self.assertRaises(exception.MigrationMovedFailure,
self.upgrade, version=v,
repository=self.repo_path)
class EndpointFilterExtension(test_sql_upgrade.SqlMigrateBase):
ENDPOINT_FILTER_MIGRATIONS = 2
def repo_package(self):
return endpoint_filter
def test_upgrade(self):
for version in range(self.ENDPOINT_FILTER_MIGRATIONS):
v = version + 1
self.assertRaises(exception.MigrationMovedFailure,
self.upgrade, version=v,
repository=self.repo_path)
class FederationExtension(test_sql_upgrade.SqlMigrateBase):
FEDERATION_MIGRATIONS = 8
def repo_package(self):
return federation
def test_upgrade(self):
for version in range(self.FEDERATION_MIGRATIONS):
v = version + 1
self.assertRaises(exception.MigrationMovedFailure,
self.upgrade, version=v,
repository=self.repo_path)
class RevokeExtension(test_sql_upgrade.SqlMigrateBase):
REVOKE_MIGRATIONS = 2
def repo_package(self):
return revoke
def test_upgrade(self):
for version in range(self.REVOKE_MIGRATIONS):
v = version + 1
self.assertRaises(exception.MigrationMovedFailure,
self.upgrade, version=v,
repository=self.repo_path)

View File

@ -48,7 +48,6 @@ from testtools import matchers
from keystone.common import sql
from keystone.common.sql import migration_helpers
import keystone.conf
from keystone import exception
from keystone.tests import unit
from keystone.tests.unit import default_fixtures
from keystone.tests.unit.ksfixtures import database
@ -1493,22 +1492,3 @@ class VersionTests(SqlMigrateBase):
db_exception.DbMigrationError,
migration_helpers._sync_common_repo,
self.max_version - 1)
def test_extension_not_controlled(self):
"""When get the version before controlling, raises DbMigrationError."""
self.assertRaises(db_exception.DbMigrationError,
migration_helpers.get_db_version,
extension='federation')
def test_unexpected_extension(self):
"""The version for a non-existent extension raises ImportError."""
extension_name = uuid.uuid4().hex
self.assertRaises(ImportError,
migration_helpers.get_db_version,
extension=extension_name)
def test_unversioned_extension(self):
"""The version for extensions without migrations raise an exception."""
self.assertRaises(exception.MigrationNotProvided,
migration_helpers.get_db_version,
extension='admin_crud')