openstack/deb-murano
Code Issues Proposed changes

Added DB migrations on Alembic

Browse Source 
This commit migrates Murano database migration framework from
sqlalchemy-migrate to Alembic. sqlalchemy-migrate is considered abandoned. All
other OpenStack projects are in  process of migration to Alembic or have
already finished migration.

This change doesn't preserve all the migration scripts, it merges them into a
single migration script. That's what we would do even without Alembic. It's a
common practice to squash migration scripts into a single one right after the
release.

NOTICE:
CLI command to run migrations changed. Now it looks like this:
$ murano-db-manage upgrade

Migration path:
1. Simple solution is just to re-create your database from scratch
2. You can stamp your database with revision to make Alembic think that it
   already applied the first migration:
   $ tox -e venv -- murano-db-manage --config-file etc/murano/murano.conf stamp --revision head

For usage instructions see:
murano/db/migraiton/alembic_migrations/README

This patch also adds opportunistic tests for migrations. These tests will run in
OpenStack Infrastructure jenkins slaves. It'll use real MySQL and Postgres databases.
Tests will run in a "snake walk" manner, which means that upgrade and downgrade path
will be tested.
Base for migration tests is copied from Nova. Please note, that at this moment we
cannot use code from oslo, because it is not documented and is not tested. Once
test_migrations_base appears in project 'oslo.db', we'll be able to remove our own
test_migrations_base and use the one from 'oslo.db'.

implements: blueprint alembic-migrations
Change-Id: I5aa978f7095efc57f2d6fad81b5553e1880ad931
This commit is contained in:
Ruslan Kamaldinov 2014-05-29 17:52:36 +04:00
parent d59a6d28e7
commit 9030bc7285
37 changed files with 1570 additions and 660 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

6
MANIFEST.in
View File

@ -10,8 +10,8 @@ include ChangeLog
include babel.cfg include babel.cfg
include tox.ini include tox.ini
include openstack-common.conf include openstack-common.conf
include muranoapi/db/migrate_repo/README include murano/db/migration/alembic.ini
include muranoapi/db/migrate_repo/migrate.cfg include murano/db/migration/alembic_migrations/README
include muranoapi/db/migrate_repo/versions/*.sql include murano/db/migration/alembic_migrations/script.py.mako
recursive-include etc * recursive-include etc *
global-exclude *.pyc global-exclude *.pyc

2
contrib/devstack/lib/murano
View File

@ -171,7 +171,7 @@ function init_murano() {
# (re)create Murano database # (re)create Murano database
recreate_database murano utf8 recreate_database murano utf8
$MURANO_BIN_DIR/murano-manage --config-file $MURANO_CONF_FILE db-sync $MURANO_BIN_DIR/murano-db-manage --config-file $MURANO_CONF_FILE upgrade
$MURANO_BIN_DIR/murano-manage --config-file $MURANO_CONF_FILE import-package $MURANO_DIR/meta/io.murano $MURANO_BIN_DIR/murano-manage --config-file $MURANO_CONF_FILE import-package $MURANO_DIR/meta/io.murano
} }

2
doc/source/install/manual.rst
View File

@ -89,7 +89,7 @@ Installing the API service and Engine
:: ::
$ tox -e venv -- murano-manage --config-file /etc/murano/murano-api.conf db-sync $ tox -e venv -- murano-db-manage --config-file /etc/murano/murano-api.conf upgrade
6. Launch Murano API service: 6. Launch Murano API service:

76
murano/cmd/db_manage.py Normal file
View File

@ -0,0 +1,76 @@
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
from oslo.config import cfg
from murano.db.migration import migration
# this forces import and registration of db related configs
from murano.db import models # noqa
from murano.openstack.common import log
CONF = cfg.CONF
class DBCommand(object):
def upgrade(self, config):
migration.upgrade(CONF.command.revision, config=config)
def downgrade(self, config):
migration.downgrade(CONF.command.revision, config=config)
def revision(self, config):
migration.revision(CONF.command.message,
CONF.command.autogenerate,
config=config)
def stamp(self, config):
migration.stamp(CONF.command.revision, config=config)
def add_command_parsers(subparsers):
command_object = DBCommand()
parser = subparsers.add_parser('upgrade')
parser.set_defaults(func=command_object.upgrade)
parser.add_argument('--revision', nargs='?')
parser = subparsers.add_parser('downgrade')
parser.set_defaults(func=command_object.downgrade)
parser.add_argument('--revision', nargs='?')
parser = subparsers.add_parser('stamp')
parser.add_argument('--revision', nargs='?')
parser.set_defaults(func=command_object.stamp)
parser = subparsers.add_parser('revision')
parser.add_argument('-m', '--message')
parser.add_argument('--autogenerate', action='store_true')
parser.set_defaults(func=command_object.revision)
command_opt = cfg.SubCommandOpt('command',
title='Command',
help='Available commands',
handler=add_command_parsers)
CONF.register_cli_opt(command_opt)
def main():
config = migration.get_alembic_config()
# attach the Murano conf to the Alembic conf
config.murano_config = CONF
CONF(project='murano')
log.setup('murano')
CONF.command.func(config)

20
murano/cmd/manage.py
View File

@ -13,7 +13,9 @@
# under the License. # under the License.
""" """
CLI interface for murano management. *** Deprecation warning ***
This file is about to be deprecated, please use python-muranoclient.
*** Deprecation warning ***
""" """
import sys import sys
@ -24,7 +26,6 @@ from oslo.config import cfg
import murano import murano
from murano.common import consts from murano.common import consts
from murano.db.catalog import api as db_catalog_api from murano.db.catalog import api as db_catalog_api
from murano.db import session as db_session
from murano.openstack.common.db import exception as db_exception from murano.openstack.common.db import exception as db_exception
from murano.openstack.common import log as logging from murano.openstack.common import log as logging
from murano.packages import load_utils from murano.packages import load_utils
@ -34,15 +35,6 @@ CONF = cfg.CONF
LOG = logging.getLogger(__name__) LOG = logging.getLogger(__name__)
# TODO(ruhe): proper error handling
def do_db_sync():
"""
Place a database under migration control and upgrade,
creating first if necessary.
"""
db_session.db_sync()
class AdminContext(object): class AdminContext(object):
def __init__(self): def __init__(self):
self.is_admin = True self.is_admin = True
@ -124,11 +116,6 @@ def do_add_category():
def add_command_parsers(subparsers): def add_command_parsers(subparsers):
parser = subparsers.add_parser('db-sync')
parser.set_defaults(func=do_db_sync)
parser.add_argument('version', nargs='?')
parser.add_argument('current_version', nargs='?')
parser = subparsers.add_parser('import-package') parser = subparsers.add_parser('import-package')
parser.set_defaults(func=do_import_package) parser.set_defaults(func=do_import_package)
parser.add_argument('directory', parser.add_argument('directory',
@ -160,6 +147,7 @@ command_opt = cfg.SubCommandOpt('command',
def main(): def main():
CONF.register_cli_opt(command_opt) CONF.register_cli_opt(command_opt)
try: try:
default_config_files = cfg.find_config_files('murano', 'murano') default_config_files = cfg.find_config_files('murano', 'murano')
CONF(sys.argv[1:], project='murano', prog='murano-manage', CONF(sys.argv[1:], project='murano', prog='murano-manage',

4
murano/db/migrate_repo/README
View File

@ -1,4 +0,0 @@
This is a database migration repository.
More information at
http://code.google.com/p/sqlalchemy-migrate/

20
murano/db/migrate_repo/migrate.cfg
View File

@ -1,20 +0,0 @@
[db_settings]
# Used to identify which repository this database is versioned under.
# You can use the name of your project.
repository_id=Murano Migrations
# The name of the database table used to track the schema version.
# This name shouldn't already be used by your project.
# If this is changed once a database is under version control, you'll need to
# change the table name in each database too.
version_table=migrate_version
# When committing a change script, Migrate will attempt to generate the
# sql for all supported databases; normally, if one of them fails - probably
# because you don't have that database installed - it is ignored and the
# commit continues, perhaps ending successfully.
# Databases in this list MUST compile successfully during a commit, or the
# entire commit will fail. List the databases your application will actually
# be using to ensure your updates to that database work properly.
# This must be a list; example: ['sqlite']
required_dbs=[]

96
murano/db/migrate_repo/versions/001_add_initial_tables.py
View File

@ -1,96 +0,0 @@
# Copyright (c) 2013 Mirantis, Inc.
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
from migrate.changeset import constraint as const
from sqlalchemy import schema
from sqlalchemy import types
meta = schema.MetaData()
def upgrade(migrate_engine):
meta.bind = migrate_engine
meta.reflect()
environment = schema.Table(
'environment',
meta,
schema.Column('id', types.String(32), primary_key=True),
schema.Column('name', types.String(255), nullable=False),
schema.Column('created', types.DateTime(), nullable=False),
schema.Column('updated', types.DateTime(), nullable=False),
schema.Column('tenant_id', types.String(32), nullable=False),
schema.Column('version', types.BigInteger, nullable=False,
server_default='0'),
schema.Column('description', types.Text(), nullable=False))
environment.create()
session = schema.Table(
'session',
meta,
schema.Column('id', types.String(32), primary_key=True),
schema.Column('environment_id', types.String(32), nullable=False),
schema.Column('created', types.DateTime, nullable=False),
schema.Column('updated', types.DateTime, nullable=False),
schema.Column('user_id', types.String(32), nullable=False),
schema.Column('version', types.BigInteger, nullable=False,
server_default='0'),
schema.Column('description', types.Text(), nullable=True),
schema.Column('state', types.Text(), nullable=False))
session.create()
environment = schema.Table('environment', meta, autoload=True)
const.ForeignKeyConstraint(columns=[session.c.environment_id],
refcolumns=[environment.c.id]).create()
deployment = schema.Table(
'deployment',
meta,
schema.Column('id', types.String(32), primary_key=True),
schema.Column('environment_id', types.String(32), nullable=False),
schema.Column('created', types.DateTime, nullable=False),
schema.Column('updated', types.DateTime, nullable=False),
schema.Column('started', types.DateTime, nullable=False),
schema.Column('description', types.Text(), nullable=True),
schema.Column('finished', types.DateTime, nullable=True))
deployment.create()
environment = schema.Table('environment', meta, autoload=True)
const.ForeignKeyConstraint(columns=[deployment.c.environment_id],
refcolumns=[environment.c.id]).create()
status = schema.Table(
'status',
meta,
schema.Column('id', types.String(32), primary_key=True),
schema.Column('created', types.DateTime, nullable=False),
schema.Column('updated', types.DateTime, nullable=False),
schema.Column('entity', types.String(10), nullable=True),
schema.Column('entity_id', types.String(32), nullable=True),
schema.Column('environment_id', types.String(32), nullable=True),
schema.Column('deployment_id', types.String(32), nullable=False),
schema.Column('text', types.Text(), nullable=False),
schema.Column('details', types.Text(), nullable=True),
schema.Column('level', types.String(32), nullable=False,
server_default='info'))
status.create()
const.ForeignKeyConstraint(columns=[status.c.deployment_id],
refcolumns=[deployment.c.id]).create()
def downgrade(migrate_engine):
meta.bind = migrate_engine
meta.drop_all()

34
murano/db/migrate_repo/versions/002_add_networking_field.py
View File

@ -1,34 +0,0 @@
# Copyright (c) 2013 Mirantis, Inc.
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
from sqlalchemy import schema
from sqlalchemy import types
meta = schema.MetaData()
def upgrade(migrate_engine):
meta.bind = migrate_engine
environment = schema.Table('environment', meta, autoload=True)
networking = schema.Column('networking',
types.Text(),
nullable=True,
default='{}')
networking.create(environment)
def downgrade(migrate_engine):
meta.bind = migrate_engine
environment = schema.Table('environment', meta, autoload=True)
environment.c.networking.drop()

41
murano/db/migrate_repo/versions/003_add_stats_table.py
View File

@ -1,41 +0,0 @@
# Copyright (c) 2013 Mirantis, Inc.
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
from sqlalchemy import schema
from sqlalchemy import types
meta = schema.MetaData()
def upgrade(migrate_engine):
meta.bind = migrate_engine
stats = schema.Table(
'apistats',
meta,
schema.Column('id', types.Integer(), primary_key=True),
schema.Column('host', types.String(80)),
schema.Column('request_count', types.BigInteger()),
schema.Column('error_count', types.BigInteger()),
schema.Column('average_response_time', types.Float()),
schema.Column('requests_per_tenant', types.Text()),
schema.Column('requests_per_second', types.Float()),
schema.Column('errors_per_second', types.Float()),
schema.Column('created', types.DateTime, nullable=False),
schema.Column('updated', types.DateTime, nullable=False))
stats.create()
def downgrade(migrate_engine):
meta.bind = migrate_engine
stats = schema.Table('apistats', meta, autoload=True)
stats.drop()

155
murano/db/migrate_repo/versions/004_add_repository_tables.py
View File

@ -1,155 +0,0 @@
# Copyright (c) 2014 Mirantis, Inc.
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
from migrate.changeset import constraint
from sqlalchemy.dialects.mysql import mysqldb
from sqlalchemy import schema
from sqlalchemy import types
meta = schema.MetaData()
class StringWithCollation(types.String):
def __init__(self, length, collation=None, **kwargs):
super(StringWithCollation, self).__init__(length, **kwargs)
self.collation = collation
def upgrade(migrate_engine):
meta.bind = migrate_engine
collation = 'ascii_general_ci' \
if isinstance(migrate_engine.dialect, mysqldb.MySQLDialect) \
else None
package = schema.Table(
'package',
meta,
schema.Column('id',
types.String(32),
primary_key=True,
nullable=False),
schema.Column('archive', types.LargeBinary),
schema.Column('fully_qualified_name',
StringWithCollation(512, collation=collation),
index=True, unique=True),
schema.Column('type', types.String(20)),
schema.Column('author', types.String(80)),
schema.Column('name', types.String(20)),
schema.Column('enabled', types.Boolean),
schema.Column('description', types.String(512)),
schema.Column('is_public', types.Boolean),
schema.Column('logo', types.LargeBinary),
schema.Column('owner_id', types.String(36)),
schema.Column('ui_definition', types.Text),
schema.Column('created', types.DateTime, nullable=False),
schema.Column('updated', types.DateTime, nullable=False),
)
package.create()
category = schema.Table(
'category',
meta,
schema.Column('id',
types.String(32),
primary_key=True,
nullable=False),
schema.Column('name',
types.String(80),
nullable=False,
index=True,
unique=True),
schema.Column('created', types.DateTime, nullable=False),
schema.Column('updated', types.DateTime, nullable=False),
)
category.create()
package_to_category = schema.Table(
'package_to_category',
meta,
schema.Column('package_id', types.String(32)),
schema.Column('category_id', types.String(32))
)
package_to_category.create()
constraint.ForeignKeyConstraint(
columns=[package_to_category.c.package_id],
refcolumns=[package.c.id]).create()
constraint.ForeignKeyConstraint(
columns=[package_to_category.c.category_id],
refcolumns=[category.c.id]).create()
tag = schema.Table(
'tag',
meta,
schema.Column('id',
types.String(32),
primary_key=True,
nullable=False),
schema.Column('name',
types.String(80),
nullable=False,
index=True,
unique=True),
schema.Column('created', types.DateTime, nullable=False),
schema.Column('updated', types.DateTime, nullable=False),
)
tag.create()
package_to_tag = schema.Table(
'package_to_tag',
meta,
schema.Column('package_id', types.String(32)),
schema.Column('tag_id', types.String(32))
)
package_to_tag.create()
constraint.ForeignKeyConstraint(
columns=[package_to_tag.c.package_id],
refcolumns=[package.c.id]).create()
constraint.ForeignKeyConstraint(
columns=[package_to_tag.c.tag_id],
refcolumns=[tag.c.id]).create()
class_definition = schema.Table(
'class_definition',
meta,
schema.Column('id',
types.String(32),
primary_key=True,
nullable=False),
schema.Column('name', types.String(80), index=True),
schema.Column('package_id', types.String(32)),
schema.Column('created', types.DateTime, nullable=False),
schema.Column('updated', types.DateTime, nullable=False),
)
class_definition.create()
constraint.ForeignKeyConstraint(columns=[class_definition.c.package_id],
refcolumns=[package.c.id]).create()
def downgrade(migrate_engine):
meta.bind = migrate_engine
package_to_category = schema.Table('package_to_category',
meta,
autoload=True)
package_to_category.drop()
package_to_tag = schema.Table('package_to_tag', meta, autoload=True)
package_to_tag.drop()
class_definition = schema.Table('class_definition', meta, autoload=True)
class_definition.drop()
tag = schema.Table('tag', meta, autoload=True)
tag.drop()
category = schema.Table('category', meta, autoload=True)
category.drop()
package = schema.Table('package', meta, autoload=True)
package.drop()

37
murano/db/migrate_repo/versions/005_add_instance_table.py
View File

@ -1,37 +0,0 @@
# Copyright (c) 2013 Mirantis, Inc.
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
from sqlalchemy import schema
from sqlalchemy import types
meta = schema.MetaData()
def upgrade(migrate_engine):
meta.bind = migrate_engine
table = schema.Table(
'instance',
meta,
schema.Column('environment_id', types.String(100), primary_key=True),
schema.Column('instance_id', types.String(100), primary_key=True),
schema.Column('instance_type', types.Integer, nullable=False),
schema.Column('created', types.Integer, nullable=False),
schema.Column('destroyed', types.Integer, nullable=True))
table.create()
def downgrade(migrate_engine):
meta.bind = migrate_engine
table = schema.Table('instance', meta, autoload=True)
table.drop()

38
murano/db/migrate_repo/versions/006_add_default_categories.py
View File

@ -1,38 +0,0 @@
# Copyright (c) 2014 Mirantis, Inc.
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
from sqlalchemy import schema
import uuid
from murano.common import consts
from murano.openstack.common import timeutils
meta = schema.MetaData()
def upgrade(migrate_engine):
meta.bind = migrate_engine
table = schema.Table('category', meta, autoload=True)
for category in consts.CATEGORIES:
now = timeutils.utcnow()
values = {'id': uuid.uuid4().hex, 'name': category, 'updated': now,
'created': now}
table.insert(values=values).execute()
def downgrade(migrate_engine):
meta.bind = migrate_engine
table = schema.Table('category', meta, autoload=True)
for category in consts.CATEGORIES:
table.delete().where(table.c.name == category).execute()

47
murano/db/migrate_repo/versions/007_instance_table_extended.py
View File

@ -1,47 +0,0 @@
# Copyright (c) 2013 Mirantis, Inc.
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
from sqlalchemy import schema
from sqlalchemy import types
meta = schema.MetaData()
def upgrade(migrate_engine):
meta.bind = migrate_engine
table = schema.Table('instance', meta, autoload=True)
table.drop()
table = schema.Table(
'instance_stats',
meta,
schema.Column('environment_id', types.String(100), primary_key=True),
schema.Column('instance_id', types.String(100), primary_key=True),
schema.Column('instance_type', types.Integer, nullable=False),
schema.Column('created', types.Integer, nullable=False),
schema.Column('destroyed', types.Integer, nullable=True),
schema.Column('type_name', types.String(512), nullable=False),
schema.Column('type_title', types.String(512)),
schema.Column('unit_count', types.Integer()),
schema.Column('tenant_id', types.String(32), nullable=False))
table.create()
def downgrade(migrate_engine):
meta.bind = migrate_engine
table = schema.Table('instance_stats', meta, autoload=True)
table.rename('instance')
table.c.type_name.drop()
table.c.type_title.drop()
table.c.unit_count.drop()
table.c.tenant_id.drop()

61
murano/db/migrate_repo/versions/008_cpu_stats.py
View File

@ -1,61 +0,0 @@
# Copyright (c) 2013 Mirantis, Inc.
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
from sqlalchemy import schema
from sqlalchemy import types
meta = schema.MetaData()
def upgrade(migrate_engine):
meta.bind = migrate_engine
table = schema.Table('apistats', meta, autoload=True, extend_existing=True)
table.drop()
stats = schema.Table(
'apistats',
meta,
schema.Column('id', types.Integer(), primary_key=True),
schema.Column('host', types.String(80)),
schema.Column('request_count', types.BigInteger()),
schema.Column('error_count', types.BigInteger()),
schema.Column('average_response_time', types.Float()),
schema.Column('requests_per_tenant', types.Text()),
schema.Column('requests_per_second', types.Float()),
schema.Column('errors_per_second', types.Float()),
schema.Column('created', types.DateTime, nullable=False),
schema.Column('updated', types.DateTime, nullable=False),
schema.Column('cpu_count', types.Integer()),
schema.Column('cpu_percent', types.Float()), extend_existing=True)
stats.create()
def downgrade(migrate_engine):
meta.bind = migrate_engine
table = schema.Table('apistats', meta, autoload=True, extend_existing=True)
table.drop()
stats = schema.Table(
'apistats',
meta,
schema.Column('id', types.Integer(), primary_key=True),
schema.Column('host', types.String(80)),
schema.Column('request_count', types.BigInteger()),
schema.Column('error_count', types.BigInteger()),
schema.Column('average_response_time', types.Float()),
schema.Column('requests_per_tenant', types.Text()),
schema.Column('requests_per_second', types.Float()),
schema.Column('errors_per_second', types.Float()),
schema.Column('created', types.DateTime, nullable=False),
schema.Column('updated', types.DateTime, nullable=False),
extend_existing=True
)
stats.create()

30
murano/db/migrate_repo/versions/009_increase_package_name_length.py
View File

@ -1,30 +0,0 @@
# Copyright (c) 2014 Hewlett-Packard Development Company, L.P.
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
from sqlalchemy import schema
from sqlalchemy import types
meta = schema.MetaData()
def upgrade(migrate_engine):
meta.bind = migrate_engine
table = schema.Table('package', meta, autoload=True)
table.c.name.alter(type=types.String(80))
def downgrade(migrate_engine):
meta.bind = migrate_engine
table = schema.Table('package', meta, autoload=True)
table.c.name.alter(type=types.String(20))

32
murano/db/migrate_repo/versions/010_add_unique_environment_constraint.py
View File

@ -1,32 +0,0 @@
# Copyright (c) 2014 Hewlett-Packard Development Company, L.P.
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
from migrate.changeset.constraint import UniqueConstraint
from sqlalchemy import schema
meta = schema.MetaData()
def upgrade(migrate_engine):
meta.bind = migrate_engine
table = schema.Table('environment', meta, autoload=True)
cons = UniqueConstraint("tenant_id", "name", table=table)
cons.create()
def downgrade(migrate_engine):
meta.bind = migrate_engine
table = schema.Table('environment', meta, autoload=True)
cons = UniqueConstraint("tenant_id", "name", table=table)
cons.drop()

0
murano/db/migrate_repo/__init__.py → murano/db/migration/__init__.py
View File

54
murano/db/migration/alembic.ini Normal file
View File

@ -0,0 +1,54 @@
# A generic, single database configuration.
[alembic]
# path to migration scripts
script_location = murano/db/migration/alembic_migrations
# template used to generate migration files
# file_template = %%(rev)s_%%(slug)s
# max length of characters to apply to the
# "slug" field
#truncate_slug_length = 40
# set to 'true' to run the environment during
# the 'revision' command, regardless of autogenerate
# revision_environment = false
sqlalchemy.url =
# Logging configuration
[loggers]
keys = root,sqlalchemy,alembic
[handlers]
keys = console
[formatters]
keys = generic
[logger_root]
level = WARN
handlers = console
qualname =
[logger_sqlalchemy]
level = WARN
handlers =
qualname = sqlalchemy.engine
[logger_alembic]
level = INFO
handlers =
qualname = alembic
[handler_console]
class = StreamHandler
args = (sys.stderr,)
level = NOTSET
formatter = generic
[formatter_generic]
format = %(levelname)-5.5s [%(name)s] %(message)s
datefmt = %H:%M:%S

15
murano/db/migration/alembic_migrations/README Normal file
View File

@ -0,0 +1,15 @@
Please see https://alembic.readthedocs.org/en/latest/index.html for general documentation
To create alembic migrations use:
$ murano-db-manage revision --message --autogenerate
Stamp db with most recent migration version, without actually running migrations
$ murano-db-manage stamp --revision head
Upgrade can be performed by:
$ murano-db-manage upgrade
$ murano-db-manage upgrade --revision head
Downgrading db:
$ murano-db-manage downgrade
$ murano-db-manage downgrade --revision base

50
murano/db/migration/alembic_migrations/env.py Normal file
View File

@ -0,0 +1,50 @@
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
# from logging.config import fileConfig
from alembic import context
from sqlalchemy import create_engine, pool
from murano.db import models
# this is the Alembic Config object, which provides
# access to the values within the .ini file in use.
config = context.config
murano_config = config.murano_config
# add your model's MetaData object here
# for 'autogenerate' support
# from myapp import mymodel
target_metadata = models.BASE.metadata
def run_migrations_online():
"""Run migrations in 'online' mode.
In this scenario we need to create an Engine
and associate a connection with the context.
"""
engine = create_engine(
murano_config.database.connection,
poolclass=pool.NullPool)
with engine.connect() as connection:
context.configure(connection=connection,
target_metadata=target_metadata)
with context.begin_transaction():
context.run_migrations()
run_migrations_online()

37
murano/db/migration/alembic_migrations/script.py.mako Normal file
View File

@ -0,0 +1,37 @@
# Copyright ${create_date.year} OpenStack Foundation.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
# implied.
# See the License for the specific language governing permissions and
# limitations under the License.
"""${message}
Revision ID: ${up_revision}
Revises: ${down_revision}
Create Date: ${create_date}
"""
# revision identifiers, used by Alembic.
revision = ${repr(up_revision)}
down_revision = ${repr(down_revision)}
from alembic import op
import sqlalchemy as sa
${imports if imports else ""}
def upgrade():
${upgrades if upgrades else "pass"}
def downgrade():
${downgrades if downgrades else "pass"}

256
murano/db/migration/alembic_migrations/versions/001_inital_version.py Normal file
View File

@ -0,0 +1,256 @@
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
# implied.
# See the License for the specific language governing permissions and
# limitations under the License.
"""empty message
Revision ID: 001
Revises: None
Create Date: 2014-05-29 16:32:33.698760
"""
# revision identifiers, used by Alembic.
revision = '001'
down_revision = None
from alembic import op
import sqlalchemy as sa
from sqlalchemy.sql.expression import table as sa_table
import uuid
from murano.common import consts
from murano.db.sqla import types as st
from murano.openstack.common import timeutils
MYSQL_ENGINE = 'InnoDB'
MYSQL_CHARSET = 'utf8'
def _create_default_categories(op):
bind = op.get_bind()
table = sa_table(
'category',
sa.Column('id', sa.String(length=36), primary_key=True),
sa.Column('created', sa.DateTime()),
sa.Column('updated', sa.DateTime()),
sa.Column('name', sa.String(length=80)))
now = timeutils.utcnow()
for category in consts.CATEGORIES:
values = {'id': uuid.uuid4().hex,
'name': category,
'updated': now,
'created': now}
bind.execute(table.insert(values=values))
def upgrade():
op.create_table(
'environment',
sa.Column('created', sa.DateTime(), nullable=False),
sa.Column('updated', sa.DateTime(), nullable=False),
sa.Column('id', sa.String(length=255), nullable=False),
sa.Column('name', sa.String(length=255), nullable=False),
sa.Column('tenant_id', sa.String(length=36), nullable=False),
sa.Column('version', sa.BigInteger(), nullable=False),
sa.Column('description', sa.Text(), nullable=False),
sa.Column('networking', sa.Text(), nullable=True),
sa.PrimaryKeyConstraint('id'),
sa.UniqueConstraint('tenant_id', 'name'),
mysql_engine=MYSQL_ENGINE,
mysql_charset=MYSQL_CHARSET
)
op.create_table(
'tag',
sa.Column('created', sa.DateTime(), nullable=False),
sa.Column('updated', sa.DateTime(), nullable=False),
sa.Column('id', sa.String(length=36), nullable=False),
sa.Column('name', sa.String(length=80), nullable=False),
sa.PrimaryKeyConstraint('id'),
sa.UniqueConstraint('name'),
mysql_engine=MYSQL_ENGINE,
mysql_charset=MYSQL_CHARSET
)
op.create_table(
'category',
sa.Column('created', sa.DateTime(), nullable=False),
sa.Column('updated', sa.DateTime(), nullable=False),
sa.Column('id', sa.String(length=36), nullable=False),
sa.Column('name', sa.String(length=80), nullable=False, index=True),
sa.PrimaryKeyConstraint('id'),
mysql_engine=MYSQL_ENGINE,
mysql_charset=MYSQL_CHARSET
)
op.create_table(
'apistats',
sa.Column('created', sa.DateTime(), nullable=False),
sa.Column('updated', sa.DateTime(), nullable=False),
sa.Column('id', sa.Integer(), nullable=False),
sa.Column('host', sa.String(length=80), nullable=True),
sa.Column('request_count', sa.BigInteger(), nullable=True),
sa.Column('error_count', sa.BigInteger(), nullable=True),
sa.Column('average_response_time', sa.Float(), nullable=True),
sa.Column('requests_per_tenant', sa.Text(), nullable=True),
sa.Column('requests_per_second', sa.Float(), nullable=True),
sa.Column('errors_per_second', sa.Float(), nullable=True),
sa.Column('cpu_count', sa.Integer(), nullable=True),
sa.Column('cpu_percent', sa.Float(), nullable=True),
sa.PrimaryKeyConstraint('id'),
mysql_engine=MYSQL_ENGINE,
mysql_charset=MYSQL_CHARSET
)
op.create_table(
'instance_stats',
sa.Column('environment_id', sa.String(length=255), nullable=False),
sa.Column('instance_id', sa.String(length=255), nullable=False),
sa.Column('instance_type', sa.Integer(), nullable=False),
sa.Column('created', sa.Integer(), nullable=False),
sa.Column('destroyed', sa.Integer(), nullable=True),
sa.Column('type_name', sa.String(length=512), nullable=False),
sa.Column('type_title', sa.String(length=512), nullable=True),
sa.Column('unit_count', sa.Integer(), nullable=True),
sa.Column('tenant_id', sa.String(length=36), nullable=False),
sa.PrimaryKeyConstraint('environment_id', 'instance_id'),
mysql_engine=MYSQL_ENGINE,
mysql_charset=MYSQL_CHARSET
)
op.create_table(
'package',
sa.Column('created', sa.DateTime(), nullable=False),
sa.Column('updated', sa.DateTime(), nullable=False),
sa.Column('id', sa.String(length=36), nullable=False),
sa.Column('archive', st.LargeBinary(), nullable=True),
sa.Column('fully_qualified_name', sa.String(length=512),
nullable=False, index=True),
sa.Column('type', sa.String(length=20), nullable=False),
sa.Column('author', sa.String(length=80), nullable=True),
sa.Column('name', sa.String(length=80), nullable=False),
sa.Column('enabled', sa.Boolean(), nullable=True),
sa.Column('description', sa.String(length=512), nullable=False),
sa.Column('is_public', sa.Boolean(), nullable=True),
sa.Column('logo', st.LargeBinary(), nullable=True),
sa.Column('owner_id', sa.String(length=36), nullable=False),
sa.Column('ui_definition', sa.Text(), nullable=True),
sa.PrimaryKeyConstraint('id'),
mysql_engine=MYSQL_ENGINE,
mysql_charset=MYSQL_CHARSET
)
op.create_table(
'session',
sa.Column('created', sa.DateTime(), nullable=False),
sa.Column('updated', sa.DateTime(), nullable=False),
sa.Column('id', sa.String(length=36), nullable=False),
sa.Column('environment_id', sa.String(length=255), nullable=True),
sa.Column('user_id', sa.String(length=36), nullable=False),
sa.Column('state', sa.String(length=36), nullable=False),
sa.Column('description', sa.Text(), nullable=False),
sa.Column('version', sa.BigInteger(), nullable=False),
sa.ForeignKeyConstraint(['environment_id'], ['environment.id'], ),
sa.PrimaryKeyConstraint('id'),
mysql_engine=MYSQL_ENGINE,
mysql_charset=MYSQL_CHARSET
)
op.create_table(
'deployment',
sa.Column('created', sa.DateTime(), nullable=False),
sa.Column('updated', sa.DateTime(), nullable=False),
sa.Column('id', sa.String(length=36), nullable=False),
sa.Column('started', sa.DateTime(), nullable=False),
sa.Column('finished', sa.DateTime(), nullable=True),
sa.Column('description', sa.Text(), nullable=False),
sa.Column('environment_id', sa.String(length=255), nullable=True),
sa.ForeignKeyConstraint(['environment_id'], ['environment.id'], ),
sa.PrimaryKeyConstraint('id'),
mysql_engine=MYSQL_ENGINE,
mysql_charset=MYSQL_CHARSET
)
op.create_table(
'class_definition',
sa.Column('created', sa.DateTime(), nullable=False),
sa.Column('updated', sa.DateTime(), nullable=False),
sa.Column('id', sa.String(length=36), nullable=False),
sa.Column('name', sa.String(length=512), nullable=False, index=True),
sa.Column('package_id', sa.String(length=36), nullable=True),
sa.ForeignKeyConstraint(['package_id'], ['package.id'], ),
sa.PrimaryKeyConstraint('id'),
mysql_engine=MYSQL_ENGINE,
mysql_charset=MYSQL_CHARSET
)
op.create_table(
'status',
sa.Column('created', sa.DateTime(), nullable=False),
sa.Column('updated', sa.DateTime(), nullable=False),
sa.Column('id', sa.String(length=36), nullable=False),
sa.Column('entity_id', sa.String(length=255), nullable=True),
sa.Column('entity', sa.String(length=10), nullable=True),
sa.Column('deployment_id', sa.String(length=36), nullable=True),
sa.Column('text', sa.Text(), nullable=False),
sa.Column('level', sa.String(length=32), nullable=False),
sa.Column('details', sa.Text(), nullable=True),
sa.ForeignKeyConstraint(['deployment_id'], ['deployment.id'], ),
sa.PrimaryKeyConstraint('id'),
mysql_engine=MYSQL_ENGINE,
mysql_charset=MYSQL_CHARSET
)
op.create_table(
'package_to_tag',
sa.Column('package_id', sa.String(length=36), nullable=False),
sa.Column('tag_id', sa.String(length=36), nullable=False),
sa.ForeignKeyConstraint(['package_id'], ['package.id'], ),
sa.ForeignKeyConstraint(['tag_id'], ['tag.id'], ondelete='CASCADE'),
mysql_engine=MYSQL_ENGINE,
mysql_charset=MYSQL_CHARSET
)
op.create_table(
'package_to_category',
sa.Column('package_id', sa.String(length=36), nullable=False),
sa.Column('category_id', sa.String(length=36), nullable=False),
sa.ForeignKeyConstraint(['category_id'],
['category.id'],
ondelete='RESTRICT'),
sa.ForeignKeyConstraint(['package_id'], ['package.id'], ),
mysql_engine=MYSQL_ENGINE,
mysql_charset=MYSQL_CHARSET
)
_create_default_categories(op)
### end Alembic commands ###
def downgrade():
op.drop_table('status')
op.drop_table('package_to_category')
op.drop_table('class_definition')
op.drop_table('deployment')
op.drop_table('package_to_tag')
op.drop_table('session')
op.drop_table('instance_stats')
op.drop_table('package')
op.drop_table('apistats')
op.drop_table('category')
op.drop_table('tag')
op.drop_table('environment')
### end Alembic commands ###

79
murano/db/migration/migration.py Normal file
View File

@ -0,0 +1,79 @@
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
import os
import alembic
from alembic import config as alembic_config
def get_alembic_config():
path = os.path.join(os.path.dirname(__file__), 'alembic.ini')
config = alembic_config.Config(path)
config.set_main_option('script_location',
'murano.db.migration:alembic_migrations')
return config
# TODO(ruhe): implement me
#def version(config=None):
# """Current database version."""
def upgrade(revision, config=None):
"""Used for upgrading database.
:param version: Desired database version
:type version: string
"""
revision = revision or 'head'
config = config or get_alembic_config()
alembic.command.upgrade(config, revision or 'head')
def downgrade(revision, config=None):
"""Used for downgrading database.
:param version: Desired database version7
:type version: string
"""
revision = revision or 'base'
config = config or get_alembic_config()
return alembic.command.downgrade(config, revision)
def stamp(revision, config=None):
"""Stamps database with provided revision.
Don't run any migrations.
:param revision: Should match one from repository or head - to stamp
database with most recent revision
:type revision: string
"""
config = config or get_alembic_config()
return alembic.command.stamp(config, revision=revision)
def revision(message=None, autogenerate=False, config=None):
"""Creates template for migration.
:param message: Text that will be used for migration title
:type message: string
:param autogenerate: If True - generates diff based on current database
state
:type autogenerate: bool
"""
config = config or get_alembic_config()
return alembic.command.revision(config, message=message,
autogenerate=autogenerate)

42
murano/db/models.py
View File

@ -108,11 +108,11 @@ class Environment(BASE, ModificationsTrackedObject):
"""Represents a Environment in the metadata-store""" """Represents a Environment in the metadata-store"""
__tablename__ = 'environment' __tablename__ = 'environment'
id = sa.Column(sa.String(32), id = sa.Column(sa.String(255),
primary_key=True, primary_key=True,
default=uuidutils.generate_uuid) default=uuidutils.generate_uuid)
name = sa.Column(sa.String(255), nullable=False) name = sa.Column(sa.String(255), nullable=False)
tenant_id = sa.Column(sa.String(32), nullable=False) tenant_id = sa.Column(sa.String(36), nullable=False)
version = sa.Column(sa.BigInteger, nullable=False, default=0) version = sa.Column(sa.BigInteger, nullable=False, default=0)
description = sa.Column(JsonBlob(), nullable=False, default={}) description = sa.Column(JsonBlob(), nullable=False, default={})
networking = sa.Column(JsonBlob(), nullable=True, default={}) networking = sa.Column(JsonBlob(), nullable=True, default={})
@ -131,10 +131,10 @@ class Environment(BASE, ModificationsTrackedObject):
class Session(BASE, ModificationsTrackedObject): class Session(BASE, ModificationsTrackedObject):
__tablename__ = 'session' __tablename__ = 'session'
id = sa.Column(sa.String(32), id = sa.Column(sa.String(36),
primary_key=True, primary_key=True,
default=uuidutils.generate_uuid) default=uuidutils.generate_uuid)
environment_id = sa.Column(sa.String(32), sa.ForeignKey('environment.id')) environment_id = sa.Column(sa.String(255), sa.ForeignKey('environment.id'))
user_id = sa.Column(sa.String(36), nullable=False) user_id = sa.Column(sa.String(36), nullable=False)
state = sa.Column(sa.String(36), nullable=False) state = sa.Column(sa.String(36), nullable=False)
@ -153,13 +153,13 @@ class Session(BASE, ModificationsTrackedObject):
class Deployment(BASE, ModificationsTrackedObject): class Deployment(BASE, ModificationsTrackedObject):
__tablename__ = 'deployment' __tablename__ = 'deployment'
id = sa.Column(sa.String(32), id = sa.Column(sa.String(36),
primary_key=True, primary_key=True,
default=uuidutils.generate_uuid) default=uuidutils.generate_uuid)
started = sa.Column(sa.DateTime, default=timeutils.utcnow, nullable=False) started = sa.Column(sa.DateTime, default=timeutils.utcnow, nullable=False)
finished = sa.Column(sa.DateTime, default=None, nullable=True) finished = sa.Column(sa.DateTime, default=None, nullable=True)
description = sa.Column(JsonBlob(), nullable=False) description = sa.Column(JsonBlob(), nullable=False)
environment_id = sa.Column(sa.String(32), sa.ForeignKey('environment.id')) environment_id = sa.Column(sa.String(255), sa.ForeignKey('environment.id'))
statuses = sa_orm.relationship("Status", backref='deployment', statuses = sa_orm.relationship("Status", backref='deployment',
cascade='save-update, merge, delete') cascade='save-update, merge, delete')
@ -177,12 +177,12 @@ class Deployment(BASE, ModificationsTrackedObject):
class Status(BASE, ModificationsTrackedObject): class Status(BASE, ModificationsTrackedObject):
__tablename__ = 'status' __tablename__ = 'status'
id = sa.Column(sa.String(32), id = sa.Column(sa.String(36),
primary_key=True, primary_key=True,
default=uuidutils.generate_uuid) default=uuidutils.generate_uuid)
entity_id = sa.Column(sa.String(32), nullable=True) entity_id = sa.Column(sa.String(255), nullable=True)
entity = sa.Column(sa.String(10), nullable=True) entity = sa.Column(sa.String(10), nullable=True)
deployment_id = sa.Column(sa.String(32), sa.ForeignKey('deployment.id')) deployment_id = sa.Column(sa.String(36), sa.ForeignKey('deployment.id'))
text = sa.Column(sa.String(), nullable=False) text = sa.Column(sa.String(), nullable=False)
level = sa.Column(sa.String(32), nullable=False) level = sa.Column(sa.String(32), nullable=False)
details = sa.Column(sa.Text(), nullable=True) details = sa.Column(sa.Text(), nullable=True)
@ -216,20 +216,20 @@ class ApiStats(BASE, ModificationsTrackedObject):
package_to_category = sa.Table('package_to_category', package_to_category = sa.Table('package_to_category',
BASE.metadata, BASE.metadata,
sa.Column('package_id', sa.Column('package_id',
sa.String(32), sa.String(36),
sa.ForeignKey('package.id')), sa.ForeignKey('package.id')),
sa.Column('category_id', sa.Column('category_id',
sa.String(32), sa.String(36),
sa.ForeignKey('category.id', sa.ForeignKey('category.id',
ondelete="RESTRICT"))) ondelete="RESTRICT")))
package_to_tag = sa.Table('package_to_tag', package_to_tag = sa.Table('package_to_tag',
BASE.metadata, BASE.metadata,
sa.Column('package_id', sa.Column('package_id',
sa.String(32), sa.String(36),
sa.ForeignKey('package.id')), sa.ForeignKey('package.id')),
sa.Column('tag_id', sa.Column('tag_id',
sa.String(32), sa.String(36),
sa.ForeignKey('tag.id', sa.ForeignKey('tag.id',
ondelete="CASCADE"))) ondelete="CASCADE")))
@ -238,16 +238,16 @@ class Instance(BASE, ModelBase):
__tablename__ = 'instance_stats' __tablename__ = 'instance_stats'
environment_id = sa.Column( environment_id = sa.Column(
sa.String(100), primary_key=True, nullable=False) sa.String(255), primary_key=True, nullable=False)
instance_id = sa.Column( instance_id = sa.Column(
sa.String(100), primary_key=True, nullable=False) sa.String(255), primary_key=True, nullable=False)
instance_type = sa.Column(sa.Integer, default=0, nullable=False) instance_type = sa.Column(sa.Integer, default=0, nullable=False)
created = sa.Column(sa.Integer, nullable=False) created = sa.Column(sa.Integer, nullable=False)
destroyed = sa.Column(sa.Integer, nullable=True) destroyed = sa.Column(sa.Integer, nullable=True)
type_name = sa.Column('type_name', sa.String(512), nullable=False) type_name = sa.Column('type_name', sa.String(512), nullable=False)
type_title = sa.Column('type_title', sa.String(512)) type_title = sa.Column('type_title', sa.String(512))
unit_count = sa.Column('unit_count', sa.Integer()) unit_count = sa.Column('unit_count', sa.Integer())
tenant_id = sa.Column('tenant_id', sa.String(32), nullable=False) tenant_id = sa.Column('tenant_id', sa.String(36), nullable=False)
def to_dict(self): def to_dict(self):
dictionary = super(Instance, self).to_dict() dictionary = super(Instance, self).to_dict()
@ -311,7 +311,7 @@ class Category(BASE, ModificationsTrackedObject):
""" """
__tablename__ = 'category' __tablename__ = 'category'
id = sa.Column(sa.String(32), id = sa.Column(sa.String(36),
primary_key=True, primary_key=True,
default=uuidutils.generate_uuid) default=uuidutils.generate_uuid)
name = sa.Column(sa.String(80), nullable=False, index=True, unique=True) name = sa.Column(sa.String(80), nullable=False, index=True, unique=True)
@ -323,7 +323,7 @@ class Tag(BASE, ModificationsTrackedObject):
""" """
__tablename__ = 'tag' __tablename__ = 'tag'
id = sa.Column(sa.String(32), id = sa.Column(sa.String(36),
primary_key=True, primary_key=True,
default=uuidutils.generate_uuid) default=uuidutils.generate_uuid)
name = sa.Column(sa.String(80), nullable=False, unique=True) name = sa.Column(sa.String(80), nullable=False, unique=True)
@ -335,11 +335,11 @@ class Class(BASE, ModificationsTrackedObject):
""" """
__tablename__ = 'class_definition' __tablename__ = 'class_definition'
id = sa.Column(sa.String(32), id = sa.Column(sa.String(36),
primary_key=True, primary_key=True,
default=uuidutils.generate_uuid) default=uuidutils.generate_uuid)
name = sa.Column(sa.String(80), nullable=False, index=True) name = sa.Column(sa.String(512), nullable=False, index=True)
package_id = sa.Column(sa.String(32), sa.ForeignKey('package.id')) package_id = sa.Column(sa.String(36), sa.ForeignKey('package.id'))
def register_models(engine): def register_models(engine):

15
murano/db/session.py
View File

@ -14,13 +14,7 @@
"""Session management functions.""" """Session management functions."""
import os
from migrate import exceptions as versioning_exceptions
from migrate.versioning import api as versioning_api
from murano.common import config from murano.common import config
from murano.db import migrate_repo
from murano.openstack.common.db.sqlalchemy import session as db_session from murano.openstack.common.db.sqlalchemy import session as db_session
from murano.openstack.common import log as logging from murano.openstack.common import log as logging
@ -48,12 +42,3 @@ def get_session(autocommit=True, expire_on_commit=False):
def get_engine(): def get_engine():
return _create_facade_lazily().get_engine() return _create_facade_lazily().get_engine()
def db_sync():
repo_path = os.path.abspath(os.path.dirname(migrate_repo.__file__))
try:
versioning_api.upgrade(CONF.database.connection, repo_path)
except versioning_exceptions.DatabaseNotControlledError:
versioning_api.version_control(CONF.database.connection, repo_path)
versioning_api.upgrade(CONF.database.connection, repo_path)

0
murano/db/migrate_repo/versions/__init__.py → murano/db/sqla/__init__.py
View File

19
murano/db/migrate_repo/manage.py → murano/db/sqla/types.py
View File

@ -1,5 +1,3 @@
# Copyright (c) 2013 Mirantis, Inc.
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may # Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain # not use this file except in compliance with the License. You may obtain
# a copy of the License at # a copy of the License at
@ -12,9 +10,18 @@
# License for the specific language governing permissions and limitations # License for the specific language governing permissions and limitations
# under the License. # under the License.
from oslo.config import cfg
import sqlalchemy as sa
from sqlalchemy.dialects import mysql
from migrate.versioning.shell import main CONF = cfg.CONF
# This should probably be a console script entry point.
if __name__ == '__main__': def _is_mysql_avail():
main(debug='False', repository='.') return CONF.database.connection.startswith('mysql')
def LargeBinary():
if _is_mysql_avail():
return mysql.LONGBLOB
return sa.LargeBinary

267
murano/openstack/common/processutils.py Normal file
View File

@ -0,0 +1,267 @@
# Copyright 2011 OpenStack Foundation.
# All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
"""
System-level utilities and helper functions.
"""
import errno
import logging as stdlib_logging
import os
import random
import shlex
import signal
from eventlet.green import subprocess
from eventlet import greenthread
import six
from murano.openstack.common.gettextutils import _
from murano.openstack.common import log as logging
LOG = logging.getLogger(__name__)
class InvalidArgumentError(Exception):
def __init__(self, message=None):
super(InvalidArgumentError, self).__init__(message)
class UnknownArgumentError(Exception):
def __init__(self, message=None):
super(UnknownArgumentError, self).__init__(message)
class ProcessExecutionError(Exception):
def __init__(self, stdout=None, stderr=None, exit_code=None, cmd=None,
description=None):
self.exit_code = exit_code
self.stderr = stderr
self.stdout = stdout
self.cmd = cmd
self.description = description
if description is None:
description = _("Unexpected error while running command.")
if exit_code is None:
exit_code = '-'
message = _('%(description)s\n'
'Command: %(cmd)s\n'
'Exit code: %(exit_code)s\n'
'Stdout: %(stdout)r\n'
'Stderr: %(stderr)r') % {'description': description,
'cmd': cmd,
'exit_code': exit_code,
'stdout': stdout,
'stderr': stderr}
super(ProcessExecutionError, self).__init__(message)
class NoRootWrapSpecified(Exception):
def __init__(self, message=None):
super(NoRootWrapSpecified, self).__init__(message)
def _subprocess_setup():
# Python installs a SIGPIPE handler by default. This is usually not what
# non-Python subprocesses expect.
signal.signal(signal.SIGPIPE, signal.SIG_DFL)
def execute(*cmd, **kwargs):
"""Helper method to shell out and execute a command through subprocess.
Allows optional retry.
:param cmd: Passed to subprocess.Popen.
:type cmd: string
:param process_input: Send to opened process.
:type process_input: string
:param check_exit_code: Single bool, int, or list of allowed exit
codes. Defaults to [0]. Raise
:class:`ProcessExecutionError` unless
program exits with one of these code.
:type check_exit_code: boolean, int, or [int]
:param delay_on_retry: True | False. Defaults to True. If set to True,
wait a short amount of time before retrying.
:type delay_on_retry: boolean
:param attempts: How many times to retry cmd.
:type attempts: int
:param run_as_root: True | False. Defaults to False. If set to True,
the command is prefixed by the command specified
in the root_helper kwarg.
:type run_as_root: boolean
:param root_helper: command to prefix to commands called with
run_as_root=True
:type root_helper: string
:param shell: whether or not there should be a shell used to
execute this command. Defaults to false.
:type shell: boolean
:param loglevel: log level for execute commands.
:type loglevel: int. (Should be stdlib_logging.DEBUG or
stdlib_logging.INFO)
:returns: (stdout, stderr) from process execution
:raises: :class:`UnknownArgumentError` on
receiving unknown arguments
:raises: :class:`ProcessExecutionError`
"""
process_input = kwargs.pop('process_input', None)
check_exit_code = kwargs.pop('check_exit_code', [0])
ignore_exit_code = False
delay_on_retry = kwargs.pop('delay_on_retry', True)
attempts = kwargs.pop('attempts', 1)
run_as_root = kwargs.pop('run_as_root', False)
root_helper = kwargs.pop('root_helper', '')
shell = kwargs.pop('shell', False)
loglevel = kwargs.pop('loglevel', stdlib_logging.DEBUG)
if isinstance(check_exit_code, bool):
ignore_exit_code = not check_exit_code
check_exit_code = [0]
elif isinstance(check_exit_code, int):
check_exit_code = [check_exit_code]
if kwargs:
raise UnknownArgumentError(_('Got unknown keyword args '
'to utils.execute: %r') % kwargs)
if run_as_root and hasattr(os, 'geteuid') and os.geteuid() != 0:
if not root_helper:
raise NoRootWrapSpecified(
message=_('Command requested root, but did not '
'specify a root helper.'))
cmd = shlex.split(root_helper) + list(cmd)
cmd = map(str, cmd)
while attempts > 0:
attempts -= 1
try:
LOG.log(loglevel, 'Running cmd (subprocess): %s',
' '.join(cmd))
_PIPE = subprocess.PIPE # pylint: disable=E1101
if os.name == 'nt':
preexec_fn = None
close_fds = False
else:
preexec_fn = _subprocess_setup
close_fds = True
obj = subprocess.Popen(cmd,
stdin=_PIPE,
stdout=_PIPE,
stderr=_PIPE,
close_fds=close_fds,
preexec_fn=preexec_fn,
shell=shell)
result = None
for _i in six.moves.range(20):
# NOTE(russellb) 20 is an arbitrary number of retries to
# prevent any chance of looping forever here.
try:
if process_input is not None:
result = obj.communicate(process_input)
else:
result = obj.communicate()
except OSError as e:
if e.errno in (errno.EAGAIN, errno.EINTR):
continue
raise
break
obj.stdin.close() # pylint: disable=E1101
_returncode = obj.returncode # pylint: disable=E1101
LOG.log(loglevel, 'Result was %s' % _returncode)
if not ignore_exit_code and _returncode not in check_exit_code:
(stdout, stderr) = result
raise ProcessExecutionError(exit_code=_returncode,
stdout=stdout,
stderr=stderr,
cmd=' '.join(cmd))
return result
except ProcessExecutionError:
if not attempts:
raise
else:
LOG.log(loglevel, '%r failed. Retrying.', cmd)
if delay_on_retry:
greenthread.sleep(random.randint(20, 200) / 100.0)
finally:
# NOTE(termie): this appears to be necessary to let the subprocess
# call clean something up in between calls, without
# it two execute calls in a row hangs the second one
greenthread.sleep(0)
def trycmd(*args, **kwargs):
"""A wrapper around execute() to more easily handle warnings and errors.
Returns an (out, err) tuple of strings containing the output of
the command's stdout and stderr. If 'err' is not empty then the
command can be considered to have failed.
:discard_warnings True | False. Defaults to False. If set to True,
then for succeeding commands, stderr is cleared
"""
discard_warnings = kwargs.pop('discard_warnings', False)
try:
out, err = execute(*args, **kwargs)
failed = False
except ProcessExecutionError as exn:
out, err = '', str(exn)
failed = True
if not failed and discard_warnings and err:
# Handle commands that output to stderr but otherwise succeed
err = ''
return out, err
def ssh_execute(ssh, cmd, process_input=None,
addl_env=None, check_exit_code=True):
LOG.debug('Running cmd (SSH): %s', cmd)
if addl_env:
raise InvalidArgumentError(_('Environment not supported over SSH'))
if process_input:
# This is (probably) fixable if we need it...
raise InvalidArgumentError(_('process_input not supported over SSH'))
stdin_stream, stdout_stream, stderr_stream = ssh.exec_command(cmd)
channel = stdout_stream.channel
# NOTE(justinsb): This seems suspicious...
# ...other SSH clients have buffering issues with this approach
stdout = stdout_stream.read()
stderr = stderr_stream.read()
stdin_stream.close()
exit_status = channel.recv_exit_status()
# exit_status == -1 if no exit code was returned
if exit_status != -1:
LOG.debug('Result was %s' % exit_status)
if check_exit_code and exit_status != 0:
raise ProcessExecutionError(exit_code=exit_status,
stdout=stdout,
stderr=stderr,
cmd=cmd)
return (stdout, stderr)

0
murano/tests/db/__init__.py Normal file
View File

0
murano/tests/db/migration/__init__.py Normal file
View File

26
murano/tests/db/migration/test_migrations.conf Normal file
View File

@ -0,0 +1,26 @@
[unit_tests]
# Set up any number of databases to test concurrently.
# The "name" used in the test is the config variable key.
# A few tests rely on one sqlite database with 'sqlite' as the key.
sqlite=sqlite://
#sqlitefile=sqlite:///test_migrations_utils.db
#mysql=mysql+mysqldb://user:pass@localhost/test_migrations_utils
#postgresql=postgresql+psycopg2://user:pass@localhost/test_migrations_utils
[migration_dbs]
# Migration DB details are listed separately as they can't be connected to
# concurrently. These databases can't be the same as above
# Note, sqlite:// is in-memory and unique each time it is spawned.
# However file sqlite's are not unique.
#sqlite=sqlite://
#sqlitefile=sqlite:///test_migrations.db
#mysql=mysql+mysqldb://user:pass@localhost/test_migrations
#postgresql=postgresql+psycopg2://user:pass@localhost/test_migrations
[walk_style]
snake_walk=yes
downgrade=yes

74
murano/tests/db/migration/test_migrations.py Normal file
View File

@ -0,0 +1,74 @@
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
# implied.
# See the License for the specific language governing permissions and
# limitations under the License.
from oslo.config import cfg
from murano.db import models # noqa
from murano.openstack.common.db.sqlalchemy import utils as db_utils
from murano.tests.db.migration import test_migrations_base as base
CONF = cfg.CONF
class TestMigrations(base.BaseWalkMigrationTestCase, base.CommonTestsMixIn):
USER = "openstack_citest"
PASSWD = "openstack_citest"
DATABASE = "openstack_citest"
def __init__(self, *args, **kwargs):
super(TestMigrations, self).__init__(*args, **kwargs)
def setUp(self):
super(TestMigrations, self).setUp()
def assertColumnExists(self, engine, table, column):
t = db_utils.get_table(engine, table)
self.assertIn(column, t.c)
def assertColumnsExists(self, engine, table, columns):
for column in columns:
self.assertColumnExists(engine, table, column)
def assertColumnCount(self, engine, table, columns):
t = db_utils.get_table(engine, table)
self.assertEqual(len(t.columns), len(columns))
def assertColumnNotExists(self, engine, table, column):
t = db_utils.get_table(engine, table)
self.assertNotIn(column, t.c)
def assertIndexExists(self, engine, table, index):
t = db_utils.get_table(engine, table)
index_names = [idx.name for idx in t.indexes]
self.assertIn(index, index_names)
def assertIndexMembers(self, engine, table, index, members):
self.assertIndexExists(engine, table, index)
t = db_utils.get_table(engine, table)
index_columns = None
for idx in t.indexes:
if idx.name == index:
index_columns = idx.columns.keys()
break
self.assertEqual(sorted(members), sorted(index_columns))
def _check_001(self, engine, data):
self.assertColumnExists(engine, 'category', 'id')
self.assertColumnExists(engine, 'environment', 'tenant_id')
self.assertIndexExists(engine,
'class_definition',
'ix_class_definition_name')

585
murano/tests/db/migration/test_migrations_base.py Normal file
View File

@ -0,0 +1,585 @@
# Copyright 2010-2011 OpenStack Foundation
# Copyright 2012-2013 IBM Corp.
# All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
#
#
# Ripped off from Nova's test_migrations.py
# The only difference between Nova and this code is usage of alembic instead
# of sqlalchemy migrations.
#
# There is an ongoing work to extact similar code to oslo incubator. Once it is
# extracted we'll be able to remove this file and use oslo.
import ConfigParser
import io
import os
from alembic import command
from alembic import config as alembic_config
from alembic import migration
from oslo.config import cfg
import six.moves.urllib.parse as urlparse
import sqlalchemy
import sqlalchemy.exc
import unittest2
import murano.db.migration
from murano.openstack.common import lockutils
from murano.openstack.common import log as logging
from murano.openstack.common import processutils
LOG = logging.getLogger(__name__)
CONF = cfg.CONF
synchronized = lockutils.synchronized_with_prefix('murano-')
def _get_connect_string(backend, user, passwd, database):
"""Try to get a connection with a very specific set of values, if we get
these then we'll run the tests, otherwise they are skipped
"""
if backend == "postgres":
backend = "postgresql+psycopg2"
elif backend == "mysql":
backend = "mysql+mysqldb"
else:
raise Exception("Unrecognized backend: '%s'" % backend)
return ("%s://%s:%s@localhost/%s" % (backend, user, passwd, database))
def _is_backend_avail(backend, user, passwd, database):
try:
connect_uri = _get_connect_string(backend, user, passwd, database)
engine = sqlalchemy.create_engine(connect_uri)
connection = engine.connect()
except Exception:
# intentionally catch all to handle exceptions even if we don't
# have any backend code loaded.
return False
else:
connection.close()
engine.dispose()
return True
def _have_mysql(user, passwd, database):
present = os.environ.get('MURANO_MYSQL_PRESENT')
if present is None:
return _is_backend_avail('mysql', user, passwd, database)
return present.lower() in ('', 'true')
def _have_postgresql(user, passwd, database):
present = os.environ.get('MURANO_TEST_POSTGRESQL_PRESENT')
if present is None:
return _is_backend_avail('postgres', user, passwd, database)
return present.lower() in ('', 'true')
def get_mysql_connection_info(conn_pieces):
database = conn_pieces.path.strip('/')
loc_pieces = conn_pieces.netloc.split('@')
host = loc_pieces[1]
auth_pieces = loc_pieces[0].split(':')
user = auth_pieces[0]
password = ""
if len(auth_pieces) > 1:
if auth_pieces[1].strip():
password = "-p\"%s\"" % auth_pieces[1]
return (user, password, database, host)
def get_pgsql_connection_info(conn_pieces):
database = conn_pieces.path.strip('/')
loc_pieces = conn_pieces.netloc.split('@')
host = loc_pieces[1]
auth_pieces = loc_pieces[0].split(':')
user = auth_pieces[0]
password = ""
if len(auth_pieces) > 1:
password = auth_pieces[1].strip()
return (user, password, database, host)
class CommonTestsMixIn(object):
"""BaseMigrationTestCase is effectively an abstract class, meant to be
derived from and not directly tested against; that's why these `test_`
methods need to be on a Mixin, so that they won't be picked up as valid
tests for BaseMigrationTestCase.
"""
def test_walk_versions(self):
for key, engine in self.engines.items():
# We start each walk with a completely blank slate.
self._reset_database(key)
self._walk_versions(engine, self.snake_walk, self.downgrade)
def test_mysql_opportunistically(self):
self._test_mysql_opportunistically()
def test_mysql_connect_fail(self):
"""Test that we can trigger a mysql connection failure and we fail
gracefully to ensure we don't break people without mysql
"""
if _is_backend_avail('mysql', "openstack_cifail", self.PASSWD,
self.DATABASE):
self.fail("Shouldn't have connected")
def test_postgresql_opportunistically(self):
self._test_postgresql_opportunistically()
def test_postgresql_connect_fail(self):
"""Test that we can trigger a postgres connection failure and we fail
gracefully to ensure we don't break people without postgres
"""
if _is_backend_avail('postgres', "openstack_cifail", self.PASSWD,
self.DATABASE):
self.fail("Shouldn't have connected")
class BaseMigrationTestCase(unittest2.TestCase):
"""Base class for testing migrations and migration utils. This sets up
and configures the databases to run tests against.
"""
# NOTE(jhesketh): It is expected that tests clean up after themselves.
# This is necessary for concurrency to allow multiple tests to work on
# one database.
# The full migration walk tests however do call the old _reset_databases()
# to throw away whatever was there so they need to operate on their own
# database that we know isn't accessed concurrently.
# Hence, BaseWalkMigrationTestCase overwrites the engine list.
USER = None
PASSWD = None
DATABASE = None
TIMEOUT_SCALING_FACTOR = 2
def __init__(self, *args, **kwargs):
super(BaseMigrationTestCase, self).__init__(*args, **kwargs)
self.DEFAULT_CONFIG_FILE = os.path.join(
os.path.dirname(__file__),
'test_migrations.conf')
# Test machines can set the MURANO_TEST_MIGRATIONS_CONF variable
# to override the location of the config file for migration testing
self.CONFIG_FILE_PATH = os.environ.get(
'MURANO_TEST_MIGRATIONS_CONF',
self.DEFAULT_CONFIG_FILE)
self.ALEMBIC_CONFIG = alembic_config.Config(
os.path.join(os.path.dirname(murano.db.migration.__file__),
'alembic.ini')
)
self.ALEMBIC_CONFIG.set_main_option(
'script_location',
'murano.db.migration:alembic_migrations')
self.ALEMBIC_CONFIG.murano_config = CONF
self.snake_walk = False
self.downgrade = False
self.test_databases = {}
self.migration = None
self.migration_api = None
def setUp(self):
super(BaseMigrationTestCase, self).setUp()
self._load_config()
def _load_config(self):
# Load test databases from the config file. Only do this
# once. No need to re-run this on each test...
LOG.debug('config_path is %s' % self.CONFIG_FILE_PATH)
if os.path.exists(self.CONFIG_FILE_PATH):
cp = ConfigParser.RawConfigParser()
try:
cp.read(self.CONFIG_FILE_PATH)
config = cp.options('unit_tests')
for key in config:
self.test_databases[key] = cp.get('unit_tests', key)
self.snake_walk = cp.getboolean('walk_style', 'snake_walk')
self.downgrade = cp.getboolean('walk_style', 'downgrade')
except ConfigParser.ParsingError as e:
self.fail("Failed to read test_migrations.conf config "
"file. Got error: %s" % e)
else:
self.fail("Failed to find test_migrations.conf config "
"file.")
self.engines = {}
for key, value in self.test_databases.items():
self.engines[key] = sqlalchemy.create_engine(value)
# NOTE(jhesketh): We only need to make sure the databases are created
# not necessarily clean of tables.
self._create_databases()
def execute_cmd(self, cmd=None):
out, err = processutils.trycmd(cmd, shell=True, discard_warnings=True)
output = out or err
LOG.debug(output)
self.assertEqual('', err,
"Failed to run: %s\n%s" % (cmd, output))
@synchronized('pgadmin', external=True, lock_path='/tmp')
def _reset_pg(self, conn_pieces):
(user, password, database, host) = \
get_pgsql_connection_info(conn_pieces)
os.environ['PGPASSWORD'] = password
os.environ['PGUSER'] = user
# note(boris-42): We must create and drop database, we can't
# drop database which we have connected to, so for such
# operations there is a special database template1.
sqlcmd = ("psql -w -U %(user)s -h %(host)s -c"
" '%(sql)s' -d template1")
sqldict = {'user': user, 'host': host}
sqldict['sql'] = ("drop database if exists %s;") % database
droptable = sqlcmd % sqldict
self.execute_cmd(droptable)
sqldict['sql'] = ("create database %s;") % database
createtable = sqlcmd % sqldict
self.execute_cmd(createtable)
os.unsetenv('PGPASSWORD')
os.unsetenv('PGUSER')
@synchronized('mysql', external=True, lock_path='/tmp')
def _reset_mysql(self, conn_pieces):
# We can execute the MySQL client to destroy and re-create
# the MYSQL database, which is easier and less error-prone
# than using SQLAlchemy to do this via MetaData...trust me.
(user, password, database, host) = \
get_mysql_connection_info(conn_pieces)
sql = ("drop database if exists %(database)s; "
"create database %(database)s;" % {'database': database})
cmd = ("mysql -u \"%(user)s\" %(password)s -h %(host)s -e \"%(sql)s\""
% {'user': user, 'password': password,
'host': host, 'sql': sql})
self.execute_cmd(cmd)
@synchronized('sqlite', external=True, lock_path='/tmp')
def _reset_sqlite(self, conn_pieces):
# We can just delete the SQLite database, which is
# the easiest and cleanest solution
db_path = conn_pieces.path.strip('/')
if os.path.exists(db_path):
os.unlink(db_path)
# No need to recreate the SQLite DB. SQLite will
# create it for us if it's not there...
def _create_databases(self):
"""Create all configured databases as needed."""
for key, engine in self.engines.items():
self._create_database(key)
def _create_database(self, key):
"""Create database if it doesn't exist."""
conn_string = self.test_databases[key]
conn_pieces = urlparse.urlparse(conn_string)
if conn_string.startswith('mysql'):
(user, password, database, host) = \
get_mysql_connection_info(conn_pieces)
sql = "create database if not exists %s;" % database
cmd = ("mysql -u \"%(user)s\" %(password)s -h %(host)s "
"-e \"%(sql)s\"" % {'user': user, 'password': password,
'host': host, 'sql': sql})
self.execute_cmd(cmd)
elif conn_string.startswith('postgresql'):
(user, password, database, host) = \
get_pgsql_connection_info(conn_pieces)
os.environ['PGPASSWORD'] = password
os.environ['PGUSER'] = user
sqlcmd = ("psql -w -U %(user)s -h %(host)s -c"
" '%(sql)s' -d template1")
sql = ("create database if not exists %s;") % database
createtable = sqlcmd % {'user': user, 'host': host, 'sql': sql}
# 0 means databases is created
# 256 means it already exists (which is fine)
# otherwise raise an error
out, err = processutils.trycmd(createtable, shell=True,
check_exit_code=[0, 256],
discard_warnings=True)
output = out or err
if err != '':
self.fail("Failed to run: %s\n%s" % (createtable, output))
os.unsetenv('PGPASSWORD')
os.unsetenv('PGUSER')
def _reset_databases(self):
"""Reset all configured databases."""
for key, engine in self.engines.items():
self._reset_database(key)
def _reset_database(self, key):
"""Reset specific database."""
engine = self.engines[key]
conn_string = self.test_databases[key]
conn_pieces = urlparse.urlparse(conn_string)
engine.dispose()
if conn_string.startswith('sqlite'):
self._reset_sqlite(conn_pieces)
elif conn_string.startswith('mysql'):
self._reset_mysql(conn_pieces)
elif conn_string.startswith('postgresql'):
self._reset_pg(conn_pieces)
class BaseWalkMigrationTestCase(BaseMigrationTestCase):
"""BaseWalkMigrationTestCase loads in an alternative set of databases for
testing against. This is necessary as the default databases can run tests
concurrently without interfering with itself. It is expected that
databases listed under [migraiton_dbs] in the configuration are only being
accessed by one test at a time. Currently only test_walk_versions accesses
the databases (and is the only method that calls _reset_database() which
is clearly problematic for concurrency).
"""
def _load_config(self):
# Load test databases from the config file. Only do this
# once. No need to re-run this on each test...
LOG.debug('config_path is %s' % self.CONFIG_FILE_PATH)
if os.path.exists(self.CONFIG_FILE_PATH):
cp = ConfigParser.RawConfigParser()
try:
cp.read(self.CONFIG_FILE_PATH)
config = cp.options('migration_dbs')
for key in config:
self.test_databases[key] = cp.get('migration_dbs', key)
self.snake_walk = cp.getboolean('walk_style', 'snake_walk')
self.downgrade = cp.getboolean('walk_style', 'downgrade')
except ConfigParser.ParsingError as e:
self.fail("Failed to read test_migrations.conf config "
"file. Got error: %s" % e)
else:
self.fail("Failed to find test_migrations.conf config "
"file.")
self.engines = {}
for key, value in self.test_databases.items():
self.engines[key] = sqlalchemy.create_engine(value)
self._create_databases()
def _configure(self, engine):
"""For each type of repository we should do some of configure steps.
For migrate_repo we should set under version control our database.
For alembic we should configure database settings. For this goal we
should use oslo.config and openstack.commom.db.sqlalchemy.session with
database functionality (reset default settings and session cleanup).
"""
CONF.set_override('connection', str(engine.url), group='database')
#session.cleanup()
def _test_mysql_opportunistically(self):
# Test that table creation on mysql only builds InnoDB tables
if not _have_mysql(self.USER, self.PASSWD, self.DATABASE):
self.skipTest("mysql not available")
# add this to the global lists to make reset work with it, it's removed
# automatically in tearDown so no need to clean it up here.
connect_string = _get_connect_string(
"mysql", self.USER, self.PASSWD, self.DATABASE)
(user, password, database, host) = \
get_mysql_connection_info(urlparse.urlparse(connect_string))
engine = sqlalchemy.create_engine(connect_string)
self.engines[database] = engine
self.test_databases[database] = connect_string
# build a fully populated mysql database with all the tables
self._reset_database(database)
self._walk_versions(engine, self.snake_walk, self.downgrade)
connection = engine.connect()
# sanity check
total = connection.execute("SELECT count(*) "
"from information_schema.TABLES "
"where TABLE_SCHEMA='%(database)s'" %
{'database': database})
self.assertTrue(total.scalar() > 0, "No tables found. Wrong schema?")
connection.close()
del(self.engines[database])
del(self.test_databases[database])
def _test_postgresql_opportunistically(self):
# Test postgresql database migration walk
if not _have_postgresql(self.USER, self.PASSWD, self.DATABASE):
self.skipTest("postgresql not available")
# add this to the global lists to make reset work with it, it's removed
# automatically in tearDown so no need to clean it up here.
connect_string = _get_connect_string(
"postgres", self.USER, self.PASSWD, self.DATABASE)
engine = sqlalchemy.create_engine(connect_string)
(user, password, database, host) = \
get_mysql_connection_info(urlparse.urlparse(connect_string))
self.engines[database] = engine
self.test_databases[database] = connect_string
# build a fully populated postgresql database with all the tables
self._reset_database(database)
self._walk_versions(engine, self.snake_walk, self.downgrade)
del(self.engines[database])
del(self.test_databases[database])
def _alembic_command(self, alembic_command, engine, *args, **kwargs):
"""Most of alembic command return data into output.
We should redefine this setting for getting info.
"""
self.ALEMBIC_CONFIG.stdout = buf = io.StringIO()
CONF.set_override('connection', str(engine.url), group='database')
#session.cleanup()
getattr(command, alembic_command)(*args, **kwargs)
res = buf.getvalue().strip()
LOG.debug('Alembic command `%s` returns: %s' % (alembic_command, res))
#session.cleanup()
return res
def _get_alembic_versions(self, engine):
"""For support of full testing of migrations
we should have an opportunity to run command step by step for each
version in repo. This method returns list of alembic_versions by
historical order.
"""
full_history = self._alembic_command('history',
engine, self.ALEMBIC_CONFIG)
# The piece of output data with version can looked as:
# 'Rev: 17738166b91 (head)' or 'Rev: 43b1a023dfaa'
alembic_history = [r.split(' ')[1] for r in full_history.split("\n")
if r.startswith("Rev")]
alembic_history.reverse()
return alembic_history
def _up_and_down_versions(self, engine):
"""Since alembic version has a random algorithm of generation
(SA-migrate has an ordered autoincrement naming) we should store
a tuple of versions (version for upgrade and version for downgrade)
for successful testing of migrations in up>down>up mode.
"""
versions = self._get_alembic_versions(engine)
return zip(versions, ['-1'] + versions)
def _walk_versions(self, engine=None, snake_walk=False,
downgrade=True):
# Determine latest version script from the repo, then
# upgrade from 1 through to the latest, with no data
# in the databases. This just checks that the schema itself
# upgrades successfully.
self._configure(engine)
up_and_down_versions = self._up_and_down_versions(engine)
for ver_up, ver_down in up_and_down_versions:
# upgrade -> downgrade -> upgrade
self._migrate_up(engine, ver_up, with_data=True)
if snake_walk:
downgraded = self._migrate_down(engine,
ver_down,
with_data=True,
next_version=ver_up)
if downgraded:
self._migrate_up(engine, ver_up)
if downgrade:
# Now walk it back down to 0 from the latest, testing
# the downgrade paths.
up_and_down_versions.reverse()
for ver_up, ver_down in up_and_down_versions:
# downgrade -> upgrade -> downgrade
downgraded = self._migrate_down(engine,
ver_down, next_version=ver_up)
if snake_walk and downgraded:
self._migrate_up(engine, ver_up)
self._migrate_down(engine, ver_down, next_version=ver_up)
def _get_version_from_db(self, engine):
"""For each type of migrate repo latest version from db
will be returned.
"""
conn = engine.connect()
try:
context = migration.MigrationContext.configure(conn)
version = context.get_current_revision() or '-1'
finally:
conn.close()
return version
def _migrate(self, engine, version, cmd):
"""Base method for manipulation with migrate repo.
It will upgrade or downgrade the actual database.
"""
self._alembic_command(cmd, engine, self.ALEMBIC_CONFIG, version)
def _migrate_down(self, engine, version, with_data=False,
next_version=None):
try:
self._migrate(engine, version, 'downgrade')
except NotImplementedError:
# NOTE(sirp): some migrations, namely release-level
# migrations, don't support a downgrade.
return False
self.assertEqual(version, self._get_version_from_db(engine))
# NOTE(sirp): `version` is what we're downgrading to (i.e. the 'target'
# version). So if we have any downgrade checks, they need to be run for
# the previous (higher numbered) migration.
if with_data:
post_downgrade = getattr(
self, "_post_downgrade_%s" % next_version, None)
if post_downgrade:
post_downgrade(engine)
return True
def _migrate_up(self, engine, version, with_data=False):
"""migrate up to a new version of the db.
We allow for data insertion and post checks at every
migration version with special _pre_upgrade_### and
_check_### functions in the main test.
"""
# NOTE(sdague): try block is here because it's impossible to debug
# where a failed data migration happens otherwise
check_version = version
try:
if with_data:
data = None
pre_upgrade = getattr(
self, "_pre_upgrade_%s" % check_version, None)
if pre_upgrade:
data = pre_upgrade(engine)
self._migrate(engine, version, 'upgrade')
self.assertEqual(version, self._get_version_from_db(engine))
if with_data:
check = getattr(self, "_check_%s" % check_version, None)
if check:
check(engine, data)
except Exception:
LOG.error("Failed to migrate to version %s on engine %s" %
(version, engine))
raise

3
requirements.txt
View File

@ -1,6 +1,7 @@
pbr>=0.6,<1.0 pbr>=0.6,<1.0
Babel>=1.3 Babel>=1.3
SQLAlchemy>=0.7.8,<=0.9.99 SQLAlchemy>=0.7.8,<=0.9.99
alembic>=0.4.1
anyjson>=0.3.3 anyjson>=0.3.3
eventlet>=0.13.0 eventlet>=0.13.0
PasteDeploy>=1.5.0 PasteDeploy>=1.5.0
@ -9,7 +10,6 @@ WebOb>=1.2.3
wsgiref>=0.1.2 wsgiref>=0.1.2
argparse argparse
ordereddict ordereddict
sqlalchemy-migrate>=0.8.2,!=0.8.4
kombu>=2.4.8 kombu>=2.4.8
lockfile>=0.8 lockfile>=0.8
pycrypto>=2.6 pycrypto>=2.6
@ -34,4 +34,3 @@ oslo.messaging>=1.3.0a9
# not listed in global requirements # not listed in global requirements
yaql>=0.2.2,<0.3 yaql>=0.2.2,<0.3
python-muranoclient>=0.5.2 python-muranoclient>=0.5.2

1
setup.cfg
View File

@ -45,6 +45,7 @@ console_scripts =
murano-api = murano.cmd.api:main murano-api = murano.cmd.api:main
murano-engine = murano.cmd.engine:main murano-engine = murano.cmd.engine:main
murano-manage = murano.cmd.manage:main murano-manage = murano.cmd.manage:main
murano-db-manage = murano.cmd.db_manage:main
[build_sphinx] [build_sphinx]
all_files = 1 all_files = 1

6
test-requirements.txt
View File

@ -4,10 +4,16 @@ coverage>=3.6
discover discover
fixtures>=0.3.14 fixtures>=0.3.14
mock>=1.0 mock>=1.0
posix_ipc
sqlalchemy-migrate>=0.8.2,!=0.8.4
testrepository>=0.0.18 testrepository>=0.0.18
testscenarios>=0.4 testscenarios>=0.4
testtools>=0.9.34 testtools>=0.9.34
unittest2 unittest2
# Some of the tests use real MySQL and Postgres databases
MySQL-python
psycopg2
# doc build requirements # doc build requirements
sphinx>=1.1.2,<1.2 sphinx>=1.1.2,<1.2