Merge trunk

This commit is contained in:
jaypipes@gmail.com 2011-02-04 19:05:23 -05:00
commit bc310ced44
23 changed files with 644 additions and 21 deletions

View File

@ -6,5 +6,6 @@ include tests/stubs.py
include tests/test_data.py
include tests/utils.py
include run_tests.py
include glance/registry/db/migrate_repo/migrate.cfg
graft doc
graft tools

130
bin/glance-manage Executable file
View File

@ -0,0 +1,130 @@
#!/usr/bin/env python
# vim: tabstop=4 shiftwidth=4 softtabstop=4
# Copyright 2010 United States Government as represented by the
# Administrator of the National Aeronautics and Space Administration.
# Copyright 2011 OpenStack LLC.
# All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
"""
Glance Management Utility
"""
# FIXME(sirp): When we have glance-admin we can consider merging this into it
# Perhaps for consistency with Nova, we would then rename glance-admin ->
# glance-manage (or the other way around)
import optparse
import os
import sys
ROOT_DIR = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
sys.path.append(ROOT_DIR)
from glance import version as glance_version
from glance.common import config
from glance.common import exception
import glance.registry.db
import glance.registry.db.migration
def create_options(parser):
"""
Sets up the CLI and config-file options that may be
parsed and program commands.
:param parser: The option parser
"""
glance.registry.db.add_options(parser)
config.add_common_options(parser)
config.add_log_options('glance-manage', parser)
def do_db_version(options, args):
"""Print database's current migration level"""
print glance.registry.db.migration.db_version(options)
def do_upgrade(options, args):
"""Upgrade the database's migration level"""
try:
db_version = args[1]
except IndexError:
db_version = None
glance.registry.db.migration.upgrade(options, version=db_version)
def do_downgrade(options, args):
"""Downgrade the database's migration level"""
try:
db_version = args[1]
except IndexError:
raise exception.MissingArgumentError(
"downgrade requires a version argument")
glance.registry.db.migration.downgrade(options, version=db_version)
def do_version_control(options, args):
"""Place a database under migration control"""
glance.registry.db.migration.version_control(options)
def do_db_sync(options, args):
"""Place a database under migration control and upgrade"""
try:
db_version = args[1]
except IndexError:
db_version = None
glance.registry.db.migration.db_sync(options, version=db_version)
def dispatch_cmd(options, args):
"""Search for do_* cmd in this module and then run it"""
cmd = args[0]
try:
cmd_func = globals()['do_%s' % cmd]
except KeyError:
sys.exit("ERROR: unrecognized command '%s'" % cmd)
try:
cmd_func(options, args)
except exception.Error, e:
sys.exit("ERROR: %s" % e)
def main():
version = '%%prog %s' % glance_version.version_string()
usage = "%prog [options] <cmd>"
oparser = optparse.OptionParser(usage, version=version)
create_options(oparser)
(options, args) = config.parse_options(oparser)
try:
config.setup_logging(options)
except RuntimeError, e:
sys.exit("ERROR: %s" % e)
if not args:
oparser.print_usage()
sys.exit(1)
dispatch_cmd(options, args)
if __name__ == '__main__':
main()

View File

@ -33,6 +33,9 @@ Kernel-outside:
<filename> <name>
"""
# FIXME(sirp): This can be merged into glance-admin when that becomes
# available
import argparse
import pprint
import sys

View File

@ -129,6 +129,8 @@ man_pages = [
('man/glanceapi', 'glance-api', u'Glance API Server',
[u'OpenStack'], 1),
('man/glanceregistry', 'glance-registry', u'Glance Registry Server',
[u'OpenStack'], 1),
('man/glancemanage', 'glance-manage', u'Glance Management Utility',
[u'OpenStack'], 1)
]

View File

@ -48,7 +48,7 @@ OPTIONS
running ``glance-api``
FILES
========
=====
None

View File

@ -0,0 +1,54 @@
=============
glance-manage
=============
-------------------------
Glance Management Utility
-------------------------
:Author: glance@lists.launchpad.net
:Date: 2010-11-16
:Copyright: OpenStack LLC
:Version: 0.1.2
:Manual section: 1
:Manual group: cloud computing
SYNOPSIS
========
glance-manage [options]
DESCRIPTION
===========
glance-manage is a utility for managing and configuring a Glance installation.
One important use of glance-manage is to setup the database. To do this run::
glance-manage db_sync
OPTIONS
=======
**General options**
**-v, --verbose**
Print more verbose output
**--sql_connection=CONN_STRING**
A proper SQLAlchemy connection string as described
`here <http://www.sqlalchemy.org/docs/05/reference/sqlalchemy/connections.html?highlight=engine#sqlalchemy.create_engine>`_
FILES
=====
None
SEE ALSO
========
* `OpenStack Glance <http://glance.openstack.org>`__
BUGS
====
* Glance is sourced in Launchpad so you can view current bugs at `OpenStack Glance <http://glance.openstack.org>`__

View File

@ -43,7 +43,7 @@ OPTIONS
`here <http://www.sqlalchemy.org/docs/05/reference/sqlalchemy/connections.html?highlight=engine#sqlalchemy.create_engine>`_
FILES
========
=====
None

View File

@ -31,8 +31,11 @@ import sys
from paste import deploy
import glance.common.exception as exception
DEFAULT_LOG_FORMAT = "%(asctime)s %(levelname)8s [%(name)s] %(message)s"
DEFAULT_LOG_DATE_FORMAT = "%Y-%m-%d %H:%M:%S"
DEFAULT_LOG_HANDLER = 'stream'
LOGGING_HANDLER_CHOICES = ['syslog', 'file', 'stream']
@ -160,7 +163,8 @@ def add_log_options(prog_name, parser):
"any other logging options specified. Please see "
"the Python logging module documentation for "
"details on logging configuration files.")
group.add_option('--log-handler', default='stream', metavar="HANDLER",
group.add_option('--log-handler', default=DEFAULT_LOG_HANDLER,
metavar="HANDLER",
choices=LOGGING_HANDLER_CHOICES,
help="What logging handler to use? "
"Default: %default")
@ -184,7 +188,7 @@ def setup_logging(options):
:param options: Mapping of typed option key/values
"""
if options['log_config']:
if options.get('log_config', None):
# Use a logging configuration file for all settings...
if os.path.exists(options['log_config']):
logging.config.fileConfig(options['log_config'])
@ -207,14 +211,16 @@ def setup_logging(options):
# Note that we use a hard-coded log format in the options
# because of Paste.Deploy bug #379
# http://trac.pythonpaste.org/pythonpaste/ticket/379
formatter = logging.Formatter(DEFAULT_LOG_FORMAT,
options['log_date_format'])
log_format = options.get('log_format', DEFAULT_LOG_FORMAT)
log_date_format = options.get('log_date_format', DEFAULT_LOG_DATE_FORMAT)
formatter = logging.Formatter(log_format, log_date_format)
if options['log_handler'] == 'syslog':
log_handler = options.get('log_handler', DEFAULT_LOG_HANDLER)
if log_handler == 'syslog':
syslog = logging.handlers.SysLogHandler(address='/dev/log')
syslog.setFormatter(formatter)
root_logger.addHandler(syslog)
elif options['log_handler'] == 'file':
elif log_handler == 'file':
logfile = options['log_file']
logdir = options['log_dir']
if logdir:
@ -223,10 +229,13 @@ def setup_logging(options):
logfile.setFormatter(formatter)
logfile.setFormatter(formatter)
root_logger.addHandler(logfile)
else:
elif log_handler == 'stream':
handler = logging.StreamHandler(sys.stdout)
handler.setFormatter(formatter)
root_logger.addHandler(handler)
else:
raise exception.BadInputError(
"unrecognized log handler '%(log_handler)s'" % locals())
# Log the options used when starting if we're in debug mode...
if debug:

View File

@ -75,6 +75,14 @@ class BadInputError(Exception):
pass
class MissingArgumentError(Error):
pass
class DatabaseMigrationError(Error):
pass
def wrap_exception(f):
def _wrap(*args, **kw):
try:

View File

@ -33,7 +33,7 @@ from glance.registry.db import models
_ENGINE = None
_MAKER = None
BASE = declarative_base()
BASE = models.BASE
# attributes common to all models
BASE_MODEL_ATTRS = set(['id', 'created_at', 'updated_at', 'deleted_at',

View File

@ -0,0 +1,4 @@
This is a database migration repository.
More information at
http://code.google.com/p/sqlalchemy-migrate/

View File

@ -0,0 +1 @@
# template repository default module

View File

@ -0,0 +1,3 @@
#!/usr/bin/env python
from migrate.versioning.shell import main
main(debug='False', repository='.')

View File

@ -0,0 +1,20 @@
[db_settings]
# Used to identify which repository this database is versioned under.
# You can use the name of your project.
repository_id=Glance Migrations
# The name of the database table used to track the schema version.
# This name shouldn't already be used by your project.
# If this is changed once a database is under version control, you'll need to
# change the table name in each database too.
version_table=migrate_version
# When committing a change script, Migrate will attempt to generate the
# sql for all supported databases; normally, if one of them fails - probably
# because you don't have that database installed - it is ignored and the
# commit continues, perhaps ending successfully.
# Databases in this list MUST compile successfully during a commit, or the
# entire commit will fail. List the databases your application will actually
# be using to ensure your updates to that database work properly.
# This must be a list; example: ['postgres','sqlite']
required_dbs=[]

View File

@ -0,0 +1,100 @@
# vim: tabstop=4 shiftwidth=4 softtabstop=4
# Copyright 2011 OpenStack LLC.
# All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
"""
Various conveniences used for migration scripts
"""
import logging
import sqlalchemy.types
from sqlalchemy.schema import MetaData
logger = logging.getLogger('glance.registry.db.migrate_repo.schema')
String = lambda length: sqlalchemy.types.String(
length=length, convert_unicode=False, assert_unicode=None,
unicode_error=None, _warn_on_bytestring=False)
Text = lambda: sqlalchemy.types.Text(
length=None, convert_unicode=False, assert_unicode=None,
unicode_error=None, _warn_on_bytestring=False)
Boolean = lambda: sqlalchemy.types.Boolean(create_constraint=True, name=None)
DateTime = lambda: sqlalchemy.types.DateTime(timezone=False)
Integer = lambda: sqlalchemy.types.Integer()
def from_migration_import(module_name, fromlist):
"""Import a migration file and return the module
:param module_name: name of migration module to import from
(ex: 001_add_images_table)
:param fromlist: list of items to import (ex: define_images_table)
:retval: module object
This bit of ugliness warrants an explanation:
As you're writing migrations, you'll frequently want to refer to
tables defined in previous migrations.
In the interest of not repeating yourself, you need a way of importing
that table into a 'future' migration.
However, tables are bound to metadata, so what you need to import is
really a table factory, which you can late-bind to your current
metadata object.
Moreover, migrations begin with a number (001...), which means they
aren't valid Python identifiers. This means we can't perform a
'normal' import on them (the Python lexer will 'splode). Instead, we
need to use __import__ magic to bring the table-factory into our
namespace.
Example Usage:
(define_images_table,) = from_migration_import(
'001_add_images_table', ['define_images_table'])
images = define_images_table(meta)
# Refer to images table
"""
module_path = 'glance.registry.db.migrate_repo.versions.%s' % module_name
module = __import__(module_path, globals(), locals(), fromlist, -1)
return [getattr(module, item) for item in fromlist]
def create_tables(tables):
for table in tables:
logger.info("creating table %(table)s" % locals())
table.create()
def drop_tables(tables):
for table in tables:
logger.info("dropping table %(table)s" % locals())
table.drop()

View File

@ -0,0 +1,55 @@
# vim: tabstop=4 shiftwidth=4 softtabstop=4
# Copyright 2011 OpenStack LLC.
# All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
from sqlalchemy.schema import (Column, MetaData, Table)
from glance.registry.db.migrate_repo.schema import (
Boolean, DateTime, Integer, String, Text, create_tables, drop_tables)
def define_images_table(meta):
images = Table('images', meta,
Column('id', Integer(), primary_key=True, nullable=False),
Column('name', String(255)),
Column('type', String(30)),
Column('size', Integer()),
Column('status', String(30), nullable=False),
Column('is_public', Boolean(), nullable=False, default=False,
index=True),
Column('location', Text()),
Column('created_at', DateTime(), nullable=False),
Column('updated_at', DateTime()),
Column('deleted_at', DateTime()),
Column('deleted', Boolean(), nullable=False, default=False,
index=True),
mysql_engine='InnoDB')
return images
def upgrade(migrate_engine):
meta = MetaData()
meta.bind = migrate_engine
tables = [define_images_table(meta)]
create_tables(tables)
def downgrade(migrate_engine):
meta = MetaData()
meta.bind = migrate_engine
tables = [define_images_table(meta)]
drop_tables(tables)

View File

@ -0,0 +1,63 @@
# vim: tabstop=4 shiftwidth=4 softtabstop=4
# Copyright 2011 OpenStack LLC.
# All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
from sqlalchemy.schema import (
Column, ForeignKey, Index, MetaData, Table, UniqueConstraint)
from glance.registry.db.migrate_repo.schema import (
Boolean, DateTime, Integer, String, Text, create_tables, drop_tables,
from_migration_import)
def define_image_properties_table(meta):
(define_images_table,) = from_migration_import(
'001_add_images_table', ['define_images_table'])
images = define_images_table(meta)
image_properties = Table('image_properties', meta,
Column('id', Integer(), primary_key=True, nullable=False),
Column('image_id', Integer(), ForeignKey('images.id'), nullable=False,
index=True),
Column('key', String(255), nullable=False),
Column('value', Text()),
Column('created_at', DateTime(), nullable=False),
Column('updated_at', DateTime()),
Column('deleted_at', DateTime()),
Column('deleted', Boolean(), nullable=False, default=False,
index=True),
UniqueConstraint('image_id', 'key'),
mysql_engine='InnoDB')
Index('ix_image_properties_image_id_key', image_properties.c.image_id,
image_properties.c.key)
return image_properties
def upgrade(migrate_engine):
meta = MetaData()
meta.bind = migrate_engine
tables = [define_image_properties_table(meta)]
create_tables(tables)
def downgrade(migrate_engine):
meta = MetaData()
meta.bind = migrate_engine
tables = [define_image_properties_table(meta)]
drop_tables(tables)

View File

@ -0,0 +1 @@
# template repository default versions module

View File

@ -0,0 +1,117 @@
# vim: tabstop=4 shiftwidth=4 softtabstop=4
# Copyright 2011 OpenStack LLC.
# All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
import logging
import os
from migrate.versioning import api as versioning_api
from migrate.versioning import exceptions as versioning_exceptions
from glance.common import exception
def db_version(options):
"""Return the database's current migration number
:param options: options dict
:retval version number
"""
repo_path = _find_migrate_repo()
sql_connection = options['sql_connection']
try:
return versioning_api.db_version(sql_connection, repo_path)
except versioning_exceptions.DatabaseNotControlledError, e:
msg = ("database '%(sql_connection)s' is not under migration control"
% locals())
raise exception.DatabaseMigrationError(msg)
def upgrade(options, version=None):
"""Upgrade the database's current migration level
:param options: options dict
:param version: version to upgrade (defaults to latest)
:retval version number
"""
db_version(options) # Ensure db is under migration control
repo_path = _find_migrate_repo()
sql_connection = options['sql_connection']
version_str = version or 'latest'
logging.info("Upgrading %(sql_connection)s to version %(version_str)s" %
locals())
return versioning_api.upgrade(sql_connection, repo_path, version)
def downgrade(options, version):
"""Downgrade the database's current migration level
:param options: options dict
:param version: version to downgrade to
:retval version number
"""
db_version(options) # Ensure db is under migration control
repo_path = _find_migrate_repo()
sql_connection = options['sql_connection']
logging.info("Downgrading %(sql_connection)s to version %(version)s" %
locals())
return versioning_api.downgrade(sql_connection, repo_path, version)
def version_control(options):
"""Place a database under migration control
:param options: options dict
"""
sql_connection = options['sql_connection']
try:
_version_control(options)
except versioning_exceptions.DatabaseAlreadyControlledError, e:
msg = ("database '%(sql_connection)s' is already under migration "
"control" % locals())
raise exception.DatabaseMigrationError(msg)
def _version_control(options):
"""Place a database under migration control
:param options: options dict
"""
repo_path = _find_migrate_repo()
sql_connection = options['sql_connection']
return versioning_api.version_control(sql_connection, repo_path)
def db_sync(options, version=None):
"""Place a database under migration control and perform an upgrade
:param options: options dict
:retval version number
"""
try:
_version_control(options)
except versioning_exceptions.DatabaseAlreadyControlledError, e:
pass
upgrade(options, version=version)
def _find_migrate_repo():
"""Get the path for the migrate repository."""
path = os.path.join(os.path.abspath(os.path.dirname(__file__)),
'migrate_repo')
assert os.path.exists(path)
return path

View File

@ -42,10 +42,11 @@ class ModelBase(object):
__protected_attributes__ = set([
"created_at", "updated_at", "deleted_at", "deleted"])
created_at = Column(DateTime, default=datetime.datetime.utcnow)
created_at = Column(DateTime, default=datetime.datetime.utcnow,
nullable=False)
updated_at = Column(DateTime, onupdate=datetime.datetime.utcnow)
deleted_at = Column(DateTime)
deleted = Column(Boolean, default=False)
deleted = Column(Boolean, nullable=False, default=False)
def save(self, session=None):
"""Save this object"""
@ -96,8 +97,8 @@ class Image(BASE, ModelBase):
name = Column(String(255))
type = Column(String(30))
size = Column(Integer)
status = Column(String(30))
is_public = Column(Boolean, default=False)
status = Column(String(30), nullable=False)
is_public = Column(Boolean, nullable=False, default=False)
location = Column(Text)
@validates('type')
@ -123,5 +124,7 @@ class ImageProperty(BASE, ModelBase):
image_id = Column(Integer, ForeignKey('images.id'), nullable=False)
image = relationship(Image, backref=backref('properties'))
key = Column(String(255), index=True)
# FIXME(sirp): KEY is a reserved word in SQL, might be a good idea to
# rename this column
key = Column(String(255), index=True, nullable=False)
value = Column(Text)

View File

@ -0,0 +1,48 @@
# vim: tabstop=4 shiftwidth=4 softtabstop=4
# Copyright 2010-2011 OpenStack, LLC
# All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
import os
import unittest
import glance.registry.db.migration as migration_api
import glance.common.config as config
class TestMigrations(unittest.TestCase):
"""Test sqlalchemy-migrate migrations"""
def setUp(self):
self.db_path = "glance_test_migration.sqlite"
self.options = dict(sql_connection="sqlite:///%s" % self.db_path,
verbose=False)
config.setup_logging(self.options)
def tearDown(self):
if os.path.exists(self.db_path):
os.unlink(self.db_path)
def test_db_sync_downgrade_then_upgrade(self):
migration_api.db_sync(self.options)
latest = migration_api.db_version(self.options)
migration_api.downgrade(self.options, latest-1)
cur_version = migration_api.db_version(self.options)
self.assertEqual(cur_version, latest-1)
migration_api.upgrade(self.options, cur_version+1)
cur_version = migration_api.db_version(self.options)
self.assertEqual(cur_version, latest)

View File

@ -13,3 +13,4 @@ sphinx
argparse
mox==0.5.0
-f http://pymox.googlecode.com/files/mox-0.5.0.tar.gz
sqlalchemy-migrate>=0.6