Browse Source

Add SQLalchemy database model

As a step towards continuous deployment and having a pecan/WSME REST
interface, split the database out into SQLalchemy-based model using
Alembic for migrations. To support that, also pull in oslo.db and use
oslo.config for config files.

Change-Id: I33a1e72700be14e28255aaa52faed70c4686a3ec
changes/39/62239/5
Monty Taylor 9 years ago committed by Ruslan Kamaldinov
parent
commit
d6066f2a75
  1. 61
      .gitignore
  2. 17
      CONTRIBUTING.rst
  3. 4
      MANIFEST.in
  4. 1
      babel.cfg
  5. 75
      doc/source/conf.py
  6. 1
      doc/source/contributing.rst
  7. 23
      doc/source/index.rst
  8. 12
      doc/source/installation.rst
  9. 1
      doc/source/readme.rst
  10. 74
      etc/storyboard.conf
  11. 8
      openstack-common.conf
  12. 5
      requirements.txt
  13. 34
      setup.cfg
  14. 19
      storyboard/__init__.py
  15. 0
      storyboard/db/__init__.py
  16. 75
      storyboard/db/migration/README
  17. 0
      storyboard/db/migration/__init__.py
  18. 52
      storyboard/db/migration/alembic.ini
  19. 1
      storyboard/db/migration/alembic_migrations/README
  20. 17
      storyboard/db/migration/alembic_migrations/__init__.py
  21. 83
      storyboard/db/migration/alembic_migrations/env.py
  22. 39
      storyboard/db/migration/alembic_migrations/script.py.mako
  23. 226
      storyboard/db/migration/alembic_migrations/versions/18708bcdc0fe_initial_version.py
  24. 3
      storyboard/db/migration/alembic_migrations/versions/README
  25. 122
      storyboard/db/migration/cli.py
  26. 235
      storyboard/db/models.py
  27. 0
      storyboard/openstack/__init__.py
  28. 0
      storyboard/openstack/common/__init__.py
  29. 16
      storyboard/openstack/common/db/__init__.py
  30. 106
      storyboard/openstack/common/db/api.py
  31. 51
      storyboard/openstack/common/db/exception.py
  32. 16
      storyboard/openstack/common/db/sqlalchemy/__init__.py
  33. 278
      storyboard/openstack/common/db/sqlalchemy/migration.py
  34. 110
      storyboard/openstack/common/db/sqlalchemy/models.py
  35. 797
      storyboard/openstack/common/db/sqlalchemy/session.py
  36. 289
      storyboard/openstack/common/db/sqlalchemy/test_migrations.py
  37. 501
      storyboard/openstack/common/db/sqlalchemy/utils.py
  38. 101
      storyboard/openstack/common/excutils.py
  39. 139
      storyboard/openstack/common/fileutils.py
  40. 373
      storyboard/openstack/common/gettextutils.py
  41. 68
      storyboard/openstack/common/importutils.py
  42. 180
      storyboard/openstack/common/jsonutils.py
  43. 47
      storyboard/openstack/common/local.py
  44. 305
      storyboard/openstack/common/lockutils.py
  45. 626
      storyboard/openstack/common/log.py
  46. 54
      storyboard/openstack/common/test.py
  47. 197
      storyboard/openstack/common/timeutils.py
  48. 17
      storyboard/stories/tests.py
  49. 0
      storyboard/tests/__init__.py
  50. 74
      storyboard/tests/base.py
  51. 98
      storyboard/tests/test_db_migration.py
  52. 28
      storyboard/tests/test_stories.py
  53. 1
      storyboard/wsgi.py
  54. 6
      test-requirements.txt
  55. 22
      tox.ini

61
.gitignore vendored

@ -1,14 +1,59 @@
*.pyc
*.py[cod]
# C extensions
*.so
# Editor save files
*~
*.swo
*.swp
*.sqlite
# Development databases
*.db
*~
local_settings.py
storyboard.db
# Packages
*.egg
*.egg-info
dist
build
eggs
parts
bin
var
sdist
develop-eggs
.installed.cfg
lib
lib64
# Installer logs
pip-log.txt
# Unit test / coverage reports
nosetests.xml
.coverage
.tox
.testrepository
.venv
# Translations
*.mo
# Mr Developer
.mr.developer.cfg
.project
.pydevproject
# Complexity
output/*.html
output/*/index.html
# Sphinx
doc/build
# pbr generates these
AUTHORS
ChangeLog
*.egg-info
/*.egg
.testrepository
# Local settings
local_settings.py

17
CONTRIBUTING.rst

@ -0,0 +1,17 @@
If you would like to contribute to the development of OpenStack,
you must follow the steps in the "If you're a developer, start here"
section of this page:
http://wiki.openstack.org/HowToContribute
Once those steps have been completed, changes to OpenStack
should be submitted for review via the Gerrit tool, following
the workflow documented at:
http://wiki.openstack.org/GerritWorkflow
Pull requests submitted through GitHub will be ignored.
Storyboard uses itself for bug tracking. Bugs should be filed at:
https://stories.openstack.org

4
MANIFEST.in

@ -1,5 +1,9 @@
include AUTHORS
include ChangeLog
include storyboard/db/migration/README
include storyboard/db/migration/alembic.ini
include storyboard/db/migration/alembic_migrations/script.py.mako
include storyboard/db/migration/alembic_migrations/versions/README
exclude .gitignore
exclude .gitreview

1
babel.cfg

@ -0,0 +1 @@
[python: **.py]

75
doc/source/conf.py

@ -0,0 +1,75 @@
# -*- coding: utf-8 -*-
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
# implied.
# See the License for the specific language governing permissions and
# limitations under the License.
import os
import sys
sys.path.insert(0, os.path.abspath('../..'))
# -- General configuration ----------------------------------------------------
# Add any Sphinx extension module names here, as strings. They can be
# extensions coming with Sphinx (named 'sphinx.ext.*') or your custom ones.
extensions = [
'sphinx.ext.autodoc',
'sphinx.ext.intersphinx',
'oslo.sphinx'
]
# autodoc generation is a bit aggressive and a nuisance when doing heavy
# text edit cycles.
# execute "export SPHINX_DEBUG=1" in your terminal to disable
# The suffix of source filenames.
source_suffix = '.rst'
# The master toctree document.
master_doc = 'index'
# General information about the project.
project = u'storyboard'
copyright = u'2013, OpenStack Foundation'
# If true, '()' will be appended to :func: etc. cross-reference text.
add_function_parentheses = True
# If true, the current module name will be prepended to all description
# unit titles (such as .. function::).
add_module_names = True
# The name of the Pygments (syntax highlighting) style to use.
pygments_style = 'sphinx'
# -- Options for HTML output --------------------------------------------------
# The theme to use for HTML and HTML Help pages. Major themes that come with
# Sphinx are currently 'default' and 'sphinxdoc'.
# html_theme_path = ["."]
# html_theme = '_theme'
# html_static_path = ['static']
# Output file base name for HTML help builder.
htmlhelp_basename = '%sdoc' % project
# Grouping the document tree into LaTeX files. List of tuples
# (source start file, target name, title, author, documentclass
# [howto/manual]).
latex_documents = [
('index',
'%s.tex' % project,
u'%s Documentation' % project,
u'OpenStack Foundation', 'manual'),
]
# Example configuration for intersphinx: refer to the Python standard library.
intersphinx_mapping = {'http://docs.python.org/': None}

1
doc/source/contributing.rst

@ -0,0 +1 @@
.. include:: ../../CONTRIBUTING.rst

23
doc/source/index.rst

@ -0,0 +1,23 @@
.. storyboard documentation master file, created by
sphinx-quickstart on Tue Jul 9 22:26:36 2013.
You can adapt this file completely to your liking, but it should at least
contain the root `toctree` directive.
Welcome to storyboard's documentation!
========================================================
Contents:
.. toctree::
:maxdepth: 2
readme
installation
contributing
Indices and tables
==================
* :ref:`genindex`
* :ref:`modindex`
* :ref:`search`

12
doc/source/installation.rst

@ -0,0 +1,12 @@
============
Installation
============
At the command line::
$ pip install storyboard
Or, if you have virtualenvwrapper installed::
$ mkvirtualenv storyboard
$ pip install storyboard

1
doc/source/readme.rst

@ -0,0 +1 @@
.. include:: ../../README.rst

74
etc/storyboard.conf

@ -0,0 +1,74 @@
[DEFAULT]
# Default log level is INFO
# verbose and debug has the same result.
# One of them will set DEBUG log level output
# debug = False
# verbose = False
# Where to store lock files
lock_path = $state_path/lock
# log_format = %(asctime)s %(levelname)8s [%(name)s] %(message)s
# log_date_format = %Y-%m-%d %H:%M:%S
# use_syslog -> syslog
# log_file and log_dir -> log_dir/log_file
# (not log_file) and log_dir -> log_dir/{binary_name}.log
# use_stderr -> stderr
# (not user_stderr) and (not log_file) -> stdout
# publish_errors -> notification system
# use_syslog = False
# syslog_log_facility = LOG_USER
# use_stderr = True
# log_file =
# log_dir =
# publish_errors = False
# Address to bind the API server
# bind_host = 0.0.0.0
# Port the bind the API server to
# bind_port = 9696
[database]
# This line MUST be changed to actually run storyboard
# Example:
# connection = mysql://root:pass@127.0.0.1:3306/storyboard
# Replace 127.0.0.1 above with the IP address of the database used by the
# main storyboard server. (Leave it as is if the database runs on this host.)
# connection = sqlite://
# The SQLAlchemy connection string used to connect to the slave database
# slave_connection =
# Database reconnection retry times - in event connectivity is lost
# set to -1 implies an infinite retry count
# max_retries = 10
# Database reconnection interval in seconds - if the initial connection to the
# database fails
# retry_interval = 10
# Minimum number of SQL connections to keep open in a pool
# min_pool_size = 1
# Maximum number of SQL connections to keep open in a pool
# max_pool_size = 10
# Timeout in seconds before idle sql connections are reaped
# idle_timeout = 3600
# If set, use this value for max_overflow with sqlalchemy
# max_overflow = 20
# Verbosity of SQL debugging information. 0=None, 100=Everything
# connection_debug = 0
# Add python stack traces to SQL as comment strings
# connection_trace = False
# If set, use this value for pool_timeout with sqlalchemy
# pool_timeout = 10

8
openstack-common.conf

@ -0,0 +1,8 @@
[DEFAULT]
# The list of modules to copy from oslo-incubator.git
module=db
module=db.sqlalchemy
# The base module to hold the copy of openstack.common
base=storyboard

5
requirements.txt

@ -5,3 +5,8 @@ django-openid-auth
markdown
python-openid
six>=1.4.1
Babel>=0.9.6
SQLAlchemy>=0.8
alembic>=0.4.1
oslo.config>=1.2.0
iso8601>=0.1.8

34
setup.cfg

@ -1,6 +1,6 @@
[metadata]
name = storyboard
summary = OpenStack StoryBoard
summary = OpenStack Story Tracking
description-file =
README.rst
author = OpenStack
@ -18,9 +18,39 @@ classifier =
Programming Language :: Python
Programming Language :: Python :: 2
Programming Language :: Python :: 2.7
Programming Language :: Python :: 2.6
Programming Language :: Python :: 3
Programming Language :: Python :: 3.3
Topic :: Internet :: WWW/HTTP
[files]
packages =
storyboard
data_files =
etc/storyboard =
etc/storyboard.conf
[entry_points]
console_scripts =
storyboard-db-manage = storyboard.db.migration.cli:main
[build_sphinx]
source-dir = doc/source
build-dir = doc/build
all_files = 1
[upload_sphinx]
upload-dir = doc/build/html
[compile_catalog]
directory = storyboard/locale
domain = storyboard
[update_catalog]
domain = storyboard
output_dir = storyboard/locale
input_file = storyboard/locale/storyboard.pot
[extract_messages]
keywords = _ gettext ngettext l_ lazy_gettext
mapping_file = babel.cfg
output_file = storyboard/locale/storyboard.pot

19
storyboard/__init__.py

@ -0,0 +1,19 @@
# -*- coding: utf-8 -*-
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
import pbr.version
__version__ = pbr.version.VersionInfo(
'storyboard').version_string()

0
storyboard/db/__init__.py

75
storyboard/db/migration/README

@ -0,0 +1,75 @@
# Copyright 2012 New Dream Network, LLC (DreamHost)
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
The migrations in the alembic_migrations/versions contain the changes needed
to migrate from older storyboard releases to newer versions. A migration occurs
by executing a script that details the changes needed to upgrade/downgrade
the database. The migration scripts are ordered so that multiple scripts
can run sequentially to update the database. The scripts are executed by
storyboard's migration wrapper which uses the Alembic library to manage the
migration.
You can upgrade to the latest database version via:
$ storyboard-db-manage --config-file /path/to/storyboard.conf upgrade head
To check the current database version:
$ storyboard-db-manage --config-file /path/to/storyboard.conf current
To create a script to run the migration offline:
$ storyboard-db-manage --config-file /path/to/storyboard.conf upgrade head --sql
To run the offline migration between specific migration versions:
$ storyboard-db-manage --config-file /path/to/storyboard.conf \
upgrade <start version>:<end version> --sql
Upgrade the database incrementally:
$ storyboard-db-manage --config-file /path/to/storyboard.conf \
upgrade --delta <# of revs>
Downgrade the database by a certain number of revisions:
$ storyboard-db-manage --config-file /path/to/storyboard.conf \
downgrade --delta <# of revs>
DEVELOPERS:
A database migration script is required when you submit a change to storyboard
that alters the database model definition. The migration script is a special
python file that includes code to update/downgrade the database to match the
changes in the model definition. Alembic will execute these scripts in order to
provide a linear migration path between revision. The storyboard-db-manage
command can be used to generate migration template for you to complete. The
operations in the template are those supported by the Alembic migration library.
$ storyboard-db-manage --config-file /path/to/storyboard.conf \
revision -m "description of revision" --autogenerate
This generates a prepopulated template with the changes needed to match the
database state with the models. You should inspect the autogenerated template
to ensure that the proper models have been altered.
In rare circumstances, you may want to start with an empty migration template
and manually author the changes necessary for an upgrade/downgrade. You can
create a blank file via:
$ storyboard-db-manage --config-file /path/to/storyboard.conf \
revision -m "description of revision"
The migration timeline should remain linear so that there is a clear path when
upgrading/downgrading. To verify that the timeline does branch, you can run
this command:
$ storyboard-db-manage --config-file /path/to/storyboard.conf check_migration
If the migration path does branch, you can find the branch point via:
$ storyboard-db-manage --config-file /path/to/storyboard.conf history

0
storyboard/db/migration/__init__.py

52
storyboard/db/migration/alembic.ini

@ -0,0 +1,52 @@
# A generic, single database configuration.
[alembic]
# path to migration scripts
script_location = %(here)s/alembic_migrations
# template used to generate migration files
# file_template = %%(rev)s_%%(slug)s
# set to 'true' to run the environment during
# the 'revision' command, regardless of autogenerate
# revision_environment = false
# default to an empty string because the storyboard migration cli will
# extract the correct value and set it programatically before alembic is fully
# invoked.
sqlalchemy.url =
# Logging configuration
[loggers]
keys = root,sqlalchemy,alembic
[handlers]
keys = console
[formatters]
keys = generic
[logger_root]
level = WARN
handlers = console
qualname =
[logger_sqlalchemy]
level = WARN
handlers =
qualname = sqlalchemy.engine
[logger_alembic]
level = INFO
handlers =
qualname = alembic
[handler_console]
class = StreamHandler
args = (sys.stderr,)
level = NOTSET
formatter = generic
[formatter_generic]
format = %(levelname)-5.5s [%(name)s] %(message)s
datefmt = %H:%M:%S

1
storyboard/db/migration/alembic_migrations/README

@ -0,0 +1 @@
Generic single-database configuration.

17
storyboard/db/migration/alembic_migrations/__init__.py

@ -0,0 +1,17 @@
# vim: tabstop=4 shiftwidth=4 softtabstop=4
#
# Copyright 2012 New Dream Network, LLC (DreamHost)
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
#
# @author: Mark McClain, DreamHost

83
storyboard/db/migration/alembic_migrations/env.py

@ -0,0 +1,83 @@
# vim: tabstop=4 shiftwidth=4 softtabstop=4
#
# Copyright 2012 New Dream Network, LLC (DreamHost)
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
# from logging.config import fileConfig
from alembic import context
from sqlalchemy import create_engine, pool
from storyboard.db import models
# this is the Alembic Config object, which provides
# access to the values within the .ini file in use.
config = context.config
storyboard_config = config.storyboard_config
# Interpret the config file for Python logging.
# This line sets up loggers basically.
# TODO(mordred): enable this once we're doing something with logging
# fileConfig(config.config_file_name)
# set the target for 'autogenerate' support
target_metadata = models.Base.metadata
def run_migrations_offline():
"""Run migrations in 'offline' mode.
This configures the context with just a URL
and not an Engine, though an Engine is acceptable
here as well. By skipping the Engine creation
we don't even need a DBAPI to be available.
Calls to context.execute() here emit the given string to the
script output.
"""
context.configure(url=storyboard_config.database.connection)
with context.begin_transaction():
context.run_migrations()
def run_migrations_online():
"""Run migrations in 'online' mode.
In this scenario we need to create an Engine
and associate a connection with the context.
"""
engine = create_engine(
storyboard_config.database.connection,
poolclass=pool.NullPool)
connection = engine.connect()
context.configure(
connection=connection,
target_metadata=target_metadata
)
try:
with context.begin_transaction():
context.run_migrations()
finally:
connection.close()
if context.is_offline_mode():
run_migrations_offline()
else:
run_migrations_online()

39
storyboard/db/migration/alembic_migrations/script.py.mako

@ -0,0 +1,39 @@
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
#
"""${message}
Revision ID: ${up_revision}
Revises: ${down_revision}
Create Date: ${create_date}
"""
# revision identifiers, used by Alembic.
revision = ${repr(up_revision)}
down_revision = ${repr(down_revision)}
from alembic import op
import sqlalchemy as sa
${imports if imports else ""}
def upgrade(active_plugins=None, options=None):
${upgrades if upgrades else "pass"}
def downgrade(active_plugins=None, options=None):
${downgrades if downgrades else "pass"}

226
storyboard/db/migration/alembic_migrations/versions/18708bcdc0fe_initial_version.py

@ -0,0 +1,226 @@
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
#
"""initial version
Revision ID: 18708bcdc0fe
Revises: None
Create Date: 2013-12-10 00:35:55.327593
"""
# revision identifiers, used by Alembic.
revision = '18708bcdc0fe'
down_revision = None
from alembic import op
import sqlalchemy as sa
def upgrade(active_plugins=None, options=None):
op.create_table(
'branches',
sa.Column('id', sa.Integer(), nullable=False),
sa.Column('created_at', sa.DateTime(), nullable=True),
sa.Column('updated_at', sa.DateTime(), nullable=True),
sa.Column('name', sa.String(length=50), nullable=True),
sa.Column(
'status',
sa.Enum(
'master', 'release', 'stable', 'unsupported',
name='branch_status'), nullable=True),
sa.Column('release_date', sa.DateTime(), nullable=True),
sa.PrimaryKeyConstraint('id'),
sa.UniqueConstraint('name', name='uniq_branch_name')
)
op.create_table(
'project_groups',
sa.Column('id', sa.Integer(), nullable=False),
sa.Column('created_at', sa.DateTime(), nullable=True),
sa.Column('updated_at', sa.DateTime(), nullable=True),
sa.Column('name', sa.String(length=50), nullable=True),
sa.Column('title', sa.Unicode(length=100), nullable=True),
sa.PrimaryKeyConstraint('id'),
sa.UniqueConstraint('name', name='uniq_group_name')
)
op.create_table(
'users',
sa.Column('id', sa.Integer(), nullable=False),
sa.Column('created_at', sa.DateTime(), nullable=True),
sa.Column('updated_at', sa.DateTime(), nullable=True),
sa.Column('username', sa.Unicode(length=30), nullable=True),
sa.Column('first_name', sa.Unicode(length=30), nullable=True),
sa.Column('last_name', sa.Unicode(length=30), nullable=True),
sa.Column('email', sa.String(length=255), nullable=True),
sa.Column('password', sa.UnicodeText(), nullable=True),
sa.Column('is_staff', sa.Boolean(), nullable=True),
sa.Column('is_active', sa.Boolean(), nullable=True),
sa.Column('is_superuser', sa.Boolean(), nullable=True),
sa.Column('last_login', sa.DateTime(), nullable=True),
sa.PrimaryKeyConstraint('id'),
sa.UniqueConstraint('email', name='uniq_user_email'),
sa.UniqueConstraint('username', name='uniq_user_username')
)
op.create_table(
'teams',
sa.Column('id', sa.Integer(), nullable=False),
sa.Column('created_at', sa.DateTime(), nullable=True),
sa.Column('updated_at', sa.DateTime(), nullable=True),
sa.Column('name', sa.Unicode(length=255), nullable=True),
sa.PrimaryKeyConstraint('id'),
sa.UniqueConstraint('name', name='uniq_team_name')
)
op.create_table(
'permissions',
sa.Column('id', sa.Integer(), nullable=False),
sa.Column('created_at', sa.DateTime(), nullable=True),
sa.Column('updated_at', sa.DateTime(), nullable=True),
sa.Column('name', sa.Unicode(length=50), nullable=True),
sa.Column('codename', sa.Unicode(length=255), nullable=True),
sa.PrimaryKeyConstraint('id'),
sa.UniqueConstraint('name', name='uniq_team_name')
)
op.create_table(
'team_membership',
sa.Column('user_id', sa.Integer(), nullable=True),
sa.Column('team_id', sa.Integer(), nullable=True),
sa.ForeignKeyConstraint(['team_id'], ['teams.id'], ),
sa.ForeignKeyConstraint(['user_id'], ['users.id'], ),
sa.PrimaryKeyConstraint()
)
op.create_table(
'user_permissions',
sa.Column('user_id', sa.Integer(), nullable=True),
sa.Column('permission_id', sa.Integer(), nullable=True),
sa.ForeignKeyConstraint(['permission_id'], ['permissions.id'], ),
sa.ForeignKeyConstraint(['user_id'], ['users.id'], ),
sa.PrimaryKeyConstraint()
)
op.create_table(
'team_permissions',
sa.Column('permission_id', sa.Integer(), nullable=True),
sa.Column('team_id', sa.Integer(), nullable=True),
sa.ForeignKeyConstraint(['team_id'], ['teams.id'], ),
sa.ForeignKeyConstraint(['permission_id'], ['permissions.id'], ),
sa.PrimaryKeyConstraint()
)
op.create_table(
'storyboard',
sa.Column('id', sa.Integer(), nullable=False),
sa.Column('created_at', sa.DateTime(), nullable=True),
sa.Column('updated_at', sa.DateTime(), nullable=True),
sa.Column('creator_id', sa.Integer(), nullable=True),
sa.Column('title', sa.Unicode(length=100), nullable=True),
sa.Column('description', sa.UnicodeText(), nullable=True),
sa.Column('is_bug', sa.Boolean(), nullable=True),
sa.Column(
'priority',
sa.Enum(
'Undefined', 'Low', 'Medium', 'High', 'Critical',
name='priority'), nullable=True),
sa.ForeignKeyConstraint(['creator_id'], ['users.id'], ),
sa.PrimaryKeyConstraint('id')
)
op.create_table(
'milestones',
sa.Column('id', sa.Integer(), nullable=False),
sa.Column('created_at', sa.DateTime(), nullable=True),
sa.Column('updated_at', sa.DateTime(), nullable=True),
sa.Column('name', sa.String(length=50), nullable=True),
sa.Column('branch_id', sa.Integer(), nullable=True),
sa.Column('released', sa.Boolean(), nullable=True),
sa.Column('undefined', sa.Boolean(), nullable=True),
sa.ForeignKeyConstraint(['branch_id'], ['branches.id'], ),
sa.PrimaryKeyConstraint('id'),
sa.UniqueConstraint('name', name='uniq_milestone_name')
)
op.create_table(
'projects',
sa.Column('id', sa.Integer(), nullable=False),
sa.Column('created_at', sa.DateTime(), nullable=True),
sa.Column('updated_at', sa.DateTime(), nullable=True),
sa.Column('name', sa.String(length=50), nullable=True),
sa.Column('description', sa.Unicode(length=100), nullable=True),
sa.Column('team_id', sa.Integer(), nullable=True),
sa.ForeignKeyConstraint(['team_id'], ['teams.id'], ),
sa.PrimaryKeyConstraint('id'),
sa.UniqueConstraint('name', name='uniq_project_name')
)
op.create_table(
'project_group_mapping',
sa.Column('project_id', sa.Integer(), nullable=True),
sa.Column('project_group_id', sa.Integer(), nullable=True),
sa.ForeignKeyConstraint(['project_group_id'], ['project_groups.id'], ),
sa.ForeignKeyConstraint(['project_id'], ['projects.id'], ),
sa.PrimaryKeyConstraint()
)
op.create_table(
'tasks',
sa.Column('id', sa.Integer(), nullable=False),
sa.Column('created_at', sa.DateTime(), nullable=True),
sa.Column('updated_at', sa.DateTime(), nullable=True),
sa.Column('title', sa.Unicode(length=100), nullable=True),
sa.Column(
'status', sa.Enum('Todo', 'In review', 'Landed'), nullable=True),
sa.Column('story_id', sa.Integer(), nullable=True),
sa.Column('project_id', sa.Integer(), nullable=True),
sa.Column('assignee_id', sa.Integer(), nullable=True),
sa.Column('milestone_id', sa.Integer(), nullable=True),
sa.ForeignKeyConstraint(['assignee_id'], ['users.id'], ),
sa.ForeignKeyConstraint(['milestone_id'], ['milestones.id'], ),
sa.ForeignKeyConstraint(['project_id'], ['projects.id'], ),
sa.ForeignKeyConstraint(['story_id'], ['storyboard.id'], ),
sa.PrimaryKeyConstraint('id')
)
op.create_table(
'comments',
sa.Column('id', sa.Integer(), nullable=False),
sa.Column('created_at', sa.DateTime(), nullable=True),
sa.Column('updated_at', sa.DateTime(), nullable=True),
sa.Column('action', sa.String(length=150), nullable=True),
sa.Column('comment_type', sa.String(length=20), nullable=True),
sa.Column('content', sa.UnicodeText(), nullable=True),
sa.Column('story_id', sa.Integer(), nullable=True),
sa.Column('author_id', sa.Integer(), nullable=True),
sa.ForeignKeyConstraint(['author_id'], ['users.id'], ),
sa.ForeignKeyConstraint(['story_id'], ['storyboard.id'], ),
sa.PrimaryKeyConstraint('id')
)
op.create_table(
'storytags',
sa.Column('id', sa.Integer(), nullable=False),
sa.Column('created_at', sa.DateTime(), nullable=True),
sa.Column('updated_at', sa.DateTime(), nullable=True),
sa.Column('name', sa.String(length=20), nullable=True),
sa.Column('story_id', sa.Integer(), nullable=True),
sa.ForeignKeyConstraint(['story_id'], ['storyboard.id'], ),
sa.PrimaryKeyConstraint('id'),
sa.UniqueConstraint('name', name='uniq_story_tags_name')
)
def downgrade(active_plugins=None, options=None):
op.drop_table('storytags')
op.drop_table('comments')
op.drop_table('tasks')
op.drop_table('project_groups')
op.drop_table('projects')
op.drop_table('milestones')
op.drop_table('storyboard')
op.drop_table('team_membership')
op.drop_table('teams')
op.drop_table('users')
op.drop_table('groups')
op.drop_table('branches')

3
storyboard/db/migration/alembic_migrations/versions/README

@ -0,0 +1,3 @@
This directory contains the migration scripts for the storyboard project. Please
see the README in storyboard/db/migration on how to use and generate new
migrations.

122
storyboard/db/migration/cli.py

@ -0,0 +1,122 @@
# -*- encoding: utf-8 -*-
#
# Copyright 2013 Hewlett-Packard Development Company, L.P.
# Copyright 2012 New Dream Network, LLC (DreamHost)
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
import gettext
import os
from alembic import command as alembic_command
from alembic import config as alembic_config
from alembic import util as alembic_util
from oslo.config import cfg
gettext.install('storyboard', unicode=1)
_db_opts = [
cfg.StrOpt('connection',
default='',
help=_('URL to database')),
]
CONF = cfg.ConfigOpts()
CONF.register_opts(_db_opts, 'database')
def do_alembic_command(config, cmd, *args, **kwargs):
try:
getattr(alembic_command, cmd)(config, *args, **kwargs)
except alembic_util.CommandError as e:
alembic_util.err(str(e))
def do_check_migration(config, cmd):
do_alembic_command(config, 'branches')
def do_upgrade_downgrade(config, cmd):
if not CONF.command.revision and not CONF.command.delta:
raise SystemExit(_('You must provide a revision or relative delta'))
revision = CONF.command.revision
if CONF.command.delta:
sign = '+' if CONF.command.name == 'upgrade' else '-'
revision = sign + str(CONF.command.delta)
else:
revision = CONF.command.revision
do_alembic_command(config, cmd, revision, sql=CONF.command.sql)
def do_stamp(config, cmd):
do_alembic_command(config, cmd,
CONF.command.revision,
sql=CONF.command.sql)
def do_revision(config, cmd):
do_alembic_command(config, cmd,
message=CONF.command.message,
autogenerate=CONF.command.autogenerate,
sql=CONF.command.sql)
def add_command_parsers(subparsers):
for name in ['current', 'history', 'branches']:
parser = subparsers.add_parser(name)
parser.set_defaults(func=do_alembic_command)
parser = subparsers.add_parser('check_migration')
parser.set_defaults(func=do_check_migration)
for name in ['upgrade', 'downgrade']:
parser = subparsers.add_parser(name)
parser.add_argument('--delta', type=int)
parser.add_argument('--sql', action='store_true')
parser.add_argument('revision', nargs='?')
parser.set_defaults(func=do_upgrade_downgrade)
parser = subparsers.add_parser('stamp')
parser.add_argument('--sql', action='store_true')
parser.add_argument('revision')
parser.set_defaults(func=do_stamp)
parser = subparsers.add_parser('revision')
parser.add_argument('-m', '--message')
parser.add_argument('--autogenerate', action='store_true')
parser.add_argument('--sql', action='store_true')
parser.set_defaults(func=do_revision)
command_opt = cfg.SubCommandOpt('command',
title='Command',
help=_('Available commands'),
handler=add_command_parsers)
CONF.register_cli_opt(command_opt)
def main():
config = alembic_config.Config(
os.path.join(os.path.dirname(__file__), 'alembic.ini')
)
config.set_main_option('script_location',
'storyboard.db.migration:alembic_migrations')
# attach the storyboard conf to the Alembic conf
config.storyboard_config = CONF
CONF()
CONF.command.func(config, CONF.command.name)

235
storyboard/db/models.py

@ -0,0 +1,235 @@
# Copyright 2013 Hewlett-Packard Development Company, L.P.
# Copyright 2013 Thierry Carrez <thierry@openstack.org>
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
"""
SQLAlchemy Models for storing storyboard
"""
import urlparse
import warnings
from oslo.config import cfg
from sqlalchemy.ext import declarative
from sqlalchemy.orm import relationship
from sqlalchemy import schema
from sqlalchemy import Boolean
from sqlalchemy import Column
from sqlalchemy import DateTime
from sqlalchemy import Enum
from sqlalchemy import ForeignKey
from sqlalchemy import Integer
from sqlalchemy import String
from sqlalchemy import Table
from sqlalchemy import Unicode
from sqlalchemy import UnicodeText
from storyboard.openstack.common.db.sqlalchemy import models
# Turn SQLAlchemy warnings into errors
warnings.simplefilter('error')
CONF = cfg.CONF
def table_args():
engine_name = urlparse.urlparse(cfg.CONF.database_connection).scheme
if engine_name == 'mysql':
return {'mysql_engine': cfg.CONF.mysql_engine,
'mysql_charset': "utf8"}
return None
class IdMixin(object):
id = Column(Integer, primary_key=True)
class StoriesBase(models.TimestampMixin,
IdMixin,
models.ModelBase):
metadata = None
@declarative.declared_attr
def __tablename__(cls):
# NOTE(jkoelker) use the pluralized name of the class as the table
return cls.__name__.lower() + 's'
def as_dict(self):
d = {}
for c in self.__table__.columns:
d[c.name] = self[c.name]
return d
Base = declarative.declarative_base(cls=StoriesBase)
user_permissions = Table(
'user_permissions', Base.metadata,
Column('user_id', Integer, ForeignKey('users.id')),
Column('permission_id', Integer, ForeignKey('permissions.id')),
)
team_permissions = Table(
'team_permissions', Base.metadata,
Column('team_id', Integer, ForeignKey('teams.id')),
Column('permission_id', Integer, ForeignKey('permissions.id')),
)
team_membership = Table(
'team_membership', Base.metadata,
Column('user_id', Integer, ForeignKey('users.id')),
Column('team_id', Integer, ForeignKey('teams.id')),
)
class User(Base):
__table_args__ = (
schema.UniqueConstraint('username', name='uniq_user_username'),
schema.UniqueConstraint('email', name='uniq_user_email'),
)
username = Column(Unicode(30))
first_name = Column(Unicode(30), nullable=True)
last_name = Column(Unicode(30), nullable=True)
email = Column(String(255))
password = Column(UnicodeText)
is_staff = Column(Boolean, default=False)
is_active = Column(Boolean, default=True)
is_superuser = Column(Boolean, default=False)
last_login = Column(DateTime)
teams = relationship("Team", secondary="team_membership")
permissions = relationship("Permission", secondary="user_permissions")
tasks = relationship('Task', backref='assignee')
class Team(Base):
__table_args__ = (
schema.UniqueConstraint('name', name='uniq_team_name'),
)
name = Column(Unicode(255))
users = relationship("User", secondary="team_membership")
permissions = relationship("Permission", secondary="team_permissions")
project_group_mapping = Table(
'project_group_mapping', Base.metadata,
Column('project_id', Integer, ForeignKey('projects.id')),
Column('project_group_id', Integer, ForeignKey('project_groups.id')),
)
class Permission(Base):
__table_args__ = (
schema.UniqueConstraint('name', name='uniq_permission_name'),
)
name = Column(Unicode(50))
codename = Column(Unicode(255))
# TODO(mordred): Do we really need name and title?
class Project(Base):
"""Represents a software project."""
__table_args__ = (
schema.UniqueConstraint('name', name='uniq_project_name'),
)
name = Column(String(50))
description = Column(Unicode(100))
team_id = Column(Integer, ForeignKey('teams.id'))
team = relationship(Team, primaryjoin=team_id == Team.id)
tasks = relationship('Task', backref='project')
class ProjectGroup(Base):
__tablename__ = 'project_groups'
__table_args__ = (
schema.UniqueConstraint('name', name='uniq_group_name'),
)
name = Column(String(50))
title = Column(Unicode(100))
projects = relationship("Project", secondary="project_group_mapping")
class Branch(Base):
# TODO(mordred): order_by release date?
_BRANCH_STATUS = ('master', 'release', 'stable', 'unsupported')
__tablename__ = 'branches'
__table_args__ = (
schema.UniqueConstraint('name', name='uniq_branch_name'),
)
id = Column(Integer, primary_key=True)
name = Column(String(50))
status = Column(Enum(*_BRANCH_STATUS, name='branch_status'))
release_date = Column(DateTime, nullable=True)
class Milestone(Base):
__table_args__ = (
schema.UniqueConstraint('name', name='uniq_milestone_name'),
)
name = Column(String(50))
branch_id = Column(Integer, ForeignKey('branches.id'))
branch = relationship(Branch, primaryjoin=branch_id == Branch.id)
released = Column(Boolean, default=False)
undefined = Column(Boolean, default=False)
tasks = relationship('Task', backref='milestone')
class Story(Base):
__tablename__ = 'stories'
_STORY_PRIORITIES = ('Undefined', 'Low', 'Medium', 'High', 'Critical')
creator_id = Column(Integer, ForeignKey('users.id'))
creator = relationship(User, primaryjoin=creator_id == User.id)
title = Column(Unicode(100))
description = Column(UnicodeText())
is_bug = Column(Boolean, default=True)
priority = Column(Enum(*_STORY_PRIORITIES, name='priority'))
tasks = relationship('Task', backref='story')
comments = relationship('Comment', backref='story')
tags = relationship('StoryTag', backref='story')
class Task(Base):
_TASK_STATUSES = ('Todo', 'In review', 'Landed')
title = Column(Unicode(100), nullable=True)
status = Column(Enum(*_TASK_STATUSES), default='Todo')
story_id = Column(Integer, ForeignKey('stories.id'))
project_id = Column(Integer, ForeignKey('projects.id'))
assignee_id = Column(Integer, ForeignKey('users.id'), nullable=True)
milestone_id = Column(Integer, ForeignKey('milestones.id'), nullable=True)
class Comment(Base):
action = Column(String(150), nullable=True)
comment_type = Column(String(20))
content = Column(UnicodeText)
story_id = Column(Integer, ForeignKey('stories.id'))
author_id = Column(Integer, ForeignKey('users.id'), nullable=True)
author = relationship('User', primaryjoin=author_id == User.id)
class StoryTag(Base):
__table_args__ = (
schema.UniqueConstraint('name', name='uniq_story_tags_name'),
)
name = Column(String(20))
story_id = Column(Integer, ForeignKey('stories.id'))

0
storyboard/openstack/__init__.py

0
storyboard/openstack/common/__init__.py

16
storyboard/openstack/common/db/__init__.py

@ -0,0 +1,16 @@
# vim: tabstop=4 shiftwidth=4 softtabstop=4
# Copyright 2012 Cloudscaling Group, Inc
# All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.

106
storyboard/openstack/common/db/api.py

@ -0,0 +1,106 @@
# vim: tabstop=4 shiftwidth=4 softtabstop=4
# Copyright (c) 2013 Rackspace Hosting
# All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
"""Multiple DB API backend support.
Supported configuration options:
The following two parameters are in the 'database' group:
`backend`: DB backend name or full module path to DB backend module.
`use_tpool`: Enable thread pooling of DB API calls.
A DB backend module should implement a method named 'get_backend' which
takes no arguments. The method can return any object that implements DB
API methods.
*NOTE*: There are bugs in eventlet when using tpool combined with
threading locks. The python logging module happens to use such locks. To
work around this issue, be sure to specify thread=False with
eventlet.monkey_patch().
A bug for eventlet has been filed here:
https://bitbucket.org/eventlet/eventlet/issue/137/
"""
import functools
from oslo.config import cfg
from storyboard.openstack.common import importutils
from storyboard.openstack.common import lockutils
db_opts = [
cfg.StrOpt('backend',
default='sqlalchemy',
deprecated_name='db_backend',
deprecated_group='DEFAULT',
help='The backend to use for db'),
cfg.BoolOpt('use_tpool',
default=False,
deprecated_name='dbapi_use_tpool',
deprecated_group='DEFAULT',
help='Enable the experimental use of thread pooling for '
'all DB API calls')
]
CONF = cfg.CONF
CONF.register_opts(db_opts, 'database')
class DBAPI(object):
def __init__(self, backend_mapping=None):
if backend_mapping is None:
backend_mapping = {}
self.__backend = None
self.__backend_mapping = backend_mapping
@lockutils.synchronized('dbapi_backend', 'storyboard-')
def __get_backend(self):
"""Get the actual backend. May be a module or an instance of
a class. Doesn't matter to us. We do this synchronized as it's
possible multiple greenthreads started very quickly trying to do
DB calls and eventlet can switch threads before self.__backend gets
assigned.
"""
if self.__backend:
# Another thread assigned it
return self.__backend
backend_name = CONF.database.backend
self.__use_tpool = CONF.database.use_tpool
if self.__use_tpool:
from eventlet import tpool
self.__tpool = tpool
# Import the untranslated name if we don't have a
# mapping.
backend_path = self.__backend_mapping.get(backend_name,
backend_name)
backend_mod = importutils.import_module(backend_path)
self.__backend = backend_mod.get_backend()
return self.__backend
def __getattr__(self, key):
backend = self.__backend or self.__get_backend()
attr = getattr(backend, key)
if not self.__use_tpool or not hasattr(attr, '__call__'):
return attr
def tpool_wrapper(*args, **kwargs):
return self.__tpool.execute(attr, *args, **kwargs)
functools.update_wrapper(tpool_wrapper, attr)