diff --git a/.gitignore b/.gitignore index b0a6235af1a..aff35a16a55 100644 --- a/.gitignore +++ b/.gitignore @@ -31,13 +31,14 @@ mypy-report tools/lintstack.head.py tools/pylint_exceptions tags + # Files created by Sphinx build doc/build doc/source/_static/cinder.conf.sample doc/source/_static/cinder.policy.yaml.sample doc/source/drivers.rst -#Files created for API reference +# Files created for API reference api-ref/build # Files created by releasenotes build @@ -45,3 +46,6 @@ releasenotes/build contrib/block-box/.db_data RELEASENOTES.rst releasenotes/notes/reno.cache + +# Files created by alembic +cinder.db diff --git a/cinder/db/alembic.ini b/cinder/db/alembic.ini index ef98e0e62e2..5461418ebdd 100644 --- a/cinder/db/alembic.ini +++ b/cinder/db/alembic.ini @@ -35,9 +35,7 @@ script_location = %(here)s/migrations # are written from script.py.mako # output_encoding = utf-8 -# Uncomment and update to your sql connection string if wishing to run -# alembic directly from command line -# sqlalchemy.url = driver://user:pass@localhost/dbname +sqlalchemy.url = sqlite:///cinder.db [post_write_hooks] # post_write_hooks defines scripts or Python functions that are run diff --git a/cinder/db/migrations/env.py b/cinder/db/migrations/env.py index 3a139ad17c8..ed44dc9232f 100644 --- a/cinder/db/migrations/env.py +++ b/cinder/db/migrations/env.py @@ -16,6 +16,8 @@ from alembic import context from sqlalchemy import engine_from_config from sqlalchemy import pool +from cinder.db.sqlalchemy import models + # this is the Alembic Config object, which provides # access to the values within the .ini file in use. config = context.config @@ -25,11 +27,13 @@ config = context.config if config.attributes.get('configure_logger', True): fileConfig(config.config_file_name) -# add your model's MetaData object here -# for 'autogenerate' support -# from myapp import mymodel -# target_metadata = mymodel.Base.metadata -target_metadata = None +target_metadata = models.BASE.metadata + + +def include_name(name, type_, parent_names): + # if there are any columns or tables that should be excluded from + # auto-generation, include them here + return True def run_migrations_offline(): @@ -45,6 +49,7 @@ def run_migrations_offline(): context.configure( url=url, target_metadata=target_metadata, + include_name=include_name, literal_binds=True, dialect_opts={"paramstyle": "named"}, ) @@ -80,7 +85,9 @@ def run_migrations_online(): with connectable.connect() as connection: context.configure( - connection=connection, target_metadata=target_metadata + connection=connection, + target_metadata=target_metadata, + include_name=include_name, ) with context.begin_transaction(): diff --git a/cinder/db/sqlalchemy/models.py b/cinder/db/sqlalchemy/models.py index 031a65d5390..1dcc82efbfc 100644 --- a/cinder/db/sqlalchemy/models.py +++ b/cinder/db/sqlalchemy/models.py @@ -37,8 +37,7 @@ CONF = cfg.CONF BASE = declarative_base() -class CinderBase(models.TimestampMixin, - models.ModelBase): +class CinderBase(models.TimestampMixin, models.ModelBase): """Base class for Cinder Models.""" __table_args__ = {'mysql_engine': 'InnoDB'} diff --git a/doc/source/admin/upgrades.rst b/doc/source/admin/upgrades.rst index 8293d93dd76..cb1ec3fb1e3 100644 --- a/doc/source/admin/upgrades.rst +++ b/doc/source/admin/upgrades.rst @@ -72,8 +72,8 @@ the background until it tells you no more migrations are needed. Note that you won't be able to apply N+1's schema migrations before completing N's online data migrations. -For information on developing your own schema migrations as part of a feature -or bugfix, refer to **TODO**. +For information on developing your own schema or data migrations as part of a +feature or bugfix, refer to :doc:`/contributor/database-migrations`. API load balancer draining ~~~~~~~~~~~~~~~~~~~~~~~~~~ diff --git a/doc/source/contributor/database-migrations.rst b/doc/source/contributor/database-migrations.rst new file mode 100644 index 00000000000..f9332295bd1 --- /dev/null +++ b/doc/source/contributor/database-migrations.rst @@ -0,0 +1,135 @@ +=================== +Database migrations +=================== + +.. note:: + + This document details how to generate database migrations as part of a new + feature or bugfix. For info on how to apply existing database migrations, + refer to the documentation for the :program:`cinder-manage db sync` + command in :doc:`/cli/cinder-manage`. + For info on the general upgrade process for a cinder deployment, refer to + :doc:`/admin/upgrades`. + +Occasionally the databases used in cinder will require schema or data +migrations. + + +Schema migrations +----------------- + +.. versionchanged:: 24.0.0 (Xena) + + The database migration engine was changed from ``sqlalchemy-migrate`` to + ``alembic``. + +The `alembic`__ database migration tool is used to manage schema migrations in +cinder. The migration files and related metadata can be found in +``cinder/db/migrations``. As discussed in :doc:`/admin/upgrades`, these can be +run by end users using the :program:`cinder-manage db sync` command. + +.. __: https://alembic.sqlalchemy.org/en/latest/ + +.. note:: + + There are also legacy migrations provided in the + ``cinder/db/legacy_migrations`` directory . These are provided to facilitate + upgrades from pre-Xena (24.0.0) deployments and will be removed in a future + release. They should not be modified or extended. + +The best reference for alembic is the `alembic documentation`__, but a small +example is provided here. You can create the migration either manually or +automatically. Manual generation might be necessary for some corner cases such +as renamed tables but auto-generation will typically handle your issues. +Examples of both are provided below. In both examples, we're going to +demonstrate how you could add a new model, ``Foo``, to the main database. + +.. __: https://alembic.sqlalchemy.org/en/latest/ + +.. code-block:: diff + + diff --git cinder/db/sqlalchemy/models.py cinder/db/sqlalchemy/models.py + index 7eab643e14..8f70bcdaca 100644 + --- cinder/db/sqlalchemy/models.py + +++ cinder/db/sqlalchemy/models.py + @@ -73,6 +73,16 @@ def MediumText(): + sqlalchemy.dialects.mysql.MEDIUMTEXT(), 'mysql') + + + +class Foo(BASE, models.SoftDeleteMixin): + + """A test-only model.""" + + + + __tablename__ = 'foo' + + + + id = sa.Column(sa.Integer, primary_key=True) + + uuid = sa.Column(sa.String(36), nullable=True) + + bar = sa.Column(sa.String(255)) + + + + + class Service(BASE, models.SoftDeleteMixin): + """Represents a running service on a host.""" + +(you might not be able to apply the diff above cleanly - this is just a demo). + +.. rubric:: Auto-generating migration scripts + +In order for alembic to compare the migrations with the underlying models, it +require a database that it can inspect and compare the models against. As such, +we first need to create a working database. We'll bypass ``cinder-manage`` for +this and go straight to the :program:`alembic` CLI. The ``alembic.ini`` file +provided in the ``cinder/db`` directory is helpfully configured to use an +SQLite database by default (``cinder.db``). Create this database and apply the +current schema, as dictated by the current migration scripts: + +.. code-block:: bash + + $ tox -e venv -- alembic -c cinder/db/alembic.ini \ + upgrade head + +Once done, you should notice the new ``cinder.db`` file in the root of the +repo. Now, let's generate the new revision: + +.. code-block:: bash + + $ tox -e venv -- alembic -c cinder/db/alembic.ini \ + revision -m "Add foo model" --autogenerate + +This will create a new file in ``cinder/db/migrations/versions`` with +``add_foo_model`` in the name including (hopefully!) the necessary changes to +add the new ``Foo`` model. You **must** inspect this file once created, since +there's a chance you'll be missing imports or something else which will need to +be manually corrected. Once you've inspected this file and made any required +changes, you can apply the migration and make sure it works: + +.. code-block:: bash + + $ tox -e venv -- alembic -c cinder/db/alembic.ini \ + upgrade head + +.. rubric:: Manually generating migration scripts + +For trickier migrations or things that alembic doesn't understand, you may need +to manually create a migration script. This is very similar to the +auto-generation step, with the exception being that you don't need to have a +database in place beforehand. As such, you can simply run: + +.. code-block:: bash + + $ tox -e venv -- alembic -c cinder/db/alembic.ini \ + revision -m "Add foo model" + +As before, this will create a new file in ``cinder/db/migrations/versions`` +with ``add_foo_model`` in the name. You can simply modify this to make whatever +changes are necessary. Once done, you can apply the migration and make sure it +works: + +.. code-block:: bash + + $ tox -e venv -- alembic -c cinder/db/alembic.ini \ + upgrade head + + +Data migrations +--------------- + +.. todo: Populate this. diff --git a/doc/source/contributor/index.rst b/doc/source/contributor/index.rst index 456aae4e4fb..37a1957650d 100644 --- a/doc/source/contributor/index.rst +++ b/doc/source/contributor/index.rst @@ -62,6 +62,7 @@ Programming HowTos and Tutorials api.apache rolling.upgrades groups + database-migrations .. _managing-development: