From 31f8ad4eb4279d356c67e9f14531ce2ca6c70ea5 Mon Sep 17 00:00:00 2001 From: Stephen Finucane Date: Fri, 24 Sep 2021 17:27:49 +0100 Subject: [PATCH] db: Enable auto-generation of database migrations Alembic does lots of new things. Provide docs for how to use this. This doesn't actually work that well at the moment because it appears our database migrations are not in sync with the models. Those issues will have to be resolved separately. Change-Id: I6645ca951114b94ce9ed5a83e5e11d53879a7cd5 Signed-off-by: Stephen Finucane --- .gitignore | 6 +- cinder/db/alembic.ini | 4 +- cinder/db/migrations/env.py | 19 ++- cinder/db/sqlalchemy/models.py | 3 +- doc/source/admin/upgrades.rst | 4 +- .../contributor/database-migrations.rst | 135 ++++++++++++++++++ doc/source/contributor/index.rst | 1 + 7 files changed, 158 insertions(+), 14 deletions(-) create mode 100644 doc/source/contributor/database-migrations.rst diff --git a/.gitignore b/.gitignore index b0a6235af1a..aff35a16a55 100644 --- a/.gitignore +++ b/.gitignore @@ -31,13 +31,14 @@ mypy-report tools/lintstack.head.py tools/pylint_exceptions tags + # Files created by Sphinx build doc/build doc/source/_static/cinder.conf.sample doc/source/_static/cinder.policy.yaml.sample doc/source/drivers.rst -#Files created for API reference +# Files created for API reference api-ref/build # Files created by releasenotes build @@ -45,3 +46,6 @@ releasenotes/build contrib/block-box/.db_data RELEASENOTES.rst releasenotes/notes/reno.cache + +# Files created by alembic +cinder.db diff --git a/cinder/db/alembic.ini b/cinder/db/alembic.ini index ef98e0e62e2..5461418ebdd 100644 --- a/cinder/db/alembic.ini +++ b/cinder/db/alembic.ini @@ -35,9 +35,7 @@ script_location = %(here)s/migrations # are written from script.py.mako # output_encoding = utf-8 -# Uncomment and update to your sql connection string if wishing to run -# alembic directly from command line -# sqlalchemy.url = driver://user:pass@localhost/dbname +sqlalchemy.url = sqlite:///cinder.db [post_write_hooks] # post_write_hooks defines scripts or Python functions that are run diff --git a/cinder/db/migrations/env.py b/cinder/db/migrations/env.py index 3a139ad17c8..ed44dc9232f 100644 --- a/cinder/db/migrations/env.py +++ b/cinder/db/migrations/env.py @@ -16,6 +16,8 @@ from alembic import context from sqlalchemy import engine_from_config from sqlalchemy import pool +from cinder.db.sqlalchemy import models + # this is the Alembic Config object, which provides # access to the values within the .ini file in use. config = context.config @@ -25,11 +27,13 @@ config = context.config if config.attributes.get('configure_logger', True): fileConfig(config.config_file_name) -# add your model's MetaData object here -# for 'autogenerate' support -# from myapp import mymodel -# target_metadata = mymodel.Base.metadata -target_metadata = None +target_metadata = models.BASE.metadata + + +def include_name(name, type_, parent_names): + # if there are any columns or tables that should be excluded from + # auto-generation, include them here + return True def run_migrations_offline(): @@ -45,6 +49,7 @@ def run_migrations_offline(): context.configure( url=url, target_metadata=target_metadata, + include_name=include_name, literal_binds=True, dialect_opts={"paramstyle": "named"}, ) @@ -80,7 +85,9 @@ def run_migrations_online(): with connectable.connect() as connection: context.configure( - connection=connection, target_metadata=target_metadata + connection=connection, + target_metadata=target_metadata, + include_name=include_name, ) with context.begin_transaction(): diff --git a/cinder/db/sqlalchemy/models.py b/cinder/db/sqlalchemy/models.py index 031a65d5390..1dcc82efbfc 100644 --- a/cinder/db/sqlalchemy/models.py +++ b/cinder/db/sqlalchemy/models.py @@ -37,8 +37,7 @@ CONF = cfg.CONF BASE = declarative_base() -class CinderBase(models.TimestampMixin, - models.ModelBase): +class CinderBase(models.TimestampMixin, models.ModelBase): """Base class for Cinder Models.""" __table_args__ = {'mysql_engine': 'InnoDB'} diff --git a/doc/source/admin/upgrades.rst b/doc/source/admin/upgrades.rst index 8293d93dd76..cb1ec3fb1e3 100644 --- a/doc/source/admin/upgrades.rst +++ b/doc/source/admin/upgrades.rst @@ -72,8 +72,8 @@ the background until it tells you no more migrations are needed. Note that you won't be able to apply N+1's schema migrations before completing N's online data migrations. -For information on developing your own schema migrations as part of a feature -or bugfix, refer to **TODO**. +For information on developing your own schema or data migrations as part of a +feature or bugfix, refer to :doc:`/contributor/database-migrations`. API load balancer draining ~~~~~~~~~~~~~~~~~~~~~~~~~~ diff --git a/doc/source/contributor/database-migrations.rst b/doc/source/contributor/database-migrations.rst new file mode 100644 index 00000000000..f9332295bd1 --- /dev/null +++ b/doc/source/contributor/database-migrations.rst @@ -0,0 +1,135 @@ +=================== +Database migrations +=================== + +.. note:: + + This document details how to generate database migrations as part of a new + feature or bugfix. For info on how to apply existing database migrations, + refer to the documentation for the :program:`cinder-manage db sync` + command in :doc:`/cli/cinder-manage`. + For info on the general upgrade process for a cinder deployment, refer to + :doc:`/admin/upgrades`. + +Occasionally the databases used in cinder will require schema or data +migrations. + + +Schema migrations +----------------- + +.. versionchanged:: 24.0.0 (Xena) + + The database migration engine was changed from ``sqlalchemy-migrate`` to + ``alembic``. + +The `alembic`__ database migration tool is used to manage schema migrations in +cinder. The migration files and related metadata can be found in +``cinder/db/migrations``. As discussed in :doc:`/admin/upgrades`, these can be +run by end users using the :program:`cinder-manage db sync` command. + +.. __: https://alembic.sqlalchemy.org/en/latest/ + +.. note:: + + There are also legacy migrations provided in the + ``cinder/db/legacy_migrations`` directory . These are provided to facilitate + upgrades from pre-Xena (24.0.0) deployments and will be removed in a future + release. They should not be modified or extended. + +The best reference for alembic is the `alembic documentation`__, but a small +example is provided here. You can create the migration either manually or +automatically. Manual generation might be necessary for some corner cases such +as renamed tables but auto-generation will typically handle your issues. +Examples of both are provided below. In both examples, we're going to +demonstrate how you could add a new model, ``Foo``, to the main database. + +.. __: https://alembic.sqlalchemy.org/en/latest/ + +.. code-block:: diff + + diff --git cinder/db/sqlalchemy/models.py cinder/db/sqlalchemy/models.py + index 7eab643e14..8f70bcdaca 100644 + --- cinder/db/sqlalchemy/models.py + +++ cinder/db/sqlalchemy/models.py + @@ -73,6 +73,16 @@ def MediumText(): + sqlalchemy.dialects.mysql.MEDIUMTEXT(), 'mysql') + + + +class Foo(BASE, models.SoftDeleteMixin): + + """A test-only model.""" + + + + __tablename__ = 'foo' + + + + id = sa.Column(sa.Integer, primary_key=True) + + uuid = sa.Column(sa.String(36), nullable=True) + + bar = sa.Column(sa.String(255)) + + + + + class Service(BASE, models.SoftDeleteMixin): + """Represents a running service on a host.""" + +(you might not be able to apply the diff above cleanly - this is just a demo). + +.. rubric:: Auto-generating migration scripts + +In order for alembic to compare the migrations with the underlying models, it +require a database that it can inspect and compare the models against. As such, +we first need to create a working database. We'll bypass ``cinder-manage`` for +this and go straight to the :program:`alembic` CLI. The ``alembic.ini`` file +provided in the ``cinder/db`` directory is helpfully configured to use an +SQLite database by default (``cinder.db``). Create this database and apply the +current schema, as dictated by the current migration scripts: + +.. code-block:: bash + + $ tox -e venv -- alembic -c cinder/db/alembic.ini \ + upgrade head + +Once done, you should notice the new ``cinder.db`` file in the root of the +repo. Now, let's generate the new revision: + +.. code-block:: bash + + $ tox -e venv -- alembic -c cinder/db/alembic.ini \ + revision -m "Add foo model" --autogenerate + +This will create a new file in ``cinder/db/migrations/versions`` with +``add_foo_model`` in the name including (hopefully!) the necessary changes to +add the new ``Foo`` model. You **must** inspect this file once created, since +there's a chance you'll be missing imports or something else which will need to +be manually corrected. Once you've inspected this file and made any required +changes, you can apply the migration and make sure it works: + +.. code-block:: bash + + $ tox -e venv -- alembic -c cinder/db/alembic.ini \ + upgrade head + +.. rubric:: Manually generating migration scripts + +For trickier migrations or things that alembic doesn't understand, you may need +to manually create a migration script. This is very similar to the +auto-generation step, with the exception being that you don't need to have a +database in place beforehand. As such, you can simply run: + +.. code-block:: bash + + $ tox -e venv -- alembic -c cinder/db/alembic.ini \ + revision -m "Add foo model" + +As before, this will create a new file in ``cinder/db/migrations/versions`` +with ``add_foo_model`` in the name. You can simply modify this to make whatever +changes are necessary. Once done, you can apply the migration and make sure it +works: + +.. code-block:: bash + + $ tox -e venv -- alembic -c cinder/db/alembic.ini \ + upgrade head + + +Data migrations +--------------- + +.. todo: Populate this. diff --git a/doc/source/contributor/index.rst b/doc/source/contributor/index.rst index 456aae4e4fb..37a1957650d 100644 --- a/doc/source/contributor/index.rst +++ b/doc/source/contributor/index.rst @@ -62,6 +62,7 @@ Programming HowTos and Tutorials api.apache rolling.upgrades groups + database-migrations .. _managing-development: