From 0250f81cdcb9e49361f24b52e76e1597a0674739 Mon Sep 17 00:00:00 2001 From: Johannes Grassler Date: Fri, 29 Jun 2018 16:00:37 +0000 Subject: [PATCH] Add monasca_db command line tool This commit adds a schema management tool for the Monasca configuration database. Apart from the usual OpenStack schema management tool subcommands (stamp, upgrade, version) it has two extra subcommands: * fingerprint: for computing a SHA1 fingerprint of the currently currently active database schema. * detect-revision: for identifiying the Alembic revision (if any) corresponding to a database schema that was created with one of the legacy SQL scripts. The data provided by the detect-revision subcommand can be used for stamping the database with Alembic version metadata when transitioning an existing Monasca configuration database to Alembic based database migrations. Story: 2001654 Task: 14341 Change-Id: Ibdd877a23ab5d6d1bbf8d83515c0197554098526 --- doc/source/admin/index.rst | 58 ++++++ doc/source/contributor/index.rst | 15 ++ monasca_api/cmd/__init__.py | 0 monasca_api/cmd/monasca_db.py | 167 ++++++++++++++++++ monasca_api/config.py | 22 ++- monasca_api/db/alembic/env.py | 21 +-- .../alembic/versions/00597b5c8325_initial.py | 12 +- monasca_api/db/fingerprint.py | 66 +++++++ setup.cfg | 1 + 9 files changed, 337 insertions(+), 25 deletions(-) create mode 100644 monasca_api/cmd/__init__.py create mode 100644 monasca_api/cmd/monasca_db.py create mode 100644 monasca_api/db/fingerprint.py diff --git a/doc/source/admin/index.rst b/doc/source/admin/index.rst index 4b1b94ad4..5f6c07a15 100644 --- a/doc/source/admin/index.rst +++ b/doc/source/admin/index.rst @@ -4,3 +4,61 @@ Administration guide .. toctree:: :maxdepth: 2 + +Schema Setup +~~~~~~~~~~~~ + +For setting up the Monasca configuration database, we provide ``monasca_db``, +an Alembic based database migration tool. Historically, the schema for the +configuration database was created by a SQL script. This SQL was changed a +couple of times, so ``monasca_db`` comes with a mechanism to detect the SQL +script revision being used to create it and stamp the database with the +matching Alembic revision. + +Setting up a new database +------------------------- + +If you are deploying Monasca from scratch, database setup is quite +straightforward: + +1. Create a database and configure access credentials with ``ALL PRIVILEGES`` + permission level on it in the Monasca API configuration file's + ``[database]`` section. + +2. Run schema migrations: ``monasca_db upgrade``. It will run all migrations up + to and including the most recent one (``head``) unless a revision to migrate + to is explicitly specified. + + +Upgrading Existing Database from Legacy Schema +---------------------------------------------- + +If you have been running an older version of Monasca, you can attempt to +identify and stamp its database schema: + +:: + + monasca_db stamp --from-fingerprint + +This command will generate a unique fingerprint for the database schema in +question and match that fingerprint with an in-code map of fingerprints to +database schema revisions. This should work for all official (shipped as part +of the ``monasca-api`` repository) schema scripts. If you used a custom +third-party schema script to set up the database, it may not be listed and +you'll get an error message similar to this one (the fingerprint hash will +vary): + +:: + + Schema fingerprint 3d45493070e3b8e6fc492d2369e51423ca4cc1ac does not match any known legacy revision. + +If this happens to you, please create a Storyboard story against the +`openstack/monasca-api project `_. +Provide the following alongside the story: + +1. A copy of or pointer to the schema SQL script being used to set up the + database. + +2. The fingerprint shown in the error message. + +3. The output of ``monasca_db fingerprint --raw``. diff --git a/doc/source/contributor/index.rst b/doc/source/contributor/index.rst index 8d7a44aa3..d2916576b 100644 --- a/doc/source/contributor/index.rst +++ b/doc/source/contributor/index.rst @@ -95,6 +95,21 @@ If any of the following applies to the patch, a release note is required: A release note is suggested if a long-standing or important bug is fixed. Otherwise, a release note is not required. +Database Migrations +------------------- + +As of the Rocky release, Monasca uses `Alembic `_ +migrations to set up its configuration database. If you need to change the +configuration database's schema, you need to create a migration to adjust the +database accordingly, as follows:: + + cd monasca_api/db/ + alembic revision + +This will create a new skeleton revision for you to edit. You will find +existing revisions to use for inspiration in the +``/monasca_api/db/alembic/versions/`` directory. + Developer reference ------------------- diff --git a/monasca_api/cmd/__init__.py b/monasca_api/cmd/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/monasca_api/cmd/monasca_db.py b/monasca_api/cmd/monasca_db.py new file mode 100644 index 000000000..a20373961 --- /dev/null +++ b/monasca_api/cmd/monasca_db.py @@ -0,0 +1,167 @@ +# Copyright 2018 SUSE Linux GmbH +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +""" + CLI interface for monasca database management. +""" + +from __future__ import print_function + +from oslo_config import cfg +from oslo_db.sqlalchemy.migration_cli.ext_alembic import AlembicExtension + +from monasca_api.common.repositories.sqla import sql_repository +from monasca_api import conf +from monasca_api.db.alembic import env +from monasca_api.db.fingerprint import Fingerprint +from monasca_api import version + +import monasca_api.config + +import sys + +CONF = cfg.CONF + +_FP_NOREVISION = ("Schema fingerprint %s does not match any known legacy " + "revision.") + +migration_config = {'alembic_ini_path': env.ini_file_path} + + +def do_detect_revision(): + fp = Fingerprint(sql_repository.get_engine()) + + if fp.revision is None: + print(_FP_NOREVISION % fp.sha1) + sys.exit(1) + else: + print(fp.revision) + + +def do_fingerprint(): + fingerprint = Fingerprint(sql_repository.get_engine()) + if CONF.command.raw: + print(fingerprint.schema_raw, end="") + else: + print(fingerprint.sha1) + + +def do_stamp(): + rev = CONF.command.revision + from_fingerprint = CONF.command.from_fingerprint + + engine = sql_repository.get_engine() + alembic_ext = AlembicExtension(engine, migration_config) + + if rev is None: + if from_fingerprint is False: + print("No revision specified. Specify --from-fingerprint to " + "attempt a guess based on the current database schema's " + "fingerprint.") + sys.exit(1) + else: + fp = Fingerprint(engine) + if fp.revision is None: + print(_FP_NOREVISION % fp.sha1) + sys.exit(1) + rev = fp.revision + + alembic_ext.stamp(rev) + + +def do_upgrade(): + engine = sql_repository.get_engine() + alembic_ext = AlembicExtension(engine, migration_config) + + rev = CONF.command.revision + db_rev = alembic_ext.version() + + fp = Fingerprint(engine) + + if fp.schema_raw != "" and db_rev is None: + print("Non-empty database schema without Alembic version metadata " + "detected. Please use the `stamp` subcommand to add version " + "metadata.") + sys.exit(1) + + alembic_ext.upgrade(rev) + + +def do_version(): + engine = sql_repository.get_engine() + alembic_ext = AlembicExtension(engine, migration_config) + + version = alembic_ext.version() + if version is None: + print("Cannot determine version. Check if this database has Alembic " + "version information. ") + sys.exit(1) + print(version) + + +def add_command_parsers(subparsers): + parser = subparsers.add_parser('fingerprint', + help="Compute SHA1 fingerprint of " + "current database schema ") + parser.add_argument('-r', '--raw', action='store_true', + help='Print raw schema dump used for ' + 'fingerprinting') + parser.set_defaults(func=do_fingerprint) + + parser = subparsers.add_parser('detect-revision', + help="Attempt to detect revision " + "matching current database " + " schema ") + parser.set_defaults(func=do_detect_revision) + + parser = subparsers.add_parser('stamp', help='Stamp database with an ' + 'Alembic revision') + parser.add_argument('revision', nargs='?', metavar='VERSION', + help='Revision to stamp database with', + default=None) + parser.add_argument('-f', '--from-fingerprint', action='store_true', + help='Try to determine VERSION from fingerprint') + parser.set_defaults(func=do_stamp) + + parser = subparsers.add_parser('upgrade', + help='Upgrade database to given or ' + 'latest revision') + parser.add_argument('revision', metavar='VERSION', nargs='?', + help='Alembic revision to upgrade database to', + default='head') + parser.add_argument('-f', '--from-fingerprint', action='store_true', + help='Try to determine VERSION from fingerprint') + parser.set_defaults(func=do_upgrade) + + parser = subparsers.add_parser('version', help="Show database's current Alembic version") + parser.set_defaults(func=do_version) + + +command_opt = cfg.SubCommandOpt('command', + title='Monasca DB manager', + help='Available commands', + handler=add_command_parsers) + + +def main(): + CONF.register_cli_opt(command_opt) + CONF(args=sys.argv[1:], + default_config_files=monasca_api.config.get_config_file(None), + prog='api', + project='monasca', + version=version.version_str) + + conf.register_opts() + + CONF.command.func() diff --git a/monasca_api/config.py b/monasca_api/config.py index f3833c3af..37910e6af 100644 --- a/monasca_api/config.py +++ b/monasca_api/config.py @@ -45,14 +45,12 @@ def parse_args(argv=None, config_file=None): argv = (argv if argv is not None else sys.argv[1:]) args = ([] if _is_running_under_gunicorn() else argv or []) - config_file = (_get_deprecated_config_file() - if config_file is None else config_file) CONF(args=args, prog='api', project='monasca', version=version.version_str, - default_config_files=[config_file] if config_file else None, + default_config_files=get_config_file(config_file), description='RESTful API for alarming in the cloud') log.setup(CONF, @@ -64,6 +62,21 @@ def parse_args(argv=None, config_file=None): _CONF_LOADED = True +def get_config_file(config_file): + """Get config file in a format suitable for CONF constructor + + Returns the config file name as a single element array. If a config file + was explicitly, specified, that file's name is returned. If there isn't and a + legacy config file is present that one is returned. Otherwise we return + None. This is what the CONF constructor expects for its + default_config_files keyword argument. + """ + if config_file is not None: + return [config_file] + + return _get_deprecated_config_file() + + def _is_running_under_gunicorn(): """Evaluates if api runs under gunicorn.""" content = filter(lambda x: x != sys.executable and _GUNICORN_MARKER in x, @@ -87,4 +100,5 @@ def _get_deprecated_config_file(): if old_files is not None and len(old_files) > 0: LOG.warning('Detected old location "/etc/monasca/api-config.conf" ' 'of main configuration file') - return old_files[0] + return [old_files[0]] + return None diff --git a/monasca_api/db/alembic/env.py b/monasca_api/db/alembic/env.py index b9a276f21..830ed8341 100644 --- a/monasca_api/db/alembic/env.py +++ b/monasca_api/db/alembic/env.py @@ -14,8 +14,8 @@ from __future__ import with_statement -import monasca_api.config import os +import sys from alembic import config as alembic_config from alembic import context @@ -23,20 +23,22 @@ from logging.config import fileConfig from monasca_api.common.repositories.sqla import models from monasca_api.common.repositories.sqla import sql_repository +import monasca_api.config ini_file_path = os.path.join(os.path.dirname(__file__), '..', 'alembic.ini') # This indicates whether we are running with a viable Alembic -# context (necessary to do skip run_migrations_online() below +# context (necessary to skip run_migrations_online() below # if sphinx imports this file without a viable Alembic # context) have_context = True try: config = context.config - # FIXME: Move this to the monasca_db entry point later. - # Load monasca-api config (from files only) - monasca_api.config.parse_args(argv=[]) + # Only load Monasca configuration if imported by alembic CLI tool (the + # monasca_db command will handle this on its own). + if os.path.basename(sys.argv[0]) == 'alembic': + monasca_api.config.parse_args(argv=[]) except AttributeError: config = alembic_config.Config(ini_file_path) have_context = False @@ -76,14 +78,5 @@ def run_migrations_online(): with context.begin_transaction(): context.run_migrations() - -def fingerprint_db(): - return - - -def stamp_db(): - return - - if have_context: run_migrations_online() diff --git a/monasca_api/db/alembic/versions/00597b5c8325_initial.py b/monasca_api/db/alembic/versions/00597b5c8325_initial.py index 1ce74d315..fb5ac170d 100644 --- a/monasca_api/db/alembic/versions/00597b5c8325_initial.py +++ b/monasca_api/db/alembic/versions/00597b5c8325_initial.py @@ -401,26 +401,25 @@ def upgrade(): op.create_table( 'sub_alarm', sa.Column('id', - sa.dialects.mysql.VARCHAR(length=36, charset='utf8mb4', + sa.dialects.mysql.VARCHAR(length=36, collation='utf8mb4_unicode_ci'), nullable=False), sa.Column('alarm_id', - sa.dialects.mysql.VARCHAR(length=36, charset='utf8mb4', + sa.dialects.mysql.VARCHAR(length=36, collation='utf8mb4_unicode_ci'), sa.ForeignKey('alarm.id', ondelete='CASCADE', name='fk_sub_alarm'), nullable=False, server_default=''), sa.Column('sub_expression_id', - sa.dialects.mysql.VARCHAR(length=36, charset='utf8mb4', + sa.dialects.mysql.VARCHAR(length=36, collation='utf8mb4_unicode_ci'), sa.ForeignKey('sub_alarm_definition.id', name='fk_sub_alarm_expr'), nullable=False, server_default=''), sa.Column('expression', - sa.dialects.mysql.LONGTEXT(charset='utf8mb4', - collation='utf8mb4_unicode_ci'), + sa.dialects.mysql.LONGTEXT(collation='utf8mb4_unicode_ci'), nullable=False), sa.Column('created_at', sa.DateTime(), @@ -428,8 +427,7 @@ def upgrade(): sa.Column('updated_at', sa.DateTime(), nullable=False), - sa.PrimaryKeyConstraint('id'), - mysql_charset='latin1') + sa.PrimaryKeyConstraint('id')) op.create_table( 'schema_migrations', diff --git a/monasca_api/db/fingerprint.py b/monasca_api/db/fingerprint.py new file mode 100644 index 000000000..b566c69a3 --- /dev/null +++ b/monasca_api/db/fingerprint.py @@ -0,0 +1,66 @@ +# Copyright 2018 SUSE Linux GmbH +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +import hashlib + +from sqlalchemy import MetaData + +# Map of SHA1 fingerprints to alembic revisions. +_REVS = {"43e5913b0272077321ab6f25ffbcda7149b6284b": "00597b5c8325", + "c4e5c870c705421faa4041405b5a895970faa434": "0cce983d957a", + "f7a79c4eea9c9d130277a64eb6d2d16587088dbb": "30181b42434b", + "529f266f7ed42929d5405616810546e4615153e8": "6b2b88f3cab4", + "857904f960af77c0554c4c38d73ed47df7c949b4": "8781a256f0c1", + "773489fb7bfa84bf2db0e1ff1ab96bce7fb4ecd7": "c2f85438d6f3", + "f29f18a30519a1bae9dcee85a604eb72886e34d3": "d8b801498850", + "dd47cb01f11cb5cd7fec6bda6a190bc10b4659a6": "f69cb3152a76", + + # Database created with UTF8 default charset + "5dda7af1fd708095e6c9298976abb1242bbd1848": "8781a256f0c1", + "7fb1ce4a60f0065505096843bfd21f4ef4c5d1e0": "f69cb3152a76"} + + +class Fingerprint(object): + + def __init__(self, engine): + metadata = MetaData(bind=engine, reflect=True) + + schema_strings = [] + + for table in metadata.sorted_tables: + # Omit this table to maintain a consistent fingerprint when + # fingerprint a migrated schema is fingerprinted. + if table.name == "alembic_version": + continue + table.metadata = None + columns = [] + for column in table.columns: + column.server_default = None + columns.append(repr(column)) + table.columns = [] + schema_strings.append(repr(table)) + + for column in columns: + schema_strings.append(" " + repr(column)) + + schema_strings.append("") + + self.schema_raw = "\n".join(schema_strings) + self.sha1 = hashlib.sha1(self.schema_raw).hexdigest() + + try: + self.revision = _REVS[self.sha1] + except KeyError: + # Fingerprint does not match any revisions + self.revision = None diff --git a/setup.cfg b/setup.cfg index ea6a15754..1159b9fb3 100644 --- a/setup.cfg +++ b/setup.cfg @@ -35,6 +35,7 @@ cassandra = [entry_points] console_scripts = monasca-api = monasca_api.api.server:launch + monasca_db = monasca_api.cmd.monasca_db:main wsgi_scripts = monasca-api-wsgi = monasca_api.api.wsgi:main