diff --git a/lower-constraints.txt b/lower-constraints.txt index 5ccbfa77387b..b57532ced6bc 100644 --- a/lower-constraints.txt +++ b/lower-constraints.txt @@ -1,4 +1,4 @@ -alembic==0.9.8 +alembic==1.5.0 amqp==2.5.0 appdirs==1.4.3 asn1crypto==0.24.0 @@ -137,7 +137,7 @@ simplejson==3.13.2 six==1.15.0 smmap2==2.0.3 sortedcontainers==2.1.0 -SQLAlchemy==1.2.19 +SQLAlchemy==1.4.13 sqlalchemy-migrate==0.13.0 sqlparse==0.2.4 statsd==3.2.2 diff --git a/requirements.txt b/requirements.txt index b4eb37f5d653..a8bed744fba2 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,9 +1,5 @@ -# The order of packages is significant, because pip processes them in the order -# of appearance. Changing the order has an impact on the overall integration -# process, which may cause wedges in the gate later. - pbr>=5.5.1 # Apache-2.0 -SQLAlchemy>=1.2.19 # MIT +SQLAlchemy>=1.4.13 # MIT decorator>=4.1.0 # BSD eventlet>=0.30.1 # MIT Jinja2>=2.10 # BSD License (3 clause) @@ -19,6 +15,7 @@ PasteDeploy>=1.5.0 # MIT Paste>=2.0.2 # MIT PrettyTable>=0.7.1 # BSD sqlalchemy-migrate>=0.13.0 # Apache-2.0 +alembic>=1.5.0 # MIT netaddr>=0.7.18 # BSD netifaces>=0.10.4 # MIT paramiko>=2.7.1 # LGPLv2.1+ diff --git a/tools/db/schema_diff.py b/tools/db/schema_diff.py deleted file mode 100755 index f6a4d5e6d725..000000000000 --- a/tools/db/schema_diff.py +++ /dev/null @@ -1,283 +0,0 @@ -#!/usr/bin/env python -# Copyright 2012 OpenStack Foundation -# -# Licensed under the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. You may obtain -# a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations -# under the License. - -""" -Utility for diff'ing two versions of the DB schema. - -Each release cycle the plan is to compact all of the migrations from that -release into a single file. This is a manual and, unfortunately, error-prone -process. To ensure that the schema doesn't change, this tool can be used to -diff the compacted DB schema to the original, uncompacted form. - -The database is specified by providing a SQLAlchemy connection URL WITHOUT the -database-name portion (that will be filled in automatically with a temporary -database name). - -The schema versions are specified by providing a git ref (a branch name or -commit hash) and a SQLAlchemy-Migrate version number: - -Run like: - - MYSQL: - - ./tools/db/schema_diff.py mysql+pymysql://root@localhost \ - master:latest my_branch:82 - - POSTGRESQL: - - ./tools/db/schema_diff.py postgresql://localhost \ - master:latest my_branch:82 - -""" - -import datetime -import glob -import os -import subprocess -import sys - -from nova.i18n import _ - - -# Dump - - -def dump_db(db_driver, db_name, db_url, migration_version, dump_filename): - if not db_url.endswith('/'): - db_url += '/' - - db_url += db_name - - db_driver.create(db_name) - try: - _migrate(db_url, migration_version) - db_driver.dump(db_name, dump_filename) - finally: - db_driver.drop(db_name) - - -# Diff - - -def diff_files(filename1, filename2): - pipeline = ['diff -U 3 %(filename1)s %(filename2)s' - % {'filename1': filename1, 'filename2': filename2}] - - # Use colordiff if available - if subprocess.call(['which', 'colordiff']) == 0: - pipeline.append('colordiff') - - pipeline.append('less -R') - - cmd = ' | '.join(pipeline) - subprocess.check_call(cmd, shell=True) - - -# Database - - -class Mysql(object): - def create(self, name): - subprocess.check_call(['mysqladmin', '-u', 'root', 'create', name]) - - def drop(self, name): - subprocess.check_call(['mysqladmin', '-f', '-u', 'root', 'drop', name]) - - def dump(self, name, dump_filename): - subprocess.check_call( - 'mysqldump -u root %(name)s > %(dump_filename)s' - % {'name': name, 'dump_filename': dump_filename}, - shell=True) - - -class Postgresql(object): - def create(self, name): - subprocess.check_call(['createdb', name]) - - def drop(self, name): - subprocess.check_call(['dropdb', name]) - - def dump(self, name, dump_filename): - subprocess.check_call( - 'pg_dump %(name)s > %(dump_filename)s' - % {'name': name, 'dump_filename': dump_filename}, - shell=True) - - -def _get_db_driver_class(db_url): - try: - return globals()[db_url.split('://')[0].capitalize()] - except KeyError: - raise Exception(_("database %s not supported") % db_url) - - -# Migrate - - -MIGRATE_REPO = os.path.join(os.getcwd(), "nova/db/main/legacy_migrations") - - -def _migrate(db_url, migration_version): - earliest_version = _migrate_get_earliest_version() - - # NOTE(sirp): sqlalchemy-migrate currently cannot handle the skipping of - # migration numbers. - _migrate_cmd( - db_url, 'version_control', str(earliest_version - 1)) - - upgrade_cmd = ['upgrade'] - if migration_version != 'latest': - upgrade_cmd.append(str(migration_version)) - - _migrate_cmd(db_url, *upgrade_cmd) - - -def _migrate_cmd(db_url, *cmd): - manage_py = os.path.join(MIGRATE_REPO, 'manage.py') - - args = ['python', manage_py] - args += cmd - args += ['--repository=%s' % MIGRATE_REPO, - '--url=%s' % db_url] - - subprocess.check_call(args) - - -def _migrate_get_earliest_version(): - versions_glob = os.path.join(MIGRATE_REPO, 'versions', '???_*.py') - - versions = [] - for path in glob.iglob(versions_glob): - filename = os.path.basename(path) - prefix = filename.split('_', 1)[0] - try: - version = int(prefix) - except ValueError: - pass - versions.append(version) - - versions.sort() - return versions[0] - - -# Git - - -def git_current_branch_name(): - ref_name = git_symbolic_ref('HEAD', quiet=True) - current_branch_name = ref_name.replace('refs/heads/', '') - return current_branch_name - - -def git_symbolic_ref(ref, quiet=False): - args = ['git', 'symbolic-ref', ref] - if quiet: - args.append('-q') - proc = subprocess.Popen(args, stdout=subprocess.PIPE) - stdout, stderr = proc.communicate() - return stdout.strip() - - -def git_checkout(branch_name): - subprocess.check_call(['git', 'checkout', branch_name]) - - -def git_has_uncommited_changes(): - return subprocess.call(['git', 'diff', '--quiet', '--exit-code']) == 1 - - -# Command - - -def die(msg): - print("ERROR: %s" % msg, file=sys.stderr) - sys.exit(1) - - -def usage(msg=None): - if msg: - print("ERROR: %s" % msg, file=sys.stderr) - - prog = "schema_diff.py" - args = ["", "", - ""] - - print("usage: %s %s" % (prog, ' '.join(args)), file=sys.stderr) - sys.exit(1) - - -def parse_options(): - try: - db_url = sys.argv[1] - except IndexError: - usage("must specify DB connection url") - - try: - orig_branch, orig_version = sys.argv[2].split(':') - except IndexError: - usage('original branch and version required (e.g. master:82)') - - try: - new_branch, new_version = sys.argv[3].split(':') - except IndexError: - usage('new branch and version required (e.g. master:82)') - - return db_url, orig_branch, orig_version, new_branch, new_version - - -def main(): - timestamp = datetime.datetime.utcnow().strftime("%Y%m%d_%H%M%S") - - ORIG_DB = 'orig_db_%s' % timestamp - NEW_DB = 'new_db_%s' % timestamp - - ORIG_DUMP = ORIG_DB + ".dump" - NEW_DUMP = NEW_DB + ".dump" - - options = parse_options() - db_url, orig_branch, orig_version, new_branch, new_version = options - - # Since we're going to be switching branches, ensure user doesn't have any - # uncommitted changes - if git_has_uncommited_changes(): - die("You have uncommitted changes. Please commit them before running " - "this command.") - - db_driver = _get_db_driver_class(db_url)() - - users_branch = git_current_branch_name() - git_checkout(orig_branch) - - try: - # Dump Original Schema - dump_db(db_driver, ORIG_DB, db_url, orig_version, ORIG_DUMP) - - # Dump New Schema - git_checkout(new_branch) - dump_db(db_driver, NEW_DB, db_url, new_version, NEW_DUMP) - - diff_files(ORIG_DUMP, NEW_DUMP) - finally: - git_checkout(users_branch) - - if os.path.exists(ORIG_DUMP): - os.unlink(ORIG_DUMP) - - if os.path.exists(NEW_DUMP): - os.unlink(NEW_DUMP) - - -if __name__ == "__main__": - main() diff --git a/tools/generate-schemas b/tools/generate-schemas deleted file mode 100755 index 4df396bf8c40..000000000000 --- a/tools/generate-schemas +++ /dev/null @@ -1,123 +0,0 @@ -#!/usr/bin/env bash -# -# Script to generate schemas for the various versions. -# -# Some setup is required, similar to the opportunistic tests. -# -# MySQL -> -# -# $ mysql -uroot -# MariaDB [(none)]> CREATE DATABASE nova -# MariaDB [(none)]> GRANT ALL PRIVILEGES ON nova.* TO 'nova'@'localhost' IDENTIFIED BY 'password'; -# MariaDB [(none)]> quit; -# -# Postgres -> -# -# $ sudo -u postgres psql -# postgres=# create user nova with createdb login password 'password'; -# postgres=# create database nova with owner nova; -# postgres=# quit; -# -# Note that you may also have to configure 'pg_hba.conf' to use password-based -# auth instead of "ident", if you haven't done so already. You can locate this -# with 'locate pg_hba.conf'. More details at -# https://ubuntu.com/server/docs/databases-postgresql - -set -o xtrace -set -e - -source .tox/py36/bin/activate -pushd nova/db/main/legacy_migrations - -INIT_VERSION=$(ls -1 versions/ | head -1 | awk -F_ '{print $1}') -INIT_VERSION=$(($INIT_VERSION-1)) - -echo "Detected init version of $INIT_VERSION" - -mkdir -p schemas -rm -f "schemas/$INIT_VERSION-*.sql" - -# -# sqlite -# - -# cleanup from previous runs - -rm -f nova.db - -# sync schema - -python manage.py version_control \ - --database 'sqlite:///nova.db' \ - --version $INIT_VERSION - -python manage.py upgrade \ - --database 'sqlite:///nova.db' - -# dump the schema - -sqlite3 nova.db << EOF -.output "schemas/${INIT_VERSION}-sqlite.sql" -.schema -.quit -EOF - -rm -f nova.db - -# -# mysql -# - -# cleanup from previous runs - -mysql -u nova -ppassword << EOF -DROP DATABASE IF EXISTS nova; -CREATE DATABASE nova; -EOF - -# sync schema - -python manage.py version_control \ - --database 'mysql+pymysql://nova:password@localhost/nova' \ - --version "$INIT_VERSION" - -python manage.py upgrade \ - --database 'mysql+pymysql://nova:password@localhost/nova' - -# dump the schema - -mysqldump --no-data --skip-comments -u nova -ppassword \ - nova > "schemas/${INIT_VERSION}-mysql.sql" - -mysql -u nova -ppassword << EOF -DROP DATABASE IF EXISTS nova; -EOF - - -# -# postgres -# - -# cleanup from previous runs - -sudo -u postgres dropdb --if-exists nova -sudo -u postgres createdb --owner=nova nova - -# sync to initial version - -python manage.py version_control \ - --database 'postgresql://nova:password@localhost/nova' \ - --version "$INIT_VERSION" - -python manage.py upgrade \ - --database 'postgresql://nova:password@localhost/nova' - -# dump the schema - -pg_dump postgresql://nova:password@localhost/nova \ - --schema-only > "schemas/${INIT_VERSION}-postgres.sql" - -sudo -u postgres dropdb --if-exists nova - -popd -deactivate diff --git a/tools/reserve-migrations.py b/tools/reserve-migrations.py deleted file mode 100755 index 373c574b69f4..000000000000 --- a/tools/reserve-migrations.py +++ /dev/null @@ -1,75 +0,0 @@ -#!/usr/bin/env python - -import argparse -import glob -import os -import subprocess - -BASE = 'nova/db/main/legacy_migrations/versions'.split('/') -API_BASE = 'nova/db/api/legacy_migrations/versions'.split('/') - -STUB = \ -"""# Licensed under the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. You may obtain -# a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations -# under the License. - -# This is a placeholder for backports. -# Do not use this number for new work. New work starts after -# all the placeholders. -# -# See this for more information: -# http://lists.openstack.org/pipermail/openstack-dev/2013-March/006827.html - - -def upgrade(migrate_engine): - pass -""" - - -def get_last_migration(base): - path = os.path.join(*tuple(base + ['[0-9]*.py'])) - migrations = sorted([os.path.split(fn)[-1] for fn in glob.glob(path)]) - return int(migrations[-1].split('_')[0]) - - -def reserve_migrations(base, number, git_add): - last = get_last_migration(base) - for i in range(last + 1, last + number + 1): - name = '%03i_placeholder.py' % i - path = os.path.join(*tuple(base + [name])) - with open(path, 'w') as f: - f.write(STUB) - print('Created %s' % path) - if git_add: - subprocess.call('git add %s' % path, shell=True) - - -def main(): - parser = argparse.ArgumentParser() - parser.add_argument('-n', '--number', default=10, - type=int, - help='Number of migrations to reserve') - parser.add_argument('-g', '--git-add', action='store_const', - const=True, default=False, - help='Automatically git-add new migrations') - parser.add_argument('-a', '--api', action='store_const', - const=True, default=False, - help='Reserve migrations for the API database') - args = parser.parse_args() - if args.api: - base = API_BASE - else: - base = BASE - reserve_migrations(base, args.number, args.git_add) - - -if __name__ == '__main__': - main() diff --git a/tox.ini b/tox.ini index d11a1d69e05f..2b512e3d8c5f 100644 --- a/tox.ini +++ b/tox.ini @@ -32,11 +32,7 @@ passenv = # there is also secret magic in subunit-trace which lets you run in a fail only # mode. To do this define the TRACE_FAILONLY environmental variable. commands = -# NOTE(gibi): The group-regex runs the matching tests in the same executor. -# These tests runs against a real mysql instance and in an IO deprived CI VM they tend to time out. -# See bug https://launchpad.net/bugs/1823251 for details. -# By running them in the same executor we can spread the IO load of these tests in time. - stestr --group-regex=nova\.tests\.unit\.db\.test_migrations\.TestNovaMigrationsMySQL run {posargs} + stestr run {posargs} env TEST_OSPROFILER=1 stestr run --combine --no-discover 'nova.tests.unit.test_profiler' stestr slowest @@ -96,13 +92,7 @@ deps = {[testenv]deps} openstack-placement>=1.0.0 commands = -# NOTE(gibi): The group-regex runs the matching tests in the same executor. -# These tests runs against a real db instance and in an IO deprived CI VM they tend to time out. -# See bug https://launchpad.net/bugs/1823251 for details. -# By running them in the same executor we can spread the IO load of these tests in time. -# NOTE(gibi): I was not able to group only the mysql tests this way as regex -# TestNovaAPIMigrations.*MySQL does not do what I expect - stestr --group-regex=nova\.tests\.functional\.db\.api\.test_migrations\.TestNovaAPIMigrations --test-path=./nova/tests/functional run {posargs} + stestr --test-path=./nova/tests/functional run {posargs} stestr slowest [testenv:functional-py36]