Browse Source

db: Final cleanups

Some things that were missed in previous patches and are thrown together
here:

- Add alembic as an explicit dependency (we were getting it transitively
  from oslo.db). We also bump the sqlalchemy dependency to a 1.4.x
  release, which is the minimum supported by our chosen version of
  alembic (more on this below)
- Remove tooling related to the old migrations
- Fix the tox whitelisting of the flaky MySQL tests

On the SQLAlchemy front, we opt for 1.4.13. Technically alembic should
support anything from 1.4.0, however, with SQLAlchemy >= 1.4.0, < 1.4.13
we see errors like the following in some tests:

  sqlalchemy.exc.InvalidRequestError: Entity namespace for
  "count(instance_mappings.id)" has no property "queued_for_delete"

There's nothing specific about this in the release notes for 1.4.13 [1]
but it definitely fixes things.

[1] https://docs.sqlalchemy.org/en/14/changelog/changelog_14.html#change-1.4.13

Change-Id: I4c8eb13f11aa7471c26a5ba326319aef245c9836
Signed-off-by: Stephen Finucane <stephenfin@redhat.com>
changes/84/800484/5
Stephen Finucane 6 months ago
parent
commit
eb728e877a
  1. 4
      lower-constraints.txt
  2. 7
      requirements.txt
  3. 283
      tools/db/schema_diff.py
  4. 123
      tools/generate-schemas
  5. 75
      tools/reserve-migrations.py
  6. 14
      tox.ini

4
lower-constraints.txt

@ -1,4 +1,4 @@
alembic==0.9.8
alembic==1.5.0
amqp==2.5.0
appdirs==1.4.3
asn1crypto==0.24.0
@ -137,7 +137,7 @@ simplejson==3.13.2
six==1.15.0
smmap2==2.0.3
sortedcontainers==2.1.0
SQLAlchemy==1.2.19
SQLAlchemy==1.4.13
sqlalchemy-migrate==0.13.0
sqlparse==0.2.4
statsd==3.2.2

7
requirements.txt

@ -1,9 +1,5 @@
# The order of packages is significant, because pip processes them in the order
# of appearance. Changing the order has an impact on the overall integration
# process, which may cause wedges in the gate later.
pbr>=5.5.1 # Apache-2.0
SQLAlchemy>=1.2.19 # MIT
SQLAlchemy>=1.4.13 # MIT
decorator>=4.1.0 # BSD
eventlet>=0.30.1 # MIT
Jinja2>=2.10 # BSD License (3 clause)
@ -19,6 +15,7 @@ PasteDeploy>=1.5.0 # MIT
Paste>=2.0.2 # MIT
PrettyTable>=0.7.1 # BSD
sqlalchemy-migrate>=0.13.0 # Apache-2.0
alembic>=1.5.0 # MIT
netaddr>=0.7.18 # BSD
netifaces>=0.10.4 # MIT
paramiko>=2.7.1 # LGPLv2.1+

283
tools/db/schema_diff.py

@ -1,283 +0,0 @@
#!/usr/bin/env python
# Copyright 2012 OpenStack Foundation
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
"""
Utility for diff'ing two versions of the DB schema.
Each release cycle the plan is to compact all of the migrations from that
release into a single file. This is a manual and, unfortunately, error-prone
process. To ensure that the schema doesn't change, this tool can be used to
diff the compacted DB schema to the original, uncompacted form.
The database is specified by providing a SQLAlchemy connection URL WITHOUT the
database-name portion (that will be filled in automatically with a temporary
database name).
The schema versions are specified by providing a git ref (a branch name or
commit hash) and a SQLAlchemy-Migrate version number:
Run like:
MYSQL:
./tools/db/schema_diff.py mysql+pymysql://root@localhost \
master:latest my_branch:82
POSTGRESQL:
./tools/db/schema_diff.py postgresql://localhost \
master:latest my_branch:82
"""
import datetime
import glob
import os
import subprocess
import sys
from nova.i18n import _
# Dump
def dump_db(db_driver, db_name, db_url, migration_version, dump_filename):
if not db_url.endswith('/'):
db_url += '/'
db_url += db_name
db_driver.create(db_name)
try:
_migrate(db_url, migration_version)
db_driver.dump(db_name, dump_filename)
finally:
db_driver.drop(db_name)
# Diff
def diff_files(filename1, filename2):
pipeline = ['diff -U 3 %(filename1)s %(filename2)s'
% {'filename1': filename1, 'filename2': filename2}]
# Use colordiff if available
if subprocess.call(['which', 'colordiff']) == 0:
pipeline.append('colordiff')
pipeline.append('less -R')
cmd = ' | '.join(pipeline)
subprocess.check_call(cmd, shell=True)
# Database
class Mysql(object):
def create(self, name):
subprocess.check_call(['mysqladmin', '-u', 'root', 'create', name])
def drop(self, name):
subprocess.check_call(['mysqladmin', '-f', '-u', 'root', 'drop', name])
def dump(self, name, dump_filename):
subprocess.check_call(
'mysqldump -u root %(name)s > %(dump_filename)s'
% {'name': name, 'dump_filename': dump_filename},
shell=True)
class Postgresql(object):
def create(self, name):
subprocess.check_call(['createdb', name])
def drop(self, name):
subprocess.check_call(['dropdb', name])
def dump(self, name, dump_filename):
subprocess.check_call(
'pg_dump %(name)s > %(dump_filename)s'
% {'name': name, 'dump_filename': dump_filename},
shell=True)
def _get_db_driver_class(db_url):
try:
return globals()[db_url.split('://')[0].capitalize()]
except KeyError:
raise Exception(_("database %s not supported") % db_url)
# Migrate
MIGRATE_REPO = os.path.join(os.getcwd(), "nova/db/main/legacy_migrations")
def _migrate(db_url, migration_version):
earliest_version = _migrate_get_earliest_version()
# NOTE(sirp): sqlalchemy-migrate currently cannot handle the skipping of
# migration numbers.
_migrate_cmd(
db_url, 'version_control', str(earliest_version - 1))
upgrade_cmd = ['upgrade']
if migration_version != 'latest':
upgrade_cmd.append(str(migration_version))
_migrate_cmd(db_url, *upgrade_cmd)
def _migrate_cmd(db_url, *cmd):
manage_py = os.path.join(MIGRATE_REPO, 'manage.py')
args = ['python', manage_py]
args += cmd
args += ['--repository=%s' % MIGRATE_REPO,
'--url=%s' % db_url]
subprocess.check_call(args)
def _migrate_get_earliest_version():
versions_glob = os.path.join(MIGRATE_REPO, 'versions', '???_*.py')
versions = []
for path in glob.iglob(versions_glob):
filename = os.path.basename(path)
prefix = filename.split('_', 1)[0]
try:
version = int(prefix)
except ValueError:
pass
versions.append(version)
versions.sort()
return versions[0]
# Git
def git_current_branch_name():
ref_name = git_symbolic_ref('HEAD', quiet=True)
current_branch_name = ref_name.replace('refs/heads/', '')
return current_branch_name
def git_symbolic_ref(ref, quiet=False):
args = ['git', 'symbolic-ref', ref]
if quiet:
args.append('-q')
proc = subprocess.Popen(args, stdout=subprocess.PIPE)
stdout, stderr = proc.communicate()
return stdout.strip()
def git_checkout(branch_name):
subprocess.check_call(['git', 'checkout', branch_name])
def git_has_uncommited_changes():
return subprocess.call(['git', 'diff', '--quiet', '--exit-code']) == 1
# Command
def die(msg):
print("ERROR: %s" % msg, file=sys.stderr)
sys.exit(1)
def usage(msg=None):
if msg:
print("ERROR: %s" % msg, file=sys.stderr)
prog = "schema_diff.py"
args = ["<db-url>", "<orig-branch:orig-version>",
"<new-branch:new-version>"]
print("usage: %s %s" % (prog, ' '.join(args)), file=sys.stderr)
sys.exit(1)
def parse_options():
try:
db_url = sys.argv[1]
except IndexError:
usage("must specify DB connection url")
try:
orig_branch, orig_version = sys.argv[2].split(':')
except IndexError:
usage('original branch and version required (e.g. master:82)')
try:
new_branch, new_version = sys.argv[3].split(':')
except IndexError:
usage('new branch and version required (e.g. master:82)')
return db_url, orig_branch, orig_version, new_branch, new_version
def main():
timestamp = datetime.datetime.utcnow().strftime("%Y%m%d_%H%M%S")
ORIG_DB = 'orig_db_%s' % timestamp
NEW_DB = 'new_db_%s' % timestamp
ORIG_DUMP = ORIG_DB + ".dump"
NEW_DUMP = NEW_DB + ".dump"
options = parse_options()
db_url, orig_branch, orig_version, new_branch, new_version = options
# Since we're going to be switching branches, ensure user doesn't have any
# uncommitted changes
if git_has_uncommited_changes():
die("You have uncommitted changes. Please commit them before running "
"this command.")
db_driver = _get_db_driver_class(db_url)()
users_branch = git_current_branch_name()
git_checkout(orig_branch)
try:
# Dump Original Schema
dump_db(db_driver, ORIG_DB, db_url, orig_version, ORIG_DUMP)
# Dump New Schema
git_checkout(new_branch)
dump_db(db_driver, NEW_DB, db_url, new_version, NEW_DUMP)
diff_files(ORIG_DUMP, NEW_DUMP)
finally:
git_checkout(users_branch)
if os.path.exists(ORIG_DUMP):
os.unlink(ORIG_DUMP)
if os.path.exists(NEW_DUMP):
os.unlink(NEW_DUMP)
if __name__ == "__main__":
main()

123
tools/generate-schemas

@ -1,123 +0,0 @@
#!/usr/bin/env bash
#
# Script to generate schemas for the various versions.
#
# Some setup is required, similar to the opportunistic tests.
#
# MySQL ->
#
# $ mysql -uroot
# MariaDB [(none)]> CREATE DATABASE nova
# MariaDB [(none)]> GRANT ALL PRIVILEGES ON nova.* TO 'nova'@'localhost' IDENTIFIED BY 'password';
# MariaDB [(none)]> quit;
#
# Postgres ->
#
# $ sudo -u postgres psql
# postgres=# create user nova with createdb login password 'password';
# postgres=# create database nova with owner nova;
# postgres=# quit;
#
# Note that you may also have to configure 'pg_hba.conf' to use password-based
# auth instead of "ident", if you haven't done so already. You can locate this
# with 'locate pg_hba.conf'. More details at
# https://ubuntu.com/server/docs/databases-postgresql
set -o xtrace
set -e
source .tox/py36/bin/activate
pushd nova/db/main/legacy_migrations
INIT_VERSION=$(ls -1 versions/ | head -1 | awk -F_ '{print $1}')
INIT_VERSION=$(($INIT_VERSION-1))
echo "Detected init version of $INIT_VERSION"
mkdir -p schemas
rm -f "schemas/$INIT_VERSION-*.sql"
#
# sqlite
#
# cleanup from previous runs
rm -f nova.db
# sync schema
python manage.py version_control \
--database 'sqlite:///nova.db' \
--version $INIT_VERSION
python manage.py upgrade \
--database 'sqlite:///nova.db'
# dump the schema
sqlite3 nova.db << EOF
.output "schemas/${INIT_VERSION}-sqlite.sql"
.schema
.quit
EOF
rm -f nova.db
#
# mysql
#
# cleanup from previous runs
mysql -u nova -ppassword << EOF
DROP DATABASE IF EXISTS nova;
CREATE DATABASE nova;
EOF
# sync schema
python manage.py version_control \
--database 'mysql+pymysql://nova:password@localhost/nova' \
--version "$INIT_VERSION"
python manage.py upgrade \
--database 'mysql+pymysql://nova:password@localhost/nova'
# dump the schema
mysqldump --no-data --skip-comments -u nova -ppassword \
nova > "schemas/${INIT_VERSION}-mysql.sql"
mysql -u nova -ppassword << EOF
DROP DATABASE IF EXISTS nova;
EOF
#
# postgres
#
# cleanup from previous runs
sudo -u postgres dropdb --if-exists nova
sudo -u postgres createdb --owner=nova nova
# sync to initial version
python manage.py version_control \
--database 'postgresql://nova:password@localhost/nova' \
--version "$INIT_VERSION"
python manage.py upgrade \
--database 'postgresql://nova:password@localhost/nova'
# dump the schema
pg_dump postgresql://nova:password@localhost/nova \
--schema-only > "schemas/${INIT_VERSION}-postgres.sql"
sudo -u postgres dropdb --if-exists nova
popd
deactivate

75
tools/reserve-migrations.py

@ -1,75 +0,0 @@
#!/usr/bin/env python
import argparse
import glob
import os
import subprocess
BASE = 'nova/db/main/legacy_migrations/versions'.split('/')
API_BASE = 'nova/db/api/legacy_migrations/versions'.split('/')
STUB = \
"""# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
# This is a placeholder for backports.
# Do not use this number for new work. New work starts after
# all the placeholders.
#
# See this for more information:
# http://lists.openstack.org/pipermail/openstack-dev/2013-March/006827.html
def upgrade(migrate_engine):
pass
"""
def get_last_migration(base):
path = os.path.join(*tuple(base + ['[0-9]*.py']))
migrations = sorted([os.path.split(fn)[-1] for fn in glob.glob(path)])
return int(migrations[-1].split('_')[0])
def reserve_migrations(base, number, git_add):
last = get_last_migration(base)
for i in range(last + 1, last + number + 1):
name = '%03i_placeholder.py' % i
path = os.path.join(*tuple(base + [name]))
with open(path, 'w') as f:
f.write(STUB)
print('Created %s' % path)
if git_add:
subprocess.call('git add %s' % path, shell=True)
def main():
parser = argparse.ArgumentParser()
parser.add_argument('-n', '--number', default=10,
type=int,
help='Number of migrations to reserve')
parser.add_argument('-g', '--git-add', action='store_const',
const=True, default=False,
help='Automatically git-add new migrations')
parser.add_argument('-a', '--api', action='store_const',
const=True, default=False,
help='Reserve migrations for the API database')
args = parser.parse_args()
if args.api:
base = API_BASE
else:
base = BASE
reserve_migrations(base, args.number, args.git_add)
if __name__ == '__main__':
main()

14
tox.ini

@ -32,11 +32,7 @@ passenv =
# there is also secret magic in subunit-trace which lets you run in a fail only
# mode. To do this define the TRACE_FAILONLY environmental variable.
commands =
# NOTE(gibi): The group-regex runs the matching tests in the same executor.
# These tests runs against a real mysql instance and in an IO deprived CI VM they tend to time out.
# See bug https://launchpad.net/bugs/1823251 for details.
# By running them in the same executor we can spread the IO load of these tests in time.
stestr --group-regex=nova\.tests\.unit\.db\.test_migrations\.TestNovaMigrationsMySQL run {posargs}
stestr run {posargs}
env TEST_OSPROFILER=1 stestr run --combine --no-discover 'nova.tests.unit.test_profiler'
stestr slowest
@ -96,13 +92,7 @@ deps =
{[testenv]deps}
openstack-placement>=1.0.0
commands =
# NOTE(gibi): The group-regex runs the matching tests in the same executor.
# These tests runs against a real db instance and in an IO deprived CI VM they tend to time out.
# See bug https://launchpad.net/bugs/1823251 for details.
# By running them in the same executor we can spread the IO load of these tests in time.
# NOTE(gibi): I was not able to group only the mysql tests this way as regex
# TestNovaAPIMigrations.*MySQL does not do what I expect
stestr --group-regex=nova\.tests\.functional\.db\.api\.test_migrations\.TestNovaAPIMigrations --test-path=./nova/tests/functional run {posargs}
stestr --test-path=./nova/tests/functional run {posargs}
stestr slowest
[testenv:functional-py36]

Loading…
Cancel
Save