Merge "db: Final cleanups"
This commit is contained in:
Zuul
Gerrit Code Review
commit
828ac05615
@ -1,4 +1,4 @@
|
||||
alembic==0.9.8
|
||||
alembic==1.5.0
|
||||
amqp==2.5.0
|
||||
appdirs==1.4.3
|
||||
asn1crypto==0.24.0
|
||||
@ -137,7 +137,7 @@ simplejson==3.13.2
|
||||
six==1.15.0
|
||||
smmap2==2.0.3
|
||||
sortedcontainers==2.1.0
|
||||
SQLAlchemy==1.2.19
|
||||
SQLAlchemy==1.4.13
|
||||
sqlalchemy-migrate==0.13.0
|
||||
sqlparse==0.2.4
|
||||
statsd==3.2.2
|
||||
|
||||
@ -1,9 +1,5 @@
|
||||
# The order of packages is significant, because pip processes them in the order
|
||||
# of appearance. Changing the order has an impact on the overall integration
|
||||
# process, which may cause wedges in the gate later.
|
||||
|
||||
pbr>=5.5.1 # Apache-2.0
|
||||
SQLAlchemy>=1.2.19 # MIT
|
||||
SQLAlchemy>=1.4.13 # MIT
|
||||
decorator>=4.1.0 # BSD
|
||||
eventlet>=0.30.1 # MIT
|
||||
Jinja2>=2.10 # BSD License (3 clause)
|
||||
@ -19,6 +15,7 @@ PasteDeploy>=1.5.0 # MIT
|
||||
Paste>=2.0.2 # MIT
|
||||
PrettyTable>=0.7.1 # BSD
|
||||
sqlalchemy-migrate>=0.13.0 # Apache-2.0
|
||||
alembic>=1.5.0 # MIT
|
||||
netaddr>=0.7.18 # BSD
|
||||
netifaces>=0.10.4 # MIT
|
||||
paramiko>=2.7.1 # LGPLv2.1+
|
||||
|
||||
@ -1,283 +0,0 @@
|
||||
#!/usr/bin/env python
|
||||
# Copyright 2012 OpenStack Foundation
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License"); you may
|
||||
# not use this file except in compliance with the License. You may obtain
|
||||
# a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
|
||||
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
|
||||
# License for the specific language governing permissions and limitations
|
||||
# under the License.
|
||||
|
||||
"""
|
||||
Utility for diff'ing two versions of the DB schema.
|
||||
|
||||
Each release cycle the plan is to compact all of the migrations from that
|
||||
release into a single file. This is a manual and, unfortunately, error-prone
|
||||
process. To ensure that the schema doesn't change, this tool can be used to
|
||||
diff the compacted DB schema to the original, uncompacted form.
|
||||
|
||||
The database is specified by providing a SQLAlchemy connection URL WITHOUT the
|
||||
database-name portion (that will be filled in automatically with a temporary
|
||||
database name).
|
||||
|
||||
The schema versions are specified by providing a git ref (a branch name or
|
||||
commit hash) and a SQLAlchemy-Migrate version number:
|
||||
|
||||
Run like:
|
||||
|
||||
MYSQL:
|
||||
|
||||
./tools/db/schema_diff.py mysql+pymysql://root@localhost \
|
||||
master:latest my_branch:82
|
||||
|
||||
POSTGRESQL:
|
||||
|
||||
./tools/db/schema_diff.py postgresql://localhost \
|
||||
master:latest my_branch:82
|
||||
|
||||
"""
|
||||
|
||||
import datetime
|
||||
import glob
|
||||
import os
|
||||
import subprocess
|
||||
import sys
|
||||
|
||||
from nova.i18n import _
|
||||
|
||||
|
||||
# Dump
|
||||
|
||||
|
||||
def dump_db(db_driver, db_name, db_url, migration_version, dump_filename):
|
||||
if not db_url.endswith('/'):
|
||||
db_url += '/'
|
||||
|
||||
db_url += db_name
|
||||
|
||||
db_driver.create(db_name)
|
||||
try:
|
||||
_migrate(db_url, migration_version)
|
||||
db_driver.dump(db_name, dump_filename)
|
||||
finally:
|
||||
db_driver.drop(db_name)
|
||||
|
||||
|
||||
# Diff
|
||||
|
||||
|
||||
def diff_files(filename1, filename2):
|
||||
pipeline = ['diff -U 3 %(filename1)s %(filename2)s'
|
||||
% {'filename1': filename1, 'filename2': filename2}]
|
||||
|
||||
# Use colordiff if available
|
||||
if subprocess.call(['which', 'colordiff']) == 0:
|
||||
pipeline.append('colordiff')
|
||||
|
||||
pipeline.append('less -R')
|
||||
|
||||
cmd = ' | '.join(pipeline)
|
||||
subprocess.check_call(cmd, shell=True)
|
||||
|
||||
|
||||
# Database
|
||||
|
||||
|
||||
class Mysql(object):
|
||||
def create(self, name):
|
||||
subprocess.check_call(['mysqladmin', '-u', 'root', 'create', name])
|
||||
|
||||
def drop(self, name):
|
||||
subprocess.check_call(['mysqladmin', '-f', '-u', 'root', 'drop', name])
|
||||
|
||||
def dump(self, name, dump_filename):
|
||||
subprocess.check_call(
|
||||
'mysqldump -u root %(name)s > %(dump_filename)s'
|
||||
% {'name': name, 'dump_filename': dump_filename},
|
||||
shell=True)
|
||||
|
||||
|
||||
class Postgresql(object):
|
||||
def create(self, name):
|
||||
subprocess.check_call(['createdb', name])
|
||||
|
||||
def drop(self, name):
|
||||
subprocess.check_call(['dropdb', name])
|
||||
|
||||
def dump(self, name, dump_filename):
|
||||
subprocess.check_call(
|
||||
'pg_dump %(name)s > %(dump_filename)s'
|
||||
% {'name': name, 'dump_filename': dump_filename},
|
||||
shell=True)
|
||||
|
||||
|
||||
def _get_db_driver_class(db_url):
|
||||
try:
|
||||
return globals()[db_url.split('://')[0].capitalize()]
|
||||
except KeyError:
|
||||
raise Exception(_("database %s not supported") % db_url)
|
||||
|
||||
|
||||
# Migrate
|
||||
|
||||
|
||||
MIGRATE_REPO = os.path.join(os.getcwd(), "nova/db/main/legacy_migrations")
|
||||
|
||||
|
||||
def _migrate(db_url, migration_version):
|
||||
earliest_version = _migrate_get_earliest_version()
|
||||
|
||||
# NOTE(sirp): sqlalchemy-migrate currently cannot handle the skipping of
|
||||
# migration numbers.
|
||||
_migrate_cmd(
|
||||
db_url, 'version_control', str(earliest_version - 1))
|
||||
|
||||
upgrade_cmd = ['upgrade']
|
||||
if migration_version != 'latest':
|
||||
upgrade_cmd.append(str(migration_version))
|
||||
|
||||
_migrate_cmd(db_url, *upgrade_cmd)
|
||||
|
||||
|
||||
def _migrate_cmd(db_url, *cmd):
|
||||
manage_py = os.path.join(MIGRATE_REPO, 'manage.py')
|
||||
|
||||
args = ['python', manage_py]
|
||||
args += cmd
|
||||
args += ['--repository=%s' % MIGRATE_REPO,
|
||||
'--url=%s' % db_url]
|
||||
|
||||
subprocess.check_call(args)
|
||||
|
||||
|
||||
def _migrate_get_earliest_version():
|
||||
versions_glob = os.path.join(MIGRATE_REPO, 'versions', '???_*.py')
|
||||
|
||||
versions = []
|
||||
for path in glob.iglob(versions_glob):
|
||||
filename = os.path.basename(path)
|
||||
prefix = filename.split('_', 1)[0]
|
||||
try:
|
||||
version = int(prefix)
|
||||
except ValueError:
|
||||
pass
|
||||
versions.append(version)
|
||||
|
||||
versions.sort()
|
||||
return versions[0]
|
||||
|
||||
|
||||
# Git
|
||||
|
||||
|
||||
def git_current_branch_name():
|
||||
ref_name = git_symbolic_ref('HEAD', quiet=True)
|
||||
current_branch_name = ref_name.replace('refs/heads/', '')
|
||||
return current_branch_name
|
||||
|
||||
|
||||
def git_symbolic_ref(ref, quiet=False):
|
||||
args = ['git', 'symbolic-ref', ref]
|
||||
if quiet:
|
||||
args.append('-q')
|
||||
proc = subprocess.Popen(args, stdout=subprocess.PIPE)
|
||||
stdout, stderr = proc.communicate()
|
||||
return stdout.strip()
|
||||
|
||||
|
||||
def git_checkout(branch_name):
|
||||
subprocess.check_call(['git', 'checkout', branch_name])
|
||||
|
||||
|
||||
def git_has_uncommited_changes():
|
||||
return subprocess.call(['git', 'diff', '--quiet', '--exit-code']) == 1
|
||||
|
||||
|
||||
# Command
|
||||
|
||||
|
||||
def die(msg):
|
||||
print("ERROR: %s" % msg, file=sys.stderr)
|
||||
sys.exit(1)
|
||||
|
||||
|
||||
def usage(msg=None):
|
||||
if msg:
|
||||
print("ERROR: %s" % msg, file=sys.stderr)
|
||||
|
||||
prog = "schema_diff.py"
|
||||
args = ["<db-url>", "<orig-branch:orig-version>",
|
||||
"<new-branch:new-version>"]
|
||||
|
||||
print("usage: %s %s" % (prog, ' '.join(args)), file=sys.stderr)
|
||||
sys.exit(1)
|
||||
|
||||
|
||||
def parse_options():
|
||||
try:
|
||||
db_url = sys.argv[1]
|
||||
except IndexError:
|
||||
usage("must specify DB connection url")
|
||||
|
||||
try:
|
||||
orig_branch, orig_version = sys.argv[2].split(':')
|
||||
except IndexError:
|
||||
usage('original branch and version required (e.g. master:82)')
|
||||
|
||||
try:
|
||||
new_branch, new_version = sys.argv[3].split(':')
|
||||
except IndexError:
|
||||
usage('new branch and version required (e.g. master:82)')
|
||||
|
||||
return db_url, orig_branch, orig_version, new_branch, new_version
|
||||
|
||||
|
||||
def main():
|
||||
timestamp = datetime.datetime.utcnow().strftime("%Y%m%d_%H%M%S")
|
||||
|
||||
ORIG_DB = 'orig_db_%s' % timestamp
|
||||
NEW_DB = 'new_db_%s' % timestamp
|
||||
|
||||
ORIG_DUMP = ORIG_DB + ".dump"
|
||||
NEW_DUMP = NEW_DB + ".dump"
|
||||
|
||||
options = parse_options()
|
||||
db_url, orig_branch, orig_version, new_branch, new_version = options
|
||||
|
||||
# Since we're going to be switching branches, ensure user doesn't have any
|
||||
# uncommitted changes
|
||||
if git_has_uncommited_changes():
|
||||
die("You have uncommitted changes. Please commit them before running "
|
||||
"this command.")
|
||||
|
||||
db_driver = _get_db_driver_class(db_url)()
|
||||
|
||||
users_branch = git_current_branch_name()
|
||||
git_checkout(orig_branch)
|
||||
|
||||
try:
|
||||
# Dump Original Schema
|
||||
dump_db(db_driver, ORIG_DB, db_url, orig_version, ORIG_DUMP)
|
||||
|
||||
# Dump New Schema
|
||||
git_checkout(new_branch)
|
||||
dump_db(db_driver, NEW_DB, db_url, new_version, NEW_DUMP)
|
||||
|
||||
diff_files(ORIG_DUMP, NEW_DUMP)
|
||||
finally:
|
||||
git_checkout(users_branch)
|
||||
|
||||
if os.path.exists(ORIG_DUMP):
|
||||
os.unlink(ORIG_DUMP)
|
||||
|
||||
if os.path.exists(NEW_DUMP):
|
||||
os.unlink(NEW_DUMP)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@ -1,123 +0,0 @@
|
||||
#!/usr/bin/env bash
|
||||
#
|
||||
# Script to generate schemas for the various versions.
|
||||
#
|
||||
# Some setup is required, similar to the opportunistic tests.
|
||||
#
|
||||
# MySQL ->
|
||||
#
|
||||
# $ mysql -uroot
|
||||
# MariaDB [(none)]> CREATE DATABASE nova
|
||||
# MariaDB [(none)]> GRANT ALL PRIVILEGES ON nova.* TO 'nova'@'localhost' IDENTIFIED BY 'password';
|
||||
# MariaDB [(none)]> quit;
|
||||
#
|
||||
# Postgres ->
|
||||
#
|
||||
# $ sudo -u postgres psql
|
||||
# postgres=# create user nova with createdb login password 'password';
|
||||
# postgres=# create database nova with owner nova;
|
||||
# postgres=# quit;
|
||||
#
|
||||
# Note that you may also have to configure 'pg_hba.conf' to use password-based
|
||||
# auth instead of "ident", if you haven't done so already. You can locate this
|
||||
# with 'locate pg_hba.conf'. More details at
|
||||
# https://ubuntu.com/server/docs/databases-postgresql
|
||||
|
||||
set -o xtrace
|
||||
set -e
|
||||
|
||||
source .tox/py36/bin/activate
|
||||
pushd nova/db/main/legacy_migrations
|
||||
|
||||
INIT_VERSION=$(ls -1 versions/ | head -1 | awk -F_ '{print $1}')
|
||||
INIT_VERSION=$(($INIT_VERSION-1))
|
||||
|
||||
echo "Detected init version of $INIT_VERSION"
|
||||
|
||||
mkdir -p schemas
|
||||
rm -f "schemas/$INIT_VERSION-*.sql"
|
||||
|
||||
#
|
||||
# sqlite
|
||||
#
|
||||
|
||||
# cleanup from previous runs
|
||||
|
||||
rm -f nova.db
|
||||
|
||||
# sync schema
|
||||
|
||||
python manage.py version_control \
|
||||
--database 'sqlite:///nova.db' \
|
||||
--version $INIT_VERSION
|
||||
|
||||
python manage.py upgrade \
|
||||
--database 'sqlite:///nova.db'
|
||||
|
||||
# dump the schema
|
||||
|
||||
sqlite3 nova.db << EOF
|
||||
.output "schemas/${INIT_VERSION}-sqlite.sql"
|
||||
.schema
|
||||
.quit
|
||||
EOF
|
||||
|
||||
rm -f nova.db
|
||||
|
||||
#
|
||||
# mysql
|
||||
#
|
||||
|
||||
# cleanup from previous runs
|
||||
|
||||
mysql -u nova -ppassword << EOF
|
||||
DROP DATABASE IF EXISTS nova;
|
||||
CREATE DATABASE nova;
|
||||
EOF
|
||||
|
||||
# sync schema
|
||||
|
||||
python manage.py version_control \
|
||||
--database 'mysql+pymysql://nova:password@localhost/nova' \
|
||||
--version "$INIT_VERSION"
|
||||
|
||||
python manage.py upgrade \
|
||||
--database 'mysql+pymysql://nova:password@localhost/nova'
|
||||
|
||||
# dump the schema
|
||||
|
||||
mysqldump --no-data --skip-comments -u nova -ppassword \
|
||||
nova > "schemas/${INIT_VERSION}-mysql.sql"
|
||||
|
||||
mysql -u nova -ppassword << EOF
|
||||
DROP DATABASE IF EXISTS nova;
|
||||
EOF
|
||||
|
||||
|
||||
#
|
||||
# postgres
|
||||
#
|
||||
|
||||
# cleanup from previous runs
|
||||
|
||||
sudo -u postgres dropdb --if-exists nova
|
||||
sudo -u postgres createdb --owner=nova nova
|
||||
|
||||
# sync to initial version
|
||||
|
||||
python manage.py version_control \
|
||||
--database 'postgresql://nova:password@localhost/nova' \
|
||||
--version "$INIT_VERSION"
|
||||
|
||||
python manage.py upgrade \
|
||||
--database 'postgresql://nova:password@localhost/nova'
|
||||
|
||||
# dump the schema
|
||||
|
||||
pg_dump postgresql://nova:password@localhost/nova \
|
||||
--schema-only > "schemas/${INIT_VERSION}-postgres.sql"
|
||||
|
||||
sudo -u postgres dropdb --if-exists nova
|
||||
|
||||
popd
|
||||
deactivate
|
||||
@ -1,75 +0,0 @@
|
||||
#!/usr/bin/env python
|
||||
|
||||
import argparse
|
||||
import glob
|
||||
import os
|
||||
import subprocess
|
||||
|
||||
BASE = 'nova/db/main/legacy_migrations/versions'.split('/')
|
||||
API_BASE = 'nova/db/api/legacy_migrations/versions'.split('/')
|
||||
|
||||
STUB = \
|
||||
"""# Licensed under the Apache License, Version 2.0 (the "License"); you may
|
||||
# not use this file except in compliance with the License. You may obtain
|
||||
# a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
|
||||
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
|
||||
# License for the specific language governing permissions and limitations
|
||||
# under the License.
|
||||
|
||||
# This is a placeholder for backports.
|
||||
# Do not use this number for new work. New work starts after
|
||||
# all the placeholders.
|
||||
#
|
||||
# See this for more information:
|
||||
# http://lists.openstack.org/pipermail/openstack-dev/2013-March/006827.html
|
||||
|
||||
|
||||
def upgrade(migrate_engine):
|
||||
pass
|
||||
"""
|
||||
|
||||
|
||||
def get_last_migration(base):
|
||||
path = os.path.join(*tuple(base + ['[0-9]*.py']))
|
||||
migrations = sorted([os.path.split(fn)[-1] for fn in glob.glob(path)])
|
||||
return int(migrations[-1].split('_')[0])
|
||||
|
||||
|
||||
def reserve_migrations(base, number, git_add):
|
||||
last = get_last_migration(base)
|
||||
for i in range(last + 1, last + number + 1):
|
||||
name = '%03i_placeholder.py' % i
|
||||
path = os.path.join(*tuple(base + [name]))
|
||||
with open(path, 'w') as f:
|
||||
f.write(STUB)
|
||||
print('Created %s' % path)
|
||||
if git_add:
|
||||
subprocess.call('git add %s' % path, shell=True)
|
||||
|
||||
|
||||
def main():
|
||||
parser = argparse.ArgumentParser()
|
||||
parser.add_argument('-n', '--number', default=10,
|
||||
type=int,
|
||||
help='Number of migrations to reserve')
|
||||
parser.add_argument('-g', '--git-add', action='store_const',
|
||||
const=True, default=False,
|
||||
help='Automatically git-add new migrations')
|
||||
parser.add_argument('-a', '--api', action='store_const',
|
||||
const=True, default=False,
|
||||
help='Reserve migrations for the API database')
|
||||
args = parser.parse_args()
|
||||
if args.api:
|
||||
base = API_BASE
|
||||
else:
|
||||
base = BASE
|
||||
reserve_migrations(base, args.number, args.git_add)
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
main()
|
||||
14
tox.ini
14
tox.ini
@ -32,11 +32,7 @@ passenv =
|
||||
# there is also secret magic in subunit-trace which lets you run in a fail only
|
||||
# mode. To do this define the TRACE_FAILONLY environmental variable.
|
||||
commands =
|
||||
# NOTE(gibi): The group-regex runs the matching tests in the same executor.
|
||||
# These tests runs against a real mysql instance and in an IO deprived CI VM they tend to time out.
|
||||
# See bug https://launchpad.net/bugs/1823251 for details.
|
||||
# By running them in the same executor we can spread the IO load of these tests in time.
|
||||
stestr --group-regex=nova\.tests\.unit\.db\.test_migrations\.TestNovaMigrationsMySQL run {posargs}
|
||||
stestr run {posargs}
|
||||
env TEST_OSPROFILER=1 stestr run --combine --no-discover 'nova.tests.unit.test_profiler'
|
||||
stestr slowest
|
||||
|
||||
@ -96,13 +92,7 @@ deps =
|
||||
{[testenv]deps}
|
||||
openstack-placement>=1.0.0
|
||||
commands =
|
||||
# NOTE(gibi): The group-regex runs the matching tests in the same executor.
|
||||
# These tests runs against a real db instance and in an IO deprived CI VM they tend to time out.
|
||||
# See bug https://launchpad.net/bugs/1823251 for details.
|
||||
# By running them in the same executor we can spread the IO load of these tests in time.
|
||||
# NOTE(gibi): I was not able to group only the mysql tests this way as regex
|
||||
# TestNovaAPIMigrations.*MySQL does not do what I expect
|
||||
stestr --group-regex=nova\.tests\.functional\.db\.api\.test_migrations\.TestNovaAPIMigrations --test-path=./nova/tests/functional run {posargs}
|
||||
stestr --test-path=./nova/tests/functional run {posargs}
|
||||
stestr slowest
|
||||
|
||||
[testenv:functional-py36]
|
||||
|
||||
Loading…
x
Reference in New Issue
Block a user