From a03b141a954c7e644f0033defdb1b5b434a7c49a Mon Sep 17 00:00:00 2001 From: Cyril Roelandt Date: Wed, 19 Mar 2014 15:02:40 +0100 Subject: [PATCH] Port to Python3 Brief summary of the modifications: * Use six for compatibility with both Python 2 and 3; * Replace UserDict.DictMixin with collections.MutableMapping; * Fix relative imports; * Use test-requirements.txt for requirements that are common to both Python 2 and 3, and test-requirements-py{2,3}.txt for version-specific requirements; * Miscellaneous fixes. * Use a specific test_db_py3.cfg file for Python 3, that only runs tests on sqlite. Thanks to Victor Stinner who co-wrote this patch. Change-Id: Ia6dc536c39d274924c21fd5bb619e8e5721e04c4 Co-Authored-By: Victor Stinner --- migrate/changeset/ansisql.py | 7 ++- migrate/changeset/databases/sqlite.py | 5 +- migrate/changeset/schema.py | 58 ++++++++++++++++--- migrate/tests/__init__.py | 3 +- migrate/tests/changeset/test_changeset.py | 9 +-- migrate/tests/fixture/__init__.py | 8 +-- migrate/tests/fixture/database.py | 17 +++--- migrate/tests/versioning/test_api.py | 6 +- migrate/tests/versioning/test_genmodel.py | 23 ++++---- migrate/tests/versioning/test_repository.py | 1 - migrate/tests/versioning/test_schema.py | 6 +- migrate/tests/versioning/test_schemadiff.py | 4 +- migrate/tests/versioning/test_script.py | 9 ++- migrate/tests/versioning/test_shell.py | 11 ++-- migrate/versioning/cfgparse.py | 2 +- migrate/versioning/genmodel.py | 8 ++- migrate/versioning/repository.py | 6 +- migrate/versioning/schema.py | 12 ++-- migrate/versioning/schemadiff.py | 5 ++ migrate/versioning/script/py.py | 11 ++-- migrate/versioning/shell.py | 12 ++-- .../templates/manage/default.py_tmpl | 3 +- .../templates/manage/pylons.py_tmpl | 3 +- migrate/versioning/util/__init__.py | 13 +++-- migrate/versioning/util/importpath.py | 2 + migrate/versioning/version.py | 7 ++- test-requirements-py2.txt | 2 + test-requirements-py3.txt | 1 + test-requirements.txt | 8 +-- test_db_py3.cfg | 15 +++++ tox.ini | 13 +++++ 31 files changed, 202 insertions(+), 88 deletions(-) create mode 100644 test-requirements-py2.txt create mode 100644 test-requirements-py3.txt create mode 100644 test_db_py3.cfg diff --git a/migrate/changeset/ansisql.py b/migrate/changeset/ansisql.py index b4509ae..a18d4ed 100644 --- a/migrate/changeset/ansisql.py +++ b/migrate/changeset/ansisql.py @@ -4,7 +4,6 @@ At the moment, this isn't so much based off of ANSI as much as things that just happen to work with multiple databases. """ -import StringIO import sqlalchemy as sa from sqlalchemy.schema import SchemaVisitor @@ -20,6 +19,7 @@ from migrate import exceptions import sqlalchemy.sql.compiler from migrate.changeset import constraint from migrate.changeset import util +from six.moves import StringIO from sqlalchemy.schema import AddConstraint, DropConstraint from sqlalchemy.sql.compiler import DDLCompiler @@ -43,11 +43,12 @@ class AlterTableVisitor(SchemaVisitor): try: return self.connection.execute(self.buffer.getvalue()) finally: - self.buffer.truncate(0) + self.buffer.seek(0) + self.buffer.truncate() def __init__(self, dialect, connection, **kw): self.connection = connection - self.buffer = StringIO.StringIO() + self.buffer = StringIO() self.preparer = dialect.identifier_preparer self.dialect = dialect diff --git a/migrate/changeset/databases/sqlite.py b/migrate/changeset/databases/sqlite.py index 6453422..a601593 100644 --- a/migrate/changeset/databases/sqlite.py +++ b/migrate/changeset/databases/sqlite.py @@ -3,7 +3,10 @@ .. _`SQLite`: http://www.sqlite.org/ """ -from UserDict import DictMixin +try: # Python 3 + from collections import MutableMapping as DictMixin +except ImportError: # Python 2 + from UserDict import DictMixin from copy import copy from sqlalchemy.databases import sqlite as sa_base diff --git a/migrate/changeset/schema.py b/migrate/changeset/schema.py index 913b90f..a0e42cc 100644 --- a/migrate/changeset/schema.py +++ b/migrate/changeset/schema.py @@ -1,10 +1,14 @@ """ Schema module providing common schema operations. """ +import abc +try: # Python 3 + from collections import MutableMapping as DictMixin +except ImportError: # Python 2 + from UserDict import DictMixin import warnings -from UserDict import DictMixin - +import six import sqlalchemy from sqlalchemy.schema import ForeignKeyConstraint @@ -163,7 +167,39 @@ def _to_index(index, table=None, engine=None): return ret -class ColumnDelta(DictMixin, sqlalchemy.schema.SchemaItem): + +# Python3: if we just use: +# +# class ColumnDelta(DictMixin, sqlalchemy.schema.SchemaItem): +# ... +# +# We get the following error: +# TypeError: metaclass conflict: the metaclass of a derived class must be a +# (non-strict) subclass of the metaclasses of all its bases. +# +# The complete inheritance/metaclass relationship list of ColumnDelta can be +# summarized by this following dot file: +# +# digraph test123 { +# ColumnDelta -> MutableMapping; +# MutableMapping -> Mapping; +# Mapping -> {Sized Iterable Container}; +# {Sized Iterable Container} -> ABCMeta[style=dashed]; +# +# ColumnDelta -> SchemaItem; +# SchemaItem -> {SchemaEventTarget Visitable}; +# SchemaEventTarget -> object; +# Visitable -> {VisitableType object} [style=dashed]; +# VisitableType -> type; +# } +# +# We need to use a metaclass that inherits from all the metaclasses of +# DictMixin and sqlalchemy.schema.SchemaItem. Let's call it "MyMeta". +class MyMeta(sqlalchemy.sql.visitors.VisitableType, abc.ABCMeta, object): + pass + + +class ColumnDelta(six.with_metaclass(MyMeta, DictMixin, sqlalchemy.schema.SchemaItem)): """Extracts the differences between two columns/column-parameters May receive parameters arranged in several different ways: @@ -229,7 +265,7 @@ class ColumnDelta(DictMixin, sqlalchemy.schema.SchemaItem): diffs = self.compare_1_column(*p, **kw) else: # Zero columns specified - if not len(p) or not isinstance(p[0], basestring): + if not len(p) or not isinstance(p[0], six.string_types): raise ValueError("First argument must be column name") diffs = self.compare_parameters(*p, **kw) @@ -254,6 +290,12 @@ class ColumnDelta(DictMixin, sqlalchemy.schema.SchemaItem): def __delitem__(self, key): raise NotImplementedError + def __len__(self): + raise NotImplementedError + + def __iter__(self): + raise NotImplementedError + def keys(self): return self.diffs.keys() @@ -332,7 +374,7 @@ class ColumnDelta(DictMixin, sqlalchemy.schema.SchemaItem): """Extracts data from p and modifies diffs""" p = list(p) while len(p): - if isinstance(p[0], basestring): + if isinstance(p[0], six.string_types): k.setdefault('name', p.pop(0)) elif isinstance(p[0], sqlalchemy.types.TypeEngine): k.setdefault('type', p.pop(0)) @@ -370,7 +412,7 @@ class ColumnDelta(DictMixin, sqlalchemy.schema.SchemaItem): return getattr(self, '_table', None) def _set_table(self, table): - if isinstance(table, basestring): + if isinstance(table, six.string_types): if self.alter_metadata: if not self.meta: raise ValueError("metadata must be specified for table" @@ -587,7 +629,7 @@ populated with defaults if isinstance(cons,(ForeignKeyConstraint, UniqueConstraint)): for col_name in cons.columns: - if not isinstance(col_name,basestring): + if not isinstance(col_name,six.string_types): col_name = col_name.name if self.name==col_name: to_drop.add(cons) @@ -622,7 +664,7 @@ populated with defaults if (getattr(self, name[:-5]) and not obj): raise InvalidConstraintError("Column.create() accepts index_name," " primary_key_name and unique_name to generate constraints") - if not isinstance(obj, basestring) and obj is not None: + if not isinstance(obj, six.string_types) and obj is not None: raise InvalidConstraintError( "%s argument for column must be constraint name" % name) diff --git a/migrate/tests/__init__.py b/migrate/tests/__init__.py index 803323e..c03fbf4 100644 --- a/migrate/tests/__init__.py +++ b/migrate/tests/__init__.py @@ -6,10 +6,11 @@ sys.path.append(os.path.dirname(os.path.abspath(__file__))) from unittest import TestCase import migrate +import six class TestVersionDefined(TestCase): def test_version(self): """Test for migrate.__version__""" - self.assertTrue(isinstance(migrate.__version__, basestring)) + self.assertTrue(isinstance(migrate.__version__, six.string_types)) self.assertTrue(len(migrate.__version__) > 0) diff --git a/migrate/tests/changeset/test_changeset.py b/migrate/tests/changeset/test_changeset.py index dcbd473..57d0380 100644 --- a/migrate/tests/changeset/test_changeset.py +++ b/migrate/tests/changeset/test_changeset.py @@ -11,6 +11,7 @@ from migrate.changeset import constraint from migrate.changeset.schema import ColumnDelta from migrate.tests import fixture from migrate.tests.fixture.warnings import catch_warnings +import six class TestAddDropColumn(fixture.DB): """Test add/drop column through all possible interfaces @@ -400,7 +401,7 @@ class TestAddDropColumn(fixture.DB): if isinstance(cons,ForeignKeyConstraint): col_names = [] for col_name in cons.columns: - if not isinstance(col_name,basestring): + if not isinstance(col_name,six.string_types): col_name = col_name.name col_names.append(col_name) result.append(col_names) @@ -612,7 +613,7 @@ class TestColumnChange(fixture.DB): self.table.drop() try: self.table.create() - except sqlalchemy.exc.SQLError, e: + except sqlalchemy.exc.SQLError: # SQLite: database schema has changed if not self.url.startswith('sqlite://'): raise @@ -621,7 +622,7 @@ class TestColumnChange(fixture.DB): if self.table.exists(): try: self.table.drop(self.engine) - except sqlalchemy.exc.SQLError,e: + except sqlalchemy.exc.SQLError: # SQLite: database schema has changed if not self.url.startswith('sqlite://'): raise @@ -843,7 +844,7 @@ class TestColumnDelta(fixture.DB): def verify(self, expected, original, *p, **k): self.delta = ColumnDelta(original, *p, **k) - result = self.delta.keys() + result = list(self.delta.keys()) result.sort() self.assertEqual(expected, result) return self.delta diff --git a/migrate/tests/fixture/__init__.py b/migrate/tests/fixture/__init__.py index cfc67b4..6b8bc48 100644 --- a/migrate/tests/fixture/__init__.py +++ b/migrate/tests/fixture/__init__.py @@ -12,7 +12,7 @@ def main(imports=None): defaultTest=None return testtools.TestProgram(defaultTest=defaultTest) -from base import Base -from migrate.tests.fixture.pathed import Pathed -from shell import Shell -from database import DB,usedb +from .base import Base +from .pathed import Pathed +from .shell import Shell +from .database import DB,usedb diff --git a/migrate/tests/fixture/database.py b/migrate/tests/fixture/database.py index 90b25d5..20ca50a 100644 --- a/migrate/tests/fixture/database.py +++ b/migrate/tests/fixture/database.py @@ -3,6 +3,9 @@ import os import logging +import sys + +import six from decorator import decorator from sqlalchemy import create_engine, Table, MetaData @@ -23,7 +26,7 @@ log = logging.getLogger(__name__) def readurls(): """read URLs from config file return a list""" # TODO: remove tmpfile since sqlite can store db in memory - filename = 'test_db.cfg' + filename = 'test_db.cfg' if six.PY2 else "test_db_py3.cfg" ret = list() tmpfile = Pathed.tmp() fullpath = os.path.join(os.curdir, filename) @@ -46,12 +49,12 @@ def is_supported(url, supported, not_supported): db = url.split(':', 1)[0] if supported is not None: - if isinstance(supported, basestring): + if isinstance(supported, six.string_types): return supported == db else: return db in supported elif not_supported is not None: - if isinstance(not_supported, basestring): + if isinstance(not_supported, six.string_types): return not_supported != db else: return not (db in not_supported) @@ -96,7 +99,7 @@ def usedb(supported=None, not_supported=None): finally: try: self._teardown() - except Exception,e: + except Exception as e: teardown_exception=e else: teardown_exception=None @@ -106,14 +109,14 @@ def usedb(supported=None, not_supported=None): 'setup: %r\n' 'teardown: %r\n' )%(setup_exception,teardown_exception)) - except Exception,e: + except Exception: failed_for.append(url) - fail = True + fail = sys.exc_info() for url in failed_for: log.error('Failed for %s', url) if fail: # cause the failure :-) - raise + six.reraise(*fail) return dec diff --git a/migrate/tests/versioning/test_api.py b/migrate/tests/versioning/test_api.py index 4a93c5c..bc4b29d 100644 --- a/migrate/tests/versioning/test_api.py +++ b/migrate/tests/versioning/test_api.py @@ -1,6 +1,8 @@ #!/usr/bin/python # -*- coding: utf-8 -*- +import six + from migrate.exceptions import * from migrate.versioning import api @@ -12,7 +14,7 @@ from migrate.tests import fixture class TestAPI(Pathed): def test_help(self): - self.assertTrue(isinstance(api.help('help'), basestring)) + self.assertTrue(isinstance(api.help('help'), six.string_types)) self.assertRaises(UsageError, api.help) self.assertRaises(UsageError, api.help, 'foobar') self.assertTrue(isinstance(api.help('create'), str)) @@ -48,7 +50,7 @@ class TestAPI(Pathed): repo = self.tmp_repos() api.create(repo, 'temp') api.version_control('sqlite:///', repo) - api.version_control('sqlite:///', unicode(repo)) + api.version_control('sqlite:///', six.text_type(repo)) def test_source(self): repo = self.tmp_repos() diff --git a/migrate/tests/versioning/test_genmodel.py b/migrate/tests/versioning/test_genmodel.py index f7924ff..f800826 100644 --- a/migrate/tests/versioning/test_genmodel.py +++ b/migrate/tests/versioning/test_genmodel.py @@ -2,6 +2,7 @@ import os +import six import sqlalchemy from sqlalchemy import * @@ -43,13 +44,12 @@ class TestSchemaDiff(fixture.DB): # so the schema diffs on the columns don't work with this test. @fixture.usedb(not_supported='ibm_db_sa') def test_functional(self): - def assertDiff(isDiff, tablesMissingInDatabase, tablesMissingInModel, tablesWithDiff): diff = schemadiff.getDiffOfModelAgainstDatabase(self.meta, self.engine, excludeTables=['migrate_version']) self.assertEqual( (diff.tables_missing_from_B, diff.tables_missing_from_A, - diff.tables_different.keys(), + list(diff.tables_different.keys()), bool(diff)), (tablesMissingInDatabase, tablesMissingInModel, @@ -97,10 +97,11 @@ class TestSchemaDiff(fixture.DB): diff = schemadiff.getDiffOfModelAgainstDatabase(MetaData(), self.engine, excludeTables=['migrate_version']) src = genmodel.ModelGenerator(diff,self.engine).genBDefinition() - exec src in locals() + namespace = {} + six.exec_(src, namespace) c1 = Table('tmp_schemadiff', self.meta, autoload=True).c - c2 = tmp_schemadiff.c + c2 = namespace['tmp_schemadiff'].c self.compare_columns_equal(c1, c2, ['type']) # TODO: get rid of ignoring type @@ -139,19 +140,19 @@ class TestSchemaDiff(fixture.DB): decls, upgradeCommands, downgradeCommands = genmodel.ModelGenerator(diff,self.engine).genB2AMigration(indent='') # decls have changed since genBDefinition - exec decls in locals() + six.exec_(decls, namespace) # migration commands expect a namespace containing migrate_engine - migrate_engine = self.engine + namespace['migrate_engine'] = self.engine # run the migration up and down - exec upgradeCommands in locals() + six.exec_(upgradeCommands, namespace) assertDiff(False, [], [], []) - exec decls in locals() - exec downgradeCommands in locals() + six.exec_(decls, namespace) + six.exec_(downgradeCommands, namespace) assertDiff(True, [], [], [self.table_name]) - exec decls in locals() - exec upgradeCommands in locals() + six.exec_(decls, namespace) + six.exec_(upgradeCommands, namespace) assertDiff(False, [], [], []) if not self.engine.name == 'oracle': diff --git a/migrate/tests/versioning/test_repository.py b/migrate/tests/versioning/test_repository.py index 0949b69..6845a0e 100644 --- a/migrate/tests/versioning/test_repository.py +++ b/migrate/tests/versioning/test_repository.py @@ -111,7 +111,6 @@ class TestVersionedRepository(fixture.Pathed): # Create a script and test again now = int(datetime.utcnow().strftime('%Y%m%d%H%M%S')) repos.create_script('') - print repos.latest self.assertEqual(repos.latest, now) def test_source(self): diff --git a/migrate/tests/versioning/test_schema.py b/migrate/tests/versioning/test_schema.py index d92eed3..5396d9d 100644 --- a/migrate/tests/versioning/test_schema.py +++ b/migrate/tests/versioning/test_schema.py @@ -4,6 +4,8 @@ import os import shutil +import six + from migrate import exceptions from migrate.versioning.schema import * from migrate.versioning import script, schemadiff @@ -163,10 +165,10 @@ class TestControlledSchema(fixture.Pathed, fixture.DB): def test_create_model(self): """Test workflow to generate create_model""" model = ControlledSchema.create_model(self.engine, self.repos, declarative=False) - self.assertTrue(isinstance(model, basestring)) + self.assertTrue(isinstance(model, six.string_types)) model = ControlledSchema.create_model(self.engine, self.repos.path, declarative=True) - self.assertTrue(isinstance(model, basestring)) + self.assertTrue(isinstance(model, six.string_types)) @fixture.usedb() def test_compare_model_to_db(self): diff --git a/migrate/tests/versioning/test_schemadiff.py b/migrate/tests/versioning/test_schemadiff.py index ec6d1dc..f45a012 100644 --- a/migrate/tests/versioning/test_schemadiff.py +++ b/migrate/tests/versioning/test_schemadiff.py @@ -27,9 +27,9 @@ class SchemaDiffBase(fixture.DB): # print diff self.assertTrue(diff) self.assertEqual(1,len(diff.tables_different)) - td = diff.tables_different.values()[0] + td = list(diff.tables_different.values())[0] self.assertEqual(1,len(td.columns_different)) - cd = td.columns_different.values()[0] + cd = list(td.columns_different.values())[0] label_width = max(len(self.name1), len(self.name2)) self.assertEqual(('Schema diffs:\n' ' table with differences: xtable\n' diff --git a/migrate/tests/versioning/test_script.py b/migrate/tests/versioning/test_script.py index d30647b..183eb7e 100644 --- a/migrate/tests/versioning/test_script.py +++ b/migrate/tests/versioning/test_script.py @@ -1,10 +1,12 @@ #!/usr/bin/env python # -*- coding: utf-8 -*- +import imp import os import sys import shutil +import six from migrate import exceptions from migrate.versioning import version, repository from migrate.versioning.script import * @@ -51,7 +53,10 @@ class TestPyScript(fixture.Pathed, fixture.DB): self.assertRaises(exceptions.ScriptError, pyscript._func, 'foobar') # clean pyc file - os.remove(script_path + 'c') + if six.PY3: + os.remove(imp.cache_from_source(script_path)) + else: + os.remove(script_path + 'c') # test deprecated upgrade/downgrade with no arguments contents = open(script_path, 'r').read() @@ -94,7 +99,7 @@ class TestPyScript(fixture.Pathed, fixture.DB): path = self.tmp_py() # Create empty file f = open(path, 'w') - f.write("def zergling():\n\tprint 'rush'") + f.write("def zergling():\n\tprint('rush')") f.close() self.assertRaises(exceptions.InvalidScriptError, self.cls.verify_module, path) # script isn't verified on creation, but on module reference diff --git a/migrate/tests/versioning/test_shell.py b/migrate/tests/versioning/test_shell.py index 743828d..62dc8e0 100644 --- a/migrate/tests/versioning/test_shell.py +++ b/migrate/tests/versioning/test_shell.py @@ -5,7 +5,8 @@ import os import sys import tempfile -from cStringIO import StringIO +import six +from six.moves import cStringIO from sqlalchemy import MetaData, Table from migrate.exceptions import * @@ -29,7 +30,7 @@ class TestShellCommands(Shell): # we can only test that we get some output for cmd in api.__all__: result = self.env.run('migrate help %s' % cmd) - self.assertTrue(isinstance(result.stdout, basestring)) + self.assertTrue(isinstance(result.stdout, six.string_types)) self.assertTrue(result.stdout) self.assertFalse(result.stderr) @@ -61,11 +62,11 @@ class TestShellCommands(Shell): def _check_error(self,args,code,expected,**kw): original = sys.stderr try: - actual = StringIO() + actual = cStringIO() sys.stderr = actual try: shell.main(args,**kw) - except SystemExit, e: + except SystemExit as e: self.assertEqual(code,e.args[0]) else: self.fail('No exception raised') @@ -502,7 +503,7 @@ class TestShellDatabase(Shell, DB): result = self.env.run('migrate create_model %s %s' % (self.url, repos_path)) temp_dict = dict() - exec result.stdout in temp_dict + six.exec_(result.stdout, temp_dict) # TODO: breaks on SA06 and SA05 - in need of total refactor - use different approach diff --git a/migrate/versioning/cfgparse.py b/migrate/versioning/cfgparse.py index ff27d67..8f1ccf9 100644 --- a/migrate/versioning/cfgparse.py +++ b/migrate/versioning/cfgparse.py @@ -2,7 +2,7 @@ Configuration parser module. """ -from ConfigParser import ConfigParser +from six.moves.configparser import ConfigParser from migrate.versioning.config import * from migrate.versioning import pathed diff --git a/migrate/versioning/genmodel.py b/migrate/versioning/genmodel.py index efff67f..4d9cd12 100644 --- a/migrate/versioning/genmodel.py +++ b/migrate/versioning/genmodel.py @@ -9,6 +9,7 @@ http://code.google.com/p/sqlautocode/ import sys import logging +import six import sqlalchemy import migrate @@ -68,7 +69,10 @@ class ModelGenerator(object): # crs: not sure if this is good idea, but it gets rid of extra # u'' - name = col.name.encode('utf8') + if six.PY3: + name = col.name + else: + name = col.name.encode('utf8') type_ = col.type for cls in col.type.__class__.__mro__: @@ -192,7 +196,7 @@ class ModelGenerator(object): downgradeCommands.append( "post_meta.tables[%(table)r].drop()" % {'table': tn}) - for (tn, td) in self.diff.tables_different.iteritems(): + for (tn, td) in six.iteritems(self.diff.tables_different): if td.columns_missing_from_A or td.columns_different: pre_table = self.diff.metadataB.tables[tn] decls.extend(self._getTableDefn( diff --git a/migrate/versioning/repository.py b/migrate/versioning/repository.py index 82aa271..b317eda 100644 --- a/migrate/versioning/repository.py +++ b/migrate/versioning/repository.py @@ -43,7 +43,7 @@ class Changeset(dict): """ In a series of upgrades x -> y, keys are version x. Sorted. """ - ret = super(Changeset, self).keys() + ret = list(super(Changeset, self).keys()) # Reverse order if downgrading ret.sort(reverse=(self.step < 1)) return ret @@ -94,7 +94,7 @@ class Repository(pathed.Pathed): cls.require_found(path) cls.require_found(os.path.join(path, cls._config)) cls.require_found(os.path.join(path, cls._versions)) - except exceptions.PathNotFoundError, e: + except exceptions.PathNotFoundError: raise exceptions.InvalidRepositoryError(path) @classmethod @@ -221,7 +221,7 @@ class Repository(pathed.Pathed): range_mod = 0 op = 'downgrade' - versions = range(start + range_mod, end + range_mod, step) + versions = range(int(start) + range_mod, int(end) + range_mod, step) changes = [self.version(v).script(database, op) for v in versions] ret = Changeset(start, step=step, *changes) return ret diff --git a/migrate/versioning/schema.py b/migrate/versioning/schema.py index 0e95b0d..b525cef 100644 --- a/migrate/versioning/schema.py +++ b/migrate/versioning/schema.py @@ -4,6 +4,7 @@ import sys import logging +import six from sqlalchemy import (Table, Column, MetaData, String, Text, Integer, create_engine) from sqlalchemy.sql import and_ @@ -24,7 +25,7 @@ class ControlledSchema(object): """A database under version control""" def __init__(self, engine, repository): - if isinstance(repository, basestring): + if isinstance(repository, six.string_types): repository = Repository(repository) self.engine = engine self.repository = repository @@ -49,7 +50,8 @@ class ControlledSchema(object): data = list(result)[0] except: cls, exc, tb = sys.exc_info() - raise exceptions.DatabaseNotControlledError, exc.__str__(), tb + six.reraise(exceptions.DatabaseNotControlledError, + exceptions.DatabaseNotControlledError(str(exc)), tb) self.version = data['version'] return data @@ -133,7 +135,7 @@ class ControlledSchema(object): """ # Confirm that the version # is valid: positive, integer, # exists in repos - if isinstance(repository, basestring): + if isinstance(repository, six.string_types): repository = Repository(repository) version = cls._validate_version(repository, version) table = cls._create_table_version(engine, repository, version) @@ -198,7 +200,7 @@ class ControlledSchema(object): """ Compare the current model against the current database. """ - if isinstance(repository, basestring): + if isinstance(repository, six.string_types): repository = Repository(repository) model = load_model(model) @@ -211,7 +213,7 @@ class ControlledSchema(object): """ Dump the current database as a Python model. """ - if isinstance(repository, basestring): + if isinstance(repository, six.string_types): repository = Repository(repository) diff = schemadiff.getDiffOfModelAgainstDatabase( diff --git a/migrate/versioning/schemadiff.py b/migrate/versioning/schemadiff.py index 689703b..d9477bf 100644 --- a/migrate/versioning/schemadiff.py +++ b/migrate/versioning/schemadiff.py @@ -99,6 +99,9 @@ class ColDiff(object): def __nonzero__(self): return self.diff + __bool__ = __nonzero__ + + class TableDiff(object): """ Container for differences in one :class:`~sqlalchemy.schema.Table` @@ -135,6 +138,8 @@ class TableDiff(object): self.columns_different ) + __bool__ = __nonzero__ + class SchemaDiff(object): """ Compute the difference between two :class:`~sqlalchemy.schema.MetaData` diff --git a/migrate/versioning/script/py.py b/migrate/versioning/script/py.py index 3a090d4..92a8f6b 100644 --- a/migrate/versioning/script/py.py +++ b/migrate/versioning/script/py.py @@ -5,7 +5,6 @@ import shutil import warnings import logging import inspect -from StringIO import StringIO import migrate from migrate.versioning import genmodel, schemadiff @@ -14,6 +13,8 @@ from migrate.versioning.template import Template from migrate.versioning.script import base from migrate.versioning.util import import_path, load_model, with_engine from migrate.exceptions import MigrateDeprecationWarning, InvalidScriptError, ScriptError +import six +from six.moves import StringIO log = logging.getLogger(__name__) __all__ = ['PythonScript'] @@ -51,7 +52,7 @@ class PythonScript(base.BaseScript): :rtype: string """ - if isinstance(repository, basestring): + if isinstance(repository, six.string_types): # oh dear, an import cycle! from migrate.versioning.repository import Repository repository = Repository(repository) @@ -96,7 +97,7 @@ class PythonScript(base.BaseScript): module = import_path(path) try: assert callable(module.upgrade) - except Exception, e: + except Exception as e: raise InvalidScriptError(path + ': %s' % str(e)) return module @@ -127,7 +128,9 @@ class PythonScript(base.BaseScript): :type engine: string :type step: int """ - if step > 0: + if step in ('downgrade', 'upgrade'): + op = step + elif step > 0: op = 'upgrade' elif step < 0: op = 'downgrade' diff --git a/migrate/versioning/shell.py b/migrate/versioning/shell.py index ad7b679..5fb86b1 100644 --- a/migrate/versioning/shell.py +++ b/migrate/versioning/shell.py @@ -12,6 +12,7 @@ from migrate import exceptions from migrate.versioning import api from migrate.versioning.config import * from migrate.versioning.util import asbool +import six alias = dict( @@ -23,7 +24,7 @@ alias = dict( def alias_setup(): global alias - for key, val in alias.iteritems(): + for key, val in six.iteritems(alias): setattr(api, key, val) alias_setup() @@ -135,7 +136,7 @@ def main(argv=None, **kwargs): override_kwargs[opt] = value # override kwargs with options if user is overwriting - for key, value in options.__dict__.iteritems(): + for key, value in six.iteritems(options.__dict__): if value is not None: override_kwargs[key] = value @@ -143,7 +144,7 @@ def main(argv=None, **kwargs): f_required = list(f_args) candidates = dict(kwargs) candidates.update(override_kwargs) - for key, value in candidates.iteritems(): + for key, value in six.iteritems(candidates): if key in f_args: f_required.remove(key) @@ -160,7 +161,7 @@ def main(argv=None, **kwargs): kwargs.update(override_kwargs) # configure options - for key, value in options.__dict__.iteritems(): + for key, value in six.iteritems(options.__dict__): kwargs.setdefault(key, value) # configure logging @@ -198,6 +199,7 @@ def main(argv=None, **kwargs): num_defaults = 0 f_args_default = f_args[len(f_args) - num_defaults:] required = list(set(f_required) - set(f_args_default)) + required.sort() if required: parser.error("Not enough arguments for command %s: %s not specified" \ % (command, ', '.join(required))) @@ -207,7 +209,7 @@ def main(argv=None, **kwargs): ret = command_func(**kwargs) if ret is not None: log.info(ret) - except (exceptions.UsageError, exceptions.KnownError), e: + except (exceptions.UsageError, exceptions.KnownError) as e: parser.error(e.args[0]) if __name__ == "__main__": diff --git a/migrate/versioning/templates/manage/default.py_tmpl b/migrate/versioning/templates/manage/default.py_tmpl index f6d75c5..e72097a 100644 --- a/migrate/versioning/templates/manage/default.py_tmpl +++ b/migrate/versioning/templates/manage/default.py_tmpl @@ -2,10 +2,11 @@ from migrate.versioning.shell import main {{py: +import six _vars = locals().copy() del _vars['__template_name__'] _vars.pop('repository_name', None) -defaults = ", ".join(["%s='%s'" % var for var in _vars.iteritems()]) +defaults = ", ".join(["%s='%s'" % var for var in six.iteritems(_vars)]) }} if __name__ == '__main__': diff --git a/migrate/versioning/templates/manage/pylons.py_tmpl b/migrate/versioning/templates/manage/pylons.py_tmpl index cc2f788..ccaac05 100644 --- a/migrate/versioning/templates/manage/pylons.py_tmpl +++ b/migrate/versioning/templates/manage/pylons.py_tmpl @@ -17,9 +17,10 @@ else: conf_path = 'development.ini' {{py: +import six _vars = locals().copy() del _vars['__template_name__'] -defaults = ", ".join(["%s='%s'" % var for var in _vars.iteritems()]) +defaults = ", ".join(["%s='%s'" % var for var in six.iteritems(_vars)]) }} conf_dict = ConfigLoader(conf_path).parser._sections['app:main'] diff --git a/migrate/versioning/util/__init__.py b/migrate/versioning/util/__init__.py index 34ec5b2..a4ddd73 100644 --- a/migrate/versioning/util/__init__.py +++ b/migrate/versioning/util/__init__.py @@ -7,6 +7,7 @@ import logging from decorator import decorator from pkg_resources import EntryPoint +import six from sqlalchemy import create_engine from sqlalchemy.engine import Engine from sqlalchemy.pool import StaticPool @@ -26,7 +27,7 @@ def load_model(dotted_name): .. versionchanged:: 0.5.4 """ - if isinstance(dotted_name, basestring): + if isinstance(dotted_name, six.string_types): if ':' not in dotted_name: # backwards compatibility warnings.warn('model should be in form of module.model:User ' @@ -39,7 +40,7 @@ def load_model(dotted_name): def asbool(obj): """Do everything to use object as bool""" - if isinstance(obj, basestring): + if isinstance(obj, six.string_types): obj = obj.strip().lower() if obj in ['true', 'yes', 'on', 'y', 't', '1']: return True @@ -87,7 +88,7 @@ def catch_known_errors(f, *a, **kw): try: return f(*a, **kw) - except exceptions.PathFoundError, e: + except exceptions.PathFoundError as e: raise exceptions.KnownError("The path %s already exists" % e.args[0]) def construct_engine(engine, **opts): @@ -112,7 +113,7 @@ def construct_engine(engine, **opts): """ if isinstance(engine, Engine): return engine - elif not isinstance(engine, basestring): + elif not isinstance(engine, six.string_types): raise ValueError("you need to pass either an existing engine or a database uri") # get options for create_engine @@ -130,7 +131,7 @@ def construct_engine(engine, **opts): kwargs['echo'] = echo # parse keyword arguments - for key, value in opts.iteritems(): + for key, value in six.iteritems(opts): if key.startswith('engine_arg_'): kwargs[key[11:]] = guess_obj_type(value) @@ -174,6 +175,6 @@ class Memoize: self.memo = {} def __call__(self, *args): - if not self.memo.has_key(args): + if args not in self.memo: self.memo[args] = self.fn(*args) return self.memo[args] diff --git a/migrate/versioning/util/importpath.py b/migrate/versioning/util/importpath.py index 0b398e1..5ab7128 100644 --- a/migrate/versioning/util/importpath.py +++ b/migrate/versioning/util/importpath.py @@ -1,6 +1,8 @@ import os import sys +from six.moves import reload_module as reload + def import_path(fullpath): """ Import a file with full path specification. Allows one to import from anywhere, something __import__ does not do. diff --git a/migrate/versioning/version.py b/migrate/versioning/version.py index 37dfbb9..cec75c0 100644 --- a/migrate/versioning/version.py +++ b/migrate/versioning/version.py @@ -9,6 +9,7 @@ import logging from migrate import exceptions from migrate.versioning import pathed, script from datetime import datetime +import six log = logging.getLogger(__name__) @@ -64,6 +65,10 @@ class VerNum(object): def __int__(self): return int(self.value) + if six.PY3: + def __hash__(self): + return hash(self.value) + class Collection(pathed.Pathed): """A collection of versioning scripts in a repository""" @@ -102,7 +107,7 @@ class Collection(pathed.Pathed): @property def latest(self): """:returns: Latest version in Collection""" - return max([VerNum(0)] + self.versions.keys()) + return max([VerNum(0)] + list(self.versions.keys())) def _next_ver_num(self, use_timestamp_numbering): if use_timestamp_numbering == True: diff --git a/test-requirements-py2.txt b/test-requirements-py2.txt new file mode 100644 index 0000000..ef53025 --- /dev/null +++ b/test-requirements-py2.txt @@ -0,0 +1,2 @@ +ibm_db_sa>=0.3.0 +MySQL-python diff --git a/test-requirements-py3.txt b/test-requirements-py3.txt new file mode 100644 index 0000000..4a06ca2 --- /dev/null +++ b/test-requirements-py3.txt @@ -0,0 +1 @@ +ibm-db-sa-py3 diff --git a/test-requirements.txt b/test-requirements.txt index a035d55..c22f516 100644 --- a/test-requirements.txt +++ b/test-requirements.txt @@ -8,21 +8,17 @@ coverage>=3.6 discover feedparser fixtures>=0.3.14 -ibm_db_sa>=0.3.0 mock>=1.0 mox>=0.5.3 -MySQL-python psycopg2 -pylint==0.25.2 python-subunit>=0.0.18 sphinx>=1.1.2,<1.2 sphinxcontrib_issuetracker testrepository>=0.0.17 testtools>=0.9.34 -# NOTE: scripttest 1.0.1 removes base_path argument to ScriptTest -scripttest==1.0 +scripttest # NOTE(rpodolyaka): This version identifier is currently necessary as # pytz otherwise does not install on pip 1.4 or higher +pylint pytz>=2010h -pysqlite diff --git a/test_db_py3.cfg b/test_db_py3.cfg new file mode 100644 index 0000000..e962fc5 --- /dev/null +++ b/test_db_py3.cfg @@ -0,0 +1,15 @@ +# test_db.cfg +# +# This file contains a list of connection strings which will be used by +# database tests. Tests will be executed once for each string in this file. +# You should be sure that the database used for the test doesn't contain any +# important data. See README for more information. +# +# The string '__tmp__' is substituted for a temporary file in each connection +# string. This is useful for sqlite tests. +sqlite:///__tmp__ +#postgresql://openstack_citest:openstack_citest@localhost/openstack_citest +#mysql://openstack_citest:openstack_citest@localhost/openstack_citest +#oracle://scott:tiger@localhost +#firebird://scott:tiger@localhost//var/lib/firebird/databases/test_migrate +#ibm_db_sa://migrate:migrate@localhost:50000/migrate diff --git a/tox.ini b/tox.ini index e247ac1..7288937 100644 --- a/tox.ini +++ b/tox.ini @@ -15,40 +15,53 @@ commands = [testenv:py26] deps = sqlalchemy>=0.9 -r{toxinidir}/test-requirements.txt + -r{toxinidir}/test-requirements-py2.txt [testenv:py27] deps = sqlalchemy>=0.9 -r{toxinidir}/test-requirements.txt + -r{toxinidir}/test-requirements-py2.txt [testenv:py26sa07] basepython = python2.6 deps = sqlalchemy>=0.7,<=0.7.99 -r{toxinidir}/test-requirements.txt + -r{toxinidir}/test-requirements-py2.txt [testenv:py26sa08] basepython = python2.6 deps = sqlalchemy>=0.8,<=0.8.99 -r{toxinidir}/test-requirements.txt + -r{toxinidir}/test-requirements-py2.txt [testenv:py26sa09] basepython = python2.6 deps = sqlalchemy>=0.9,<=0.9.99 -r{toxinidir}/test-requirements.txt + -r{toxinidir}/test-requirements-py2.txt [testenv:py27sa07] basepython = python2.7 deps = sqlalchemy>=0.7,<=0.7.99 -r{toxinidir}/test-requirements.txt + -r{toxinidir}/test-requirements-py2.txt [testenv:py27sa08] basepython = python2.7 deps = sqlalchemy>=0.8,<=0.8.99 -r{toxinidir}/test-requirements.txt + -r{toxinidir}/test-requirements-py2.txt [testenv:py27sa09] basepython = python2.7 deps = sqlalchemy>=0.9,<=0.9.99 -r{toxinidir}/test-requirements.txt + -r{toxinidir}/test-requirements-py2.txt + +[testenv:py33] +deps = sqlalchemy>=0.9 + -r{toxinidir}/test-requirements.txt + -r{toxinidir}/test-requirements-py3.txt [testenv:pep8] commands = flake8