Move to MySQL 5.7 and SQLAlchemy>=1.1.0

In an effort to support JSON Path variable
queries, we need to:
* Pin to MySQL 5.7
* Use JSON column type
* Use a newer SQLAlchemy library.

Change-Id: I98ae021c9cd9755f51e67e56135db164e8ae1f50
Partial-Bug: 1671116
Depends-On: I36b7ae9cf68aeef0e5dbcb1f17db52b81efd6ffe

Dockerfile

@@ -41,11 +41,17 @@ ADD https://bootstrap.pypa.io/get-pip.py /root/get-pip.py
 # Install pip
 RUN python3.5 /root/get-pip.py
 
-# Install Mariadb
-RUN apt-get install -y mariadb-server mariadb-client
+# Install MySQL 5.7
+ENV MYSQL_ROOTPW root
+RUN echo "mysql-server mysql-server/root_password password $MYSQL_ROOTPW" | debconf-set-selections && \
+    echo "mysql-server mysql-server/root_password_again password $MYSQL_ROOTPW" | debconf-set-selections
+RUN apt-get install -y mysql-server-5.7 mysql-client-5.7
+RUN service mysql start && \
+    mysqladmin -u root -p"$MYSQL_ROOTPW" password '' && \
+    service mysql stop
 
 # Change mysql bind address
-RUN sed -i -e"s/^bind-address\s*=\s*127.0.0.1/bind-address = 0.0.0.0/" /etc/mysql/mariadb.conf.d/50-server.cnf
+RUN sed -i -e"s/^bind-address\s*=\s*127.0.0.1/bind-address = 0.0.0.0/" /etc/mysql/mysql.conf.d/mysqld.cnf
 
 # Install MySQL-python
 RUN apt-get install -y libmysqlclient-dev python-mysqldb

craton/db/sqlalchemy/alembic/versions/ffdc1a500db1_craton_inventory_init.py

@@ -32,7 +32,7 @@ def upgrade():
         sa.Column('updated_at', sa.DateTime, nullable=True),
         sa.Column('association_id', sa.Integer),
         sa.Column('key_', sa.String(length=255), nullable=False),
-        sa.Column('value_', sqlalchemy_utils.types.json.JSONType,
+        sa.Column('value_', sa.JSON,
                   nullable=True),
         sa.PrimaryKeyConstraint('association_id', 'key_'),
         sa.ForeignKeyConstraint(
@@ -118,7 +118,7 @@ def upgrade():
         sa.Column('api_key', sa.String(length=36), nullable=True),
         sa.Column('is_root', sa.Boolean, nullable=True),
         sa.Column('is_admin', sa.Boolean, nullable=True),
-        sa.Column('roles', sqlalchemy_utils.types.json.JSONType,
+        sa.Column('roles', sa.JSON,
                   nullable=True),
         sa.PrimaryKeyConstraint('id'),
         sa.UniqueConstraint(
@@ -263,7 +263,7 @@ def upgrade():
             primary_key=True),
         sa.Column('model_name', sa.String(length=255), nullable=True),
         sa.Column('os_version', sa.String(length=255), nullable=True),
-        sa.Column('vlans', sqlalchemy_utils.types.json.JSONType,
+        sa.Column('vlans', sa.JSON,
                   nullable=True)
     )
     op.create_table(

craton/db/sqlalchemy/models.py

@@ -19,14 +19,13 @@ import itertools
 
 from oslo_db.sqlalchemy import models
 from sqlalchemy import (
-    Boolean, Column, ForeignKey, Integer, String, Text, UniqueConstraint)
+    Boolean, Column, ForeignKey, Integer, String, Text, UniqueConstraint, JSON)
 from sqlalchemy.ext.associationproxy import association_proxy
 from sqlalchemy.ext.declarative import declarative_base, declared_attr
 from sqlalchemy.ext.declarative.api import _declarative_constructor
 from sqlalchemy.orm import backref, object_mapper, relationship, validates
 from sqlalchemy.orm.collections import attribute_mapped_collection
 from sqlalchemy_utils.types.ip_address import IPAddressType
-from sqlalchemy_utils.types.json import JSONType
 from sqlalchemy_utils.types.uuid import UUIDType
 
 from craton import exceptions
@@ -110,7 +109,7 @@ class Variable(Base):
     # MySQL.  This difference in naming is only visible in the use of
     # raw SQL.
     key = Column('key_', String(255), primary_key=True)
-    value = Column('value_', JSONType)
+    value = Column('value_', JSON)
     association = relationship(
         VariableAssociation, back_populates='variables',
     )
@@ -263,7 +262,7 @@ class User(Base, VariableMixin):
     is_root = Column(Boolean, default=False)
     # admin = project context admin
     is_admin = Column(Boolean, default=False)
-    roles = Column(JSONType)
+    roles = Column(JSON)
 
     project = relationship('Project', back_populates='users')
 
@@ -529,7 +528,7 @@ class NetworkDevice(Device):
     # network device specific properties
     model_name = Column(String(255), nullable=True)
     os_version = Column(String(255), nullable=True)
-    vlans = Column(JSONType)
+    vlans = Column(JSON)
 
     __mapper_args__ = {
         'polymorphic_identity': 'network_devices',

doc/source/dev/install.rst

@@ -8,6 +8,8 @@ Installation
 
 .. note:: *This is a Python3 project.*
 
+.. note:: *This project requires MySQL 5.7, until a stable release of MariaDB with JSON function support is available*
+
 ---------------------
 Ubuntu 16.04 (Xenial)
 ---------------------

requirements.txt

@@ -19,7 +19,7 @@ oslo.utils>=3.20.0 # Apache-2.0
 PasteDeploy>=1.5.0 # MIT
 Paste # MIT
 pbr>=2.0.0 # Apache-2.0
-SQLAlchemy<1.1.0,>=1.0.10 # MIT
+SQLAlchemy>=1.1.0
 SQLAlchemy-Utils # BSD License
 PyMySQL>=0.7.6 # MIT License
 stevedore>=1.20.0 # Apache-2.0

test-requirements.txt

@@ -9,6 +9,7 @@ Flask-RESTful>=0.3.5 # BSD
 fixtures>=3.0.0 # Apache-2.0/BSD
 jsonschema!=2.5.0,<3.0.0,>=2.0.0 # MIT
 nose # LGPL
+nose-exclude  # LGPL
 mock>=2.0 # BSD
 oslotest>=1.10.0 # Apache-2.0
 sphinx>=1.5.1 # BSD

tox.ini

@@ -16,12 +16,13 @@ whitelist_externals = find
 [testenv:py35]
 commands =
     {[testenv]commands}
-    nosetests -v --where={toxinidir}/craton/tests/unit
+    # NOTE(thomasem): Exclude DB unit tests, since SQLite no longer works. Will forklift to functional with real DB.
+    nosetests -v --where={toxinidir}/craton/tests/unit --exclude-dir {toxinidir}/craton/tests/unit/db
 
 [testenv:functional]
 commands =
     {[testenv]commands}
-    nosetests -v []
+    nosetests -v --exclude-dir {toxinidir}/craton/tests/unit/db []
 
 [testenv:pep8]
 commands = flake8 {posargs}