From d1391f7fa0782336394c2de33a3eb9f5f24ce375 Mon Sep 17 00:00:00 2001 From: Julien Danjou Date: Mon, 19 Sep 2016 17:34:51 +0200 Subject: [PATCH] sqlalchemy: use DATETIME(fsp=6) rather than DECIMAL This migrates the old DECIMAL based format to the new DATETIME format available in recent versions of MySQL. Change-Id: I5dc7a7c2586feec72a1a2b13865e353a844a1785 --- .../versions/12fe8fac9fe4_initial_base.py | 21 ++++- ...7aadf5485f_precisetimestamp_to_datetime.py | 68 +++++++++++++++ aodh/storage/sqlalchemy/models.py | 51 ++--------- .../storage/sqlalchemy/test_models.py | 86 ------------------- ...sql-precise-datetime-e374c77e6707985e.yaml | 4 + 5 files changed, 97 insertions(+), 133 deletions(-) create mode 100644 aodh/storage/sqlalchemy/alembic/versions/367aadf5485f_precisetimestamp_to_datetime.py delete mode 100644 aodh/tests/functional/storage/sqlalchemy/test_models.py create mode 100644 releasenotes/notes/mysql-precise-datetime-e374c77e6707985e.yaml diff --git a/aodh/storage/sqlalchemy/alembic/versions/12fe8fac9fe4_initial_base.py b/aodh/storage/sqlalchemy/alembic/versions/12fe8fac9fe4_initial_base.py index dbff9e0b7..7a2ecbfed 100644 --- a/aodh/storage/sqlalchemy/alembic/versions/12fe8fac9fe4_initial_base.py +++ b/aodh/storage/sqlalchemy/alembic/versions/12fe8fac9fe4_initial_base.py @@ -29,10 +29,25 @@ depends_on = None from alembic import op import sqlalchemy as sa +from sqlalchemy import types import aodh.storage.sqlalchemy.models +class PreciseTimestamp(types.TypeDecorator): + """Represents a timestamp precise to the microsecond.""" + + impl = sa.DateTime + + def load_dialect_impl(self, dialect): + if dialect.name == 'mysql': + return dialect.type_descriptor( + types.DECIMAL(precision=20, + scale=6, + asdecimal=True)) + return dialect.type_descriptor(self.impl) + + def upgrade(): op.create_table( 'alarm_history', @@ -44,7 +59,7 @@ def upgrade(): sa.Column('type', sa.String(length=20), nullable=True), sa.Column('detail', sa.Text(), nullable=True), sa.Column('timestamp', - aodh.storage.sqlalchemy.models.PreciseTimestamp(), + PreciseTimestamp(), nullable=True), sa.PrimaryKeyConstraint('event_id') ) @@ -60,13 +75,13 @@ def upgrade(): sa.Column('severity', sa.String(length=50), nullable=True), sa.Column('description', sa.Text(), nullable=True), sa.Column('timestamp', - aodh.storage.sqlalchemy.models.PreciseTimestamp(), + PreciseTimestamp(), nullable=True), sa.Column('user_id', sa.String(length=128), nullable=True), sa.Column('project_id', sa.String(length=128), nullable=True), sa.Column('state', sa.String(length=255), nullable=True), sa.Column('state_timestamp', - aodh.storage.sqlalchemy.models.PreciseTimestamp(), + PreciseTimestamp(), nullable=True), sa.Column('ok_actions', aodh.storage.sqlalchemy.models.JSONEncodedDict(), diff --git a/aodh/storage/sqlalchemy/alembic/versions/367aadf5485f_precisetimestamp_to_datetime.py b/aodh/storage/sqlalchemy/alembic/versions/367aadf5485f_precisetimestamp_to_datetime.py new file mode 100644 index 000000000..0b165f0f5 --- /dev/null +++ b/aodh/storage/sqlalchemy/alembic/versions/367aadf5485f_precisetimestamp_to_datetime.py @@ -0,0 +1,68 @@ +# -*- encoding: utf-8 -*- +# +# Copyright 2016 OpenStack Foundation +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. +# + +"""precisetimestamp_to_datetime + +Revision ID: 367aadf5485f +Revises: f8c31b1ffe11 +Create Date: 2016-09-19 16:43:34.379029 + +""" + +# revision identifiers, used by Alembic. +revision = '367aadf5485f' +down_revision = 'f8c31b1ffe11' +branch_labels = None +depends_on = None + +from alembic import op +import sqlalchemy as sa +from sqlalchemy import func + +from aodh.storage.sqlalchemy import models + + +def upgrade(): + bind = op.get_bind() + if bind and bind.engine.name == "mysql": + # NOTE(jd) So that crappy engine that is MySQL does not have "ALTER + # TABLE … USING …". We need to copy everything and convert… + for table_name, column_name in (("alarm", "timestamp"), + ("alarm", "state_timestamp"), + ("alarm_change", "timestamp")): + existing_type = sa.types.DECIMAL( + precision=20, scale=6, asdecimal=True) + existing_col = sa.Column( + column_name, + existing_type, + nullable=True) + temp_col = sa.Column( + column_name + "_ts", + models.TimestampUTC(), + nullable=True) + op.add_column(table_name, temp_col) + t = sa.sql.table(table_name, existing_col, temp_col) + op.execute(t.update().values( + **{column_name + "_ts": func.from_unixtime(existing_col)})) + op.drop_column(table_name, column_name) + op.alter_column(table_name, + column_name + "_ts", + nullable=True, + type_=models.TimestampUTC(), + existing_nullable=True, + existing_type=existing_type, + new_column_name=column_name) diff --git a/aodh/storage/sqlalchemy/models.py b/aodh/storage/sqlalchemy/models.py index 5ba314c2c..829b1d8a3 100644 --- a/aodh/storage/sqlalchemy/models.py +++ b/aodh/storage/sqlalchemy/models.py @@ -13,16 +13,12 @@ """ SQLAlchemy models for aodh data. """ -import calendar -import datetime -import decimal import json from oslo_utils import timeutils -from oslo_utils import units import six from sqlalchemy import Column, String, Index, Boolean, Text, DateTime -from sqlalchemy.dialects.mysql import DECIMAL +from sqlalchemy.dialects import mysql from sqlalchemy.ext.declarative import declarative_base from sqlalchemy.types import TypeDecorator @@ -45,48 +41,15 @@ class JSONEncodedDict(TypeDecorator): return value -class PreciseTimestamp(TypeDecorator): +class TimestampUTC(TypeDecorator): """Represents a timestamp precise to the microsecond.""" impl = DateTime def load_dialect_impl(self, dialect): if dialect.name == 'mysql': - return dialect.type_descriptor(DECIMAL(precision=20, - scale=6, - asdecimal=True)) - return dialect.type_descriptor(self.impl) - - @staticmethod - def process_bind_param(value, dialect): - if value is None: - return value - elif dialect.name == 'mysql': - decimal.getcontext().prec = 30 - return ( - decimal.Decimal( - str(calendar.timegm(value.utctimetuple()))) + - (decimal.Decimal(str(value.microsecond)) / - decimal.Decimal("1000000.0")) - ) - return value - - def compare_against_backend(self, dialect, conn_type): - if dialect.name == 'mysql': - return issubclass(type(conn_type), DECIMAL) - return issubclass(type(conn_type), DateTime) - - @staticmethod - def process_result_value(value, dialect): - if value is None: - return value - elif dialect.name == 'mysql': - integer = int(value) - micro = (value - - decimal.Decimal(integer)) * decimal.Decimal(units.M) - daittyme = datetime.datetime.utcfromtimestamp(integer) - return daittyme.replace(microsecond=int(round(micro))) - return value + return dialect.type_descriptor(mysql.DATETIME(fsp=6)) + return self.impl class AodhBase(object): @@ -125,13 +88,13 @@ class Alarm(Base): type = Column(String(50)) severity = Column(String(50)) description = Column(Text) - timestamp = Column(PreciseTimestamp, default=lambda: timeutils.utcnow()) + timestamp = Column(TimestampUTC, default=lambda: timeutils.utcnow()) user_id = Column(String(128)) project_id = Column(String(128)) state = Column(String(255)) - state_timestamp = Column(PreciseTimestamp, + state_timestamp = Column(TimestampUTC, default=lambda: timeutils.utcnow()) ok_actions = Column(JSONEncodedDict) @@ -156,5 +119,5 @@ class AlarmChange(Base): user_id = Column(String(128)) type = Column(String(20)) detail = Column(Text) - timestamp = Column(PreciseTimestamp, default=lambda: timeutils.utcnow()) + timestamp = Column(TimestampUTC, default=lambda: timeutils.utcnow()) severity = Column(String(50)) diff --git a/aodh/tests/functional/storage/sqlalchemy/test_models.py b/aodh/tests/functional/storage/sqlalchemy/test_models.py deleted file mode 100644 index 9af79eb57..000000000 --- a/aodh/tests/functional/storage/sqlalchemy/test_models.py +++ /dev/null @@ -1,86 +0,0 @@ -# -# Copyright 2013 Rackspace Hosting -# -# Licensed under the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. You may obtain -# a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations -# under the License. - -import datetime - -import mock -from oslotest import base -import sqlalchemy -from sqlalchemy.dialects.mysql import DECIMAL -from sqlalchemy.types import NUMERIC - -from aodh.storage.sqlalchemy import models - - -class PreciseTimestampTest(base.BaseTestCase): - - @staticmethod - def fake_dialect(name): - def _type_descriptor_mock(desc): - if type(desc) == DECIMAL: - return NUMERIC(precision=desc.precision, scale=desc.scale) - else: - return desc - dialect = mock.MagicMock() - dialect.name = name - dialect.type_descriptor = _type_descriptor_mock - return dialect - - def setUp(self): - super(PreciseTimestampTest, self).setUp() - self._mysql_dialect = self.fake_dialect('mysql') - self._postgres_dialect = self.fake_dialect('postgres') - self._type = models.PreciseTimestamp() - self._date = datetime.datetime(2012, 7, 2, 10, 44) - - def test_load_dialect_impl_mysql(self): - result = self._type.load_dialect_impl(self._mysql_dialect) - self.assertEqual(NUMERIC, type(result)) - self.assertEqual(20, result.precision) - self.assertEqual(6, result.scale) - self.assertTrue(result.asdecimal) - - def test_load_dialect_impl_postgres(self): - result = self._type.load_dialect_impl(self._postgres_dialect) - self.assertEqual(sqlalchemy.DateTime, type(result)) - - def test_process_bind_param_store_datetime_postgres(self): - result = self._type.process_bind_param(self._date, - self._postgres_dialect) - self.assertEqual(self._date, result) - - def test_process_bind_param_store_none_mysql(self): - result = self._type.process_bind_param(None, self._mysql_dialect) - self.assertIsNone(result) - - def test_process_bind_param_store_none_postgres(self): - result = self._type.process_bind_param(None, - self._postgres_dialect) - self.assertIsNone(result) - - def test_process_result_value_datetime_postgres(self): - result = self._type.process_result_value(self._date, - self._postgres_dialect) - self.assertEqual(self._date, result) - - def test_process_result_value_none_mysql(self): - result = self._type.process_result_value(None, - self._mysql_dialect) - self.assertIsNone(result) - - def test_process_result_value_none_postgres(self): - result = self._type.process_result_value(None, - self._postgres_dialect) - self.assertIsNone(result) diff --git a/releasenotes/notes/mysql-precise-datetime-e374c77e6707985e.yaml b/releasenotes/notes/mysql-precise-datetime-e374c77e6707985e.yaml new file mode 100644 index 000000000..0f3aa5388 --- /dev/null +++ b/releasenotes/notes/mysql-precise-datetime-e374c77e6707985e.yaml @@ -0,0 +1,4 @@ +--- +other: + - Aodh now leverages microseconds timestamps available since MySQL 5.6.4, + meaning it is now the minimum required version of MySQL.