Modified utils to implement all time calculations

Added support for oslo.utils.
Added tests for time calculations functions.
Modified code to reflect changes and remove direct usages of timeutils
and datetime.
All calculations are now enforced on UTC datetimes or timestamps.

Change-Id: I2e8add7e9d619c81cdeb25a066ea4e0bf8327792

cloudkitty/collector/__init__.py

@@ -16,11 +16,10 @@
 # @author: Stéphane Albert
 #
 import abc
-import datetime
 
 import six
 
-import cloudkitty.utils as utils
+import cloudkitty.utils as ck_utils
 
 
 class TransformerDependencyError(Exception):
@@ -72,19 +71,16 @@ class BaseCollector(object):
 
     @staticmethod
     def last_month():
-        now = datetime.datetime.now()
-        month_end = (datetime.datetime(now.year, now.month, 1) -
-                     datetime.timedelta(days=1))
-        month_start = month_end.replace(day=1)
-        start_ts = utils.dt2ts(month_start)
-        end_ts = utils.dt2ts(month_end)
+        month_start = ck_utils.get_month_start()
+        month_end = ck_utils.get_month_end()
+        start_ts = ck_utils.dt2ts(month_start)
+        end_ts = ck_utils.dt2ts(month_end)
         return start_ts, end_ts
 
     @staticmethod
     def current_month():
-        now = datetime.now()
-        month_start = datetime(now.year, now.month, 1)
-        return utils.dt2ts(month_start)
+        month_start = ck_utils.get_month_start()
+        return ck_utils.dt2ts(month_start)
 
     def retrieve(self, resource, start, end=None, project_id=None,
                  q_filter=None):

cloudkitty/collector/ceilometer.py

@@ -15,11 +15,10 @@
 #
 # @author: Stéphane Albert
 #
-import datetime
-
 from ceilometerclient import client as cclient
 
 from cloudkitty import collector
+from cloudkitty import utils as ck_utils
 
 
 class ResourceNotFound(Exception):
@@ -111,12 +110,12 @@ class CeilometerCollector(collector.BaseCollector):
     def get_active_instances(self, start, end=None, project_id=None,
                              q_filter=None):
         """Instance that were active during the timespan."""
-        start_iso = datetime.datetime.fromtimestamp(start).isoformat()
+        start_iso = ck_utils.ts2iso(start)
         req_filter = self.gen_filter(op='ge', timestamp=start_iso)
         if project_id:
             req_filter.extend(self.gen_filter(project=project_id))
         if end:
-            end_iso = datetime.datetime.fromtimestamp(end).isoformat()
+            end_iso = ck_utils.ts2iso(end)
             req_filter.extend(self.gen_filter(op='le', timestamp=end_iso))
         if isinstance(q_filter, list):
             req_filter.extend(q_filter)

cloudkitty/orchestrator.py

@@ -16,8 +16,6 @@
 #
 # @author: Stéphane Albert
 #
-import time
-
 import eventlet
 from keystoneclient.v2_0 import client as kclient
 from oslo.config import cfg
@@ -152,9 +150,10 @@ class Orchestrator(object):
     def _check_state(self):
         timestamp = self.storage.get_state()
         if not timestamp:
-            return ck_utils.get_this_month_timestamp()
+            month_start = ck_utils.get_month_start()
+            return ck_utils.dt2ts(month_start)
 
-        now = int(time.time() + time.timezone)
+        now = ck_utils.utcnow_ts()
         next_timestamp = timestamp + CONF.collect.period
         wait_time = CONF.collect.wait_periods * CONF.collect.period
         if next_timestamp + wait_time < now:

cloudkitty/storage/__init__.py

@@ -16,12 +16,13 @@
 # @author: Stéphane Albert
 #
 import abc
-import datetime
 
 from oslo.config import cfg
 import six
 from stevedore import driver
 
+from cloudkitty import utils as ck_utils
+
 STORAGES_NAMESPACE = 'cloudkitty.storage.backends'
 storage_opts = [
     cfg.StrOpt('backend',
@@ -153,10 +154,8 @@ class BaseStorage(object):
             if self.usage_start is None:
                 self.usage_start = usage_start
                 self.usage_end = usage_start + self._period
-                self.usage_start_dt = (
-                    datetime.datetime.fromtimestamp(self.usage_start))
-                self.usage_end_dt = (
-                    datetime.datetime.fromtimestamp(self.usage_end))
+                self.usage_start_dt = ck_utils.ts2dt(self.usage_start)
+                self.usage_end_dt = ck_utils.ts2dt(self.usage_end)
 
             self._dispatch(data)
 

cloudkitty/storage/sqlalchemy/__init__.py

@@ -15,7 +15,6 @@
 #
 # @author: Stéphane Albert
 #
-import datetime
 import json
 
 from oslo.db.sqlalchemy import utils
@@ -70,7 +69,7 @@ class SQLAlchemyStorage(storage.BaseStorage):
         model = models.RatedDataFrame
 
         # Boundary calculation
-        month_start = ck_utils.get_this_month()
+        month_start = ck_utils.get_month_start()
         month_end = ck_utils.get_next_month()
 
         session = db.get_session()
@@ -96,8 +95,8 @@ class SQLAlchemyStorage(storage.BaseStorage):
             model,
             session
         ).filter(
-            model.begin >= datetime.datetime.fromtimestamp(begin),
-            model.end <= datetime.datetime.fromtimestamp(end)
+            model.begin >= ck_utils.ts2dt(begin),
+            model.end <= ck_utils.ts2dt(end)
         )
         for cur_filter in filters:
             q = q.filter(getattr(model, cur_filter) == filters[cur_filter])

cloudkitty/tests/test_utils.py

@@ -0,0 +1,138 @@
+# -*- coding: utf-8 -*-
+# Copyright 2014 Objectif Libre
+#
+#    Licensed under the Apache License, Version 2.0 (the "License"); you may
+#    not use this file except in compliance with the License. You may obtain
+#    a copy of the License at
+#
+#         http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+#    License for the specific language governing permissions and limitations
+#    under the License.
+#
+# @author: Stéphane Albert
+#
+import datetime
+import unittest
+
+import mock
+from oslo.utils import timeutils
+
+from cloudkitty import utils as ck_utils
+
+
+def iso2dt(iso_str):
+    return timeutils.parse_isotime(iso_str)
+
+
+class UtilsTimeCalculationsTest(unittest.TestCase):
+    def setUp(self):
+        self.date_ts = 1416219015
+        self.date_iso = '2014-11-17T10:10:15Z'
+        self.date_params = {'year': 2014,
+                            'month': 11,
+                            'day': 17,
+                            'hour': 10,
+                            'minute': 10,
+                            'second': 15}
+        self.date_tz_params = {'year': 2014,
+                               'month': 10,
+                               'day': 26,
+                               'hour': 2,
+                               'minute': 00,
+                               'second': 00}
+
+    def test_dt2ts(self):
+        date = datetime.datetime(**self.date_params)
+        trans_ts = ck_utils.dt2ts(date)
+        self.assertEqual(self.date_ts, trans_ts)
+
+    def test_iso2dt(self):
+        date = datetime.datetime(**self.date_params)
+        trans_dt = ck_utils.iso2dt(self.date_iso)
+        self.assertEqual(date, trans_dt)
+
+    def test_ts2iso(self):
+        trans_iso = ck_utils.ts2iso(self.date_ts)
+        self.assertEqual(self.date_iso, trans_iso)
+
+    def test_dt2iso(self):
+        date = datetime.datetime(**self.date_params)
+        trans_iso = ck_utils.dt2iso(date)
+        self.assertEqual(self.date_iso, trans_iso)
+
+    @mock.patch.object(ck_utils, 'utcnow',
+                       return_value=iso2dt('2014-01-31T00:00:00Z'))
+    def test_month_start_without_dt(self, patch_utcnow_mock):
+        date = datetime.datetime(2014, 1, 1)
+        trans_dt = ck_utils.get_month_start()
+        self.assertEqual(date, trans_dt)
+        patch_utcnow_mock.assert_called_once()
+
+    @mock.patch.object(ck_utils, 'utcnow',
+                       return_value=iso2dt('2014-01-15T00:00:00Z'))
+    def test_month_end_without_dt(self, patch_utcnow_mock):
+        date = datetime.datetime(2014, 1, 31)
+        trans_dt = ck_utils.get_month_end()
+        self.assertEqual(date, trans_dt)
+        patch_utcnow_mock.assert_called_once()
+
+    @mock.patch.object(ck_utils, 'utcnow',
+                       return_value=iso2dt('2014-01-31T00:00:00Z'))
+    def test_get_last_month_without_dt(self, patch_utcnow_mock):
+        date = datetime.datetime(2013, 12, 1)
+        trans_dt = ck_utils.get_last_month()
+        self.assertEqual(date, trans_dt)
+        patch_utcnow_mock.assert_called_once()
+
+    @mock.patch.object(ck_utils, 'utcnow',
+                       return_value=iso2dt('2014-01-31T00:00:00Z'))
+    def test_get_next_month_without_dt(self, patch_utcnow_mock):
+        date = datetime.datetime(2014, 2, 1)
+        trans_dt = ck_utils.get_next_month()
+        self.assertEqual(date, trans_dt)
+        patch_utcnow_mock.assert_called_once()
+
+    def test_get_last_month_leap(self):
+        base_date = datetime.datetime(2016, 3, 31)
+        date = datetime.datetime(2016, 2, 1)
+        trans_dt = ck_utils.get_last_month(base_date)
+        self.assertEqual(date, trans_dt)
+
+    def test_get_next_month_leap(self):
+        base_date = datetime.datetime(2016, 1, 31)
+        date = datetime.datetime(2016, 2, 1)
+        trans_dt = ck_utils.get_next_month(base_date)
+        self.assertEqual(date, trans_dt)
+
+    def test_add_month_leap(self):
+        base_date = datetime.datetime(2016, 1, 31)
+        date = datetime.datetime(2016, 3, 3)
+        trans_dt = ck_utils.add_month(base_date, False)
+        self.assertEqual(date, trans_dt)
+
+    def test_add_month_keep_leap(self):
+        base_date = datetime.datetime(2016, 1, 31)
+        date = datetime.datetime(2016, 2, 29)
+        trans_dt = ck_utils.add_month(base_date)
+        self.assertEqual(date, trans_dt)
+
+    def test_sub_month_leap(self):
+        base_date = datetime.datetime(2016, 3, 31)
+        date = datetime.datetime(2016, 3, 3)
+        trans_dt = ck_utils.sub_month(base_date, False)
+        self.assertEqual(date, trans_dt)
+
+    def test_sub_month_keep_leap(self):
+        base_date = datetime.datetime(2016, 3, 31)
+        date = datetime.datetime(2016, 2, 29)
+        trans_dt = ck_utils.sub_month(base_date)
+        self.assertEqual(date, trans_dt)
+
+    def test_load_timestamp(self):
+        calc_dt = ck_utils.iso2dt(self.date_iso)
+        check_dt = ck_utils.ts2dt(self.date_ts)
+        self.assertEqual(calc_dt, check_dt)

cloudkitty/utils.py

@@ -15,36 +15,111 @@
 #
 # @author: Stéphane Albert
 #
+"""
+Time calculations functions
+
+We're mostly using oslo.utils for time calculations but we're encapsulating it
+to ease maintenance in case of library modifications.
+"""
 import calendar
 import datetime
-import time
 
-import iso8601
+from oslo.utils import timeutils
 
 
 def dt2ts(orig_dt):
-    return int(time.mktime(orig_dt.timetuple()))
+    """Translate a datetime into a timestamp."""
+    return calendar.timegm(orig_dt.timetuple())
 
 
 def iso2dt(iso_date):
-    return iso8601.parse_date(iso_date)
+    """iso8601 format to datetime."""
+    iso_dt = timeutils.parse_isotime(iso_date)
+    trans_dt = timeutils.normalize_time(iso_dt)
+    return trans_dt
 
 
-def get_this_month():
-    now = datetime.datetime.utcnow()
-    month_start = datetime.datetime(now.year, now.month, 1)
+def ts2dt(timestamp):
+    """timestamp to datetime format."""
+    if not isinstance(timestamp, float):
+        timestamp = float(timestamp)
+    return datetime.datetime.utcfromtimestamp(timestamp)
+
+
+def ts2iso(timestamp):
+    """timestamp to is8601 format."""
+    if not isinstance(timestamp, float):
+        timestamp = float(timestamp)
+    return timeutils.iso8601_from_timestamp(timestamp)
+
+
+def dt2iso(orig_dt):
+    """datetime to is8601 format."""
+    return timeutils.isotime(orig_dt)
+
+
+def utcnow():
+    """Returns a datetime for the current utc time."""
+    return timeutils.utcnow()
+
+
+def utcnow_ts():
+    """Returns a timestamp for the current utc time."""
+    return timeutils.utcnow_ts()
+
+
+def get_month_days(dt):
+    return calendar.monthrange(dt.year, dt.month)[1]
+
+
+def add_days(base_dt, days, stay_on_month=True):
+    if stay_on_month:
+        max_days = get_month_days(base_dt)
+        if days > max_days:
+            return get_month_end(base_dt)
+    return base_dt + datetime.timedelta(days=days)
+
+
+def add_month(dt, stay_on_month=True):
+    next_month = get_next_month(dt)
+    return add_days(next_month, dt.day, stay_on_month)
+
+
+def sub_month(dt, stay_on_month=True):
+    prev_month = get_last_month(dt)
+    return add_days(prev_month, dt.day, stay_on_month)
+
+
+def get_month_start(dt=None):
+    if not dt:
+        dt = utcnow()
+    month_start = datetime.datetime(dt.year, dt.month, 1)
     return month_start
 
 
-def get_this_month_timestamp():
-    return dt2ts(get_this_month())
+def get_month_start_timestamp(dt=None):
+    return dt2ts(get_month_start(dt))
 
 
-def get_next_month():
-    start_dt = get_this_month()
-    next_dt = start_dt + datetime.timedelta(calendar.mdays[start_dt.month])
-    return next_dt
+def get_month_end(dt=None):
+    month_start = get_month_start(dt)
+    days_of_month = get_month_days(month_start)
+    month_end = month_start.replace(day=days_of_month)
+    return month_end
 
 
-def get_next_month_timestamp():
-    return dt2ts(get_next_month())
+def get_last_month(dt=None):
+    if not dt:
+        dt = utcnow()
+    month_end = get_month_start(dt) - datetime.timedelta(days=1)
+    return get_month_start(month_end)
+
+
+def get_next_month(dt=None):
+    month_end = get_month_end(dt)
+    next_month = month_end + datetime.timedelta(days=1)
+    return next_month
+
+
+def get_next_month_timestamp(dt=None):
+    return dt2ts(get_next_month(dt))

cloudkitty/write_orchestrator.py

@@ -123,7 +123,8 @@ class WriteOrchestrator(object):
 
     def restart_month(self):
         self._load_state_manager_data()
-        self.usage_end = ck_utils.get_this_month_timestamp()
+        month_start = ck_utils.get_month_start()
+        self.usage_end = ck_utils.dt2ts(month_start)
         self._update_state_manager_data()
 
     def process(self):

cloudkitty/writer/__init__.py

@@ -16,11 +16,11 @@
 # @author: Stéphane Albert
 #
 import abc
-import datetime
 
 import six
 
 from cloudkitty import state
+from cloudkitty import utils as ck_utils
 
 
 @six.add_metaclass(abc.ABCMeta)
@@ -80,9 +80,9 @@ class BaseReportWriter(object):
     def _get_state_manager_timeframe(self):
         timeframe = self._sm.get_state()
         self.usage_start = timeframe
-        self.usage_start_dt = datetime.datetime.fromtimestamp(timeframe)
+        self.usage_start_dt = ck_utils.ts2dt(timeframe)
         self.usage_end = timeframe + self._period
-        self.usage_end_dt = datetime.datetime.fromtimestamp(self.usage_end)
+        self.usage_end_dt = ck_utils.ts2dt(self.usage_end)
         metadata = self._sm.get_metadata()
         self.total = metadata.get('total', 0)
 
@@ -150,10 +150,8 @@ class BaseReportWriter(object):
         if self.usage_start is None:
             self.usage_start = start
             self.usage_end = start + self._period
-            self.usage_start_dt = datetime.datetime.fromtimestamp(
-                self.usage_start)
-            self.usage_end_dt = datetime.datetime.fromtimestamp(
-                self.usage_end)
+            self.usage_start_dt = ck_utils.ts2dt(self.usage_start)
+            self.usage_end_dt = ck_utils.ts2dt(self.usage_end)
 
         self._update(data)
 

requirements.txt

@@ -11,6 +11,7 @@ oslo.config>=1.2.0
 oslo.messaging
 oslo.db
 oslo.i18n
+oslo.utils
 sqlalchemy
 six>=1.7.0
 stevedore

test-requirements.txt

@@ -1,4 +1,5 @@
 hacking>=0.9.2,<0.10
+coverage>=3.6
 discover
 testtools
 testscenarios