diff --git a/glance/common/utils.py b/glance/common/utils.py index f4a5216ee8..4af74e4b6b 100644 --- a/glance/common/utils.py +++ b/glance/common/utils.py @@ -31,12 +31,14 @@ import socket import sys import uuid +import iso8601 + from glance.common import exception logger = logging.getLogger(__name__) -TIME_FORMAT = "%Y-%m-%dT%H:%M:%SZ" +TIME_FORMAT = "%Y-%m-%dT%H:%M:%S" def chunkreadable(iter, chunk_size=65536): @@ -191,13 +193,29 @@ def is_uuid_like(value): def isotime(at=None): + """Stringify time in ISO 8601 format""" if not at: at = datetime.datetime.utcnow() - return at.strftime(TIME_FORMAT) + str = at.strftime(TIME_FORMAT) + tz = at.tzinfo.tzname(None) if at.tzinfo else 'UTC' + str += ('Z' if tz == 'UTC' else tz) + return str def parse_isotime(timestr): - return datetime.datetime.strptime(timestr, TIME_FORMAT) + """Parse time from ISO 8601 format""" + try: + return iso8601.parse_date(timestr) + except iso8601.ParseError as e: + raise ValueError(e.message) + except TypeError as e: + raise ValueError(e.message) + + +def normalize_time(timestamp): + """Normalize time in arbitrary timezone to UTC""" + offset = timestamp.utcoffset() + return timestamp.replace(tzinfo=None) - offset if offset else timestamp def safe_mkdirs(path): diff --git a/glance/registry/db/api.py b/glance/registry/db/api.py index ac75ecd7e7..78eef7cb5f 100644 --- a/glance/registry/db/api.py +++ b/glance/registry/db/api.py @@ -228,7 +228,9 @@ def image_get_all(context, filters=None, marker=None, limit=None, showing_deleted = False if 'changes-since' in filters: - changes_since = filters.pop('changes-since') + # normalize timestamp to UTC, as sqlalchemy doesn't appear to + # respect timezone offsets + changes_since = utils.normalize_time(filters.pop('changes-since')) query = query.filter(models.Image.updated_at > changes_since) showing_deleted = True diff --git a/glance/tests/functional/test_api.py b/glance/tests/functional/test_api.py index 39f663c49a..1fa6f289c1 100644 --- a/glance/tests/functional/test_api.py +++ b/glance/tests/functional/test_api.py @@ -905,9 +905,23 @@ class TestApi(functional.FunctionalTest): self.assertEqual(image['name'], "My Image!") # 16. GET /images with past changes-since filter - dt1 = datetime.datetime.utcnow() - datetime.timedelta(1) - iso1 = utils.isotime(dt1) - params = "changes-since=%s" % iso1 + yesterday = utils.isotime(datetime.datetime.utcnow() - + datetime.timedelta(1)) + params = "changes-since=%s" % yesterday + path = "http://%s:%d/v1/images?%s" % ("0.0.0.0", self.api_port, params) + response, content = http.request(path, 'GET') + self.assertEqual(response.status, 200) + data = json.loads(content) + self.assertEqual(len(data['images']), 3) + + # one timezone west of Greenwich equates to an hour ago + # taking care to pre-urlencode '+' as '%2B', otherwise the timezone + # '+' is wrongly decoded as a space + # TODO(eglynn): investigate '+' --> decoding, an artifact + # of WSGI/webob dispatch? + now = datetime.datetime.utcnow() + hour_ago = now.strftime('%Y-%m-%dT%H:%M:%S%%2B01:00') + params = "changes-since=%s" % hour_ago path = "http://%s:%d/v1/images?%s" % ("0.0.0.0", self.api_port, params) response, content = http.request(path, 'GET') self.assertEqual(response.status, 200) @@ -915,9 +929,19 @@ class TestApi(functional.FunctionalTest): self.assertEqual(len(data['images']), 3) # 17. GET /images with future changes-since filter - dt2 = datetime.datetime.utcnow() + datetime.timedelta(1) - iso2 = utils.isotime(dt2) - params = "changes-since=%s" % iso2 + tomorrow = utils.isotime(datetime.datetime.utcnow() + + datetime.timedelta(1)) + params = "changes-since=%s" % tomorrow + path = "http://%s:%d/v1/images?%s" % ("0.0.0.0", self.api_port, params) + response, content = http.request(path, 'GET') + self.assertEqual(response.status, 200) + data = json.loads(content) + self.assertEqual(len(data['images']), 0) + + # one timezone east of Greenwich equates to an hour from now + now = datetime.datetime.utcnow() + hour_hence = now.strftime('%Y-%m-%dT%H:%M:%S-01:00') + params = "changes-since=%s" % hour_hence path = "http://%s:%d/v1/images?%s" % ("0.0.0.0", self.api_port, params) response, content = http.request(path, 'GET') self.assertEqual(response.status, 200) diff --git a/glance/tests/unit/test_api.py b/glance/tests/unit/test_api.py index bc9f7ae288..036d4a1156 100644 --- a/glance/tests/unit/test_api.py +++ b/glance/tests/unit/test_api.py @@ -1268,8 +1268,9 @@ class TestRegistryAPI(base.IsolatedUnitTest): dt2 = datetime.datetime.utcnow() + datetime.timedelta(1) iso2 = utils.isotime(dt2) - dt3 = datetime.datetime.utcnow() + datetime.timedelta(2) - iso3 = utils.isotime(dt3) + image_ts = datetime.datetime.utcnow() + datetime.timedelta(2) + hour_before = image_ts.strftime('%Y-%m-%dT%H:%M:%S%%2B01:00') + hour_after = image_ts.strftime('%Y-%m-%dT%H:%M:%S-01:00') dt4 = datetime.datetime.utcnow() + datetime.timedelta(3) iso4 = utils.isotime(dt4) @@ -1296,8 +1297,8 @@ class TestRegistryAPI(base.IsolatedUnitTest): 'name': 'fake image #4', 'size': 20, 'checksum': None, - 'created_at': dt3, - 'updated_at': dt3} + 'created_at': image_ts, + 'updated_at': image_ts} db_api.image_create(self.context, extra_fixture) @@ -1331,6 +1332,25 @@ class TestRegistryAPI(base.IsolatedUnitTest): self.assertEquals(len(images), 1) self.assertEqual(images[0]['id'], UUID4) + # Expect 1 images (0 deleted) + req = webob.Request.blank('/images/detail?changes-since=%s' % + hour_before) + res = req.get_response(self.api) + self.assertEquals(res.status_int, 200) + res_dict = json.loads(res.body) + images = res_dict['images'] + self.assertEquals(len(images), 1) + self.assertEqual(images[0]['id'], UUID4) + + # Expect 0 images (0 deleted) + req = webob.Request.blank('/images/detail?changes-since=%s' % + hour_after) + res = req.get_response(self.api) + self.assertEquals(res.status_int, 200) + res_dict = json.loads(res.body) + images = res_dict['images'] + self.assertEquals(len(images), 0) + # Expect 0 images (0 deleted) req = webob.Request.blank('/images/detail?changes-since=%s' % iso4) res = req.get_response(self.api) @@ -2367,8 +2387,9 @@ class TestGlanceAPI(base.IsolatedUnitTest): dt2 = datetime.datetime.utcnow() + datetime.timedelta(1) iso2 = utils.isotime(dt2) - dt3 = datetime.datetime.utcnow() + datetime.timedelta(2) - iso3 = utils.isotime(dt3) + image_ts = datetime.datetime.utcnow() + datetime.timedelta(2) + hour_before = image_ts.strftime('%Y-%m-%dT%H:%M:%S%%2B01:00') + hour_after = image_ts.strftime('%Y-%m-%dT%H:%M:%S-01:00') dt4 = datetime.datetime.utcnow() + datetime.timedelta(3) iso4 = utils.isotime(dt4) @@ -2395,8 +2416,8 @@ class TestGlanceAPI(base.IsolatedUnitTest): 'name': 'fake image #4', 'size': 20, 'checksum': None, - 'created_at': dt3, - 'updated_at': dt3} + 'created_at': image_ts, + 'updated_at': image_ts} db_api.image_create(self.context, extra_fixture) @@ -2430,6 +2451,25 @@ class TestGlanceAPI(base.IsolatedUnitTest): self.assertEquals(len(images), 1) self.assertEqual(images[0]['id'], UUID4) + # Expect 1 images (0 deleted) + req = webob.Request.blank('/images/detail?changes-since=%s' % + hour_before) + res = req.get_response(self.api) + self.assertEquals(res.status_int, 200) + res_dict = json.loads(res.body) + images = res_dict['images'] + self.assertEquals(len(images), 1) + self.assertEqual(images[0]['id'], UUID4) + + # Expect 0 images (0 deleted) + req = webob.Request.blank('/images/detail?changes-since=%s' % + hour_after) + res = req.get_response(self.api) + self.assertEquals(res.status_int, 200) + res_dict = json.loads(res.body) + images = res_dict['images'] + self.assertEquals(len(images), 0) + # Expect 0 images (0 deleted) req = webob.Request.blank('/images/detail?changes-since=%s' % iso4) res = req.get_response(self.api) diff --git a/glance/tests/unit/test_utils.py b/glance/tests/unit/test_utils.py index aab61cd49b..851311164d 100644 --- a/glance/tests/unit/test_utils.py +++ b/glance/tests/unit/test_utils.py @@ -17,6 +17,8 @@ import unittest +import iso8601 + from glance.common import utils @@ -45,3 +47,108 @@ class TestUtils(unittest.TestCase): def test_is_uuid_like_fails(self): fixture = 'pants' self.assertFalse(utils.is_uuid_like(fixture)) + + +class TestIso8601Time(unittest.TestCase): + + def _instaneous(self, timestamp, yr, mon, day, hr, min, sec, micro): + self.assertEquals(timestamp.year, yr) + self.assertEquals(timestamp.month, mon) + self.assertEquals(timestamp.day, day) + self.assertEquals(timestamp.hour, hr) + self.assertEquals(timestamp.minute, min) + self.assertEquals(timestamp.second, sec) + self.assertEquals(timestamp.microsecond, micro) + + def _do_test(self, str, yr, mon, day, hr, min, sec, micro, shift): + DAY_SECONDS = 24 * 60 * 60 + timestamp = utils.parse_isotime(str) + self._instaneous(timestamp, yr, mon, day, hr, min, sec, micro) + offset = timestamp.tzinfo.utcoffset(None) + self.assertEqual(offset.seconds + offset.days * DAY_SECONDS, shift) + + def test_zulu(self): + str = '2012-02-14T20:53:07Z' + self._do_test(str, 2012, 02, 14, 20, 53, 7, 0, 0) + + def test_zulu_micros(self): + str = '2012-02-14T20:53:07.123Z' + self._do_test(str, 2012, 02, 14, 20, 53, 7, 123000, 0) + + def test_offset_east(self): + str = '2012-02-14T20:53:07+04:30' + offset = 4.5 * 60 * 60 + self._do_test(str, 2012, 02, 14, 20, 53, 7, 0, offset) + + def test_offset_east_micros(self): + str = '2012-02-14T20:53:07.42+04:30' + offset = 4.5 * 60 * 60 + self._do_test(str, 2012, 02, 14, 20, 53, 7, 420000, offset) + + def test_offset_west(self): + str = '2012-02-14T20:53:07-05:30' + offset = -5.5 * 60 * 60 + self._do_test(str, 2012, 02, 14, 20, 53, 7, 0, offset) + + def test_offset_west_micros(self): + str = '2012-02-14T20:53:07.654321-05:30' + offset = -5.5 * 60 * 60 + self._do_test(str, 2012, 02, 14, 20, 53, 7, 654321, offset) + + def test_compare(self): + zulu = utils.parse_isotime('2012-02-14T20:53:07') + east = utils.parse_isotime('2012-02-14T20:53:07-01:00') + west = utils.parse_isotime('2012-02-14T20:53:07+01:00') + self.assertTrue(east > west) + self.assertTrue(east > zulu) + self.assertTrue(zulu > west) + + def test_compare_micros(self): + zulu = utils.parse_isotime('2012-02-14T20:53:07.6544') + east = utils.parse_isotime('2012-02-14T19:53:07.654321-01:00') + west = utils.parse_isotime('2012-02-14T21:53:07.655+01:00') + self.assertTrue(east < west) + self.assertTrue(east < zulu) + self.assertTrue(zulu < west) + + def test_zulu_roundtrip(self): + str = '2012-02-14T20:53:07Z' + zulu = utils.parse_isotime(str) + self.assertEquals(zulu.tzinfo, iso8601.iso8601.UTC) + self.assertEquals(utils.isotime(zulu), str) + + def test_east_roundtrip(self): + str = '2012-02-14T20:53:07-07:00' + east = utils.parse_isotime(str) + self.assertEquals(east.tzinfo.tzname(None), '-07:00') + self.assertEquals(utils.isotime(east), str) + + def test_west_roundtrip(self): + str = '2012-02-14T20:53:07+11:30' + west = utils.parse_isotime(str) + self.assertEquals(west.tzinfo.tzname(None), '+11:30') + self.assertEquals(utils.isotime(west), str) + + def test_now_roundtrip(self): + str = utils.isotime() + now = utils.parse_isotime(str) + self.assertEquals(now.tzinfo, iso8601.iso8601.UTC) + self.assertEquals(utils.isotime(now), str) + + def test_zulu_normalize(self): + str = '2012-02-14T20:53:07Z' + zulu = utils.parse_isotime(str) + normed = utils.normalize_time(zulu) + self._instaneous(normed, 2012, 2, 14, 20, 53, 07, 0) + + def test_east_normalize(self): + str = '2012-02-14T20:53:07-07:00' + east = utils.parse_isotime(str) + normed = utils.normalize_time(east) + self._instaneous(normed, 2012, 2, 15, 03, 53, 07, 0) + + def test_west_normalize(self): + str = '2012-02-14T20:53:07+21:00' + west = utils.parse_isotime(str) + normed = utils.normalize_time(west) + self._instaneous(normed, 2012, 2, 13, 23, 53, 07, 0) diff --git a/tools/pip-requires b/tools/pip-requires index 78d0c7d984..ebf5bb9aae 100644 --- a/tools/pip-requires +++ b/tools/pip-requires @@ -21,6 +21,7 @@ xattr>=0.6.0 kombu pycrypto>=2.1.0alpha1 pysendfile==2.0.0 +iso8601>=0.1.4 # The following allow Keystone to be installed in the venv