openstack/glance
Code Issues Proposed changes

Implementing changes-since param in api & registry

Browse Source 
Change-Id: I1d462b555f20ae6b28968b257cc5c22de65276e7
This commit is contained in:
Brian Waldon 2011-09-07 14:12:05 -04:00
parent e7d06e409e
commit 8805302edb
10 changed files with 506 additions and 30 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

13
bin/glance
View File

@ -100,8 +100,6 @@ def get_image_fields_from_args(args):
raise RuntimeError(msg)
fields[pieces[0]] = pieces[1]
fields = dict([(k.lower().replace('-', '_'), v)
for k, v in fields.items()])
return fields
@ -114,7 +112,8 @@ def get_image_filters_from_args(args):
return FAILURE
SUPPORTED_FILTERS = ['name', 'disk_format', 'container_format', 'status',
'min_ram', 'min_disk', 'size_min', 'size_max']
'min_ram', 'min_disk', 'size_min', 'size_max',
'changes-since']
filters = {}
for (key, value) in fields.items():
if key not in SUPPORTED_FILTERS:
@ -480,8 +479,8 @@ Returns basic information for all public images
a Glance server knows about. Provided fields are
handled as query filters. Supported filters
include 'name', 'disk_format', 'container_format',
'status', 'size_min', and 'size_max.' Any extra
fields are treated as image metadata properties"""
'status', 'size_min', 'size_max' and 'changes-since.'
Any extra fields are treated as image metadata properties"""
client = get_client(options)
filters = get_image_filters_from_args(args)
limit = options.limit
@ -539,8 +538,8 @@ Returns detailed information for all public images
a Glance server knows about. Provided fields are
handled as query filters. Supported filters
include 'name', 'disk_format', 'container_format',
'status', 'size_min', and 'size_max.' Any extra
fields are treated as image metadata properties"""
'status', 'size_min', 'size_max' and 'changes-since.'
Any extra fields are treated as image metadata properties"""
client = get_client(options)
filters = get_image_filters_from_args(args)
limit = options.limit

2
glance/api/v1/images.py
View File

@ -56,7 +56,7 @@ logger = logging.getLogger('glance.api.v1.images')
SUPPORTED_FILTERS = ['name', 'status', 'container_format', 'disk_format',
'min_ram', 'min_disk', 'size_min', 'size_max',
'is_public']
'is_public', 'changes-since']
SUPPORTED_PARAMS = ('limit', 'marker', 'sort_key', 'sort_dir')

24
glance/registry/db/api.py
View File

@ -150,10 +150,12 @@ def image_get(context, image_id, session=None):
options(joinedload(models.Image.members)).\
filter_by(id=image_id)
# filter out deleted images if context disallows it
if not can_show_deleted(context):
query = query.filter_by(deleted=False)
image = query.one()
except exc.NoResultFound:
raise exception.NotFound("No image found with ID %s" % image_id)
@ -182,16 +184,7 @@ def image_get_all(context, filters=None, marker=None, limit=None,
session = get_session()
query = session.query(models.Image).\
options(joinedload(models.Image.properties)).\
options(joinedload(models.Image.members)).\
filter(models.Image.status != 'killed')
if not can_show_deleted(context) or 'deleted' not in filters:
query = query.filter_by(deleted=False)
else:
query = query.filter_by(deleted=filters['deleted'])
if 'deleted' in filters:
del filters['deleted']
options(joinedload(models.Image.members))
sort_dir_func = {
'asc': asc,
@ -223,6 +216,17 @@ def image_get_all(context, filters=None, marker=None, limit=None,
query = query.filter(the_filter[0])
del filters['is_public']
if 'changes-since' in filters:
changes_since = filters.pop('changes-since')
query = query.filter(models.Image.updated_at > changes_since)
if 'deleted' in filters:
deleted_filter = filters.pop('deleted')
query = query.filter_by(deleted=deleted_filter)
# TODO(bcwaldon): handle this logic in registry server
if not deleted_filter:
query = query.filter(models.Image.status != 'killed')
for (k, v) in filters.pop('properties', {}).items():
query = query.filter(models.Image.properties.any(name=k, value=v))

82
glance/registry/db/migrate_repo/versions/010_default_update_at.py Normal file
View File

@ -0,0 +1,82 @@
# vim: tabstop=4 shiftwidth=4 softtabstop=4
# Copyright 2011 OpenStack LLC.
# All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
from migrate.changeset import *
from sqlalchemy import *
from glance.registry.db.migrate_repo.schema import from_migration_import
def get_images_table(meta):
"""
No changes to the images table from 008...
"""
(get_images_table,) = from_migration_import(
'008_add_image_members_table', ['get_images_table'])
images = get_images_table(meta)
return images
def get_image_properties_table(meta):
"""
No changes to the image properties table from 008...
"""
(get_image_properties_table,) = from_migration_import(
'008_add_image_members_table', ['get_image_properties_table'])
image_properties = get_image_properties_table(meta)
return image_properties
def get_image_members_table(meta):
"""
No changes to the image members table from 008...
"""
(get_image_members_table,) = from_migration_import(
'008_add_image_members_table', ['get_image_members_table'])
images = get_image_members_table(meta)
return images
def upgrade(migrate_engine):
meta = MetaData()
meta.bind = migrate_engine
images_table = get_images_table(meta)
# set updated_at to created_at if equal to None
conn = migrate_engine.connect()
conn.execute(
images_table.update(
images_table.c.updated_at == None,
{images_table.c.updated_at: images_table.c.created_at}))
def downgrade(migrate_engine):
meta = MetaData()
meta.bind = migrate_engine
images_table = get_images_table(meta)
# set updated_at to None if equal to created_at
conn = migrate_engine.connect()
conn.execute(
images_table.update(
images_table.c.updated_at == images_table.c.created_at,
{images_table.c.updated_at: None}))

3
glance/registry/db/models.py
View File

@ -44,7 +44,8 @@ class ModelBase(object):
created_at = Column(DateTime, default=datetime.datetime.utcnow,
nullable=False)
updated_at = Column(DateTime, onupdate=datetime.datetime.utcnow)
updated_at = Column(DateTime, default=datetime.datetime.utcnow,
nullable=False, onupdate=datetime.datetime.utcnow)
deleted_at = Column(DateTime)
deleted = Column(Boolean, nullable=False, default=False)

33
glance/registry/server.py
View File

@ -38,7 +38,8 @@ DISPLAY_FIELDS_IN_INDEX = ['id', 'name', 'size',
'checksum']
SUPPORTED_FILTERS = ['name', 'status', 'container_format', 'disk_format',
'min_ram', 'min_disk', 'size_min', 'size_max']
'min_ram', 'min_disk', 'size_min', 'size_max',
'changes-since']
SUPPORTED_SORT_KEYS = ('name', 'status', 'container_format', 'disk_format',
'size', 'id', 'created_at', 'updated_at')
@ -148,14 +149,9 @@ class Controller(object):
if req.context.is_admin:
# Only admin gets to look for non-public images
filters['is_public'] = self._get_is_public(req)
# The same for deleted
filters['deleted'] = self._parse_deleted_filter(req)
else:
filters['is_public'] = True
# NOTE(jkoelker): This is technically unnecessary since the db
# api will force deleted=False if its not an
# admin context. But explicit > implicit.
filters['deleted'] = False
for param in req.str_params:
if param in SUPPORTED_FILTERS:
filters[param] = req.str_params.get(param)
@ -163,6 +159,23 @@ class Controller(object):
_param = param[9:]
properties[_param] = req.str_params.get(param)
if 'changes-since' in filters:
isotime = filters['changes-since']
try:
filters['changes-since'] = utils.parse_isotime(isotime)
except ValueError:
raise exc.HTTPBadRequest(_("Unrecognized changes-since value"))
# only allow admins to filter on 'deleted'
if req.context.is_admin:
deleted_filter = self._parse_deleted_filter(req)
if deleted_filter is not None:
filters['deleted'] = deleted_filter
elif 'changes-since' not in filters:
filters['deleted'] = False
elif 'changes-since' not in filters:
filters['deleted'] = False
if len(properties) > 0:
filters['properties'] = properties
@ -249,9 +262,9 @@ class Controller(object):
def _parse_deleted_filter(self, req):
"""Parse deleted into something usable."""
deleted = req.str_params.get('deleted', False)
if not deleted:
return False
deleted = req.str_params.get('deleted')
if deleted is None:
return None
return utils.bool_from_string(deleted)
def show(self, req, id):

22
glance/tests/functional/test_api.py
View File

@ -17,12 +17,14 @@
"""Functional test case that utilizes httplib2 against the API server"""
import datetime
import hashlib
import httplib2
import json
import os
import tempfile
from glance.common import utils
from glance.tests import functional
from glance.tests.utils import execute, skip_if_disabled
@ -853,6 +855,26 @@ class TestApi(functional.FunctionalTest):
self.assertEqual(image['properties']['pants'], "are on")
self.assertEqual(image['name'], "My Image!")
# 14. GET /images with past changes-since filter
dt1 = datetime.datetime.utcnow() - datetime.timedelta(1)
iso1 = utils.isotime(dt1)
params = "changes-since=%s" % iso1
path = "http://%s:%d/v1/images?%s" % ("0.0.0.0", self.api_port, params)
response, content = http.request(path, 'GET')
self.assertEqual(response.status, 200)
data = json.loads(content)
self.assertEqual(len(data['images']), 3)
# 15. GET /images with future changes-since filter
dt2 = datetime.datetime.utcnow() + datetime.timedelta(1)
iso2 = utils.isotime(dt2)
params = "changes-since=%s" % iso2
path = "http://%s:%d/v1/images?%s" % ("0.0.0.0", self.api_port, params)
response, content = http.request(path, 'GET')
self.assertEqual(response.status, 200)
data = json.loads(content)
self.assertEqual(len(data['images']), 0)
self.stop_servers()
@skip_if_disabled

27
glance/tests/functional/test_bin_glance.py
View File

@ -17,10 +17,12 @@
"""Functional test case that utilizes the bin/glance CLI tool"""
import datetime
import os
import tempfile
import unittest
from glance.common import utils
from glance.tests import functional
from glance.tests.utils import execute
@ -396,7 +398,30 @@ class TestBinGlance(functional.FunctionalTest):
self.assertEqual(1, len(image_lines))
self.assertTrue(image_lines[0].startswith('2'))
# 9. Ensure details call also respects filters
# 9. Check past changes-since
dt1 = datetime.datetime.utcnow() - datetime.timedelta(1)
iso1 = utils.isotime(dt1)
cmd = "changes-since=%s" % iso1
exitcode, out, err = execute("%s %s" % (_index_cmd, cmd))
self.assertEqual(0, exitcode)
image_lines = out.split("\n")[2:-1]
self.assertEqual(3, len(image_lines))
self.assertTrue(image_lines[0].startswith('3'))
self.assertTrue(image_lines[1].startswith('2'))
self.assertTrue(image_lines[2].startswith('1'))
# 10. Check future changes-since
dt2 = datetime.datetime.utcnow() + datetime.timedelta(1)
iso2 = utils.isotime(dt2)
cmd = "changes-since=%s" % iso2
exitcode, out, err = execute("%s %s" % (_index_cmd, cmd))
self.assertEqual(0, exitcode)
image_lines = out.split("\n")[2:-1]
self.assertEqual(0, len(image_lines))
# 11. Ensure details call also respects filters
_details_cmd = "%s details" % (_base_cmd,)
cmd = "foo=bar"
exitcode, out, err = execute("%s %s" % (_details_cmd, cmd))

182
glance/tests/unit/test_api.py
View File

@ -27,12 +27,14 @@ import webob
from glance.api import v1 as server
from glance.common import context
from glance.common import utils
from glance.registry import context as rcontext
from glance.registry import server as rserver
from glance.registry.db import api as db_api
from glance.registry.db import models as db_models
from glance.tests import stubs
OPTIONS = {'sql_connection': 'sqlite://',
'verbose': False,
'debug': False,
@ -1208,6 +1210,96 @@ class TestRegistryAPI(unittest.TestCase):
for image in images:
self.assertTrue(image['size'] <= 19 and image['size'] >= 18)
def test_get_details_filter_changes_since(self):
"""
Tests that the /images/detail registry API returns list of
public images that have a size less than or equal to size_max
"""
dt1 = datetime.datetime.utcnow() - datetime.timedelta(1)
iso1 = utils.isotime(dt1)
dt2 = datetime.datetime.utcnow() + datetime.timedelta(1)
iso2 = utils.isotime(dt2)
dt3 = datetime.datetime.utcnow() + datetime.timedelta(2)
iso3 = utils.isotime(dt3)
dt4 = datetime.datetime.utcnow() + datetime.timedelta(3)
iso4 = utils.isotime(dt4)
extra_fixture = {'id': 3,
'status': 'active',
'is_public': True,
'disk_format': 'vhd',
'container_format': 'ovf',
'name': 'fake image #3',
'size': 18,
'checksum': None}
db_api.image_create(self.context, extra_fixture)
db_api.image_destroy(self.context, 3)
extra_fixture = {'id': 4,
'status': 'active',
'is_public': True,
'disk_format': 'ami',
'container_format': 'ami',
'name': 'fake image #4',
'size': 20,
'checksum': None,
'created_at': dt3,
'updated_at': dt3}
db_api.image_create(self.context, extra_fixture)
# Check a standard list, 4 images in db (2 deleted)
req = webob.Request.blank('/images/detail')
res = req.get_response(self.api)
self.assertEquals(res.status_int, 200)
res_dict = json.loads(res.body)
images = res_dict['images']
self.assertEquals(len(images), 2)
self.assertEqual(images[0]['id'], 4)
self.assertEqual(images[1]['id'], 2)
# Expect 3 images (1 deleted)
req = webob.Request.blank('/images/detail?changes-since=%s' % iso1)
res = req.get_response(self.api)
self.assertEquals(res.status_int, 200)
res_dict = json.loads(res.body)
images = res_dict['images']
self.assertEquals(len(images), 3)
self.assertEqual(images[0]['id'], 4)
self.assertEqual(images[1]['id'], 3) # deleted
self.assertEqual(images[2]['id'], 2)
# Expect 1 images (0 deleted)
req = webob.Request.blank('/images/detail?changes-since=%s' % iso2)
res = req.get_response(self.api)
self.assertEquals(res.status_int, 200)
res_dict = json.loads(res.body)
images = res_dict['images']
self.assertEquals(len(images), 1)
self.assertEqual(images[0]['id'], 4)
# Expect 0 images (0 deleted)
req = webob.Request.blank('/images/detail?changes-since=%s' % iso4)
res = req.get_response(self.api)
self.assertEquals(res.status_int, 200)
res_dict = json.loads(res.body)
images = res_dict['images']
self.assertEquals(len(images), 0)
# Bad request (empty changes-since param)
req = webob.Request.blank('/images/detail?changes-since=')
res = req.get_response(self.api)
self.assertEquals(res.status_int, 400)
# Bad request (invalid changes-since param)
req = webob.Request.blank('/images/detail?changes-since=2011-09-05')
res = req.get_response(self.api)
self.assertEquals(res.status_int, 400)
def test_get_details_filter_property(self):
"""
Tests that the /images/detail registry API returns list of
@ -1981,6 +2073,96 @@ class TestGlanceAPI(unittest.TestCase):
self.assertEquals(int(images[1]['id']), 2)
self.assertEquals(int(images[2]['id']), 4)
def test_get_details_filter_changes_since(self):
"""
Tests that the /images/detail registry API returns list of
public images that have a size less than or equal to size_max
"""
dt1 = datetime.datetime.utcnow() - datetime.timedelta(1)
iso1 = utils.isotime(dt1)
dt2 = datetime.datetime.utcnow() + datetime.timedelta(1)
iso2 = utils.isotime(dt2)
dt3 = datetime.datetime.utcnow() + datetime.timedelta(2)
iso3 = utils.isotime(dt3)
dt4 = datetime.datetime.utcnow() + datetime.timedelta(3)
iso4 = utils.isotime(dt4)
extra_fixture = {'id': 3,
'status': 'active',
'is_public': True,
'disk_format': 'vhd',
'container_format': 'ovf',
'name': 'fake image #3',
'size': 18,
'checksum': None}
db_api.image_create(self.context, extra_fixture)
db_api.image_destroy(self.context, 3)
extra_fixture = {'id': 4,
'status': 'active',
'is_public': True,
'disk_format': 'ami',
'container_format': 'ami',
'name': 'fake image #4',
'size': 20,
'checksum': None,
'created_at': dt3,
'updated_at': dt3}
db_api.image_create(self.context, extra_fixture)
# Check a standard list, 4 images in db (2 deleted)
req = webob.Request.blank('/images/detail')
res = req.get_response(self.api)
self.assertEquals(res.status_int, 200)
res_dict = json.loads(res.body)
images = res_dict['images']
self.assertEquals(len(images), 2)
self.assertEqual(images[0]['id'], 4)
self.assertEqual(images[1]['id'], 2)
# Expect 3 images (1 deleted)
req = webob.Request.blank('/images/detail?changes-since=%s' % iso1)
res = req.get_response(self.api)
self.assertEquals(res.status_int, 200)
res_dict = json.loads(res.body)
images = res_dict['images']
self.assertEquals(len(images), 3)
self.assertEqual(images[0]['id'], 4)
self.assertEqual(images[1]['id'], 3) # deleted
self.assertEqual(images[2]['id'], 2)
# Expect 1 images (0 deleted)
req = webob.Request.blank('/images/detail?changes-since=%s' % iso2)
res = req.get_response(self.api)
self.assertEquals(res.status_int, 200)
res_dict = json.loads(res.body)
images = res_dict['images']
self.assertEquals(len(images), 1)
self.assertEqual(images[0]['id'], 4)
# Expect 0 images (0 deleted)
req = webob.Request.blank('/images/detail?changes-since=%s' % iso4)
res = req.get_response(self.api)
self.assertEquals(res.status_int, 200)
res_dict = json.loads(res.body)
images = res_dict['images']
self.assertEquals(len(images), 0)
# Bad request (empty changes-since param)
req = webob.Request.blank('/images/detail?changes-since=')
res = req.get_response(self.api)
self.assertEquals(res.status_int, 400)
# Bad request (invalid changes-since param)
req = webob.Request.blank('/images/detail?changes-since=2011-09-05')
res = req.get_response(self.api)
self.assertEquals(res.status_int, 400)
def test_image_is_checksummed(self):
"""Test that the image contents are checksummed properly"""
fixture_headers = {'x-image-meta-store': 'file',

148
glance/tests/unit/test_clients.py
View File

@ -27,6 +27,7 @@ import webob
from glance import client
from glance.common import context
from glance.common import exception
from glance.common import utils
from glance.registry.db import api as db_api
from glance.registry.db import models as db_models
from glance.registry import client as rclient
@ -721,6 +722,89 @@ class TestRegistryClient(unittest.TestCase):
for image in images:
self.assertTrue(image['size'] >= 20)
def test_get_image_details_with_changes_since(self):
"""Tests that a detailed call can be filtered by size_min"""
dt1 = datetime.datetime.utcnow() - datetime.timedelta(1)
iso1 = utils.isotime(dt1)
dt2 = datetime.datetime.utcnow() + datetime.timedelta(1)
iso2 = utils.isotime(dt2)
dt3 = datetime.datetime.utcnow() + datetime.timedelta(2)
iso3 = utils.isotime(dt3)
dt4 = datetime.datetime.utcnow() + datetime.timedelta(3)
iso4 = utils.isotime(dt4)
extra_fixture = {'id': 3,
'status': 'active',
'is_public': True,
'disk_format': 'vhd',
'container_format': 'ovf',
'name': 'fake image #3',
'size': 18,
'checksum': None}
db_api.image_create(self.context, extra_fixture)
db_api.image_destroy(self.context, 3)
extra_fixture = {'id': 4,
'status': 'active',
'is_public': True,
'disk_format': 'ami',
'container_format': 'ami',
'name': 'fake image #4',
'size': 20,
'checksum': None,
'created_at': dt3,
'updated_at': dt3}
db_api.image_create(self.context, extra_fixture)
# Check a standard list, 4 images in db (2 deleted)
images = self.client.get_images_detailed(filters={})
self.assertEquals(len(images), 2)
self.assertEqual(images[0]['id'], 4)
self.assertEqual(images[1]['id'], 2)
# Expect 3 images (1 deleted)
filters = {'changes-since': iso1}
images = self.client.get_images(filters=filters)
self.assertEquals(len(images), 3)
self.assertEqual(images[0]['id'], 4)
self.assertEqual(images[1]['id'], 3) # deleted
self.assertEqual(images[2]['id'], 2)
# Expect 1 images (0 deleted)
filters = {'changes-since': iso2}
images = self.client.get_images_detailed(filters=filters)
self.assertEquals(len(images), 1)
self.assertEqual(images[0]['id'], 4)
# Expect 0 images (0 deleted)
filters = {'changes-since': iso4}
images = self.client.get_images(filters=filters)
self.assertEquals(len(images), 0)
def test_get_image_details_with_changes_since(self):
"""Tests that a detailed call can be filtered by changes-since"""
extra_fixture = {'id': 3,
'status': 'saving',
'is_public': True,
'disk_format': 'vhd',
'container_format': 'ovf',
'name': 'new name! #123',
'size': 20,
'checksum': None}
db_api.image_create(self.context, extra_fixture)
images = self.client.get_images_detailed(filters={'size_min': 20})
self.assertEquals(len(images), 1)
for image in images:
self.assertTrue(image['size'] >= 20)
def test_get_image_details_by_property(self):
"""Tests that a detailed call can be filtered by a property"""
extra_fixture = {'id': 3,
@ -1302,6 +1386,70 @@ class TestClient(unittest.TestCase):
for image in images:
self.assertEquals('new name! #123', image['name'])
def test_get_image_details_with_changes_since(self):
"""Tests that a detailed call can be filtered by size_min"""
dt1 = datetime.datetime.utcnow() - datetime.timedelta(1)
iso1 = utils.isotime(dt1)
dt2 = datetime.datetime.utcnow() + datetime.timedelta(1)
iso2 = utils.isotime(dt2)
dt3 = datetime.datetime.utcnow() + datetime.timedelta(2)
iso3 = utils.isotime(dt3)
dt4 = datetime.datetime.utcnow() + datetime.timedelta(3)
iso4 = utils.isotime(dt4)
extra_fixture = {'id': 3,
'status': 'active',
'is_public': True,
'disk_format': 'vhd',
'container_format': 'ovf',
'name': 'fake image #3',
'size': 18,
'checksum': None}
db_api.image_create(self.context, extra_fixture)
db_api.image_destroy(self.context, 3)
extra_fixture = {'id': 4,
'status': 'active',
'is_public': True,
'disk_format': 'ami',
'container_format': 'ami',
'name': 'fake image #4',
'size': 20,
'checksum': None,
'created_at': dt3,
'updated_at': dt3}
db_api.image_create(self.context, extra_fixture)
# Check a standard list, 4 images in db (2 deleted)
images = self.client.get_images_detailed(filters={})
self.assertEquals(len(images), 2)
self.assertEqual(images[0]['id'], 4)
self.assertEqual(images[1]['id'], 2)
# Expect 3 images (1 deleted)
filters = {'changes-since': iso1}
images = self.client.get_images(filters=filters)
self.assertEquals(len(images), 3)
self.assertEqual(images[0]['id'], 4)
self.assertEqual(images[1]['id'], 3) # deleted
self.assertEqual(images[2]['id'], 2)
# Expect 1 images (0 deleted)
filters = {'changes-since': iso2}
images = self.client.get_images_detailed(filters=filters)
self.assertEquals(len(images), 1)
self.assertEqual(images[0]['id'], 4)
# Expect 0 images (0 deleted)
filters = {'changes-since': iso4}
images = self.client.get_images(filters=filters)
self.assertEquals(len(images), 0)
def test_get_image_details_by_property(self):
"""Tests that a detailed call can be filtered by a property"""
extra_fixture = {'id': 3,