Merge "Apply image location selection strategy"

glance/api/policy.py

@@ -16,6 +16,7 @@
 
 """Policy Engine For Glance"""
 
+import copy
 import os.path
 
 from oslo.config import cfg
@@ -304,6 +305,14 @@ class ImageLocationsProxy(object):
         self.context = context
         self.policy = policy
 
+    def __copy__(self):
+        return type(self)(self.locations, self.context, self.policy)
+
+    def __deepcopy__(self, memo):
+        # NOTE(zhiyan): Only copy location entries, others can be reused.
+        return type(self)(copy.deepcopy(self.locations, memo),
+                          self.context, self.policy)
+
     def _get_checker(action, func_name):
         def _checker(self, *args, **kwargs):
             self.policy.enforce(self.context, action, {})

glance/api/v2/images.py

@@ -21,6 +21,7 @@ import webob.exc
 
 from glance.api import policy
 from glance.common import exception
+from glance.common import location_strategy
 from glance.common import utils
 from glance.common import wsgi
 import glance.db
@@ -582,7 +583,10 @@ class ResponseSerializer(wsgi.JSONResponseSerializer):
                     image_view['locations'] = []
 
             if CONF.show_image_direct_url and image.locations:
-                image_view['direct_url'] = image.locations[0]['url']
+                # Choose best location configured strategy
+                best_location = (
+                    location_strategy.choose_best_location(image.locations))
+                image_view['direct_url'] = best_location['url']
 
             image_view['tags'] = list(image.tags)
             image_view['self'] = self._get_image_href(image)

glance/db/__init__.py

@@ -20,6 +20,7 @@ from oslo.config import cfg
 
 from glance.common import crypt
 from glance.common import exception
+from glance.common import location_strategy
 import glance.domain
 import glance.domain.proxy
 from glance.openstack.common import importutils
@@ -105,7 +106,7 @@ class ImageRepo(object):
             min_disk=db_image['min_disk'],
             min_ram=db_image['min_ram'],
             protected=db_image['protected'],
-            locations=locations,
+            locations=location_strategy.get_ordered_locations(locations),
             checksum=db_image['checksum'],
             owner=db_image['owner'],
             disk_format=db_image['disk_format'],

glance/quota/__init__.py

@@ -12,6 +12,7 @@
 #    License for the specific language governing permissions and limitations
 #    under the License.
 
+import copy
 
 from oslo.config import cfg
 
@@ -242,6 +243,14 @@ class QuotaImageLocationsProxy(object):
                                       self.db_api)
         _enforce_image_location_quota(self.image, locations)
 
+    def __copy__(self):
+        return type(self)(self.image, self.context, self.db_api)
+
+    def __deepcopy__(self, memo):
+        # NOTE(zhiyan): Only copy location entries, others can be reused.
+        self.image.locations = copy.deepcopy(self.locations, memo)
+        return type(self)(self.image, self.context, self.db_api)
+
     def append(self, object):
         self._check_user_storage_quota([object])
         return self.locations.append(object)

glance/store/__init__.py

@@ -14,6 +14,7 @@
 #    under the License.
 
 import collections
+import copy
 import sys
 
 from oslo.config import cfg
@@ -622,6 +623,15 @@ class StoreLocations(collections.MutableSequence):
     def __iter__(self):
         return iter(self.value)
 
+    def __copy__(self):
+        return type(self)(self.image_proxy, self.value)
+
+    def __deepcopy__(self, memo):
+        # NOTE(zhiyan): Only copy location entries, others can be reused.
+        value = copy.deepcopy(self.value, memo)
+        self.image_proxy.image.locations = value
+        return type(self)(self.image_proxy, value)
+
 
 def _locations_proxy(target, attr):
     """

glance/tests/functional/__init__.py

@@ -322,6 +322,9 @@ class ApiServer(Server):
         self.user_storage_quota = 0
         self.lock_path = self.test_dir
 
+        self.location_strategy = 'location_order'
+        self.store_type_location_strategy_preference = ""
+
         self.conf_base = """[DEFAULT]
 verbose = %(verbose)s
 debug = %(debug)s
@@ -379,8 +382,11 @@ image_member_quota=%(image_member_quota)s
 image_property_quota=%(image_property_quota)s
 image_tag_quota=%(image_tag_quota)s
 image_location_quota=%(image_location_quota)s
+location_strategy=%(location_strategy)s
 [paste_deploy]
 flavor = %(deployment_flavor)s
+[store_type_location_strategy]
+store_type_preference = %(store_type_location_strategy_preference)s
 """
         self.paste_conf_base = """[pipeline:glance-api]
 pipeline = versionnegotiation gzip unauthenticated-context rootapp

glance/tests/functional/v2/test_images.py

@@ -14,12 +14,15 @@
 #    under the License.
 
 import os
+import signal
+import tempfile
 import uuid
 
 import requests
 
 from glance.openstack.common import jsonutils
 from glance.tests import functional
+from glance.tests.functional.store import test_http
 
 
 TENANT1 = str(uuid.uuid4())
@@ -1976,6 +1979,160 @@ class TestImageDirectURLVisibility(functional.FunctionalTest):
         self.stop_servers()
 
 
+class TestImageLocationSelectionStrategy(functional.FunctionalTest):
+
+    def setUp(self):
+        super(TestImageLocationSelectionStrategy, self).setUp()
+        self.cleanup()
+        self.api_server.deployment_flavor = 'noauth'
+        self.foo_image_file = tempfile.NamedTemporaryFile()
+        self.foo_image_file.write("foo image file")
+        self.foo_image_file.flush()
+        self.addCleanup(self.foo_image_file.close)
+        ret = test_http.http_server("foo_image_id", "foo_image")
+        self.http_server_pid, self.http_port = ret
+
+    def tearDown(self):
+        if self.http_server_pid is not None:
+            os.kill(self.http_server_pid, signal.SIGKILL)
+
+        super(TestImageLocationSelectionStrategy, self).tearDown()
+
+    def _url(self, path):
+        return 'http://127.0.0.1:%d%s' % (self.api_port, path)
+
+    def _headers(self, custom_headers=None):
+        base_headers = {
+            'X-Identity-Status': 'Confirmed',
+            'X-Auth-Token': '932c5c84-02ac-4fe5-a9ba-620af0e2bb96',
+            'X-User-Id': 'f9a41d13-0c13-47e9-bee2-ce4e8bfe958e',
+            'X-Tenant-Id': TENANT1,
+            'X-Roles': 'member',
+        }
+        base_headers.update(custom_headers or {})
+        return base_headers
+
+    def test_image_locations_with_order_strategy(self):
+        self.api_server.show_image_direct_url = True
+        self.api_server.show_multiple_locations = True
+        self.image_location_quota = 10
+        self.api_server.location_strategy = 'location_order'
+        preference = "http, swift, filesystem"
+        self.api_server.store_type_location_strategy_preference = preference
+        self.start_servers(**self.__dict__.copy())
+
+        # Create an image
+        path = self._url('/v2/images')
+        headers = self._headers({'content-type': 'application/json'})
+        data = jsonutils.dumps({'name': 'image-1', 'type': 'kernel',
+                                'foo': 'bar', 'disk_format': 'aki',
+                                'container_format': 'aki'})
+        response = requests.post(path, headers=headers, data=data)
+        self.assertEqual(201, response.status_code)
+
+        # Get the image id
+        image = jsonutils.loads(response.text)
+        image_id = image['id']
+
+        # Image locations should not be visible before location is set
+        path = self._url('/v2/images/%s' % image_id)
+        headers = self._headers({'Content-Type': 'application/json'})
+        response = requests.get(path, headers=headers)
+        self.assertEqual(200, response.status_code)
+        image = jsonutils.loads(response.text)
+        self.assertTrue('locations' in image)
+        self.assertTrue(image["locations"] == [])
+
+       # Update image locations via PATCH
+        path = self._url('/v2/images/%s' % image_id)
+        media_type = 'application/openstack-images-v2.1-json-patch'
+        headers = self._headers({'content-type': media_type})
+        values = [{'url': 'file://%s' % self.foo_image_file.name,
+                   'metadata': {'idx': '1'}},
+                  {'url': 'http://127.0.0.1:%s/foo_image' % self.http_port,
+                   'metadata': {'idx': '0'}}]
+        doc = [{'op': 'replace',
+                'path': '/locations',
+                'value': values}]
+        data = jsonutils.dumps(doc)
+        response = requests.patch(path, headers=headers, data=data)
+        self.assertEqual(200, response.status_code)
+
+        # Image locations should be visible
+        path = self._url('/v2/images/%s' % image_id)
+        headers = self._headers({'Content-Type': 'application/json'})
+        response = requests.get(path, headers=headers)
+        self.assertEqual(200, response.status_code)
+        image = jsonutils.loads(response.text)
+        self.assertTrue('locations' in image)
+        self.assertEqual(image['locations'], values)
+        self.assertTrue('direct_url' in image)
+        self.assertEqual(image['direct_url'], values[0]['url'])
+
+        self.stop_servers()
+
+    def test_image_locatons_with_store_type_strategy(self):
+        self.api_server.show_image_direct_url = True
+        self.api_server.show_multiple_locations = True
+        self.image_location_quota = 10
+        self.api_server.location_strategy = 'store_type'
+        preference = "http, swift, filesystem"
+        self.api_server.store_type_location_strategy_preference = preference
+        self.start_servers(**self.__dict__.copy())
+
+        # Create an image
+        path = self._url('/v2/images')
+        headers = self._headers({'content-type': 'application/json'})
+        data = jsonutils.dumps({'name': 'image-1', 'type': 'kernel',
+                                'foo': 'bar', 'disk_format': 'aki',
+                                'container_format': 'aki'})
+        response = requests.post(path, headers=headers, data=data)
+        self.assertEqual(201, response.status_code)
+
+        # Get the image id
+        image = jsonutils.loads(response.text)
+        image_id = image['id']
+
+        # Image locations should not be visible before location is set
+        path = self._url('/v2/images/%s' % image_id)
+        headers = self._headers({'Content-Type': 'application/json'})
+        response = requests.get(path, headers=headers)
+        self.assertEqual(200, response.status_code)
+        image = jsonutils.loads(response.text)
+        self.assertTrue('locations' in image)
+        self.assertTrue(image["locations"] == [])
+
+       # Update image locations via PATCH
+        path = self._url('/v2/images/%s' % image_id)
+        media_type = 'application/openstack-images-v2.1-json-patch'
+        headers = self._headers({'content-type': media_type})
+        values = [{'url': 'file://%s' % self.foo_image_file.name,
+                   'metadata': {'idx': '1'}},
+                  {'url': 'http://127.0.0.1:%s/foo_image' % self.http_port,
+                   'metadata': {'idx': '0'}}]
+        doc = [{'op': 'replace',
+                'path': '/locations',
+                'value': values}]
+        data = jsonutils.dumps(doc)
+        response = requests.patch(path, headers=headers, data=data)
+        self.assertEqual(200, response.status_code)
+
+        values.sort(key=lambda loc: int(loc['metadata']['idx']))
+
+        # Image locations should be visible
+        path = self._url('/v2/images/%s' % image_id)
+        headers = self._headers({'Content-Type': 'application/json'})
+        response = requests.get(path, headers=headers)
+        self.assertEqual(200, response.status_code)
+        image = jsonutils.loads(response.text)
+        self.assertTrue('locations' in image)
+        self.assertEqual(image['locations'], values)
+        self.assertTrue('direct_url' in image)
+        self.assertEqual(image['direct_url'], values[0]['url'])
+
+        self.stop_servers()
+
+
 class TestImageMembers(functional.FunctionalTest):
 
     def setUp(self):