Update create_object to handled chunked data

We try to calculate len for data, which breaks with chunked uploads, but we only use that with filename. Move the code so that we don't try to calculate length of things where we don't need to. We still need to figure out streaming uploads and large objects. This changes several unit tests due to the re-ordering of when get_object_segment_size gets called. It's not necessary to call it for the data path, so it was moved to later, which puts it after the container create. For data paths this results in one less call. For file paths, the calls are the same but the /info call happens later. Change-Id: I841d7049ff2f05e00a31fecbc27dda6a0007be16
2018-07-24 15:00:14 -05:00 · 2018-07-24 15:00:14 -05:00 · 94b2bf06de
commit 94b2bf06de
parent 9281c5b755
4 changed files with 47 additions and 55 deletions
--- a/openstack/cloud/openstackcloud.py
+++ b/openstack/cloud/openstackcloud.py
@ -7514,16 +7514,6 @@ class OpenStackCloud(_normalize.Normalizer):
        if not filename and data is None:
            filename = name

-        # segment_size gets used as a step value in a range call, so needs
-        # to be an int
-        if segment_size:
-            segment_size = int(segment_size)
-        segment_size = self.get_object_segment_size(segment_size)
-        if filename:
-            file_size = os.path.getsize(filename)
-        else:
-            file_size = len(data)
-
        if generate_checksums and (md5 is None or sha256 is None):
            (md5, sha256) = self._get_file_hashes(filename)
        if md5:
@ -7545,6 +7535,13 @@ class OpenStackCloud(_normalize.Normalizer):

            return self._upload_object_data(endpoint, data, headers)

+        # segment_size gets used as a step value in a range call, so needs
+        # to be an int
+        if segment_size:
+            segment_size = int(segment_size)
+        segment_size = self.get_object_segment_size(segment_size)
+        file_size = os.path.getsize(filename)
+
        if self.is_object_stale(container, name, filename, md5, sha256):

            self.log.debug(
--- a/openstack/tests/unit/cloud/test_image.py
+++ b/openstack/tests/unit/cloud/test_image.py
@ -364,13 +364,6 @@ class TestImage(BaseTestImage):
                 uri=self.get_mock_url(
                     'image', append=['images'], base_url_append='v2'),
                 json={'images': []}),
-            dict(method='GET',
-                 # This is explicitly not using get_mock_url because that
-                 # gets us a project-id oriented URL.
-                 uri='https://object-store.example.com/info',
-                 json=dict(
-                     swift={'max_file_size': 1000},
-                     slo={'min_segment_size': 500})),
            dict(method='HEAD',
                 uri='{endpoint}/{container}'.format(
                     endpoint=endpoint, container=self.container_name),
@ -395,6 +388,13 @@ class TestImage(BaseTestImage):
                     'X-Trans-Id': 'tx60ec128d9dbf44b9add68-0058543271dfw1',
                     'X-Container-Bytes-Used': '0',
                     'Content-Type': 'text/plain; charset=utf-8'}),
+            dict(method='GET',
+                 # This is explicitly not using get_mock_url because that
+                 # gets us a project-id oriented URL.
+                 uri='https://object-store.example.com/info',
+                 json=dict(
+                     swift={'max_file_size': 1000},
+                     slo={'min_segment_size': 500})),
            dict(method='HEAD',
                 uri='{endpoint}/{container}/{object}'.format(
                     endpoint=endpoint, container=self.container_name,
--- a/openstack/tests/unit/cloud/test_object.py
+++ b/openstack/tests/unit/cloud/test_object.py
@ -415,11 +415,6 @@ class TestObjectUploads(BaseTestObject):
    def test_create_object(self):

        self.register_uris([
-            dict(method='GET',
-                 uri='https://object-store.example.com/info',
-                 json=dict(
-                     swift={'max_file_size': 1000},
-                     slo={'min_segment_size': 500})),
            dict(method='HEAD',
                 uri='{endpoint}/{container}'.format(
                     endpoint=self.endpoint,
@ -447,6 +442,11 @@ class TestObjectUploads(BaseTestObject):
                     'X-Trans-Id': 'tx60ec128d9dbf44b9add68-0058543271dfw1',
                     'X-Container-Bytes-Used': '0',
                     'Content-Type': 'text/plain; charset=utf-8'}),
+            dict(method='GET',
+                 uri='https://object-store.example.com/info',
+                 json=dict(
+                     swift={'max_file_size': 1000},
+                     slo={'min_segment_size': 500})),
            dict(method='HEAD',
                 uri='{endpoint}/{container}/{object}'.format(
                     endpoint=self.endpoint, container=self.container,
@ -473,11 +473,6 @@ class TestObjectUploads(BaseTestObject):
    def test_create_directory_marker_object(self):

        self.register_uris([
-            dict(method='GET',
-                 uri='https://object-store.example.com/info',
-                 json=dict(
-                     swift={'max_file_size': 1000},
-                     slo={'min_segment_size': 500})),
            dict(method='HEAD',
                 uri='{endpoint}/{container}'.format(
                     endpoint=self.endpoint,
@ -527,12 +522,6 @@ class TestObjectUploads(BaseTestObject):
        min_file_size = 1

        uris_to_mock = [
-            dict(method='GET',
-                 uri='https://object-store.example.com/info',
-                 json=dict(
-                     swift={'max_file_size': max_file_size},
-                     slo={'min_segment_size': min_file_size})),
-
            dict(method='HEAD',
                 uri='{endpoint}/{container}'.format(
                     endpoint=self.endpoint,
@ -562,6 +551,11 @@ class TestObjectUploads(BaseTestObject):
                     'X-Trans-Id': 'tx60ec128d9dbf44b9add68-0058543271dfw1',
                     'X-Container-Bytes-Used': '0',
                     'Content-Type': 'text/plain; charset=utf-8'}),
+            dict(method='GET',
+                 uri='https://object-store.example.com/info',
+                 json=dict(
+                     swift={'max_file_size': max_file_size},
+                     slo={'min_segment_size': min_file_size})),
            dict(method='HEAD',
                 uri='{endpoint}/{container}/{object}'.format(
                     endpoint=self.endpoint, container=self.container,
@ -613,10 +607,6 @@ class TestObjectUploads(BaseTestObject):
        min_file_size = 1

        uris_to_mock = [
-            dict(method='GET', uri='https://object-store.example.com/info',
-                 json=dict(
-                     swift={'max_file_size': max_file_size},
-                     slo={'min_segment_size': min_file_size})),
            dict(method='HEAD',
                 uri='{endpoint}/{container}'.format(
                     endpoint=self.endpoint,
@ -646,6 +636,10 @@ class TestObjectUploads(BaseTestObject):
                     'X-Trans-Id': 'tx60ec128d9dbf44b9add68-0058543271dfw1',
                     'X-Container-Bytes-Used': '0',
                     'Content-Type': 'text/plain; charset=utf-8'}),
+            dict(method='GET', uri='https://object-store.example.com/info',
+                 json=dict(
+                     swift={'max_file_size': max_file_size},
+                     slo={'min_segment_size': min_file_size})),
            dict(method='HEAD',
                 uri='{endpoint}/{container}/{object}'.format(
                     endpoint=self.endpoint,
@ -732,10 +726,6 @@ class TestObjectUploads(BaseTestObject):
        min_file_size = 1

        self.register_uris([
-            dict(method='GET', uri='https://object-store.example.com/info',
-                 json=dict(
-                     swift={'max_file_size': max_file_size},
-                     slo={'min_segment_size': min_file_size})),
            dict(method='HEAD',
                 uri='{endpoint}/{container}'.format(
                     endpoint=self.endpoint,
@ -765,6 +755,10 @@ class TestObjectUploads(BaseTestObject):
                     'X-Trans-Id': 'tx60ec128d9dbf44b9add68-0058543271dfw1',
                     'X-Container-Bytes-Used': '0',
                     'Content-Type': 'text/plain; charset=utf-8'}),
+            dict(method='GET', uri='https://object-store.example.com/info',
+                 json=dict(
+                     swift={'max_file_size': max_file_size},
+                     slo={'min_segment_size': min_file_size})),
            dict(method='HEAD',
                 uri='{endpoint}/{container}/{object}'.format(
                     endpoint=self.endpoint,
@ -816,10 +810,6 @@ class TestObjectUploads(BaseTestObject):
        min_file_size = 1

        self.register_uris([
-            dict(method='GET', uri='https://object-store.example.com/info',
-                 json=dict(
-                     swift={'max_file_size': max_file_size},
-                     slo={'min_segment_size': min_file_size})),
            dict(method='HEAD',
                 uri='{endpoint}/{container}'.format(
                     endpoint=self.endpoint,
@ -849,6 +839,10 @@ class TestObjectUploads(BaseTestObject):
                     'X-Trans-Id': 'tx60ec128d9dbf44b9add68-0058543271dfw1',
                     'X-Container-Bytes-Used': '0',
                     'Content-Type': 'text/plain; charset=utf-8'}),
+            dict(method='GET', uri='https://object-store.example.com/info',
+                 json=dict(
+                     swift={'max_file_size': max_file_size},
+                     slo={'min_segment_size': min_file_size})),
            dict(method='HEAD',
                 uri='{endpoint}/{container}/{object}'.format(
                     endpoint=self.endpoint,
@ -950,11 +944,6 @@ class TestObjectUploads(BaseTestObject):
    def test_create_object_skip_checksum(self):

        self.register_uris([
-            dict(method='GET',
-                 uri='https://object-store.example.com/info',
-                 json=dict(
-                     swift={'max_file_size': 1000},
-                     slo={'min_segment_size': 500})),
            dict(method='HEAD',
                 uri='{endpoint}/{container}'.format(
                     endpoint=self.endpoint,
@ -982,6 +971,11 @@ class TestObjectUploads(BaseTestObject):
                     'X-Trans-Id': 'tx60ec128d9dbf44b9add68-0058543271dfw1',
                     'X-Container-Bytes-Used': '0',
                     'Content-Type': 'text/plain; charset=utf-8'}),
+            dict(method='GET',
+                 uri='https://object-store.example.com/info',
+                 json=dict(
+                     swift={'max_file_size': 1000},
+                     slo={'min_segment_size': 500})),
            dict(method='HEAD',
                 uri='{endpoint}/{container}/{object}'.format(
                     endpoint=self.endpoint, container=self.container,
@ -1005,11 +999,6 @@ class TestObjectUploads(BaseTestObject):
    def test_create_object_data(self):

        self.register_uris([
-            dict(method='GET',
-                 uri='https://object-store.example.com/info',
-                 json=dict(
-                     swift={'max_file_size': 1000},
-                     slo={'min_segment_size': 500})),
            dict(method='HEAD',
                 uri='{endpoint}/{container}'.format(
                     endpoint=self.endpoint,
--- a/releasenotes/notes/object-chunked-data-ee619b7d4759b8d2.yaml
+++ b/releasenotes/notes/object-chunked-data-ee619b7d4759b8d2.yaml
@ -0,0 +1,6 @@
+---
+fixes:
+  - |
+    Fixed an issue where passing an iterator to the ``data`` parameter of
+    ``create_object`` for chunked uploads failed due to attempting to
+    calculate the length of the data.