openstack/openstacksdk
Code Issues Proposed changes

Merge "Optimize code by moving misc functions to utils" into feature/r1

Browse Source
This commit is contained in:
Zuul 2021-09-08 18:27:37 +00:00 committed by Gerrit Code Review
commit 5f9cc4fe00
5 changed files with 53 additions and 61 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

57
openstack/cloud/_object_store.py
View File

@ -15,7 +15,6 @@
# openstack.resource.Resource.list and openstack.resource2.Resource.list
import collections
import concurrent.futures
import hashlib
import json
import os
import types # noqa
@ -196,39 +195,6 @@ class ObjectStoreCloudMixin(_normalize.Normalizer):
raise exc.OpenStackCloudException(
"Could not determine container access for ACL: %s." % acl)
def _get_file_hashes(self, filename):
file_key = "{filename}:{mtime}".format(
filename=filename,
mtime=os.stat(filename).st_mtime)
if file_key not in self._file_hash_cache:
self.log.debug(
'Calculating hashes for %(filename)s', {'filename': filename})
(md5, sha256) = (None, None)
with open(filename, 'rb') as file_obj:
(md5, sha256) = self._calculate_data_hashes(file_obj)
self._file_hash_cache[file_key] = dict(
md5=md5, sha256=sha256)
self.log.debug(
"Image file %(filename)s md5:%(md5)s sha256:%(sha256)s",
{'filename': filename,
'md5': self._file_hash_cache[file_key]['md5'],
'sha256': self._file_hash_cache[file_key]['sha256']})
return (self._file_hash_cache[file_key]['md5'],
self._file_hash_cache[file_key]['sha256'])
def _calculate_data_hashes(self, data):
md5 = utils.md5(usedforsecurity=False)
sha256 = hashlib.sha256()
if hasattr(data, 'read'):
for chunk in iter(lambda: data.read(8192), b''):
md5.update(chunk)
sha256.update(chunk)
else:
md5.update(data)
sha256.update(data)
return (md5.hexdigest(), sha256.hexdigest())
@_utils.cache_on_arguments()
def get_object_capabilities(self):
"""Get infomation about the object-storage service
@ -293,13 +259,13 @@ class ObjectStoreCloudMixin(_normalize.Normalizer):
return True
if not (file_md5 or file_sha256):
(file_md5, file_sha256) = self._get_file_hashes(filename)
(file_md5, file_sha256) = utils._get_file_hashes(filename)
md5_key = metadata.get(
self._OBJECT_MD5_KEY, metadata.get(self._SHADE_OBJECT_MD5_KEY, ''))
sha256_key = metadata.get(
self._OBJECT_SHA256_KEY, metadata.get(
self._SHADE_OBJECT_SHA256_KEY, ''))
up_to_date = self._hashes_up_to_date(
up_to_date = utils._hashes_up_to_date(
md5=file_md5, sha256=file_sha256,
md5_key=md5_key, sha256_key=sha256_key)
@ -405,7 +371,7 @@ class ObjectStoreCloudMixin(_normalize.Normalizer):
filename = name
if generate_checksums and (md5 is None or sha256 is None):
(md5, sha256) = self._get_file_hashes(filename)
(md5, sha256) = utils._get_file_hashes(filename)
if md5:
headers[self._OBJECT_MD5_KEY] = md5 or ''
if sha256:
@ -857,20 +823,3 @@ class ObjectStoreCloudMixin(_normalize.Normalizer):
# can try again
retries.append(completed.result())
return results, retries
def _hashes_up_to_date(self, md5, sha256, md5_key, sha256_key):
'''Compare md5 and sha256 hashes for being up to date
md5 and sha256 are the current values.
md5_key and sha256_key are the previous values.
'''
up_to_date = False
if md5 and md5_key == md5:
up_to_date = True
if sha256 and sha256_key == sha256:
up_to_date = True
if md5 and md5_key != md5:
up_to_date = False
if sha256 and sha256_key != sha256:
up_to_date = False
return up_to_date

7
openstack/image/_base_proxy.py
View File

@ -15,6 +15,7 @@ import os
from openstack import exceptions
from openstack import proxy
from openstack import utils
class BaseImageProxy(proxy.Proxy, metaclass=abc.ABCMeta):
@ -151,9 +152,9 @@ class BaseImageProxy(proxy.Proxy, metaclass=abc.ABCMeta):
'direct binary object')
if not (md5 or sha256) and validate_checksum:
if filename:
(md5, sha256) = self._connection._get_file_hashes(filename)
(md5, sha256) = utils._get_file_hashes(filename)
elif data and isinstance(data, bytes):
(md5, sha256) = self._connection._calculate_data_hashes(data)
(md5, sha256) = utils._calculate_data_hashes(data)
if allow_duplicates:
current_image = None
else:
@ -167,7 +168,7 @@ class BaseImageProxy(proxy.Proxy, metaclass=abc.ABCMeta):
sha256_key = props.get(
self._IMAGE_SHA256_KEY,
props.get(self._SHADE_IMAGE_SHA256_KEY, ''))
up_to_date = self._connection._hashes_up_to_date(
up_to_date = utils._hashes_up_to_date(
md5=md5, sha256=sha256,
md5_key=md5_key, sha256_key=sha256_key)
if up_to_date:

7
openstack/object_store/v1/_proxy.py
View File

@ -27,6 +27,7 @@ from openstack.object_store.v1 import container as _container
from openstack.object_store.v1 import info as _info
from openstack.object_store.v1 import obj as _obj
from openstack import proxy
from openstack import utils
DEFAULT_OBJECT_SEGMENT_SIZE = 1073741824 # 1GB
DEFAULT_MAX_FILE_SIZE = (5 * 1024 * 1024 * 1024 + 2) / 2
@ -353,7 +354,7 @@ class Proxy(proxy.Proxy):
filename = name
if generate_checksums and (md5 is None or sha256 is None):
(md5, sha256) = self._connection._get_file_hashes(filename)
(md5, sha256) = utils._get_file_hashes(filename)
if md5:
headers[self._connection._OBJECT_MD5_KEY] = md5 or ''
if sha256:
@ -513,14 +514,14 @@ class Proxy(proxy.Proxy):
if not (file_md5 or file_sha256):
(file_md5, file_sha256) = \
self._connection._get_file_hashes(filename)
utils._get_file_hashes(filename)
md5_key = metadata.get(
self._connection._OBJECT_MD5_KEY,
metadata.get(self._connection._SHADE_OBJECT_MD5_KEY, ''))
sha256_key = metadata.get(
self._connection._OBJECT_SHA256_KEY, metadata.get(
self._connection._SHADE_OBJECT_SHA256_KEY, ''))
up_to_date = self._connection._hashes_up_to_date(
up_to_date = utils._hashes_up_to_date(
md5=file_md5, sha256=file_sha256,
md5_key=md5_key, sha256_key=sha256_key)

3
openstack/tests/unit/cloud/test_object.py
View File

@ -25,6 +25,7 @@ from openstack.object_store.v1 import _proxy
from openstack.object_store.v1 import container
from openstack.object_store.v1 import obj
from openstack.tests.unit import base
from openstack import utils
class BaseTestObject(base.TestCase):
@ -654,7 +655,7 @@ class TestObjectUploads(BaseTestObject):
self.object_file = tempfile.NamedTemporaryFile(delete=False)
self.object_file.write(self.content)
self.object_file.close()
(self.md5, self.sha256) = self.cloud._get_file_hashes(
(self.md5, self.sha256) = utils._get_file_hashes(
self.object_file.name)
self.endpoint = self.cloud._object_store_client.get_endpoint()

40
openstack/utils.py
View File

@ -233,6 +233,24 @@ def maximum_supported_microversion(adapter, client_maximum):
return discover.version_to_string(result)
def _hashes_up_to_date(md5, sha256, md5_key, sha256_key):
'''Compare md5 and sha256 hashes for being up to date
md5 and sha256 are the current values.
md5_key and sha256_key are the previous values.
'''
up_to_date = False
if md5 and md5_key == md5:
up_to_date = True
if sha256 and sha256_key == sha256:
up_to_date = True
if md5 and md5_key != md5:
up_to_date = False
if sha256 and sha256_key != sha256:
up_to_date = False
return up_to_date
try:
_test_md5 = hashlib.md5(usedforsecurity=False) # nosec
@ -253,6 +271,28 @@ except TypeError:
return hashlib.md5(string) # nosec
def _calculate_data_hashes(data):
_md5 = md5(usedforsecurity=False)
_sha256 = hashlib.sha256()
if hasattr(data, 'read'):
for chunk in iter(lambda: data.read(8192), b''):
_md5.update(chunk)
_sha256.update(chunk)
else:
_md5.update(data)
_sha256.update(data)
return (_md5.hexdigest(), _sha256.hexdigest())
def _get_file_hashes(filename):
(_md5, _sha256) = (None, None)
with open(filename, 'rb') as file_obj:
(_md5, _sha256) = _calculate_data_hashes(file_obj)
return (_md5, _sha256)
class TinyDAG:
"""Tiny DAG