Merge branch 'master' into feature/mpu

Signed-off-by: Tim Burke <tim.burke@gmail.com>
Change-Id: I359490aa178738d697534edcfaa40fca4f0c51bb

.zuul.yaml

@@ -16,7 +16,6 @@
 - job:
     name: swift-tox-py37
     parent: swift-tox-base
-    nodeset: ubuntu-bionic
     description: |
       Run unit-tests for swift under cPython version 3.7.
 
@@ -26,6 +25,7 @@
     vars:
       tox_envlist: py37
       bindep_profile: test py37
+      python_use_pyenv: True
       python_version: 3.7
     post-run: tools/playbooks/common/cover-post.yaml
 
@@ -489,10 +489,9 @@
 - job:
     name: swift-tox-lower-constraints
     parent: openstack-tox-lower-constraints
-    # This seems defensible for a l-c job
-    nodeset: ubuntu-bionic
     vars:
       bindep_profile: test py37
+      python_use_pyenv: True
       python_version: 3.7
       tox_environment:
         TMPDIR: '{{ ansible_env.HOME }}/xfstmp'
@@ -722,6 +721,7 @@
         - swift-tox-py37
         - swift-tox-py39
         - swift-tox-py312
+        - swift-tox-py313
         - swift-tox-func-py312
         - swift-tox-func-encryption-py312
         - swift-tox-func-ec-py312

swift/common/utils/checksum.py

@@ -25,12 +25,6 @@ import socket
 import struct
 import zlib
 
-try:
-    import pyeclib  # noqa
-    from importlib.metadata import files as pkg_files  # py38+
-except ImportError:
-    pkg_files = None
-
 
 # See if anycrc is available...
 if anycrc:
@@ -41,17 +35,30 @@ else:
     crc64nvme_anycrc = None
 
 
-# If isal is available system-wide, great!
+def find_isal():
-isal_lib = ctypes.util.find_library('isal')
+    # If isal is available system-wide, great!
-if isal_lib is None and pkg_files is not None:
+    isal_lib = ctypes.util.find_library('isal')
+    if isal_lib is None:
         # py38+: Hopefully pyeclib was installed from a manylinux wheel
         # with isal baked in?
-    isal_libs = [f for f in pkg_files('pyeclib')
+        try:
+            import pyeclib  # noqa
+            from importlib.metadata import files as pkg_files  # py38+
+        except ImportError:
+            pass
+        else:
+            pyeclib_files = pkg_files('pyeclib')
+            if not pyeclib_files:
+                # see https://docs.python.org/3/library/importlib.metadata.html
+                raise RuntimeError('pyeclib installed but missing files')
+            isal_libs = [f for f in pyeclib_files
                          if f.name.startswith("libisal")]
             if len(isal_libs) == 1:
                 isal_lib = isal_libs[0].locate()
+    return ctypes.CDLL(isal_lib) if isal_lib else None
 
-isal = ctypes.CDLL(isal_lib) if isal_lib else None
+
+isal = find_isal()
 
 if hasattr(isal, 'crc32_iscsi'):  # isa-l >= 2.16
     isal.crc32_iscsi.argtypes = [ctypes.c_char_p, ctypes.c_int, ctypes.c_uint]

swift/common/utils/timestamp.py

@@ -90,7 +90,7 @@ class Timestamp(object):
             timestamp = timestamp.decode('ascii')
         if isinstance(timestamp, str):
             base, base_offset = timestamp.partition('_')[::2]
-            self.timestamp = float(base)
+            float_timestamp = float(base)
             if '_' in base_offset:
                 raise ValueError('invalid literal for int() with base 16: '
                                  '%r' % base_offset)
@@ -99,7 +99,7 @@ class Timestamp(object):
             else:
                 self.offset = 0
         else:
-            self.timestamp = float(timestamp)
+            float_timestamp = float(timestamp)
             self.offset = getattr(timestamp, 'offset', 0)
         # increment offset
         if offset >= 0:
@@ -108,14 +108,15 @@ class Timestamp(object):
             raise ValueError('offset must be non-negative')
         if self.offset > MAX_OFFSET:
             raise ValueError('offset must be smaller than %d' % MAX_OFFSET)
-        self.raw = int(round(self.timestamp / PRECISION))
+        self.raw = int(round(float_timestamp / PRECISION))
         # add delta
         if delta:
             self.raw = self.raw + delta
             if self.raw <= 0:
                 raise ValueError(
                     'delta must be greater than %d' % (-1 * self.raw))
-            self.timestamp = float(self.raw * PRECISION)
+
+        self.timestamp = round(float(self.raw * PRECISION), 5)
         if check_bounds:
             if self.timestamp < 0:
                 raise ValueError('timestamp cannot be negative')

test/unit/common/middleware/s3api/test_utils.py

@@ -121,14 +121,19 @@ class TestS3Timestamp(unittest.TestCase):
         # integer
         ts = utils.S3Timestamp(1)
         self.assertEqual(expected, ts.s3xmlformat)
+        # microseconds digits are not included in Timestamp.normal so do not
+        # cause the timestamp to be rounded up
+        ts = utils.S3Timestamp(1.000001)
+        self.assertEqual(expected, ts.s3xmlformat)
+
         # milliseconds unit should be rounded up
         expected = '1970-01-01T00:00:02.000Z'
         ts = utils.S3Timestamp(1.1)
         self.assertEqual(expected, ts.s3xmlformat)
-        # float (microseconds) should be floored too
+        # float (deca-microseconds) should be rounded up too
-        ts = utils.S3Timestamp(1.000001)
+        ts = utils.S3Timestamp(1.000010)
         self.assertEqual(expected, ts.s3xmlformat)
-        # Bigger float (milliseconds) should be floored too
+        # Bigger float (milliseconds) should be rounded up too
         ts = utils.S3Timestamp(1.9)
         self.assertEqual(expected, ts.s3xmlformat)
 

test/unit/common/utils/test_checksum.py

@@ -13,6 +13,7 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
+import sys
 import unittest
 from unittest import mock
 
@@ -23,6 +24,19 @@ from test.debug_logger import debug_logger
 from test.unit import requires_crc32c, requires_crc64nvme
 
 
+class TestModuleFunctions(unittest.TestCase):
+    @unittest.skipIf(
+        sys.version_info.major == 3 and sys.version_info.minor < 8,
+        "importlib.metadata not available until py3.8")
+    def test_find_isal_pyeclib_dist_missing_files(self):
+        with mock.patch('ctypes.util.find_library', return_value=None):
+            with mock.patch('importlib.metadata.files', return_value=None):
+                with self.assertRaises(RuntimeError) as cm:
+                    checksum.find_isal()
+        self.assertEqual('pyeclib installed but missing files',
+                         str(cm.exception))
+
+
 # If you're curious about the 0xe3069283, see "check" at
 # https://reveng.sourceforge.io/crc-catalogue/17plus.htm#crc.cat.crc-32-iscsi
 class TestCRC32C(unittest.TestCase):

test/unit/common/utils/test_timestamp.py

@@ -172,10 +172,13 @@ class TestTimestamp(unittest.TestCase):
     def test_ceil(self):
         self.assertEqual(0.0, timestamp.Timestamp(0).ceil())
         self.assertEqual(1.0, timestamp.Timestamp(0.00001).ceil())
-        self.assertEqual(1.0, timestamp.Timestamp(0.000001).ceil())
+        self.assertEqual(0.0, timestamp.Timestamp(0.000001).ceil())
+        self.assertEqual(1.0, timestamp.Timestamp(0.000006).ceil())
         self.assertEqual(12345678.0, timestamp.Timestamp(12345678.0).ceil())
-        self.assertEqual(12345679.0,
+        self.assertEqual(12345678.0,
                          timestamp.Timestamp(12345678.000001).ceil())
+        self.assertEqual(12345679.0,
+                         timestamp.Timestamp(12345678.000006).ceil())
 
     def test_not_equal(self):
         ts = '1402436408.91203_0000000000000001'
@@ -332,11 +335,17 @@ class TestTimestamp(unittest.TestCase):
         expected = 140243640891203
         ts = timestamp.Timestamp(1402436408.91203)
         self.assertEqual(expected, ts.raw)
+        self.assertEqual('1402436408.91203', ts.normal)
 
         # 'raw' does not include offset
         ts = timestamp.Timestamp(1402436408.91203, 0xf0)
         self.assertEqual(expected, ts.raw)
 
+        expected = 175507756652338
+        ts = timestamp.Timestamp(1755077566.523385)
+        self.assertEqual(expected, ts.raw)
+        self.assertEqual('1755077566.52338', ts.normal)
+
     def test_delta(self):
         def _assertWithinBounds(expected, timestamp):
             tolerance = 0.00001
@@ -801,6 +810,23 @@ class TestTimestamp(unittest.TestCase):
         self.assertEqual(caught.exception.args[0],
                          'Cannot invert timestamps with offsets')
 
+    def test_inversion_reversibility(self):
+        ts = timestamp.Timestamp(1755077566.523385)
+        inv = ~ts
+        inv_inv = ~inv
+        self.assertEqual(ts, inv_inv)
+        self.assertEqual(ts.normal, inv_inv.normal)
+
+        inv_inv_inv = ~inv_inv
+        self.assertEqual(inv, inv_inv_inv)
+        self.assertEqual(inv.normal, inv_inv_inv.normal)
+
+        ts = timestamp.Timestamp.now()
+        inv = ~ts
+        inv_inv = ~inv
+        self.assertEqual(ts, inv_inv)
+        self.assertEqual(ts.normal, inv_inv.normal)
+
 
 class TestTimestampEncoding(unittest.TestCase):