Option to ignore mtime metadata entry.

Currently, the swiftclient upload command passes a custom metadata
header for each object (called object-meta-mtime), whose value is
the current UNIX timestamp. When downloading such an object with the
swiftclient, the mtime header is parsed and passed as the atime and
mtime for the newly created file.

There are use-cases where this is not desired, for example when using
tmp or scratch directories in which files older than a specific date
are deleted. This commit provides a boolean option for ignoring the
mtime header.

Change-Id: If60b389aa910c6f1969b999b5d3b6d0940375686

doc/manpages/swift.1

@@ -102,6 +102,9 @@ with \-\-no-download actually not to write anything to disk.
 The \-\-ignore-checksum is an option that turns off checksum validation.
 You can specify optional headers with the repeatable cURL-like option
 \-H [\-\-header]. For more details and options see swift download \-\-help.
+The \-\-ignore\-mtime option ignores the x\-object\-meta\-mtime metadata entry
+on the object (if present) and instead creates the downloaded files with
+fresh atime and mtime values.
 .RE
 
 \fBdelete\fR [\fIcommand-options\fR] [\fIcontainer\fR] [\fIobject\fR] [\fIobject\fR] [...]

doc/source/cli.rst

@@ -188,7 +188,9 @@ Download
        option to redirect the output to a specific file or ``-`` to
        redirect to stdout. The ``--ignore-checksum`` is an option that turn off
        checksum validation. You can specify optional headers with the repeatable
-       cURL-like option ``-H [--header <name:value>]``.
+       cURL-like option ``-H [--header <name:value>]``. ``--ignore-mtime`` ignores the
+       ``x-object-meta-mtime`` metadata entry on the object (if present) and instead
+       creates the downloaded files with fresh atime and mtime values.
 
 Delete
 ------

swiftclient/service.py

@@ -203,6 +203,7 @@ _default_local_options = {
     'shuffle': False,
     'destination': None,
     'fresh_metadata': False,
+    'ignore_mtime': False,
 }
 
 POLICY = 'X-Storage-Policy'
@@ -1240,7 +1241,8 @@ class SwiftService(object):
                 bytes_read = obj_body.bytes_read()
                 if fp is not None:
                     fp.close()
-                    if 'x-object-meta-mtime' in headers and not no_file:
+                    if ('x-object-meta-mtime' in headers and not no_file
+                            and not options['ignore_mtime']):
                         try:
                             mtime = float(headers['x-object-meta-mtime'])
                         except ValueError:

swiftclient/shell.py

@@ -272,6 +272,9 @@ Optional arguments:
                         script to multiple servers). Enable this option to
                         submit download jobs to the thread pool in the order
                         they are listed in the object store.
+  --ignore-mtime        Ignore the 'X-Object-Meta-Mtime' header when
+                        downloading an object. Instead, create atime and mtime
+                        with fresh timestamps.
 '''.strip("\n")
 
 
@@ -332,6 +335,12 @@ def st_download(parser, args, output_manager):
         'nightly automated download script to multiple servers). Enable this '
         'option to submit download jobs to the thread pool in the order they '
         'are listed in the object store.')
+    parser.add_argument(
+        '--ignore-mtime', action='store_true', dest='ignore_mtime',
+        default=False, help='By default, the object-meta-mtime header is used '
+        'to store the access and modified timestamp for the downloaded file. '
+        'With this option, the header is ignored and the timestamps are '
+        'created freshly.')
     (options, args) = parse_args(parser, args)
     args = args[1:]
     if options['out_file'] == '-':

tests/unit/test_service.py

@@ -2051,6 +2051,52 @@ class TestServiceDownload(_TestServiceBase):
         )
         self.assertEqual(expected_r, actual_r)
 
+    def test_download_object_job_ignore_mtime(self):
+        mock_conn = self._get_mock_connection()
+        objcontent = six.BytesIO(b'objcontent')
+        mock_conn.get_object.side_effect = [
+            ({'content-type': 'text/plain',
+              'etag': '2cbbfe139a744d6abbe695e17f3c1991',
+              'x-object-meta-mtime': '1454113727.682512'},
+             objcontent)
+        ]
+        expected_r = self._get_expected({
+            'success': True,
+            'start_time': 1,
+            'finish_time': 2,
+            'headers_receipt': 3,
+            'auth_end_time': 4,
+            'read_length': len(b'objcontent'),
+        })
+
+        with mock.patch.object(builtins, 'open') as mock_open, \
+                mock.patch('swiftclient.service.utime') as mock_utime:
+            written_content = Mock()
+            mock_open.return_value = written_content
+            s = SwiftService()
+            _opts = self.opts.copy()
+            _opts['no_download'] = False
+            _opts['ignore_mtime'] = True
+            actual_r = s._download_object_job(
+                mock_conn, 'test_c', 'test_o', _opts)
+            actual_r = dict(  # Need to override the times we got from the call
+                actual_r,
+                **{
+                    'start_time': 1,
+                    'finish_time': 2,
+                    'headers_receipt': 3
+                }
+            )
+            mock_open.assert_called_once_with('test_o', 'wb')
+            self.assertEqual([], mock_utime.mock_calls)
+            written_content.write.assert_called_once_with(b'objcontent')
+
+        mock_conn.get_object.assert_called_once_with(
+            'test_c', 'test_o', resp_chunk_size=65536, headers={},
+            response_dict={}
+        )
+        self.assertEqual(expected_r, actual_r)
+
     def test_download_object_job_exception(self):
         mock_conn = self._get_mock_connection()
         mock_conn.get_object = Mock(side_effect=self.exc)