Hide the file descriptor and disk write methodology for PUTs

Towards moving the DiskFile class into place as the API definition for
pluggable DiskFile backends, we hide the file descriptor and the
method of writing data to disks. The mkstemp() method has been renamed
to writer(), and no longer returns an fd, but a new object that
encapsulates the state tracked for writes. This new object is then
used directly to perform the reminder of the write operations and
application of required semantics.

Change-Id: Ib37ed37b34a2ce6b442d69f83ca011c918114434
Signed-off-by: Peter Portante <peter.portante@redhat.com>

test/unit/obj/test_auditor.py

@@ -62,17 +62,17 @@ class TestAuditor(unittest.TestCase):
         self.auditor = auditor.AuditorWorker(self.conf, self.logger)
         data = '0' * 1024
         etag = md5()
-        with self.disk_file.mkstemp() as fd:
-            os.write(fd, data)
+        with self.disk_file.writer() as writer:
+            writer.write(data)
             etag.update(data)
             etag = etag.hexdigest()
             timestamp = str(normalize_timestamp(time.time()))
             metadata = {
                 'ETag': etag,
                 'X-Timestamp': timestamp,
-                'Content-Length': str(os.fstat(fd).st_size),
+                'Content-Length': str(os.fstat(writer.fd).st_size),
             }
-            self.disk_file.put(fd, 1024, metadata)
+            writer.put(metadata)
             pre_quarantines = self.auditor.quarantines
 
             self.auditor.object_audit(
@@ -80,7 +80,7 @@ class TestAuditor(unittest.TestCase):
                 'sda', '0')
             self.assertEquals(self.auditor.quarantines, pre_quarantines)
 
-            os.write(fd, 'extra_data')
+            os.write(writer.fd, 'extra_data')
             self.auditor.object_audit(
                 os.path.join(self.disk_file.datadir, timestamp + '.data'),
                 'sda', '0')
@@ -91,35 +91,39 @@ class TestAuditor(unittest.TestCase):
         data = '0' * 1024
         etag = md5()
         timestamp = str(normalize_timestamp(time.time()))
-        with self.disk_file.mkstemp() as fd:
-            os.write(fd, data)
+        with self.disk_file.writer() as writer:
+            writer.write(data)
             etag.update(data)
             etag = etag.hexdigest()
             metadata = {
                 'ETag': etag,
                 'X-Timestamp': timestamp,
-                'Content-Length': str(os.fstat(fd).st_size),
+                'Content-Length': str(os.fstat(writer.fd).st_size),
             }
-            self.disk_file.put(fd, 1024, metadata)
+            writer.put(metadata)
             pre_quarantines = self.auditor.quarantines
-            # remake so it will have metadata
-            self.disk_file = DiskFile(self.devices, 'sda', '0', 'a', 'c', 'o',
-                                      self.logger)
 
-            self.auditor.object_audit(
-                os.path.join(self.disk_file.datadir, timestamp + '.data'),
-                'sda', '0')
-            self.assertEquals(self.auditor.quarantines, pre_quarantines)
-            etag = md5()
-            etag.update('1' + '0' * 1023)
-            etag = etag.hexdigest()
-            metadata['ETag'] = etag
-            write_metadata(fd, metadata)
+        # remake so it will have metadata
+        self.disk_file = DiskFile(self.devices, 'sda', '0', 'a', 'c', 'o',
+                                  self.logger)
 
-            self.auditor.object_audit(
-                os.path.join(self.disk_file.datadir, timestamp + '.data'),
-                'sda', '0')
-            self.assertEquals(self.auditor.quarantines, pre_quarantines + 1)
+        self.auditor.object_audit(
+            os.path.join(self.disk_file.datadir, timestamp + '.data'),
+            'sda', '0')
+        self.assertEquals(self.auditor.quarantines, pre_quarantines)
+        etag = md5()
+        etag.update('1' + '0' * 1023)
+        etag = etag.hexdigest()
+        metadata['ETag'] = etag
+
+        with self.disk_file.writer() as writer:
+            writer.write(data)
+            writer.put(metadata)
+
+        self.auditor.object_audit(
+            os.path.join(self.disk_file.datadir, timestamp + '.data'),
+            'sda', '0')
+        self.assertEquals(self.auditor.quarantines, pre_quarantines + 1)
 
     def test_object_audit_no_meta(self):
         timestamp = str(normalize_timestamp(time.time()))
@@ -152,17 +156,16 @@ class TestAuditor(unittest.TestCase):
         pre_quarantines = self.auditor.quarantines
         data = '0' * 1024
         etag = md5()
-        with self.disk_file.mkstemp() as fd:
-            os.write(fd, data)
+        with self.disk_file.writer() as writer:
+            writer.write(data)
             etag.update(data)
             etag = etag.hexdigest()
             metadata = {
                 'ETag': etag,
                 'X-Timestamp': timestamp,
-                'Content-Length': str(os.fstat(fd).st_size),
+                'Content-Length': str(os.fstat(writer.fd).st_size),
             }
-            self.disk_file.put(fd, 1024, metadata)
-            self.disk_file.close()
+            writer.put(metadata)
         self.auditor.audit_all_objects()
         self.assertEquals(self.auditor.quarantines, pre_quarantines)
 
@@ -172,18 +175,17 @@ class TestAuditor(unittest.TestCase):
         pre_quarantines = self.auditor.quarantines
         data = '0' * 1024
         etag = md5()
-        with self.disk_file.mkstemp() as fd:
-            os.write(fd, data)
+        with self.disk_file.writer() as writer:
+            writer.write(data)
             etag.update(data)
             etag = etag.hexdigest()
             metadata = {
                 'ETag': etag,
                 'X-Timestamp': timestamp,
-                'Content-Length': str(os.fstat(fd).st_size),
+                'Content-Length': str(os.fstat(writer.fd).st_size),
             }
-            self.disk_file.put(fd, 1024, metadata)
-            self.disk_file.close()
-            os.write(fd, 'extra_data')
+            writer.put(metadata)
+            os.write(writer.fd, 'extra_data')
         self.auditor.audit_all_objects()
         self.assertEquals(self.auditor.quarantines, pre_quarantines + 1)
 
@@ -193,34 +195,32 @@ class TestAuditor(unittest.TestCase):
         pre_quarantines = self.auditor.quarantines
         data = '0' * 10
         etag = md5()
-        with self.disk_file.mkstemp() as fd:
-            os.write(fd, data)
+        with self.disk_file.writer() as writer:
+            writer.write(data)
             etag.update(data)
             etag = etag.hexdigest()
             metadata = {
                 'ETag': etag,
                 'X-Timestamp': timestamp,
-                'Content-Length': str(os.fstat(fd).st_size),
+                'Content-Length': str(os.fstat(writer.fd).st_size),
             }
-            self.disk_file.put(fd, 10, metadata)
-            self.disk_file.close()
+            writer.put(metadata)
         self.auditor.audit_all_objects()
         self.disk_file = DiskFile(self.devices, 'sdb', '0', 'a', 'c',
                                   'ob', self.logger)
         data = '1' * 10
         etag = md5()
-        with self.disk_file.mkstemp() as fd:
-            os.write(fd, data)
+        with self.disk_file.writer() as writer:
+            writer.write(data)
             etag.update(data)
             etag = etag.hexdigest()
             metadata = {
                 'ETag': etag,
                 'X-Timestamp': timestamp,
-                'Content-Length': str(os.fstat(fd).st_size),
+                'Content-Length': str(os.fstat(writer.fd).st_size),
             }
-            self.disk_file.put(fd, 10, metadata)
-            self.disk_file.close()
-            os.write(fd, 'extra_data')
+            writer.put(metadata)
+            os.write(writer.fd, 'extra_data')
         self.auditor.audit_all_objects()
         self.assertEquals(self.auditor.quarantines, pre_quarantines + 1)
 
@@ -229,21 +229,21 @@ class TestAuditor(unittest.TestCase):
         self.auditor.log_time = 0
         data = '0' * 1024
         etag = md5()
-        with self.disk_file.mkstemp() as fd:
-            os.write(fd, data)
+        with self.disk_file.writer() as writer:
+            writer.write(data)
             etag.update(data)
             etag = etag.hexdigest()
             metadata = {
                 'ETag': etag,
                 'X-Timestamp': str(normalize_timestamp(time.time())),
-                'Content-Length': str(os.fstat(fd).st_size),
+                'Content-Length': str(os.fstat(writer.fd).st_size),
             }
-            self.disk_file.put(fd, 1024, metadata)
+            writer.put(metadata)
             etag = md5()
             etag.update('1' + '0' * 1023)
             etag = etag.hexdigest()
             metadata['ETag'] = etag
-            write_metadata(fd, metadata)
+            write_metadata(writer.fd, metadata)
 
         quarantine_path = os.path.join(self.devices,
                                        'sda', 'quarantined', 'objects')
@@ -268,18 +268,18 @@ class TestAuditor(unittest.TestCase):
             fp.close()
 
         etag = md5()
-        with self.disk_file.mkstemp() as fd:
+        with self.disk_file.writer() as writer:
             etag = etag.hexdigest()
             metadata = {
                 'ETag': etag,
                 'X-Timestamp': str(normalize_timestamp(time.time())),
                 'Content-Length': 10,
             }
-            self.disk_file.put(fd, 10, metadata)
+            writer.put(metadata)
             etag = md5()
             etag = etag.hexdigest()
             metadata['ETag'] = etag
-            write_metadata(fd, metadata)
+            write_metadata(writer.fd, metadata)
         if self.disk_file.data_file:
             return self.disk_file.data_file
         return ts_file_path

test/unit/obj/test_server.py

@@ -145,7 +145,7 @@ class TestDiskFile(unittest.TestCase):
         tmpdir = os.path.join(self.testdir, 'sda1', 'tmp')
         os.rmdir(tmpdir)
         with object_server.DiskFile(self.testdir, 'sda1', '0', 'a', 'c',
-                                    'o', FakeLogger()).mkstemp():
+                                    'o', FakeLogger()).writer() as writer:
             self.assert_(os.path.exists(tmpdir))
 
     def test_iter_hook(self):
@@ -153,7 +153,7 @@ class TestDiskFile(unittest.TestCase):
         def hook():
             hook_call_count[0] += 1
 
-        df = self._get_data_file(fsize=65, csize=8, iter_hook=hook)
+        df = self._get_disk_file(fsize=65, csize=8, iter_hook=hook)
         for _ in df:
             pass
 
@@ -204,7 +204,7 @@ class TestDiskFile(unittest.TestCase):
         self.assert_(os.path.isdir(double_uuid_path))
         self.assert_('-' in os.path.basename(double_uuid_path))
 
-    def _get_data_file(self, invalid_type=None, obj_name='o',
+    def _get_disk_file(self, invalid_type=None, obj_name='o',
                        fsize=1024, csize=8, extension='.data', ts=None,
                        iter_hook=None):
         '''returns a DiskFile'''
@@ -216,25 +216,25 @@ class TestDiskFile(unittest.TestCase):
             timestamp = ts
         else:
             timestamp = str(normalize_timestamp(time()))
-        with df.mkstemp() as fd:
-            os.write(fd, data)
+        with df.writer() as writer:
+            writer.write(data)
             etag.update(data)
             etag = etag.hexdigest()
             metadata = {
                 'ETag': etag,
                 'X-Timestamp': timestamp,
-                'Content-Length': str(os.fstat(fd).st_size),
+                'Content-Length': str(os.fstat(writer.fd).st_size),
             }
-            df.put(fd, fsize, metadata, extension=extension)
+            writer.put(metadata, extension=extension)
             if invalid_type == 'ETag':
                 etag = md5()
                 etag.update('1' + '0' * (fsize - 1))
                 etag = etag.hexdigest()
                 metadata['ETag'] = etag
-                object_server.write_metadata(fd, metadata)
+                object_server.write_metadata(writer.fd, metadata)
             if invalid_type == 'Content-Length':
                 metadata['Content-Length'] = fsize - 1
-                object_server.write_metadata(fd, metadata)
+                object_server.write_metadata(writer.fd, metadata)
 
         df = object_server.DiskFile(self.testdir, 'sda1', '0', 'a', 'c',
                                     obj_name, FakeLogger(),
@@ -248,43 +248,43 @@ class TestDiskFile(unittest.TestCase):
         return df
 
     def test_quarantine_valids(self):
-        df = self._get_data_file(obj_name='1')
+        df = self._get_disk_file(obj_name='1')
         for chunk in df:
             pass
         self.assertFalse(df.quarantined_dir)
 
-        df = self._get_data_file(obj_name='2', csize=1)
+        df = self._get_disk_file(obj_name='2', csize=1)
         for chunk in df:
             pass
         self.assertFalse(df.quarantined_dir)
 
-        df = self._get_data_file(obj_name='3', csize=100000)
+        df = self._get_disk_file(obj_name='3', csize=100000)
         for chunk in df:
             pass
         self.assertFalse(df.quarantined_dir)
 
     def run_quarantine_invalids(self, invalid_type):
-        df = self._get_data_file(invalid_type=invalid_type, obj_name='1')
+        df = self._get_disk_file(invalid_type=invalid_type, obj_name='1')
         for chunk in df:
             pass
         self.assertTrue(df.quarantined_dir)
-        df = self._get_data_file(invalid_type=invalid_type,
+        df = self._get_disk_file(invalid_type=invalid_type,
                                  obj_name='2', csize=1)
         for chunk in df:
             pass
         self.assertTrue(df.quarantined_dir)
-        df = self._get_data_file(invalid_type=invalid_type,
+        df = self._get_disk_file(invalid_type=invalid_type,
                                  obj_name='3', csize=100000)
         for chunk in df:
             pass
         self.assertTrue(df.quarantined_dir)
-        df = self._get_data_file(invalid_type=invalid_type, obj_name='4')
+        df = self._get_disk_file(invalid_type=invalid_type, obj_name='4')
         self.assertFalse(df.quarantined_dir)
-        df = self._get_data_file(invalid_type=invalid_type, obj_name='5')
+        df = self._get_disk_file(invalid_type=invalid_type, obj_name='5')
         for chunk in df.app_iter_range(0, df.unit_test_len):
             pass
         self.assertTrue(df.quarantined_dir)
-        df = self._get_data_file(invalid_type=invalid_type, obj_name='6')
+        df = self._get_disk_file(invalid_type=invalid_type, obj_name='6')
         for chunk in df.app_iter_range(0, df.unit_test_len + 100):
             pass
         self.assertTrue(df.quarantined_dir)
@@ -293,15 +293,15 @@ class TestDiskFile(unittest.TestCase):
         # in a quarantine, even if the whole file isn't check-summed
         if invalid_type in ('Zero-Byte', 'Content-Length'):
             expected_quar = True
-        df = self._get_data_file(invalid_type=invalid_type, obj_name='7')
+        df = self._get_disk_file(invalid_type=invalid_type, obj_name='7')
         for chunk in df.app_iter_range(1, df.unit_test_len):
             pass
         self.assertEquals(bool(df.quarantined_dir), expected_quar)
-        df = self._get_data_file(invalid_type=invalid_type, obj_name='8')
+        df = self._get_disk_file(invalid_type=invalid_type, obj_name='8')
         for chunk in df.app_iter_range(0, df.unit_test_len - 1):
             pass
         self.assertEquals(bool(df.quarantined_dir), expected_quar)
-        df = self._get_data_file(invalid_type=invalid_type, obj_name='8')
+        df = self._get_disk_file(invalid_type=invalid_type, obj_name='8')
         for chunk in df.app_iter_range(1, df.unit_test_len + 1):
             pass
         self.assertEquals(bool(df.quarantined_dir), expected_quar)
@@ -312,20 +312,20 @@ class TestDiskFile(unittest.TestCase):
         self.run_quarantine_invalids('Zero-Byte')
 
     def test_quarantine_deleted_files(self):
-        df = self._get_data_file(invalid_type='Content-Length',
+        df = self._get_disk_file(invalid_type='Content-Length',
                                  extension='.data')
         df.close()
         self.assertTrue(df.quarantined_dir)
-        df = self._get_data_file(invalid_type='Content-Length',
+        df = self._get_disk_file(invalid_type='Content-Length',
                                  extension='.ts')
         df.close()
         self.assertFalse(df.quarantined_dir)
-        df = self._get_data_file(invalid_type='Content-Length',
+        df = self._get_disk_file(invalid_type='Content-Length',
                                  extension='.ts')
         self.assertRaises(DiskFileNotExist, df.get_data_file_size)
 
     def test_put_metadata(self):
-        df = self._get_data_file()
+        df = self._get_disk_file()
         ts = time()
         metadata = { 'X-Timestamp': ts, 'X-Object-Meta-test': 'data' }
         df.put_metadata(metadata)
@@ -335,7 +335,7 @@ class TestDiskFile(unittest.TestCase):
         self.assertTrue(exp_name in set(dl))
 
     def test_put_metadata_ts(self):
-        df = self._get_data_file()
+        df = self._get_disk_file()
         ts = time()
         metadata = { 'X-Timestamp': ts, 'X-Object-Meta-test': 'data' }
         df.put_metadata(metadata, tombstone=True)
@@ -345,9 +345,9 @@ class TestDiskFile(unittest.TestCase):
         self.assertTrue(exp_name in set(dl))
 
     def test_unlinkold(self):
-        df1 = self._get_data_file()
+        df1 = self._get_disk_file()
         future_time = str(normalize_timestamp(time() + 100))
-        df2 = self._get_data_file(ts=future_time)
+        df2 = self._get_disk_file(ts=future_time)
         self.assertEquals(len(os.listdir(df1.datadir)), 2)
         df1.unlinkold(future_time)
         self.assertEquals(len(os.listdir(df1.datadir)), 1)
@@ -358,7 +358,7 @@ class TestDiskFile(unittest.TestCase):
         def err():
             raise Exception("bad")
 
-        df = self._get_data_file(fsize=1024 * 1024 * 2)
+        df = self._get_disk_file(fsize=1024 * 1024 * 2)
         df._handle_close_quarantine = err
         for chunk in df:
             pass
@@ -367,7 +367,7 @@ class TestDiskFile(unittest.TestCase):
         self.assertEquals(len(df.logger.log_dict['error']), 1)
 
     def test_quarantine_twice(self):
-        df = self._get_data_file(invalid_type='Content-Length',
+        df = self._get_disk_file(invalid_type='Content-Length',
                                  extension='.data')
         self.assert_(os.path.isfile(df.data_file))
         quar_dir = df.quarantine()