Fix open() calls to include explicit 'b' hints + cleanup

Signed-off-by: Tushar Gohad <tushar.gohad@intel.com>

setup.py

@@ -78,7 +78,7 @@ def _construct_jerasure_buildenv():
 
 # utility routine
 def _read_file_as_str(name):
-    with open(name, 'r') as f:
+    with open(name, "rt") as f:
         s = f.readline().strip()
     return s
 

src/python/pyeclib/ec_test.py

@@ -69,8 +69,6 @@ class TestPyECLibDriver(unittest.TestCase):
             os.mkdir("./test_files")
 
         for size_str in self.file_sizes:
-            filename = "test_file.%s" % size_str
-            fp = open("test_files/%s" % filename, "w")
 
             size_desc = size_str.split("-")
 
@@ -84,8 +82,9 @@ class TestPyECLibDriver(unittest.TestCase):
             buffer = ''.join(random.choice(string.letters)
                              for i in range(size))
 
-            fp.write(buffer)
+            filename = "test_file.%s" % size_str
-            fp.close()
+            with open("test_files/%s" % filename, "wb") as fp:
+                fp.write(buffer)
 
     def tearDown(self):
         for size_str in self.file_sizes:
@@ -425,10 +424,10 @@ class TestPyECLibDriver(unittest.TestCase):
 
         for pyeclib_driver in pyeclib_drivers:
             for file_size in self.file_sizes:
-                filename = "test_file.%s" % file_size
-                fp = open("test_files/%s" % filename, "r")
 
-                whole_file_str = fp.read()
+                filename = "test_file.%s" % file_size
+                with open("test_files/%s" % filename, "r") as fp:
+                    whole_file_str = fp.read()
 
                 orig_fragments = pyeclib_driver.encode(whole_file_str)
 

src/python/test/pyeclib_test.py

@@ -54,8 +54,6 @@ def setup(file_sizes):
         os.mkdir("./test_files")
 
     for size_str in file_sizes:
-        filename = "test_file.%s" % size_str
-        fp = open("test_files/%s" % filename, "w")
 
         size_desc = size_str.split("-")
 
@@ -68,8 +66,9 @@ def setup(file_sizes):
 
         buffer = ''.join(random.choice(string.letters) for i in range(size))
 
-        fp.write(buffer)
+        filename = "test_file.%s" % size_str
-        fp.close()
+        with open(("test_files/%s" % filename), "wb") as fp:
+            fp.write(buffer)
 
 
 def cleanup(file_sizes):
@@ -80,13 +79,13 @@ def cleanup(file_sizes):
 
 
 def time_encode(num_data, num_parity, w, type, file_size, iterations):
-    filename = "test_file.%s" % file_size
-    fp = open("test_files/%s" % filename, "r")
     timer = Timer()
     sum = 0
     handle = pyeclib_c.init(num_data, num_parity, w, type)
 
-    whole_file_str = fp.read()
+    filename = "test_file.%s" % file_size
+    with open("test_files/%s" % filename, "rb") as fp:
+        whole_file_str = fp.read()
 
     timer.start()
     for i in range(iterations):
@@ -97,13 +96,13 @@ def time_encode(num_data, num_parity, w, type, file_size, iterations):
 
 
 def time_decode(num_data, num_parity, w, type, file_size, iterations, hd):
-    filename = "test_file.%s" % file_size
-    fp = open("test_files/%s" % filename, "r")
     timer = Timer()
     sum = 0
     handle = pyeclib_c.init(num_data, num_parity, w, type)
 
-    whole_file_str = fp.read()
+    filename = "test_file.%s" % file_size
+    with open("test_files/%s" % filename, "rb") as fp:
+        whole_file_str = fp.read()
 
     fragments = pyeclib_c.encode(handle, whole_file_str)
 
@@ -131,14 +130,12 @@ def time_decode(num_data, num_parity, w, type, file_size, iterations, hd):
 
         for j in range(num_data + num_parity):
             if orig_fragments[j] != decoded_fragments[j]:
-                fd_orig = open("orig_fragments", "w")
+                with open("orig_fragments", "wb") as fd_orig:
-                fd_decoded = open("decoded_fragments", "w")
+                    fd_orig.write(orig_fragments[j])
 
-                fd_orig.write(orig_fragments[j])
+                with open("decoded_fragments", "wb") as fd_decoded:
-                fd_decoded.write(decoded_fragments[j])
+                    fd_decoded.write(decoded_fragments[j])
 
-                fd_orig.close()
-                fd_decoded.close()
                 print(("Fragment %d was not reconstructed!!!" % j))
                 sys.exit(2)
 
@@ -148,13 +145,13 @@ def time_decode(num_data, num_parity, w, type, file_size, iterations, hd):
 
 
 def test_reconstruct(num_data, num_parity, w, type, file_size, iterations):
-    filename = "test_file.%s" % file_size
-    fp = open("test_files/%s" % filename, "r")
     timer = Timer()
     sum = 0
     handle = pyeclib_c.init(num_data, num_parity, w, type)
 
-    whole_file_str = fp.read()
+    filename = "test_file.%s" % file_size
+    with open("test_files/%s" % filename, "rb") as fp:
+        whole_file_str = fp.read()
 
     orig_fragments = pyeclib_c.encode(handle, whole_file_str)
 
@@ -187,13 +184,10 @@ def test_reconstruct(num_data, num_parity, w, type, file_size, iterations):
         fragments = decoded_fragments
 
         if orig_fragments[missing_idxs[0]] != reconstructed_fragment:
-            fd_orig = open("orig_fragments", "w")
+            with open("orig_fragments", "wb") as fd_orig:
-            fd_decoded = open("decoded_fragments", "w")
+                fd_orig.write(orig_fragments[missing_idxs[0]])
-
+            with open("decoded_fragments", "wb") as fd_decoded:
-            fd_orig.write(orig_fragments[missing_idxs[0]])
+                fd_decoded.write(reconstructed_fragment)
-            fd_decoded.write(reconstructed_fragment)
-            fd_orig.close()
-            fd_decoded.close()
             print(("Fragment %d was not reconstructed!!!" % missing_idxs[0]))
             sys.exit(2)
 
@@ -202,11 +196,11 @@ def test_reconstruct(num_data, num_parity, w, type, file_size, iterations):
 
 def test_get_fragment_partition(
         num_data, num_parity, w, type, file_size, iterations):
-    filename = "test_file.%s" % file_size
-    fp = open("test_files/%s" % filename, "r")
     handle = pyeclib_c.init(num_data, num_parity, w, type)
 
-    whole_file_str = fp.read()
+    filename = "test_file.%s" % file_size
+    with open("test_files/%s" % filename, "rb") as fp:
+        whole_file_str = fp.read()
 
     fragments = pyeclib_c.encode(handle, whole_file_str)
 
@@ -235,11 +229,11 @@ def test_get_fragment_partition(
 
 
 def test_fragments_to_string(num_data, num_parity, w, type, file_size):
-    filename = "test_file.%s" % file_size
-    fp = open("test_files/%s" % filename, "r")
     handle = pyeclib_c.init(num_data, num_parity, w, type)
 
-    whole_file_str = fp.read()
+    filename = "test_file.%s" % file_size
+    with open(("test_files/%s" % filename), "rb") as fp:
+        whole_file_str = fp.read()
 
     fragments = pyeclib_c.encode(handle, whole_file_str)
 

tools/pyeclib_conf_tool.py

@@ -235,16 +235,12 @@ for scheme in schemes:
     # Generate a new string for each test
     file_str = ''.join(
         random.choice(
-            string.ascii_uppercase +
+            string.ascii_uppercase + string.digits) for x in range(args.s))
-            string.digits) for x in range(
-            args.s))
 
     try:
         ec_driver = ECDriver(
             "pyeclib.core.ECPyECLibDriver",
-            k=scheme.k,
+            k=scheme.k, m=scheme.m, type=scheme.type)
-            m=scheme.m,
-            type=scheme.type)
     except Exception as e:
         print("Scheme %s is not defined (%s)." % (scheme, e))
         continue

tools/pyeclib_decode.py

@@ -46,25 +46,18 @@ print args.fragments
 print args.filename
 
 ec_driver = ECDriver(
-    "pyeclib.core.ECPyECLibDriver",
+    "pyeclib.core.ECPyECLibDriver", k=args.k, m=args.m, type=args.type)
-    k=args.k,
-    m=args.m,
-    type=args.type)
 
 fragment_list = []
 
+# read fragments
 for fragment in args.fragments:
-    fp = open("%s" % fragment, "r")
+    with open(("%s" % fragment), "rb") as fp:
-
+        fragment_list.append(fp.read())
-    fragment_list.append(fp.read())
-
-    fp.close()
-
-
-fp = open("%s.decoded" % args.filename, "w")
 
+# decode
 decoded_file = ec_driver.decode(fragment_list)
 
-fp.write(decoded_file)
+# write
-
+with open("%s.decoded" % args.filename, "wb") as fp:
-fp.close()
+    fp.write(decoded_file)

tools/pyeclib_encode.py

@@ -46,21 +46,18 @@ print args.filename
 
 ec_driver = ECDriver(
     "pyeclib.core.ECPyECLibDriver",
-    k=args.k,
+    k=args.k, m=args.m, type=args.type)
-    m=args.m,
-    type=args.type)
 
-fp = open("%s/%s" % (args.file_dir, args.filename), "r")
+# read
-
+with open(("%s/%s" % (args.file_dir, args.filename)), "rb") as fp:
-whole_file_str = fp.read()
+    whole_file_str = fp.read()
 
+# encode
 fragments = ec_driver.encode(whole_file_str)
 
-fp.close()
+# store
-
 i = 0
 for fragment in fragments:
-    fp = open("%s/%s.%d" % (args.fragment_dir, args.filename, i), "w")
+    with open("%s/%s.%d" % (args.fragment_dir, args.filename, i), "wb") as fp:
-    fp.write(fragment)
+        fp.write(fragment)
-    fp.close()
     i += 1

tools/pyeclib_fragments_needed.py

@@ -42,9 +42,7 @@ args = parser.parse_args()
 
 ec_driver = ECDriver(
     "pyeclib.core.ECPyECLibDriver",
-    k=args.k,
+    k=args.k, m=args.m, type=args.type)
-    m=args.m,
-    type=args.type)
 
 fragments_needed = ec_driver.fragments_needed(args.missing_fragments)