diff --git a/setup.py b/setup.py index 3d42bfa..28e909e 100644 --- a/setup.py +++ b/setup.py @@ -78,7 +78,7 @@ def _construct_jerasure_buildenv(): # utility routine def _read_file_as_str(name): - with open(name, 'r') as f: + with open(name, "rt") as f: s = f.readline().strip() return s diff --git a/src/python/pyeclib/ec_test.py b/src/python/pyeclib/ec_test.py index a71b0f4..7304c4f 100644 --- a/src/python/pyeclib/ec_test.py +++ b/src/python/pyeclib/ec_test.py @@ -69,8 +69,6 @@ class TestPyECLibDriver(unittest.TestCase): os.mkdir("./test_files") for size_str in self.file_sizes: - filename = "test_file.%s" % size_str - fp = open("test_files/%s" % filename, "w") size_desc = size_str.split("-") @@ -84,8 +82,9 @@ class TestPyECLibDriver(unittest.TestCase): buffer = ''.join(random.choice(string.letters) for i in range(size)) - fp.write(buffer) - fp.close() + filename = "test_file.%s" % size_str + with open("test_files/%s" % filename, "wb") as fp: + fp.write(buffer) def tearDown(self): for size_str in self.file_sizes: @@ -425,10 +424,10 @@ class TestPyECLibDriver(unittest.TestCase): for pyeclib_driver in pyeclib_drivers: for file_size in self.file_sizes: - filename = "test_file.%s" % file_size - fp = open("test_files/%s" % filename, "r") - whole_file_str = fp.read() + filename = "test_file.%s" % file_size + with open("test_files/%s" % filename, "r") as fp: + whole_file_str = fp.read() orig_fragments = pyeclib_driver.encode(whole_file_str) diff --git a/src/python/test/pyeclib_test.py b/src/python/test/pyeclib_test.py index f250b35..669a6fc 100644 --- a/src/python/test/pyeclib_test.py +++ b/src/python/test/pyeclib_test.py @@ -54,8 +54,6 @@ def setup(file_sizes): os.mkdir("./test_files") for size_str in file_sizes: - filename = "test_file.%s" % size_str - fp = open("test_files/%s" % filename, "w") size_desc = size_str.split("-") @@ -68,8 +66,9 @@ def setup(file_sizes): buffer = ''.join(random.choice(string.letters) for i in range(size)) - fp.write(buffer) - fp.close() + filename = "test_file.%s" % size_str + with open(("test_files/%s" % filename), "wb") as fp: + fp.write(buffer) def cleanup(file_sizes): @@ -80,13 +79,13 @@ def cleanup(file_sizes): def time_encode(num_data, num_parity, w, type, file_size, iterations): - filename = "test_file.%s" % file_size - fp = open("test_files/%s" % filename, "r") timer = Timer() sum = 0 handle = pyeclib_c.init(num_data, num_parity, w, type) - whole_file_str = fp.read() + filename = "test_file.%s" % file_size + with open("test_files/%s" % filename, "rb") as fp: + whole_file_str = fp.read() timer.start() for i in range(iterations): @@ -97,13 +96,13 @@ def time_encode(num_data, num_parity, w, type, file_size, iterations): def time_decode(num_data, num_parity, w, type, file_size, iterations, hd): - filename = "test_file.%s" % file_size - fp = open("test_files/%s" % filename, "r") timer = Timer() sum = 0 handle = pyeclib_c.init(num_data, num_parity, w, type) - whole_file_str = fp.read() + filename = "test_file.%s" % file_size + with open("test_files/%s" % filename, "rb") as fp: + whole_file_str = fp.read() fragments = pyeclib_c.encode(handle, whole_file_str) @@ -131,14 +130,12 @@ def time_decode(num_data, num_parity, w, type, file_size, iterations, hd): for j in range(num_data + num_parity): if orig_fragments[j] != decoded_fragments[j]: - fd_orig = open("orig_fragments", "w") - fd_decoded = open("decoded_fragments", "w") + with open("orig_fragments", "wb") as fd_orig: + fd_orig.write(orig_fragments[j]) - fd_orig.write(orig_fragments[j]) - fd_decoded.write(decoded_fragments[j]) + with open("decoded_fragments", "wb") as fd_decoded: + fd_decoded.write(decoded_fragments[j]) - fd_orig.close() - fd_decoded.close() print(("Fragment %d was not reconstructed!!!" % j)) sys.exit(2) @@ -148,13 +145,13 @@ def time_decode(num_data, num_parity, w, type, file_size, iterations, hd): def test_reconstruct(num_data, num_parity, w, type, file_size, iterations): - filename = "test_file.%s" % file_size - fp = open("test_files/%s" % filename, "r") timer = Timer() sum = 0 handle = pyeclib_c.init(num_data, num_parity, w, type) - whole_file_str = fp.read() + filename = "test_file.%s" % file_size + with open("test_files/%s" % filename, "rb") as fp: + whole_file_str = fp.read() orig_fragments = pyeclib_c.encode(handle, whole_file_str) @@ -187,13 +184,10 @@ def test_reconstruct(num_data, num_parity, w, type, file_size, iterations): fragments = decoded_fragments if orig_fragments[missing_idxs[0]] != reconstructed_fragment: - fd_orig = open("orig_fragments", "w") - fd_decoded = open("decoded_fragments", "w") - - fd_orig.write(orig_fragments[missing_idxs[0]]) - fd_decoded.write(reconstructed_fragment) - fd_orig.close() - fd_decoded.close() + with open("orig_fragments", "wb") as fd_orig: + fd_orig.write(orig_fragments[missing_idxs[0]]) + with open("decoded_fragments", "wb") as fd_decoded: + fd_decoded.write(reconstructed_fragment) print(("Fragment %d was not reconstructed!!!" % missing_idxs[0])) sys.exit(2) @@ -202,11 +196,11 @@ def test_reconstruct(num_data, num_parity, w, type, file_size, iterations): def test_get_fragment_partition( num_data, num_parity, w, type, file_size, iterations): - filename = "test_file.%s" % file_size - fp = open("test_files/%s" % filename, "r") handle = pyeclib_c.init(num_data, num_parity, w, type) - whole_file_str = fp.read() + filename = "test_file.%s" % file_size + with open("test_files/%s" % filename, "rb") as fp: + whole_file_str = fp.read() fragments = pyeclib_c.encode(handle, whole_file_str) @@ -235,11 +229,11 @@ def test_get_fragment_partition( def test_fragments_to_string(num_data, num_parity, w, type, file_size): - filename = "test_file.%s" % file_size - fp = open("test_files/%s" % filename, "r") handle = pyeclib_c.init(num_data, num_parity, w, type) - whole_file_str = fp.read() + filename = "test_file.%s" % file_size + with open(("test_files/%s" % filename), "rb") as fp: + whole_file_str = fp.read() fragments = pyeclib_c.encode(handle, whole_file_str) diff --git a/tools/pyeclib_conf_tool.py b/tools/pyeclib_conf_tool.py index 4d9d522..5f6a964 100644 --- a/tools/pyeclib_conf_tool.py +++ b/tools/pyeclib_conf_tool.py @@ -235,16 +235,12 @@ for scheme in schemes: # Generate a new string for each test file_str = ''.join( random.choice( - string.ascii_uppercase + - string.digits) for x in range( - args.s)) + string.ascii_uppercase + string.digits) for x in range(args.s)) try: ec_driver = ECDriver( "pyeclib.core.ECPyECLibDriver", - k=scheme.k, - m=scheme.m, - type=scheme.type) + k=scheme.k, m=scheme.m, type=scheme.type) except Exception as e: print("Scheme %s is not defined (%s)." % (scheme, e)) continue diff --git a/tools/pyeclib_decode.py b/tools/pyeclib_decode.py index 07b118c..acc313d 100644 --- a/tools/pyeclib_decode.py +++ b/tools/pyeclib_decode.py @@ -46,25 +46,18 @@ print args.fragments print args.filename ec_driver = ECDriver( - "pyeclib.core.ECPyECLibDriver", - k=args.k, - m=args.m, - type=args.type) + "pyeclib.core.ECPyECLibDriver", k=args.k, m=args.m, type=args.type) fragment_list = [] +# read fragments for fragment in args.fragments: - fp = open("%s" % fragment, "r") - - fragment_list.append(fp.read()) - - fp.close() - - -fp = open("%s.decoded" % args.filename, "w") + with open(("%s" % fragment), "rb") as fp: + fragment_list.append(fp.read()) +# decode decoded_file = ec_driver.decode(fragment_list) -fp.write(decoded_file) - -fp.close() +# write +with open("%s.decoded" % args.filename, "wb") as fp: + fp.write(decoded_file) diff --git a/tools/pyeclib_encode.py b/tools/pyeclib_encode.py index a61f023..fdfae80 100644 --- a/tools/pyeclib_encode.py +++ b/tools/pyeclib_encode.py @@ -46,21 +46,18 @@ print args.filename ec_driver = ECDriver( "pyeclib.core.ECPyECLibDriver", - k=args.k, - m=args.m, - type=args.type) + k=args.k, m=args.m, type=args.type) -fp = open("%s/%s" % (args.file_dir, args.filename), "r") - -whole_file_str = fp.read() +# read +with open(("%s/%s" % (args.file_dir, args.filename)), "rb") as fp: + whole_file_str = fp.read() +# encode fragments = ec_driver.encode(whole_file_str) -fp.close() - +# store i = 0 for fragment in fragments: - fp = open("%s/%s.%d" % (args.fragment_dir, args.filename, i), "w") - fp.write(fragment) - fp.close() + with open("%s/%s.%d" % (args.fragment_dir, args.filename, i), "wb") as fp: + fp.write(fragment) i += 1 diff --git a/tools/pyeclib_fragments_needed.py b/tools/pyeclib_fragments_needed.py index 782d59e..4bcc754 100644 --- a/tools/pyeclib_fragments_needed.py +++ b/tools/pyeclib_fragments_needed.py @@ -42,9 +42,7 @@ args = parser.parse_args() ec_driver = ECDriver( "pyeclib.core.ECPyECLibDriver", - k=args.k, - m=args.m, - type=args.type) + k=args.k, m=args.m, type=args.type) fragments_needed = ec_driver.fragments_needed(args.missing_fragments)