openstack/deb-python-pyeclib
Code Issues Proposed changes

Fix open() calls to include explicit 'b' hints + cleanup

Browse Source 
Signed-off-by: Tushar Gohad <tushar.gohad@intel.com>
This commit is contained in:
Tushar Gohad
2014-04-08 13:14:22 -07:00
parent e5d1dcaad8
commit c341d5aee5
7 changed files with 52 additions and 75 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
setup.py
View File

@@ -78,7 +78,7 @@ def _construct_jerasure_buildenv():
# utility routine
def _read_file_as_str(name):
with open(name, 'r') as f:
with open(name, "rt") as f:
s = f.readline().strip()
return s

13
src/python/pyeclib/ec_test.py
View File

@@ -69,8 +69,6 @@ class TestPyECLibDriver(unittest.TestCase):
os.mkdir("./test_files")
for size_str in self.file_sizes:
filename = "test_file.%s" % size_str
fp = open("test_files/%s" % filename, "w")
size_desc = size_str.split("-")
@@ -84,8 +82,9 @@ class TestPyECLibDriver(unittest.TestCase):
buffer = ''.join(random.choice(string.letters)
for i in range(size))
fp.write(buffer)
fp.close()
filename = "test_file.%s" % size_str
with open("test_files/%s" % filename, "wb") as fp:
fp.write(buffer)
def tearDown(self):
for size_str in self.file_sizes:
@@ -425,10 +424,10 @@ class TestPyECLibDriver(unittest.TestCase):
for pyeclib_driver in pyeclib_drivers:
for file_size in self.file_sizes:
filename = "test_file.%s" % file_size
fp = open("test_files/%s" % filename, "r")
whole_file_str = fp.read()
filename = "test_file.%s" % file_size
with open("test_files/%s" % filename, "r") as fp:
whole_file_str = fp.read()
orig_fragments = pyeclib_driver.encode(whole_file_str)

58
src/python/test/pyeclib_test.py
View File

@@ -54,8 +54,6 @@ def setup(file_sizes):
os.mkdir("./test_files")
for size_str in file_sizes:
filename = "test_file.%s" % size_str
fp = open("test_files/%s" % filename, "w")
size_desc = size_str.split("-")
@@ -68,8 +66,9 @@ def setup(file_sizes):
buffer = ''.join(random.choice(string.letters) for i in range(size))
fp.write(buffer)
fp.close()
filename = "test_file.%s" % size_str
with open(("test_files/%s" % filename), "wb") as fp:
fp.write(buffer)
def cleanup(file_sizes):
@@ -80,13 +79,13 @@ def cleanup(file_sizes):
def time_encode(num_data, num_parity, w, type, file_size, iterations):
filename = "test_file.%s" % file_size
fp = open("test_files/%s" % filename, "r")
timer = Timer()
sum = 0
handle = pyeclib_c.init(num_data, num_parity, w, type)
whole_file_str = fp.read()
filename = "test_file.%s" % file_size
with open("test_files/%s" % filename, "rb") as fp:
whole_file_str = fp.read()
timer.start()
for i in range(iterations):
@@ -97,13 +96,13 @@ def time_encode(num_data, num_parity, w, type, file_size, iterations):
def time_decode(num_data, num_parity, w, type, file_size, iterations, hd):
filename = "test_file.%s" % file_size
fp = open("test_files/%s" % filename, "r")
timer = Timer()
sum = 0
handle = pyeclib_c.init(num_data, num_parity, w, type)
whole_file_str = fp.read()
filename = "test_file.%s" % file_size
with open("test_files/%s" % filename, "rb") as fp:
whole_file_str = fp.read()
fragments = pyeclib_c.encode(handle, whole_file_str)
@@ -131,14 +130,12 @@ def time_decode(num_data, num_parity, w, type, file_size, iterations, hd):
for j in range(num_data + num_parity):
if orig_fragments[j] != decoded_fragments[j]:
fd_orig = open("orig_fragments", "w")
fd_decoded = open("decoded_fragments", "w")
with open("orig_fragments", "wb") as fd_orig:
fd_orig.write(orig_fragments[j])
fd_orig.write(orig_fragments[j])
fd_decoded.write(decoded_fragments[j])
with open("decoded_fragments", "wb") as fd_decoded:
fd_decoded.write(decoded_fragments[j])
fd_orig.close()
fd_decoded.close()
print(("Fragment %d was not reconstructed!!!" % j))
sys.exit(2)
@@ -148,13 +145,13 @@ def time_decode(num_data, num_parity, w, type, file_size, iterations, hd):
def test_reconstruct(num_data, num_parity, w, type, file_size, iterations):
filename = "test_file.%s" % file_size
fp = open("test_files/%s" % filename, "r")
timer = Timer()
sum = 0
handle = pyeclib_c.init(num_data, num_parity, w, type)
whole_file_str = fp.read()
filename = "test_file.%s" % file_size
with open("test_files/%s" % filename, "rb") as fp:
whole_file_str = fp.read()
orig_fragments = pyeclib_c.encode(handle, whole_file_str)
@@ -187,13 +184,10 @@ def test_reconstruct(num_data, num_parity, w, type, file_size, iterations):
fragments = decoded_fragments
if orig_fragments[missing_idxs[0]] != reconstructed_fragment:
fd_orig = open("orig_fragments", "w")
fd_decoded = open("decoded_fragments", "w")
fd_orig.write(orig_fragments[missing_idxs[0]])
fd_decoded.write(reconstructed_fragment)
fd_orig.close()
fd_decoded.close()
with open("orig_fragments", "wb") as fd_orig:
fd_orig.write(orig_fragments[missing_idxs[0]])
with open("decoded_fragments", "wb") as fd_decoded:
fd_decoded.write(reconstructed_fragment)
print(("Fragment %d was not reconstructed!!!" % missing_idxs[0]))
sys.exit(2)
@@ -202,11 +196,11 @@ def test_reconstruct(num_data, num_parity, w, type, file_size, iterations):
def test_get_fragment_partition(
num_data, num_parity, w, type, file_size, iterations):
filename = "test_file.%s" % file_size
fp = open("test_files/%s" % filename, "r")
handle = pyeclib_c.init(num_data, num_parity, w, type)
whole_file_str = fp.read()
filename = "test_file.%s" % file_size
with open("test_files/%s" % filename, "rb") as fp:
whole_file_str = fp.read()
fragments = pyeclib_c.encode(handle, whole_file_str)
@@ -235,11 +229,11 @@ def test_get_fragment_partition(
def test_fragments_to_string(num_data, num_parity, w, type, file_size):
filename = "test_file.%s" % file_size
fp = open("test_files/%s" % filename, "r")
handle = pyeclib_c.init(num_data, num_parity, w, type)
whole_file_str = fp.read()
filename = "test_file.%s" % file_size
with open(("test_files/%s" % filename), "rb") as fp:
whole_file_str = fp.read()
fragments = pyeclib_c.encode(handle, whole_file_str)

8
tools/pyeclib_conf_tool.py
View File

@@ -235,16 +235,12 @@ for scheme in schemes:
# Generate a new string for each test
file_str = ''.join(
random.choice(
string.ascii_uppercase +
string.digits) for x in range(
args.s))
string.ascii_uppercase + string.digits) for x in range(args.s))
try:
ec_driver = ECDriver(
"pyeclib.core.ECPyECLibDriver",
k=scheme.k,
m=scheme.m,
type=scheme.type)
k=scheme.k, m=scheme.m, type=scheme.type)
except Exception as e:
print("Scheme %s is not defined (%s)." % (scheme, e))
continue

23
tools/pyeclib_decode.py
View File

@@ -46,25 +46,18 @@ print args.fragments
print args.filename
ec_driver = ECDriver(
"pyeclib.core.ECPyECLibDriver",
k=args.k,
m=args.m,
type=args.type)
"pyeclib.core.ECPyECLibDriver", k=args.k, m=args.m, type=args.type)
fragment_list = []
# read fragments
for fragment in args.fragments:
fp = open("%s" % fragment, "r")
fragment_list.append(fp.read())
fp.close()
fp = open("%s.decoded" % args.filename, "w")
with open(("%s" % fragment), "rb") as fp:
fragment_list.append(fp.read())
# decode
decoded_file = ec_driver.decode(fragment_list)
fp.write(decoded_file)
fp.close()
# write
with open("%s.decoded" % args.filename, "wb") as fp:
fp.write(decoded_file)

19
tools/pyeclib_encode.py
View File

@@ -46,21 +46,18 @@ print args.filename
ec_driver = ECDriver(
"pyeclib.core.ECPyECLibDriver",
k=args.k,
m=args.m,
type=args.type)
k=args.k, m=args.m, type=args.type)
fp = open("%s/%s" % (args.file_dir, args.filename), "r")
whole_file_str = fp.read()
# read
with open(("%s/%s" % (args.file_dir, args.filename)), "rb") as fp:
whole_file_str = fp.read()
# encode
fragments = ec_driver.encode(whole_file_str)
fp.close()
# store
i = 0
for fragment in fragments:
fp = open("%s/%s.%d" % (args.fragment_dir, args.filename, i), "w")
fp.write(fragment)
fp.close()
with open("%s/%s.%d" % (args.fragment_dir, args.filename, i), "wb") as fp:
fp.write(fragment)
i += 1

4
tools/pyeclib_fragments_needed.py
View File

@@ -42,9 +42,7 @@ args = parser.parse_args()
ec_driver = ECDriver(
"pyeclib.core.ECPyECLibDriver",
k=args.k,
m=args.m,
type=args.type)
k=args.k, m=args.m, type=args.type)
fragments_needed = ec_driver.fragments_needed(args.missing_fragments)