diff --git a/.gitignore b/.gitignore deleted file mode 100644 index efbe7aa..0000000 --- a/.gitignore +++ /dev/null @@ -1,53 +0,0 @@ -# Byte-compiled / optimized / DLL files -__pycache__/ -*.py[cod] - -# C extensions -*.so - -# Distribution / packaging -.Python -env/ -build/ -develop-eggs/ -dist/ -eggs/ -lib/ -lib64/ -parts/ -sdist/ -var/ -*.egg-info/ -.installed.cfg -*.egg - -# PyInstaller -# Usually these files are written by a python script from a template -# before PyInstaller builds the exe, so as to inject date/other infos into it. -*.manifest -*.spec - -# Installer logs -pip-log.txt -pip-delete-this-directory.txt - -# Unit test / coverage reports -htmlcov/ -.tox/ -.coverage -.cache -nosetests.xml -coverage.xml - -# Translations -*.mo -*.pot - -# Django stuff: -*.log - -# Sphinx documentation -docs/_build/ - -# PyBuilder -target/ diff --git a/.gitreview b/.gitreview deleted file mode 100644 index 42dc129..0000000 --- a/.gitreview +++ /dev/null @@ -1,4 +0,0 @@ -[gerrit] -host=review.openstack.org -port=29418 -project=openstack/pyeclib.git diff --git a/.mailmap b/.mailmap deleted file mode 100644 index 8f65196..0000000 --- a/.mailmap +++ /dev/null @@ -1,8 +0,0 @@ -Kevin Greenan -Kevin Greenan -Kevin Greenan -Tushar Gohad -Tushar Gohad -Eric Lambert -Kota Tsuyuzaki -Kota Tsuyuzaki diff --git a/.travis.yml b/.travis.yml deleted file mode 100644 index 082cf55..0000000 --- a/.travis.yml +++ /dev/null @@ -1,14 +0,0 @@ -language: python -python: - - "2.7" - - "3.4" -script: python setup.py install && nosetests test/ && tox -branches: - only: - - master -before_install: - - echo "deb http://archive.ubuntu.com/ubuntu precise-backports main restricted universe multiverse" | sudo tee -a /etc/apt/sources.list - - sudo apt-get update - - sudo apt-get install -y liberasurecode-dev libffi-dev python-setuptools - - sudo easy_install pip - - pip install tox diff --git a/.unittests b/.unittests deleted file mode 100755 index 0265036..0000000 --- a/.unittests +++ /dev/null @@ -1,10 +0,0 @@ -#!/bin/bash - -TOP_DIR=$(python -c "import os; print os.path.dirname(os.path.realpath('$0'))") - -python -c 'from distutils.version import LooseVersion as Ver; import nose, sys; sys.exit(0 if Ver(nose.__version__) >= Ver("1.2.0") else 1)' -cd $TOP_DIR/test -nosetests $@ -rvalue=$? -cd - -exit $rvalue diff --git a/AUTHORS b/AUTHORS deleted file mode 100644 index bba00a2..0000000 --- a/AUTHORS +++ /dev/null @@ -1,22 +0,0 @@ -Original Authors ----------------- -Kevin Greenan (kmgreen2@gmail.com) -Tushar Gohad (tushar.gohad@intel.com) - -Contributors ------------- -Clay Gerrard (clay.gerrard@gmail.com) -Davanum Srinivas (davanum@gmail.com) -Eric Lambert (eric_lambert@xyratex.com) -Jim Cheung (jim.cheung@phazr.io) -John Dickinson (me@not.mn) -Kota Tsuyuzaki (bloodeagle40234@gmail.com) -Mark Storer (Mark.Storer@evault.com) -Ondřej Nový (ondrej.novy@firma.seznam.cz) -Pete Zaitcev (zaitcev@kotori.zaitcev.us) -Thiago da Silva (thiago@redhat.com) -Timur Alperovich (timuralp@swiftstack.com) -Tim Burke (tim.burke@gmail.com) -Victor Stinner (vstinner@redhat.com) -Yuan Zhou (yuan.zhou@intel.com) - diff --git a/ChangeLog b/ChangeLog deleted file mode 100644 index 3ad5e89..0000000 --- a/ChangeLog +++ /dev/null @@ -1,116 +0,0 @@ -New in 1.5.0 ------------- - - * Added support for Phazr.IO libphazr library. - * Fixed error handling that caused segfaults. - * Use version number exposed by liberasurecode. - * various other minor bugs fixes and improvements. - - -New in 1.4.0 ------------- - - * Added support for ISA-L Cauchy - * Fixed memory leak in get_metadata - * Added soft warning log line when using liberasurecode <1.3.1 - -New in 1.3.1 ------------- - - * Updated name in setup.py to work with release tooling. - - -New in 1.3.0 ------------- - - * Updated liberasurecode dependency to 1.2.0. - - * Fixed memory leak in get_segment_info (Launchpad bug #1604335). - - * Properly return an error code if liberasurecode returns an - invalid fragment size. - - * ECDriver() now requries "k" and "m" values to be passed in. - - * Fix some requirements and installation instruction. - - -New in 1.2.1 ------------- - - * Eliminate spurious syslog messages and added cleaner - mechanism for querying all available backends on a system. - - * Moved source code hosting from bitbucket to Openstack infra. - This is first release with Openstack Infra - -New in 1.2.0 ------------- - - * Remove integrated liberasurecode - the prereq library is now - available in major deb/rpm based distros - - * Eliminate liberasurecode-related rpath checks, library searches - and architecture validation - - * Changes for compatibility with liberasurecode versions prior to - 1.1.0 - - * Make VALID_EC_TYPES a runtime property - dynamically look for - liberasurecode EC schemes available at runtime - - * More comprehensive unit test coverage - - * Add travis-ci build config for automated build/tests - -New in 1.1.0 ------------- - - * Eliminate pyeclib dependency on alloc functions internal to - liberasurecode - - * Update include subdirs to be explicit - - * Update internal liberasurecode version to 1.1.0 - -New in 1.0.9 ------------- - - * Eliminate rpath handling in setup.py - - * Clean py34 shared libraries created during build - - * Fix integer truncation issue with PyBuildValue on Big Endian - systems by explicitly casting the size argument passed in to - Py_ssize_t. Also fix import issue with the API test where - older versions of Python fail to import. - - * Add --install-liberasurecode option to setup.py. Requested - by Red Hat/Debian package maintainers. - - * Update bundled liberasurecode version to 1.0.9 - - -New in 1.0.8 ------------- - - * Support for a new Reed-Soloman backend (liberasurecode_rs_vand) - - naive, non-accelerated version, native to liberasurecode - - * Single version, with liberasurecode distributed in the package, - installed if necessary. No versions going forward with Jerasure - included. Use 'liberasurecode_rs_vand' for default test backend. - - * Test code refactor for eliminating duplicated code, add cases - for liberasurecode_rs_vand and making jerasure/isa_l test cases - conditional. - - * Better Python3 support - - * setup.py enhancements - - improved library path detection on Mac OS X (and Linux) including - workarounds for Mac OS X dyld bugs for library search paths - - handling installroot better for optional liberasurecode installs - - * tox support for automated py27 and py34 testing - diff --git a/License.txt b/License.txt deleted file mode 100644 index 2d3f241..0000000 --- a/License.txt +++ /dev/null @@ -1,23 +0,0 @@ -Copyright (c) 2013-2015, Kevin Greenan (kmgreen2@gmail.com) -Copyright (c) 2013-2015, Tushar Gohad (tusharsg@gmail.com) -All rights reserved. - -Redistribution and use in source and binary forms, with or without -modification, are permitted provided that the following conditions are met: - -Redistributions of source code must retain the above copyright notice, this -list of conditions and the following disclaimer. - -Redistributions in binary form must reproduce the above copyright notice, this -list of conditions and the following disclaimer in the documentation and/or -other materials provided with the distribution. THIS SOFTWARE IS PROVIDED BY -THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED -WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF -MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO -EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, -INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, -BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, -DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF -LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE -OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF -ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. diff --git a/MANIFEST.in b/MANIFEST.in deleted file mode 100644 index e4701c1..0000000 --- a/MANIFEST.in +++ /dev/null @@ -1,3 +0,0 @@ -recursive-include src * -recursive-include test * -recursive-include tools * diff --git a/Makefile b/Makefile deleted file mode 100644 index 473d658..0000000 --- a/Makefile +++ /dev/null @@ -1,52 +0,0 @@ -# Copyright (c) 2014, Tushar Gohad (tusharsg@gmail.com) -# All rights reserved. -# -# Redistribution and use in source and binary forms, with or without -# modification, are permitted provided that the following conditions are met: -# -# Redistributions of source code must retain the above copyright notice, this -# list of conditions and the following disclaimer. -# -# Redistributions in binary form must reproduce the above copyright notice, -# this list of conditions and the following disclaimer in the documentation -# and/or other materials provided with the distribution. THIS SOFTWARE IS -# PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS -# OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES -# OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN -# NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY -# DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES -# (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; -# LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND -# ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT -# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF -# THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - -TOPDIR := $(PWD) - -.PHONY: build test - -build: - python setup.py build - -install: build - python setup.py install - -UNITS := test/test_pyeclib_api.py test/test_pyeclib_c.py - -test: build - $(eval SONAMES := $(shell find $(abs_top_builddir) -name '*.so')) - $(eval SODIRS := $(dir $(SONAMES))) - $(eval LD_LIBRARY_PATH := LD_LIBRARY_PATH="$(subst / ,/:,$(SODIRS))") - $(eval DYLD_LIBRARY_PATH := DYLD_LIBRARY_PATH="$(subst / ,/:,$(SODIRS))") - $(eval DYLD_FALLBACK_LIBRARY_PATH := DYLD_FALLBACK_LIBRARY_PATH="$(subst / ,/:,$(SODIRS))") - rm -rf cover .coverage - @$(LD_LIBRARY_PATH) $(DYLD_LIBRARY_PATH) $(DYLD_FALLBACK_LIBRARY_PATH) \ - nosetests --exe --with-coverage \ - --cover-package pyeclib --cover-erase \ - --cover-html --cover-html-dir=${TOPDIR}/cover \ - $(UNITS) - -clean: - -rm -f pyeclib_c*.so - -rm -rf build - python setup.py clean diff --git a/README b/README new file mode 100644 index 0000000..8fcd2b2 --- /dev/null +++ b/README @@ -0,0 +1,14 @@ +This project is no longer maintained. + +The contents of this repository are still available in the Git +source code management system. To see the contents of this +repository before it reached its end of life, please check out the +previous commit with "git checkout HEAD^1". + +For ongoing work on maintaining OpenStack packages in the Debian +distribution, please see the Debian OpenStack packaging team at +https://wiki.debian.org/OpenStack/. + +For any further questions, please email +openstack-dev@lists.openstack.org or join #openstack-dev on +Freenode. diff --git a/README.rst b/README.rst deleted file mode 100644 index 584b87d..0000000 --- a/README.rst +++ /dev/null @@ -1,285 +0,0 @@ -PyEClib -------- - -This library provides a simple Python interface for implementing erasure codes -and is known to work with Python v2.6, 2.7 and 3.x. - -To obtain the best possible performance, the library utilizes liberasurecode, -which is a C based erasure code library. Please let us know if you have any -issues building or installing (email: kmgreen2@gmail.com or tusharsg@gmail.com). - -PyECLib supports a variety of Erasure Coding backends including the standard Reed -Soloman implementations provided by Jerasure [1], liberasurecode [3] and Intel -ISA-L [4]. It also provides support for a flat XOR-based encoder and decoder -(part of liberasurecode) - a class of HD Combination Codes based on "Flat -XOR-based erasure codes in storage systems: Constructions, efficient recovery, -and tradeoffs" in IEEE MSST 2010[2]). These codes are well-suited to archival -use-cases, have a simple construction and require a minimum number of -participating disks during single-disk reconstruction (think XOR-based LRC code). - -Examples of using PyECLib are provided in the "tools" directory: - - Command-line encoder:: - - tools/pyeclib_encode.py - - Command-line decoder:: - - tools/pyeclib_decode.py - - Utility to determine what is needed to reconstruct missing fragments:: - - tools/pyeclib_fragments_needed.py - - A configuration utility to help compare available EC schemes in terms of - performance and redundancy:: - - tools/pyeclib_conf_tool.py - -PyEClib initialization:: - - ec_driver = ECDriver(k=, - m=, - ec_type=)) - -Supported ``ec_type`` values: - - * ``liberasurecode_rs_vand`` => Vandermonde Reed-Solomon encoding, software-only backend implemented by liberasurecode [3] - * ``jerasure_rs_vand`` => Vandermonde Reed-Solomon encoding, based on Jerasure [1] - * ``jerasure_rs_cauchy`` => Cauchy Reed-Solomon encoding (Jerasure variant), based on Jerasure [1] - * ``flat_xor_hd_3``, ``flat_xor_hd_4`` => Flat-XOR based HD combination codes, liberasurecode [3] - * ``isa_l_rs_vand`` => Intel Storage Acceleration Library (ISA-L) - SIMD accelerated Erasure Coding backends [4] - * ``isa_l_rs_cauchy`` => Cauchy Reed-Solomon encoding (ISA-L variant) [4] - * ``shss`` => NTT Lab Japan's Erasure Coding Library [5] - * ``libphazr`` => Phazr.IO's erasure code library with built-in privacy [6] - - -The Python API supports the following functions: - -- EC Encode - - Encode N bytes of a data object into k (data) + m (parity) fragments:: - - def encode(self, data_bytes) - - input: data_bytes - input data object (bytes) - returns: list of fragments (bytes) - throws: - ECBackendInstanceNotAvailable - if the backend library cannot be found - ECBackendNotSupported - if the backend is not supported by PyECLib (see ec_types above) - ECInvalidParameter - if invalid parameters were provided - ECOutOfMemory - if the process has run out of memory - ECDriverError - if an unknown error occurs - -- EC Decode - - Decode between k and k+m fragments into original object:: - - def decode(self, fragment_payloads) - - input: list of fragment_payloads (bytes) - returns: decoded object (bytes) - throws: - ECBackendInstanceNotAvailable - if the backend library cannot be found - ECBackendNotSupported - if the backend is not supported by PyECLib (see ec_types above) - ECInvalidParameter - if invalid parameters were provided - ECOutOfMemory - if the process has run out of memory - ECInsufficientFragments - if an insufficient set of fragments has been provided (e.g. not enough) - ECInvalidFragmentMetadata - if the fragment headers appear to be corrupted - ECDriverError - if an unknown error occurs - - -*Note*: ``bytes`` is a synonym to ``str`` in Python 2.6, 2.7. -In Python 3.x, ``bytes`` and ``str`` types are non-interchangeable and care -needs to be taken when handling input to and output from the ``encode()`` and -``decode()`` routines. - - -- EC Reconstruct - - Reconstruct "missing_fragment_indexes" using "available_fragment_payloads":: - - def reconstruct(self, available_fragment_payloads, missing_fragment_indexes) - - input: available_fragment_payloads - list of fragment payloads - input: missing_fragment_indexes - list of indexes to reconstruct - output: list of reconstructed fragments corresponding to missing_fragment_indexes - throws: - ECBackendInstanceNotAvailable - if the backend library cannot be found - ECBackendNotSupported - if the backend is not supported by PyECLib (see ec_types above) - ECInvalidParameter - if invalid parameters were provided - ECOutOfMemory - if the process has run out of memory - ECInsufficientFragments - if an insufficient set of fragments has been provided (e.g. not enough) - ECInvalidFragmentMetadata - if the fragment headers appear to be corrupted - ECDriverError - if an unknown error occurs - - -- Minimum parity fragments needed for durability gurantees:: - - def min_parity_fragments_needed(self) - - NOTE: Currently hard-coded to 1, so this can only be trusted for MDS codes, such as - Reed-Solomon. - - output: minimum number of additional fragments needed to be synchronously written to tolerate - the loss of any one fragment (similar guarantees to 2 out of 3 with 3x replication) - throws: - ECBackendInstanceNotAvailable - if the backend library cannot be found - ECBackendNotSupported - if the backend is not supported by PyECLib (see ec_types above) - ECInvalidParameter - if invalid parameters were provided - ECOutOfMemory - if the process has run out of memory - ECDriverError - if an unknown error occurs - - -- Fragments needed for EC Reconstruct - - Return the indexes of fragments needed to reconstruct "missing_fragment_indexes":: - - def fragments_needed(self, missing_fragment_indexes) - - input: list of missing_fragment_indexes - output: list of fragments needed to reconstruct fragments listed in missing_fragment_indexes - throws: - ECBackendInstanceNotAvailable - if the backend library cannot be found - ECBackendNotSupported - if the backend is not supported by PyECLib (see ec_types above) - ECInvalidParameter - if invalid parameters were provided - ECOutOfMemory - if the process has run out of memory - ECDriverError - if an unknown error occurs - - -- Get EC Metadata - - Return an opaque header known by the underlying library or a formatted header (Python dict):: - - def get_metadata(self, fragment, formatted = 0) - - input: raw fragment payload - input: boolean specifying if returned header is opaque buffer or formatted string - output: fragment header (opaque or formatted) - throws: - ECBackendInstanceNotAvailable - if the backend library cannot be found - ECBackendNotSupported - if the backend is not supported by PyECLib (see ec_types above) - ECInvalidParameter - if invalid parameters were provided - ECOutOfMemory - if the process has run out of memory - ECDriverError - if an unknown error occurs - -- Verify EC Stripe Consistency - - Use opaque buffers from get_metadata() to verify a the consistency of a stripe:: - - def verify_stripe_metadata(self, fragment_metadata_list) - - intput: list of opaque fragment headers - output: formatted string containing the 'status' (0 is success) and 'reason' if verification fails - throws: - ECBackendInstanceNotAvailable - if the backend library cannot be found - ECBackendNotSupported - if the backend is not supported by PyECLib (see ec_types above) - ECInvalidParameter - if invalid parameters were provided - ECOutOfMemory - if the process has run out of memory - ECDriverError - if an unknown error occurs - - -- Get EC Segment Info - - Return a dict with the keys - segment_size, last_segment_size, fragment_size, last_fragment_size and num_segments:: - - def get_segment_info(self, data_len, segment_size) - - input: total data_len of the object to store - input: target segment size used to segment the object into multiple EC stripes - output: a dict with keys - segment_size, last_segment_size, fragment_size, last_fragment_size and num_segments - throws: - ECBackendInstanceNotAvailable - if the backend library cannot be found - ECBackendNotSupported - if the backend is not supported by PyECLib (see ec_types above) - ECInvalidParameter - if invalid parameters were provided - ECOutOfMemory - if the process has run out of memory - ECDriverError - if an unknown error occurs - - -- Get EC Segment Info given a list of ranges, data length and segment size:: - - def get_segment_info_byterange(self, ranges, data_len, segment_size) - - input: byte ranges - input: total data_len of the object to store - input: target segment size used to segment the object into multiple EC stripes - output: (see below) - throws: - ECBackendInstanceNotAvailable - if the backend library cannot be found - ECBackendNotSupported - if the backend is not supported by PyECLib (see ec_types above) - ECInvalidParameter - if invalid parameters were provided - ECOutOfMemory - if the process has run out of memory - ECDriverError - if an unknown error occurs - - Assume a range request is given for an object with segment size 3K and - a 1 MB file:: - - Ranges = (0, 1), (1, 12), (10, 1000), (0, segment_size-1), - (1, segment_size+1), (segment_size-1, 2*segment_size) - - This will return a map keyed on the ranges, where there is a recipe - given for each range:: - - { - (0, 1): {0: (0, 1)}, - (10, 1000): {0: (10, 1000)}, - (1, 12): {0: (1, 12)}, - (0, 3071): {0: (0, 3071)}, - (3071, 6144): {0: (3071, 3071), 1: (0, 3071), 2: (0, 0)}, - (1, 3073): {0: (1, 3071), 1: (0,0)} - } - - -Quick Start - - Install pre-requisites:: - - * Python 2.6, 2.7 or 3.x (including development packages), argparse, setuptools - * liberasurecode v1.2.0 or greater [3] - * Erasure code backend libraries, gf-complete and Jerasure [1],[2], ISA-L [4] etc - - An example for ubuntu to install dependency packages:: - - $ sudo apt-get install build-essential python-dev python-pip liberasurecode-dev - $ sudo pip install -U bindep -r test-requirements.txt - - If you want to confirm all dependency packages installed successfully, try:: - - $ sudo bindep -f bindep.txt - - *Note*: currently liberasurecode-dev/liberasurecode-devel in package repo is older than v1.2.0 - - Install PyECLib:: - - $ sudo python setup.py install - - Run test suite included:: - - $ ./.unittests - - If all of this works, then you should be good to go. If not, send us an email! - - If the test suite fails because it cannot find any of the shared libraries, - then you probably need to add /usr/local/lib to the path searched when loading - libraries. The best way to do this (on Linux) is to add '/usr/local/lib' to:: - - /etc/ld.so.conf - - and then make sure to run:: - - $ sudo ldconfig - - -References - - [1] Jerasure, C library that supports erasure coding in storage applications, http://jerasure.org - - [2] Greenan, Kevin M et al, "Flat XOR-based erasure codes in storage systems", http://www.kaymgee.com/Kevin_Greenan/Publications_files/greenan-msst10.pdf - - [3] liberasurecode, C API abstraction layer for erasure coding backends, https://github.com/openstack/liberasurecode - - [4] Intel(R) Storage Acceleration Library (Open Source Version), https://01.org/intel%C2%AE-storage-acceleration-library-open-source-version - - [5] Kota Tsuyuzaki , "NTT SHSS Erasure Coding backend" - - [6] Jim Cheung , "Phazr.IO libphazr erasure code backend with built-in privacy" diff --git a/bindep.txt b/bindep.txt deleted file mode 100644 index 9302ef5..0000000 --- a/bindep.txt +++ /dev/null @@ -1,10 +0,0 @@ -# This is a cross-platform list tracking distribution packages needed by tests; -# see http://docs.openstack.org/infra/bindep/ for additional information. - -build-essential [platform:dpkg] -gcc [platform:rpm] -make [platform:rpm] -liberasurecode-dev [platform:dpkg] -liberasurecode-devel [platform:rpm] -python-dev [platform:dpkg] -python-devel [platform:rpm] diff --git a/doc/source/conf.py b/doc/source/conf.py deleted file mode 100644 index 8d51341..0000000 --- a/doc/source/conf.py +++ /dev/null @@ -1,242 +0,0 @@ -# -*- coding: utf-8 -*- -# -# PyECLib documentation build configuration file, created by -# sphinx-quickstart on Thu Aug 11 19:59:50 2016. -# -# This file is execfile()d with the current directory set to its containing dir. -# -# Note that not all possible configuration values are present in this -# autogenerated file. -# -# All configuration values have a default; values that are commented out -# serve to show the default. - -import sys, os - -# If extensions (or modules to document with autodoc) are in another directory, -# add these directories to sys.path here. If the directory is relative to the -# documentation root, use os.path.abspath to make it absolute, like shown here. -#sys.path.insert(0, os.path.abspath('.')) - -# -- General configuration ----------------------------------------------------- - -# If your documentation needs a minimal Sphinx version, state it here. -#needs_sphinx = '1.0' - -# Add any Sphinx extension module names here, as strings. They can be extensions -# coming with Sphinx (named 'sphinx.ext.*') or your custom ones. -extensions = [] - -# Add any paths that contain templates here, relative to this directory. -templates_path = ['_templates'] - -# The suffix of source filenames. -source_suffix = '.rst' - -# The encoding of source files. -#source_encoding = 'utf-8-sig' - -# The master toctree document. -master_doc = 'index' - -# General information about the project. -project = u'PyECLib' -copyright = u'2016, Kevin Greenan, Tushar Gohad, Kota Tsuyuzaki' - -# The version info for the project you're documenting, acts as replacement for -# |version| and |release|, also used in various other places throughout the -# built documents. -# -# The short X.Y version. -version = '1.5.0' -# The full version, including alpha/beta/rc tags. -release = '1.5.0' - -# The language for content autogenerated by Sphinx. Refer to documentation -# for a list of supported languages. -#language = None - -# There are two options for replacing |today|: either, you set today to some -# non-false value, then it is used: -#today = '' -# Else, today_fmt is used as the format for a strftime call. -#today_fmt = '%B %d, %Y' - -# List of patterns, relative to source directory, that match files and -# directories to ignore when looking for source files. -exclude_patterns = [] - -# The reST default role (used for this markup: `text`) to use for all documents. -#default_role = None - -# If true, '()' will be appended to :func: etc. cross-reference text. -#add_function_parentheses = True - -# If true, the current module name will be prepended to all description -# unit titles (such as .. function::). -#add_module_names = True - -# If true, sectionauthor and moduleauthor directives will be shown in the -# output. They are ignored by default. -#show_authors = False - -# The name of the Pygments (syntax highlighting) style to use. -pygments_style = 'sphinx' - -# A list of ignored prefixes for module index sorting. -#modindex_common_prefix = [] - - -# -- Options for HTML output --------------------------------------------------- - -# The theme to use for HTML and HTML Help pages. See the documentation for -# a list of builtin themes. -html_theme = 'default' - -# Theme options are theme-specific and customize the look and feel of a theme -# further. For a list of options available for each theme, see the -# documentation. -#html_theme_options = {} - -# Add any paths that contain custom themes here, relative to this directory. -#html_theme_path = [] - -# The name for this set of Sphinx documents. If None, it defaults to -# " v documentation". -#html_title = None - -# A shorter title for the navigation bar. Default is the same as html_title. -#html_short_title = None - -# The name of an image file (relative to this directory) to place at the top -# of the sidebar. -#html_logo = None - -# The name of an image file (within the static path) to use as favicon of the -# docs. This file should be a Windows icon file (.ico) being 16x16 or 32x32 -# pixels large. -#html_favicon = None - -# Add any paths that contain custom static files (such as style sheets) here, -# relative to this directory. They are copied after the builtin static files, -# so a file named "default.css" will overwrite the builtin "default.css". -html_static_path = ['_static'] - -# If not '', a 'Last updated on:' timestamp is inserted at every page bottom, -# using the given strftime format. -#html_last_updated_fmt = '%b %d, %Y' - -# If true, SmartyPants will be used to convert quotes and dashes to -# typographically correct entities. -#html_use_smartypants = True - -# Custom sidebar templates, maps document names to template names. -#html_sidebars = {} - -# Additional templates that should be rendered to pages, maps page names to -# template names. -#html_additional_pages = {} - -# If false, no module index is generated. -#html_domain_indices = True - -# If false, no index is generated. -#html_use_index = True - -# If true, the index is split into individual pages for each letter. -#html_split_index = False - -# If true, links to the reST sources are added to the pages. -#html_show_sourcelink = True - -# If true, "Created using Sphinx" is shown in the HTML footer. Default is True. -#html_show_sphinx = True - -# If true, "(C) Copyright ..." is shown in the HTML footer. Default is True. -#html_show_copyright = True - -# If true, an OpenSearch description file will be output, and all pages will -# contain a tag referring to it. The value of this option must be the -# base URL from which the finished HTML is served. -#html_use_opensearch = '' - -# This is the file name suffix for HTML files (e.g. ".xhtml"). -#html_file_suffix = None - -# Output file base name for HTML help builder. -htmlhelp_basename = 'PyECLibdoc' - - -# -- Options for LaTeX output -------------------------------------------------- - -latex_elements = { -# The paper size ('letterpaper' or 'a4paper'). -#'papersize': 'letterpaper', - -# The font size ('10pt', '11pt' or '12pt'). -#'pointsize': '10pt', - -# Additional stuff for the LaTeX preamble. -#'preamble': '', -} - -# Grouping the document tree into LaTeX files. List of tuples -# (source start file, target name, title, author, documentclass [howto/manual]). -latex_documents = [ - ('index', 'PyECLib.tex', u'PyECLib Documentation', - u'Kevin Greenan, Tushar Gohad, Kota Tsuyuzaki', 'manual'), -] - -# The name of an image file (relative to this directory) to place at the top of -# the title page. -#latex_logo = None - -# For "manual" documents, if this is true, then toplevel headings are parts, -# not chapters. -#latex_use_parts = False - -# If true, show page references after internal links. -#latex_show_pagerefs = False - -# If true, show URL addresses after external links. -#latex_show_urls = False - -# Documents to append as an appendix to all manuals. -#latex_appendices = [] - -# If false, no module index is generated. -#latex_domain_indices = True - - -# -- Options for manual page output -------------------------------------------- - -# One entry per manual page. List of tuples -# (source start file, name, description, authors, manual section). -man_pages = [ - ('index', 'pyeclib', u'PyECLib Documentation', - [u'Kevin Greenan, Tushar Gohad, Kota Tsuyuzaki'], 1) -] - -# If true, show URL addresses after external links. -#man_show_urls = False - - -# -- Options for Texinfo output ------------------------------------------------ - -# Grouping the document tree into Texinfo files. List of tuples -# (source start file, target name, title, author, -# dir menu entry, description, category) -texinfo_documents = [ - ('index', 'PyECLib', u'PyECLib Documentation', - u'Kevin Greenan, Tushar Gohad, Kota Tsuyuzaki', 'PyECLib', 'One line description of project.', - 'Miscellaneous'), -] - -# Documents to append as an appendix to all manuals. -#texinfo_appendices = [] - -# If false, no module index is generated. -#texinfo_domain_indices = True - -# How to display URL addresses: 'footnote', 'no', or 'inline'. -#texinfo_show_urls = 'footnote' diff --git a/doc/source/index.rst b/doc/source/index.rst deleted file mode 100644 index 35d6333..0000000 --- a/doc/source/index.rst +++ /dev/null @@ -1,22 +0,0 @@ -.. PyECLib documentation master file, created by - sphinx-quickstart on Thu Aug 11 19:59:50 2016. - You can adapt this file completely to your liking, but it should at least - contain the root `toctree` directive. - -Welcome to PyECLib's documentation! -=================================== - -Contents: - -.. toctree:: - :maxdepth: 2 - - - -Indices and tables -================== - -* :ref:`genindex` -* :ref:`modindex` -* :ref:`search` - diff --git a/pyeclib/__init__.py b/pyeclib/__init__.py deleted file mode 100644 index fc3df4e..0000000 --- a/pyeclib/__init__.py +++ /dev/null @@ -1,22 +0,0 @@ -# Copyright (c) 2013, Kevin Greenan (kmgreen2@gmail.com) -# All rights reserved. -# -# Redistribution and use in source and binary forms, with or without -# modification, are permitted provided that the following conditions are met: -# -# Redistributions of source code must retain the above copyright notice, this -# list of conditions and the following disclaimer. -# -# Redistributions in binary form must reproduce the above copyright notice, -# this list of conditions and the following disclaimer in the documentation -# and/or other materials provided with the distribution. THIS SOFTWARE IS -# PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS -# OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES -# OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN -# NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY -# DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES -# (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; -# LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND -# ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT -# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF -# THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. diff --git a/pyeclib/core.py b/pyeclib/core.py deleted file mode 100644 index c7e38df..0000000 --- a/pyeclib/core.py +++ /dev/null @@ -1,289 +0,0 @@ -# Copyright (c) 2013, 2014, Kevin Greenan (kmgreen2@gmail.com) -# All rights reserved. -# -# Redistribution and use in source and binary forms, with or without -# modification, are permitted provided that the following conditions are met: -# -# Redistributions of source code must retain the above copyright notice, this -# list of conditions and the following disclaimer. -# -# Redistributions in binary form must reproduce the above copyright notice, -# this list of conditions and the following disclaimer in the documentation -# and/or other materials provided with the distribution. THIS SOFTWARE IS -# PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS -# OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES -# OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN -# NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY -# DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES -# (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; -# LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND -# ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT -# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF -# THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - -from .ec_iface import ECDriverError -from .ec_iface import ECInsufficientFragments -from .ec_iface import PyECLib_FRAGHDRCHKSUM_Types - -import math -import pyeclib_c -import sys - -pyver = float('%s.%s' % sys.version_info[:2]) - - -class ECPyECLibDriver(object): - - def __init__(self, k, m, hd, ec_type, - chksum_type=PyECLib_FRAGHDRCHKSUM_Types.none, - validate=False): - self.k = k - self.m = m - self.hd = hd - self.ec_type = ec_type - self.chksum_type = chksum_type - - self.inline_chksum = 0 - self.algsig_chksum = 0 - # crc32 is the only inline checksum type currently supported - if self.chksum_type is PyECLib_FRAGHDRCHKSUM_Types.inline_crc32: - self.inline_chksum = 1 - - name = self.ec_type.name - - self.handle = pyeclib_c.init( - self.k, - self.m, - ec_type.value, - self.hd, - self.inline_chksum, - self.algsig_chksum, - validate) - - def encode(self, data_bytes): - return pyeclib_c.encode(self.handle, data_bytes) - - def _validate_and_return_fragment_size(self, fragments): - if len(fragments) == 0 or len(fragments[0]) == 0: - return -1 - fragment_len = len(fragments[0]) - for fragment in fragments[1:]: - if len(fragment) != fragment_len: - return -1 - return fragment_len - - def decode(self, fragment_payloads, ranges=None, - force_metadata_checks=False): - _fragment_payloads = list(fragment_payloads) - fragment_len = self._validate_and_return_fragment_size( - _fragment_payloads) - if fragment_len < 0: - raise ECDriverError( - "Invalid fragment payload in ECPyECLibDriver.decode") - - if len(_fragment_payloads) < self.k: - raise ECInsufficientFragments( - "Not enough fragments given in ECPyECLibDriver.decode") - - return pyeclib_c.decode(self.handle, _fragment_payloads, - fragment_len, ranges, force_metadata_checks) - - def reconstruct(self, fragment_payloads, indexes_to_reconstruct): - _fragment_payloads = list(fragment_payloads) - fragment_len = self._validate_and_return_fragment_size( - _fragment_payloads) - if fragment_len < 0: - raise ECDriverError( - "Invalid fragment payload in ECPyECLibDriver.reconstruct") - - reconstructed_data = [] - - # Reconstruct the data, then the parity - # The parity cannot be reconstructed until - # after all data is reconstructed - indexes_to_reconstruct.sort() - _indexes_to_reconstruct = indexes_to_reconstruct[:] - - while len(_indexes_to_reconstruct) > 0: - index = _indexes_to_reconstruct.pop(0) - reconstructed = pyeclib_c.reconstruct( - self.handle, _fragment_payloads, fragment_len, index) - reconstructed_data.append(reconstructed) - _fragment_payloads.append(reconstructed) - - return reconstructed_data - - def fragments_needed(self, reconstruct_indexes, exclude_indexes): - required_fragments = pyeclib_c.get_required_fragments( - self.handle, reconstruct_indexes, exclude_indexes) - return required_fragments - - def min_parity_fragments_needed(self): - """ FIXME - fix this to return a function of HD """ - return 1 - - def get_metadata(self, fragment, formatted=0): - fragment_metadata = pyeclib_c.get_metadata(self.handle, fragment, - formatted) - return fragment_metadata - - def verify_stripe_metadata(self, fragment_metadata_list): - success = pyeclib_c.check_metadata(self.handle, fragment_metadata_list) - return success - - def get_segment_info(self, data_len, segment_size): - segment_info = pyeclib_c.get_segment_info(self.handle, data_len, - segment_size) - return segment_info - - -class ECNullDriver(object): - - def __init__(self, k, m, hd, ec_type=None, chksum_type=None, - validate=False): - self.k = k - self.m = m - self.hd = hd - - def encode(self, data_bytes): - pass - - def decode(self, fragment_payloads, ranges, force_metadata_checks): - pass - - def reconstruct(self, available_fragment_payloads, - missing_fragment_indexes): - pass - - def fragments_needed(self, missing_fragment_indexes): - pass - - def get_metadata(self, fragment, formatted=0): - pass - - def min_parity_fragments_needed(self): - pass - - def verify_stripe_metadata(self, fragment_metadata_list): - pass - - def get_segment_info(self, data_len, segment_size): - pass - - -# -# A striping-only driver for EC. This is -# pretty much RAID 0. -# -class ECStripingDriver(object): - - def __init__(self, k, m, hd, ec_type=None, chksum_type=None, - validate=False): - """Stripe an arbitrary-sized string into k fragments - :param k: the number of data fragments to stripe - :param m: the number of parity fragments to stripe - :raises: ECDriverError if there is an error during encoding - """ - self.k = k - - if m != 0: - raise ECDriverError("This driver only supports m=0") - - self.m = m - self.hd = hd - - def encode(self, data_bytes): - """Stripe an arbitrary-sized string into k fragments - :param data_bytes: the buffer to encode - :returns: a list of k buffers (data only) - :raises: ECDriverError if there is an error during encoding - """ - # Main fragment size - fragment_size = math.ceil(len(data_bytes) / float(self.k)) - - # Size of last fragment - last_fragment_size = len(data_bytes) - (fragment_size * self.k - 1) - - fragments = [] - offset = 0 - for i in range(self.k - 1): - fragments.append(data_bytes[offset:fragment_size]) - offset += fragment_size - - fragments.append(data_bytes[offset:last_fragment_size]) - - return fragments - - def decode(self, fragment_payloads, ranges=None, - force_metadata_checks=False): - """Convert a k-fragment data stripe into a string - :param fragment_payloads: fragments (in order) to convert into a string - :param ranges (unsupported): range decode - :param force_metadata_checks (unsupported): verify fragment metadata - :returns: a string containing original data - :raises: ECDriverError if there is an error during decoding - """ - if ranges is not None: - raise ECDriverError("Decode does not support range requests in the" - " striping driver.") - if force_metadata_checks is not False: - raise ECDriverError( - "Decode does not support metadata integrity checks in the " - " striping driver.") - if len(fragment_payloads) != self.k: - raise ECInsufficientFragments( - "Decode requires %d fragments, %d fragments were given" % - (len(fragment_payloads), self.k)) - - ret_string = '' - - for fragment in fragment_payloads: - ret_string += fragment - - return ret_string - - def reconstruct(self, available_fragment_payloads, - missing_fragment_indexes): - """We cannot reconstruct a fragment using other fragments. This means - that reconstruction means all fragments must be specified, otherwise we - cannot reconstruct and must raise an error. - :param available_fragment_payloads: available fragments (in order) - :param missing_fragment_indexes: indexes of missing fragments - :returns: a string containing the original data - :raises: ECDriverError if there is an error during reconstruction - """ - if len(available_fragment_payloads) != self.k: - raise ECDriverError( - "Reconstruction requires %d fragments, %d fragments given" % - (len(available_fragment_payloads), self.k)) - - return available_fragment_payloads - - def fragments_needed(self, missing_fragment_indexes): - """By definition, all missing fragment indexes are needed to - reconstruct, so just return the list handed to this function. - :param missing_fragment_indexes: indexes of missing fragments - :returns: missing_fragment_indexes - """ - return missing_fragment_indexes - - def min_parity_fragments_needed(self): - pass - - def get_metadata(self, fragment, formatted=0): - """This driver does not include fragment metadata, so return empty - string - :param fragment: a fragment - :returns: empty string - """ - return '' - - def verify_stripe_metadata(self, fragment_metadata_list): - """This driver does not include fragment metadata, so return true - :param fragment_metadata_list: a list of fragments - :returns: True - """ - return True - - def get_segment_info(self, data_len, segment_size): - pass diff --git a/pyeclib/ec_iface.py b/pyeclib/ec_iface.py deleted file mode 100644 index ed7a7a6..0000000 --- a/pyeclib/ec_iface.py +++ /dev/null @@ -1,559 +0,0 @@ -# Copyright (c) 2013-2014, Kevin Greenan (kmgreen2@gmail.com) -# Copyright (c) 2014, Tushar Gohad (tushar.gohad@intel.com) -# All rights reserved. -# -# Redistribution and use in source and binary forms, with or without -# modification, are permitted provided that the following conditions are met: -# -# Redistributions of source code must retain the above copyright notice, this -# list of conditions and the following disclaimer. -# -# Redistributions in binary form must reproduce the above copyright notice, -# this list of conditions and the following disclaimer in the documentation -# and/or other materials provided with the distribution. THIS SOFTWARE IS -# PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS -# OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES -# OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN -# NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY -# DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES -# (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; -# LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND -# ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT -# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF -# THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - -from .enum import Enum -from .enum import unique -from .utils import create_instance -from .utils import positive_int_value -from pyeclib_c import get_liberasurecode_version - -import logging -from logging.handlers import SysLogHandler -logger = logging.getLogger('pyeclib') -syslog_handler = SysLogHandler() -logger.addHandler(syslog_handler) - - -def check_backend_available(backend_name): - try: - from pyeclib_c import check_backend_available - - if backend_name.startswith('flat_xor_hd'): - int_type = PyECLib_EC_Types.get_by_name('flat_xor_hd') - else: - int_type = PyECLib_EC_Types.get_by_name(backend_name) - if not int_type: - return False - return check_backend_available(int_type.value) - except ImportError: - # check_backend_available has been supported since - # liberasurecode>=1.2.0 so we need to define the func for older - # liberasurecode version - - # select available k, m values - if backend_name.startswith('flat_xor_hd'): - k, m = (10, 5) - else: - k, m = (10, 4) - try: - driver = ECDriver(ec_type=backend_name, k=k, m=m) - except ECDriverError: - return False - return True - - -def PyECLibVersion(z, y, x): - return (((z) << 16) + ((y) << 8) + (x)) - -PYECLIB_MAJOR = 1 -PYECLIB_MINOR = 1 -PYECLIB_REV = 2 -PYECLIB_VERSION = PyECLibVersion(PYECLIB_MAJOR, PYECLIB_MINOR, - PYECLIB_REV) - - -PYECLIB_MAX_DATA = 32 -PYECLIB_MAX_PARITY = 32 - - -@unique -class PyECLibEnum(Enum): - - def describe(self): - # returns supported types - return list(self) - - @classmethod - def has_enum(cls, name): - # returns True if name is a valid member of the enum - try: - cls.__getattr__(name) - except AttributeError: - return False - return True - - @classmethod - def get_by_name(cls, name): - try: - obj = cls.__getattr__(name) - except AttributeError: - return None - return obj - - @classmethod - def names(cls): - return [name for name, value in cls.__members__.items()] - - @classmethod - def values(cls): - return [value for name, value in cls.__members__.items()] - - def __str__(self): - return "%s: %d" % (self.name, self.value) - - -# Erasure Code backends supported as of this PyECLib API rev -class PyECLib_EC_Types(PyECLibEnum): - # Note: the Enum start value defaults to 1 as the starting value and not 0 - # 0 is False in the boolean sense but enum members evaluate to True - jerasure_rs_vand = 1 - jerasure_rs_cauchy = 2 - flat_xor_hd = 3 - isa_l_rs_vand = 4 - shss = 5 - liberasurecode_rs_vand = 6 - isa_l_rs_cauchy = 7 - libphazr = 8 - - -# Output of Erasure (en)Coding process are data "fragments". Fragment data -# integrity checks are provided by a checksum embedded in a header (prepend) -# for each fragment. - -# The following Enum defines the schemes supported for fragment checksums. -# The checksum type is "none" unless specified. -class PyECLib_FRAGHDRCHKSUM_Types(PyECLibEnum): - # Note: the Enum start value defaults to 1 as the starting value and not 0 - # 0 is False in the boolean sense but enum members evaluate to True - none = 1 - inline_crc32 = 2 - - -# Main ECDriver class -class ECDriver(object): - - def __init__(self, *args, **kwargs): - self.k = -1 - self.m = -1 - self.hd = -1 - self.ec_type = None - self.chksum_type = None - self.validate = False - - for required in ('k', 'm'): - if required not in kwargs: - raise ECDriverError( - "Invalid Argument: %s is required" % required) - - for (key, value) in kwargs.items(): - if key == "k": - try: - self.k = positive_int_value(value) - except ValueError: - raise ECDriverError( - "Invalid number of data fragments (k)") - elif key == "m": - try: - self.m = positive_int_value(value) - except ValueError: - raise ECDriverError( - "Invalid number of parity fragments (m)") - elif key == "ec_type": - if value in ["flat_xor_hd", "flat_xor_hd_3", "flat_xor_hd_4"]: - if value == "flat_xor_hd" or value == "flat_xor_hd_3": - self.hd = 3 - elif value == "flat_xor_hd_4": - self.hd = 4 - value = "flat_xor_hd" - elif value == "libphazr": - self.hd = 1 - if PyECLib_EC_Types.has_enum(value): - self.ec_type = PyECLib_EC_Types.get_by_name(value) - else: - raise ECBackendNotSupported( - "%s is not a valid EC type for PyECLib!" % value) - elif key == "chksum_type": - if PyECLib_FRAGHDRCHKSUM_Types.has_enum(value): - self.chksum_type = \ - PyECLib_FRAGHDRCHKSUM_Types.get_by_name(value) - else: - raise ECDriverError( - "%s is not a valid checksum type for PyECLib!" % value) - elif key == "validate": - # validate if the ec type is available (runtime check) - self.validate = value - - if self.hd == -1: - self.hd = self.m - - self.library_import_str = kwargs.pop('library_import_str', - 'pyeclib.core.ECPyECLibDriver') - # - # Instantiate EC backend driver - # - self.ec_lib_reference = create_instance( - self.library_import_str, - k=self.k, - m=self.m, - hd=self.hd, - ec_type=self.ec_type, - chksum_type=self.chksum_type, - validate=int(self.validate) - ) - - # - # Verify that the imported library implements the required functions - # - required_methods = [ - 'decode', - 'encode', - 'reconstruct', - 'fragments_needed', - 'min_parity_fragments_needed', - 'get_metadata', - 'verify_stripe_metadata', - 'get_segment_info', - ] - - missing_methods = ' '.join( - method for method in required_methods - if not callable(getattr(self.ec_lib_reference, method, None))) - - if missing_methods: - raise ECDriverError( - "The following required methods are not implemented " - "in %s: %s" % (self.library_import_str, missing_methods)) - - def __repr__(self): - return '%s(ec_type=%r, k=%r, m=%r)' % ( - type(self).__name__, - 'flat_xor_hd_%s' % self.hd if self.ec_type.name == 'flat_xor_hd' - else self.ec_type.name, - self.k, - self.m) - - def encode(self, data_bytes): - """ - Encode an arbitrary-sized string - :param data_bytes: the buffer to encode - :returns: a list of buffers (first k entries are data and - the last m are parity) - :raises: ECDriverError if there is an error during encoding - """ - return self.ec_lib_reference.encode(data_bytes) - - def decode(self, fragment_payloads, ranges=None, - force_metadata_checks=False): - """ - Decode a set of fragments into a buffer that represents the original - buffer passed into encode(). - - :param fragment_payloads: a list of buffers representing a subset of - the list generated by encode() - :param ranges (optional): a list of byte ranges to return instead of - the entire buffer - :param force_metadata_checks (optional): validate collective integrity - of the fragments before trying to decode - :returns: a buffer - :raises: ECDriverError if there is an error during decoding - """ - return self.ec_lib_reference.decode(fragment_payloads, ranges, - force_metadata_checks) - - def reconstruct(self, available_fragment_payloads, - missing_fragment_indexes): - """ - Reconstruct a missing fragment from a subset of available fragments. - - :param available_fragment_payloads: a list of buffers representing - a subset of the list generated - by encode() - :param missing_fragment_indexes: a list of integers representing - the indexes of the fragments to be - reconstructed. - :param destination_index: the index of the element to reconstruct - :returns: a list of buffers (ordered by fragment index) containing - the reconstructed payload associated with the indexes - provided in missing_fragment_indexes - :raises: ECDriverError if there is an error during decoding or there - are not sufficient fragments to decode - """ - return self.ec_lib_reference.reconstruct( - available_fragment_payloads, missing_fragment_indexes) - - def fragments_needed(self, reconstruction_indexes, - exclude_indexes=[]): - """ - Determine which fragments are needed to reconstruct some subset of - missing fragments. - - :param reconstruction_indexes: a list of integers representing the - indexes of the fragments to be - reconstructed. - :param exclude_indexes: a list of integers representing the - indexes of the fragments to be - excluded from the reconstruction - equations. - :returns: a list containing fragment indexes that can be used to - reconstruct the missing fragments. - :raises: ECDriverError if there is an error during decoding or there - are not sufficient fragments to decode - """ - return self.ec_lib_reference.fragments_needed(reconstruction_indexes, - exclude_indexes) - - def min_parity_fragments_needed(self): - return self.ec_lib_reference.min_parity_fragments_needed() - - def get_metadata(self, fragment, formatted=0): - """ - Get opaque metadata for a fragment. The metadata is opaque to the - client, but meaningful to the underlying library. It is used to verify - stripes in verify_stripe_metadata(). - - :param fragment: a buffer representing a single fragment generated by - the encode() function. - :returns: an opaque buffer to be passed into verify_stripe_metadata() - :raises: ECDriverError if there was a problem getting the metadata. - """ - return self.ec_lib_reference.get_metadata(fragment, formatted) - - def verify_stripe_metadata(self, fragment_metadata_list): - """ - Verify a subset of fragments generated by encode() - - :param fragment_metadata_list: a list of buffers representing the - metadata from a subset of the framgments - generated by encode(). - :returns: 'None' if the metadata is consistent. - a list of fragment indexes corresponding to inconsistent - fragments - :raises: ECDriverError if there was a problem verifying the metadata - - """ - return self.ec_lib_reference.verify_stripe_metadata( - fragment_metadata_list) - - def get_segment_info(self, data_len, segment_size): - """ - Get segmentation info for a given data length and - segment size. - - Semment info returns a dict with the following keys: - - segment_size: size of the payload to give to encode() - last_segment_size: size of the payload to give to encode() - fragment_size: the fragment size returned by encode() - last_fragment_size: the fragment size returned by encode() - num_segments: number of segments - - This allows the caller to prepare requests - when segmenting a data stream to be EC'd. - - Since the data length will rarely be aligned - to the segment size, the last segment will be - a different size than the others. - - There are restrictions on the length given to encode(), - so calling this before encode is highly recommended when - segmenting a data stream. - """ - return self.ec_lib_reference.get_segment_info(data_len, segment_size) - - # - # Map of segment indexes with a list of tuples - # - def get_segment_info_byterange(self, ranges, data_len, segment_size): - """ - Get segmentation info for a byterange request, given a data length and - segment size. - - This will return a map-of-maps that represents a recipe describing - the segments and ranges within each segment needed to satisfy a range - request. - - Assume a range request is given for an object with segment size 3K and - a 1 MB file: - - Ranges = (0, 1), (1, 12), (10, 1000), (0, segment_size-1), - (1, segment_size+1), (segment_size-1, 2*segment_size) - - This will return a map keyed on the ranges, where there is a recipe - given for each range: - - { - (0, 1): {0: (0, 1)}, - (10, 1000): {0: (10, 1000)}, - (1, 12): {0: (1, 12)}, - (0, 3071): {0: (0, 3071)}, - (3071, 6144): {0: (3071, 3071), 1: (0, 3071), 2: (0, 0)}, - (1, 3073): {0: (1, 3071), 1: (0,0)} - } - - """ - - segment_info = self.ec_lib_reference.get_segment_info( - data_len, segment_size) - - segment_size = segment_info['segment_size'] - - sorted_ranges = ranges[:] - sorted_ranges.sort(key=lambda obj: obj[0]) - - recipe = {} - - for r in ranges: - segment_map = {} - begin_off = r[0] - end_off = r[1] - begin_segment = begin_off // segment_size - end_segment = end_off // segment_size - - if begin_segment == end_segment: - begin_relative_off = begin_off % segment_size - end_relative_off = end_off % segment_size - segment_map[begin_segment] = (begin_relative_off, - end_relative_off) - else: - begin_relative_off = begin_off % segment_size - end_relative_off = end_off % segment_size - - segment_map[begin_segment] = (begin_relative_off, - segment_size - 1) - - for middle_segment in range(begin_segment + 1, end_segment): - segment_map[middle_segment] = (0, segment_size - 1) - - segment_map[end_segment] = (0, end_relative_off) - - recipe[r] = segment_map - - return recipe - - -# PyECLib Exceptions - -# Generic ECDriverException -class ECDriverError(Exception): - def __init__(self, error): - try: - self.error_str = str(error) - except Exception: - self.error_str = 'Error retrieving the error message from %s' \ - % error.__class__.__name__ - - def __str__(self): - return self.error_str - - -# More specific exceptions, mapped to liberasurecode error codes - -# Specified EC backend is not supported by PyECLib/liberasurecode -class ECBackendNotSupported(ECDriverError): - pass - - -# Unsupported EC method -class ECMethodNotImplemented(ECDriverError): - pass - - -# liberasurecode backend init error -class ECBackendInitializationError(ECDriverError): - pass - - -# Specified backend instance is invalid/unavailable -class ECBackendInstanceNotAvailable(ECDriverError): - pass - - -# Specified backend instance is busy -class ECBackendInstanceInUse(ECDriverError): - pass - - -# Invalid parameter passed to a method -class ECInvalidParameter(ECDriverError): - pass - - -# Invalid or incompatible fragment metadata -class ECInvalidFragmentMetadata(ECDriverError): - pass - - -# Fragment checksum verification failed -class ECBadFragmentChecksum(ECDriverError): - pass - - -# Insufficient fragments specified for decode or reconstruct operation -class ECInsufficientFragments(ECDriverError): - pass - - -# Out of memory -class ECOutOfMemory(ECDriverError): - pass - - -# PyECLib helper for "available" EC types -ALL_EC_TYPES = [ - 'jerasure_rs_vand', - 'jerasure_rs_cauchy', - 'flat_xor_hd_3', - 'flat_xor_hd_4', - 'isa_l_rs_vand', - 'shss', - 'liberasurecode_rs_vand', - 'isa_l_rs_cauchy', - 'libphazr', -] - - -def _PyECLibValidECTypes(): - available_ec_types = [] - for _type in ALL_EC_TYPES: - if check_backend_available(_type): - available_ec_types.append(_type) - return available_ec_types - - -VALID_EC_TYPES = _PyECLibValidECTypes() - - -def _liberasurecode_version(): - version_int = get_liberasurecode_version() - version_hex_str = hex(version_int) - version_hex_str = version_hex_str.lstrip('0x') - major = str(int(version_hex_str[-6:-4])) - minor = str(int(version_hex_str[-4:-2])) - rev = str(int(version_hex_str[-2:])) - version_str = '.'.join([major, minor, rev]) - - # liberasurecode < 1.3.1 should be incompatible but - # just warn until packagers build the required version - # See https://bugs.launchpad.net/swift/+bug/1639691 in detail - required_version = ((1 << 16) + (3 << 8) + 1) - if version_int < required_version: - logger.warning( - 'DEPRECATED WARNING: your liberasurecode ' - '%s will be deprecated in the near future because of the issue ' - 'https://bugs.launchpad.net/swift/+bug/1639691; ' - 'Please upgrade to >=1.3.1 and rebuild pyeclib to suppress ' - 'this message' % version_str) - return version_str - -LIBERASURECODE_VERSION = _liberasurecode_version() diff --git a/pyeclib/enum.py b/pyeclib/enum.py deleted file mode 100644 index d0c858d..0000000 --- a/pyeclib/enum.py +++ /dev/null @@ -1,807 +0,0 @@ -# Original credit: enum34 project https://pypi.python.org/pypi/enum34 -# Changes for PyECLib (c) Tushar Gohad (tusharsg@gmail.com) -# -# Redistribution and use in source and binary forms, with or without -# modification, are permitted provided that the following conditions are met: -# -# Redistributions of source code must retain the above copyright notice, this -# list of conditions and the following disclaimer. -# -# Redistributions in binary form must reproduce the above copyright notice, -# this list of conditions and the following disclaimer in the documentation -# and/or other materials provided with the distribution. THIS SOFTWARE IS -# PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS -# OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES -# OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN -# NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY -# DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES -# (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; -# LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND -# ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT -# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF -# THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - -"""Python Enumerations""" - -import sys as _sys - -__all__ = ['Enum', 'IntEnum', 'unique'] - -pyver = float('%s.%s' % _sys.version_info[:2]) - -try: - any -except NameError: - def any(iterable): - for element in iterable: - if element: - return True - return False - -try: - from collections import OrderedDict -except ImportError: - OrderedDict = None - -try: - basestring -except NameError: - # In Python 2 basestring is the ancestor of both str and unicode - # in Python 3 it's just str, but was missing in 3.1 - basestring = str - - -class _RouteClassAttributeToGetattr(object): - """Route attribute access on a class to __getattr__. - - This is a descriptor, used to define attributes that act differently when - accessed through an instance and through a class. Instance access remains - normal, but access to an attribute through a class will be routed to the - class's __getattr__ method; this is done by raising AttributeError. - - """ - def __init__(self, fget=None): - self.fget = fget - - def __get__(self, instance, ownerclass=None): - if instance is None: - raise AttributeError() - return self.fget(instance) - - def __set__(self, instance, value): - raise AttributeError("can't set attribute") - - def __delete__(self, instance): - raise AttributeError("can't delete attribute") - - -def _is_descriptor(obj): - """Returns True if obj is a descriptor, False otherwise.""" - return (hasattr(obj, '__get__') or - hasattr(obj, '__set__') or - hasattr(obj, '__delete__')) - - -def _is_dunder(name): - """Returns True if a __dunder__ name, False otherwise.""" - return (name[:2] == name[-2:] == '__' and - name[2:3] != '_' and - name[-3:-2] != '_' and - len(name) > 4) - - -def _is_sunder(name): - """Returns True if a _sunder_ name, False otherwise.""" - return (name[0] == name[-1] == '_' and - name[1:2] != '_' and - name[-2:-1] != '_' and - len(name) > 2) - - -def _make_class_unpicklable(cls): - """Make the given class un-picklable.""" - def _break_on_call_reduce(self, protocol=None): - raise TypeError('%r cannot be pickled' % self) - cls.__reduce_ex__ = _break_on_call_reduce - cls.__module__ = '' - - -class _EnumDict(dict): - """Track enum member order and ensure member names are not reused. - - EnumMeta will use the names found in self._member_names as the - enumeration member names. - - """ - def __init__(self): - super(_EnumDict, self).__init__() - self._member_names = [] - - def __setitem__(self, key, value): - """Changes anything not dundered or not a descriptor. - - If a descriptor is added with the same name as an enum member, the name - is removed from _member_names (this may leave a hole in the numerical - sequence of values). - - If an enum member name is used twice, an error is raised; duplicate - values are not checked for. - - Single underscore (sunder) names are reserved. - - Note: in 3.x __order__ is simply discarded as a not necessary piece - leftover from 2.x - - """ - if pyver >= 3.0 and key == '__order__': - return - if _is_sunder(key): - raise ValueError('_names_ are reserved for future Enum use') - elif _is_dunder(key): - pass - elif key in self._member_names: - # descriptor overwriting an enum? - raise TypeError('Attempted to reuse key: %r' % key) - elif not _is_descriptor(value): - if key in self: - # enum overwriting a descriptor? - raise TypeError('Key already defined as: %r' % self[key]) - self._member_names.append(key) - super(_EnumDict, self).__setitem__(key, value) - - -# Dummy value for Enum as EnumMeta explicity checks for it, but of course until -# EnumMeta finishes running the first time the Enum class doesn't exist. This -# is also why there are checks in EnumMeta like `if Enum is not None` -Enum = None - - -class EnumMeta(type): - """Metaclass for Enum""" - @classmethod - def __prepare__(metacls, cls, bases): - return _EnumDict() - - def __new__(metacls, cls, bases, classdict): - # an Enum class is final once enumeration items have been defined; it - # cannot be mixed with other types (int, float, etc.) if it has an - # inherited __new__ unless a new __new__ is defined (or the resulting - # class will fail). - if type(classdict) is dict: - original_dict = classdict - classdict = _EnumDict() - for k, v in original_dict.items(): - classdict[k] = v - - member_type, first_enum = metacls._get_mixins_(bases) - __new__, save_new, use_args = metacls._find_new_( - classdict, member_type, first_enum) - # save enum items into separate mapping so they don't get baked into - # the new class - members = dict((k, classdict[k]) for k in classdict._member_names) - for name in classdict._member_names: - del classdict[name] - - # py2 support for definition order - __order__ = classdict.get('__order__') - if __order__ is None: - if pyver < 3.0: - __order__ = [name for (name, value) in - sorted(members.items(), key=lambda item: item[1])] - else: - __order__ = classdict._member_names - else: - del classdict['__order__'] - if pyver < 3.0: - __order__ = __order__.replace(',', ' ').split() - aliases = [name for name in members if name not in __order__] - __order__ += aliases - - # check for illegal enum names (any others?) - invalid_names = set(members) & set(['mro']) - if invalid_names: - raise ValueError('Invalid enum member name(s): %s' % ( - ', '.join(invalid_names), )) - - # create our new Enum type - enum_class = super(EnumMeta, metacls).__new__(metacls, cls, bases, - classdict) - enum_class._member_names_ = [] # names in random order - if OrderedDict is not None: - enum_class._member_map_ = OrderedDict() - else: - enum_class._member_map_ = {} # name->value map - enum_class._member_type_ = member_type - - # Reverse value->name map for hashable values. - enum_class._value2member_map_ = {} - - # instantiate them, checking for duplicates as we go - # we instantiate first instead of checking for duplicates first in case - # a custom __new__ is doing something funky with the values -- such as - # auto-numbering ;) - if __new__ is None: - __new__ = enum_class.__new__ - for member_name in __order__: - value = members[member_name] - if not isinstance(value, tuple): - args = (value, ) - else: - args = value - if member_type is tuple: # special case for tuple enums - args = (args, ) # wrap it one more time - if not use_args or not args: - enum_member = __new__(enum_class) - if not hasattr(enum_member, '_value_'): - enum_member._value_ = value - else: - enum_member = __new__(enum_class, *args) - if not hasattr(enum_member, '_value_'): - enum_member._value_ = member_type(*args) - value = enum_member._value_ - enum_member._name_ = member_name - enum_member.__objclass__ = enum_class - enum_member.__init__(*args) - # If another member with the same value was already defined, the - # new member becomes an alias to the existing one. - for name, canonical_member in enum_class._member_map_.items(): - if canonical_member.value == enum_member._value_: - enum_member = canonical_member - break - else: - # Aliases don't appear in member names (only in __members__). - enum_class._member_names_.append(member_name) - enum_class._member_map_[member_name] = enum_member - try: - # This may fail if value is not hashable. We can't add the - # value to the map, and by-value lookups for this value will be - # linear. - enum_class._value2member_map_[value] = enum_member - except TypeError: - pass - - # If a custom type is mixed into the Enum, and it does not know how - # to pickle itself, pickle.dumps will succeed but pickle.loads will - # fail. Rather than have the error show up later and possibly far - # from the source, sabotage the pickle protocol for this class so - # that pickle.dumps also fails. - # - # However, if the new class implements its own __reduce_ex__, do not - # sabotage -- it's on them to make sure it works correctly. We use - # __reduce_ex__ instead of any of the others as it is preferred by - # pickle over __reduce__, and it handles all pickle protocols. - unpicklable = False - if '__reduce_ex__' not in classdict: - if member_type is not object: - methods = ('__getnewargs_ex__', '__getnewargs__', - '__reduce_ex__', '__reduce__') - if not any(m in member_type.__dict__ for m in methods): - _make_class_unpicklable(enum_class) - unpicklable = True - - # double check that repr and friends are not the mixin's or various - # things break (such as pickle) - for name in ('__repr__', '__str__', '__format__', '__reduce_ex__'): - class_method = getattr(enum_class, name) - # obj_method = getattr(member_type, name, None) - enum_method = getattr(first_enum, name, None) - if name not in classdict and class_method is not enum_method: - if name == '__reduce_ex__' and unpicklable: - continue - setattr(enum_class, name, enum_method) - - # method resolution and int's are not playing nice - # Python's less than 2.6 use __cmp__ - - if pyver < 2.6: - - if issubclass(enum_class, int): - setattr(enum_class, '__cmp__', getattr(int, '__cmp__')) - - elif pyver < 3.0: - - if issubclass(enum_class, int): - for method in ( - '__le__', - '__lt__', - '__gt__', - '__ge__', - '__eq__', - '__ne__', - '__hash__',): - setattr(enum_class, method, getattr(int, method)) - - # replace any other __new__ with our own (as long as Enum is not None, - # anyway) -- again, this is to support pickle - if Enum is not None: - # if the user defined their own __new__, save it before it gets - # clobbered in case they subclass later - if save_new: - setattr(enum_class, '__member_new__', - enum_class.__dict__['__new__']) - setattr(enum_class, '__new__', Enum.__dict__['__new__']) - return enum_class - - def __call__(cls, value, names=None, module=None, type=None): - """Either returns an existing member, or creates a new enum class. - - This method is used both when an enum class is given a value to match - to an enumeration member (i.e. Color(3)) and for the functional API - (i.e. Color = Enum('Color', names='red green blue')). - - When used for the functional API: `module`, if set, will be stored in - the new class' __module__ attribute; `type`, if set, will be mixed in - as the first base class. - - Note: if `module` is not set this routine will attempt to discover the - calling module by walking the frame stack; if this is unsuccessful - the resulting class will not be pickleable. - - """ - if names is None: # simple value lookup - return cls.__new__(cls, value) - # otherwise, functional API: we're creating a new Enum type - return cls._create_(value, names, module=module, type=type) - - def __contains__(cls, member): - return isinstance(member, cls) and member.name in cls._member_map_ - - def __delattr__(cls, attr): - # nicer error message when someone tries to delete an attribute - # (see issue19025). - if attr in cls._member_map_: - raise AttributeError( - "%s: cannot delete Enum member." % cls.__name__) - super(EnumMeta, cls).__delattr__(attr) - - def __dir__(self): - return (['__class__', '__doc__', '__members__', '__module__'] + - self._member_names_) - - @property - def __members__(cls): - """Returns a mapping of member name->value. - - This mapping lists all enum members, including aliases. Note that this - is a copy of the internal mapping. - - """ - return cls._member_map_.copy() - - def __getattr__(cls, name): - """Return the enum member matching `name` - - We use __getattr__ instead of descriptors or inserting into the enum - class' __dict__ in order to support `name` and `value` being both - properties for enum members (which live in the class' __dict__) and - enum members themselves. - - """ - if _is_dunder(name): - raise AttributeError(name) - try: - return cls._member_map_[name] - except KeyError: - raise AttributeError(name) - - def __getitem__(cls, name): - return cls._member_map_[name] - - def __iter__(cls): - return (cls._member_map_[name] - for name in cls._member_names_) - - def __reversed__(cls): - return (cls._member_map_[name] - for name in reversed(cls._member_names_)) - - def __len__(cls): - return len(cls._member_names_) - - def __repr__(cls): - return "" % cls.__name__ - - def __setattr__(cls, name, value): - """Block attempts to reassign Enum members. - - A simple assignment to the class namespace only changes one of the - several possible ways to get an Enum member from the Enum class, - resulting in an inconsistent Enumeration. - - """ - member_map = cls.__dict__.get('_member_map_', {}) - if name in member_map: - raise AttributeError('Cannot reassign members.') - super(EnumMeta, cls).__setattr__(name, value) - - def _create_(cls, class_name, names=None, module=None, type=None): - """Convenience method to create a new Enum class. - - `names` can be: - - * A string containing member names, separated either with spaces or - commas. Values are auto-numbered from 1. - * An iterable of member names. Values are auto-numbered from 1. - * An iterable of (member name, value) pairs. - * A mapping of member name -> value. - - """ - metacls = cls.__class__ - if type is None: - bases = (cls, ) - else: - bases = (type, cls) - classdict = metacls.__prepare__(class_name, bases) - __order__ = [] - - # special processing needed for names? - if isinstance(names, basestring): - names = names.replace(',', ' ').split() - if isinstance(names, (tuple, list)) and isinstance(names[0], - basestring): - names = [(e, i + 1) for (i, e) in enumerate(names)] - - # Here, names is either an iterable of (name, value) or a mapping. - for item in names: - if isinstance(item, basestring): - member_name, member_value = item, names[item] - else: - member_name, member_value = item - classdict[member_name] = member_value - __order__.append(member_name) - # only set __order__ in classdict if name/value was not from a mapping - if not isinstance(item, basestring): - classdict['__order__'] = ' '.join(__order__) - enum_class = metacls.__new__(metacls, class_name, bases, classdict) - - # TODO: replace the frame hack if a blessed way to know the calling - # module is ever developed - if module is None: - try: - module = _sys._getframe(2).f_globals['__name__'] - except (AttributeError, ValueError): - pass - if module is None: - _make_class_unpicklable(enum_class) - else: - enum_class.__module__ = module - - return enum_class - - @staticmethod - def _get_mixins_(bases): - """Returns the type for creating enum members, and the first inherited - enum class. - - bases: the tuple of bases that was given to __new__ - - """ - if not bases or Enum is None: - return object, Enum - - # double check that we are not subclassing a class with existing - # enumeration members; while we're at it, see if any other data - # type has been mixed in so we can use the correct __new__ - member_type = first_enum = None - for base in bases: - if (base is not Enum and - issubclass(base, Enum) and - base._member_names_): - raise TypeError("Cannot extend enumerations") - # base is now the last base in bases - if not issubclass(base, Enum): - raise TypeError("new enumerations must be created as " - "`ClassName([mixin_type,] enum_type)`") - - # get correct mix-in type (either mix-in type of Enum subclass, or - # first base if last base is Enum) - if not issubclass(bases[0], Enum): - member_type = bases[0] # first data type - first_enum = bases[-1] # enum type - else: - for base in bases[0].__mro__: - # most common: (IntEnum, int, Enum, object) - # possible: (, , - # , , - # ) - if issubclass(base, Enum): - if first_enum is None: - first_enum = base - else: - if member_type is None: - member_type = base - - return member_type, first_enum - - if pyver < 3.0: - @staticmethod - def _find_new_(classdict, member_type, first_enum): - """Returns the __new__ to be used for creating the enum members. - - classdict: the class dictionary given to __new__ - member_type: the data type whose __new__ will be used by default - first_enum: enumeration to check for an overriding __new__ - - """ - # now find the correct __new__, checking to see of one was defined - # by the user; also check earlier enum classes in case a __new__ - # was saved as __member_new__ - __new__ = classdict.get('__new__', None) - if __new__: - return None, True, True # __new__, save_new, use_args - - N__new__ = getattr(None, '__new__') - O__new__ = getattr(object, '__new__') - if Enum is None: - E__new__ = N__new__ - else: - E__new__ = Enum.__dict__['__new__'] - # check all possibles for __member_new__ before falling back to - # __new__ - for method in ('__member_new__', '__new__'): - for possible in (member_type, first_enum): - try: - target = possible.__dict__[method] - except (AttributeError, KeyError): - target = getattr(possible, method, None) - if target not in [ - None, - N__new__, - O__new__, - E__new__, ]: - if method == '__member_new__': - classdict['__new__'] = target - return None, False, True - if isinstance(target, staticmethod): - target = target.__get__(member_type) - __new__ = target - break - if __new__ is not None: - break - else: - __new__ = object.__new__ - - # if a non-object.__new__ is used then whatever value/tuple was - # assigned to the enum member name will be passed to __new__ and - # to the new enum member's __init__ - if __new__ is object.__new__: - use_args = False - else: - use_args = True - - return __new__, False, use_args - else: - @staticmethod - def _find_new_(classdict, member_type, first_enum): - """Returns the __new__ to be used for creating the enum members. - - classdict: the class dictionary given to __new__ - member_type: the data type whose __new__ will be used by default - first_enum: enumeration to check for an overriding __new__ - - """ - # now find the correct __new__, checking to see of one was defined - # by the user; also check earlier enum classes in case a __new__ - # was saved as __member_new__ - __new__ = classdict.get('__new__', None) - - # should __new__ be saved as __member_new__ later? - save_new = __new__ is not None - - if __new__ is None: - # check all possibles for __member_new__ before falling back to - # __new__ - for method in ('__member_new__', '__new__'): - for possible in (member_type, first_enum): - target = getattr(possible, method, None) - if target not in ( - None, - None.__new__, - object.__new__, - Enum.__new__, ): - __new__ = target - break - if __new__ is not None: - break - else: - __new__ = object.__new__ - - # if a non-object.__new__ is used then whatever value/tuple was - # assigned to the enum member name will be passed to __new__ and - # to the new enum member's __init__ - if __new__ is object.__new__: - use_args = False - else: - use_args = True - - return __new__, save_new, use_args - - -######################################################## -# In order to support Python 2 and 3 with a single -# codebase we have to create the Enum methods separately -# and then use the `type(name, bases, dict)` method to -# create the class. -######################################################## -temp_enum_dict = {} -temp_enum_dict['__doc__'] = "Generic enumeration.\n\n "\ - "Derive from this class to define new enumerations.\n\n" - - -def __new__(cls, value): - # all enum instances are actually created during class construction - # without calling this method; this method is called by the metaclass' - # __call__ (i.e. Color(3) ), and by pickle - if type(value) is cls: - # For lookups like Color(Color.red) - value = value.value - # return value - # by-value search for a matching enum member - # see if it's in the reverse mapping (for hashable values) - try: - if value in cls._value2member_map_: - return cls._value2member_map_[value] - except TypeError: - # not there, now do long search -- O(n) behavior - for member in cls._member_map_.values(): - if member.value == value: - return member - raise ValueError("%s is not a valid %s" % (value, cls.__name__)) -temp_enum_dict['__new__'] = __new__ -del __new__ - - -def __repr__(self): - return "<%s.%s: %r>" % ( - self.__class__.__name__, self._name_, self._value_) -temp_enum_dict['__repr__'] = __repr__ -del __repr__ - - -def __str__(self): - return "%s.%s" % (self.__class__.__name__, self._name_) -temp_enum_dict['__str__'] = __str__ -del __str__ - - -def __dir__(self): - added_behavior = [m for m in self.__class__.__dict__ if m[0] != '_'] - return (['__class__', '__doc__', '__module__', 'name', 'value'] + - added_behavior) -temp_enum_dict['__dir__'] = __dir__ -del __dir__ - - -def __format__(self, format_spec): - # mixed-in Enums should use the mixed-in type's __format__, otherwise - # we can get strange results with the Enum name showing up instead of - # the value - - # pure Enum branch - if self._member_type_ is object: - cls = str - val = str(self) - # mix-in branch - else: - cls = self._member_type_ - val = self.value - return cls.__format__(val, format_spec) -temp_enum_dict['__format__'] = __format__ -del __format__ - - -#################################### -# Python's less than 2.6 use __cmp__ - -if pyver < 2.6: - - def __cmp__(self, other): - if type(other) is self.__class__: - if self is other: - return 0 - return -1 - return NotImplemented - raise TypeError("unorderable types: %s() and %s()" % - (self.__class__.__name__, other.__class__.__name__)) - temp_enum_dict['__cmp__'] = __cmp__ - del __cmp__ - -else: - - def __le__(self, other): - raise TypeError("unorderable types: %s() <= %s()" % - (self.__class__.__name__, other.__class__.__name__)) - temp_enum_dict['__le__'] = __le__ - del __le__ - - def __lt__(self, other): - raise TypeError("unorderable types: %s() < %s()" % - (self.__class__.__name__, other.__class__.__name__)) - temp_enum_dict['__lt__'] = __lt__ - del __lt__ - - def __ge__(self, other): - raise TypeError("unorderable types: %s() >= %s()" % - (self.__class__.__name__, other.__class__.__name__)) - temp_enum_dict['__ge__'] = __ge__ - del __ge__ - - def __gt__(self, other): - raise TypeError("unorderable types: %s() > %s()" % - (self.__class__.__name__, other.__class__.__name__)) - temp_enum_dict['__gt__'] = __gt__ - del __gt__ - - -def __eq__(self, other): - if type(other) is self.__class__: - return self is other - return NotImplemented -temp_enum_dict['__eq__'] = __eq__ -del __eq__ - - -def __ne__(self, other): - if type(other) is self.__class__: - return self is not other - return NotImplemented -temp_enum_dict['__ne__'] = __ne__ -del __ne__ - - -def __hash__(self): - return hash(self._name_) -temp_enum_dict['__hash__'] = __hash__ -del __hash__ - - -def __reduce_ex__(self, proto): - return self.__class__, (self._value_, ) -temp_enum_dict['__reduce_ex__'] = __reduce_ex__ -del __reduce_ex__ - -# _RouteClassAttributeToGetattr is used to provide access to the `name` -# and `value` properties of enum members while keeping some measure of -# protection from modification, while still allowing for an enumeration -# to have members named `name` and `value`. This works because enumeration -# members are not set directly on the enum class -- __getattr__ is -# used to look them up. - - -@_RouteClassAttributeToGetattr -def name(self): - return self._name_ -temp_enum_dict['name'] = name -del name - - -@_RouteClassAttributeToGetattr -def value(self): - return self._value_ -temp_enum_dict['value'] = value -del value - -Enum = EnumMeta('Enum', (object, ), temp_enum_dict) -del temp_enum_dict - -# Enum has now been created -########################### - - -class IntEnum(int, Enum): - """Enum where members are also (and must be) ints""" - - -def unique(enumeration): - """Class decorator that ensures only unique members exist - in an enumeration.""" - duplicates = [] - for name, member in enumeration.__members__.items(): - if name != member.name: - duplicates.append((name, member.name)) - if duplicates: - duplicate_names = ', '.join( - ["%s -> %s" % (alias, name) for (alias, name) in duplicates]) - raise ValueError('duplicate names found in %r: %s' % - (enumeration, duplicate_names)) - return enumeration diff --git a/pyeclib/utils.py b/pyeclib/utils.py deleted file mode 100644 index 3b7e535..0000000 --- a/pyeclib/utils.py +++ /dev/null @@ -1,75 +0,0 @@ -# Copyright (c) 2013, 2014, Kevin Greenan (kmgreen2@gmail.com) -# Copyright (c) 2014, Tushar Gohad (tusharsg@gmail.com) -# All rights reserved. -# -# Redistribution and use in source and binary forms, with or without -# modification, are permitted provided that the following conditions are met: -# -# Redistributions of source code must retain the above copyright notice, this -# list of conditions and the following disclaimer. -# -# Redistributions in binary form must reproduce the above copyright notice, -# this list of conditions and the following disclaimer in the documentation -# and/or other materials provided with the distribution. THIS SOFTWARE IS -# PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS -# OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES -# OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN -# NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY -# DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES -# (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; -# LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND -# ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT -# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF -# THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - -import sys -import traceback - - -def positive_int_value(param): - # Returns value as a positive int or raises ValueError otherwise - try: - value = int(param) - assert value > 0 - except (TypeError, ValueError, AssertionError): - # Handle: TypeError for 'None', ValueError for non-int strings - # and AssertionError for values <= 0 - raise ValueError('Must be an integer > 0, not "%s".' % param) - return value - - -def import_class(import_str): - """ - Returns a class from a string that specifies a module and/or class - - :param import_str: import path, e.g. 'httplib.HTTPConnection' - :returns imported object - :raises: ImportedError if the class does not exist or the path is invalid - """ - (mod_str, separator, class_str) = import_str.rpartition('.') - try: - __import__(mod_str) - return getattr(sys.modules[mod_str], class_str) - except (ValueError, AttributeError): - raise ImportError('Class %s cannot be found (%)' % - (class_str, - traceback.format_exception(*sys.exc_info()))) - - -def create_instance(import_str, *args, **kwargs): - """ - Returns instance of class which imported by import path. - - :param import_str: import path of class - :param \*args: indexed arguments for new instance - :param \*\*kwargs: keyword arguments for new instance - :returns: instance of imported class which instantiated with - arguments *args and **kwargs - """ - try: - object_class = import_class(import_str) - except Exception: - raise - instance = object_class(*args, **kwargs) - - return instance diff --git a/setup.py b/setup.py deleted file mode 100644 index 66e3470..0000000 --- a/setup.py +++ /dev/null @@ -1,196 +0,0 @@ -#!/usr/bin/env python - -# Copyright (c) 2013-2015, Kevin Greenan (kmgreen2@gmail.com) -# Copyright (c) 2013-2015, Tushar Gohad (tusharsg@gmail.com) -# All rights reserved. -# -# Redistribution and use in source and binary forms, with or without -# modification, are permitted provided that the following conditions are met: -# -# Redistributions of source code must retain the above copyright notice, this -# list of conditions and the following disclaimer. -# -# Redistributions in binary form must reproduce the above copyright notice, -# this list of conditions and the following disclaimer in the documentation -# and/or other materials provided with the distribution. THIS SOFTWARE IS -# PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS -# OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES -# OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN -# NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY -# DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES -# (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; -# LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND -# ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT -# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF -# THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - -import os -import platform -import sys - -from ctypes import * -from ctypes.util import * -from distutils.command.build import build as _build -from distutils.command.clean import clean as _clean -from distutils.sysconfig import EXEC_PREFIX as _exec_prefix -from distutils.sysconfig import get_python_lib -from distutils.sysconfig import get_python_inc - -try: - from setuptools import setup -except ImportError: - from distribute_setup import use_setuptools - use_setuptools() - from setuptools import setup - -from setuptools import Extension -from setuptools.command.install import install as _install - -platform_str = platform.platform() -default_python_incdir = get_python_inc() - - -# this is to be used only for library existence/version checks, -# not for rpath handling -def _find_library(name): - target_lib = find_library(name) - if platform_str.find("Darwin") > -1: - target_lib = os.path.abspath(target_lib) - if os.path.islink(target_lib): - p = os.readlink(target_lib) - if os.path.isabs(p): - target_lib = p - else: - target_lib = os.path.join(os.path.dirname(target_lib), p) - # return absolute path to the library if found - return target_lib - - -class build(_build): - - def check_liberasure(self): - library_basename = "liberasurecode" - library_version = "1" - library = library_basename + "-" + library_version - library_url = "https://github.com/openstack/liberasurecode" - - found_path = _find_library("erasurecode") - if found_path: - if found_path.endswith(library_version) or \ - found_path.find(library_version + ".") > -1: - # call 1.x.x the only compatible version for now - return - - if platform_str.find("Darwin") > -1: - liberasure_file = \ - library_basename + "." + library_version + ".dylib" - else: - liberasure_file = \ - library_basename + ".so." + library_version - - print("**************************************************************") - print("*** ") - print("*** Can not locate %s" % (liberasure_file)) - print("*** ") - print("*** Install - ") - print("*** Manual: %s" % library_url) - print("*** Fedora/Red Hat variants: liberasurecode-devel") - print("*** Debian/Ubuntu variants: liberasurecode-dev") - print("*** ") - print("**************************************************************") - - sys.exit(-1) - - def run(self): - self.check_liberasure() - _build.run(self) - - -class clean(_clean): - - def run(self): - _clean.run(self) - - -class install(_install): - - def run(self): - - install_cmd = self.distribution.get_command_obj('install') - install_lib = self.distribution.get_command_obj('install_lib') - for cmd in (install_lib, install_cmd): - cmd.ensure_finalized() - - # ensure that the paths are absolute so we don't get lost - opts = {'exec_prefix': install_cmd.exec_prefix, - 'root': install_cmd.root} - for optname, value in list(opts.items()): - if value is not None: - opts[optname] = os.path.abspath(value) - - installroot = install_lib.install_dir - _install.run(self) - - # Another Mac-ism... If the libraries are installed - # in a strange place, DYLD_LIRBARY_PATH needs to be - # updated. - if platform_str.find("Darwin") > -1: - ldpath_str = "DYLD_LIBRARY_PATH" - else: - ldpath_str = "LD_LIBRARY_PATH" - print("***************************************************") - print("** ") - print("** PyECLib libraries have been installed to: ") - print("** %s" % installroot) - print("** ") - print("** Any user using this library must update: ") - print("** %s" % ldpath_str) - print("** ") - print("** Run 'ldconfig' or place this line: ") - print("** export %s=%s" % (ldpath_str, "%s" - % installroot)) - print("** ") - print("** into .bashrc, .profile, or the appropriate shell") - print("** start-up script! Also look at ldconfig(8) man ") - print("** page for a more static LD configuration ") - print("** ") - print("***************************************************") - - -module = Extension('pyeclib_c', - define_macros=[('MAJOR VERSION', '1'), - ('MINOR VERSION', '5')], - include_dirs=[default_python_incdir, - 'src/c/pyeclib_c', - '/usr/include', - '/usr/include/liberasurecode', - '%s/include/liberasurecode' % sys.prefix, - '%s/include' % sys.prefix], - libraries=['erasurecode'], - # The extra arguments are for debugging - # extra_compile_args=['-g', '-O0'], - sources=['src/c/pyeclib_c/pyeclib_c.c']) - -setup(name='pyeclib', - version='1.5.0', - author='Kevin Greenan', - author_email='kmgreen2@gmail.com', - maintainer='Kevin Greenan and Tushar Gohad', - maintainer_email='kmgreen2@gmail.com, tusharsg@gmail.com', - url='http://git.openstack.org/cgit/openstack/pyeclib/', - bugtrack_url='https://bugs.launchpad.net/pyeclib', - description='This library provides a simple Python interface for \ - implementing erasure codes. To obtain the best possible \ - performance, the underlying erasure code algorithms are \ - written in C.', - platforms='Linux', - license='BSD', - ext_modules=[module], - packages=['pyeclib'], - package_dir={'pyeclib': 'pyeclib'}, - cmdclass={'build': build, 'install': install, 'clean': clean}, - py_modules=['pyeclib.ec_iface', 'pyeclib.core'], - command_options={ - 'build_sphinx': { - 'build_dir': ('setup.py', 'doc/build')}}, - test_suite='test') diff --git a/src/c/pyeclib_c/capsulethunk.h b/src/c/pyeclib_c/capsulethunk.h deleted file mode 100644 index 6dca150..0000000 --- a/src/c/pyeclib_c/capsulethunk.h +++ /dev/null @@ -1,160 +0,0 @@ -/* - * Copyright (c) 2013-2014, Kevin Greenan (kmgreen2@gmail.com) - * Copyright (c) 2014, Tushar Gohad (tusharsg@gmail.com) - * All rights reserved. - * - * Redistribution and use in source and binary forms, with or without - * modification, are permitted provided that the following conditions are met: - * - * Redistributions of source code must retain the above copyright notice, this - * list of conditions and the following disclaimer. - * - * Redistributions in binary form must reproduce the above copyright notice, this - * list of conditions and the following disclaimer in the documentation and/or - * other materials provided with the distribution. THIS SOFTWARE IS PROVIDED BY - * THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED - * WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF - * MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO - * EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, - * INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, - * BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, - * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF - * LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE - * OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF - * ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - */ - -#ifndef __CAPSULETHUNK_H -#define __CAPSULETHUNK_H - -#if ( (PY_VERSION_HEX < 0x02070000) \ - || ((PY_VERSION_HEX >= 0x03000000) \ - && (PY_VERSION_HEX < 0x03010000)) ) - -#define __PyCapsule_GetField(capsule, field, default_value) \ - ( PyCapsule_CheckExact(capsule) \ - ? (((PyCObject *)capsule)->field) \ - : (default_value) \ - ) \ - -#define __PyCapsule_SetField(capsule, field, value) \ - ( PyCapsule_CheckExact(capsule) \ - ? (((PyCObject *)capsule)->field = value), 1 \ - : 0 \ - ) \ - - -#define PyCapsule_Type PyCObject_Type - -#define PyCapsule_CheckExact(capsule) (PyCObject_Check(capsule)) -#define PyCapsule_IsValid(capsule, name) (PyCObject_Check(capsule)) - - -#define PyCapsule_New(pointer, name, destructor) \ - (PyCObject_FromVoidPtr(pointer, destructor)) - - -#define PyCapsule_GetPointer(capsule, name) \ - (PyCObject_AsVoidPtr(capsule)) - -/* Don't call PyCObject_SetPointer here, it fails if there's a destructor */ -#define PyCapsule_SetPointer(capsule, pointer) \ - __PyCapsule_SetField(capsule, cobject, pointer) - - -#define PyCapsule_GetDestructor(capsule) \ - __PyCapsule_GetField(capsule, destructor) - -#define PyCapsule_SetDestructor(capsule, dtor) \ - __PyCapsule_SetField(capsule, destructor, dtor) - - -/* - * Sorry, there's simply no place - * to store a Capsule "name" in a CObject. - */ -#define PyCapsule_GetName(capsule) NULL - -static int -PyCapsule_SetName(PyObject *capsule, const char *unused) -{ - unused = unused; - PyErr_SetString(PyExc_NotImplementedError, - "can't use PyCapsule_SetName with CObjects"); - return 1; -} - - - -#define PyCapsule_GetContext(capsule) \ - __PyCapsule_GetField(capsule, descr) - -#define PyCapsule_SetContext(capsule, context) \ - __PyCapsule_SetField(capsule, descr, context) - - -static void * -PyCapsule_Import(const char *name, int no_block) -{ - PyObject *object = NULL; - void *return_value = NULL; - char *trace; - size_t name_length = (strlen(name) + 1) * sizeof(char); - char *name_dup = (char *)PyMem_MALLOC(name_length); - - if (!name_dup) { - return NULL; - } - - memcpy(name_dup, name, name_length); - - trace = name_dup; - while (trace) { - char *dot = strchr(trace, '.'); - if (dot) { - *dot++ = '\0'; - } - - if (object == NULL) { - if (no_block) { - object = PyImport_ImportModuleNoBlock(trace); - } else { - object = PyImport_ImportModule(trace); - if (!object) { - PyErr_Format(PyExc_ImportError, - "PyCapsule_Import could not " - "import module \"%s\"", trace); - } - } - } else { - PyObject *object2 = PyObject_GetAttrString(object, trace); - Py_DECREF(object); - object = object2; - } - if (!object) { - goto EXIT; - } - - trace = dot; - } - - if (PyCObject_Check(object)) { - PyCObject *cobject = (PyCObject *)object; - return_value = cobject->cobject; - } else { - PyErr_Format(PyExc_AttributeError, - "PyCapsule_Import \"%s\" is not valid", - name); - } - -EXIT: - Py_XDECREF(object); - if (name_dup) { - PyMem_FREE(name_dup); - } - return return_value; -} - -#endif /* #if PY_VERSION_HEX < 0x02070000 */ - -#endif /* __CAPSULETHUNK_H */ diff --git a/src/c/pyeclib_c/pyeclib_c.c b/src/c/pyeclib_c/pyeclib_c.c deleted file mode 100644 index 4e2646c..0000000 --- a/src/c/pyeclib_c/pyeclib_c.c +++ /dev/null @@ -1,1243 +0,0 @@ -/* - * Copyright (c) 2013-2014, Kevin Greenan (kmgreen2@gmail.com) - * Copyright (c) 2014, Tushar Gohad (tusharsg@gmail.com) - * All rights reserved. - * - * Redistribution and use in source and binary forms, with or without - * modification, are permitted provided that the following conditions are met: - * - * Redistributions of source code must retain the above copyright notice, this - * list of conditions and the following disclaimer. - * - * Redistributions in binary form must reproduce the above copyright notice, this - * list of conditions and the following disclaimer in the documentation and/or - * other materials provided with the distribution. THIS SOFTWARE IS PROVIDED BY - * THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED - * WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF - * MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO - * EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, - * INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, - * BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, - * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF - * LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE - * OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF - * ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - */ - -#include -#include -#include -#include -#include -#include - -#if ( LIBERASURECODE_VERSION < _VERSION(1,1,0) ) -#include -#endif - -/* Compat layer for python <= 2.6 */ -#include "capsulethunk.h" - -#include - -/* Python 3 compatibility macros */ -#if PY_MAJOR_VERSION >= 3 - #define PYTHON3 - #define MOD_ERROR_VAL NULL - #define MOD_SUCCESS_VAL(val) val - #define MOD_INIT(name) PyMODINIT_FUNC PyInit_##name(void) - #define MOD_DEF(ob, name, doc, methods) \ - static struct PyModuleDef moduledef = { \ - PyModuleDef_HEAD_INIT, name, doc, -1, methods, }; \ - ob = PyModule_Create(&moduledef); - #define PY_BUILDVALUE_OBJ_LEN(obj, objlen) \ - Py_BuildValue("y#", obj, (Py_ssize_t)objlen) - #define PyInt_FromLong PyLong_FromLong - #define PyString_FromString PyUnicode_FromString - #define ENCODE_ARGS "Oy#" - #define GET_METADATA_ARGS "Oy#i" -#else - #define MOD_ERROR_VAL - #define MOD_SUCCESS_VAL(val) - #define MOD_INIT(name) void init##name(void) - #define MOD_DEF(ob, name, doc, methods) \ - ob = Py_InitModule3(name, methods, doc); - #define PY_BUILDVALUE_OBJ_LEN(obj, objlen) \ - Py_BuildValue("s#", obj, (Py_ssize_t)objlen) - #define ENCODE_ARGS "Os#" - #define GET_METADATA_ARGS "Os#i" -#endif - - -typedef struct pyeclib_byte_range { - uint64_t offset; - uint64_t length; -} pyeclib_byte_range_t; - -/** - * Prototypes for Python/C API methods - */ -static PyObject * pyeclib_c_init(PyObject *self, PyObject *args); -static void pyeclib_c_destructor(PyObject *obj); -static PyObject * pyeclib_c_get_segment_info(PyObject *self, PyObject *args); -static PyObject * pyeclib_c_encode(PyObject *self, PyObject *args); -static PyObject * pyeclib_c_reconstruct(PyObject *self, PyObject *args); -static PyObject * pyeclib_c_decode(PyObject *self, PyObject *args); -static PyObject * pyeclib_c_get_metadata(PyObject *self, PyObject *args); -static PyObject * pyeclib_c_check_metadata(PyObject *self, PyObject *args); -static PyObject * pyeclib_c_liberasurecode_version(PyObject *self, PyObject *args); - -static PyObject *import_class(const char *module, const char *cls) -{ - PyObject *s = PyImport_ImportModule(module); - if (s == NULL) { - return NULL; - } - return (PyObject *) PyObject_GetAttrString(s, cls); -} - -/** - * Allocate a buffer of a specific size and set its' contents - * to the specified value. - * - * @param size integer size in bytes of buffer to allocate - * @param value - * @return pointer to start of allocated buffer or NULL on error - */ -void * alloc_and_set_buffer(int size, int value) { - void * buf = NULL; /* buffer to allocate and return */ - - /* Allocate and zero the buffer, or set the appropriate error */ - buf = malloc((size_t) size); - if (buf) { - buf = memset(buf, value, (size_t) size); - } - return buf; -} - -/** - * Allocate a zero-ed buffer of a specific size. - * - * @param size integer size in bytes of buffer to allocate - * @return pointer to start of allocated buffer or NULL on error - */ -void * alloc_zeroed_buffer(int size) -{ - return alloc_and_set_buffer(size, 0); -} - -/** - * Deallocate memory buffer if it's not NULL. This methods returns NULL so - * that you can free and reset a buffer using a single line as follows: - * - * my_ptr = check_and_free_buffer(my_ptr); - * - * @return NULL - */ -void * check_and_free_buffer(void * buf) -{ - if (buf) - free(buf); - return NULL; -} - -void pyeclib_c_seterr(int ret, const char * prefix) { - char *err_class; - char *err_msg; - char err[255]; - - // If any error was previously set, we're explicitly ignoring it - // to raise something new - PyErr_Clear(); - - switch (ret) { - case -EBACKENDNOTAVAIL: - err_class = "ECBackendInstanceNotAvailable"; - err_msg = "Backend instance not found"; - break; - case -EINSUFFFRAGS: - err_class = "ECInsufficientFragments"; - err_msg = "Insufficient number of fragments"; - break; - case -EBACKENDNOTSUPP: - err_class = "ECBackendNotSupported"; - err_msg = "Backend not supported"; - break; - case -EINVALIDPARAMS: - err_class = "ECInvalidParameter"; - err_msg = "Invalid arguments"; - break; - case -EBADCHKSUM: - err_class = "ECBadFragmentChecksum"; - err_msg = "Fragment integrity check failed"; - break; - case -EBADHEADER: - err_class = "ECInvalidFragmentMetadata"; - err_msg = "Fragment integrity check failed"; - break; - case -ENOMEM: - err_class = "ECOutOfMemory"; - err_msg = "Out of memory"; - break; - default: - err_class = "ECDriverError"; - err_msg = "Unknown error"; - break; - } - PyObject *eo = import_class("pyeclib.ec_iface", err_class); - if (eo != NULL) { - snprintf(err, 255, - "%s ERROR: %s. Please inspect syslog for liberasurecode error report.", - prefix, err_msg); - PyErr_SetString(eo, err); - } -} - -static int stderr_fd; -static fpos_t stderr_fpos; -#ifndef _PATH_DEVNULL -#define _PATH_DEVNULL "/dev/null" -#endif - -static void redirect_stderr(void) -{ - fflush(stderr); - fgetpos(stderr, &stderr_fpos); - stderr_fd = dup(fileno(stderr)); - freopen(_PATH_DEVNULL, "w", stderr); -} - -static void restore_stderr(void) -{ - fflush(stderr); - dup2(stderr_fd, fileno(stderr)); - close(stderr_fd); - clearerr(stderr); - fsetpos(stderr, &stderr_fpos); /* for C9X */ -} - -/** - * Constructor method for creating a new pyeclib object using the given parameters. - * - * @param k integer number of data elements - * @param m integer number of checksum elements - * @param hd hamming distance - * @param backend_id erasure coding backend - * @param use_inline_chksum type of inline fragment header checksum - * @param use_algsig_chksum use algorithmic signature for fragment header checksum - * @param validate only validate backend and params, close handle immediately - * @return pointer to PyObject or NULL on error - */ -static PyObject * -pyeclib_c_init(PyObject *self, PyObject *args) -{ - pyeclib_t *pyeclib_handle = NULL; - PyObject *pyeclib_obj_handle = NULL; - int k, m, hd = 0, validate = 0; - int use_inline_chksum = 0, use_algsig_chksum = 0; - const ec_backend_id_t backend_id; - - /* Obtain and validate the method parameters */ - if (!PyArg_ParseTuple(args, "iii|iiiii", - &k, &m, &backend_id, &hd, &use_inline_chksum, - &use_algsig_chksum, &validate)) { - pyeclib_c_seterr(-EINVALIDPARAMS, "pyeclib_c_init"); - return NULL; - } - - /* Allocate and initialize the pyeclib object */ - pyeclib_handle = (pyeclib_t *) alloc_zeroed_buffer(sizeof(pyeclib_t)); - if (NULL == pyeclib_handle) { - pyeclib_c_seterr(-ENOMEM, "pyeclib_c_init"); - goto cleanup; - } - - pyeclib_handle->ec_args.k = k; - pyeclib_handle->ec_args.m = m; - pyeclib_handle->ec_args.hd = hd; - pyeclib_handle->ec_args.ct = use_inline_chksum ? CHKSUM_CRC32 : CHKSUM_NONE; - - if (validate) - redirect_stderr(); - - pyeclib_handle->ec_desc = liberasurecode_instance_create(backend_id, &(pyeclib_handle->ec_args)); - if (pyeclib_handle->ec_desc <= 0) { - /* liberasurecode returns status in ec_desc as one of the error codes - * (LIBERASURECODE_ERROR_CODES) defined in erasurecode.h */ - pyeclib_c_seterr(pyeclib_handle->ec_desc, "pyeclib_c_init"); - goto cleanup; - } - - /* Prepare the python object to return */ -#ifdef Py_CAPSULE_H - pyeclib_obj_handle = PyCapsule_New(pyeclib_handle, PYECC_HANDLE_NAME, - pyeclib_c_destructor); -#else - pyeclib_obj_handle = PyCObject_FromVoidPtrAndDesc( - pyeclib_handle, (void *) PYECC_HANDLE_NAME, - pyeclib_c_destructor); -#endif /* Py_CAPSULE_H */ - - /* Clean up the allocated memory on error, or update the ref count */ - if (pyeclib_obj_handle == NULL) { - pyeclib_c_seterr(-EINVALIDPARAMS, "pyeclib_c_init"); - goto cleanup; - } else { - Py_INCREF(pyeclib_obj_handle); - } - -exit: - if (validate) - restore_stderr(); - return pyeclib_obj_handle; - -cleanup: - check_and_free_buffer(pyeclib_handle); - pyeclib_obj_handle = NULL; - goto exit; -} - - -/** - * Destructor method for cleaning up pyeclib object. - */ -static void -pyeclib_c_destructor(PyObject *obj) -{ - pyeclib_t *pyeclib_handle = NULL; /* pyeclib object to destroy */ - - if (!PyCapsule_CheckExact(obj)) { - pyeclib_c_seterr(-1, "pyeclib_c_destructor"); - return; - } - - pyeclib_handle = (pyeclib_t*)PyCapsule_GetPointer(obj, PYECC_HANDLE_NAME); - if (pyeclib_handle == NULL) { - pyeclib_c_seterr(-1, "pyeclib_c_destructor"); - } else { - check_and_free_buffer(pyeclib_handle); - } - return; -} - - -/** - * This function takes data length and a segment size and returns an object - * containing: - * - * segment_size: size of the payload to give to encode() - * last_segment_size: size of the payload to give to encode() - * fragment_size: the fragment size returned by encode() - * last_fragment_size: the fragment size returned by encode() - * num_segments: number of segments - * - * This allows the caller to prepare requests when segmenting a data stream - * to be EC'd. - * - * Since the data length will rarely be aligned to the segment size, the last - * segment will be a different size than the others. - * - * There are restrictions on the length given to encode(), so calling this - * before encode is highly recommended when segmenting a data stream. - * - * Minimum segment size depends on the underlying EC type (if it is less - * than this, then the last segment will be slightly larger than the others, - * otherwise it will be smaller). - * - * @param pyeclib_obj_handle - * @param data_len integer length of data in bytes - * @param segment_size integer length of segment in bytes - * @return a python dictionary with segment information - * - */ -static PyObject * -pyeclib_c_get_segment_info(PyObject *self, PyObject *args) -{ - PyObject *pyeclib_obj_handle = NULL; - pyeclib_t *pyeclib_handle = NULL; - PyObject *ret_dict = NULL; /* python dictionary to return */ - - int data_len; /* data length from user in bytes */ - int segment_size, last_segment_size; /* segment sizes in bytes */ - int num_segments; /* total number of segments */ - int fragment_size, last_fragment_size; /* fragment sizes in bytes */ - int min_segment_size; /* EC algorithm's min. size (B) */ - - /* Obtain and validate the method parameters */ - if (!PyArg_ParseTuple(args, "Oii", &pyeclib_obj_handle, &data_len, &segment_size)) { - pyeclib_c_seterr(-EINVALIDPARAMS, "pyeclib_c_get_segment_info"); - return NULL; - } - pyeclib_handle = (pyeclib_t*)PyCapsule_GetPointer(pyeclib_obj_handle, PYECC_HANDLE_NAME); - if (pyeclib_handle == NULL) { - pyeclib_c_seterr(-EINVALIDPARAMS, "pyeclib_c_get_segment_info"); - return NULL; - } - - /* The minimum segment size depends on the EC algorithm */ - min_segment_size = liberasurecode_get_minimum_encode_size(pyeclib_handle->ec_desc); - if (min_segment_size < 0) { - pyeclib_c_seterr(-EINVALIDPARAMS, "pyeclib_c_get_segment_info"); - return NULL; - } - - /* Get the number of segments */ - num_segments = (int)ceill((double)data_len / segment_size); - - /* - * If there are two segments and the last is smaller than the - * minimum size, then combine into a single segment - */ - if (num_segments == 2 && data_len < (segment_size + min_segment_size)) { - num_segments--; - } - - /* Compute the fragment size from the segment size */ - if (num_segments == 1) { - /* - * There is one fragment, or two fragments, where the second is - * smaller than the min_segment_size, just create one segment - */ - - /* - * This will retrieve fragment_size calculated by liberasurecode with - * specified backend. - */ - - fragment_size = liberasurecode_get_fragment_size(pyeclib_handle->ec_desc, data_len); - if (fragment_size < 0) { - pyeclib_c_seterr(-EINVALIDPARAMS, "pyeclib_c_get_segment_info"); - return NULL; - } - - /* Segment size is the user-provided segment size */ - segment_size = data_len; - last_fragment_size = fragment_size; - last_segment_size = segment_size; - } else { - /* - * There will be at least 2 segments, where the last exceeeds - * the minimum segment size. - */ - - fragment_size = liberasurecode_get_fragment_size(pyeclib_handle->ec_desc, segment_size); - if (fragment_size < 0) { - pyeclib_c_seterr(-EINVALIDPARAMS, "pyeclib_c_get_segment_info"); - return NULL; - } - - last_segment_size = data_len - (segment_size * (num_segments - 1)); - - /* - * The last segment is lower than the minimum size, so combine it - * with the previous fragment - */ - if (last_segment_size < min_segment_size) { - // assert(num_segments > 2)? - - /* Add current "last segment" to second to last segment */ - num_segments--; - last_segment_size = last_segment_size + segment_size; - } - - last_fragment_size = liberasurecode_get_fragment_size(pyeclib_handle->ec_desc, last_segment_size); - if (fragment_size < 0) { - pyeclib_c_seterr(-EINVALIDPARAMS, "pyeclib_c_get_segment_info"); - return NULL; - } - } - - /* Add header to fragment sizes */ - last_fragment_size += sizeof(fragment_header_t); - fragment_size += sizeof(fragment_header_t); - - /* Create and return the python dictionary of segment info */ - ret_dict = Py_BuildValue( - "{s:i, s:i, s:i, s:i, s:i}", - "segment_size", segment_size, - "last_segment_size", last_segment_size, - "fragment_size", fragment_size, - "last_fragment_size", last_fragment_size, - "num_segments", num_segments); - if (NULL == ret_dict) { - goto error; - } - -exit: - return ret_dict; - -error: - // To prevent unexpected call, this is placed after return call - pyeclib_c_seterr(-ENOMEM, "pyeclib_c_get_segment_info"); - Py_XDECREF(ret_dict); - ret_dict = NULL; - goto exit; - -} - - -/** - * Erasure encode a data buffer. - * - * @param pyeclib_obj_handle - * @param data to encode - * @return python list of encoded data and parity elements - */ -static PyObject * -pyeclib_c_encode(PyObject *self, PyObject *args) -{ - PyObject *pyeclib_obj_handle = NULL; - pyeclib_t *pyeclib_handle= NULL; - char **encoded_data = NULL; /* array of k data buffers */ - char **encoded_parity = NULL; /* array of m parity buffers */ - PyObject *list_of_strips = NULL; /* list of encoded strips to return */ - char *data; /* param, data buffer to encode */ - int data_len; /* param, length of data buffer */ - uint64_t fragment_len; /* length, in bytes of the fragments */ - int i; /* a counter */ - int ret = 0; - - /* Assume binary data (force "byte array" input) */ - if (!PyArg_ParseTuple(args, ENCODE_ARGS, &pyeclib_obj_handle, &data, &data_len)) { - pyeclib_c_seterr(-EINVALIDPARAMS, "pyeclib_c_encode"); - return NULL; - } - pyeclib_handle = (pyeclib_t*)PyCapsule_GetPointer(pyeclib_obj_handle, PYECC_HANDLE_NAME); - if (pyeclib_handle == NULL) { - pyeclib_c_seterr(-EINVALIDPARAMS, "pyeclib_c_encode"); - return NULL; - } - - ret = liberasurecode_encode(pyeclib_handle->ec_desc, data, data_len, &encoded_data, &encoded_parity, &fragment_len); - if (ret < 0) { - pyeclib_c_seterr(ret, "pyeclib_c_encode"); - return NULL; - } - - /* Create the python list of fragments to return */ - list_of_strips = PyList_New(pyeclib_handle->ec_args.k + pyeclib_handle->ec_args.m); - if (NULL == list_of_strips) { - pyeclib_c_seterr(-ENOMEM, "pyeclib_c_encode"); - return NULL; - } - - /* Add data fragments to the python list to return */ - for (i = 0; i < pyeclib_handle->ec_args.k; i++) { - PyList_SET_ITEM(list_of_strips, i, - PY_BUILDVALUE_OBJ_LEN(encoded_data[i], fragment_len)); - } - - /* Add parity fragments to the python list to return */ - for (i = 0; i < pyeclib_handle->ec_args.m; i++) { - PyList_SET_ITEM(list_of_strips, pyeclib_handle->ec_args.k + i, - PY_BUILDVALUE_OBJ_LEN(encoded_parity[i], fragment_len)); - } - - liberasurecode_encode_cleanup(pyeclib_handle->ec_desc, encoded_data, encoded_parity); - - return list_of_strips; -} - - -/** - * Return a list of lists with valid rebuild indexes given an EC algorithm - * and a list of missing indexes. - * - * @param pyeclib_obj_handle - * @param reconstruct_list list of missing fragments to reconstruct - * @param exclude_list list of fragments to exclude from reconstruction - * @return a list of lists of indexes to rebuild data from - */ -static PyObject * -pyeclib_c_get_required_fragments(PyObject *self, PyObject *args) -{ - PyObject *pyeclib_obj_handle = NULL; - pyeclib_t *pyeclib_handle = NULL; - PyObject *reconstruct_list = NULL; /* list of missing fragments to reconstruct */ - PyObject *exclude_list = NULL; /* list of fragments to exclude from reconstruction */ - PyObject *fragment_idx_list = NULL; /* list of req'd indexes to return */ - int *c_reconstruct_list = NULL; /* c-array of missing indexes */ - int *c_exclude_list = NULL; /* c-array of missing indexes */ - int num_missing; /* size of passed in missing list */ - int num_exclude; /* size of passed in exclude list */ - int i = 0; /* counters */ - int k, m; /* EC algorithm parameters */ - int *fragments_needed = NULL; /* indexes of xor code fragments */ - int ret; /* return value for xor code */ - - /* Obtain and validate the method parameters */ - if (!PyArg_ParseTuple(args, "OOO", &pyeclib_obj_handle, &reconstruct_list, &exclude_list)) { - pyeclib_c_seterr(-EINVALIDPARAMS, "pyeclib_c_get_required_fragments"); - return NULL; - } - pyeclib_handle = (pyeclib_t*)PyCapsule_GetPointer(pyeclib_obj_handle, PYECC_HANDLE_NAME); - if (pyeclib_handle == NULL) { - pyeclib_c_seterr(-EINVALIDPARAMS, "pyeclib_c_get_required_fragments"); - return NULL; - } - k = pyeclib_handle->ec_args.k; - m = pyeclib_handle->ec_args.m; - - /* Generate -1 terminated c-array and bitmap of missing indexes */ - num_missing = (int) PyList_Size(reconstruct_list); - c_reconstruct_list = (int *) alloc_zeroed_buffer((num_missing + 1) * sizeof(int)); - if (NULL == c_reconstruct_list) { - pyeclib_c_seterr(-ENOMEM, "pyeclib_c_get_required_fragments"); - return NULL; - } - c_reconstruct_list[num_missing] = -1; - for (i = 0; i < num_missing; i++) { - PyObject *obj_idx = PyList_GetItem(reconstruct_list, i); - long idx = PyLong_AsLong(obj_idx); - c_reconstruct_list[i] = (int) idx; - } - - num_exclude = (int) PyList_Size(exclude_list); - c_exclude_list = (int *) alloc_zeroed_buffer((num_exclude + 1) * sizeof(int)); - if (NULL == c_exclude_list) { - pyeclib_c_seterr(-ENOMEM, "pyeclib_c_get_required_fragments"); - goto exit; - } - c_exclude_list[num_exclude] = -1; - for (i = 0; i < num_exclude; i++) { - PyObject *obj_idx = PyList_GetItem(exclude_list, i); - long idx = PyLong_AsLong(obj_idx); - c_exclude_list[i] = (int) idx; - } - - fragments_needed = alloc_zeroed_buffer(sizeof(int) * (k + m)); - if (NULL == fragments_needed) { - pyeclib_c_seterr(-ENOMEM, "pyeclib_c_get_required_fragments"); - goto exit; - } - - ret = liberasurecode_fragments_needed(pyeclib_handle->ec_desc, c_reconstruct_list, - c_exclude_list, fragments_needed); - if (ret < 0) { - pyeclib_c_seterr(ret, "pyeclib_c_get_required_fragments"); - goto exit; - } - - /* Post-process into a Python list */ - fragment_idx_list = PyList_New(0); - if (NULL == fragment_idx_list) { - pyeclib_c_seterr(-ENOMEM, "pyeclib_c_get_required_fragments"); - goto exit; - } - - for (i = 0; fragments_needed[i] > -1; i++) { - PyList_Append(fragment_idx_list, Py_BuildValue("i", fragments_needed[i])); - } - -exit: - check_and_free_buffer(c_reconstruct_list); - check_and_free_buffer(c_exclude_list); - check_and_free_buffer(fragments_needed); - - return fragment_idx_list; -} - - -/** - * Reconstruct a missing fragment from the the remaining fragments. - * - * TODO: If we are reconstructing a parity element, ensure that all of the - * data elements are available! - * - * @param pyeclib_obj_handle - * @param data_list k length list of data elements - * @param parity_list m length list of parity elements - * @param missing_idx_list list of the indexes of missing elements - * @param destination_idx index of fragment to reconstruct - * @param fragment_size size in bytes of the fragments - * @return reconstructed destination fragment or NULL on error - */ -static PyObject * -pyeclib_c_reconstruct(PyObject *self, PyObject *args) -{ - PyObject *pyeclib_obj_handle = NULL; - pyeclib_t *pyeclib_handle = NULL; - PyObject *fragments = NULL; /* param, list of fragments */ - PyObject *reconstructed = NULL; /* reconstructed object to return */ - char * c_reconstructed = NULL; /* C string of reconstructed fragment */ - int fragment_len; /* param, size in bytes of fragment */ - int num_fragments; /* number of fragments passed in */ - char **c_fragments = NULL; /* C array containing the fragment payloads */ - int destination_idx; /* param, index to reconstruct */ - int ret; /* decode matrix creation return val */ - int i = 0; /* a counter */ - - /* Obtain and validate the method parameters */ - if (!PyArg_ParseTuple(args, "OOii", &pyeclib_obj_handle, &fragments, - &fragment_len, &destination_idx)) { - pyeclib_c_seterr(-EINVALIDPARAMS, "pyeclib_c_reconstruct"); - return NULL; - } - pyeclib_handle = (pyeclib_t*)PyCapsule_GetPointer(pyeclib_obj_handle, PYECC_HANDLE_NAME); - if (pyeclib_handle == NULL) { - pyeclib_c_seterr(-EINVALIDPARAMS, "pyeclib_c_reconstruct"); - return NULL; - } - - /* Pre-processing Python data structures */ - if (!PyList_Check(fragments)) { - pyeclib_c_seterr(-EINVALIDPARAMS, "pyeclib_c_reconstruct"); - return NULL; - } - - num_fragments = PyList_Size(fragments); - - c_fragments = (char **) alloc_zeroed_buffer(sizeof(char *) * num_fragments); - if (NULL == c_fragments) { - pyeclib_c_seterr(-ENOMEM, "pyeclib_c_reconstruct"); - goto error; - } - - c_reconstructed = (char*) alloc_zeroed_buffer(sizeof(char) * fragment_len); - if (NULL == c_fragments) { - pyeclib_c_seterr(-ENOMEM, "pyeclib_c_reconstruct"); - goto error; - } - - /* Put the fragments into an array of C strings */ - for (i = 0; i < num_fragments; i++) { - PyObject *tmp_data = PyList_GetItem(fragments, i); - Py_ssize_t len = 0; - PyBytes_AsStringAndSize(tmp_data, &(c_fragments[i]), &len); - } - - ret = liberasurecode_reconstruct_fragment(pyeclib_handle->ec_desc, - c_fragments, - num_fragments, - fragment_len, - destination_idx, - c_reconstructed); - if (ret < 0) { - pyeclib_c_seterr(ret, "pyeclib_c_reconstruct"); - reconstructed = NULL; - } else { - reconstructed = PY_BUILDVALUE_OBJ_LEN(c_reconstructed, fragment_len); - } - - goto out; - -error: - reconstructed = NULL; - -out: - check_and_free_buffer(c_fragments); - check_and_free_buffer(c_reconstructed); - - return reconstructed; -} - -/** - * Decode a set of fragments into the original string - * - * @param pyeclib_obj_handle - * @param data_list k length list of data elements - * @param parity_list m length list of parity elements - * @param missing_idx_list list of the indexes of missing elements - * @param fragment_size size in bytes of the fragments - * @return list of fragments - */ -static PyObject * -pyeclib_c_decode(PyObject *self, PyObject *args) -{ - PyObject *pyeclib_obj_handle = NULL; - pyeclib_t *pyeclib_handle = NULL; - PyObject *fragments = NULL; /* param, list of missing indexes */ - PyObject *ret_payload = NULL; /* object to store original payload or ranges of payload */ - PyObject *ranges = NULL; /* a list of tuples that represent byte ranges */ - PyObject *metadata_checks_obj = NULL; /* boolean specifying if headers should be validated before decode */ - pyeclib_byte_range_t *c_ranges = NULL; /* the byte ranges */ - int num_ranges = 0; /* number of specified ranges */ - int fragment_len; /* param, size in bytes of fragment */ - char **c_fragments = NULL; /* k length array of data buffers */ - int num_fragments; /* param, number of fragments */ - char *c_orig_payload = NULL; /* buffer to store original payload in */ - uint64_t range_payload_size = 0; /* length of buffer used to store byte ranges */ - uint64_t orig_data_size = 0; /* data size in bytes ,from fragment hdr */ - int i = 0; /* counters */ - int force_metadata_checks = 0; /* validate the fragment headers before decoding */ - int ret = 0; - - /* Obtain and validate the method parameters */ - if (!PyArg_ParseTuple(args, "OOi|OO",&pyeclib_obj_handle, &fragments, - &fragment_len, &ranges, &metadata_checks_obj)) { - pyeclib_c_seterr(-EINVALIDPARAMS, "pyeclib_c_decode"); - return NULL; - } - - /* If Python filled in None, we may get a reference. If so, set the pointer to NULL */ - if (NULL != ranges && ranges == Py_None) { - ranges = NULL; - } - - /* Liberasurecode wants an integer, so convert the PyBool to an integer */ - if (NULL != metadata_checks_obj && PyObject_IsTrue(metadata_checks_obj)) { - force_metadata_checks = 1; - } - - pyeclib_handle = (pyeclib_t*)PyCapsule_GetPointer(pyeclib_obj_handle, PYECC_HANDLE_NAME); - if (pyeclib_handle == NULL) { - pyeclib_c_seterr(-EINVALIDPARAMS, "pyeclib_c_decode"); - return NULL; - } - if (!PyList_Check(fragments)) { - pyeclib_c_seterr(-EINVALIDPARAMS, "pyeclib_c_decode"); - return NULL; - } - - num_fragments = PyList_Size(fragments); - - if (ranges) { - num_ranges = PyList_Size(ranges); - } - - if (pyeclib_handle->ec_args.k > num_fragments) { - pyeclib_c_seterr(-EINSUFFFRAGS, "pyeclib_c_decode"); - return NULL; - } - - if (num_ranges > 0) { - c_ranges = (pyeclib_byte_range_t*)malloc(sizeof(pyeclib_byte_range_t) * num_ranges); - if (NULL == c_ranges) { - pyeclib_c_seterr(-ENOMEM, "pyeclib_c_decode"); - goto error; - } - for (i = 0; i < num_ranges; i++) { - PyObject *tuple = PyList_GetItem(ranges, i); - if (PyTuple_Size(tuple) == 2) { - PyObject *py_begin = PyTuple_GetItem(tuple, 0); - PyObject *py_end = PyTuple_GetItem(tuple, 1); - long begin, end; - - if (PyLong_Check(py_begin)) - begin = PyLong_AsLong(py_begin); -#ifndef PYTHON3 - else if (PyInt_Check(py_begin)) - begin = PyInt_AsLong(py_begin); -#endif - else { - pyeclib_c_seterr(-EINVALIDPARAMS, "pyeclib_c_decode invalid range"); - goto error; - } - if (PyLong_Check(py_end)) - end = PyLong_AsLong(py_end); -#ifndef PYTHON3 - else if (PyInt_Check(py_end)) - end = PyInt_AsLong(py_end); -#endif - else { - pyeclib_c_seterr(-EINVALIDPARAMS, "pyeclib_c_decode invalid range"); - goto error; - } - - c_ranges[i].offset = begin; - c_ranges[i].length = end - begin + 1; - - range_payload_size += c_ranges[i].length; - } else { - pyeclib_c_seterr(-EINVALIDPARAMS, "pyeclib_c_decode invalid range"); - goto error; - } - } - } - - c_fragments = (char **) alloc_zeroed_buffer(sizeof(char *) * num_fragments); - if (NULL == c_fragments) { - goto error; - } - - /* Put the fragments into an array of C strings */ - for (i = 0; i < num_fragments; i++) { - PyObject *tmp_data = PyList_GetItem(fragments, i); - Py_ssize_t len = 0; - PyBytes_AsStringAndSize(tmp_data, &(c_fragments[i]), &len); - } - - ret = liberasurecode_decode(pyeclib_handle->ec_desc, - c_fragments, - num_fragments, - fragment_len, - force_metadata_checks, - &c_orig_payload, - &orig_data_size); - - if (ret < 0) { - pyeclib_c_seterr(ret, "pyeclib_c_decode"); - goto error; - } - - if (num_ranges == 0) { - ret_payload = PY_BUILDVALUE_OBJ_LEN(c_orig_payload, orig_data_size); - } else { - ret_payload = PyList_New(num_ranges); - if (NULL == ret_payload) { - pyeclib_c_seterr(-ENOMEM, "pyeclib_c_decode"); - goto error; - } - range_payload_size = 0; - for (i = 0; i < num_ranges; i++) { - /* Check that range is within the original buffer */ - if (c_ranges[i].offset + c_ranges[i].length > orig_data_size) { - pyeclib_c_seterr(-EINVALIDPARAMS, "pyeclib_c_decode invalid range"); - goto error; - } - PyList_SET_ITEM(ret_payload, i, - PY_BUILDVALUE_OBJ_LEN(c_orig_payload + c_ranges[i].offset, c_ranges[i].length)); - } - } - - goto exit; - -error: - ret_payload = NULL; - -exit: - check_and_free_buffer(c_fragments); - check_and_free_buffer(c_ranges); - liberasurecode_decode_cleanup(pyeclib_handle->ec_desc, c_orig_payload); - - return ret_payload; -} - - -static const char* chksum_type_to_str(uint8_t chksum_type) -{ - const char *chksum_type_str = NULL; - switch (chksum_type) - { - case CHKSUM_NONE: - chksum_type_str = "none\0"; - break; - case CHKSUM_CRC32: - chksum_type_str = "crc32\0"; - break; - case CHKSUM_MD5: - chksum_type_str = "md5\0"; - break; - default: - chksum_type_str = "unknown\0"; - } - - return chksum_type_str; -} - -static int chksum_length(uint8_t chksum_type) -{ - int length = 0; - switch (chksum_type) - { - case CHKSUM_NONE: - // None - break; - case CHKSUM_CRC32: - // CRC 32 - length = 4; - break; - case CHKSUM_MD5: - // MD5 - length = 16; - break; - default: - length = 0; - } - return length; -} - -static const char* backend_id_to_str(uint8_t backend_id) -{ - const char *backend_id_str = NULL; - switch (backend_id) - { - case 0: - backend_id_str = "null\0"; - break; - case 1: - backend_id_str = "jerasure_rs_vand\0"; - break; - case 2: - backend_id_str = "jerasure_rs_cauchy\0"; - break; - case 3: - backend_id_str = "flat_xor_hd\0"; - break; - case 4: - backend_id_str = "isa_l_rs_vand\0"; - break; - case 6: - backend_id_str = "liberasurecode_rs_vand\0"; - break; - case 7: - backend_id_str = "isa_l_rs_cauchy\0"; - break; - case 8: - backend_id_str = "libphazr\0"; - break; - default: - backend_id_str = "unknown\0"; - } - - return backend_id_str; -} - -static char* -hex_encode_string(char *buf, uint32_t buf_len) -{ - char *hex_encoded_buf = (char*)alloc_zeroed_buffer((buf_len * 2) + 1); - char *hex_encoded_ptr = hex_encoded_buf; - int i; - - for (i = 0; i < buf_len; i++) { - hex_encoded_ptr += sprintf(hex_encoded_ptr, "%.2x", (unsigned char)buf[i]); - } - - hex_encoded_buf[buf_len * 2] = 0; - - return hex_encoded_buf; -} - -static PyObject* -fragment_metadata_to_dict(fragment_metadata_t *fragment_metadata) -{ - const char *chksum_type_str = chksum_type_to_str(fragment_metadata->chksum_type); - char *encoded_chksum = hex_encode_string((char*)fragment_metadata->chksum, - chksum_length(fragment_metadata->chksum_type)); - const char *backend_id_str = backend_id_to_str(fragment_metadata->backend_id); - PyObject* metadata_dict = Py_BuildValue( - "{s:k, s:k, s:K, s:s, s:s, s:B, s:s, s:k}", - "index", fragment_metadata->idx, - "size", fragment_metadata->size, - "orig_data_size", fragment_metadata->orig_data_size, - "chksum_type", chksum_type_str, - "chksum", encoded_chksum, - "chksum_mismatch", fragment_metadata->chksum_mismatch, - "backend_id", backend_id_str, - "backend_version", fragment_metadata->backend_version); - encoded_chksum = check_and_free_buffer(encoded_chksum); - if (metadata_dict == NULL) { - pyeclib_c_seterr(-ENOMEM, "fragment_metadata_to_dict"); - return NULL; - } - return metadata_dict; -} - -/** - * Obtain the metadata from a fragment. - * - * @param pyeclib_obj_handle - * @param data fragment from user to extract metadata from - * @param data_len size in bytes of the data fragment - * @return fragment metadata or NULL on error - */ -static PyObject * -pyeclib_c_get_metadata(PyObject *self, PyObject *args) -{ - PyObject *pyeclib_obj_handle = NULL; - pyeclib_t* pyeclib_handle = NULL; - char *fragment = NULL; /* param, fragment from caller */ - fragment_metadata_t c_fragment_metadata; /* structure to hold metadata */ - PyObject *fragment_metadata = NULL; /* metadata object to return */ - int fragment_len; /* fragment length */ - int formatted; /* format the metadata in a dict */ - int ret; - - /* Obtain and validate the method parameters */ - if (!PyArg_ParseTuple(args, GET_METADATA_ARGS, &pyeclib_obj_handle, &fragment, &fragment_len, &formatted)) { - pyeclib_c_seterr(-EINVALIDPARAMS, "pyeclib_c_get_metadata"); - return NULL; - } - pyeclib_handle = (pyeclib_t*)PyCapsule_GetPointer(pyeclib_obj_handle, PYECC_HANDLE_NAME); - if (pyeclib_handle == NULL) { - pyeclib_c_seterr(-EINVALIDPARAMS, "pyeclib_c_get_metadata"); - return NULL; - } - - ret = liberasurecode_get_fragment_metadata(fragment, &c_fragment_metadata); - - if (ret < 0) { - pyeclib_c_seterr(ret, "pyeclib_c_get_metadata"); - fragment_metadata = NULL; - } else { - if (formatted) { - fragment_metadata = fragment_metadata_to_dict(&c_fragment_metadata); - } else { - fragment_metadata = PY_BUILDVALUE_OBJ_LEN((char*)&c_fragment_metadata, - sizeof(fragment_metadata_t)); - } - } - - return fragment_metadata; -} - -/** - * Confirm the health of the fragment metadata. - * - * TODO: Return a list containing tuples (index, problem). An empty list means - * everything is OK. - * - * @param pyeclib_obj_handle - * @param fragment_metadata_list list of fragment metadata headers - * @return dictionary containing 'status', 'reason' and 'bad_fragments', depending - * on the status - * NULL if the metadata could not be checked by the liberasurecode - */ -static PyObject* -pyeclib_c_check_metadata(PyObject *self, PyObject *args) -{ - PyObject *pyeclib_obj_handle = NULL; - pyeclib_t *pyeclib_handle = NULL; - PyObject *fragment_metadata_list = NULL; /* param, fragment metadata */ - fragment_metadata_t *c_fragment_metadata = NULL; /* metadata buffer for a single fragment */ - char **c_fragment_metadata_list = NULL; /* c version of metadata */ - int num_fragments; /* k + m from EC algorithm */ - int k, m; /* EC algorithm params */ - int size; /* size for buf allocation */ - int ret = -1; /* c return value */ - PyObject *ret_obj = NULL; /* python long to return */ - int i = 0; /* a counter */ - - - /* Obtain and validate the method parameters */ - if (!PyArg_ParseTuple(args, "OO", &pyeclib_obj_handle, &fragment_metadata_list)) { - pyeclib_c_seterr(-EINVALIDPARAMS, "pyeclib_c_encode"); - return NULL; - } - pyeclib_handle = (pyeclib_t*)PyCapsule_GetPointer(pyeclib_obj_handle, PYECC_HANDLE_NAME); - if (pyeclib_handle == NULL) { - pyeclib_c_seterr(-EINVALIDPARAMS, "pyeclib_c_encode"); - return NULL; - } - k = pyeclib_handle->ec_args.k; - m = pyeclib_handle->ec_args.m; - num_fragments = k + m; - if (num_fragments != PyList_Size(fragment_metadata_list)) { - pyeclib_c_seterr(-EINVALIDPARAMS, "pyeclib_c_encode"); - return NULL; - } - - /* Allocate space for fragment signatures */ - size = sizeof(char * ) * num_fragments; - c_fragment_metadata_list = (char **) alloc_zeroed_buffer(size); - if (NULL == c_fragment_metadata_list) { - pyeclib_c_seterr(-ENOMEM, "pyeclib_c_encode"); - goto error; - } - - /* Populate the metadata into a C array */ - for (i = 0; i < num_fragments; i++) { - PyObject *tmp_data = PyList_GetItem(fragment_metadata_list, i); - Py_ssize_t len = 0; - PyBytes_AsStringAndSize(tmp_data, &(c_fragment_metadata_list[i]), &len); - } - - ret = liberasurecode_verify_stripe_metadata(pyeclib_handle->ec_desc, c_fragment_metadata_list, - num_fragments); - - ret_obj = PyDict_New(); - if (ret == 0) { - PyDict_SetItemString(ret_obj, "status", PyLong_FromLong((long)0)); - } else if (ret == -EINVALIDPARAMS) { - PyDict_SetItemString(ret_obj, "status", PyLong_FromLong((long)-EINVALIDPARAMS)); - PyDict_SetItemString(ret_obj, "reason", PyString_FromString("Invalid arguments")); - goto error; - } else if (ret == -EBADCHKSUM) { - PyDict_SetItemString(ret_obj, "status", PyLong_FromLong((long)-EINVALIDPARAMS)); - PyDict_SetItemString(ret_obj, "reason", PyString_FromString("Bad checksum")); - PyObject *bad_chksums = PyList_New(0); - for (i = 0; i < num_fragments; i++) { - c_fragment_metadata = (fragment_metadata_t*)c_fragment_metadata_list[i]; - if (c_fragment_metadata->chksum_mismatch == 1) { - PyList_Append(bad_chksums, PyLong_FromLong((long)c_fragment_metadata->idx)); - } - } - PyDict_SetItemString(ret_obj, "bad_fragments", bad_chksums); - } - - goto exit; - -error: - ret_obj = NULL; - -exit: - free(c_fragment_metadata_list); - - return ret_obj; -} - -static PyObject* -pyeclib_c_check_backend_available(PyObject *self, PyObject *args) -{ - const ec_backend_id_t backend_id; - - if (!PyArg_ParseTuple(args, "i", &backend_id)) { - pyeclib_c_seterr(-EINVALIDPARAMS, "pyeclib_c_check_backend_available"); - return NULL; - } - - if (liberasurecode_backend_available(backend_id)) { - Py_RETURN_TRUE; - } - - Py_RETURN_FALSE; -} - -static PyObject* -pyeclib_c_liberasurecode_version(PyObject *self, PyObject *args) { - void *hLib; - char *err; - uint32_t (*hGetVersion)(void); - - dlerror(); - hLib = dlopen("liberasurecode.so", RTLD_LAZY); - /* It's important that we clear the last error before calling dysym */ - err = dlerror(); - if (err) { - /* This should never actually get hit; since we're using various - symbols already, liberasurecode.so should *already* be loaded. */ - return PyInt_FromLong(LIBERASURECODE_VERSION); - } - - hGetVersion = dlsym(hLib, "liberasurecode_get_version"); - err = dlerror(); - if (err) { - /* This is the important bit. Old version, doesn't have get_version - support; fall back to old behavior. */ - dlclose(hLib); - return PyInt_FromLong(LIBERASURECODE_VERSION); - } - - uint32_t version = (*hGetVersion)(); - dlclose(hLib); - return Py_BuildValue("k", version); -} - -static PyMethodDef PyECLibMethods[] = { - {"init", pyeclib_c_init, METH_VARARGS, "Initialize a new erasure encoder/decoder"}, - {"encode", pyeclib_c_encode, METH_VARARGS, "Create parity using source data"}, - {"decode", pyeclib_c_decode, METH_VARARGS, "Recover all lost data/parity"}, - {"reconstruct", pyeclib_c_reconstruct, METH_VARARGS, "Recover selective data/parity"}, - {"get_required_fragments", pyeclib_c_get_required_fragments, METH_VARARGS, "Return the fragments required to reconstruct a set of missing fragments"}, - {"get_segment_info", pyeclib_c_get_segment_info, METH_VARARGS, "Return segment and fragment size information needed when encoding a segmented stream"}, - {"get_metadata", pyeclib_c_get_metadata, METH_VARARGS, "Get the integrity checking metadata for a fragment"}, - {"check_metadata", pyeclib_c_check_metadata, METH_VARARGS, "Check the integrity checking metadata for a set of fragments"}, - {"get_liberasurecode_version", pyeclib_c_liberasurecode_version, METH_NOARGS, "Get libersaurecode version in use"}, -#if ( LIBERASURECODE_VERSION >= _VERSION(1,2,0) ) - {"check_backend_available", pyeclib_c_check_backend_available, METH_VARARGS, "Check if a backend is available"}, -#endif - {NULL, NULL, 0, NULL} /* Sentinel */ -}; - -MOD_INIT(pyeclib_c) -{ - PyObject *m; - - MOD_DEF(m, "pyeclib_c", NULL, PyECLibMethods); - - if (m == NULL) - return MOD_ERROR_VAL; - - return MOD_SUCCESS_VAL(m); -} diff --git a/src/c/pyeclib_c/pyeclib_c.h b/src/c/pyeclib_c/pyeclib_c.h deleted file mode 100644 index 4b7b3db..0000000 --- a/src/c/pyeclib_c/pyeclib_c.h +++ /dev/null @@ -1,43 +0,0 @@ -/* - * Copyright (c) 2013-2014, Kevin Greenan (kmgreen2@gmail.com) - * Copyright (c) 2014, Tushar Gohad (tusharsg@gmail.com) - * All rights reserved. - * - * Redistribution and use in source and binary forms, with or without - * modification, are permitted provided that the following conditions are met: - * - * Redistributions of source code must retain the above copyright notice, this - * list of conditions and the following disclaimer. - * - * Redistributions in binary form must reproduce the above copyright notice, this - * list of conditions and the following disclaimer in the documentation and/or - * other materials provided with the distribution. THIS SOFTWARE IS PROVIDED BY - * THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED - * WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF - * MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO - * EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, - * INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, - * BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, - * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF - * LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE - * OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF - * ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - */ - -#ifndef __PYEC_LIB_C_H_ -#define __PYEC_LIB_C_H_ - -/* For exact-width integer types */ -#include - -typedef struct pyeclib_s -{ - int ec_desc; - struct ec_args ec_args; -} pyeclib_t; - - -#define PYECC_HANDLE_NAME "pyeclib_handle" - -#endif - diff --git a/test-requirements.txt b/test-requirements.txt deleted file mode 100644 index 6d929e8..0000000 --- a/test-requirements.txt +++ /dev/null @@ -1,8 +0,0 @@ -# The order of packages is significant, because pip processes them in the order -# of appearance. Changing the order has an impact on the overall integration -# process, which may cause wedges in the gate later. - -coverage -nose -six>=1.9.0 -sphinx>=1.1.2,!=1.2.0,!=1.3b1,<1.3 # BSD diff --git a/test/__init__.py b/test/__init__.py deleted file mode 100644 index e69de29..0000000 diff --git a/test/ec_pyeclib_file_test.sh b/test/ec_pyeclib_file_test.sh deleted file mode 100755 index 446ad62..0000000 --- a/test/ec_pyeclib_file_test.sh +++ /dev/null @@ -1,94 +0,0 @@ -#!/bin/bash - -# Copyright (c) 2013, Kevin Greenan (kmgreen2@gmail.com) -# All rights reserved. -# -# Redistribution and use in source and binary forms, with or without -# modification, are permitted provided that the following conditions are met: -# -# Redistributions of source code must retain the above copyright notice, this -# list of conditions and the following disclaimer. -# -# Redistributions in binary form must reproduce the above copyright notice, this -# list of conditions and the following disclaimer in the documentation and/or -# other materials provided with the distribution. THIS SOFTWARE IS PROVIDED BY -# THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED -# WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF -# MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO -# EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, -# INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, -# BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, -# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF -# LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE -# OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF -# ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -# -FILE_DIR=./test_files -DECODED_DIR=./decoded_files -FRAGMENT_DIR=./test_fragments -FILES=`echo ${FILE_DIR}` -TOOLS_DIR=../tools - -if [ ! -d ${DECODED_DIR} ]; then - mkdir ${DECODED_DIR} -fi - -if [ ! -d ${FRAGMENT_DIR} ]; then - mkdir ${FRAGMENT_DIR} -fi - -TYPES=" -jerasure_rs_vand -jerasure_rs_cauchy -flat_xor_hd_3 -flat_xor_hd_4 -isa_l_rs_vand -liberasurecode_rs_vand -" -NUM_DATAS="10 11 12" -RS_NUM_PARITIES="2 3 4" -XOR_NUM_PARITIES="6" - -# Weird stuff happens when we try to rebuild and -# are beyond the FT of the code... Make it fail -# gracefully - -for TYPE in ${TYPES}; do - for NUM_DATA in ${NUM_DATAS}; do - rm ${DECODED_DIR}/* - rm ${FRAGMENT_DIR}/* - NUM_PARITIES=${RS_NUM_PARITIES} - if [[ ${TYPE} == "flat_xor_hd"* ]]; then - NUM_PARITIES=${XOR_NUM_PARITIES} - fi - for NUM_PARITY in ${NUM_PARITIES}; do - let NUM_TOTAL=$(( NUM_DATA + NUM_PARITY)) - FAULT_TOL=${NUM_PARITY} - if [[ ${TYPE} == "flat_xor_hd"* ]]; then - FAULT_TOL="2" - fi - for file in `cd ${FILES}; echo *; cd ..`; do - python ${TOOLS_DIR}/pyeclib_encode.py ${NUM_DATA} ${NUM_PARITY} ${TYPE} ${FILE_DIR} ${file} ${FRAGMENT_DIR} - done - - for file in `cd ${FILES}; echo *; cd ..`; do - fragments=( `echo ${FRAGMENT_DIR}/${file}.*` ) - let i=0 - while (( $i < ${FAULT_TOL} )); do - index=$(( RANDOM % NUM_TOTAL )) - fragments[${index}]="" - let i=$i+1 - done - python ${TOOLS_DIR}/pyeclib_decode.py ${NUM_DATA} ${NUM_PARITY} ${TYPE} ${fragments[*]} ${DECODED_DIR}/${file} - diff ${FILE_DIR}/${file} ${DECODED_DIR}/${file}.decoded - if [[ $? != 0 ]]; then - echo "${FILE_DIR}/${file} != ${DECODED_DIR}/${file}.decoded" - exit 1 - fi - done - done - done -done - -rm ${DECODED_DIR}/* -rm ${FRAGMENT_DIR}/* diff --git a/test/run_core_tests_manual.py b/test/run_core_tests_manual.py deleted file mode 100644 index e19b024..0000000 --- a/test/run_core_tests_manual.py +++ /dev/null @@ -1,92 +0,0 @@ -# Copyright (c) 2013, Kevin Greenan (kmgreen2@gmail.com) -# All rights reserved. -# -# Redistribution and use in source and binary forms, with or without -# modification, are permitted provided that the following conditions are met: -# -# Redistributions of source code must retain the above copyright notice, this -# list of conditions and the following disclaimer. -# -# Redistributions in binary form must reproduce the above copyright notice, -# this list of conditions and the following disclaimer in the documentation -# and/or other materials provided with the distribution. THIS SOFTWARE IS -# PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS -# OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES -# OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN -# NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY -# DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES -# (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; -# LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND -# ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT -# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF -# THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - -import os -import sys - -run_under_valgrind = False -test_cmd_prefix = "" -log_filename_prefix = "" - - -class CoreTests(): - - def __init__(self, *args): - self.pyeclib_core_test = "test_pyeclib_c.py" - self.pyeclib_iface_test = "test_pyeclib_api.py" - - def setUp(self): - # Determine which directory we're in - dirs = os.getcwd().split('/') - if dirs[-1] == 'test': - self.pyeclib_test_dir = "." - else: - self.pyeclib_test_dir = "./test" - - # Create the array of tests to run - self.py_test_dirs = [ - (self.pyeclib_test_dir, self.pyeclib_core_test), - (self.pyeclib_test_dir, self.pyeclib_iface_test) - ] - - def tearDown(self): - pass - - def invoke_core_tests(self): - cur_dir = os.getcwd() - print("\n") - for (dir, test) in self.py_test_dirs: - sys.stdout.write("Running test %s ... " % test) - sys.stdout.flush() - os.chdir(dir) - if os.path.isfile(test): - pythonpath = "PYTHONPATH=%s:%s" % \ - (cur_dir, os.path.dirname(cur_dir)) - ret = os.system( - "%s %s python %s >%s/%s.%s.out 2>&1" % - (pythonpath, test_cmd_prefix, test, cur_dir, - log_filename_prefix, test)) - - assert(0 == ret) - os.system("rm -f *.pyc") - os.chdir(cur_dir) - print('ok') - else: - print('failed') - - -# Invoke this script as "python test_core_valgrind.py" -# for the "valgrind" variant -# (test_core_valgrind.py is a symlink to test_core.py) -if __name__ == "__main__": - if '_valgrind' in sys.argv[0]: - if (0 != os.system("which valgrind")): - print("You don't appear to have 'valgrind' installed") - sys.exit(-1) - run_under_valgrind = True - test_cmd_prefix = "valgrind --leak-check=full " - log_filename_prefix = "valgrind" - coretests = CoreTests() - coretests.setUp() - coretests.invoke_core_tests() - coretests.tearDown() diff --git a/test/run_core_tests_manual_valgrind.py b/test/run_core_tests_manual_valgrind.py deleted file mode 120000 index f409632..0000000 --- a/test/run_core_tests_manual_valgrind.py +++ /dev/null @@ -1 +0,0 @@ -run_core_tests_manual.py \ No newline at end of file diff --git a/test/test_files/ames-msst06.pdf b/test/test_files/ames-msst06.pdf deleted file mode 100644 index 9ddbfa9..0000000 Binary files a/test/test_files/ames-msst06.pdf and /dev/null differ diff --git a/test/test_files/bhatnagar-hotemnets08.pdf b/test/test_files/bhatnagar-hotemnets08.pdf deleted file mode 100644 index c65247a..0000000 Binary files a/test/test_files/bhatnagar-hotemnets08.pdf and /dev/null differ diff --git a/test/test_files/greenan-dsn08.pdf b/test/test_files/greenan-dsn08.pdf deleted file mode 100644 index 85c6d9f..0000000 Binary files a/test/test_files/greenan-dsn08.pdf and /dev/null differ diff --git a/test/test_files/greenan-emsoft06.pdf b/test/test_files/greenan-emsoft06.pdf deleted file mode 100644 index 168a7ff..0000000 Binary files a/test/test_files/greenan-emsoft06.pdf and /dev/null differ diff --git a/test/test_files/greenan-hotdep07.pdf b/test/test_files/greenan-hotdep07.pdf deleted file mode 100644 index 1100a1d..0000000 Binary files a/test/test_files/greenan-hotdep07.pdf and /dev/null differ diff --git a/test/test_files/greenan-hotdep08.pdf b/test/test_files/greenan-hotdep08.pdf deleted file mode 100644 index 6a8f31f..0000000 Binary files a/test/test_files/greenan-hotdep08.pdf and /dev/null differ diff --git a/test/test_files/greenan-hotdep09.pdf b/test/test_files/greenan-hotdep09.pdf deleted file mode 100644 index 59fa23b..0000000 Binary files a/test/test_files/greenan-hotdep09.pdf and /dev/null differ diff --git a/test/test_files/greenan-hotstorage10.pdf b/test/test_files/greenan-hotstorage10.pdf deleted file mode 100644 index 4ff1c17..0000000 Binary files a/test/test_files/greenan-hotstorage10.pdf and /dev/null differ diff --git a/test/test_files/greenan-mascots08.pdf b/test/test_files/greenan-mascots08.pdf deleted file mode 100644 index 0d4e9dc..0000000 Binary files a/test/test_files/greenan-mascots08.pdf and /dev/null differ diff --git a/test/test_files/greenan-msst10.pdf b/test/test_files/greenan-msst10.pdf deleted file mode 100644 index 70038b1..0000000 Binary files a/test/test_files/greenan-msst10.pdf and /dev/null differ diff --git a/test/test_files/greenan-sisw05.pdf b/test/test_files/greenan-sisw05.pdf deleted file mode 100644 index c8c2855..0000000 Binary files a/test/test_files/greenan-sisw05.pdf and /dev/null differ diff --git a/test/test_files/greenan-storagess07.pdf b/test/test_files/greenan-storagess07.pdf deleted file mode 100644 index 6365827..0000000 Binary files a/test/test_files/greenan-storagess07.pdf and /dev/null differ diff --git a/test/test_files/li-fast12.pdf b/test/test_files/li-fast12.pdf deleted file mode 100644 index e009956..0000000 Binary files a/test/test_files/li-fast12.pdf and /dev/null differ diff --git a/test/test_files/li-fast2012.pdf b/test/test_files/li-fast2012.pdf deleted file mode 100644 index e009956..0000000 Binary files a/test/test_files/li-fast2012.pdf and /dev/null differ diff --git a/test/test_files/plank-fast2013.pdf b/test/test_files/plank-fast2013.pdf deleted file mode 100644 index eb1db88..0000000 Binary files a/test/test_files/plank-fast2013.pdf and /dev/null differ diff --git a/test/test_files/portegys-pdpta03.pdf b/test/test_files/portegys-pdpta03.pdf deleted file mode 100644 index 1d6feef..0000000 Binary files a/test/test_files/portegys-pdpta03.pdf and /dev/null differ diff --git a/test/test_files/storer-fast08.pdf b/test/test_files/storer-fast08.pdf deleted file mode 100644 index c81ef51..0000000 Binary files a/test/test_files/storer-fast08.pdf and /dev/null differ diff --git a/test/test_files/storer-pdsw08.pdf b/test/test_files/storer-pdsw08.pdf deleted file mode 100644 index 595d167..0000000 Binary files a/test/test_files/storer-pdsw08.pdf and /dev/null differ diff --git a/test/test_files/storer-storagess06.pdf b/test/test_files/storer-storagess06.pdf deleted file mode 100644 index 69bea64..0000000 Binary files a/test/test_files/storer-storagess06.pdf and /dev/null differ diff --git a/test/test_files/storer-storagess08.pdf b/test/test_files/storer-storagess08.pdf deleted file mode 100644 index fc60fce..0000000 Binary files a/test/test_files/storer-storagess08.pdf and /dev/null differ diff --git a/test/test_files/storer-usenix07.pdf b/test/test_files/storer-usenix07.pdf deleted file mode 100644 index 1658e3a..0000000 Binary files a/test/test_files/storer-usenix07.pdf and /dev/null differ diff --git a/test/test_pyeclib_api.py b/test/test_pyeclib_api.py deleted file mode 100644 index e8732af..0000000 --- a/test/test_pyeclib_api.py +++ /dev/null @@ -1,812 +0,0 @@ -# Copyright (c) 2013, Kevin Greenan (kmgreen2@gmail.com) -# All rights reserved. -# -# Redistribution and use in source and binary forms, with or without -# modification, are permitted provided that the following conditions are met: -# -# Redistributions of source code must retain the above copyright notice, this -# list of conditions and the following disclaimer. -# -# Redistributions in binary form must reproduce the above copyright notice, -# this list of conditions and the following disclaimer in the documentation -# and/or other materials provided with the distribution. THIS SOFTWARE IS -# PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS -# OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES -# OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN -# NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY -# DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES -# (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; -# LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND -# ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT -# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF -# THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - -import os -import random -from string import ascii_letters, ascii_uppercase, digits -import sys -import tempfile -import unittest - -from distutils.version import StrictVersion -from itertools import combinations - -import pyeclib.ec_iface -from pyeclib.ec_iface import ECBackendNotSupported -from pyeclib.ec_iface import ECDriver -from pyeclib.ec_iface import ECDriverError -from pyeclib.ec_iface import ECInsufficientFragments -from pyeclib.ec_iface import ECInvalidFragmentMetadata -from pyeclib.ec_iface import ECInvalidParameter -from pyeclib.ec_iface import PyECLib_EC_Types -from pyeclib.ec_iface import ALL_EC_TYPES -from pyeclib.ec_iface import VALID_EC_TYPES -from pyeclib.ec_iface import LIBERASURECODE_VERSION -import resource - - -if sys.version < '3': - def b2i(b): - return ord(b) -else: - def b2i(b): - return b - - -class TestNullDriver(unittest.TestCase): - - def setUp(self): - self.null_driver = ECDriver( - library_import_str="pyeclib.core.ECNullDriver", k=8, m=2) - - def tearDown(self): - pass - - def test_null_driver(self): - self.null_driver.encode('') - self.null_driver.decode([]) - - -class TestStripeDriver(unittest.TestCase): - - def setUp(self): - self.stripe_driver = ECDriver( - library_import_str="pyeclib.core.ECStripingDriver", k=8, m=0) - - def tearDown(self): - pass - - -class TestPyECLibDriver(unittest.TestCase): - - def __init__(self, *args): - # Create the temp files needed for testing - self.file_sizes = ["100-K"] - self.files = {} - self.num_iterations = 100 - self._create_tmp_files() - - unittest.TestCase.__init__(self, *args) - - def _create_tmp_files(self): - """ - Create the temporary files needed for testing. Use the tempfile - package so that the files will be automatically removed during - garbage collection. - """ - for size_str in self.file_sizes: - # Determine the size of the file to create - size_desc = size_str.split("-") - size = int(size_desc[0]) - if size_desc[1] == 'M': - size *= 1000000 - elif size_desc[1] == 'K': - size *= 1000 - - # Create the dictionary of files to test with - buf = ''.join(random.choice(ascii_letters) for i in range(size)) - if sys.version_info >= (3,): - buf = buf.encode('ascii') - tmp_file = tempfile.NamedTemporaryFile() - tmp_file.write(buf) - self.files[size_str] = tmp_file - - def setUp(self): - # Ensure that the file offset is set to the head of the file - for _, tmp_file in self.files.items(): - tmp_file.seek(0, 0) - - def tearDown(self): - pass - - def test_invalid_km_args(self): - for ec_type in VALID_EC_TYPES: - # missing k - with self.assertRaises(ECDriverError) as err_context: - ECDriver(ec_type=ec_type, m=1) - - self.assertEqual(str(err_context.exception), - "Invalid Argument: k is required") - - # missing m - with self.assertRaises(ECDriverError) as err_context: - ECDriver(ec_type=ec_type, k=1) - - self.assertEqual(str(err_context.exception), - "Invalid Argument: m is required") - - with self.assertRaises(ECDriverError) as err_context: - # k is smaller than 1 - ECDriver(ec_type=ec_type, k=-100, m=1) - self.assertEqual(str(err_context.exception), - "Invalid number of data fragments (k)") - - with self.assertRaises(ECDriverError) as err_context: - # m is smaller than 1 - ECDriver(ec_type=ec_type, k=1, m=-100) - self.assertEqual(str(err_context.exception), - "Invalid number of parity fragments (m)") - - def test_valid_ec_types(self): - # Build list of available types and compare to VALID_EC_TYPES - available_ec_types = [] - for _type in ALL_EC_TYPES: - try: - if _type is 'shss': - _k = 10 - _m = 4 - elif _type is 'libphazr': - _k = 4 - _m = 4 - else: - _k = 10 - _m = 5 - ECDriver(k=_k, m=_m, ec_type=_type, validate=True) - available_ec_types.append(_type) - except Exception: - # ignore any errors, assume backend not available - pass - self.assertEqual(available_ec_types, VALID_EC_TYPES) - - def test_valid_algo(self): - print("") - for _type in ALL_EC_TYPES: - # Check if this algo works - if _type not in VALID_EC_TYPES: - print("Skipping test for %s backend" % _type) - continue - try: - if _type is 'shss': - _instance = ECDriver(k=10, m=4, ec_type=_type) - elif _type is 'libphazr': - _instance = ECDriver(k=4, m=4, ec_type=_type) - else: - _instance = ECDriver(k=10, m=5, ec_type=_type) - except ECDriverError: - self.fail("%s algorithm not supported" % _type) - - self.assertRaises(ECBackendNotSupported, ECDriver, k=10, m=5, - ec_type="invalid_algo") - - def get_pyeclib_testspec(self, csum="none"): - pyeclib_drivers = [] - _type1 = 'jerasure_rs_vand' - if _type1 in VALID_EC_TYPES: - pyeclib_drivers.append(ECDriver(k=12, m=2, ec_type=_type1, - chksum_type=csum)) - pyeclib_drivers.append(ECDriver(k=11, m=2, ec_type=_type1, - chksum_type=csum)) - pyeclib_drivers.append(ECDriver(k=10, m=2, ec_type=_type1, - chksum_type=csum)) - pyeclib_drivers.append(ECDriver(k=8, m=4, ec_type=_type1, - chksum_type=csum)) - _type2 = 'liberasurecode_rs_vand' - if _type2 in VALID_EC_TYPES: - pyeclib_drivers.append(ECDriver(k=12, m=2, ec_type=_type2, - chksum_type=csum)) - pyeclib_drivers.append(ECDriver(k=11, m=2, ec_type=_type2, - chksum_type=csum)) - pyeclib_drivers.append(ECDriver(k=10, m=2, ec_type=_type2, - chksum_type=csum)) - pyeclib_drivers.append(ECDriver(k=8, m=4, ec_type=_type2, - chksum_type=csum)) - _type3_1 = 'flat_xor_hd' - if _type3_1 in VALID_EC_TYPES: - pyeclib_drivers.append(ECDriver(k=12, m=6, ec_type=_type3_1, - chksum_type=csum)) - pyeclib_drivers.append(ECDriver(k=10, m=5, ec_type=_type3_1, - chksum_type=csum)) - _type3_2 = 'flat_xor_hd_4' - if _type3_2 in VALID_EC_TYPES: - pyeclib_drivers.append(ECDriver(k=12, m=6, ec_type=_type3_2, - chksum_type=csum)) - pyeclib_drivers.append(ECDriver(k=10, m=5, ec_type=_type3_2, - chksum_type=csum)) - _type4 = 'shss' - if _type4 in VALID_EC_TYPES: - pyeclib_drivers.append(ECDriver(k=10, m=4, ec_type=_type4, - chksum_type=csum)) - pyeclib_drivers.append(ECDriver(k=20, m=4, ec_type=_type4, - chksum_type=csum)) - pyeclib_drivers.append(ECDriver(k=11, m=7, ec_type=_type4, - chksum_type=csum)) - - _type5 = 'isa_l_rs_vand' - if _type5 in VALID_EC_TYPES: - pyeclib_drivers.append(ECDriver(k=12, m=2, ec_type=_type5, - chksum_type=csum)) - pyeclib_drivers.append(ECDriver(k=11, m=2, ec_type=_type5, - chksum_type=csum)) - pyeclib_drivers.append(ECDriver(k=10, m=2, ec_type=_type5, - chksum_type=csum)) - pyeclib_drivers.append(ECDriver(k=8, m=4, ec_type=_type5, - chksum_type=csum)) - _type6 = 'isa_l_rs_cauchy' - if _type6 in VALID_EC_TYPES: - pyeclib_drivers.append(ECDriver(k=12, m=2, ec_type=_type6, - chksum_type=csum)) - pyeclib_drivers.append(ECDriver(k=11, m=2, ec_type=_type6, - chksum_type=csum)) - pyeclib_drivers.append(ECDriver(k=10, m=2, ec_type=_type6, - chksum_type=csum)) - pyeclib_drivers.append(ECDriver(k=8, m=4, ec_type=_type6, - chksum_type=csum)) - pyeclib_drivers.append(ECDriver(k=11, m=7, ec_type=_type6, - chksum_type=csum)) - - _type7 = 'libphazr' - if _type7 in VALID_EC_TYPES: - pyeclib_drivers.append(ECDriver(k=4, m=4, ec_type=_type7, - chksum_type=csum)) - return pyeclib_drivers - - def test_small_encode(self): - pyeclib_drivers = self.get_pyeclib_testspec() - encode_strs = [b"a", b"hello", b"hellohyhi", b"yo"] - - for pyeclib_driver in pyeclib_drivers: - for encode_str in encode_strs: - encoded_fragments = pyeclib_driver.encode(encode_str) - decoded_str = pyeclib_driver.decode(encoded_fragments) - - self.assertTrue(decoded_str == encode_str) - - def test_encode_invalid_params(self): - pyeclib_drivers = self.get_pyeclib_testspec() - encode_args = [u'\U0001F0A1', 3, object(), None, {}, []] - - for pyeclib_driver in pyeclib_drivers: - for encode_str in encode_args: - with self.assertRaises(ECInvalidParameter): - pyeclib_driver.encode(encode_str) - - def test_attribute_error_in_the_error_handling(self): - pyeclib_drivers = self.get_pyeclib_testspec() - self.assertGreater(len(pyeclib_drivers), 0) - pyeclib_driver = pyeclib_drivers[0] - - del pyeclib.ec_iface.ECInvalidParameter - try: - with self.assertRaises(AttributeError): # !! - pyeclib_driver.encode(3) - finally: - pyeclib.ec_iface.ECInvalidParameter = ECInvalidParameter - - def test_import_error_in_the_error_handling(self): - pyeclib_drivers = self.get_pyeclib_testspec() - self.assertGreater(len(pyeclib_drivers), 0) - pyeclib_driver = pyeclib_drivers[0] - - from six.moves import builtins - real_import = builtins.__import__ - - def fake_import(*a, **kw): - raise ImportError - - builtins.__import__ = fake_import - try: - with self.assertRaises(ImportError): # !! - pyeclib_driver.encode(3) - finally: - builtins.__import__ = real_import - - def test_decode_reconstruct_with_fragment_iterator(self): - pyeclib_drivers = self.get_pyeclib_testspec() - encode_strs = [b"a", b"hello", b"hellohyhi", b"yo"] - - for pyeclib_driver in pyeclib_drivers: - for encode_str in encode_strs: - encoded_fragments = pyeclib_driver.encode(encode_str) - - idxs_to_remove = random.sample(range( - pyeclib_driver.k + pyeclib_driver.m), 2) - available_fragments = encoded_fragments[:] - for i in sorted(idxs_to_remove, reverse=True): - available_fragments.pop(i) - - frag_iter = iter(available_fragments) - decoded_str = pyeclib_driver.decode(frag_iter) - self.assertEqual(decoded_str, encode_str) - - # Since the iterator is exhausted, we can't decode again - with self.assertRaises(ECDriverError) as exc_mgr: - pyeclib_driver.decode(frag_iter) - self.assertEqual( - 'Invalid fragment payload in ECPyECLibDriver.decode', - str(exc_mgr.exception)) - - frag_iter = iter(available_fragments) - reconstructed_fragments = pyeclib_driver.reconstruct( - frag_iter, idxs_to_remove) - self.assertEqual(len(reconstructed_fragments), - len(idxs_to_remove)) - for i, data in zip(idxs_to_remove, reconstructed_fragments): - self.assertEqual(data, encoded_fragments[i]) - - # Since the iterator is exhausted, we can't decode again - with self.assertRaises(ECDriverError) as exc_mgr: - pyeclib_driver.reconstruct(frag_iter, idxs_to_remove) - self.assertEqual( - 'Invalid fragment payload in ECPyECLibDriver.reconstruct', - str(exc_mgr.exception)) - - def check_metadata_formatted(self, k, m, ec_type, chksum_type): - - if ec_type not in VALID_EC_TYPES: - return - - filesize = 1024 * 1024 * 3 - file_str = ''.join(random.choice(ascii_letters) - for i in range(filesize)) - file_bytes = file_str.encode('utf-8') - - pyeclib_driver = ECDriver(k=k, m=m, ec_type=ec_type, - chksum_type=chksum_type) - - fragments = pyeclib_driver.encode(file_bytes) - - f = 0 - for fragment in fragments: - metadata = pyeclib_driver.get_metadata(fragment, 1) - if 'index' in metadata: - self.assertEqual(metadata['index'], f) - else: - self.assertTrue(False) - - if 'chksum_mismatch' in metadata: - self.assertEqual(metadata['chksum_mismatch'], 0) - else: - self.assertTrue(False) - - if 'backend_id' in metadata: - self.assertEqual(metadata['backend_id'], ec_type) - else: - self.assertTrue(False) - - if 'orig_data_size' in metadata: - self.assertEqual(metadata['orig_data_size'], 3145728) - else: - self.assertTrue(False) - - if 'chksum_type' in metadata: - self.assertEqual(metadata['chksum_type'], 'crc32') - else: - self.assertTrue(False) - - if 'backend_version' not in metadata: - self.assertTrue(False) - - if 'chksum' not in metadata: - self.assertTrue(False) - - if 'size' not in metadata: - self.assertTrue(False) - - f += 1 - - def test_get_metadata_formatted(self): - self.check_metadata_formatted(12, 2, "jerasure_rs_vand", - "inline_crc32") - self.check_metadata_formatted(12, 2, "liberasurecode_rs_vand", - "inline_crc32") - self.check_metadata_formatted(8, 4, "liberasurecode_rs_vand", - "inline_crc32") - - def test_verify_fragment_inline_chksum_fail(self): - pyeclib_drivers = self.get_pyeclib_testspec("inline_crc32") - filesize = 1024 * 1024 * 3 - file_str = ''.join(random.choice(ascii_letters) - for i in range(filesize)) - file_bytes = file_str.encode('utf-8') - - for pyeclib_driver in pyeclib_drivers: - fragments = pyeclib_driver.encode(file_bytes) - - fragment_metadata_list = [] - - first_fragment_to_corrupt = random.randint(0, len(fragments)) - num_to_corrupt = 2 - fragments_to_corrupt = [ - (first_fragment_to_corrupt + i) % len(fragments) - for i in range(num_to_corrupt + 1)] - fragments_to_corrupt.sort() - - i = 0 - for fragment in fragments: - if i in fragments_to_corrupt: - corrupted_fragment = fragment[:100] +\ - (str(chr((b2i(fragment[100]) + 0x1) - % 128))).encode('utf-8') + fragment[101:] - fragment_metadata_list.append( - pyeclib_driver.get_metadata(corrupted_fragment)) - else: - fragment_metadata_list.append( - pyeclib_driver.get_metadata(fragment)) - i += 1 - - expected_ret_value = {"status": -206, - "reason": "Bad checksum", - "bad_fragments": fragments_to_corrupt} - self.assertEqual( - pyeclib_driver.verify_stripe_metadata(fragment_metadata_list), - expected_ret_value) - - def test_verify_fragment_inline_chksum_succeed(self): - pyeclib_drivers = self.get_pyeclib_testspec("inline_crc32") - - filesize = 1024 * 1024 * 3 - file_str = ''.join(random.choice(ascii_letters) - for i in range(filesize)) - file_bytes = file_str.encode('utf-8') - - for pyeclib_driver in pyeclib_drivers: - fragments = pyeclib_driver.encode(file_bytes) - - fragment_metadata_list = [] - - for fragment in fragments: - fragment_metadata_list.append( - pyeclib_driver.get_metadata(fragment)) - - expected_ret_value = {"status": 0} - - self.assertTrue(pyeclib_driver.verify_stripe_metadata( - fragment_metadata_list) == expected_ret_value) - - def test_get_segment_byterange_info(self): - pyeclib_drivers = self.get_pyeclib_testspec() - if not pyeclib_drivers: - return - - file_size = 1024 * 1024 - segment_size = 3 * 1024 - - ranges = [ - (0, 1), - (1, 12), - (10, 1000), - (0, segment_size - 1), - (1, segment_size + 1), - (segment_size - 1, 2 * segment_size)] - - expected_results = {} - - expected_results[(0, 1)] = {0: (0, 1)} - expected_results[(1, 12)] = {0: (1, 12)} - expected_results[(10, 1000)] = {0: (10, 1000)} - expected_results[(0, segment_size - 1)] = {0: (0, segment_size - 1)} - expected_results[(1, segment_size + 1) - ] = {0: (1, segment_size - 1), 1: (0, 1)} - expected_results[ - (segment_size - 1, 2 * segment_size)] = { - 0: (segment_size - 1, segment_size - 1), - 1: (0, segment_size - 1), - 2: (0, 0)} - - results = pyeclib_drivers[0].get_segment_info_byterange( - ranges, file_size, segment_size) - - for exp_result_key in expected_results: - self.assertIn(exp_result_key, results) - self.assertTrue( - len(results[exp_result_key]) == - len(expected_results[exp_result_key])) - exp_result_map = expected_results[exp_result_key] - for segment_key in exp_result_map: - self.assertIn(segment_key, results[exp_result_key]) - self.assertTrue( - results[exp_result_key][segment_key] == - expected_results[exp_result_key][segment_key]) - - def test_get_segment_info(self): - pyeclib_drivers = self.get_pyeclib_testspec() - - file_sizes = [ - 1024 * 1024, - 2 * 1024 * 1024, - 10 * 1024 * 1024, - 10 * 1024 * 1024 + 7] - segment_sizes = [3 * 1024, 1024 * 1024] - segment_strings = {} - - # - # Generate some test segments for each segment size. - # Use 2 * segment size, because last segment may be - # greater than segment_size - # - char_set = ascii_uppercase + digits - for segment_size in segment_sizes: - segment_strings[segment_size] = ''.join( - random.choice(char_set) for i in range(segment_size * 2)) - - for pyeclib_driver in pyeclib_drivers: - for file_size in file_sizes: - for segment_size in segment_sizes: - # - # Compute the segment info - # - segment_info = pyeclib_driver.get_segment_info( - file_size, - segment_size) - - num_segments = segment_info['num_segments'] - segment_size = segment_info['segment_size'] - fragment_size = segment_info['fragment_size'] - last_segment_size = segment_info['last_segment_size'] - last_fragment_size = segment_info['last_fragment_size'] - - computed_file_size = ( - (num_segments - 1) * segment_size) + last_segment_size - - # - # Verify that the segment sizes add up - # - self.assertTrue(computed_file_size == file_size) - - encoded_fragments = pyeclib_driver.encode( - (segment_strings[segment_size][ - :segment_size]).encode('utf-8')) - - # - # Verify the fragment size - # - self.assertTrue(fragment_size == len(encoded_fragments[0])) - - if last_segment_size > 0: - encoded_fragments = pyeclib_driver.encode( - (segment_strings[segment_size][ - :last_segment_size]).encode('utf-8')) - - # - # Verify the last fragment size, if there is one - # - self.assertTrue( - last_fragment_size == len( - encoded_fragments[0])) - - def test_greedy_decode_reconstruct_combination(self): - # the testing spec defined at get_pyeclib_testspec() method - # and if you want to test either other parameters or backends, - # you can add the spec you want to test there. - pyeclib_drivers = self.get_pyeclib_testspec() - orig_data = os.urandom(1024 ** 2) - for pyeclib_driver in pyeclib_drivers: - encoded = pyeclib_driver.encode(orig_data) - # make all fragment like (index, frag_data) format to feed - # to combinations - frags = [(i, frag) for i, frag in enumerate(encoded)] - num_frags = pyeclib_driver.k + pyeclib_driver.m - - if pyeclib_driver.ec_type == PyECLib_EC_Types.flat_xor_hd: - # flat_xord_hd is guaranteed to work with 2 or 3 failures - tolerable_failures = pyeclib_driver.hd - 1 - else: - # ... while others can tolerate more - tolerable_failures = pyeclib_driver.m - - for check_frags_tuples in combinations( - frags, num_frags - tolerable_failures): - # extract check_frags_tuples from [(index, data bytes), ...] - # to [index, index, ...] and [data bytes, data bytes, ...] - indexes, check_frags = zip(*check_frags_tuples) - decoded = pyeclib_driver.decode(check_frags) - self.assertEqual( - orig_data, decoded, - "assertion fail in decode %s from:%s" % - (pyeclib_driver, indexes)) - holes = [index for index in range(num_frags) if - index not in indexes] - for hole in holes: - reconed = pyeclib_driver.reconstruct( - check_frags, [hole])[0] - self.assertEqual( - frags[hole][1], reconed, - "assertion fail in reconstruct %s target:%s " - "from:%s" % (pyeclib_driver, hole, indexes)) - - def test_rs(self): - pyeclib_drivers = self.get_pyeclib_testspec() - - for pyeclib_driver in pyeclib_drivers: - for file_size in self.file_sizes: - tmp_file = self.files[file_size] - tmp_file.seek(0) - whole_file_bytes = tmp_file.read() - - encode_input = whole_file_bytes - orig_fragments = pyeclib_driver.encode(encode_input) - - for iter in range(self.num_iterations): - num_missing = 2 - idxs_to_remove = random.sample(range( - pyeclib_driver.k + pyeclib_driver.m), num_missing) - fragments = orig_fragments[:] - - # Reverse sort the list, so we can always - # remove from the original index - idxs_to_remove.sort(reverse=True) - for idx in idxs_to_remove: - fragments.pop(idx) - - # - # Test decoder - # - decoded_string = pyeclib_driver.decode(fragments) - - self.assertTrue(encode_input == decoded_string) - - # - # Test reconstructor - # - reconstructed_fragments = pyeclib_driver.reconstruct( - fragments, - idxs_to_remove) - self.assertEqual(len(reconstructed_fragments), - len(idxs_to_remove)) - for idx, frag_data in zip(idxs_to_remove, - reconstructed_fragments): - self.assertEqual( - frag_data, orig_fragments[idx], - 'Failed to reconstruct fragment %d!' % idx) - - # - # Test decode with integrity checks - # - first_fragment_to_corrupt = random.randint( - 0, len(fragments)) - num_to_corrupt = min(len(fragments), pyeclib_driver.m + 1) - fragments_to_corrupt = [ - (first_fragment_to_corrupt + i) % len(fragments) - for i in range(num_to_corrupt)] - - if StrictVersion(LIBERASURECODE_VERSION) < \ - StrictVersion('1.2.0'): - # if liberasurecode is older than the version supports - # fragment integrity check, skip following test - continue - i = 0 - for fragment in fragments: - if i in fragments_to_corrupt: - corrupted_fragment = ( - "0" * len(fragment)).encode('utf-8') - fragments[i] = corrupted_fragment - i += 1 - - self.assertRaises(ECInvalidFragmentMetadata, - pyeclib_driver.decode, - fragments, force_metadata_checks=True) - - def get_available_backend(self, k, m, ec_type, chksum_type="inline_crc32"): - if ec_type[:11] == "flat_xor_hd": - return ECDriver(k=k, m=m, ec_type="flat_xor_hd", - chksum_type=chksum_type) - elif ec_type in VALID_EC_TYPES: - return ECDriver(k=k, m=m, ec_type=ec_type, chksum_type=chksum_type) - else: - return None - - def test_liberasurecode_insufficient_frags_error(self): - file_size = self.file_sizes[0] - tmp_file = self.files[file_size] - tmp_file.seek(0) - whole_file_bytes = tmp_file.read() - for ec_type in ['flat_xor_hd_3', 'liberasurecode_rs_vand']: - if ec_type in VALID_EC_TYPES: - pyeclib_driver = self.get_available_backend( - k=10, m=5, ec_type=ec_type) - fragments = pyeclib_driver.encode(whole_file_bytes) - self.assertRaises(ECInsufficientFragments, - pyeclib_driver.reconstruct, - [fragments[0]], [1, 2, 3, 4, 5, 6]) - - def test_min_parity_fragments_needed(self): - pyeclib_drivers = [] - for ec_type in ['flat_xor_hd_3', 'liberasurecode_rs_vand']: - if ec_type in VALID_EC_TYPES: - pyeclib_drivers.append(ECDriver(k=10, m=5, ec_type=ec_type)) - self.assertTrue( - pyeclib_drivers[0].min_parity_fragments_needed() == 1) - - def test_pyeclib_driver_repr_expression(self): - pyeclib_drivers = self.get_pyeclib_testspec() - for driver in pyeclib_drivers: - if driver.ec_type.name == 'flat_xor_hd': - name = 'flat_xor_hd_%s' % driver.hd - else: - name = driver.ec_type.name - - self.assertEqual( - "ECDriver(ec_type='%s', k=%s, m=%s)" % - (name, driver.k, driver.m), repr(driver)) - - def test_get_segment_info_memory_usage(self): - for ec_driver in self.get_pyeclib_testspec(): - self._test_get_segment_info_memory_usage(ec_driver) - - def _test_get_segment_info_memory_usage(self, ec_driver): - # 1. Preapre the expected memory allocation - ec_driver.get_segment_info(1024 * 1024, 1024 * 1024) - loop_range = range(1000) - - # 2. Get current memory usage - usage = resource.getrusage(resource.RUSAGE_SELF)[2] - - # 3. Loop to call get_segment_info - for x in loop_range: - ec_driver.get_segment_info(1024 * 1024, 1024 * 1024) - - # 4. memory usage shoudln't be increased - self.assertEqual(usage, resource.getrusage(resource.RUSAGE_SELF)[2], - 'Memory usage is increased unexpectedly %s - %s' % - (usage, resource.getrusage(resource.RUSAGE_SELF)[2])) - - def test_get_metadata_memory_usage(self): - for ec_driver in self.get_pyeclib_testspec(): - self._test_get_metadata_memory_usage(ec_driver) - - def _test_get_metadata_memory_usage(self, ec_driver): - # 1. Prepare the expected memory allocation - encoded = ec_driver.encode(b'aaa') - ec_driver.get_metadata(encoded[0], formatted=True) - loop_range = range(400000) - - # 2. Get current memory usage - baseline_usage = resource.getrusage(resource.RUSAGE_SELF)[2] - - # 3. Loop to call get_metadata - for x in loop_range: - ec_driver.get_metadata(encoded[0], formatted=True) - - # 4. memory usage shouldn't increase - new_usage = resource.getrusage(resource.RUSAGE_SELF)[2] - self.assertEqual(baseline_usage, new_usage, - 'Memory usage is increased unexpectedly %s -> %s' % - (baseline_usage, new_usage)) - - -class TestBackendsEnabled(unittest.TestCase): - '''Based on TestPyECLibDriver.test_valid_algo above, but these tests - should *always* either pass or skip.''' - class __metaclass__(type): - def __new__(meta, cls_name, cls_bases, cls_dict): - for ec_type in ALL_EC_TYPES: - def dummy(self, ec_type=ec_type): - if ec_type not in VALID_EC_TYPES: - raise unittest.SkipTest - if ec_type == 'shss': - k = 10 - m = 4 - elif ec_type == 'libphazr': - k = 4 - m = 4 - else: - k = 10 - m = 5 - ECDriver(k=k, m=m, ec_type=ec_type) - dummy.__name__ = 'test_%s_available' % ec_type - cls_dict[dummy.__name__] = dummy - return type.__new__(meta, cls_name, cls_bases, cls_dict) - - -if __name__ == '__main__': - unittest.main() diff --git a/test/test_pyeclib_c.py b/test/test_pyeclib_c.py deleted file mode 100644 index 04e2444..0000000 --- a/test/test_pyeclib_c.py +++ /dev/null @@ -1,473 +0,0 @@ -# Copyright (c) 2013, Kevin Greenan (kmgreen2@gmail.com) -# All rights reserved. -# -# Redistribution and use in source and binary forms, with or without -# modification, are permitted provided that the following conditions are met: -# -# Redistributions of source code must retain the above copyright notice, this -# list of conditions and the following disclaimer. -# -# Redistributions in binary form must reproduce the above copyright notice, -# this list of conditions and the following disclaimer in the documentation -# and/or other materials provided with the distribution. THIS SOFTWARE IS -# PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS -# OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES -# OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN -# NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY -# DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES -# (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; -# LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND -# ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT -# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF -# THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - -import random -from string import ascii_letters -import sys -import tempfile -import time -import unittest - -import pyeclib_c -from pyeclib.ec_iface import PyECLib_EC_Types -from pyeclib.ec_iface import VALID_EC_TYPES - - -class Timer: - - def __init__(self): - self.start_time = 0 - self.end_time = 0 - - def start(self): - self.start_time = time.time() - - def stop(self): - self.end_time = time.time() - - def curr_delta(self): - return self.end_time - self.start_time - - def stop_and_return(self): - self.end_time = time.time() - return self.curr_delta() - - -def require_backend(backend): - return unittest.skipIf(backend not in VALID_EC_TYPES, - "%s backend is not available" % backend) - - -class TestPyECLib(unittest.TestCase): - - def __init__(self, *args): - self.num_datas = [12, 12, 12] - self.num_parities = [2, 3, 4] - self.iterations = 100 - - # EC algorithm and config parameters - self.rs_types = [(PyECLib_EC_Types.jerasure_rs_vand), - (PyECLib_EC_Types.jerasure_rs_cauchy), - (PyECLib_EC_Types.isa_l_rs_vand), - (PyECLib_EC_Types.liberasurecode_rs_vand), - (PyECLib_EC_Types.isa_l_rs_cauchy)] - self.xor_types = [(PyECLib_EC_Types.flat_xor_hd, 12, 6, 4), - (PyECLib_EC_Types.flat_xor_hd, 10, 5, 4), - (PyECLib_EC_Types.flat_xor_hd, 10, 5, 3)] - self.shss = [(PyECLib_EC_Types.shss, 6, 3), - (PyECLib_EC_Types.shss, 10, 4), - (PyECLib_EC_Types.shss, 20, 4), - (PyECLib_EC_Types.shss, 11, 7)] - self.libphazr = [(PyECLib_EC_Types.libphazr, 4, 4)] - - # Input temp files for testing - self.sizes = ["101-K", "202-K", "303-K"] - self.files = {} - self._create_tmp_files() - - unittest.TestCase.__init__(self, *args) - - def _create_tmp_files(self): - """ - Create the temporary files needed for testing. Use the tempfile - package so that the files will be automatically removed during - garbage collection. - """ - for size_str in self.sizes: - # Determine the size of the file to create - size_desc = size_str.split("-") - size = int(size_desc[0]) - if size_desc[1] == 'M': - size *= 1000000 - elif size_desc[1] == 'K': - size *= 1000 - - # Create the dictionary of files to test with - buf = ''.join(random.choice(ascii_letters) for i in range(size)) - if sys.version_info >= (3,): - buf = buf.encode('ascii') - tmp_file = tempfile.NamedTemporaryFile('w+b') - tmp_file.write(buf) - self.files[size_str] = tmp_file - - def get_tmp_file(self, name): - """ - Acquire a temp file from the dictionary of pre-built, random files - with the seek position to the head of the file. - """ - tmp_file = self.files.get(name, None) - - if tmp_file: - tmp_file.seek(0, 0) - - return tmp_file - - def setUp(self): - # Ensure that the file offset is set to the head of the file - for _, tmp_file in self.files.items(): - tmp_file.seek(0, 0) - - def tearDown(self): - pass - - def iter_available_types(self, ec_types): - found_one = False - for ec_type in ec_types: - if ec_type.name not in VALID_EC_TYPES: - continue - found_one = True - yield ec_type - if not found_one: - type_list = ', '.join(t.name for t in ec_types) - raise unittest.SkipTest('No backend available in types: %r' % - type_list) - - def time_encode(self, num_data, num_parity, ec_type, hd, - file_size, iterations): - """ - :return average encode time - """ - timer = Timer() - handle = pyeclib_c.init(num_data, num_parity, ec_type, hd) - whole_file_bytes = self.get_tmp_file(file_size).read() - - timer.start() - for l in range(iterations): - pyeclib_c.encode(handle, whole_file_bytes) - tsum = timer.stop_and_return() - - return tsum / iterations - - def time_decode(self, - num_data, num_parity, ec_type, hd, - file_size, iterations): - """ - :return 2-tuple, (success, average decode time) - """ - timer = Timer() - tsum = 0 - handle = pyeclib_c.init(num_data, num_parity, ec_type, hd) - whole_file_bytes = self.get_tmp_file(file_size).read() - success = True - - fragments = pyeclib_c.encode(handle, whole_file_bytes) - orig_fragments = fragments[:] - - for i in range(iterations): - num_missing = hd - 1 - for j in range(num_missing): - num_frags_left = len(fragments) - idx = random.randint(0, num_frags_left - 1) - fragments.pop(idx) - - timer.start() - decoded_file_bytes = pyeclib_c.decode(handle, - fragments, - len(fragments[0])) - tsum += timer.stop_and_return() - - fragments = orig_fragments[:] - - if whole_file_bytes != decoded_file_bytes: - success = False - - return success, tsum / iterations - - def time_range_decode(self, - num_data, num_parity, ec_type, hd, - file_size, iterations): - """ - :return 2-tuple, (success, average decode time) - """ - timer = Timer() - tsum = 0 - handle = pyeclib_c.init(num_data, num_parity, ec_type, hd) - whole_file_bytes = self.get_tmp_file(file_size).read() - success = True - - begins = [int(random.randint(0, len(whole_file_bytes) - 1)) - for i in range(3)] - ends = [int(random.randint(begins[i], len(whole_file_bytes))) - for i in range(3)] - - ranges = list(zip(begins, ends)) - - fragments = pyeclib_c.encode(handle, whole_file_bytes) - orig_fragments = fragments[:] - - for i in range(iterations): - num_missing = hd - 1 - for j in range(num_missing): - num_frags_left = len(fragments) - idx = random.randint(0, num_frags_left - 1) - fragments.pop(idx) - - timer.start() - decoded_file_ranges = pyeclib_c.decode(handle, - fragments, - len(fragments[0]), - ranges) - tsum += timer.stop_and_return() - - fragments = orig_fragments[:] - - range_offset = 0 - for r in ranges: - if whole_file_bytes[ - r[0]: r[1] + 1] != decoded_file_ranges[range_offset]: - success = False - range_offset += 1 - - return success, tsum / iterations - - def time_reconstruct(self, - num_data, num_parity, ec_type, hd, - file_size, iterations): - """ - :return 2-tuple, (success, average reconstruct time) - """ - timer = Timer() - tsum = 0 - handle = pyeclib_c.init(num_data, num_parity, ec_type, hd) - whole_file_bytes = self.get_tmp_file(file_size).read() - success = True - - orig_fragments = pyeclib_c.encode(handle, whole_file_bytes) - - for i in range(iterations): - fragments = orig_fragments[:] - num_missing = 1 - missing_idxs = [] - for j in range(num_missing): - num_frags_left = len(fragments) - idx = random.randint(0, num_frags_left - 1) - while idx in missing_idxs: - idx = random.randint(0, num_frags_left - 1) - missing_idxs.append(idx) - fragments.pop(idx) - - timer.start() - reconstructed_fragment = pyeclib_c.reconstruct(handle, - fragments, - len(fragments[0]), - missing_idxs[0]) - tsum += timer.stop_and_return() - - if orig_fragments[missing_idxs[0]] != reconstructed_fragment: - success = False - # Output the fragments for debugging - with open("orig_fragments", "wb") as fd_orig: - fd_orig.write(orig_fragments[missing_idxs[0]]) - with open("decoded_fragments", "wb") as fd_decoded: - fd_decoded.write(reconstructed_fragment) - print("Fragment %d was not reconstructed!!!" % missing_idxs[0]) - sys.exit(2) - - return success, tsum / iterations - - def get_throughput(self, avg_time, size_str): - size_desc = size_str.split("-") - size = float(size_desc[0]) - - if avg_time == 0: - return '? (test finished too fast to calculate throughput)' - - if size_desc[1] == 'M': - throughput = size / avg_time - elif size_desc[1] == 'K': - throughput = (size / 1000.0) / avg_time - - return format(throughput, '.10g') - - @require_backend("flat_xor_hd_3") - def test_xor_code(self): - for (ec_type, k, m, hd) in self.xor_types: - print("\nRunning tests for flat_xor_hd k=%d, m=%d, hd=%d" % - (k, m, hd)) - - for size_str in self.sizes: - avg_time = self.time_encode(k, m, ec_type.value, hd, - size_str, - self.iterations) - print("Encode (%s): %s" % - (size_str, self.get_throughput(avg_time, size_str))) - - for size_str in self.sizes: - success, avg_time = self.time_decode(k, m, ec_type.value, hd, - size_str, - self.iterations) - self.assertTrue(success) - print("Decode (%s): %s" % - (size_str, self.get_throughput(avg_time, size_str))) - - for size_str in self.sizes: - success, avg_time = self.time_reconstruct( - k, m, ec_type.value, hd, size_str, self.iterations) - self.assertTrue(success) - print("Reconstruct (%s): %s" % - (size_str, self.get_throughput(avg_time, size_str))) - - @require_backend("shss") - def test_shss(self): - for (ec_type, k, m) in self.shss: - print(("\nRunning tests for %s k=%d, m=%d" % (ec_type, k, m))) - - success = self._test_get_required_fragments(k, m, ec_type) - self.assertTrue(success) - - for size_str in self.sizes: - avg_time = self.time_encode(k, m, ec_type.value, 0, - size_str, - self.iterations) - print("Encode (%s): %s" % - (size_str, self.get_throughput(avg_time, size_str))) - - for size_str in self.sizes: - success, avg_time = self.time_decode(k, m, ec_type.value, 0, - size_str, - self.iterations) - self.assertTrue(success) - print("Decode (%s): %s" % - (size_str, self.get_throughput(avg_time, size_str))) - - for size_str in self.sizes: - success, avg_time = self.time_reconstruct( - k, m, ec_type.value, 0, size_str, self.iterations) - self.assertTrue(success) - print("Reconstruct (%s): %s" % - (size_str, self.get_throughput(avg_time, size_str))) - - @require_backend("libphazr") - def test_libphazr(self): - for (ec_type, k, m) in self.libphazr: - print(("\nRunning tests for %s k=%d, m=%d" % (ec_type, k, m))) - - success = self._test_get_required_fragments(k, m, ec_type) - self.assertTrue(success) - - for size_str in self.sizes: - avg_time = self.time_encode(k, m, ec_type.value, 0, - size_str, - self.iterations) - print("Encode (%s): %s" % - (size_str, self.get_throughput(avg_time, size_str))) - - for size_str in self.sizes: - success, avg_time = self.time_decode(k, m, ec_type.value, 0, - size_str, - self.iterations) - self.assertTrue(success) - print("Decode (%s): %s" % - (size_str, self.get_throughput(avg_time, size_str))) - - for size_str in self.sizes: - success, avg_time = self.time_reconstruct( - k, m, ec_type.value, 0, size_str, self.iterations) - self.assertTrue(success) - print("Reconstruct (%s): %s" % - (size_str, self.get_throughput(avg_time, size_str))) - - def _test_get_required_fragments(self, num_data, num_parity, ec_type): - """ - :return boolean, True if all tests passed - """ - handle = pyeclib_c.init(num_data, num_parity, ec_type.value) - success = True - - # - # MDS codes need any k fragments - # - if ec_type in ["jerasure_rs_vand", "jerasure_rs_cauchy", - "liberasurecode_rs_vand"]: - expected_fragments = [i for i in range(num_data + num_parity)] - missing_fragments = [] - - # - # Remove between 1 and num_parity - # - for i in range(random.randint(0, num_parity - 1)): - missing_fragment = random.sample(expected_fragments, 1)[0] - missing_fragments.append(missing_fragment) - expected_fragments.remove(missing_fragment) - - expected_fragments = expected_fragments[:num_data] - required_fragments = pyeclib_c.get_required_fragments( - handle, - missing_fragments, []) - - if expected_fragments != required_fragments: - success = False - print("Unexpected required fragments list " - "(exp != req): %s != %s" % ( - expected_fragments, required_fragments)) - - return success - - def test_codes(self): - for ec_type in self.iter_available_types(self.rs_types): - for i in range(len(self.num_datas)): - success = self._test_get_required_fragments( - self.num_datas[i], self.num_parities[i], ec_type) - self.assertTrue(success) - - for i in range(len(self.num_datas)): - for size_str in self.sizes: - avg_time = self.time_encode( - self.num_datas[i], self.num_parities[i], ec_type.value, - self.num_parities[i] + 1, size_str, self.iterations) - - print("Encode (%s): %s" % - (size_str, self.get_throughput(avg_time, size_str))) - - for i in range(len(self.num_datas)): - for size_str in self.sizes: - success, avg_time = self.time_decode( - self.num_datas[i], self.num_parities[i], ec_type.value, - self.num_parities[i] + 1, size_str, self.iterations) - - self.assertTrue(success) - print("Decode (%s): %s" % - (size_str, self.get_throughput(avg_time, size_str))) - - for i in range(len(self.num_datas)): - for size_str in self.sizes: - success, avg_time = self.time_range_decode( - self.num_datas[i], self.num_parities[i], ec_type.value, - self.num_parities[i] + 1, size_str, self.iterations) - - self.assertTrue(success) - print("Range Decode (%s): %s" % - (size_str, self.get_throughput(avg_time, size_str))) - - for i in range(len(self.num_datas)): - for size_str in self.sizes: - success, avg_time = self.time_reconstruct( - self.num_datas[i], self.num_parities[i], ec_type.value, - self.num_parities[i] + 1, size_str, self.iterations) - self.assertTrue(success) - print("Reconstruct (%s): %s" % - (size_str, self.get_throughput(avg_time, size_str))) - - -if __name__ == "__main__": - unittest.main() diff --git a/tools/pyeclib_conf_tool.py b/tools/pyeclib_conf_tool.py deleted file mode 100644 index 7f7a837..0000000 --- a/tools/pyeclib_conf_tool.py +++ /dev/null @@ -1,266 +0,0 @@ -# Copyright (c) 2013, Kevin Greenan (kmgreen2@gmail.com) -# All rights reserved. -# -# Redistribution and use in source and binary forms, with or without -# modification, are permitted provided that the following conditions are met: -# -# Redistributions of source code must retain the above copyright notice, this -# list of conditions and the following disclaimer. -# -# Redistributions in binary form must reproduce the above copyright notice, -# this list of conditions and the following disclaimer in the documentation -# and/or other materials provided with the distribution. THIS SOFTWARE IS -# PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS -# OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES -# OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN -# NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY -# DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES -# (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; -# LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND -# ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT -# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF -# THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - - -# PyEClib Companion tool -# Goal: When defining an EC pool, help cluster admin make an informed choice -# between available EC implementations. Generate sample swift.conf + swift- -# ring-builder hints. -# -# Suggested features: -# -# - List the "EC types" supported - EC algorithms -# - List implementations of each EC type available on the platform -# (dumb-software-only, software with SIMD acceleration, -# specialized hardware, etc). -# - Benchmark each algorithm with possible implementation and display -# performance numbers. -# - Generate sample EC policy entry (for inclusion in swift.conf) for the -# best performing algorithm + implementation. (And optionally provide swift- -# ring-builder hints). -# -# Suggested EC policy entry format: -# -# ======== swift.conf ============ -# [storage-policy:10] -# type = erasure_coding -# name = ec_jerasure_rs_cauchy_12_2 -# ec_type = jerasure_rs_cauchy -# ec_k = 12 -# ec_m = 2 -# ============================ -# -# (ec_type values are one of those available within PyEClib) - -# -# User input: Num data, num parity, average file size -# Output: Ordered list of options and their corresponding conf entries -# (limit 10) -# - -from pyeclib.ec_iface import ECDriver -import random -import string -import sys -import argparse -import time -import math - - -class Timer: - - def __init__(self): - self.start_time = 0 - self.end_time = 0 - - def reset(self): - self.start_time = 0 - self.end_time = 0 - - def start(self): - self.start_time = time.time() - - def stop(self): - self.end_time = time.time() - - def curr_delta(self): - return self.end_time - self.start_time - - def stop_and_return(self): - self.end_time = time.time() - return self.curr_delta() - - -def nCr(n, r): - f = math.factorial - return f(n) / f(r) / f(n - r) - - -class ECScheme: - - def __init__(self, k, m, ec_type): - self.k = k - self.m = m - self.ec_type = ec_type - - def __str__(self): - return "k=%d m=%d ec_type=%s" % (self.k, self.m, self.ec_type) - -valid_flat_xor_hd_3 = [(6, 6), (7, 6), (8, 6), (9, 6), - (10, 6), (11, 6), (12, 6), (13, 6), - (14, 6), (15, 6)] - -valid_flat_xor_hd_4 = [(6, 6), (7, 6), (8, 6), (9, 6), - (10, 6), (11, 6), (12, 6), (13, 6), - (14, 6), (15, 6), (16, 6), (17, 6), - (18, 6), (19, 6), (20, 6)] - - -def get_viable_schemes( - max_num_frags, minimum_rate, avg_stripe_size, fault_tolerance): - - list_of_schemes = [] - - # - # Get min_k from (minimum_rate * max_num_frags) - # - min_k = int(math.ceil(minimum_rate * max_num_frags)) - - # - # Get min_m from the fault tolerance - # - min_m = fault_tolerance - - # - # Is not information theoretically possible - # - if (min_k + min_m) > max_num_frags: - return list_of_schemes - - # - # Iterate over EC(k, max_num_frags-k) k \in [min_k, n-min_m] - # - for k in range(min_k, max_num_frags - min_m + 1): - list_of_schemes.append( - ECScheme(k, max_num_frags - k, "jerasure_rs_vand")) - - list_of_schemes.append( - ECScheme(k, max_num_frags - k, "jerasure_rs_cauchy")) - - # - # The XOR codes are a little tricker - # (only check if fault_tolerance = 2 or 3) - # - # Constraint for 2: k <= (m choose 2) - # Constraint for 3: k <= (m choose 3) - # - # The '3' flat_xor_hd_3 (and '4' in flat_xor_hd_4) refers to the Hamming - # distance, which means the code guarantees the reconstruction of any - # 2 lost fragments (or 3 in the case of flat_xor_hd_4). - # - # So, only consider the XOR code if the fault_tolerance matches and - # the additional constraint is met - # - if fault_tolerance == 2: - max_k = nCr(max_num_frags - k, 2) - if k <= max_k and (k, max_num_frags - k) in valid_flat_xor_hd_3: - list_of_schemes.append( - ECScheme(k, max_num_frags - k, "flat_xor_hd_3")) - - if fault_tolerance == 3: - max_k = nCr(max_num_frags - k, 3) - if k <= max_k and (k, max_num_frags - k) in valid_flat_xor_hd_4: - list_of_schemes.append( - ECScheme(k, max_num_frags - k, "flat_xor_hd_4")) - - return list_of_schemes - - -parser = argparse.ArgumentParser( - description='PyECLib tool to evaluate viable EC options, benchmark them ' - 'and report results with the appropriate conf entries.') -parser.add_argument( - '-n', - type=int, - help='max number of fragments', - required=True) -parser.add_argument('-f', type=int, help='fault tolerance', required=True) -parser.add_argument( - '-r', - type=float, - help='minimum coding rate (num_data / num_data+num_parity)', - required=True) -parser.add_argument('-s', type=int, help='average stripe size', required=True) -parser.add_argument( - '-l', - type=int, - help='set limit on number of entries returned (default = 10)', - default=10, -) - -args = parser.parse_args(sys.argv[1:]) - -MB = 1024 * 1024 - -# Generate a buffer of size 's' -if args.s > 10 * MB: - print("s must be smaller than 10 MB.") - sys.exit(1) - -# Instantiate the timer -timer = Timer() - -return_limit = args.l - -schemes = get_viable_schemes(args.n, args.r, args.s, args.f) - -# Results will be List[(ec_type, throughput)] -results = [] - -# Num iterations -num_iterations = 10 - -for scheme in schemes: - print(scheme) - - # Generate a new string for each test - file_str = ''.join( - random.choice( - string.ascii_uppercase + string.digits) for x in range(args.s)) - - try: - ec_driver = ECDriver(k=scheme.k, m=scheme.m, ec_type=scheme.ec_type) - except Exception as e: - print("Scheme %s is not defined (%s)." % (scheme, e)) - continue - - timer.start() - - for i in range(num_iterations): - ec_driver.encode(file_str) - - duration = timer.stop_and_return() - - results.append((scheme, duration)) - - timer.reset() - -print(results) -results.sort(lambda x, y: (int)((1000 * x[1]) - (1000 * y[1]))) - -for i in range(len(results)): - if i > return_limit: - break - - print("\n\nPerf Rank #%d:" % i) - print(" ======== To Use this Policy, Copy and Paste Text (not including " - "this header and footer) to Swift Conf ========") - print(" type = erasure_coding") - print(" name = %s_%d_%d" % (results[i][0].ec_type, - results[i][0].k, results[i][0].m)) - print(" ec_type = %s" % results[i][0].ec_type) - print(" ec_k = %s" % results[i][0].k) - print(" ec_m = %s" % results[i][0].m) - print(" ================================================================" - "==============================================") - results[i] diff --git a/tools/pyeclib_decode.py b/tools/pyeclib_decode.py deleted file mode 100644 index 707a63a..0000000 --- a/tools/pyeclib_decode.py +++ /dev/null @@ -1,56 +0,0 @@ -# Copyright (c) 2013, Kevin Greenan (kmgreen2@gmail.com) -# All rights reserved. -# -# Redistribution and use in source and binary forms, with or without -# modification, are permitted provided that the following conditions are met: -# -# Redistributions of source code must retain the above copyright notice, this -# list of conditions and the following disclaimer. -# -# Redistributions in binary form must reproduce the above copyright notice, -# this list of conditions and the following disclaimer in the documentation -# and/or other materials provided with the distribution. THIS SOFTWARE IS -# PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS -# OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES -# OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN -# NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY -# DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES -# (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; -# LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND -# ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT -# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF -# THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - -from pyeclib.ec_iface import ECDriver -import argparse - -parser = argparse.ArgumentParser(description='Decoder for PyECLib.') -parser.add_argument('k', type=int, help='number of data elements') -parser.add_argument('m', type=int, help='number of parity elements') -parser.add_argument('ec_type', help='EC algorithm used') -parser.add_argument('fragments', metavar='fragment', nargs='+', - help='fragments to decode') -parser.add_argument('filename', help='output file') - -args = parser.parse_args() - -print("k = %d, m = %d" % (args.k, args.m)) -print("ec_type = %s" % args.ec_type) -print("fragments = %s" % args.fragments) -print("filename = %s" % args.filename) - -ec_driver = ECDriver(k=args.k, m=args.m, ec_type=args.ec_type) - -fragment_list = [] - -# read fragments -for fragment in args.fragments: - with open(("%s" % fragment), "rb") as fp: - fragment_list.append(fp.read()) - -# decode -decoded_file = ec_driver.decode(fragment_list) - -# write -with open("%s.decoded" % args.filename, "wb") as fp: - fp.write(decoded_file) diff --git a/tools/pyeclib_encode.py b/tools/pyeclib_encode.py deleted file mode 100644 index 093d1cf..0000000 --- a/tools/pyeclib_encode.py +++ /dev/null @@ -1,55 +0,0 @@ -# Copyright (c) 2013, Kevin Greenan (kmgreen2@gmail.com) -# All rights reserved. -# -# Redistribution and use in source and binary forms, with or without -# modification, are permitted provided that the following conditions are met: -# -# Redistributions of source code must retain the above copyright notice, this -# list of conditions and the following disclaimer. -# -# Redistributions in binary form must reproduce the above copyright notice, -# this list of conditions and the following disclaimer in the documentation -# and/or other materials provided with the distribution. THIS SOFTWARE IS -# PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS -# OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES -# OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN -# NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY -# DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES -# (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; -# LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND -# ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT -# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF -# THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - -from pyeclib.ec_iface import ECDriver -import argparse - -parser = argparse.ArgumentParser(description='Encoder for PyECLib.') -parser.add_argument('k', type=int, help='number of data elements') -parser.add_argument('m', type=int, help='number of parity elements') -parser.add_argument('ec_type', help='EC algorithm used') -parser.add_argument('file_dir', help='directory with the file') -parser.add_argument('filename', help='file to encode') -parser.add_argument('fragment_dir', help='directory to drop encoded fragments') - -args = parser.parse_args() - -print("k = %d, m = %d" % (args.k, args.m)) -print("ec_type = %s" % args.ec_type) -print("filename = %s" % args.filename) - -ec_driver = ECDriver(k=args.k, m=args.m, ec_type=args.ec_type) - -# read -with open(("%s/%s" % (args.file_dir, args.filename)), "rb") as fp: - whole_file_str = fp.read() - -# encode -fragments = ec_driver.encode(whole_file_str) - -# store -i = 0 -for fragment in fragments: - with open("%s/%s.%d" % (args.fragment_dir, args.filename, i), "wb") as fp: - fp.write(fragment) - i += 1 diff --git a/tools/pyeclib_fragments_needed.py b/tools/pyeclib_fragments_needed.py deleted file mode 100644 index 48d1bdc..0000000 --- a/tools/pyeclib_fragments_needed.py +++ /dev/null @@ -1,47 +0,0 @@ -# Copyright (c) 2013, Kevin Greenan (kmgreen2@gmail.com) -# All rights reserved. -# -# Redistribution and use in source and binary forms, with or without -# modification, are permitted provided that the following conditions are met: -# -# Redistributions of source code must retain the above copyright notice, this -# list of conditions and the following disclaimer. -# -# Redistributions in binary form must reproduce the above copyright notice, -# this list of conditions and the following disclaimer in the documentation -# and/or other materials provided with the distribution. THIS SOFTWARE IS -# PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS -# OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES -# OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN -# NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY -# DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES -# (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; -# LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND -# ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT -# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF -# THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - -import pyeclib -from pyeclib.ec_iface import ECDriver -import random -import string -import sys -import os -import argparse - -parser = argparse.ArgumentParser( - description='PyECLib tool to determine fragment indexes needed to ' - 'recover missing fragments.') -parser.add_argument('k', type=int, help='number of data elements') -parser.add_argument('m', type=int, help='number of parity elements') -parser.add_argument('ec_type', help='EC algorithm used') -parser.add_argument('missing_fragments', type=int, metavar='missing_fragment', - nargs='+', help='missing_fragments') - -args = parser.parse_args() - -ec_driver = ECDriver(k=args.k, m=args.m, ec_type=args.ec_type) - -fragments_needed = ec_driver.fragments_needed(args.missing_fragments) - -print(fragments_needed) diff --git a/tox.ini b/tox.ini deleted file mode 100644 index 2f2f09c..0000000 --- a/tox.ini +++ /dev/null @@ -1,20 +0,0 @@ -[tox] -envlist = py27,py35,pep8 - -[testenv] -deps = - -r{toxinidir}/test-requirements.txt -commands= - nosetests --no-path-adjustment -v test/ - -[testenv:pep8] -deps= - pep8 -commands= - pep8 pyeclib/ setup.py test/ - -[testenv:venv] -commands = {posargs} - -[testenv:docs] -commands = python setup.py build_sphinx