Run codespell to fix typos
This fixes the spell fix by; $ tox -e codespell -- -q 2 -S '*.csv,*.po,*.yaml' -w Change-Id: I11a3142d6eef40bec2324b98bfe4b0d871779f64
This commit is contained in:
parent
f565b62fa5
commit
88369fee13
@ -14,7 +14,7 @@ storlets (0.6.0)
|
|||||||
* Minor improvements and bug fixes
|
* Minor improvements and bug fixes
|
||||||
|
|
||||||
storlets (0.5.0, Openstack Pike)
|
storlets (0.5.0, Openstack Pike)
|
||||||
* Extending IPython intgration
|
* Extending IPython integration
|
||||||
|
|
||||||
* Storlet execution on GET with multiple files using a single stream
|
* Storlet execution on GET with multiple files using a single stream
|
||||||
using MULTIPART MIME
|
using MULTIPART MIME
|
||||||
@ -49,7 +49,7 @@ storlets (0.3.0, Openstack Newton)
|
|||||||
|
|
||||||
* Introduction of Storlets ACLs. The storlet middleware now supports
|
* Introduction of Storlets ACLs. The storlet middleware now supports
|
||||||
a POST verb for allowing a certain user READ access to a container
|
a POST verb for allowing a certain user READ access to a container
|
||||||
ony via a certain storlet. Thus, a user which has no read access
|
only via a certain storlet. Thus, a user which has no read access
|
||||||
to a container can get access only if the request has the appropriate
|
to a container can get access only if the request has the appropriate
|
||||||
storlet specified in the X-Run-Storlet header.
|
storlet specified in the X-Run-Storlet header.
|
||||||
|
|
||||||
|
@ -78,7 +78,7 @@ Code Organization
|
|||||||
- doc/source/: Documentation
|
- doc/source/: Documentation
|
||||||
- install/: Installation scripts
|
- install/: Installation scripts
|
||||||
|
|
||||||
- swift/: Scripts for invoking an extenral Swift ansible installation scrpts
|
- swift/: Scripts for invoking an extenral Swift ansible installation scripts
|
||||||
- storlets/: Scripts for installing storlets over a Swift cluster
|
- storlets/: Scripts for installing storlets over a Swift cluster
|
||||||
|
|
||||||
- tests/: Unit and functional tests
|
- tests/: Unit and functional tests
|
||||||
|
@ -220,7 +220,7 @@ public class CSVStorlet implements IStorlet, PushdownStorletConstants, SparkInde
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
if (rangeBytesLeft > 0) {
|
if (rangeBytesLeft > 0) {
|
||||||
Utils.doubleLogPrint(logger,"got a null line with more bytes ot read: " + String.valueOf(rangeBytesLeft));
|
Utils.doubleLogPrint(logger,"got a null line with more bytes to read: " + String.valueOf(rangeBytesLeft));
|
||||||
}
|
}
|
||||||
|
|
||||||
if ( (line == null) && (rangeBytesLeft > 1024)) {
|
if ( (line == null) && (rangeBytesLeft > 1024)) {
|
||||||
|
@ -191,10 +191,10 @@ public class IdentityStorlet implements IStorlet {
|
|||||||
}
|
}
|
||||||
os.close();
|
os.close();
|
||||||
} catch (Exception e) {
|
} catch (Exception e) {
|
||||||
log.emitLog("Copying data from inut stream to output stream failed: "
|
log.emitLog("Copying data from input stream to output stream failed: "
|
||||||
+ e.getMessage());
|
+ e.getMessage());
|
||||||
throw new StorletException(
|
throw new StorletException(
|
||||||
"Copying data from inut stream to output stream failed: "
|
"Copying data from input stream to output stream failed: "
|
||||||
+ e.getMessage());
|
+ e.getMessage());
|
||||||
} finally {
|
} finally {
|
||||||
try {
|
try {
|
||||||
|
@ -105,7 +105,7 @@ public class MultiInputStorlet implements IStorlet {
|
|||||||
try {
|
try {
|
||||||
psis.close();
|
psis.close();
|
||||||
} catch (Exception e) {
|
} catch (Exception e) {
|
||||||
log.emitLog("Falied to close input steram " + e.getMessage());
|
log.emitLog("Failed to close input stream " + e.getMessage());
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@ -116,7 +116,7 @@ public class MultiInputStorlet implements IStorlet {
|
|||||||
try {
|
try {
|
||||||
os.close();
|
os.close();
|
||||||
} catch (IOException e) {
|
} catch (IOException e) {
|
||||||
log.emitLog("Falied to close input steram " + e.getMessage());
|
log.emitLog("Failed to close input stream " + e.getMessage());
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
log.emitLog("MultiInputStorlet Invocation done");
|
log.emitLog("MultiInputStorlet Invocation done");
|
||||||
|
@ -153,7 +153,7 @@ function _install_docker {
|
|||||||
add_user_to_group $STORLETS_SWIFT_RUNTIME_USER docker
|
add_user_to_group $STORLETS_SWIFT_RUNTIME_USER docker
|
||||||
|
|
||||||
if [ $STORLETS_SWIFT_RUNTIME_USER == $USER ]; then
|
if [ $STORLETS_SWIFT_RUNTIME_USER == $USER ]; then
|
||||||
# NOTE(takashi): We need this workaroud because we can't reload
|
# NOTE(takashi): We need this workaround because we can't reload
|
||||||
# user-group relationship in bash scripts
|
# user-group relationship in bash scripts
|
||||||
DOCKER_UNIX_SOCKET=/var/run/docker.sock
|
DOCKER_UNIX_SOCKET=/var/run/docker.sock
|
||||||
sudo chown $USER:$USER $DOCKER_UNIX_SOCKET
|
sudo chown $USER:$USER $DOCKER_UNIX_SOCKET
|
||||||
|
@ -231,10 +231,10 @@ An additional header ('X-Run-Storlet') must be provided to inform the system to
|
|||||||
'X-Auth-Token': {authorization_token}
|
'X-Auth-Token': {authorization_token}
|
||||||
|
|
||||||
In the PUT case the storlet acts upon the object appearing in the 'X-Copy-From' header, creating the object appearing in the request path.
|
In the PUT case the storlet acts upon the object appearing in the 'X-Copy-From' header, creating the object appearing in the request path.
|
||||||
In the COPY case the storlet acts upon the object appeairng in the requets path, crating the object appearing in the 'Destination' header.
|
In the COPY case the storlet acts upon the object appeairng in the request path, crating the object appearing in the 'Destination' header.
|
||||||
|
|
||||||
Independently of the verb used to invoke a copy, one can add the 'X-Storlet-Extra-Resources' header. Thus, one can e.g. create an
|
Independently of the verb used to invoke a copy, one can add the 'X-Storlet-Extra-Resources' header. Thus, one can e.g. create an
|
||||||
object which is a concatenation of the copy source and the extra resources. As with the invocation upon downlod, when using extra
|
object which is a concatenation of the copy source and the extra resources. As with the invocation upon download, when using extra
|
||||||
resources, the storlet is invoked on a proxy node.
|
resources, the storlet is invoked on a proxy node.
|
||||||
|
|
||||||
Currently, specifying any of the headers below while invoking a storlet upon copy will result in '400 Bad Request'
|
Currently, specifying any of the headers below while invoking a storlet upon copy will result in '400 Bad Request'
|
||||||
|
@ -196,7 +196,7 @@ An additional header ('X-Run-Storlet') must be provided to inform the system to
|
|||||||
'X-Auth-Token': {authorization_token}
|
'X-Auth-Token': {authorization_token}
|
||||||
|
|
||||||
In the PUT case the storlet acts upon the object appearing in the 'X-Copy-From' header, creating the object appearing in the request path.
|
In the PUT case the storlet acts upon the object appearing in the 'X-Copy-From' header, creating the object appearing in the request path.
|
||||||
In the COPY case the storlet acts upon the object appeairng in the requets path, crating the object appearing in the 'Destination' header.
|
In the COPY case the storlet acts upon the object appeairng in the request path, crating the object appearing in the 'Destination' header.
|
||||||
|
|
||||||
Specifying any of the headers below while invoking a storlet upon copy will result in '400 Bad Request'
|
Specifying any of the headers below while invoking a storlet upon copy will result in '400 Bad Request'
|
||||||
- 'X-Copy-From-Account'
|
- 'X-Copy-From-Account'
|
||||||
|
@ -175,7 +175,7 @@ with the following content:
|
|||||||
FROM ubuntu:22.04
|
FROM ubuntu:22.04
|
||||||
MAINTAINER root
|
MAINTAINER root
|
||||||
|
|
||||||
# The following operations shoud be defined in one line
|
# The following operations should be defined in one line
|
||||||
# to prevent docker images from including apt cache file.
|
# to prevent docker images from including apt cache file.
|
||||||
RUN apt-get update && \
|
RUN apt-get update && \
|
||||||
apt-get install python3 && \
|
apt-get install python3 && \
|
||||||
@ -245,7 +245,7 @@ Configure Swift to work with the middleware components
|
|||||||
Step 1: Edit the proxy server config file /etc/swift/proxy-server.conf and
|
Step 1: Edit the proxy server config file /etc/swift/proxy-server.conf and
|
||||||
do the following:
|
do the following:
|
||||||
|
|
||||||
#. Add the storlet_handler to the proxy server pipline just before the slo middleware as shown below:
|
#. Add the storlet_handler to the proxy server pipeline just before the slo middleware as shown below:
|
||||||
|
|
||||||
::
|
::
|
||||||
|
|
||||||
@ -269,7 +269,7 @@ In a SAIO environment these would be:
|
|||||||
/etc/swift/object-server/1.conf through /etc/swift/object-server/4.conf
|
/etc/swift/object-server/1.conf through /etc/swift/object-server/4.conf
|
||||||
otherwise the file is typically /etc/swift/object-server.conf
|
otherwise the file is typically /etc/swift/object-server.conf
|
||||||
|
|
||||||
1. Add the storlet_handler to the object server pipline just before the slo object-server as shown below:
|
1. Add the storlet_handler to the object server pipeline just before the slo object-server as shown below:
|
||||||
|
|
||||||
::
|
::
|
||||||
|
|
||||||
|
@ -41,7 +41,7 @@ Deploy a Python storlet
|
|||||||
General background on storlets deployment is found `here <http://storlets.readthedocs.io/en/latest/writing_and_deploying_storlets.html#storlet-deployment-guidelines>`__.
|
General background on storlets deployment is found `here <http://storlets.readthedocs.io/en/latest/writing_and_deploying_storlets.html#storlet-deployment-guidelines>`__.
|
||||||
|
|
||||||
In a new notebook cell, enter the '%%storletapp' directive
|
In a new notebook cell, enter the '%%storletapp' directive
|
||||||
followed by the storlet name. Followng that type the storlet code.
|
followed by the storlet name. Following that type the storlet code.
|
||||||
Below is an example of a simple 'identitiy' storlet.
|
Below is an example of a simple 'identitiy' storlet.
|
||||||
Executing the cell will deploy the storlet into Swift.
|
Executing the cell will deploy the storlet into Swift.
|
||||||
|
|
||||||
@ -96,7 +96,7 @@ General information on storlet invocation can be found `here <http://storlets.re
|
|||||||
|
|
||||||
Here is how an invocation works:
|
Here is how an invocation works:
|
||||||
|
|
||||||
#. Define an optional dictionay variable params that would hold the invocation parameters:
|
#. Define an optional dictionary variable params that would hold the invocation parameters:
|
||||||
|
|
||||||
::
|
::
|
||||||
|
|
||||||
|
@ -37,7 +37,7 @@ The storlet engine is tightly coupled with accounts in Swift in the following ma
|
|||||||
The Docker image
|
The Docker image
|
||||||
----------------
|
----------------
|
||||||
As mentioned above there is a Docker image per account that is enabled for storlets.
|
As mentioned above there is a Docker image per account that is enabled for storlets.
|
||||||
At a high level this image containes:
|
At a high level this image containers:
|
||||||
|
|
||||||
#. A Java run time environment. This is needed when you run storlets written in Java
|
#. A Java run time environment. This is needed when you run storlets written in Java
|
||||||
#. A daemon factory. A Python process that starts as part of the Docker container bring up.
|
#. A daemon factory. A Python process that starts as part of the Docker container bring up.
|
||||||
|
@ -1,7 +1,7 @@
|
|||||||
Storlets Terminology
|
Storlets Terminology
|
||||||
====================
|
====================
|
||||||
The overall storlets mechanism involves a lot of moving parts as well as
|
The overall storlets mechanism involves a lot of moving parts as well as
|
||||||
poeple or roles involved throughout the usage of the system. We give below
|
people or roles involved throughout the usage of the system. We give below
|
||||||
a definition of the various terms used throughout the documentation.
|
a definition of the various terms used throughout the documentation.
|
||||||
|
|
||||||
Storlet
|
Storlet
|
||||||
|
@ -1,11 +1,11 @@
|
|||||||
Queriable "Secondary Storage" Data
|
Queryable "Secondary Storage" Data
|
||||||
==================================
|
==================================
|
||||||
It is said that the primary use case for object stores is to serve as secondary
|
It is said that the primary use case for object stores is to serve as secondary
|
||||||
storage. With the increasing amount of data being gathered and analysed
|
storage. With the increasing amount of data being gathered and analysed
|
||||||
(have someone said IoT?) much of this data will make it to secondary storage.
|
(have someone said IoT?) much of this data will make it to secondary storage.
|
||||||
|
|
||||||
Being kept on secondary storage does not mean that the data does not
|
Being kept on secondary storage does not mean that the data does not
|
||||||
need to be queriable anymore: A recent identified trend may be searched
|
need to be queryable anymore: A recent identified trend may be searched
|
||||||
for in older data that was moved to secondary storage. Storlets allow an
|
for in older data that was moved to secondary storage. Storlets allow an
|
||||||
efficient and simple querying of data that resides in Swift.
|
efficient and simple querying of data that resides in Swift.
|
||||||
|
|
||||||
@ -14,7 +14,7 @@ known practice to aggregate data as it gets older. Storlets can serve
|
|||||||
as 'in place' data aggregators.
|
as 'in place' data aggregators.
|
||||||
|
|
||||||
Below are more concrete use cases that fall under the definition of
|
Below are more concrete use cases that fall under the definition of
|
||||||
efficient queriable secondary storage data.
|
efficient queryable secondary storage data.
|
||||||
|
|
||||||
Pushing down SQL filtering from Spark
|
Pushing down SQL filtering from Spark
|
||||||
-------------------------------------
|
-------------------------------------
|
||||||
|
@ -11,7 +11,7 @@ once that information is obscured. Some examples are:
|
|||||||
#. Smart electricity meters raw data can be made available to usage analysis
|
#. Smart electricity meters raw data can be made available to usage analysis
|
||||||
applications if the data is being averaged. The raw data is sensitive as it can reveal
|
applications if the data is being averaged. The raw data is sensitive as it can reveal
|
||||||
e.g. the time at which the residents are at home.
|
e.g. the time at which the residents are at home.
|
||||||
#. Pictures of landscape having poeple in them can be made available to e.g. google earth
|
#. Pictures of landscape having people in them can be made available to e.g. google earth
|
||||||
if the faces are blurred.
|
if the faces are blurred.
|
||||||
#. 3D designs can be made available to manufacturers after a slicing lossy transformation.
|
#. 3D designs can be made available to manufacturers after a slicing lossy transformation.
|
||||||
|
|
||||||
|
@ -95,7 +95,7 @@ Below are specific guidelines for deploying a Java storlet:
|
|||||||
#. The name of the Java storlet being uploaded must be of the form <name>-<version>, e.g.
|
#. The name of the Java storlet being uploaded must be of the form <name>-<version>, e.g.
|
||||||
identitystorlet-1.0.jar
|
identitystorlet-1.0.jar
|
||||||
|
|
||||||
#. The 'X-Object-Meta-Storlet-Main' metadata key shold be the name of the class implementing
|
#. The 'X-Object-Meta-Storlet-Main' metadata key should be the name of the class implementing
|
||||||
the IStorlet interface. E.g. "org.openstack.storlet.identity.IdentityStorlet"
|
the IStorlet interface. E.g. "org.openstack.storlet.identity.IdentityStorlet"
|
||||||
|
|
||||||
Deploying a Java Dependency
|
Deploying a Java Dependency
|
||||||
|
@ -68,14 +68,14 @@ Deploying a Python Storlet
|
|||||||
--------------------------
|
--------------------------
|
||||||
Below are specific guidelines for deploying a Python storlet:
|
Below are specific guidelines for deploying a Python storlet:
|
||||||
|
|
||||||
#. The object name of the python module containing the storlet class implemetation
|
#. The object name of the python module containing the storlet class implementation
|
||||||
must end with .py
|
must end with .py
|
||||||
|
|
||||||
#. Any python modules that the class implementation is dependent on should be uploaded as separate .py(s).
|
#. Any python modules that the class implementation is dependent on should be uploaded as separate .py(s).
|
||||||
|
|
||||||
#. The 'X-Object-Meta-Storlet-Main' metadata key shold be of the form: <module_name>.<class_name>. For example,
|
#. The 'X-Object-Meta-Storlet-Main' metadata key should be of the form: <module_name>.<class_name>. For example,
|
||||||
if the storlet name is SimpleStorlet and it resides in simple_storlet.py, then the
|
if the storlet name is SimpleStorlet and it resides in simple_storlet.py, then the
|
||||||
'X-Object-Meta-Storlet-Main' metadata key shold be "simple_storlet.SimpleStorlet"
|
'X-Object-Meta-Storlet-Main' metadata key should be "simple_storlet.SimpleStorlet"
|
||||||
|
|
||||||
Deploying a Python Dependency
|
Deploying a Python Dependency
|
||||||
-----------------------------
|
-----------------------------
|
||||||
|
@ -106,7 +106,7 @@ We support two types of dependencies:
|
|||||||
Storlet Object Metadata
|
Storlet Object Metadata
|
||||||
-----------------------
|
-----------------------
|
||||||
Uploading a storlet must be done to a designated container, called by default "storlet". The uploaded object
|
Uploading a storlet must be done to a designated container, called by default "storlet". The uploaded object
|
||||||
must carry the following metadata. See the specific langauge guides for more information.
|
must carry the following metadata. See the specific language guides for more information.
|
||||||
|
|
||||||
::
|
::
|
||||||
|
|
||||||
|
@ -248,7 +248,7 @@ use = egg:swift#recon
|
|||||||
# points and report the result after a full scan.
|
# points and report the result after a full scan.
|
||||||
# object_size_stats =
|
# object_size_stats =
|
||||||
|
|
||||||
# Note: Put it at the beginning of the pipleline to profile all middleware. But
|
# Note: Put it at the beginning of the pipeline to profile all middleware. But
|
||||||
# it is safer to put this after healthcheck.
|
# it is safer to put this after healthcheck.
|
||||||
[filter:xprofile]
|
[filter:xprofile]
|
||||||
use = egg:swift#xprofile
|
use = egg:swift#xprofile
|
||||||
|
@ -111,7 +111,7 @@ use = egg:swift#proxy
|
|||||||
# post_quorum_timeout = 0.5
|
# post_quorum_timeout = 0.5
|
||||||
#
|
#
|
||||||
# How long without an error before a node's error count is reset. This will
|
# How long without an error before a node's error count is reset. This will
|
||||||
# also be how long before a node is reenabled after suppression is triggered.
|
# also be how long before a node is re-enabled after suppression is triggered.
|
||||||
# error_suppression_interval = 60
|
# error_suppression_interval = 60
|
||||||
#
|
#
|
||||||
# How many errors can accumulate before a node is temporarily ignored.
|
# How many errors can accumulate before a node is temporarily ignored.
|
||||||
|
@ -43,3 +43,9 @@ console_scripts =
|
|||||||
sbus = storlets.sbus.cli:main
|
sbus = storlets.sbus.cli:main
|
||||||
storlets-daemon = storlets.agent.daemon.server:main
|
storlets-daemon = storlets.agent.daemon.server:main
|
||||||
storlets-daemon-factory = storlets.agent.daemon_factory.server:main
|
storlets-daemon-factory = storlets.agent.daemon_factory.server:main
|
||||||
|
|
||||||
|
[codespell]
|
||||||
|
quiet-level = 4
|
||||||
|
# Words to ignore:
|
||||||
|
# cna: Intel CNA card
|
||||||
|
ignore-words-list = cna
|
||||||
|
@ -60,7 +60,7 @@ public class SBusBackend {
|
|||||||
strLogLevel = "OFF";
|
strLogLevel = "OFF";
|
||||||
break;
|
break;
|
||||||
default:
|
default:
|
||||||
strLogLevel = "WARNINIG";
|
strLogLevel = "WARNING";
|
||||||
break;
|
break;
|
||||||
}
|
}
|
||||||
SBusJNIObj_.startLogger(strLogLevel, contId);
|
SBusJNIObj_.startLogger(strLogLevel, contId);
|
||||||
|
@ -89,7 +89,7 @@ public class ServerSBusInDatagram {
|
|||||||
* },
|
* },
|
||||||
* ...
|
* ...
|
||||||
* ]
|
* ]
|
||||||
* All the values in the above JSON elemens are strings.
|
* All the values in the above JSON elements are strings.
|
||||||
* Once constructed the class provides all necessary accessors to the parsed
|
* Once constructed the class provides all necessary accessors to the parsed
|
||||||
* fields.
|
* fields.
|
||||||
* @param msg the raw mwssage consisting of the string encoded json formats
|
* @param msg the raw mwssage consisting of the string encoded json formats
|
||||||
|
@ -124,7 +124,7 @@ public class SDaemon {
|
|||||||
return;
|
return;
|
||||||
|
|
||||||
storletTaskFactory_ = new STaskFactory(storlet, logger_);
|
storletTaskFactory_ = new STaskFactory(storlet, logger_);
|
||||||
logger_.trace("Instanciating SBus");
|
logger_.trace("Instantiating SBus");
|
||||||
sbus_ = new SBus(strContId);
|
sbus_ = new SBus(strContId);
|
||||||
sbus_.startLogger();
|
sbus_.startLogger();
|
||||||
try {
|
try {
|
||||||
|
@ -32,7 +32,7 @@ import java.util.concurrent.*;
|
|||||||
/*----------------------------------------------------------------------------
|
/*----------------------------------------------------------------------------
|
||||||
* SExecutionManager
|
* SExecutionManager
|
||||||
*
|
*
|
||||||
* This class manages tread workers to execute storlet application
|
* This class manages thread workers to execute storlet application
|
||||||
* */
|
* */
|
||||||
public class SExecutionManager {
|
public class SExecutionManager {
|
||||||
|
|
||||||
|
@ -34,11 +34,11 @@ class CommandResponse(Exception):
|
|||||||
|
|
||||||
def __init__(self, status, message, iterable=True, task_id=None):
|
def __init__(self, status, message, iterable=True, task_id=None):
|
||||||
"""
|
"""
|
||||||
Constract CommandResponse instance
|
Construct CommandResponse instance
|
||||||
|
|
||||||
:param status: task status
|
:param status: task status
|
||||||
:param message: message to be returned and logged
|
:param message: message to be returned and logged
|
||||||
:param iterable: wheter we can keep SDaemon process running
|
:param iterable: whether we can keep SDaemon process running
|
||||||
:param task_id: ID assigned to the requested task
|
:param task_id: ID assigned to the requested task
|
||||||
"""
|
"""
|
||||||
self.status = status
|
self.status = status
|
||||||
@ -184,7 +184,7 @@ class SBusServer(object):
|
|||||||
Main loop to run storlet application
|
Main loop to run storlet application
|
||||||
|
|
||||||
:returns: EXIT_SUCCESS when the loop exists normally
|
:returns: EXIT_SUCCESS when the loop exists normally
|
||||||
EXIT_FAILURE when some error occurd in main loop
|
EXIT_FAILURE when an error occurs in main loop
|
||||||
"""
|
"""
|
||||||
sbus = SBus()
|
sbus = SBus()
|
||||||
fd = sbus.create(self.sbus_path)
|
fd = sbus.create(self.sbus_path)
|
||||||
|
@ -180,7 +180,7 @@ class StorletRangeInputFile(StorletInputFile):
|
|||||||
self.start = start
|
self.start = start
|
||||||
self.end = end
|
self.end = end
|
||||||
# TODO(takashi): Currently we use range input file only for zero copy
|
# TODO(takashi): Currently we use range input file only for zero copy
|
||||||
# case, so can execute seek on fd. Myabe we need some
|
# case, so can execute seek on fd. Maybe we need some
|
||||||
# mechanism to confirm the fd is seekable.
|
# mechanism to confirm the fd is seekable.
|
||||||
self.obj_file.seek(self.start, 0)
|
self.obj_file.seek(self.start, 0)
|
||||||
self.point = self.start
|
self.point = self.start
|
||||||
|
@ -250,7 +250,7 @@ def main():
|
|||||||
parser.add_argument('sbus_path', help='the path to unix domain socket')
|
parser.add_argument('sbus_path', help='the path to unix domain socket')
|
||||||
parser.add_argument('log_level', help='log level')
|
parser.add_argument('log_level', help='log level')
|
||||||
parser.add_argument('pool_size', type=int,
|
parser.add_argument('pool_size', type=int,
|
||||||
help='the maximun thread numbers used swapns for '
|
help='the maximum thread numbers used swapns for '
|
||||||
'one storlet application')
|
'one storlet application')
|
||||||
parser.add_argument('container_id', help='container id')
|
parser.add_argument('container_id', help='container id')
|
||||||
opts = parser.parse_args()
|
opts = parser.parse_args()
|
||||||
|
@ -336,7 +336,7 @@ class StorletDaemonFactory(SBusServer):
|
|||||||
"""
|
"""
|
||||||
Kill every one.
|
Kill every one.
|
||||||
|
|
||||||
:param try_all: wheather we try to kill all process if we fail to
|
:param try_all: whether we try to kill all process if we fail to
|
||||||
stop some of the storlet daemons
|
stop some of the storlet daemons
|
||||||
:raises SDaemonError: when failed to kill one of the storlet daemons
|
:raises SDaemonError: when failed to kill one of the storlet daemons
|
||||||
"""
|
"""
|
||||||
@ -361,7 +361,7 @@ class StorletDaemonFactory(SBusServer):
|
|||||||
"""
|
"""
|
||||||
send HALT command to every spawned process
|
send HALT command to every spawned process
|
||||||
|
|
||||||
:param try_all: wheather we try to kill all process if we fail to
|
:param try_all: whether we try to kill all process if we fail to
|
||||||
stop some of the storlet daemons
|
stop some of the storlet daemons
|
||||||
:returns: a list of the terminated storlet daemons
|
:returns: a list of the terminated storlet daemons
|
||||||
:raises SDaemonError: when failed to kill one of the storlet daemons
|
:raises SDaemonError: when failed to kill one of the storlet daemons
|
||||||
@ -540,5 +540,5 @@ def main():
|
|||||||
sys.exit(factory.main_loop())
|
sys.exit(factory.main_loop())
|
||||||
|
|
||||||
except Exception:
|
except Exception:
|
||||||
logger.eception('Unhandled exception')
|
logger.exception('Unhandled exception')
|
||||||
sys.exit(EXIT_FAILURE)
|
sys.exit(EXIT_FAILURE)
|
||||||
|
@ -197,7 +197,7 @@ class StorletGatewayDocker(StorletGatewayBase):
|
|||||||
|
|
||||||
def invocation_flow(self, sreq, extra_sources=None):
|
def invocation_flow(self, sreq, extra_sources=None):
|
||||||
"""
|
"""
|
||||||
Invoke the backend protocl with gateway
|
Invoke the backend protocol with gateway
|
||||||
|
|
||||||
:param sreq: StorletRequest instance
|
:param sreq: StorletRequest instance
|
||||||
:param extra_sources (WIP): A list of StorletRequest instance to gather
|
:param extra_sources (WIP): A list of StorletRequest instance to gather
|
||||||
@ -267,7 +267,7 @@ class StorletGatewayDocker(StorletGatewayBase):
|
|||||||
:params sreq: DockerStorletRequest instance
|
:params sreq: DockerStorletRequest instance
|
||||||
:params is_storlet: True if the object is a storlet object
|
:params is_storlet: True if the object is a storlet object
|
||||||
False if the object is a dependency object
|
False if the object is a dependency object
|
||||||
:returns: Wheather the Docker container was updated with obj_name
|
:returns: Whether the Docker container was updated with obj_name
|
||||||
"""
|
"""
|
||||||
# Determine the cache we are to work with
|
# Determine the cache we are to work with
|
||||||
# e.g. dependency or storlet
|
# e.g. dependency or storlet
|
||||||
@ -295,7 +295,7 @@ class StorletGatewayDocker(StorletGatewayBase):
|
|||||||
elif is_storlet:
|
elif is_storlet:
|
||||||
# The cache_target_path exists, we test if it is up-to-date
|
# The cache_target_path exists, we test if it is up-to-date
|
||||||
# with the metadata we got.
|
# with the metadata we got.
|
||||||
# We mention that this is currenlty applicable for storlets
|
# We mention that this is currently applicable for storlets
|
||||||
# only, and not for dependencies.
|
# only, and not for dependencies.
|
||||||
# This will change when we will head dependencies as well
|
# This will change when we will head dependencies as well
|
||||||
fstat = os.stat(cache_target_path)
|
fstat = os.stat(cache_target_path)
|
||||||
@ -309,7 +309,7 @@ class StorletGatewayDocker(StorletGatewayBase):
|
|||||||
|
|
||||||
if update_cache:
|
if update_cache:
|
||||||
# If the cache needs to be updated, then get on with it
|
# If the cache needs to be updated, then get on with it
|
||||||
# bring the object from storge
|
# bring the object from storage
|
||||||
data_iter, perm = get_func(obj_name)
|
data_iter, perm = get_func(obj_name)
|
||||||
|
|
||||||
if perm:
|
if perm:
|
||||||
|
@ -105,7 +105,7 @@ class RunTimePaths(object):
|
|||||||
|
|
||||||
def __init__(self, scope, conf):
|
def __init__(self, scope, conf):
|
||||||
"""
|
"""
|
||||||
Constract RunTimePaths instance
|
Construct RunTimePaths instance
|
||||||
|
|
||||||
:param scope: scope name to be used as container name
|
:param scope: scope name to be used as container name
|
||||||
:param conf: gateway conf
|
:param conf: gateway conf
|
||||||
@ -190,15 +190,15 @@ class RunTimePaths(object):
|
|||||||
"""---------------------------------------------------------------------------
|
"""---------------------------------------------------------------------------
|
||||||
Docker Stateful Container API
|
Docker Stateful Container API
|
||||||
The RunTimeSandbox serve as an API between the Docker Gateway and
|
The RunTimeSandbox serve as an API between the Docker Gateway and
|
||||||
a re-usable per scope sandbox
|
a reusable per scope sandbox
|
||||||
---------------------------------------------------------------------------"""
|
---------------------------------------------------------------------------"""
|
||||||
|
|
||||||
|
|
||||||
class RunTimeSandbox(object):
|
class RunTimeSandbox(object):
|
||||||
"""
|
"""
|
||||||
The RunTimeSandbox represents a re-usable per scope sandbox.
|
The RunTimeSandbox represents a reusable per scope sandbox.
|
||||||
|
|
||||||
The sandbox is re-usable in the sense that it can run several storlet
|
The sandbox is reusable in the sense that it can run several storlet
|
||||||
daemons.
|
daemons.
|
||||||
|
|
||||||
The following methods are supported:
|
The following methods are supported:
|
||||||
@ -579,7 +579,7 @@ class StorletInvocationProtocol(object):
|
|||||||
@property
|
@property
|
||||||
def remote_fds(self):
|
def remote_fds(self):
|
||||||
"""
|
"""
|
||||||
A list of sbus file descirptors passed to remote side
|
A list of sbus file descriptors passed to remote side
|
||||||
"""
|
"""
|
||||||
storlets_metadata = {}
|
storlets_metadata = {}
|
||||||
if self.srequest.has_range:
|
if self.srequest.has_range:
|
||||||
@ -647,7 +647,7 @@ class StorletInvocationProtocol(object):
|
|||||||
raise
|
raise
|
||||||
# TODO(kota_): fd might be closed already, so if already
|
# TODO(kota_): fd might be closed already, so if already
|
||||||
# closed, OSError will be raised. we need more refactor to
|
# closed, OSError will be raised. we need more refactor to
|
||||||
# keep clean the file discriptors.
|
# keep clean the file descriptors.
|
||||||
pass
|
pass
|
||||||
|
|
||||||
def _close_remote_side_descriptors(self):
|
def _close_remote_side_descriptors(self):
|
||||||
|
@ -43,7 +43,7 @@ class SBusClient(object):
|
|||||||
|
|
||||||
def _parse_response(self, str_response):
|
def _parse_response(self, str_response):
|
||||||
"""
|
"""
|
||||||
Parse response string recieved from container side
|
Parse response string received from container side
|
||||||
|
|
||||||
:param str_response: response string
|
:param str_response: response string
|
||||||
:returns: SBusResponse instance
|
:returns: SBusResponse instance
|
||||||
|
@ -65,7 +65,7 @@ class SwiftFileManager(FileManager):
|
|||||||
@property
|
@property
|
||||||
def client(self):
|
def client(self):
|
||||||
# TODO(kota_): IMO, we need to make this to self._client environ to
|
# TODO(kota_): IMO, we need to make this to self._client environ to
|
||||||
# get rid of redundant instanciation
|
# get rid of redundant instantiation
|
||||||
return InternalClient(self.conf_file, 'SA', 1)
|
return InternalClient(self.conf_file, 'SA', 1)
|
||||||
|
|
||||||
def _get_object(self, container, obj, headers=None):
|
def _get_object(self, container, obj, headers=None):
|
||||||
@ -187,7 +187,7 @@ class StorletBaseHandler(object):
|
|||||||
"""
|
"""
|
||||||
:param request: swob.Request instance
|
:param request: swob.Request instance
|
||||||
:param conf: middleware conf dict
|
:param conf: middleware conf dict
|
||||||
:param gateway_conf: gatway conf dict
|
:param gateway_conf: gateway conf dict
|
||||||
:param app: wsgi Application
|
:param app: wsgi Application
|
||||||
:param logger: logger instance
|
:param logger: logger instance
|
||||||
"""
|
"""
|
||||||
@ -354,7 +354,7 @@ class StorletBaseHandler(object):
|
|||||||
|
|
||||||
def is_slo_response(self, resp):
|
def is_slo_response(self, resp):
|
||||||
"""
|
"""
|
||||||
Determins whether the response is a slo one
|
Determines whether the response is a slo one
|
||||||
|
|
||||||
:param resp: swob.Response instance
|
:param resp: swob.Response instance
|
||||||
:return: Whenther the response is a slo one
|
:return: Whenther the response is a slo one
|
||||||
@ -373,7 +373,7 @@ class StorletBaseHandler(object):
|
|||||||
|
|
||||||
def _update_storlet_parameters_from_headers(self):
|
def _update_storlet_parameters_from_headers(self):
|
||||||
"""
|
"""
|
||||||
Extract parameters for header (an alternative to parmeters through
|
Extract parameters for header (an alternative to parameters through
|
||||||
the query string)
|
the query string)
|
||||||
|
|
||||||
"""
|
"""
|
||||||
@ -442,7 +442,7 @@ class StorletBaseHandler(object):
|
|||||||
|
|
||||||
self._set_metadata_in_headers(new_headers, sresp.user_metadata)
|
self._set_metadata_in_headers(new_headers, sresp.user_metadata)
|
||||||
response = Response(headers=new_headers, app_iter=sresp.data_iter,
|
response = Response(headers=new_headers, app_iter=sresp.data_iter,
|
||||||
reuqest=self.request)
|
request=self.request)
|
||||||
except StorletRuntimeException:
|
except StorletRuntimeException:
|
||||||
response = HTTPServiceUnavailable()
|
response = HTTPServiceUnavailable()
|
||||||
|
|
||||||
@ -490,7 +490,7 @@ class StorletBaseHandler(object):
|
|||||||
Build a storlet_gatway.common.stob.StorletRequest (or its child)
|
Build a storlet_gatway.common.stob.StorletRequest (or its child)
|
||||||
instance for storlet invocation
|
instance for storlet invocation
|
||||||
|
|
||||||
:param req: an instane of swift.common.swob.Request
|
:param req: an instance of swift.common.swob.Request
|
||||||
:param sheaders: swift.common.swob.HeaderKeyDict instance which
|
:param sheaders: swift.common.swob.HeaderKeyDict instance which
|
||||||
includes object metadata information to be passed
|
includes object metadata information to be passed
|
||||||
to storlet daemon
|
to storlet daemon
|
||||||
|
@ -47,7 +47,7 @@ class StorletObjectHandler(StorletBaseHandler):
|
|||||||
|
|
||||||
def is_symlink_response(self, resp):
|
def is_symlink_response(self, resp):
|
||||||
"""
|
"""
|
||||||
Determins whether the response is a symlink one
|
Determines whether the response is a symlink one
|
||||||
|
|
||||||
:param resp: swob.Response instance
|
:param resp: swob.Response instance
|
||||||
:return: Whenther the response is a slo one
|
:return: Whenther the response is a slo one
|
||||||
@ -69,7 +69,7 @@ class StorletObjectHandler(StorletBaseHandler):
|
|||||||
options = super(StorletObjectHandler, self).\
|
options = super(StorletObjectHandler, self).\
|
||||||
_get_storlet_invocation_options(req)
|
_get_storlet_invocation_options(req)
|
||||||
|
|
||||||
# If the request is a storlet request with an simgle input range, we
|
# If the request is a storlet request with an single input range, we
|
||||||
# pass range parameters to storlet gateway, to realize range handling
|
# pass range parameters to storlet gateway, to realize range handling
|
||||||
# with keepling zero copy
|
# with keepling zero copy
|
||||||
if self.is_storlet_range_request and \
|
if self.is_storlet_range_request and \
|
||||||
|
@ -127,7 +127,7 @@ class StorletProxyHandler(StorletBaseHandler):
|
|||||||
|
|
||||||
def is_symlink_response(self, resp):
|
def is_symlink_response(self, resp):
|
||||||
"""
|
"""
|
||||||
Determins whether the response is a symlink one
|
Determines whether the response is a symlink one
|
||||||
|
|
||||||
:param resp: swob.Response instance
|
:param resp: swob.Response instance
|
||||||
:return: Whenther the response is a slo one
|
:return: Whenther the response is a slo one
|
||||||
@ -159,7 +159,7 @@ class StorletProxyHandler(StorletBaseHandler):
|
|||||||
|
|
||||||
def _validate_registration(self, req):
|
def _validate_registration(self, req):
|
||||||
"""
|
"""
|
||||||
Validate parameters about storlet/dependency object when registrating
|
Validate parameters about storlet/dependency object when registering
|
||||||
|
|
||||||
:params req: swob.Request instance
|
:params req: swob.Request instance
|
||||||
:raises ValueError: If some parameters are wrong
|
:raises ValueError: If some parameters are wrong
|
||||||
@ -190,7 +190,7 @@ class StorletProxyHandler(StorletBaseHandler):
|
|||||||
storlet ACL update
|
storlet ACL update
|
||||||
|
|
||||||
:params req: swob.Request instance
|
:params req: swob.Request instance
|
||||||
:return: the resulting acl string that hould be added
|
:return: the resulting acl string that should be added
|
||||||
:raises HTTPBadRequest: If a header is missing or mulformed
|
:raises HTTPBadRequest: If a header is missing or mulformed
|
||||||
"""
|
"""
|
||||||
# Make sure we are not meddling with the storlet containers
|
# Make sure we are not meddling with the storlet containers
|
||||||
@ -293,8 +293,8 @@ class StorletProxyHandler(StorletBaseHandler):
|
|||||||
|
|
||||||
def gather_extra_sources(self):
|
def gather_extra_sources(self):
|
||||||
# (kota_): I know this is a crazy hack to set the resp
|
# (kota_): I know this is a crazy hack to set the resp
|
||||||
# dinamically so that this is a temprorary way to make sure
|
# dynamically so that this is a temporary way to make sure
|
||||||
# the capability, this aboslutely needs cleanup more genelic
|
# the capability, this absolutely needs cleanup more genelic
|
||||||
if 'X-Storlet-Extra-Resources' in self.request.headers:
|
if 'X-Storlet-Extra-Resources' in self.request.headers:
|
||||||
try:
|
try:
|
||||||
resources = list_from_csv(
|
resources = list_from_csv(
|
||||||
|
@ -41,7 +41,7 @@ class StorletHandlerMiddleware(object):
|
|||||||
Generate Handler class based on execution_server parameter
|
Generate Handler class based on execution_server parameter
|
||||||
|
|
||||||
:param exec_server: Where this storlet_middleware is running.
|
:param exec_server: Where this storlet_middleware is running.
|
||||||
This should value shoud be 'proxy' or 'object'
|
This should value should be 'proxy' or 'object'
|
||||||
:raise ValueError: If exec_server is invalid
|
:raise ValueError: If exec_server is invalid
|
||||||
"""
|
"""
|
||||||
if exec_server == 'proxy':
|
if exec_server == 'proxy':
|
||||||
@ -80,7 +80,7 @@ class StorletHandlerMiddleware(object):
|
|||||||
self.logger.exception('Internal request timed out')
|
self.logger.exception('Internal request timed out')
|
||||||
raise HTTPInternalServerError(body='Internal request timed out')
|
raise HTTPInternalServerError(body='Internal request timed out')
|
||||||
except HTTPException:
|
except HTTPException:
|
||||||
# TODO(takashi): Shoud we generate this log for all error?
|
# TODO(takashi): Should we generate this log for all error?
|
||||||
# (ex. 404 when the object is not found)
|
# (ex. 404 when the object is not found)
|
||||||
self.logger.exception('Storlet execution failed')
|
self.logger.exception('Storlet execution failed')
|
||||||
raise
|
raise
|
||||||
|
@ -87,7 +87,7 @@ MESSAGES = {
|
|||||||
|
|
||||||
|
|
||||||
def usage():
|
def usage():
|
||||||
print("Useage: deploy_storlet.py <path to conf>")
|
print("Usage: deploy_storlet.py <path to conf>")
|
||||||
|
|
||||||
|
|
||||||
def main(argv):
|
def main(argv):
|
||||||
|
@ -39,7 +39,7 @@ class Response(object):
|
|||||||
Response object to return the object to ipython cell
|
Response object to return the object to ipython cell
|
||||||
|
|
||||||
:param status: int for status code
|
:param status: int for status code
|
||||||
:param headers: a dict for repsonse headers
|
:param headers: a dict for response headers
|
||||||
:param body_iter: an iterator object which takes the body content from
|
:param body_iter: an iterator object which takes the body content from
|
||||||
"""
|
"""
|
||||||
def __init__(self, status, headers, body_iter=None):
|
def __init__(self, status, headers, body_iter=None):
|
||||||
@ -190,7 +190,7 @@ class StorletMagics(Magics):
|
|||||||
)
|
)
|
||||||
@magic_arguments.argument(
|
@magic_arguments.argument(
|
||||||
'--print-result', action='store_true', default=False,
|
'--print-result', action='store_true', default=False,
|
||||||
help='Print result objet to stdout. Note that this may be a large'
|
help='Print result object to stdout. Note that this may be a large'
|
||||||
'binary depends on your app'
|
'binary depends on your app'
|
||||||
)
|
)
|
||||||
@cell_magic
|
@cell_magic
|
||||||
|
@ -55,7 +55,7 @@ class FakeStorletFileOut(FakeStorletFile):
|
|||||||
if self._metadata is not None:
|
if self._metadata is not None:
|
||||||
raise IOError('Sending metadata twice is not allowed')
|
raise IOError('Sending metadata twice is not allowed')
|
||||||
self._metadata = {}
|
self._metadata = {}
|
||||||
# expect the incomming metadata should be dict
|
# expect the incoming metadata should be dict
|
||||||
self._metadata.update(metadata)
|
self._metadata.update(metadata)
|
||||||
|
|
||||||
def read(self):
|
def read(self):
|
||||||
|
@ -45,7 +45,7 @@ def put_storlet_object(url, token, storlet, dependencies, storlet_main_class,
|
|||||||
|
|
||||||
:param url: swift endpoint url
|
:param url: swift endpoint url
|
||||||
:param token: token string to access to swift
|
:param token: token string to access to swift
|
||||||
:param storlet: storlet file to be registerd
|
:param storlet: storlet file to be registered
|
||||||
:param dependencies: a list of dependency files
|
:param dependencies: a list of dependency files
|
||||||
:param storlet_main_class: name of the storlet main class
|
:param storlet_main_class: name of the storlet main class
|
||||||
:param language: storlet language. default value is Java
|
:param language: storlet language. default value is Java
|
||||||
@ -86,7 +86,7 @@ def deploy_storlet(url, token, storlet, storlet_main_class, dependencies,
|
|||||||
|
|
||||||
:param url: swift endpoint url
|
:param url: swift endpoint url
|
||||||
:param token: token string to access swift
|
:param token: token string to access swift
|
||||||
:param storlet: storlet file to be registerd
|
:param storlet: storlet file to be registered
|
||||||
:param dependencies: a list of dependency files to be registered
|
:param dependencies: a list of dependency files to be registered
|
||||||
:param language: storlet language. default value is Java
|
:param language: storlet language. default value is Java
|
||||||
:param version: storlet language version. default is 3 for python
|
:param version: storlet language version. default is 3 for python
|
||||||
|
@ -65,7 +65,7 @@ class TestJupyterExcecution(unittest.TestCase):
|
|||||||
if 'text' in node:
|
if 'text' in node:
|
||||||
for line in node['text'].split('\n'):
|
for line in node['text'].split('\n'):
|
||||||
if line:
|
if line:
|
||||||
# NOTE: extract bytes lines appreared in the cell
|
# NOTE: extract bytes lines appeared in the cell
|
||||||
# in PY2, it's NOT needed but in PY3 it is required
|
# in PY2, it's NOT needed but in PY3 it is required
|
||||||
# because PY3 explicitly add 'b' prefix to bytes
|
# because PY3 explicitly add 'b' prefix to bytes
|
||||||
# objects
|
# objects
|
||||||
|
@ -139,7 +139,7 @@ class TestPartitionsIdentityStorlet(StorletJavaFunctionalTest):
|
|||||||
|
|
||||||
def test_first_range(self):
|
def test_first_range(self):
|
||||||
for max_record_line in range(5):
|
for max_record_line in range(5):
|
||||||
# The maximium line length in this section is 14
|
# The maximum line length in this section is 14
|
||||||
# includuing the new line. and so any max length >=14
|
# includuing the new line. and so any max length >=14
|
||||||
# should be good.
|
# should be good.
|
||||||
self._test_first_range(80)
|
self._test_first_range(80)
|
||||||
|
@ -114,7 +114,7 @@ def create_fake_sbus_class(scenario):
|
|||||||
"""
|
"""
|
||||||
class FakeSBus(object):
|
class FakeSBus(object):
|
||||||
"""
|
"""
|
||||||
Fake SBus communication according feeded scenario.
|
Fake SBus communication according fed scenario.
|
||||||
The scenario must be ordered by calls
|
The scenario must be ordered by calls
|
||||||
"""
|
"""
|
||||||
def __init__(self):
|
def __init__(self):
|
||||||
|
@ -477,7 +477,7 @@ class TestStorletDockerGateway(unittest.TestCase):
|
|||||||
# prepare nested mock patch
|
# prepare nested mock patch
|
||||||
# SBus -> mock SBus.send() for container communication
|
# SBus -> mock SBus.send() for container communication
|
||||||
# os.read -> mock reading the file descriptor from container
|
# os.read -> mock reading the file descriptor from container
|
||||||
# select.slect -> mock fd communication which can be readable
|
# select.select -> mock fd communication which can be readable
|
||||||
@mock.patch('storlets.gateway.gateways.docker.runtime.SBusClient')
|
@mock.patch('storlets.gateway.gateways.docker.runtime.SBusClient')
|
||||||
@mock.patch('storlets.gateway.gateways.docker.runtime.os.read',
|
@mock.patch('storlets.gateway.gateways.docker.runtime.os.read',
|
||||||
mock_read)
|
mock_read)
|
||||||
|
@ -222,7 +222,7 @@ class TestRuntimePaths(unittest.TestCase):
|
|||||||
self.assertEqual(
|
self.assertEqual(
|
||||||
'/var/lib/storlets/storlets/scopes/account/Storlet-1.0.jar',
|
'/var/lib/storlets/storlets/scopes/account/Storlet-1.0.jar',
|
||||||
runtime_paths.get_host_storlet_dir(storlet_id))
|
runtime_paths.get_host_storlet_dir(storlet_id))
|
||||||
# And this one is a mount poit in sand box?
|
# And this one is a mount point in sand box?
|
||||||
self.assertEqual('/home/swift/Storlet-1.0.jar',
|
self.assertEqual('/home/swift/Storlet-1.0.jar',
|
||||||
runtime_paths.get_sbox_storlet_dir(storlet_id))
|
runtime_paths.get_sbox_storlet_dir(storlet_id))
|
||||||
|
|
||||||
|
@ -202,7 +202,7 @@ class TestStorletObjectHandler(unittest.TestCase):
|
|||||||
handler = self.handler_class(
|
handler = self.handler_class(
|
||||||
req, self.conf, self.gateway_conf, mock.MagicMock(),
|
req, self.conf, self.gateway_conf, mock.MagicMock(),
|
||||||
mock.MagicMock())
|
mock.MagicMock())
|
||||||
# FIXME: stil hold api version 0 at ObjectHandler but will be
|
# FIXME: still hold api version 0 at ObjectHandler but will be
|
||||||
# deprecated if it's never used.
|
# deprecated if it's never used.
|
||||||
self.assertEqual('0', handler.api_version)
|
self.assertEqual('0', handler.api_version)
|
||||||
self.assertEqual('acc', handler.account)
|
self.assertEqual('acc', handler.account)
|
||||||
|
@ -95,7 +95,7 @@ class TestStorletMiddlewareProxy(BaseTestStorletMiddleware):
|
|||||||
self.assertEqual('HEAD', calls[0][0])
|
self.assertEqual('HEAD', calls[0][0])
|
||||||
self.assertEqual(storlet, calls[0][1])
|
self.assertEqual(storlet, calls[0][1])
|
||||||
|
|
||||||
# The last one is exexution GET call
|
# The last one is execution GET call
|
||||||
self.assertEqual(target, calls[-1][1])
|
self.assertEqual(target, calls[-1][1])
|
||||||
self.assertIn('X-Run-Storlet', calls[-1][2])
|
self.assertIn('X-Run-Storlet', calls[-1][2])
|
||||||
|
|
||||||
|
@ -17,7 +17,7 @@ import unittest
|
|||||||
|
|
||||||
|
|
||||||
class TestStorletHandlerMiddleware(unittest.TestCase):
|
class TestStorletHandlerMiddleware(unittest.TestCase):
|
||||||
# TODO(takashi): should implemente some unit test cases
|
# TODO(takashi): should implement some unit test cases
|
||||||
pass
|
pass
|
||||||
|
|
||||||
|
|
||||||
|
@ -44,7 +44,7 @@ class FakeConnection(object):
|
|||||||
return (self._fake_headers, resp_body)
|
return (self._fake_headers, resp_body)
|
||||||
|
|
||||||
# Those 3 methods are just for entry point difference from the caller
|
# Those 3 methods are just for entry point difference from the caller
|
||||||
# but all methods returns same response format with updateing response_dict
|
# but all methods returns same response format with updating response_dict
|
||||||
def get_object(self, *args, **kwargs):
|
def get_object(self, *args, **kwargs):
|
||||||
return self._return_fake_response(**kwargs)
|
return self._return_fake_response(**kwargs)
|
||||||
|
|
||||||
|
@ -23,7 +23,7 @@ class TestDeployStorlet(unittest.TestCase):
|
|||||||
# - no class found in the jar
|
# - no class found in the jar
|
||||||
# - not a class in the list
|
# - not a class in the list
|
||||||
# - fail to open the jar file
|
# - fail to open the jar file
|
||||||
# (e.g. no such a file or directry)
|
# (e.g. no such a file or directory)
|
||||||
def test_deploy_storlet_main_java(self):
|
def test_deploy_storlet_main_java(self):
|
||||||
class MockZipFile(mock.MagicMock):
|
class MockZipFile(mock.MagicMock):
|
||||||
def infolist(self):
|
def infolist(self):
|
||||||
|
9
tox.ini
9
tox.ini
@ -99,3 +99,12 @@ passenv =
|
|||||||
[testenv:releasenotes]
|
[testenv:releasenotes]
|
||||||
deps = {[testenv:docs]deps}
|
deps = {[testenv:docs]deps}
|
||||||
commands = sphinx-build -a -W -E -d releasenotes/build/doctrees -b html releasenotes/source releasenotes/build/html
|
commands = sphinx-build -a -W -E -d releasenotes/build/doctrees -b html releasenotes/source releasenotes/build/html
|
||||||
|
|
||||||
|
[testenv:codespell]
|
||||||
|
description =
|
||||||
|
Run codespell to check spelling
|
||||||
|
deps = codespell
|
||||||
|
# note(JayF): {posargs} lets us run `tox -ecodespell -- -w` to get codespell
|
||||||
|
# to correct spelling issues in our code it's aware of.
|
||||||
|
commands =
|
||||||
|
codespell {posargs}
|
||||||
|
Loading…
Reference in New Issue
Block a user