added storlet for GZIP compression / decompression
Change-Id: Ib949e426ad97a08dfc415bf731d6893ac7391065
This commit is contained in:
parent
7c09cc2f81
commit
0897780292
|
@ -0,0 +1,47 @@
|
|||
<!--
|
||||
Copyright IBM Corp. 2015, 2015 All Rights Reserved
|
||||
Licensed under the Apache License, Version 2.0 (the "License");
|
||||
you may not use this file except in compliance with the License.
|
||||
You may obtain a copy of the License at
|
||||
|
||||
http://www.apache.org/licenses/LICENSE-2.0
|
||||
|
||||
Unless required by applicable law or agreed to in writing, software
|
||||
distributed under the License is distributed on an "AS IS" BASIS,
|
||||
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
See the License for the specific language governing permissions and
|
||||
Limitations under the License.
|
||||
-->
|
||||
|
||||
<project>
|
||||
|
||||
<target name="clean">
|
||||
<delete dir="bin" />
|
||||
</target>
|
||||
|
||||
<target name="java">
|
||||
<mkdir dir="bin" />
|
||||
<javac srcdir="src" destdir="bin"
|
||||
classpath="../../Engine/SCommon/bin/SCommon.jar"
|
||||
includeantruntime="false" />
|
||||
</target>
|
||||
|
||||
<target name="text">
|
||||
<exec dir="bin" executable="wget">
|
||||
<arg line="https://github.com/openstack/storlets -O input.txt" />
|
||||
</exec>
|
||||
</target>
|
||||
|
||||
<target name="jar" depends="java">
|
||||
<jar destfile="compressstorlet-1.0.jar" basedir="bin">
|
||||
<manifest>
|
||||
<attribute name="Main-Class"
|
||||
value="com.ibm.storlet.IdentityStorlet" />
|
||||
</manifest>
|
||||
</jar>
|
||||
<move file="compressstorlet-1.0.jar" todir="bin" />
|
||||
</target>
|
||||
|
||||
<target name="build" depends="clean, jar, text">
|
||||
</target>
|
||||
</project>
|
|
@ -0,0 +1,96 @@
|
|||
/*----------------------------------------------------------------------------
|
||||
* Copyright IBM Corp. 2015, 2015 All Rights Reserved
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* Limitations under the License.
|
||||
* ---------------------------------------------------------------------------
|
||||
*/
|
||||
|
||||
/*============================================================================
|
||||
07-Jan-2016 cdoron Initial implementation.
|
||||
===========================================================================*/
|
||||
|
||||
package com.ibm.storlet.compress;
|
||||
|
||||
import java.io.IOException;
|
||||
import java.util.ArrayList;
|
||||
import java.util.HashMap;
|
||||
import java.util.Map;
|
||||
import java.io.InputStream;
|
||||
import java.io.OutputStream;
|
||||
import java.util.zip.GZIPOutputStream;
|
||||
import java.util.zip.GZIPInputStream;
|
||||
|
||||
import com.ibm.storlet.common.IStorlet;
|
||||
import com.ibm.storlet.common.StorletException;
|
||||
import com.ibm.storlet.common.StorletInputStream;
|
||||
import com.ibm.storlet.common.StorletLogger;
|
||||
import com.ibm.storlet.common.StorletObjectOutputStream;
|
||||
import com.ibm.storlet.common.StorletContainerHandle;
|
||||
import com.ibm.storlet.common.StorletOutputStream;
|
||||
import com.ibm.storlet.common.StorletUtils;
|
||||
|
||||
public class CompressStorlet implements IStorlet {
|
||||
@Override
|
||||
public void invoke(ArrayList<StorletInputStream> inputStreams,
|
||||
ArrayList<StorletOutputStream> outputStreams,
|
||||
Map<String, String> parameters, StorletLogger log)
|
||||
throws StorletException {
|
||||
log.emitLog("CompressStorlet Invoked");
|
||||
|
||||
StorletInputStream sis = inputStreams.get(0);
|
||||
InputStream is = sis.getStream();
|
||||
HashMap<String, String> metadata = sis.getMetadata();
|
||||
|
||||
final int COMPRESS = 0;
|
||||
final int UNCOMPRESS = 1;
|
||||
|
||||
int action = COMPRESS;
|
||||
/*
|
||||
* Get optional action flag
|
||||
*/
|
||||
String action_str = parameters.get("action");
|
||||
if (action_str != null && action_str.equals("uncompress"))
|
||||
{
|
||||
action = UNCOMPRESS;
|
||||
}
|
||||
|
||||
StorletObjectOutputStream storletObjectOutputStream = (StorletObjectOutputStream)outputStreams.get(0);
|
||||
storletObjectOutputStream.setMetadata(metadata);
|
||||
OutputStream outputStream = storletObjectOutputStream.getStream();
|
||||
try {
|
||||
byte[] buffer = new byte[65536];
|
||||
int len;
|
||||
if (action == COMPRESS) {
|
||||
GZIPOutputStream gzipOS = new GZIPOutputStream(outputStream);
|
||||
while((len=is.read(buffer)) != -1) {
|
||||
gzipOS.write(buffer, 0, len);
|
||||
}
|
||||
gzipOS.close();
|
||||
} else {
|
||||
GZIPInputStream gzipIS = new GZIPInputStream(is);
|
||||
while((len = gzipIS.read(buffer)) != -1) {
|
||||
outputStream.write(buffer, 0, len);
|
||||
}
|
||||
gzipIS.close();
|
||||
}
|
||||
} catch (IOException e) {
|
||||
log.emitLog("CompressExample - raised IOException: " + e.getMessage());
|
||||
} finally {
|
||||
try {
|
||||
is.close();
|
||||
outputStream.close();
|
||||
} catch (IOException e) {
|
||||
}
|
||||
}
|
||||
log.emitLog("CompressStorlet Invocation done");
|
||||
}
|
||||
}
|
|
@ -72,6 +72,7 @@
|
|||
<fileset dir="StorletSamples/IdentityStorlet" includes="build.xml" />
|
||||
<fileset dir="StorletSamples/TestMetadataStorlet" includes="build.xml" />
|
||||
<fileset dir="StorletSamples/HalfStorlet" includes="build.xml" />
|
||||
<fileset dir="StorletSamples/CompressStorlet" includes="build.xml" />
|
||||
</subant>
|
||||
</sequential>
|
||||
</macrodef>
|
||||
|
|
|
@ -0,0 +1,67 @@
|
|||
'''-------------------------------------------------------------------------
|
||||
Copyright IBM Corp. 2015, 2015 All Rights Reserved
|
||||
Licensed under the Apache License, Version 2.0 (the "License");
|
||||
you may not use this file except in compliance with the License.
|
||||
You may obtain a copy of the License at
|
||||
|
||||
http://www.apache.org/licenses/LICENSE-2.0
|
||||
|
||||
Unless required by applicable law or agreed to in writing, software
|
||||
distributed under the License is distributed on an "AS IS" BASIS,
|
||||
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
See the License for the specific language governing permissions and
|
||||
Limitations under the License.
|
||||
-------------------------------------------------------------------------'''
|
||||
|
||||
from swiftclient import client as c
|
||||
from __init__ import StorletFunctionalTest
|
||||
|
||||
|
||||
class TestCompressStorlet(StorletFunctionalTest):
|
||||
def setUp(self):
|
||||
self.storlet_dir = 'CompressStorlet'
|
||||
self.storlet_name = 'compressstorlet-1.0.jar'
|
||||
self.storlet_main = 'com.ibm.storlet.compress.CompressStorlet'
|
||||
self.storlet_log = ''
|
||||
self.headers = {}
|
||||
self.storlet_file = 'input.txt'
|
||||
self.container = 'myobjects'
|
||||
self.dep_names = []
|
||||
super(TestCompressStorlet, self).setUp()
|
||||
|
||||
def test_put(self):
|
||||
headers = {'X-Run-Storlet': self.storlet_name}
|
||||
querystring = "action=compress"
|
||||
|
||||
with open('../../StorletSamples/CompressStorlet/bin/input.txt',
|
||||
'r') as myfile:
|
||||
data = myfile.read()
|
||||
|
||||
response = dict()
|
||||
c.put_object(self.url, self.token, self.container, self.storlet_file,
|
||||
data, None, None, None,
|
||||
"application/octet-stream", headers, None, None,
|
||||
querystring, response)
|
||||
|
||||
querystring = "action=uncompress"
|
||||
|
||||
original_headers, original_content = \
|
||||
c.get_object(self.url, self.token, self.container,
|
||||
self.storlet_file,
|
||||
response_dict=dict())
|
||||
|
||||
object_length = int(original_headers['content-length'])
|
||||
|
||||
self.assertTrue(object_length < len(data))
|
||||
|
||||
processed_headers, returned_content = \
|
||||
c.get_object(self.url, self.token, self.container,
|
||||
self.storlet_file,
|
||||
query_string=querystring, response_dict=dict(),
|
||||
headers=headers, resp_chunk_size=object_length)
|
||||
processed_content = ''
|
||||
for chunk in returned_content:
|
||||
if chunk:
|
||||
processed_content += chunk
|
||||
|
||||
self.assertEqual(data, processed_content)
|
Loading…
Reference in New Issue