Add a SparkWrapper class

This class takes the name of an XML file to add to the default Hadoop
configuration for a Spark job before invoking the job's main class.

This class is based on the MainWrapper used for Oozie Java actions
but is simpler. There is no need to modify the SecurityManager, and
the name of the file may be anything. This class is broken out
separately so that the wrappers can vary independently in
separate JARs.

Change-Id: I5f34fcf525f4f16b095d52fab5059b9be59896f7
Partial-Implements: blueprint edp-spark-swift-integration
This commit is contained in:
Trevor McKay 2015-01-16 14:11:35 -05:00
parent 13658d970b
commit d4b2f376e4
3 changed files with 90 additions and 0 deletions

View File

@ -0,0 +1,14 @@
===========================================
Sources for main function wrapper for Spark
===========================================
The Hadoop configuration for a Spark job must be modified if
the Spark job is going to access Swift paths. Specifically,
the Hadoop configuration must contain values that allow
the job to authenticate to the Swift service.
This wrapper adds a specified xml file to the default Hadoop
Configuration resource list and then calls the specified
main class. Any necessary Hadoop configuration values can
be added to the xml file. This allows the main class to
be run and access Swift paths without alteration.

View File

@ -0,0 +1,54 @@
<?xml version="1.0" encoding="UTF-8"?>
<!--
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License. See accompanying LICENSE file.
-->
<project xmlns="http://maven.apache.org/POM/4.0.0"
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/maven-v4_0_0.xsd">
<modelVersion>4.0.0</modelVersion>
<groupId>org.openstack.sahara.edp</groupId>
<artifactId>edp-spark-wrapper</artifactId>
<version>1.0.0-SNAPSHOT</version>
<name>EDP Wrapper for Spark</name>
<packaging>jar</packaging>
<properties>
<file.encoding>UTF-8</file.encoding>
<downloadSources>true</downloadSources>
<project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
</properties>
<build>
<plugins>
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-compiler-plugin</artifactId>
<configuration>
<source>1.6</source>
<target>1.6</target>
</configuration>
</plugin>
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-checkstyle-plugin</artifactId>
<configuration>
<configLocation>file://${basedir}/../hadoop-swiftfs/checkstyle.xml</configLocation>
<failOnViolation>false</failOnViolation>
<format>xml</format>
<format>html</format>
</configuration>
</plugin>
</plugins>
</build>
</project>

View File

@ -0,0 +1,22 @@
package org.openstack.sahara.edp;
import java.lang.reflect.InvocationTargetException;
import java.lang.reflect.Method;
import java.security.Permission;
import java.util.Arrays;
public class SparkWrapper {
public static void main(String[] args) throws Throwable {
Class<?> configClass
= Class.forName("org.apache.hadoop.conf.Configuration");
Method method = configClass.getMethod("addDefaultResource", String.class);
method.invoke(null, args[0]);
Class<?> mainClass = Class.forName(args[1]);
Method mainMethod = mainClass.getMethod("main", String[].class);
String[] newArgs = Arrays.copyOfRange(args, 2, args.length);
mainMethod.invoke(null, (Object) newArgs);
}
}