Add a SparkWrapper class
This class takes the name of an XML file to add to the default Hadoop configuration for a Spark job before invoking the job's main class. This class is based on the MainWrapper used for Oozie Java actions but is simpler. There is no need to modify the SecurityManager, and the name of the file may be anything. This class is broken out separately so that the wrappers can vary independently in separate JARs. Change-Id: I5f34fcf525f4f16b095d52fab5059b9be59896f7 Partial-Implements: blueprint edp-spark-swift-integration
This commit is contained in:
parent
13658d970b
commit
d4b2f376e4
|
@ -0,0 +1,14 @@
|
|||
===========================================
|
||||
Sources for main function wrapper for Spark
|
||||
===========================================
|
||||
|
||||
The Hadoop configuration for a Spark job must be modified if
|
||||
the Spark job is going to access Swift paths. Specifically,
|
||||
the Hadoop configuration must contain values that allow
|
||||
the job to authenticate to the Swift service.
|
||||
|
||||
This wrapper adds a specified xml file to the default Hadoop
|
||||
Configuration resource list and then calls the specified
|
||||
main class. Any necessary Hadoop configuration values can
|
||||
be added to the xml file. This allows the main class to
|
||||
be run and access Swift paths without alteration.
|
|
@ -0,0 +1,54 @@
|
|||
<?xml version="1.0" encoding="UTF-8"?>
|
||||
<!--
|
||||
Licensed under the Apache License, Version 2.0 (the "License");
|
||||
you may not use this file except in compliance with the License.
|
||||
You may obtain a copy of the License at
|
||||
|
||||
http://www.apache.org/licenses/LICENSE-2.0
|
||||
|
||||
Unless required by applicable law or agreed to in writing, software
|
||||
distributed under the License is distributed on an "AS IS" BASIS,
|
||||
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
See the License for the specific language governing permissions and
|
||||
limitations under the License. See accompanying LICENSE file.
|
||||
-->
|
||||
<project xmlns="http://maven.apache.org/POM/4.0.0"
|
||||
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
|
||||
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/maven-v4_0_0.xsd">
|
||||
<modelVersion>4.0.0</modelVersion>
|
||||
|
||||
<groupId>org.openstack.sahara.edp</groupId>
|
||||
<artifactId>edp-spark-wrapper</artifactId>
|
||||
<version>1.0.0-SNAPSHOT</version>
|
||||
<name>EDP Wrapper for Spark</name>
|
||||
<packaging>jar</packaging>
|
||||
|
||||
<properties>
|
||||
<file.encoding>UTF-8</file.encoding>
|
||||
<downloadSources>true</downloadSources>
|
||||
<project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
|
||||
</properties>
|
||||
|
||||
<build>
|
||||
<plugins>
|
||||
<plugin>
|
||||
<groupId>org.apache.maven.plugins</groupId>
|
||||
<artifactId>maven-compiler-plugin</artifactId>
|
||||
<configuration>
|
||||
<source>1.6</source>
|
||||
<target>1.6</target>
|
||||
</configuration>
|
||||
</plugin>
|
||||
<plugin>
|
||||
<groupId>org.apache.maven.plugins</groupId>
|
||||
<artifactId>maven-checkstyle-plugin</artifactId>
|
||||
<configuration>
|
||||
<configLocation>file://${basedir}/../hadoop-swiftfs/checkstyle.xml</configLocation>
|
||||
<failOnViolation>false</failOnViolation>
|
||||
<format>xml</format>
|
||||
<format>html</format>
|
||||
</configuration>
|
||||
</plugin>
|
||||
</plugins>
|
||||
</build>
|
||||
</project>
|
|
@ -0,0 +1,22 @@
|
|||
package org.openstack.sahara.edp;
|
||||
|
||||
import java.lang.reflect.InvocationTargetException;
|
||||
import java.lang.reflect.Method;
|
||||
import java.security.Permission;
|
||||
import java.util.Arrays;
|
||||
|
||||
public class SparkWrapper {
|
||||
|
||||
public static void main(String[] args) throws Throwable {
|
||||
|
||||
Class<?> configClass
|
||||
= Class.forName("org.apache.hadoop.conf.Configuration");
|
||||
Method method = configClass.getMethod("addDefaultResource", String.class);
|
||||
method.invoke(null, args[0]);
|
||||
|
||||
Class<?> mainClass = Class.forName(args[1]);
|
||||
Method mainMethod = mainClass.getMethod("main", String[].class);
|
||||
String[] newArgs = Arrays.copyOfRange(args, 2, args.length);
|
||||
mainMethod.invoke(null, (Object) newArgs);
|
||||
}
|
||||
}
|
Loading…
Reference in New Issue