From 1e04f8653188e4b1d6d476082ec3e44fe9dadffb Mon Sep 17 00:00:00 2001 From: Ethan Gafford Date: Thu, 4 Jun 2015 13:22:04 -0400 Subject: [PATCH] Modifying Swift Paths for EDP Examples To make things more convenient for users of the JSON EDP examples, doc and JSON payloads have been modified to use the same tree structure as the edp-examples themselves, allowing one upload to Swift. Change-Id: I114a6e93cb0194eab72edb065e712b9976c6b60e --- .../json-api-examples/v1.1/README.rst | 37 +++++++++++++++---- .../data-sources/create.swift-pig-input.json | 2 +- .../data-sources/create.swift-pig-output.json | 2 +- .../v1.1/job-binaries/create.pig-job.json | 2 +- .../v1.1/job-binaries/create.pig-udf.json | 2 +- 5 files changed, 33 insertions(+), 12 deletions(-) diff --git a/etc/edp-examples/json-api-examples/v1.1/README.rst b/etc/edp-examples/json-api-examples/v1.1/README.rst index 7e7ff7ee..35462722 100644 --- a/etc/edp-examples/json-api-examples/v1.1/README.rst +++ b/etc/edp-examples/json-api-examples/v1.1/README.rst @@ -48,6 +48,15 @@ When the preconditions for a given example specify that you must have "an active Hadoop cluster", that cluster must be running an Oozie process in all cases, as sahara's EDP jobs are scheduled through Oozie in all Hadoop plugins. +Swift Objects +------------- + +Several of the examples here call for objects in Swift. To upload all of the +required files for all examples, use the following: + + $ cd etc/edp-examples + $ swift upload edp-examples ./* + Swift credentials ----------------- @@ -58,6 +67,17 @@ store credentials. .. _swift proxy: http://docs.openstack.org/developer/sahara/userdoc/advanced.configuration.guide.html +Swift containers +---------------- + +You may note that the container references in the data source creation JSON +examples for Swift have the ``.sahara`` suffix on the container name, even +though the data must be uploaded to the non-suffixed container. This suffix +informs Hadoop that ``sahara`` is the provider for this data source. See the +`hadoop swift`_ documentation for more information. + +.. _hadoop swift: http://docs.openstack.org/developer/sahara/userdoc/hadoop-swift.html + REST API usage -------------- @@ -71,6 +91,7 @@ will be useful throughout these exercises. .. _Sahara EDP API Documentation: http://developer.openstack.org/api-ref-data-processing-v1.1.html + Example 1: Pig, using swift =========================== @@ -85,11 +106,11 @@ This example assumes the following: container ``edp-examples``, as follows: * The file at ``edp-examples/edp-pig/trim-spaces/example.pig`` is stored - at path ``pig-job/example.pig``. - * The file at ``edp-examples/edp-pig/trim-spaces/udf.jar`` is stored at - path ``pig-job/udf.jar``. + at path ``swift://edp-examples/edp-pig/trim-spaces/example.pig``. + * The file at ``edp-pig/trim-spaces/udf.jar`` is stored at + path ``swift://edp-examples/edp-pig/trim-spaces/udf.jar``. * The file at ``edp-examples/edp-pig/trim-spaces/data/input`` is stored at - path ``pig-job/data/input``. + path ``swift://edp-examples/edp-pig/trim-spaces/data/input``. Steps ----- @@ -132,8 +153,8 @@ This example assumes the following: master node's HDFS available at URL ``hdfs://hadoop-cluster-master-001:8020/``. 3. In the demo user's project, the file at - ``edp-examples/edp-mapreduce/edp-mapreduce.jar`` is stored in swift in the - container ``edp-examples``, at path ``edp-mapreduce/edp-mapreduce.jar``. + ``edp-examples/edp-mapreduce/edp-mapreduce.jar`` is stored in swift, at + path ``swift://edp-examples/edp-mapreduce/edp-mapreduce.jar``. 4. A text file exists in your Hadoop cluster's HDFS at path ``/user/edp-examples/edp-map-reduce/input``. @@ -200,8 +221,8 @@ This example assumes the following: 1. Usage of an OpenStack user named "demo", with password "password". 2. An active Spark cluster exists in the demo user's project. 3. In the demo user's project, the file at - ``edp-examples/edp-spark/spark-example.jar`` is stored in swift in the - container ``edp-examples``, at path ``edp-spark/spark-example.jar``. + ``edp-examples/edp-spark/spark-example.jar`` is stored in swift, at path + ``swift://edp-examples/edp-spark/spark-example.jar``. Steps ----- diff --git a/etc/edp-examples/json-api-examples/v1.1/data-sources/create.swift-pig-input.json b/etc/edp-examples/json-api-examples/v1.1/data-sources/create.swift-pig-input.json index 72fb796d..cbfe4891 100644 --- a/etc/edp-examples/json-api-examples/v1.1/data-sources/create.swift-pig-input.json +++ b/etc/edp-examples/json-api-examples/v1.1/data-sources/create.swift-pig-input.json @@ -2,7 +2,7 @@ "name": "demo-pig-input", "description": "A data source for Pig input, stored in Swift", "type": "swift", - "url": "swift://edp-examples.sahara/pig-job/data/input", + "url": "swift://edp-examples.sahara/edp-pig/trim-spaces/data/input", "credentials": { "user": "demo", "password": "password" diff --git a/etc/edp-examples/json-api-examples/v1.1/data-sources/create.swift-pig-output.json b/etc/edp-examples/json-api-examples/v1.1/data-sources/create.swift-pig-output.json index d48095e5..4430a4b6 100644 --- a/etc/edp-examples/json-api-examples/v1.1/data-sources/create.swift-pig-output.json +++ b/etc/edp-examples/json-api-examples/v1.1/data-sources/create.swift-pig-output.json @@ -2,7 +2,7 @@ "name": "demo-pig-output", "description": "A data source for Pig output, stored in Swift", "type": "swift", - "url": "swift://edp-examples.sahara/pig-job/data/output", + "url": "swift://edp-examples.sahara/edp-pig/trim-spaces/data/output", "credentials": { "user": "demo", "password": "password" diff --git a/etc/edp-examples/json-api-examples/v1.1/job-binaries/create.pig-job.json b/etc/edp-examples/json-api-examples/v1.1/job-binaries/create.pig-job.json index af3c0529..30cfafe5 100644 --- a/etc/edp-examples/json-api-examples/v1.1/job-binaries/create.pig-job.json +++ b/etc/edp-examples/json-api-examples/v1.1/job-binaries/create.pig-job.json @@ -1,7 +1,7 @@ { "name": "example.pig", "description": "An example pig script", - "url": "swift://edp-examples/pig-job/example.pig", + "url": "swift://edp-examples/edp-pig/trim-spaces/example.pig", "extra": { "user": "demo", "password": "password" diff --git a/etc/edp-examples/json-api-examples/v1.1/job-binaries/create.pig-udf.json b/etc/edp-examples/json-api-examples/v1.1/job-binaries/create.pig-udf.json index 689900fe..93ad3085 100644 --- a/etc/edp-examples/json-api-examples/v1.1/job-binaries/create.pig-udf.json +++ b/etc/edp-examples/json-api-examples/v1.1/job-binaries/create.pig-udf.json @@ -1,7 +1,7 @@ { "name": "udf.jar", "description": "An example pig UDF library", - "url": "swift://edp-examples/pig-job/udf.jar", + "url": "swift://edp-examples/edp-pig/trim-spaces/udf.jar", "extra": { "user": "demo", "password": "password"