Move default templates to framework repo

For packaging we need move default templates to scenario folder

Change-Id: Id2913ff6de09977b4afaf8d779620b0a438eb5a1

etc/scenario/gate/edp.yaml.mako

@@ -3,13 +3,13 @@ edp_jobs_flow:
     - type: Pig
       input_datasource:
         type: swift
-        source: etc/edp-examples/edp-pig/trim-spaces/data/input
+        source: sahara_tests/scenario/defaults/edp-examples/edp-pig/trim-spaces/data/input
       output_datasource:
         type: hdfs
         destination: /user/hadoop/edp-output
       main_lib:
         type: swift
-        source: etc/edp-examples/edp-pig/trim-spaces/example.pig
+        source: sahara_tests/scenario/defaults/edp-examples/edp-pig/trim-spaces/example.pig
       additional_libs:
         - type: swift
-          source: etc/edp-examples/edp-pig/trim-spaces/udf.jar
+          source: sahara_tests/scenario/defaults/edp-examples/edp-pig/trim-spaces/udf.jar

sahara_tests/scenario/defaults/ambari-2.3.yaml.mako

@@ -48,7 +48,7 @@ clusters:
           - type: Spark
             main_lib:
               type: database
-              source: etc/edp-examples/edp-spark/spark-kafka-example.jar
+              source: sahara_tests/scenario/defaults/edp-examples/edp-spark/spark-kafka-example.jar
             args:
               - '{zookeeper_list}'
               - '{topic}'

sahara_tests/scenario/defaults/edp-examples/json-api-examples/v1.1/README.rst

@@ -54,7 +54,7 @@ Swift Objects
 Several of the examples here call for objects in Swift. To upload all of the
 required files for all examples, use the following:
 
-    $ cd etc/edp-examples
+    $ cd sahara_tests/scenario/defaults/edp-examples
     $ swift upload edp-examples ./*
 
 Swift credentials

sahara_tests/scenario/defaults/edp.yaml.mako

@@ -3,27 +3,27 @@ edp_jobs_flow:
     - type: Pig
       input_datasource:
         type: swift
-        source: etc/edp-examples/edp-pig/trim-spaces/data/input
+        source: sahara_tests/scenario/defaults/edp-examples/edp-pig/trim-spaces/data/input
       output_datasource:
         type: swift
         destination: edp-output
       main_lib:
         type: swift
-        source: etc/edp-examples/edp-pig/trim-spaces/example.pig
+        source: sahara_tests/scenario/defaults/edp-examples/edp-pig/trim-spaces/example.pig
       additional_libs:
         - type: swift
-          source: etc/edp-examples/edp-pig/trim-spaces/udf.jar
+          source: sahara_tests/scenario/defaults/edp-examples/edp-pig/trim-spaces/udf.jar
   mapreduce_job:
     - type: MapReduce
       input_datasource:
         type: swift
-        source: etc/edp-examples/edp-pig/trim-spaces/data/input
+        source: sahara_tests/scenario/defaults/edp-examples/edp-pig/trim-spaces/data/input
       output_datasource:
         type: hdfs
         destination: /user/hadoop/edp-output
       additional_libs:
         - type: database
-          source:  etc/edp-examples/edp-mapreduce/edp-mapreduce.jar
+          source:  sahara_tests/scenario/defaults/edp-examples/edp-mapreduce/edp-mapreduce.jar
       configs:
         mapred.map.class: org.apache.oozie.example.SampleMapper
         mapred.reduce.class: org.apache.oozie.example.SampleReducer
@@ -32,7 +32,7 @@ edp_jobs_flow:
     - type: MapReduce.Streaming
       input_datasource:
         type: swift
-        source: etc/edp-examples/edp-pig/trim-spaces/data/input
+        source: sahara_tests/scenario/defaults/edp-examples/edp-pig/trim-spaces/data/input
       output_datasource:
         type: hdfs
         destination: /user/hadoop/edp-output
@@ -43,7 +43,7 @@ edp_jobs_flow:
     - type: Java
       additional_libs:
         - type: database
-          source: etc/edp-examples/hadoop2/edp-java/hadoop-mapreduce-examples-2.6.0.jar
+          source: sahara_tests/scenario/defaults/edp-examples/hadoop2/edp-java/hadoop-mapreduce-examples-2.6.0.jar
       configs:
        edp.java.main_class: org.apache.hadoop.examples.QuasiMonteCarlo
       args:
@@ -53,11 +53,11 @@ edp_jobs_flow:
     - type: Hive
       main_lib:
         type: swift
-        source: etc/edp-examples/edp-hive/script.q
+        source: sahara_tests/scenario/defaults/edp-examples/edp-hive/script.q
       input_datasource:
         type: hdfs
         hdfs_username: hadoop
-        source: etc/edp-examples/edp-hive/input.csv
+        source: sahara_tests/scenario/defaults/edp-examples/edp-hive/input.csv
       output_datasource:
         type: hdfs
         destination: /user/edp-output
@@ -65,7 +65,7 @@ edp_jobs_flow:
     - type: Spark
       main_lib:
         type: database
-        source: etc/edp-examples/edp-spark/spark-example.jar
+        source: sahara_tests/scenario/defaults/edp-examples/edp-spark/spark-example.jar
       configs:
         edp.java.main_class: org.apache.spark.examples.SparkPi
       args:
@@ -74,10 +74,10 @@ edp_jobs_flow:
     - type: Spark
       input_datasource:
         type: swift
-        source: etc/edp-examples/edp-spark/sample_input.txt
+        source: sahara_tests/scenario/defaults/edp-examples/edp-spark/sample_input.txt
       main_lib:
         type: database
-        source: etc/edp-examples/edp-spark/spark-wordcount.jar
+        source: sahara_tests/scenario/defaults/edp-examples/edp-spark/spark-wordcount.jar
       configs:
         edp.java.main_class: sahara.edp.spark.SparkWordCount
         edp.spark.adapt_for_swift: true
@@ -89,34 +89,34 @@ edp_jobs_flow:
     - type: Pig
       input_datasource:
         type: maprfs
-        source: etc/edp-examples/edp-pig/trim-spaces/data/input
+        source: sahara_tests/scenario/defaults/edp-examples/edp-pig/trim-spaces/data/input
       output_datasource:
         type: maprfs
         destination: /user/hadoop/edp-output
       main_lib:
         type: swift
-        source: etc/edp-examples/edp-pig/trim-spaces/example.pig
+        source: sahara_tests/scenario/defaults/edp-examples/edp-pig/trim-spaces/example.pig
       additional_libs:
         - type: swift
-          source: etc/edp-examples/edp-pig/trim-spaces/udf.jar
+          source: sahara_tests/scenario/defaults/edp-examples/edp-pig/trim-spaces/udf.jar
   mapr:
     - type: MapReduce
       input_datasource:
         type: maprfs
-        source: etc/edp-examples/edp-pig/trim-spaces/data/input
+        source: sahara_tests/scenario/defaults/edp-examples/edp-pig/trim-spaces/data/input
       output_datasource:
         type: maprfs
         destination: /user/hadoop/edp-output
       additional_libs:
         - type: database
-          source:  etc/edp-examples/edp-mapreduce/edp-mapreduce.jar
+          source:  sahara_tests/scenario/defaults/edp-examples/edp-mapreduce/edp-mapreduce.jar
       configs:
         mapred.mapper.class: org.apache.oozie.example.SampleMapper
         mapred.reducer.class: org.apache.oozie.example.SampleReducer
     - type: MapReduce.Streaming
       input_datasource:
         type: maprfs
-        source: etc/edp-examples/edp-pig/trim-spaces/data/input
+        source: sahara_tests/scenario/defaults/edp-examples/edp-pig/trim-spaces/data/input
       output_datasource:
         type: maprfs
         destination: /user/hadoop/edp-output
@@ -126,7 +126,7 @@ edp_jobs_flow:
     - type: Java
       additional_libs:
         - type: database
-          source: etc/edp-examples/hadoop2/edp-java/hadoop-mapreduce-examples-2.6.0.jar
+          source: sahara_tests/scenario/defaults/edp-examples/hadoop2/edp-java/hadoop-mapreduce-examples-2.6.0.jar
       configs:
        edp.java.main_class: org.apache.hadoop.examples.QuasiMonteCarlo
       args:

sahara_tests/scenario/defaults/liberty/ambari-2.3.yaml.mako

@@ -48,7 +48,7 @@ clusters:
           - type: Spark
             main_lib:
               type: database
-              source: etc/edp-examples/edp-spark/spark-kafka-example.jar
+              source: sahara_tests/scenario/defaults/edp-examples/edp-spark/spark-kafka-example.jar
             args:
               - '{zookeeper_list}'
               - '{topic}'

sahara_tests/scenario/runner.py

@@ -31,7 +31,7 @@ import yaml
 from sahara_tests.scenario import validation
 
 
-TEST_TEMPLATE_DIR = 'etc/scenario/defaults/'
+TEST_TEMPLATE_DIR = 'sahara_tests/scenario/defaults/'
 DEFAULT_TEMPLATE_VARS = [os.path.join(TEST_TEMPLATE_DIR,
                                       'credentials.yaml.mako'),
                          os.path.join(TEST_TEMPLATE_DIR,

sahara_tests/unit/scenario/test_base.py

@@ -86,7 +86,8 @@ class TestBase(testtools.TestCase):
                                               'data-processing-local',
                                           'sahara_url':
                                               'http://sahara_host:8386/v1.1',
-                                          'ssl_cert': '/etc/tests/cert.crt',
+                                          'ssl_cert': 'sahara_tests/scenario/'
+                                          'defaults/tests/cert.crt',
                                           'ssl_verify': True}
         self.base_scenario.plugin_opts = {'plugin_name': 'vanilla',
                                           'hadoop_version': '2.7.1'}
@@ -127,7 +128,8 @@ class TestBase(testtools.TestCase):
                         "type": "Pig",
                         "input_datasource": {
                             "type": "swift",
-                            "source": "etc/edp-examples/edp-pig/"
+                            "source": "sahara_tests/scenario/defaults/"
+                                      "edp-examples/edp-pig/"
                                       "top-todoers/data/input"
                         },
                         "output_datasource": {
@@ -136,7 +138,8 @@ class TestBase(testtools.TestCase):
                         },
                         "main_lib": {
                             "type": "swift",
-                            "source": "etc/edp-examples/edp-pig/"
+                            "source": "sahara_tests/scenario/defaults/"
+                                      "edp-examples/edp-pig/"
                                       "top-todoers/example.pig"
                         }
                     }
@@ -176,13 +179,10 @@ class TestBase(testtools.TestCase):
                                   session=fake_session,
                                   service_type='data-processing-local',
                                   sahara_url='http://sahara_host:8386/v1.1')
-        swift.assert_called_with(auth_version='2.0',
-                                 user='admin',
-                                 key='nova',
-                                 insecure=False,
-                                 cacert='/etc/tests/cert.crt',
-                                 tenant_name='admin',
-                                 authurl='http://localhost:5000/v2.0')
+        swift.assert_called_with(
+            auth_version='2.0', user='admin', key='nova', insecure=False,
+            cacert='sahara_tests/scenario/defaults/tests/cert.crt',
+            tenant_name='admin', authurl='http://localhost:5000/v2.0')
 
         nova.assert_called_with('2', session=fake_session)
         neutron.assert_called_with('2.0', session=fake_session)
@@ -193,9 +193,9 @@ class TestBase(testtools.TestCase):
                                   project_name='admin',
                                   user_domain_name='default',
                                   project_domain_name='default')
-        m_session.assert_called_with(auth=fake_auth,
-                                     cert='/etc/tests/cert.crt',
-                                     verify=True)
+        m_session.assert_called_with(
+            auth=fake_auth,
+            cert='sahara_tests/scenario/defaults/tests/cert.crt', verify=True)
 
     @mock.patch('sahara_tests.scenario.clients.NeutronClient.get_network_id',
                 return_value='mock_net')
@@ -394,7 +394,8 @@ class TestBase(testtools.TestCase):
                 "type": "Pig",
                 "input_datasource": {
                     "type": "swift",
-                    "source": "etc/edp-examples/edp-pig/top-todoers/"
+                    "source": "sahara_tests/scenario/defaults/edp-examples/"
+                              "edp-pig/top-todoers/"
                               "data/input"
                 },
                 "output_datasource": {
@@ -403,7 +404,8 @@ class TestBase(testtools.TestCase):
                 },
                 "main_lib": {
                     "type": "swift",
-                    "source": "etc/edp-examples/edp-pig/top-todoers/"
+                    "source": "sahara_tests/scenario/defaults/edp-examples/"
+                              "edp-pig/top-todoers/"
                               "example.pig"
                 }
             }
@@ -536,7 +538,8 @@ class TestBase(testtools.TestCase):
         self.assertEqual(output_path,
                          self.base_scenario._create_dfs_data(None, output_path,
                                                              None, 'hdfs'))
-        input_path = 'etc/edp-examples/edp-pig/trim-spaces/data/input'
+        input_path = ('sahara_tests/scenario/defaults/edp-examples/edp-pig/'
+                      'trim-spaces/data/input')
         with mock.patch(
             'sahara_tests.scenario.clients.SaharaClient.get_cluster',
             return_value=FakeResponse(node_groups=[

sahara_tests/unit/scenario/test_runner.py

@@ -93,7 +93,7 @@ class RunnerUnitTest(testtools.TestCase):
                 "sahara_url": "http://127.0.0.1",
                 "os_password": "changed_nova",
                 "os_tenant": "changed_admin",
-                "ssl_cert": "/etc/tests/cert.crt"
+                "ssl_cert": "sahara_tests/scenario/defaults/tests/cert.crt"
             },
             "network": {
                 "type": "neutron",
@@ -116,7 +116,8 @@ class RunnerUnitTest(testtools.TestCase):
                         "type": "Pig",
                         "input_datasource": {
                             "type": "swift",
-                            "source": "etc/edp-examples/edp-pig/top-todoers/"
+                            "source": "sahara_tests/scenario/defaults/"
+                                      "edp-examples/edp-pig/top-todoers/"
                                       "data/input"
                         },
                         "output_datasource": {
@@ -125,7 +126,8 @@ class RunnerUnitTest(testtools.TestCase):
                         },
                         "main_lib": {
                             "type": "swift",
-                            "source": "etc/edp-examples/edp-pig/top-todoers/"
+                            "source": "sahara_tests/scenario/defaults/"
+                                      "edp-examples/edp-pig/top-todoers/"
                                       "example.pig"
                         }
                     },
@@ -167,7 +169,7 @@ class RunnerUnitTest(testtools.TestCase):
                 "os_password": "changed_nova",
                 "os_tenant": "changed_admin",
                 "sahara_service_type": "data-processing",
-                "ssl_cert": "/etc/tests/cert.crt",
+                "ssl_cert": "sahara_tests/scenario/defaults/tests/cert.crt",
                 "ssl_verify": False
             },
         }
@@ -191,13 +193,15 @@ class RunnerUnitTest(testtools.TestCase):
                     'edp_jobs_flow': [
                         {
                             'main_lib': {
-                                'source': 'etc/edp-examples/edp-pig/'
+                                'source': 'sahara_tests/scenario/defaults/'
+                                          'edp-examples/edp-pig/'
                                           'top-todoers/example.pig',
                                 'type': 'swift'
                             },
                             'type': 'Pig',
                             'input_datasource': {
-                                'source': 'etc/edp-examples/edp-pig/'
+                                'source': 'sahara_tests/scenario/defaults/'
+                                          'edp-examples/edp-pig/'
                                           'top-todoers/data/input',
                                 'type': 'swift'
                             },

sahara_tests/unit/scenario/vanilla2_7_1.yaml

@@ -5,7 +5,7 @@ credentials:
     os_password: nova
     os_tenant: admin
     os_auth_url: http://127.0.0.1:5000/v2.0
-    ssl_cert: /etc/test/cert.crt
+    ssl_cert: sahara_tests/scenario/defaults/test/cert.crt
     ssl_verify: True
 
 network:
@@ -58,17 +58,17 @@ edp_jobs_flow:
         - type: Pig
           input_datasource:
               type: swift
-              source: etc/edp-examples/edp-pig/top-todoers/data/input
+              source: sahara_tests/scenario/defaults/edp-examples/edp-pig/top-todoers/data/input
           output_datasource:
               type: hdfs
               destination: /user/hadoop/edp-output
           main_lib:
               type: swift
-              source: etc/edp-examples/edp-pig/top-todoers/example.pig
+              source: sahara_tests/scenario/defaults/edp-examples/edp-pig/top-todoers/example.pig
         - type: Java
           additional_libs:
               - type: database
-                source: etc/edp-examples/hadoop2/edp-java/hadoop-mapreduce-examples-2.7.1.jar
+                source: sahara_tests/scenario/defaults/edp-examples/hadoop2/edp-java/hadoop-mapreduce-examples-2.7.1.jar
           configs:
               edp.java.main_class: org.apache.hadoop.examples.QuasiMonteCarlo
           args:
@@ -80,10 +80,10 @@ edp_jobs_flow:
               mapred.reducer.class: org.apache.oozie.example.SampleReducer
           additional_libs:
               - type: database
-                source: etc/edp-examples/edp-java/edp-java.jar
+                source: sahara_tests/scenario/defaults/edp-examples/edp-java/edp-java.jar
           input_datasource:
               type: swift
-              source: etc/edp-examples/edp-pig/top-todoers/data/input
+              source: sahara_tests/scenario/defaults/edp-examples/edp-pig/top-todoers/data/input
           output_datasource:
               type: hdfs
               destination: /user/hadoop/edp-output
@@ -93,17 +93,17 @@ edp_jobs_flow:
                 edp.streaming.reducer: /usr/bin/wc
           input_datasource:
               type: swift
-              source: etc/edp-examples/edp-pig/top-todoers/data/input
+              source: sahara_tests/scenario/defaults/edp-examples/edp-pig/top-todoers/data/input
           output_datasource:
               type: hdfs
               destination: /user/hadoop/edp-output
         - type: Hive
           input_datasource:
               type: swift
-              source: etc/edp-examples/edp-hive/input.csv
+              source: sahara_tests/scenario/defaults/edp-examples/edp-hive/input.csv
           output_datasource:
               type: hdfs
               destination: /user/hadoop/edp-hive/
           main_lib:
               type: swift
-              source: etc/edp-examples/edp-hive/script.q
+              source: sahara_tests/scenario/defaults/edp-examples/edp-hive/script.q

sahara_tests/unit/scenario/vanilla2_7_1.yaml.mako

@@ -5,7 +5,7 @@ credentials:
     os_password: ${OS_PASSWORD}
     os_tenant: ${OS_TENANT_NAME}
     os_auth_url: ${OS_AUTH_URL}
-    ssl_cert: /etc/tests/cert.crt
+    ssl_cert: sahara_tests/scenario/defaults/tests/cert.crt
     ssl_verify: True
 
 network:
@@ -56,17 +56,17 @@ edp_jobs_flow:
         - type: Pig
           input_datasource:
               type: swift
-              source: etc/edp-examples/edp-pig/top-todoers/data/input
+              source: sahara_tests/scenario/defaults/edp-examples/edp-pig/top-todoers/data/input
           output_datasource:
               type: hdfs
               destination: /user/hadoop/edp-output
           main_lib:
               type: swift
-              source: etc/edp-examples/edp-pig/top-todoers/example.pig
+              source: sahara_tests/scenario/defaults/edp-examples/edp-pig/top-todoers/example.pig
         - type: Java
           additional_libs:
               - type: database
-                source: etc/edp-examples/hadoop2/edp-java/hadoop-mapreduce-examples-2.7.1.jar
+                source: sahara_tests/scenario/defaults/edp-examples/hadoop2/edp-java/hadoop-mapreduce-examples-2.7.1.jar
           configs:
               edp.java.main_class: org.apache.hadoop.examples.QuasiMonteCarlo
           args:
@@ -78,10 +78,10 @@ edp_jobs_flow:
               mapred.reducer.class: org.apache.oozie.example.SampleReducer
           additional_libs:
               - type: database
-                source: etc/edp-examples/edp-java/edp-java.jar
+                source: sahara_tests/scenario/defaults/edp-examples/edp-java/edp-java.jar
           input_datasource:
               type: swift
-              source: etc/edp-examples/edp-pig/top-todoers/data/input
+              source: sahara_tests/scenario/defaults/edp-examples/edp-pig/top-todoers/data/input
           output_datasource:
               type: hdfs
               destination: /user/hadoop/edp-output
@@ -91,30 +91,30 @@ edp_jobs_flow:
                 edp.streaming.reducer: /usr/bin/wc
           input_datasource:
               type: swift
-              source: etc/edp-examples/edp-pig/top-todoers/data/input
+              source: sahara_tests/scenario/defaults/edp-examples/edp-pig/top-todoers/data/input
           output_datasource:
               type: hdfs
               destination: /user/hadoop/edp-output
         - type: Hive
           input_datasource:
               type: swift
-              source: etc/edp-examples/edp-hive/input.csv
+              source: sahara_tests/scenario/defaults/edp-examples/edp-hive/input.csv
           output_datasource:
               type: hdfs
               destination: /user/hadoop/edp-hive/
           main_lib:
               type: swift
-              source: etc/edp-examples/edp-hive/script.q
+              source: sahara_tests/scenario/defaults/edp-examples/edp-hive/script.q
         - type: MapReduce
           configs:
               mapred.mapper.class: org.apache.oozie.example.SampleMapper
               mapred.reducer.class: org.apache.oozie.example.SampleReducer
           additional_libs:
               - type: database
-                source: etc/edp-examples/edp-java/edp-java.jar
+                source: sahara_tests/scenario/defaults/edp-examples/edp-java/edp-java.jar
           input_datasource:
               type: swift
-              source: etc/edp-examples/edp-pig/top-todoers/data/input
+              source: sahara_tests/scenario/defaults/edp-examples/edp-pig/top-todoers/data/input
           output_datasource:
               type: hdfs
               destination: /user/hadoop/edp-output