diff --git a/etc/rest-api-samples/edp/data_source_create.json b/etc/rest-api-samples/edp/data_source_create.json index bbb629e4..a4248938 100644 --- a/etc/rest-api-samples/edp/data_source_create.json +++ b/etc/rest-api-samples/edp/data_source_create.json @@ -1,6 +1,6 @@ { "description": "This is input", - "url": "swift://container.savanna/text", + "url": "swift://container.sahara/text", "credentials": { "password": "swordfish", "user": "admin" diff --git a/savanna/plugins/hdp/versions/version_1_3_2/resources/ambari-config-resource.json b/savanna/plugins/hdp/versions/version_1_3_2/resources/ambari-config-resource.json index 808e70b3..00924ae0 100644 --- a/savanna/plugins/hdp/versions/version_1_3_2/resources/ambari-config-resource.json +++ b/savanna/plugins/hdp/versions/version_1_3_2/resources/ambari-config-resource.json @@ -234,7 +234,7 @@ "default_value": true, "description": "savanna provider public attribute", "is_optional": true, - "name": "fs.swift.service.savanna.public", + "name": "fs.swift.service.sahara.public", "scope": "cluster" }, { @@ -243,7 +243,7 @@ "default_value": "8080", "description": " ", "is_optional": true, - "name": "fs.swift.service.savanna.http.port", + "name": "fs.swift.service.sahara.http.port", "scope": "cluster" }, { @@ -252,7 +252,7 @@ "default_value": "443", "description": " ", "is_optional": true, - "name": "fs.swift.service.savanna.https.port", + "name": "fs.swift.service.sahara.https.port", "scope": "cluster" } ] diff --git a/savanna/plugins/hdp/versions/version_1_3_2/resources/default-cluster.template b/savanna/plugins/hdp/versions/version_1_3_2/resources/default-cluster.template index 2c3324af..84b98a9e 100644 --- a/savanna/plugins/hdp/versions/version_1_3_2/resources/default-cluster.template +++ b/savanna/plugins/hdp/versions/version_1_3_2/resources/default-cluster.template @@ -438,11 +438,11 @@ { "name" : "fs.swift.blocksize", "value" : "32768" }, { "name" : "fs.swift.partsize", "value" : "4718592" }, { "name" : "fs.swift.requestsize", "value" : "64" }, - { "name" : "fs.swift.service.savanna.public", "value" : "true" }, - { "name" : "fs.swift.service.savanna.http.port", "value" : "8080" }, - { "name" : "fs.swift.service.savanna.https.port", "value" : "443" }, - { "name" : "fs.swift.service.savanna.auth.url", "value" : "None" }, - { "name" : "fs.swift.service.savanna.tenant", "value" : "None"}, + { "name" : "fs.swift.service.sahara.public", "value" : "true" }, + { "name" : "fs.swift.service.sahara.http.port", "value" : "8080" }, + { "name" : "fs.swift.service.sahara.https.port", "value" : "443" }, + { "name" : "fs.swift.service.sahara.auth.url", "value" : "None" }, + { "name" : "fs.swift.service.sahara.tenant", "value" : "None"}, { "name" : "hadoop.proxyuser.hive.groups", "value" : "users" }, { "name" : "hadoop.proxyuser.hive.hosts", "value" : "%HIVE_HOST%" }, { "name" : "hadoop.proxyuser.hcat.groups", "value" : "users" }, diff --git a/savanna/service/edp/workflow_creator/workflow_factory.py b/savanna/service/edp/workflow_creator/workflow_factory.py index e8f6a107..bca5c83a 100644 --- a/savanna/service/edp/workflow_creator/workflow_factory.py +++ b/savanna/service/edp/workflow_creator/workflow_factory.py @@ -28,8 +28,8 @@ from savanna.utils import xmlutils conductor = c.API -swift_username = 'fs.swift.service.savanna.username' -swift_password = 'fs.swift.service.savanna.password' +swift_username = 'fs.swift.service.sahara.username' +swift_password = 'fs.swift.service.sahara.password' class BaseFactory(object): diff --git a/savanna/swift/resources/conf-template.xml b/savanna/swift/resources/conf-template.xml index 258afe27..eb629d95 100644 --- a/savanna/swift/resources/conf-template.xml +++ b/savanna/swift/resources/conf-template.xml @@ -1,51 +1,51 @@ - + - + - fs.swift.service.savanna.auth.url + fs.swift.service.sahara.auth.url - fs.swift.service.savanna.tenant + fs.swift.service.sahara.tenant - fs.swift.service.savanna.username + fs.swift.service.sahara.username - fs.swift.service.savanna.password + fs.swift.service.sahara.password - fs.swift.service.savanna.http.port + fs.swift.service.sahara.http.port 8080 - fs.swift.service.savanna.https.port + fs.swift.service.sahara.https.port 443 - fs.swift.service.savanna.public + fs.swift.service.sahara.public true - fs.swift.service.savanna.auth.endpoint.prefix + fs.swift.service.sahara.auth.endpoint.prefix /endpoints/AUTH_ - + - fs.swift.service.savanna.region + fs.swift.service.sahara.region - fs.swift.service.savanna.apikey + fs.swift.service.sahara.apikey - + diff --git a/savanna/swift/swift_helper.py b/savanna/swift/swift_helper.py index 9b9955e3..758a4ee9 100644 --- a/savanna/swift/swift_helper.py +++ b/savanna/swift/swift_helper.py @@ -24,8 +24,8 @@ from savanna.utils import xmlutils as x LOG = logging.getLogger(__name__) CONF = cfg.CONF -HADOOP_SWIFT_AUTH_URL = 'fs.swift.service.savanna.auth.url' -HADOOP_SWIFT_TENANT = 'fs.swift.service.savanna.tenant' +HADOOP_SWIFT_AUTH_URL = 'fs.swift.service.sahara.auth.url' +HADOOP_SWIFT_TENANT = 'fs.swift.service.sahara.tenant' def _retrieve_tenant(): diff --git a/savanna/tests/integration/tests/edp.py b/savanna/tests/integration/tests/edp.py index 666859a4..905839f6 100644 --- a/savanna/tests/integration/tests/edp.py +++ b/savanna/tests/integration/tests/edp.py @@ -92,8 +92,8 @@ class EDPTest(base.ITestCase): self.savanna.data_sources.delete(output_id) def _add_swift_configs(self, configs): - swift_user = "fs.swift.service.savanna.username" - swift_passw = "fs.swift.service.savanna.password" + swift_user = "fs.swift.service.sahara.username" + swift_passw = "fs.swift.service.sahara.password" if "configs" not in configs: configs["configs"] = {} @@ -131,8 +131,8 @@ class EDPTest(base.ITestCase): lib_binary_list = [] job_binary_internal_list = [] - swift_input_url = 'swift://%s.savanna/input' % container_name - swift_output_url = 'swift://%s.savanna/output' % container_name + swift_input_url = 'swift://%s.sahara/input' % container_name + swift_output_url = 'swift://%s.sahara/output' % container_name # Java jobs don't use data sources. Input/output paths must # be passed as args with corresponding username/password configs diff --git a/savanna/tests/integration/tests/resources/swift_test_script.sh b/savanna/tests/integration/tests/resources/swift_test_script.sh index efcf69d5..af15d160 100644 --- a/savanna/tests/integration/tests/resources/swift_test_script.sh +++ b/savanna/tests/integration/tests/resources/swift_test_script.sh @@ -53,10 +53,10 @@ check_swift_availability() { sudo -u $HADOOP_USER bash -c "hadoop dfs -copyFromLocal /tmp/test-file /swift-test/" check_return_code_after_command_execution -clean_hdfs `echo "$?"` - sudo -u $HADOOP_USER bash -c "hadoop distcp -D fs.swift.service.savanna.username=$OS_USERNAME -D fs.swift.service.savanna.tenant=$OS_TENANT_NAME -D fs.swift.service.savanna.password=$OS_PASSWORD /swift-test/test-file swift://$SWIFT_CONTAINER_NAME.savanna/" + sudo -u $HADOOP_USER bash -c "hadoop distcp -D fs.swift.service.sahara.username=$OS_USERNAME -D fs.swift.service.sahara.tenant=$OS_TENANT_NAME -D fs.swift.service.sahara.password=$OS_PASSWORD /swift-test/test-file swift://$SWIFT_CONTAINER_NAME.sahara/" check_return_code_after_command_execution -clean_hdfs `echo "$?"` - sudo -u $HADOOP_USER bash -c "hadoop distcp -D fs.swift.service.savanna.username=$OS_USERNAME -D fs.swift.service.savanna.tenant=$OS_TENANT_NAME -D fs.swift.service.savanna.password=$OS_PASSWORD swift://$SWIFT_CONTAINER_NAME.savanna/test-file /swift-test/swift-test-file" + sudo -u $HADOOP_USER bash -c "hadoop distcp -D fs.swift.service.sahara.username=$OS_USERNAME -D fs.swift.service.sahara.tenant=$OS_TENANT_NAME -D fs.swift.service.sahara.password=$OS_PASSWORD swift://$SWIFT_CONTAINER_NAME.sahara/test-file /swift-test/swift-test-file" check_return_code_after_command_execution -clean_hdfs `echo "$?"` sudo -u $HADOOP_USER bash -c "hadoop dfs -copyToLocal /swift-test/swift-test-file /tmp/swift-test-file" diff --git a/savanna/tests/unit/conductor/test_resource.py b/savanna/tests/unit/conductor/test_resource.py index b416f79a..17649225 100644 --- a/savanna/tests/unit/conductor/test_resource.py +++ b/savanna/tests/unit/conductor/test_resource.py @@ -67,7 +67,7 @@ SAMPLE_JOB_BINARY_DICT = { "name": "bob", "tenant_id": "6b859fb8d1f44e8eafdfb91f21309b5f", "updated_at": "null", - "url": "swift-internal://bob.savanna/job" + "url": "swift-internal://bob.sahara/job" } @@ -75,7 +75,7 @@ SAMPLE_DATA_SOURCE = { 'name': 'input', 'description': 'some input', 'type': 'swift', - 'url': 'swift://tmckay.savanna', + 'url': 'swift://tmckay.sahara', 'credentials': { 'username': 'me', 'password': 'password' diff --git a/savanna/tests/unit/plugins/hdp/test_clusterspec.py b/savanna/tests/unit/plugins/hdp/test_clusterspec.py index eab2fe51..5eac6a4f 100644 --- a/savanna/tests/unit/plugins/hdp/test_clusterspec.py +++ b/savanna/tests/unit/plugins/hdp/test_clusterspec.py @@ -599,7 +599,7 @@ class ClusterSpecTest(unittest2.TestCase): self.assertEqual( 'true', cluster_config.configurations['core-site'] - ['fs.swift.service.savanna.location-aware']) + ['fs.swift.service.sahara.location-aware']) self.assertEqual( 'org.apache.hadoop.net.ScriptBasedMapping', cluster_config.configurations['core-site'] diff --git a/savanna/tests/unit/plugins/hdp/test_clusterspec_hdp2.py b/savanna/tests/unit/plugins/hdp/test_clusterspec_hdp2.py index a09ce64b..5cac770e 100644 --- a/savanna/tests/unit/plugins/hdp/test_clusterspec_hdp2.py +++ b/savanna/tests/unit/plugins/hdp/test_clusterspec_hdp2.py @@ -644,7 +644,7 @@ class ClusterSpecTestForHDP2(unittest2.TestCase): self.assertEqual( 'true', cluster_config.configurations['core-site'] - ['fs.swift.service.savanna.location-aware']) + ['fs.swift.service.sahara.location-aware']) self.assertEqual( 'org.apache.hadoop.net.ScriptBasedMapping', cluster_config.configurations['core-site'] diff --git a/savanna/tests/unit/service/edp/test_job_manager.py b/savanna/tests/unit/service/edp/test_job_manager.py index d47c0003..df16382c 100644 --- a/savanna/tests/unit/service/edp/test_job_manager.py +++ b/savanna/tests/unit/service/edp/test_job_manager.py @@ -96,8 +96,8 @@ class TestJobManager(base.SavannaWithDbTestCase): job, job_exec = _create_all_stack('Pig') job_binary.return_value = {"name": "script.pig"} - input_data = _create_data_source('swift://ex.savanna/i') - output_data = _create_data_source('swift://ex.savanna/o') + input_data = _create_data_source('swift://ex.sahara/i') + output_data = _create_data_source('swift://ex.sahara/o') creator = workflow_factory.get_creator(job) @@ -105,17 +105,17 @@ class TestJobManager(base.SavannaWithDbTestCase): input_data, output_data) self.assertIn(""" - INPUT=swift://ex.savanna/i - OUTPUT=swift://ex.savanna/o""", res) + INPUT=swift://ex.sahara/i + OUTPUT=swift://ex.sahara/o""", res) self.assertIn(""" - fs.swift.service.savanna.password + fs.swift.service.sahara.password admin1 - fs.swift.service.savanna.username + fs.swift.service.sahara.username admin """, res) @@ -129,7 +129,7 @@ class TestJobManager(base.SavannaWithDbTestCase): job, job_exec = _create_all_stack('Pig') job_binary.return_value = {"name": "script.pig"} - input_data = _create_data_source('swift://ex.savanna/i') + input_data = _create_data_source('swift://ex.sahara/i') output_data = _create_data_source('hdfs://user/hadoop/out') creator = workflow_factory.get_creator(job) @@ -139,17 +139,17 @@ class TestJobManager(base.SavannaWithDbTestCase): self.assertIn(""" - fs.swift.service.savanna.password + fs.swift.service.sahara.password admin1 - fs.swift.service.savanna.username + fs.swift.service.sahara.username admin """, res) input_data = _create_data_source('hdfs://user/hadoop/in') - output_data = _create_data_source('swift://ex.savanna/o') + output_data = _create_data_source('swift://ex.sahara/o') creator = workflow_factory.get_creator(job) @@ -159,11 +159,11 @@ class TestJobManager(base.SavannaWithDbTestCase): self.assertIn(""" - fs.swift.service.savanna.password + fs.swift.service.sahara.password admin1 - fs.swift.service.savanna.username + fs.swift.service.sahara.username admin """, res) @@ -196,8 +196,8 @@ class TestJobManager(base.SavannaWithDbTestCase): job, job_exec = _create_all_stack(job_type, configs) - input_data = _create_data_source('swift://ex.savanna/i') - output_data = _create_data_source('swift://ex.savanna/o') + input_data = _create_data_source('swift://ex.sahara/i') + output_data = _create_data_source('swift://ex.sahara/o') creator = workflow_factory.get_creator(job) @@ -214,24 +214,24 @@ class TestJobManager(base.SavannaWithDbTestCase): self.assertIn(""" mapred.output.dir - swift://ex.savanna/o + swift://ex.sahara/o """, res) self.assertIn(""" mapred.input.dir - swift://ex.savanna/i + swift://ex.sahara/i """, res) self.assertIn(""" - fs.swift.service.savanna.password + fs.swift.service.sahara.password admin1 """, res) self.assertIn(""" - fs.swift.service.savanna.username + fs.swift.service.sahara.username admin """, res) @@ -259,11 +259,11 @@ class TestJobManager(base.SavannaWithDbTestCase): self.assertIn(""" - fs.swift.service.savanna.password + fs.swift.service.sahara.password admin1 - fs.swift.service.savanna.username + fs.swift.service.sahara.username admin @@ -278,8 +278,8 @@ class TestJobManager(base.SavannaWithDbTestCase): job, job_exec = _create_all_stack('Hive') job_binary.return_value = {"name": "script.q"} - input_data = _create_data_source('swift://ex.savanna/i') - output_data = _create_data_source('swift://ex.savanna/o') + input_data = _create_data_source('swift://ex.sahara/i') + output_data = _create_data_source('swift://ex.sahara/o') creator = workflow_factory.get_creator(job) @@ -290,23 +290,23 @@ class TestJobManager(base.SavannaWithDbTestCase): /user/hadoop/conf/hive-site.xml - fs.swift.service.savanna.password + fs.swift.service.sahara.password admin1 - fs.swift.service.savanna.username + fs.swift.service.sahara.username admin - INPUT=swift://ex.savanna/i - OUTPUT=swift://ex.savanna/o""", res) + INPUT=swift://ex.sahara/i + OUTPUT=swift://ex.sahara/o""", res) def _build_workflow_with_conf_common(self, job_type): job, _ = _create_all_stack(job_type) - input_data = _create_data_source('swift://ex.savanna/i') - output_data = _create_data_source('swift://ex.savanna/o') + input_data = _create_data_source('swift://ex.sahara/i') + output_data = _create_data_source('swift://ex.sahara/o') job_exec = _create_job_exec(job.id, job_type, configs={"configs": {'c': 'f'}}) @@ -325,13 +325,13 @@ class TestJobManager(base.SavannaWithDbTestCase): self.assertIn(""" mapred.input.dir - swift://ex.savanna/i + swift://ex.sahara/i """, res) self.assertIn(""" mapred.output.dir - swift://ex.savanna/o + swift://ex.sahara/o """, res) def test_build_workflow_for_job_mapreduce_with_conf(self): diff --git a/savanna/tests/unit/service/validation/edp/test_job_binary.py b/savanna/tests/unit/service/validation/edp/test_job_binary.py index fbf79e72..6a5b4e96 100644 --- a/savanna/tests/unit/service/validation/edp/test_job_binary.py +++ b/savanna/tests/unit/service/validation/edp/test_job_binary.py @@ -41,7 +41,7 @@ class TestJobBinaryValidation(u.ValidationTestCase): self._assert_create_object_validation( data={ "name": "j_o_w", - "url": su.SWIFT_INTERNAL_PREFIX+"o.savanna/k" + "url": su.SWIFT_INTERNAL_PREFIX+"o.sahara/k" }, bad_req_i=(1, "BAD_JOB_BINARY", "To work with JobBinary located in internal " @@ -52,7 +52,7 @@ class TestJobBinaryValidation(u.ValidationTestCase): self._assert_create_object_validation( data={ "name": "j_o_w", - "url": su.OLD_SWIFT_INTERNAL_PREFIX+"o.savanna/k" + "url": su.OLD_SWIFT_INTERNAL_PREFIX+"o.sahara/k" }, bad_req_i=(1, "BAD_JOB_BINARY", "To work with JobBinary located in internal " diff --git a/savanna/tests/unit/swift/test_swift_helper.py b/savanna/tests/unit/swift/test_swift_helper.py index 0a595636..ac4a38da 100644 --- a/savanna/tests/unit/swift/test_swift_helper.py +++ b/savanna/tests/unit/swift/test_swift_helper.py @@ -20,7 +20,7 @@ from savanna.tests.unit import base GENERAL_PREFIX = "fs.swift." -SERVICE_PREFIX = "service.savanna." +SERVICE_PREFIX = "service.sahara." GENERAL = ["impl", "connect.timeout", "socket.timeout", "connect.retry.count", "connect.throttle.delay", @@ -43,7 +43,7 @@ class SwiftIntegrationTestCase(base.SavannaTestCase): result = h.get_swift_configs() self.assertEqual(7, len(result)) - self.assertIn({'name': "fs.swift.service.savanna.tenant", + self.assertIn({'name': "fs.swift.service.sahara.tenant", 'value': 'test_tenant', 'description': ''}, result) - self.assertIn({'name': "fs.swift.service.savanna.http.port", + self.assertIn({'name': "fs.swift.service.sahara.http.port", 'value': '8080', 'description': ''}, result) diff --git a/savanna/topology/resources/core-template.xml b/savanna/topology/resources/core-template.xml index 43f5788d..c886ad90 100644 --- a/savanna/topology/resources/core-template.xml +++ b/savanna/topology/resources/core-template.xml @@ -17,7 +17,7 @@ - fs.swift.service.savanna.location-aware + fs.swift.service.sahara.location-aware true