From 40fd2aa34e4f02f165f1017e78799e99367cf4eb Mon Sep 17 00:00:00 2001 From: Telles Nobrega Date: Wed, 2 Jan 2019 12:34:13 -0300 Subject: [PATCH] Plugins splitted from sahara core Change-Id: I43e0beec6508f93a436a150749bfa23571986b9d --- .stestr.conf | 2 +- README.rst | 35 ++++ doc/requirements.txt | 9 + lower-constraints.txt | 162 ++++++++++++++++++ requirements.txt | 1 + {sahara => sahara_plugin_mapr}/__init__.py | 0 {sahara => sahara_plugin_mapr}/i18n.py | 0 .../plugins/__init__.py | 0 .../plugins/mapr/__init__.py | 0 .../plugins/mapr/abstract/__init__.py | 0 .../plugins/mapr/abstract/cluster_context.py | 0 .../mapr/abstract/cluster_validator.py | 0 .../plugins/mapr/abstract/configurer.py | 0 .../plugins/mapr/abstract/health_checker.py | 0 .../plugins/mapr/abstract/node_manager.py | 0 .../plugins/mapr/abstract/version_handler.py | 0 .../plugins/mapr/base/__init__.py | 0 .../mapr/base/base_cluster_configurer.py | 35 ++-- .../plugins/mapr/base/base_cluster_context.py | 33 ++-- .../mapr/base/base_cluster_validator.py | 6 +- .../plugins/mapr/base/base_edp_engine.py | 11 +- .../plugins/mapr/base/base_health_checker.py | 12 +- .../plugins/mapr/base/base_node_manager.py | 42 ++--- .../plugins/mapr/base/base_version_handler.py | 20 +-- .../plugins/mapr/domain/__init__.py | 0 .../plugins/mapr/domain/configuration_file.py | 13 +- .../plugins/mapr/domain/distro.py | 0 .../plugins/mapr/domain/node_process.py | 8 +- .../plugins/mapr/domain/service.py | 17 +- .../plugins/mapr/images.py | 3 +- .../plugins/mapr/plugin.py | 4 +- .../plugins/mapr/resources/add_mapr_repo.sh | 0 .../mapr/resources/add_security_repos.sh | 0 .../mapr/resources/create_disk_list_file.sh | 0 .../resources/images/centos/disable_firewall | 0 .../resources/images/centos/epel_dependencies | 0 .../centos/install_mapr_core_repository | 0 .../images/centos/install_mapr_eco_repository | 0 .../resources/images/centos/install_openjdk | 0 .../resources/images/centos/install_scala | 0 .../mapr/resources/images/centos/requiretty | 0 .../images/centos/selinux_permissive | 0 .../resources/images/centos/update_repository | 0 .../resources/images/common/configure_extjs | 0 .../mapr/resources/images/common/oracle_java | 0 .../images/common/resources/package_utils.sh | 0 .../images/common/resources/packages.json | 0 .../images/common/resources/spec_5.1.0.json | 0 .../images/common/resources/spec_5.2.0.json | 0 .../common/resources/spec_5.2.0_ubuntu.json | 0 .../images/common/resources/versions.py | 0 .../plugins/mapr/resources/images/image.yaml | 0 .../ubuntu/install_mapr_core_repository | 0 .../images/ubuntu/install_mapr_dependencies | 0 .../images/ubuntu/install_mapr_eco_repository | 0 .../resources/images/ubuntu/install_openjdk | 0 .../resources/images/ubuntu/install_scala | 0 .../resources/images/ubuntu/update_repository | 0 .../plugins/mapr/resources/install_java.sh | 0 .../plugins/mapr/resources/install_mysql.sh | 0 .../mapr/resources/install_mysql_client.sh | 0 .../plugins/mapr/resources/install_scala.sh | 0 .../plugins/mapr/resources/topology.sh | 0 .../plugins/mapr/services/__init__.py | 0 .../plugins/mapr/services/drill/__init__.py | 0 .../plugins/mapr/services/drill/drill.py | 8 +- .../plugins/mapr/services/flume/__init__.py | 0 .../plugins/mapr/services/flume/flume.py | 6 +- .../plugins/mapr/services/hbase/__init__.py | 0 .../plugins/mapr/services/hbase/hbase.py | 8 +- .../hbase/resources/hbase-default.json | 0 .../plugins/mapr/services/hive/__init__.py | 0 .../plugins/mapr/services/hive/hive.py | 27 +-- .../services/hive/resources/hive-default.xml | 0 .../services/hive/resources/sentry-db.xml | 0 .../hive/resources/sentry-default.xml | 0 .../plugins/mapr/services/httpfs/__init__.py | 0 .../plugins/mapr/services/httpfs/httpfs.py | 6 +- .../httpfs/resources/httpfs-default.json | 0 .../plugins/mapr/services/hue/__init__.py | 0 .../plugins/mapr/services/hue/hue.py | 52 +++--- .../hue/resources/hue_3.10.0.template | 0 .../services/hue/resources/hue_3.6.0.template | 0 .../services/hue/resources/hue_3.7.0.template | 0 .../services/hue/resources/hue_3.8.1.template | 0 .../services/hue/resources/hue_3.9.0.template | 0 .../hue/resources/hue_sh_3.10.0.template | 0 .../hue/resources/hue_sh_3.7.0.template | 0 .../hue/resources/hue_sh_3.8.1.template | 0 .../hue/resources/hue_sh_3.9.0.template | 0 .../hue/resources/livy_conf_3.10.0.template | 0 .../hue/resources/livy_sh_3.10.0.template | 0 .../plugins/mapr/services/impala/__init__.py | 0 .../plugins/mapr/services/impala/impala.py | 20 +-- .../impala/resources/impala-env.sh.j2 | 0 .../plugins/mapr/services/kafka/__init__.py | 0 .../plugins/mapr/services/kafka/kafka.py | 6 +- .../plugins/mapr/services/mahout/__init__.py | 0 .../plugins/mapr/services/mahout/mahout.py | 6 +- .../mapr/services/management/__init__.py | 0 .../mapr/services/management/management.py | 8 +- .../mapr/services/mapreduce/__init__.py | 0 .../mapr/services/mapreduce/mapreduce.py | 12 +- .../resources/mapreduce-cluster.json | 0 .../plugins/mapr/services/maprfs/__init__.py | 0 .../plugins/mapr/services/maprfs/maprfs.py | 29 ++-- .../mapr/services/maprfs/resources/cldb.conf | 0 .../plugins/mapr/services/mysql/__init__.py | 0 .../plugins/mapr/services/mysql/mysql.py | 13 +- .../mysql/resources/create_database.sql | 0 .../services/mysql/resources/grant_access.sql | 0 .../services/mysql/resources/hive_schema.sql | 0 .../plugins/mapr/services/oozie/__init__.py | 0 .../plugins/mapr/services/oozie/oozie.py | 18 +- .../oozie/resources/oozie-default.json | 0 .../plugins/mapr/services/pig/__init__.py | 0 .../plugins/mapr/services/pig/pig.py | 6 +- .../plugins/mapr/services/sentry/__init__.py | 0 .../sentry/resources/global-policy.ini | 0 .../sentry/resources/sentry-default.xml | 0 .../plugins/mapr/services/sentry/sentry.py | 22 +-- .../plugins/mapr/services/spark/__init__.py | 0 .../spark/resources/spark-env.template | 0 .../plugins/mapr/services/spark/spark.py | 22 +-- .../plugins/mapr/services/sqoop/__init__.py | 0 .../plugins/mapr/services/sqoop/sqoop2.py | 6 +- .../plugins/mapr/services/swift/__init__.py | 0 .../swift/resources/hadoop-swift-latest.jar | Bin .../swift/resources/swift-default.json | 0 .../plugins/mapr/services/swift/swift.py | 10 +- .../plugins/mapr/services/yarn/__init__.py | 0 .../services/yarn/resources/yarn-cluster.json | 0 .../services/yarn/resources/yarn-node.json | 0 .../plugins/mapr/services/yarn/yarn.py | 12 +- .../plugins/mapr/util/__init__.py | 0 .../plugins/mapr/util/commands.py | 0 .../plugins/mapr/util/event_log.py | 12 +- .../plugins/mapr/util/general.py | 12 +- .../plugins/mapr/util/maprfs_helper.py | 2 +- .../plugins/mapr/util/password_utils.py | 8 +- .../plugins/mapr/util/service_utils.py | 4 +- .../plugins/mapr/util/validation_utils.py | 23 ++- .../plugins/mapr/versions/__init__.py | 0 .../mapr/versions/v5_2_0_mrv2/__init__.py | 0 .../mapr/versions/v5_2_0_mrv2/context.py | 4 +- .../versions/v5_2_0_mrv2/version_handler.py | 44 ++--- .../mapr/versions/version_handler_factory.py | 2 +- .../tests/__init__.py | 2 +- .../tests/unit/__init__.py | 0 .../tests/unit/base.py | 36 +--- .../tests/unit/plugins/__init__.py | 0 .../tests/unit/plugins/mapr/__init__.py | 0 .../unit/plugins/mapr/test_base_handler.py | 25 +-- .../unit/plugins/mapr/test_cluster_context.py | 29 ++-- .../unit/plugins/mapr/test_config_files.py | 4 +- .../tests/unit/plugins/mapr/test_distro.py | 4 +- .../utils/__init__.py | 0 .../utils/patches.py | 0 setup.cfg | 85 ++------- tox.ini | 83 ++------- 160 files changed, 581 insertions(+), 508 deletions(-) create mode 100644 README.rst create mode 100644 doc/requirements.txt create mode 100644 lower-constraints.txt rename {sahara => sahara_plugin_mapr}/__init__.py (100%) rename {sahara => sahara_plugin_mapr}/i18n.py (100%) rename {sahara => sahara_plugin_mapr}/plugins/__init__.py (100%) rename {sahara => sahara_plugin_mapr}/plugins/mapr/__init__.py (100%) rename {sahara => sahara_plugin_mapr}/plugins/mapr/abstract/__init__.py (100%) rename {sahara => sahara_plugin_mapr}/plugins/mapr/abstract/cluster_context.py (100%) rename {sahara => sahara_plugin_mapr}/plugins/mapr/abstract/cluster_validator.py (100%) rename {sahara => sahara_plugin_mapr}/plugins/mapr/abstract/configurer.py (100%) rename {sahara => sahara_plugin_mapr}/plugins/mapr/abstract/health_checker.py (100%) rename {sahara => sahara_plugin_mapr}/plugins/mapr/abstract/node_manager.py (100%) rename {sahara => sahara_plugin_mapr}/plugins/mapr/abstract/version_handler.py (100%) rename {sahara => sahara_plugin_mapr}/plugins/mapr/base/__init__.py (100%) rename {sahara => sahara_plugin_mapr}/plugins/mapr/base/base_cluster_configurer.py (94%) rename {sahara => sahara_plugin_mapr}/plugins/mapr/base/base_cluster_context.py (94%) rename {sahara => sahara_plugin_mapr}/plugins/mapr/base/base_cluster_validator.py (85%) rename {sahara => sahara_plugin_mapr}/plugins/mapr/base/base_edp_engine.py (93%) rename {sahara => sahara_plugin_mapr}/plugins/mapr/base/base_health_checker.py (92%) rename {sahara => sahara_plugin_mapr}/plugins/mapr/base/base_node_manager.py (85%) rename {sahara => sahara_plugin_mapr}/plugins/mapr/base/base_version_handler.py (91%) rename {sahara => sahara_plugin_mapr}/plugins/mapr/domain/__init__.py (100%) rename {sahara => sahara_plugin_mapr}/plugins/mapr/domain/configuration_file.py (94%) rename {sahara => sahara_plugin_mapr}/plugins/mapr/domain/distro.py (100%) rename {sahara => sahara_plugin_mapr}/plugins/mapr/domain/node_process.py (96%) rename {sahara => sahara_plugin_mapr}/plugins/mapr/domain/service.py (94%) rename {sahara => sahara_plugin_mapr}/plugins/mapr/images.py (94%) rename {sahara => sahara_plugin_mapr}/plugins/mapr/plugin.py (97%) rename {sahara => sahara_plugin_mapr}/plugins/mapr/resources/add_mapr_repo.sh (100%) rename {sahara => sahara_plugin_mapr}/plugins/mapr/resources/add_security_repos.sh (100%) rename {sahara => sahara_plugin_mapr}/plugins/mapr/resources/create_disk_list_file.sh (100%) rename {sahara => sahara_plugin_mapr}/plugins/mapr/resources/images/centos/disable_firewall (100%) rename {sahara => sahara_plugin_mapr}/plugins/mapr/resources/images/centos/epel_dependencies (100%) rename {sahara => sahara_plugin_mapr}/plugins/mapr/resources/images/centos/install_mapr_core_repository (100%) rename {sahara => sahara_plugin_mapr}/plugins/mapr/resources/images/centos/install_mapr_eco_repository (100%) rename {sahara => sahara_plugin_mapr}/plugins/mapr/resources/images/centos/install_openjdk (100%) rename {sahara => sahara_plugin_mapr}/plugins/mapr/resources/images/centos/install_scala (100%) rename {sahara => sahara_plugin_mapr}/plugins/mapr/resources/images/centos/requiretty (100%) rename {sahara => sahara_plugin_mapr}/plugins/mapr/resources/images/centos/selinux_permissive (100%) rename {sahara => sahara_plugin_mapr}/plugins/mapr/resources/images/centos/update_repository (100%) rename {sahara => sahara_plugin_mapr}/plugins/mapr/resources/images/common/configure_extjs (100%) rename {sahara => sahara_plugin_mapr}/plugins/mapr/resources/images/common/oracle_java (100%) rename {sahara => sahara_plugin_mapr}/plugins/mapr/resources/images/common/resources/package_utils.sh (100%) rename {sahara => sahara_plugin_mapr}/plugins/mapr/resources/images/common/resources/packages.json (100%) rename {sahara => sahara_plugin_mapr}/plugins/mapr/resources/images/common/resources/spec_5.1.0.json (100%) rename {sahara => sahara_plugin_mapr}/plugins/mapr/resources/images/common/resources/spec_5.2.0.json (100%) rename {sahara => sahara_plugin_mapr}/plugins/mapr/resources/images/common/resources/spec_5.2.0_ubuntu.json (100%) rename {sahara => sahara_plugin_mapr}/plugins/mapr/resources/images/common/resources/versions.py (100%) rename {sahara => sahara_plugin_mapr}/plugins/mapr/resources/images/image.yaml (100%) rename {sahara => sahara_plugin_mapr}/plugins/mapr/resources/images/ubuntu/install_mapr_core_repository (100%) rename {sahara => sahara_plugin_mapr}/plugins/mapr/resources/images/ubuntu/install_mapr_dependencies (100%) rename {sahara => sahara_plugin_mapr}/plugins/mapr/resources/images/ubuntu/install_mapr_eco_repository (100%) rename {sahara => sahara_plugin_mapr}/plugins/mapr/resources/images/ubuntu/install_openjdk (100%) rename {sahara => sahara_plugin_mapr}/plugins/mapr/resources/images/ubuntu/install_scala (100%) rename {sahara => sahara_plugin_mapr}/plugins/mapr/resources/images/ubuntu/update_repository (100%) rename {sahara => sahara_plugin_mapr}/plugins/mapr/resources/install_java.sh (100%) rename {sahara => sahara_plugin_mapr}/plugins/mapr/resources/install_mysql.sh (100%) rename {sahara => sahara_plugin_mapr}/plugins/mapr/resources/install_mysql_client.sh (100%) rename {sahara => sahara_plugin_mapr}/plugins/mapr/resources/install_scala.sh (100%) rename {sahara => sahara_plugin_mapr}/plugins/mapr/resources/topology.sh (100%) rename {sahara => sahara_plugin_mapr}/plugins/mapr/services/__init__.py (100%) rename {sahara => sahara_plugin_mapr}/plugins/mapr/services/drill/__init__.py (100%) rename {sahara => sahara_plugin_mapr}/plugins/mapr/services/drill/drill.py (91%) rename {sahara => sahara_plugin_mapr}/plugins/mapr/services/flume/__init__.py (100%) rename {sahara => sahara_plugin_mapr}/plugins/mapr/services/flume/flume.py (86%) rename {sahara => sahara_plugin_mapr}/plugins/mapr/services/hbase/__init__.py (100%) rename {sahara => sahara_plugin_mapr}/plugins/mapr/services/hbase/hbase.py (93%) rename {sahara => sahara_plugin_mapr}/plugins/mapr/services/hbase/resources/hbase-default.json (100%) rename {sahara => sahara_plugin_mapr}/plugins/mapr/services/hive/__init__.py (100%) rename {sahara => sahara_plugin_mapr}/plugins/mapr/services/hive/hive.py (88%) rename {sahara => sahara_plugin_mapr}/plugins/mapr/services/hive/resources/hive-default.xml (100%) rename {sahara => sahara_plugin_mapr}/plugins/mapr/services/hive/resources/sentry-db.xml (100%) rename {sahara => sahara_plugin_mapr}/plugins/mapr/services/hive/resources/sentry-default.xml (100%) rename {sahara => sahara_plugin_mapr}/plugins/mapr/services/httpfs/__init__.py (100%) rename {sahara => sahara_plugin_mapr}/plugins/mapr/services/httpfs/httpfs.py (85%) rename {sahara => sahara_plugin_mapr}/plugins/mapr/services/httpfs/resources/httpfs-default.json (100%) rename {sahara => sahara_plugin_mapr}/plugins/mapr/services/hue/__init__.py (100%) rename {sahara => sahara_plugin_mapr}/plugins/mapr/services/hue/hue.py (87%) rename {sahara => sahara_plugin_mapr}/plugins/mapr/services/hue/resources/hue_3.10.0.template (100%) rename {sahara => sahara_plugin_mapr}/plugins/mapr/services/hue/resources/hue_3.6.0.template (100%) rename {sahara => sahara_plugin_mapr}/plugins/mapr/services/hue/resources/hue_3.7.0.template (100%) rename {sahara => sahara_plugin_mapr}/plugins/mapr/services/hue/resources/hue_3.8.1.template (100%) rename {sahara => sahara_plugin_mapr}/plugins/mapr/services/hue/resources/hue_3.9.0.template (100%) rename {sahara => sahara_plugin_mapr}/plugins/mapr/services/hue/resources/hue_sh_3.10.0.template (100%) rename {sahara => sahara_plugin_mapr}/plugins/mapr/services/hue/resources/hue_sh_3.7.0.template (100%) rename {sahara => sahara_plugin_mapr}/plugins/mapr/services/hue/resources/hue_sh_3.8.1.template (100%) rename {sahara => sahara_plugin_mapr}/plugins/mapr/services/hue/resources/hue_sh_3.9.0.template (100%) rename {sahara => sahara_plugin_mapr}/plugins/mapr/services/hue/resources/livy_conf_3.10.0.template (100%) rename {sahara => sahara_plugin_mapr}/plugins/mapr/services/hue/resources/livy_sh_3.10.0.template (100%) rename {sahara => sahara_plugin_mapr}/plugins/mapr/services/impala/__init__.py (100%) rename {sahara => sahara_plugin_mapr}/plugins/mapr/services/impala/impala.py (91%) rename {sahara => sahara_plugin_mapr}/plugins/mapr/services/impala/resources/impala-env.sh.j2 (100%) rename {sahara => sahara_plugin_mapr}/plugins/mapr/services/kafka/__init__.py (100%) rename {sahara => sahara_plugin_mapr}/plugins/mapr/services/kafka/kafka.py (91%) rename {sahara => sahara_plugin_mapr}/plugins/mapr/services/mahout/__init__.py (100%) rename {sahara => sahara_plugin_mapr}/plugins/mapr/services/mahout/mahout.py (88%) rename {sahara => sahara_plugin_mapr}/plugins/mapr/services/management/__init__.py (100%) rename {sahara => sahara_plugin_mapr}/plugins/mapr/services/management/management.py (87%) rename {sahara => sahara_plugin_mapr}/plugins/mapr/services/mapreduce/__init__.py (100%) rename {sahara => sahara_plugin_mapr}/plugins/mapr/services/mapreduce/mapreduce.py (92%) rename {sahara => sahara_plugin_mapr}/plugins/mapr/services/mapreduce/resources/mapreduce-cluster.json (100%) rename {sahara => sahara_plugin_mapr}/plugins/mapr/services/maprfs/__init__.py (100%) rename {sahara => sahara_plugin_mapr}/plugins/mapr/services/maprfs/maprfs.py (85%) rename {sahara => sahara_plugin_mapr}/plugins/mapr/services/maprfs/resources/cldb.conf (100%) rename {sahara => sahara_plugin_mapr}/plugins/mapr/services/mysql/__init__.py (100%) rename {sahara => sahara_plugin_mapr}/plugins/mapr/services/mysql/mysql.py (96%) rename {sahara => sahara_plugin_mapr}/plugins/mapr/services/mysql/resources/create_database.sql (100%) rename {sahara => sahara_plugin_mapr}/plugins/mapr/services/mysql/resources/grant_access.sql (100%) rename {sahara => sahara_plugin_mapr}/plugins/mapr/services/mysql/resources/hive_schema.sql (100%) rename {sahara => sahara_plugin_mapr}/plugins/mapr/services/oozie/__init__.py (100%) rename {sahara => sahara_plugin_mapr}/plugins/mapr/services/oozie/oozie.py (92%) rename {sahara => sahara_plugin_mapr}/plugins/mapr/services/oozie/resources/oozie-default.json (100%) rename {sahara => sahara_plugin_mapr}/plugins/mapr/services/pig/__init__.py (100%) rename {sahara => sahara_plugin_mapr}/plugins/mapr/services/pig/pig.py (87%) rename {sahara => sahara_plugin_mapr}/plugins/mapr/services/sentry/__init__.py (100%) rename {sahara => sahara_plugin_mapr}/plugins/mapr/services/sentry/resources/global-policy.ini (100%) rename {sahara => sahara_plugin_mapr}/plugins/mapr/services/sentry/resources/sentry-default.xml (100%) rename {sahara => sahara_plugin_mapr}/plugins/mapr/services/sentry/sentry.py (89%) rename {sahara => sahara_plugin_mapr}/plugins/mapr/services/spark/__init__.py (100%) rename {sahara => sahara_plugin_mapr}/plugins/mapr/services/spark/resources/spark-env.template (100%) rename {sahara => sahara_plugin_mapr}/plugins/mapr/services/spark/spark.py (94%) rename {sahara => sahara_plugin_mapr}/plugins/mapr/services/sqoop/__init__.py (100%) rename {sahara => sahara_plugin_mapr}/plugins/mapr/services/sqoop/sqoop2.py (87%) rename {sahara => sahara_plugin_mapr}/plugins/mapr/services/swift/__init__.py (100%) rename {sahara => sahara_plugin_mapr}/plugins/mapr/services/swift/resources/hadoop-swift-latest.jar (100%) rename {sahara => sahara_plugin_mapr}/plugins/mapr/services/swift/resources/swift-default.json (100%) rename {sahara => sahara_plugin_mapr}/plugins/mapr/services/swift/swift.py (86%) rename {sahara => sahara_plugin_mapr}/plugins/mapr/services/yarn/__init__.py (100%) rename {sahara => sahara_plugin_mapr}/plugins/mapr/services/yarn/resources/yarn-cluster.json (100%) rename {sahara => sahara_plugin_mapr}/plugins/mapr/services/yarn/resources/yarn-node.json (100%) rename {sahara => sahara_plugin_mapr}/plugins/mapr/services/yarn/yarn.py (92%) rename {sahara => sahara_plugin_mapr}/plugins/mapr/util/__init__.py (100%) rename {sahara => sahara_plugin_mapr}/plugins/mapr/util/commands.py (100%) rename {sahara => sahara_plugin_mapr}/plugins/mapr/util/event_log.py (82%) rename {sahara => sahara_plugin_mapr}/plugins/mapr/util/general.py (95%) rename {sahara => sahara_plugin_mapr}/plugins/mapr/util/maprfs_helper.py (98%) rename {sahara => sahara_plugin_mapr}/plugins/mapr/util/password_utils.py (93%) rename {sahara => sahara_plugin_mapr}/plugins/mapr/util/service_utils.py (93%) rename {sahara => sahara_plugin_mapr}/plugins/mapr/util/validation_utils.py (94%) rename {sahara => sahara_plugin_mapr}/plugins/mapr/versions/__init__.py (100%) rename {sahara => sahara_plugin_mapr}/plugins/mapr/versions/v5_2_0_mrv2/__init__.py (100%) rename {sahara => sahara_plugin_mapr}/plugins/mapr/versions/v5_2_0_mrv2/context.py (94%) rename {sahara => sahara_plugin_mapr}/plugins/mapr/versions/v5_2_0_mrv2/version_handler.py (54%) rename {sahara => sahara_plugin_mapr}/plugins/mapr/versions/version_handler_factory.py (95%) rename {sahara => sahara_plugin_mapr}/tests/__init__.py (94%) rename {sahara => sahara_plugin_mapr}/tests/unit/__init__.py (100%) rename {sahara => sahara_plugin_mapr}/tests/unit/base.py (67%) rename {sahara => sahara_plugin_mapr}/tests/unit/plugins/__init__.py (100%) rename {sahara => sahara_plugin_mapr}/tests/unit/plugins/mapr/__init__.py (100%) rename {sahara => sahara_plugin_mapr}/tests/unit/plugins/mapr/test_base_handler.py (82%) rename {sahara => sahara_plugin_mapr}/tests/unit/plugins/mapr/test_cluster_context.py (92%) rename {sahara => sahara_plugin_mapr}/tests/unit/plugins/mapr/test_config_files.py (98%) rename {sahara => sahara_plugin_mapr}/tests/unit/plugins/mapr/test_distro.py (92%) rename {sahara => sahara_plugin_mapr}/utils/__init__.py (100%) rename {sahara => sahara_plugin_mapr}/utils/patches.py (100%) diff --git a/.stestr.conf b/.stestr.conf index d81c293..6f9fff9 100644 --- a/.stestr.conf +++ b/.stestr.conf @@ -1,3 +1,3 @@ [DEFAULT] -test_path=./sahara/tests/unit +test_path=./sahara_plugin_mapr/tests/unit top_dir=./ diff --git a/README.rst b/README.rst new file mode 100644 index 0000000..92f6253 --- /dev/null +++ b/README.rst @@ -0,0 +1,35 @@ +======================== +Team and repository tags +======================== + +.. image:: https://governance.openstack.org/tc/badges/sahara.svg + :target: https://governance.openstack.org/tc/reference/tags/index.html + +.. Change things from this point on + +OpenStack Data Processing ("Sahara") project +============================================ + +Sahara at wiki.openstack.org: https://wiki.openstack.org/wiki/Sahara + +Storyboard project: https://storyboard.openstack.org/#!/project/935 + +Sahara docs site: https://docs.openstack.org/sahara/latest/ + +Roadmap: https://wiki.openstack.org/wiki/Sahara/Roadmap + +Quickstart guide: https://docs.openstack.org/sahara/latest/user/quickstart.html + +How to participate: https://docs.openstack.org/sahara/latest/contributor/how-to-participate.html + +Source: https://git.openstack.org/cgit/openstack/sahara + +Bugs and feature requests: https://storyboard.openstack.org/#!/project/935 + +Release notes: https://docs.openstack.org/releasenotes/sahara/ + +License +------- + +Apache License Version 2.0 http://www.apache.org/licenses/LICENSE-2.0 + diff --git a/doc/requirements.txt b/doc/requirements.txt new file mode 100644 index 0000000..9cff004 --- /dev/null +++ b/doc/requirements.txt @@ -0,0 +1,9 @@ +# The order of packages is significant, because pip processes them in the order +# of appearance. Changing the order has an impact on the overall integration +# process, which may cause wedges in the gate later. +openstackdocstheme>=1.18.1 # Apache-2.0 +os-api-ref>=1.4.0 # Apache-2.0 +reno>=2.5.0 # Apache-2.0 +sphinx!=1.6.6,!=1.6.7,>=1.6.2 # BSD +sphinxcontrib-httpdomain>=1.3.0 # BSD +whereto>=0.3.0 # Apache-2.0 diff --git a/lower-constraints.txt b/lower-constraints.txt new file mode 100644 index 0000000..efaf3f8 --- /dev/null +++ b/lower-constraints.txt @@ -0,0 +1,162 @@ +alabaster==0.7.10 +alembic==0.8.10 +amqp==2.2.2 +appdirs==1.4.3 +asn1crypto==0.24.0 +astroid==1.3.8 +Babel==2.3.4 +bandit==1.1.0 +bashate==0.5.1 +bcrypt==3.1.4 +botocore==1.5.1 +cachetools==2.0.1 +castellan==0.16.0 +certifi==2018.1.18 +cffi==1.11.5 +chardet==3.0.4 +click==6.7 +cliff==2.11.0 +cmd2==0.8.1 +contextlib2==0.5.5 +coverage==4.0 +cryptography==2.1.4 +debtcollector==1.19.0 +decorator==4.2.1 +deprecation==2.0 +doc8==0.6.0 +docutils==0.14 +dogpile.cache==0.6.5 +dulwich==0.19.0 +enum-compat==0.0.2 +eventlet==0.18.2 +extras==1.0.0 +fasteners==0.14.1 +fixtures==3.0.0 +flake8==2.6.2 +Flask==1.0.2 +future==0.16.0 +futurist==1.6.0 +gitdb2==2.0.3 +GitPython==2.1.8 +greenlet==0.4.13 +hacking==1.1.0 +idna==2.6 +imagesize==1.0.0 +iso8601==0.1.11 +itsdangerous==0.24 +Jinja2==2.10 +jmespath==0.9.3 +jsonpatch==1.21 +jsonpointer==2.0 +jsonschema==2.6.0 +keystoneauth1==3.4.0 +keystonemiddleware==4.17.0 +kombu==4.1.0 +linecache2==1.0.0 +logilab-common==1.4.1 +Mako==1.0.7 +MarkupSafe==1.0 +mccabe==0.2.1 +mock==2.0.0 +monotonic==1.4 +mox3==0.25.0 +msgpack==0.5.6 +munch==2.2.0 +netaddr==0.7.19 +netifaces==0.10.6 +openstackdocstheme==1.18.1 +openstacksdk==0.12.0 +os-api-ref==1.4.0 +os-client-config==1.29.0 +os-service-types==1.2.0 +osc-lib==1.10.0 +oslo.cache==1.29.0 +oslo.concurrency==3.26.0 +oslo.config==5.2.0 +oslo.context==2.19.2 +oslo.db==4.27.0 +oslo.i18n==3.15.3 +oslo.log==3.36.0 +oslo.messaging==5.29.0 +oslo.middleware==3.31.0 +oslo.policy==1.30.0 +oslo.rootwrap==5.8.0 +oslo.serialization==2.18.0 +oslo.service==1.24.0 +oslo.upgradecheck==0.1.0 +oslo.utils==3.33.0 +oslotest==3.2.0 +packaging==17.1 +paramiko==2.0.0 +Paste==2.0.3 +PasteDeploy==1.5.2 +pbr==2.0.0 +pika-pool==0.1.3 +pika==0.10.0 +prettytable==0.7.2 +psycopg2==2.6.2 +pyasn1==0.4.2 +pycadf==2.7.0 +pycparser==2.18 +pycodestyle==2.4.0 +pyflakes==0.8.1 +Pygments==2.2.0 +pyinotify==0.9.6 +pylint==1.4.5 +PyMySQL==0.7.6 +PyNaCl==1.2.1 +pyOpenSSL==17.5.0 +pyparsing==2.2.0 +pyperclip==1.6.0 +python-barbicanclient==4.6.0 +python-cinderclient==3.3.0 +python-dateutil==2.7.0 +python-editor==1.0.3 +python-glanceclient==2.8.0 +python-heatclient==1.10.0 +python-keystoneclient==3.8.0 +python-manilaclient==1.16.0 +python-mimeparse==1.6.0 +python-neutronclient==6.7.0 +python-novaclient==9.1.0 +python-openstackclient==3.14.0 +python-saharaclient==1.4.0 +python-subunit==1.2.0 +python-swiftclient==3.2.0 +pytz==2018.3 +PyYAML==3.12 +reno==2.5.0 +repoze.lru==0.7 +requests==2.14.2 +requestsexceptions==1.4.0 +restructuredtext-lint==1.1.3 +rfc3986==1.1.0 +Routes==2.4.1 +simplejson==3.13.2 +six==1.10.0 +smmap2==2.0.3 +snowballstemmer==1.2.1 +Sphinx==1.6.2 +sphinxcontrib-httpdomain==1.3.0 +sphinxcontrib-websupport==1.0.1 +sqlalchemy-migrate==0.11.0 +SQLAlchemy==1.0.10 +sqlparse==0.2.4 +statsd==3.2.2 +stestr==1.0.0 +stevedore==1.20.0 +Tempita==0.5.2 +tenacity==4.9.0 +testresources==2.0.0 +testscenarios==0.4 +testtools==2.2.0 +tooz==1.58.0 +traceback2==1.4.0 +unittest2==1.1.0 +urllib3==1.22 +vine==1.1.4 +voluptuous==0.11.1 +warlock==1.3.0 +WebOb==1.7.1 +Werkzeug==0.14.1 +wrapt==1.10.11 diff --git a/requirements.txt b/requirements.txt index e052228..4cdb2b2 100644 --- a/requirements.txt +++ b/requirements.txt @@ -39,6 +39,7 @@ python-swiftclient>=3.2.0 # Apache-2.0 python-neutronclient>=6.7.0 # Apache-2.0 python-heatclient>=1.10.0 # Apache-2.0 python-glanceclient>=2.8.0 # Apache-2.0 +sahara six>=1.10.0 # MIT stevedore>=1.20.0 # Apache-2.0 SQLAlchemy!=1.1.5,!=1.1.6,!=1.1.7,!=1.1.8,>=1.0.10 # MIT diff --git a/sahara/__init__.py b/sahara_plugin_mapr/__init__.py similarity index 100% rename from sahara/__init__.py rename to sahara_plugin_mapr/__init__.py diff --git a/sahara/i18n.py b/sahara_plugin_mapr/i18n.py similarity index 100% rename from sahara/i18n.py rename to sahara_plugin_mapr/i18n.py diff --git a/sahara/plugins/__init__.py b/sahara_plugin_mapr/plugins/__init__.py similarity index 100% rename from sahara/plugins/__init__.py rename to sahara_plugin_mapr/plugins/__init__.py diff --git a/sahara/plugins/mapr/__init__.py b/sahara_plugin_mapr/plugins/mapr/__init__.py similarity index 100% rename from sahara/plugins/mapr/__init__.py rename to sahara_plugin_mapr/plugins/mapr/__init__.py diff --git a/sahara/plugins/mapr/abstract/__init__.py b/sahara_plugin_mapr/plugins/mapr/abstract/__init__.py similarity index 100% rename from sahara/plugins/mapr/abstract/__init__.py rename to sahara_plugin_mapr/plugins/mapr/abstract/__init__.py diff --git a/sahara/plugins/mapr/abstract/cluster_context.py b/sahara_plugin_mapr/plugins/mapr/abstract/cluster_context.py similarity index 100% rename from sahara/plugins/mapr/abstract/cluster_context.py rename to sahara_plugin_mapr/plugins/mapr/abstract/cluster_context.py diff --git a/sahara/plugins/mapr/abstract/cluster_validator.py b/sahara_plugin_mapr/plugins/mapr/abstract/cluster_validator.py similarity index 100% rename from sahara/plugins/mapr/abstract/cluster_validator.py rename to sahara_plugin_mapr/plugins/mapr/abstract/cluster_validator.py diff --git a/sahara/plugins/mapr/abstract/configurer.py b/sahara_plugin_mapr/plugins/mapr/abstract/configurer.py similarity index 100% rename from sahara/plugins/mapr/abstract/configurer.py rename to sahara_plugin_mapr/plugins/mapr/abstract/configurer.py diff --git a/sahara/plugins/mapr/abstract/health_checker.py b/sahara_plugin_mapr/plugins/mapr/abstract/health_checker.py similarity index 100% rename from sahara/plugins/mapr/abstract/health_checker.py rename to sahara_plugin_mapr/plugins/mapr/abstract/health_checker.py diff --git a/sahara/plugins/mapr/abstract/node_manager.py b/sahara_plugin_mapr/plugins/mapr/abstract/node_manager.py similarity index 100% rename from sahara/plugins/mapr/abstract/node_manager.py rename to sahara_plugin_mapr/plugins/mapr/abstract/node_manager.py diff --git a/sahara/plugins/mapr/abstract/version_handler.py b/sahara_plugin_mapr/plugins/mapr/abstract/version_handler.py similarity index 100% rename from sahara/plugins/mapr/abstract/version_handler.py rename to sahara_plugin_mapr/plugins/mapr/abstract/version_handler.py diff --git a/sahara/plugins/mapr/base/__init__.py b/sahara_plugin_mapr/plugins/mapr/base/__init__.py similarity index 100% rename from sahara/plugins/mapr/base/__init__.py rename to sahara_plugin_mapr/plugins/mapr/base/__init__.py diff --git a/sahara/plugins/mapr/base/base_cluster_configurer.py b/sahara_plugin_mapr/plugins/mapr/base/base_cluster_configurer.py similarity index 94% rename from sahara/plugins/mapr/base/base_cluster_configurer.py rename to sahara_plugin_mapr/plugins/mapr/base/base_cluster_configurer.py index b4e4c66..03cb418 100644 --- a/sahara/plugins/mapr/base/base_cluster_configurer.py +++ b/sahara_plugin_mapr/plugins/mapr/base/base_cluster_configurer.py @@ -18,24 +18,23 @@ import abc from oslo_log import log as logging import six -from sahara import conductor -from sahara import context -from sahara.i18n import _ -import sahara.plugins.mapr.abstract.configurer as ac -from sahara.plugins.mapr.domain import distro as d -from sahara.plugins.mapr.domain import service as srvc -import sahara.plugins.mapr.services.management.management as mng -import sahara.plugins.mapr.services.mapreduce.mapreduce as mr -from sahara.plugins.mapr.services.maprfs import maprfs -from sahara.plugins.mapr.services.mysql import mysql -import sahara.plugins.mapr.services.yarn.yarn as yarn -from sahara.plugins.mapr.util import event_log as el -import sahara.plugins.mapr.util.general as util -import sahara.plugins.mapr.util.password_utils as pu -import sahara.utils.files as files +from sahara.plugins import conductor +from sahara.plugins import context +import sahara.plugins.utils as utils +from sahara_plugin_mapr.i18n import _ +import sahara_plugin_mapr.plugins.mapr.abstract.configurer as ac +from sahara_plugin_mapr.plugins.mapr.domain import distro as d +from sahara_plugin_mapr.plugins.mapr.domain import service as srvc +import sahara_plugin_mapr.plugins.mapr.services.management.management as mng +import sahara_plugin_mapr.plugins.mapr.services.mapreduce.mapreduce as mr +from sahara_plugin_mapr.plugins.mapr.services.maprfs import maprfs +from sahara_plugin_mapr.plugins.mapr.services.mysql import mysql +import sahara_plugin_mapr.plugins.mapr.services.yarn.yarn as yarn +from sahara_plugin_mapr.plugins.mapr.util import event_log as el +import sahara_plugin_mapr.plugins.mapr.util.general as util +import sahara_plugin_mapr.plugins.mapr.util.password_utils as pu LOG = logging.getLogger(__name__) -conductor = conductor.API _JAVA_HOME = '/usr/java/jdk1.7.0_51' _CONFIGURE_SH_TIMEOUT = 600 @@ -173,7 +172,7 @@ class BaseConfigurer(ac.AbstractConfigurer): topology_map = "\n".join(topology_map) + "\n" data_path = "%s/topology.data" % cluster_context.mapr_home - script = files.get_file_text(_TOPO_SCRIPT) + script = utils.get_file_text(_TOPO_SCRIPT, 'sahara_plugin_mapr') script_path = '%s/topology.sh' % cluster_context.mapr_home @el.provision_event() @@ -310,7 +309,7 @@ class BaseConfigurer(ac.AbstractConfigurer): 'port': mysql.MySQL.MYSQL_SERVER_PORT, }) - with context.ThreadGroup() as tg: + with context.PluginsThreadGroup() as tg: for instance in instances: tg.spawn('configure-sh-%s' % instance.id, self._configure_sh_instance, cluster_context, diff --git a/sahara/plugins/mapr/base/base_cluster_context.py b/sahara_plugin_mapr/plugins/mapr/base/base_cluster_context.py similarity index 94% rename from sahara/plugins/mapr/base/base_cluster_context.py rename to sahara_plugin_mapr/plugins/mapr/base/base_cluster_context.py index b568423..bd032fc 100644 --- a/sahara/plugins/mapr/base/base_cluster_context.py +++ b/sahara_plugin_mapr/plugins/mapr/base/base_cluster_context.py @@ -17,21 +17,20 @@ import collections from oslo_config import cfg -import sahara.exceptions as e -from sahara.i18n import _ -import sahara.plugins.mapr.abstract.cluster_context as cc -import sahara.plugins.mapr.domain.configuration_file as bcf -import sahara.plugins.mapr.domain.distro as distro -import sahara.plugins.mapr.services.management.management as mng -import sahara.plugins.mapr.services.maprfs.maprfs as mfs -import sahara.plugins.mapr.services.oozie.oozie as oozie -from sahara.plugins.mapr.services.swift import swift -import sahara.plugins.mapr.services.yarn.yarn as yarn -import sahara.plugins.mapr.util.general as g -import sahara.plugins.mapr.util.service_utils as su +import sahara.plugins.exceptions as e +from sahara.plugins import topology_helper as th import sahara.plugins.utils as u -from sahara.topology import topology_helper as th -import sahara.utils.configs as sahara_configs +from sahara_plugin_mapr.i18n import _ +import sahara_plugin_mapr.plugins.mapr.abstract.cluster_context as cc +import sahara_plugin_mapr.plugins.mapr.domain.configuration_file as bcf +import sahara_plugin_mapr.plugins.mapr.domain.distro as distro +import sahara_plugin_mapr.plugins.mapr.services.management.management as mng +import sahara_plugin_mapr.plugins.mapr.services.maprfs.maprfs as mfs +import sahara_plugin_mapr.plugins.mapr.services.oozie.oozie as oozie +from sahara_plugin_mapr.plugins.mapr.services.swift import swift +import sahara_plugin_mapr.plugins.mapr.services.yarn.yarn as yarn +import sahara_plugin_mapr.plugins.mapr.util.general as g +import sahara_plugin_mapr.plugins.mapr.util.service_utils as su CONF = cfg.CONF CONF.import_opt("enable_data_locality", "sahara.topology.topology_helper") @@ -275,12 +274,12 @@ class BaseClusterContext(cc.AbstractClusterContext): def get_service(self, node_process): ui_name = self.get_service_name_by_node_process(node_process) if ui_name is None: - raise e.InvalidDataException( + raise e.PluginInvalidDataException( _('Service not found in services list')) version = self.get_chosen_service_version(ui_name) service = self._find_service_instance(ui_name, version) if service is None: - raise e.InvalidDataException(_('Can not map service')) + raise e.PluginInvalidDataException(_('Can not map service')) return service def _find_service_instance(self, ui_name, version): @@ -417,7 +416,7 @@ class BaseClusterContext(cc.AbstractClusterContext): services = self.get_cluster_services(node_group) user_configs = node_group.configuration() default_configs = self.get_services_configs_dict(services) - return sahara_configs.merge_configs(default_configs, user_configs) + return u.merge_configs(default_configs, user_configs) def get_config_files(self, node_group): services = self.get_cluster_services(node_group) diff --git a/sahara/plugins/mapr/base/base_cluster_validator.py b/sahara_plugin_mapr/plugins/mapr/base/base_cluster_validator.py similarity index 85% rename from sahara/plugins/mapr/base/base_cluster_validator.py rename to sahara_plugin_mapr/plugins/mapr/base/base_cluster_validator.py index 9c7473b..cfc4919 100644 --- a/sahara/plugins/mapr/base/base_cluster_validator.py +++ b/sahara_plugin_mapr/plugins/mapr/base/base_cluster_validator.py @@ -13,9 +13,9 @@ # under the License. -import sahara.plugins.mapr.abstract.cluster_validator as v -import sahara.plugins.mapr.util.validation_utils as vu -import sahara.plugins.mapr.versions.version_handler_factory as vhf +import sahara_plugin_mapr.plugins.mapr.abstract.cluster_validator as v +import sahara_plugin_mapr.plugins.mapr.util.validation_utils as vu +import sahara_plugin_mapr.plugins.mapr.versions.version_handler_factory as vhf class BaseValidator(v.AbstractValidator): diff --git a/sahara/plugins/mapr/base/base_edp_engine.py b/sahara_plugin_mapr/plugins/mapr/base/base_edp_engine.py similarity index 93% rename from sahara/plugins/mapr/base/base_edp_engine.py rename to sahara_plugin_mapr/plugins/mapr/base/base_edp_engine.py index 095a174..c536326 100644 --- a/sahara/plugins/mapr/base/base_edp_engine.py +++ b/sahara_plugin_mapr/plugins/mapr/base/base_edp_engine.py @@ -15,15 +15,14 @@ import os -from sahara import context -import sahara.plugins.mapr.util.maprfs_helper as mfs -import sahara.plugins.mapr.versions.version_handler_factory as vhf +from sahara.plugins import context +from sahara.plugins import edp from sahara.service.edp.job_binaries import manager as jb_manager -import sahara.service.edp.oozie.engine as e -from sahara.utils import edp +import sahara_plugin_mapr.plugins.mapr.util.maprfs_helper as mfs +import sahara_plugin_mapr.plugins.mapr.versions.version_handler_factory as vhf -class MapROozieJobEngine(e.OozieJobEngine): +class MapROozieJobEngine(edp.PluginsOozieJobEngine): def __init__(self, cluster): super(MapROozieJobEngine, self).__init__(cluster) self.cluster_context = self._get_cluster_context(self.cluster) diff --git a/sahara/plugins/mapr/base/base_health_checker.py b/sahara_plugin_mapr/plugins/mapr/base/base_health_checker.py similarity index 92% rename from sahara/plugins/mapr/base/base_health_checker.py rename to sahara_plugin_mapr/plugins/mapr/base/base_health_checker.py index 26d4d98..0221410 100644 --- a/sahara/plugins/mapr/base/base_health_checker.py +++ b/sahara_plugin_mapr/plugins/mapr/base/base_health_checker.py @@ -15,12 +15,12 @@ import functools -from sahara.i18n import _ -import sahara.plugins.mapr.abstract.health_checker as hc -from sahara.plugins.mapr.domain import node_process as np -from sahara.plugins.mapr.services.management import management -from sahara.plugins.mapr.services.spark import spark -from sahara.service.health import health_check_base +from sahara.plugins import health_check_base +from sahara_plugin_mapr.i18n import _ +import sahara_plugin_mapr.plugins.mapr.abstract.health_checker as hc +from sahara_plugin_mapr.plugins.mapr.domain import node_process as np +from sahara_plugin_mapr.plugins.mapr.services.management import management +from sahara_plugin_mapr.plugins.mapr.services.spark import spark class BaseHealthChecker(hc.AbstractHealthChecker): diff --git a/sahara/plugins/mapr/base/base_node_manager.py b/sahara_plugin_mapr/plugins/mapr/base/base_node_manager.py similarity index 85% rename from sahara/plugins/mapr/base/base_node_manager.py rename to sahara_plugin_mapr/plugins/mapr/base/base_node_manager.py index 094a32e..c3f785e 100644 --- a/sahara/plugins/mapr/base/base_node_manager.py +++ b/sahara_plugin_mapr/plugins/mapr/base/base_node_manager.py @@ -19,14 +19,14 @@ from oslo_log import log as logging from oslo_serialization import jsonutils as json from oslo_utils import timeutils -from sahara import context -from sahara.i18n import _ +from sahara.plugins import context import sahara.plugins.exceptions as ex -import sahara.plugins.mapr.abstract.node_manager as s -import sahara.plugins.mapr.services.management.management as mng -import sahara.plugins.mapr.services.maprfs.maprfs as mfs -import sahara.plugins.mapr.util.event_log as el -from sahara.utils import cluster_progress_ops as cpo +from sahara.plugins import utils +from sahara_plugin_mapr.i18n import _ +import sahara_plugin_mapr.plugins.mapr.abstract.node_manager as s +import sahara_plugin_mapr.plugins.mapr.services.management.management as mng +import sahara_plugin_mapr.plugins.mapr.services.maprfs.maprfs as mfs +import sahara_plugin_mapr.plugins.mapr.util.event_log as el LOG = logging.getLogger(__name__) @@ -80,27 +80,29 @@ class BaseNodeManager(s.AbstractNodeManager): others = filter( lambda i: not cluster_context.check_for_process(i, mfs.CLDB), instances) - cpo.add_provisioning_step(cluster_context.cluster.id, - _("Start ZooKeepers nodes"), len(zookeepers)) + utils.add_provisioning_step(cluster_context.cluster.id, + _("Start ZooKeepers nodes"), + len(zookeepers)) self._start_zk_nodes(zookeepers) - cpo.add_provisioning_step(cluster_context.cluster.id, - _("Start CLDB nodes"), len(cldbs)) + utils.add_provisioning_step(cluster_context.cluster.id, + _("Start CLDB nodes"), len(cldbs)) self._start_cldb_nodes(cldbs) if others: - cpo.add_provisioning_step(cluster_context.cluster.id, - _("Start non-CLDB nodes"), - len(list(others))) + utils.add_provisioning_step(cluster_context.cluster.id, + _("Start non-CLDB nodes"), + len(list(others))) self._start_non_cldb_nodes(others) self._await_cldb(cluster_context, instances) def stop(self, cluster_context, instances=None): instances = instances or cluster_context.get_instances() zookeepers = cluster_context.filter_instances(instances, mng.ZOOKEEPER) - cpo.add_provisioning_step(cluster_context.cluster.id, - _("Stop ZooKeepers nodes"), len(zookeepers)) + utils.add_provisioning_step(cluster_context.cluster.id, + _("Stop ZooKeepers nodes"), + len(zookeepers)) self._stop_zk_nodes(zookeepers) - cpo.add_provisioning_step(cluster_context.cluster.id, - _("Stop Warden nodes"), len(instances)) + utils.add_provisioning_step(cluster_context.cluster.id, + _("Stop Warden nodes"), len(instances)) self._stop_warden_on_nodes(instances) def _await_cldb(self, cluster_context, instances=None, timeout=600): @@ -133,13 +135,13 @@ class BaseNodeManager(s.AbstractNodeManager): raise ex.HadoopProvisionError(_("CLDB failed to start")) def _start_nodes(self, instances, sys_service): - with context.ThreadGroup() as tg: + with context.PluginsThreadGroup() as tg: for instance in instances: tg.spawn('start-%s-%s' % (sys_service, instance.id), self._start_service, instance, sys_service) def _stop_nodes(self, instances, sys_service): - with context.ThreadGroup() as tg: + with context.PluginsThreadGroup() as tg: for instance in instances: tg.spawn('stop-%s-%s' % (sys_service, instance.id), self._stop_service, instance, sys_service) diff --git a/sahara/plugins/mapr/base/base_version_handler.py b/sahara_plugin_mapr/plugins/mapr/base/base_version_handler.py similarity index 91% rename from sahara/plugins/mapr/base/base_version_handler.py rename to sahara_plugin_mapr/plugins/mapr/base/base_version_handler.py index b070c96..a25fd39 100644 --- a/sahara/plugins/mapr/base/base_version_handler.py +++ b/sahara_plugin_mapr/plugins/mapr/base/base_version_handler.py @@ -15,23 +15,23 @@ import collections as c -from sahara.i18n import _ -import sahara.plugins.mapr.abstract.version_handler as vh -import sahara.plugins.mapr.base.base_cluster_configurer as base_conf -import sahara.plugins.mapr.base.base_cluster_validator as bv -import sahara.plugins.mapr.base.base_edp_engine as edp -import sahara.plugins.mapr.base.base_health_checker as health -import sahara.plugins.mapr.base.base_node_manager as bs -from sahara.plugins.mapr import images -import sahara.plugins.mapr.util.general as util import sahara.plugins.provisioning as p import sahara.plugins.utils as u +from sahara_plugin_mapr.i18n import _ +import sahara_plugin_mapr.plugins.mapr.abstract.version_handler as vh +import sahara_plugin_mapr.plugins.mapr.base.base_cluster_configurer as b_conf +import sahara_plugin_mapr.plugins.mapr.base.base_cluster_validator as bv +import sahara_plugin_mapr.plugins.mapr.base.base_edp_engine as edp +import sahara_plugin_mapr.plugins.mapr.base.base_health_checker as health +import sahara_plugin_mapr.plugins.mapr.base.base_node_manager as bs +from sahara_plugin_mapr.plugins.mapr import images +import sahara_plugin_mapr.plugins.mapr.util.general as util class BaseVersionHandler(vh.AbstractVersionHandler): def __init__(self): self._validator = bv.BaseValidator() - self._configurer = base_conf.BaseConfigurer() + self._configurer = b_conf.BaseConfigurer() self._health_checker = health.BaseHealthChecker() self._node_manager = bs.BaseNodeManager() self._version = None diff --git a/sahara/plugins/mapr/domain/__init__.py b/sahara_plugin_mapr/plugins/mapr/domain/__init__.py similarity index 100% rename from sahara/plugins/mapr/domain/__init__.py rename to sahara_plugin_mapr/plugins/mapr/domain/__init__.py diff --git a/sahara/plugins/mapr/domain/configuration_file.py b/sahara_plugin_mapr/plugins/mapr/domain/configuration_file.py similarity index 94% rename from sahara/plugins/mapr/domain/configuration_file.py rename to sahara_plugin_mapr/plugins/mapr/domain/configuration_file.py index 4dc751b..e428520 100644 --- a/sahara/plugins/mapr/domain/configuration_file.py +++ b/sahara_plugin_mapr/plugins/mapr/domain/configuration_file.py @@ -20,9 +20,9 @@ import re import jinja2 as j2 import six -import sahara.exceptions as e -from sahara.i18n import _ -import sahara.utils.xmlutils as xml +import sahara.plugins.exceptions as e +import sahara.plugins.utils as utils +from sahara_plugin_mapr.i18n import _ @six.add_metaclass(abc.ABCMeta) @@ -88,12 +88,12 @@ class HadoopXML(BaseConfigurationFile): super(HadoopXML, self).__init__(file_name) def parse(self, content): - configs = xml.parse_hadoop_xml_with_name_and_value(content) + configs = utils.parse_hadoop_xml_with_name_and_value(content) for cfg in configs: self.add_property(cfg["name"], cfg["value"]) def render(self): - return xml.create_hadoop_xml(self._config_dict) + return utils.create_hadoop_xml(self._config_dict) class RawFile(BaseConfigurationFile): @@ -141,7 +141,8 @@ class TemplateFile(BaseConfigurationFile): if template: return template.render(arg_dict) else: - raise e.InvalidDataException(_('Template object must be defined')) + raise e.PluginsInvalidDataException( + _('Template object must be defined')) def render(self): return self._j2_render(self._template, self._config_dict) diff --git a/sahara/plugins/mapr/domain/distro.py b/sahara_plugin_mapr/plugins/mapr/domain/distro.py similarity index 100% rename from sahara/plugins/mapr/domain/distro.py rename to sahara_plugin_mapr/plugins/mapr/domain/distro.py diff --git a/sahara/plugins/mapr/domain/node_process.py b/sahara_plugin_mapr/plugins/mapr/domain/node_process.py similarity index 96% rename from sahara/plugins/mapr/domain/node_process.py rename to sahara_plugin_mapr/plugins/mapr/domain/node_process.py index fd2e267..ff33c21 100644 --- a/sahara/plugins/mapr/domain/node_process.py +++ b/sahara_plugin_mapr/plugins/mapr/domain/node_process.py @@ -15,9 +15,9 @@ import oslo_serialization.jsonutils as json -from sahara.i18n import _ -import sahara.plugins.mapr.util.general as util -from sahara.utils import poll_utils as polls +from sahara.plugins import utils as plugin_utils +from sahara_plugin_mapr.i18n import _ +import sahara_plugin_mapr.plugins.mapr.util.general as util WARDEN_MANAGED_CMD = ('sudo -u mapr maprcli node services' @@ -77,7 +77,7 @@ class NodeProcess(object): 'instance': instance.instance_name, 'status': status.name, } - return polls.poll( + return plugin_utils.poll( get_status=lambda: self.status(instance) == status, operation_name=operation_name.format(**args), timeout=timeout, diff --git a/sahara/plugins/mapr/domain/service.py b/sahara_plugin_mapr/plugins/mapr/domain/service.py similarity index 94% rename from sahara/plugins/mapr/domain/service.py rename to sahara_plugin_mapr/plugins/mapr/domain/service.py index 12124c9..742a13f 100644 --- a/sahara/plugins/mapr/domain/service.py +++ b/sahara_plugin_mapr/plugins/mapr/domain/service.py @@ -17,15 +17,14 @@ from oslo_log import log as logging from oslo_serialization import jsonutils as json import six -import sahara.exceptions as e -from sahara.i18n import _ import sahara.plugins.exceptions as ex -from sahara.plugins.mapr.util import commands as cmd -from sahara.plugins.mapr.util import event_log as el -from sahara.plugins.mapr.util import general as g -from sahara.plugins.mapr.util import service_utils as su import sahara.plugins.provisioning as p -from sahara.utils import files +from sahara.plugins import utils +from sahara_plugin_mapr.i18n import _ +from sahara_plugin_mapr.plugins.mapr.util import commands as cmd +from sahara_plugin_mapr.plugins.mapr.util import event_log as el +from sahara_plugin_mapr.plugins.mapr.util import general as g +from sahara_plugin_mapr.plugins.mapr.util import service_utils as su LOG = logging.getLogger(__name__) @@ -160,7 +159,7 @@ class Service(object): return {self.ui_name: result} def _load_config_file(self, file_path=None): - return json.loads(files.get_file_text(file_path)) + return json.loads(utils.get_file_text(file_path, 'sahara_plugin_mapr')) def get_config_files(self, cluster_context, configs, instance=None): return [] @@ -180,7 +179,7 @@ class Service(object): raise ex.HadoopProvisionError(_("Config missing 'name'")) if conf_value is None: - raise e.InvalidDataException( + raise ex.PluginInvalidDataException( _("Config '%s' missing 'value'") % conf_name) if high_priority or item.get('priority', 2) == 1: diff --git a/sahara/plugins/mapr/images.py b/sahara_plugin_mapr/plugins/mapr/images.py similarity index 94% rename from sahara/plugins/mapr/images.py rename to sahara_plugin_mapr/plugins/mapr/images.py index aacf9ca..aede47d 100644 --- a/sahara/plugins/mapr/images.py +++ b/sahara_plugin_mapr/plugins/mapr/images.py @@ -19,7 +19,8 @@ from sahara.plugins import utils as plugin_utils _validator = images.SaharaImageValidator.from_yaml( 'plugins/mapr/resources/images/image.yaml', - resource_roots=['plugins/mapr/resources/images']) + resource_roots=['plugins/mapr/resources/images'], + package='sahara_plugin_mapr') def get_image_arguments(): diff --git a/sahara/plugins/mapr/plugin.py b/sahara_plugin_mapr/plugins/mapr/plugin.py similarity index 97% rename from sahara/plugins/mapr/plugin.py rename to sahara_plugin_mapr/plugins/mapr/plugin.py index ddc0197..743d2ea 100644 --- a/sahara/plugins/mapr/plugin.py +++ b/sahara_plugin_mapr/plugins/mapr/plugin.py @@ -13,9 +13,9 @@ # under the License. -from sahara.i18n import _ -import sahara.plugins.mapr.versions.version_handler_factory as vhf import sahara.plugins.provisioning as p +from sahara_plugin_mapr.i18n import _ +import sahara_plugin_mapr.plugins.mapr.versions.version_handler_factory as vhf class MapRPlugin(p.ProvisioningPluginBase): diff --git a/sahara/plugins/mapr/resources/add_mapr_repo.sh b/sahara_plugin_mapr/plugins/mapr/resources/add_mapr_repo.sh similarity index 100% rename from sahara/plugins/mapr/resources/add_mapr_repo.sh rename to sahara_plugin_mapr/plugins/mapr/resources/add_mapr_repo.sh diff --git a/sahara/plugins/mapr/resources/add_security_repos.sh b/sahara_plugin_mapr/plugins/mapr/resources/add_security_repos.sh similarity index 100% rename from sahara/plugins/mapr/resources/add_security_repos.sh rename to sahara_plugin_mapr/plugins/mapr/resources/add_security_repos.sh diff --git a/sahara/plugins/mapr/resources/create_disk_list_file.sh b/sahara_plugin_mapr/plugins/mapr/resources/create_disk_list_file.sh similarity index 100% rename from sahara/plugins/mapr/resources/create_disk_list_file.sh rename to sahara_plugin_mapr/plugins/mapr/resources/create_disk_list_file.sh diff --git a/sahara/plugins/mapr/resources/images/centos/disable_firewall b/sahara_plugin_mapr/plugins/mapr/resources/images/centos/disable_firewall similarity index 100% rename from sahara/plugins/mapr/resources/images/centos/disable_firewall rename to sahara_plugin_mapr/plugins/mapr/resources/images/centos/disable_firewall diff --git a/sahara/plugins/mapr/resources/images/centos/epel_dependencies b/sahara_plugin_mapr/plugins/mapr/resources/images/centos/epel_dependencies similarity index 100% rename from sahara/plugins/mapr/resources/images/centos/epel_dependencies rename to sahara_plugin_mapr/plugins/mapr/resources/images/centos/epel_dependencies diff --git a/sahara/plugins/mapr/resources/images/centos/install_mapr_core_repository b/sahara_plugin_mapr/plugins/mapr/resources/images/centos/install_mapr_core_repository similarity index 100% rename from sahara/plugins/mapr/resources/images/centos/install_mapr_core_repository rename to sahara_plugin_mapr/plugins/mapr/resources/images/centos/install_mapr_core_repository diff --git a/sahara/plugins/mapr/resources/images/centos/install_mapr_eco_repository b/sahara_plugin_mapr/plugins/mapr/resources/images/centos/install_mapr_eco_repository similarity index 100% rename from sahara/plugins/mapr/resources/images/centos/install_mapr_eco_repository rename to sahara_plugin_mapr/plugins/mapr/resources/images/centos/install_mapr_eco_repository diff --git a/sahara/plugins/mapr/resources/images/centos/install_openjdk b/sahara_plugin_mapr/plugins/mapr/resources/images/centos/install_openjdk similarity index 100% rename from sahara/plugins/mapr/resources/images/centos/install_openjdk rename to sahara_plugin_mapr/plugins/mapr/resources/images/centos/install_openjdk diff --git a/sahara/plugins/mapr/resources/images/centos/install_scala b/sahara_plugin_mapr/plugins/mapr/resources/images/centos/install_scala similarity index 100% rename from sahara/plugins/mapr/resources/images/centos/install_scala rename to sahara_plugin_mapr/plugins/mapr/resources/images/centos/install_scala diff --git a/sahara/plugins/mapr/resources/images/centos/requiretty b/sahara_plugin_mapr/plugins/mapr/resources/images/centos/requiretty similarity index 100% rename from sahara/plugins/mapr/resources/images/centos/requiretty rename to sahara_plugin_mapr/plugins/mapr/resources/images/centos/requiretty diff --git a/sahara/plugins/mapr/resources/images/centos/selinux_permissive b/sahara_plugin_mapr/plugins/mapr/resources/images/centos/selinux_permissive similarity index 100% rename from sahara/plugins/mapr/resources/images/centos/selinux_permissive rename to sahara_plugin_mapr/plugins/mapr/resources/images/centos/selinux_permissive diff --git a/sahara/plugins/mapr/resources/images/centos/update_repository b/sahara_plugin_mapr/plugins/mapr/resources/images/centos/update_repository similarity index 100% rename from sahara/plugins/mapr/resources/images/centos/update_repository rename to sahara_plugin_mapr/plugins/mapr/resources/images/centos/update_repository diff --git a/sahara/plugins/mapr/resources/images/common/configure_extjs b/sahara_plugin_mapr/plugins/mapr/resources/images/common/configure_extjs similarity index 100% rename from sahara/plugins/mapr/resources/images/common/configure_extjs rename to sahara_plugin_mapr/plugins/mapr/resources/images/common/configure_extjs diff --git a/sahara/plugins/mapr/resources/images/common/oracle_java b/sahara_plugin_mapr/plugins/mapr/resources/images/common/oracle_java similarity index 100% rename from sahara/plugins/mapr/resources/images/common/oracle_java rename to sahara_plugin_mapr/plugins/mapr/resources/images/common/oracle_java diff --git a/sahara/plugins/mapr/resources/images/common/resources/package_utils.sh b/sahara_plugin_mapr/plugins/mapr/resources/images/common/resources/package_utils.sh similarity index 100% rename from sahara/plugins/mapr/resources/images/common/resources/package_utils.sh rename to sahara_plugin_mapr/plugins/mapr/resources/images/common/resources/package_utils.sh diff --git a/sahara/plugins/mapr/resources/images/common/resources/packages.json b/sahara_plugin_mapr/plugins/mapr/resources/images/common/resources/packages.json similarity index 100% rename from sahara/plugins/mapr/resources/images/common/resources/packages.json rename to sahara_plugin_mapr/plugins/mapr/resources/images/common/resources/packages.json diff --git a/sahara/plugins/mapr/resources/images/common/resources/spec_5.1.0.json b/sahara_plugin_mapr/plugins/mapr/resources/images/common/resources/spec_5.1.0.json similarity index 100% rename from sahara/plugins/mapr/resources/images/common/resources/spec_5.1.0.json rename to sahara_plugin_mapr/plugins/mapr/resources/images/common/resources/spec_5.1.0.json diff --git a/sahara/plugins/mapr/resources/images/common/resources/spec_5.2.0.json b/sahara_plugin_mapr/plugins/mapr/resources/images/common/resources/spec_5.2.0.json similarity index 100% rename from sahara/plugins/mapr/resources/images/common/resources/spec_5.2.0.json rename to sahara_plugin_mapr/plugins/mapr/resources/images/common/resources/spec_5.2.0.json diff --git a/sahara/plugins/mapr/resources/images/common/resources/spec_5.2.0_ubuntu.json b/sahara_plugin_mapr/plugins/mapr/resources/images/common/resources/spec_5.2.0_ubuntu.json similarity index 100% rename from sahara/plugins/mapr/resources/images/common/resources/spec_5.2.0_ubuntu.json rename to sahara_plugin_mapr/plugins/mapr/resources/images/common/resources/spec_5.2.0_ubuntu.json diff --git a/sahara/plugins/mapr/resources/images/common/resources/versions.py b/sahara_plugin_mapr/plugins/mapr/resources/images/common/resources/versions.py similarity index 100% rename from sahara/plugins/mapr/resources/images/common/resources/versions.py rename to sahara_plugin_mapr/plugins/mapr/resources/images/common/resources/versions.py diff --git a/sahara/plugins/mapr/resources/images/image.yaml b/sahara_plugin_mapr/plugins/mapr/resources/images/image.yaml similarity index 100% rename from sahara/plugins/mapr/resources/images/image.yaml rename to sahara_plugin_mapr/plugins/mapr/resources/images/image.yaml diff --git a/sahara/plugins/mapr/resources/images/ubuntu/install_mapr_core_repository b/sahara_plugin_mapr/plugins/mapr/resources/images/ubuntu/install_mapr_core_repository similarity index 100% rename from sahara/plugins/mapr/resources/images/ubuntu/install_mapr_core_repository rename to sahara_plugin_mapr/plugins/mapr/resources/images/ubuntu/install_mapr_core_repository diff --git a/sahara/plugins/mapr/resources/images/ubuntu/install_mapr_dependencies b/sahara_plugin_mapr/plugins/mapr/resources/images/ubuntu/install_mapr_dependencies similarity index 100% rename from sahara/plugins/mapr/resources/images/ubuntu/install_mapr_dependencies rename to sahara_plugin_mapr/plugins/mapr/resources/images/ubuntu/install_mapr_dependencies diff --git a/sahara/plugins/mapr/resources/images/ubuntu/install_mapr_eco_repository b/sahara_plugin_mapr/plugins/mapr/resources/images/ubuntu/install_mapr_eco_repository similarity index 100% rename from sahara/plugins/mapr/resources/images/ubuntu/install_mapr_eco_repository rename to sahara_plugin_mapr/plugins/mapr/resources/images/ubuntu/install_mapr_eco_repository diff --git a/sahara/plugins/mapr/resources/images/ubuntu/install_openjdk b/sahara_plugin_mapr/plugins/mapr/resources/images/ubuntu/install_openjdk similarity index 100% rename from sahara/plugins/mapr/resources/images/ubuntu/install_openjdk rename to sahara_plugin_mapr/plugins/mapr/resources/images/ubuntu/install_openjdk diff --git a/sahara/plugins/mapr/resources/images/ubuntu/install_scala b/sahara_plugin_mapr/plugins/mapr/resources/images/ubuntu/install_scala similarity index 100% rename from sahara/plugins/mapr/resources/images/ubuntu/install_scala rename to sahara_plugin_mapr/plugins/mapr/resources/images/ubuntu/install_scala diff --git a/sahara/plugins/mapr/resources/images/ubuntu/update_repository b/sahara_plugin_mapr/plugins/mapr/resources/images/ubuntu/update_repository similarity index 100% rename from sahara/plugins/mapr/resources/images/ubuntu/update_repository rename to sahara_plugin_mapr/plugins/mapr/resources/images/ubuntu/update_repository diff --git a/sahara/plugins/mapr/resources/install_java.sh b/sahara_plugin_mapr/plugins/mapr/resources/install_java.sh similarity index 100% rename from sahara/plugins/mapr/resources/install_java.sh rename to sahara_plugin_mapr/plugins/mapr/resources/install_java.sh diff --git a/sahara/plugins/mapr/resources/install_mysql.sh b/sahara_plugin_mapr/plugins/mapr/resources/install_mysql.sh similarity index 100% rename from sahara/plugins/mapr/resources/install_mysql.sh rename to sahara_plugin_mapr/plugins/mapr/resources/install_mysql.sh diff --git a/sahara/plugins/mapr/resources/install_mysql_client.sh b/sahara_plugin_mapr/plugins/mapr/resources/install_mysql_client.sh similarity index 100% rename from sahara/plugins/mapr/resources/install_mysql_client.sh rename to sahara_plugin_mapr/plugins/mapr/resources/install_mysql_client.sh diff --git a/sahara/plugins/mapr/resources/install_scala.sh b/sahara_plugin_mapr/plugins/mapr/resources/install_scala.sh similarity index 100% rename from sahara/plugins/mapr/resources/install_scala.sh rename to sahara_plugin_mapr/plugins/mapr/resources/install_scala.sh diff --git a/sahara/plugins/mapr/resources/topology.sh b/sahara_plugin_mapr/plugins/mapr/resources/topology.sh similarity index 100% rename from sahara/plugins/mapr/resources/topology.sh rename to sahara_plugin_mapr/plugins/mapr/resources/topology.sh diff --git a/sahara/plugins/mapr/services/__init__.py b/sahara_plugin_mapr/plugins/mapr/services/__init__.py similarity index 100% rename from sahara/plugins/mapr/services/__init__.py rename to sahara_plugin_mapr/plugins/mapr/services/__init__.py diff --git a/sahara/plugins/mapr/services/drill/__init__.py b/sahara_plugin_mapr/plugins/mapr/services/drill/__init__.py similarity index 100% rename from sahara/plugins/mapr/services/drill/__init__.py rename to sahara_plugin_mapr/plugins/mapr/services/drill/__init__.py diff --git a/sahara/plugins/mapr/services/drill/drill.py b/sahara_plugin_mapr/plugins/mapr/services/drill/drill.py similarity index 91% rename from sahara/plugins/mapr/services/drill/drill.py rename to sahara_plugin_mapr/plugins/mapr/services/drill/drill.py index de2e9f3..efebb4c 100644 --- a/sahara/plugins/mapr/services/drill/drill.py +++ b/sahara_plugin_mapr/plugins/mapr/services/drill/drill.py @@ -13,10 +13,10 @@ # under the License. -import sahara.plugins.mapr.domain.node_process as np -import sahara.plugins.mapr.domain.service as s -import sahara.plugins.mapr.util.commands as cmd -import sahara.plugins.mapr.util.validation_utils as vu +import sahara_plugin_mapr.plugins.mapr.domain.node_process as np +import sahara_plugin_mapr.plugins.mapr.domain.service as s +import sahara_plugin_mapr.plugins.mapr.util.commands as cmd +import sahara_plugin_mapr.plugins.mapr.util.validation_utils as vu DRILL = np.NodeProcess( name='drill-bits', diff --git a/sahara/plugins/mapr/services/flume/__init__.py b/sahara_plugin_mapr/plugins/mapr/services/flume/__init__.py similarity index 100% rename from sahara/plugins/mapr/services/flume/__init__.py rename to sahara_plugin_mapr/plugins/mapr/services/flume/__init__.py diff --git a/sahara/plugins/mapr/services/flume/flume.py b/sahara_plugin_mapr/plugins/mapr/services/flume/flume.py similarity index 86% rename from sahara/plugins/mapr/services/flume/flume.py rename to sahara_plugin_mapr/plugins/mapr/services/flume/flume.py index ba0d7c2..2c35d10 100644 --- a/sahara/plugins/mapr/services/flume/flume.py +++ b/sahara_plugin_mapr/plugins/mapr/services/flume/flume.py @@ -13,9 +13,9 @@ # under the License. -import sahara.plugins.mapr.domain.node_process as np -import sahara.plugins.mapr.domain.service as s -import sahara.plugins.mapr.util.validation_utils as vu +import sahara_plugin_mapr.plugins.mapr.domain.node_process as np +import sahara_plugin_mapr.plugins.mapr.domain.service as s +import sahara_plugin_mapr.plugins.mapr.util.validation_utils as vu FLUME = np.NodeProcess( diff --git a/sahara/plugins/mapr/services/hbase/__init__.py b/sahara_plugin_mapr/plugins/mapr/services/hbase/__init__.py similarity index 100% rename from sahara/plugins/mapr/services/hbase/__init__.py rename to sahara_plugin_mapr/plugins/mapr/services/hbase/__init__.py diff --git a/sahara/plugins/mapr/services/hbase/hbase.py b/sahara_plugin_mapr/plugins/mapr/services/hbase/hbase.py similarity index 93% rename from sahara/plugins/mapr/services/hbase/hbase.py rename to sahara_plugin_mapr/plugins/mapr/services/hbase/hbase.py index d608d3f..81613ce 100644 --- a/sahara/plugins/mapr/services/hbase/hbase.py +++ b/sahara_plugin_mapr/plugins/mapr/services/hbase/hbase.py @@ -13,10 +13,10 @@ # under the License. -import sahara.plugins.mapr.domain.configuration_file as bcf -import sahara.plugins.mapr.domain.node_process as np -import sahara.plugins.mapr.domain.service as s -import sahara.plugins.mapr.util.validation_utils as vu +import sahara_plugin_mapr.plugins.mapr.domain.configuration_file as bcf +import sahara_plugin_mapr.plugins.mapr.domain.node_process as np +import sahara_plugin_mapr.plugins.mapr.domain.service as s +import sahara_plugin_mapr.plugins.mapr.util.validation_utils as vu HBASE_MASTER = np.NodeProcess( diff --git a/sahara/plugins/mapr/services/hbase/resources/hbase-default.json b/sahara_plugin_mapr/plugins/mapr/services/hbase/resources/hbase-default.json similarity index 100% rename from sahara/plugins/mapr/services/hbase/resources/hbase-default.json rename to sahara_plugin_mapr/plugins/mapr/services/hbase/resources/hbase-default.json diff --git a/sahara/plugins/mapr/services/hive/__init__.py b/sahara_plugin_mapr/plugins/mapr/services/hive/__init__.py similarity index 100% rename from sahara/plugins/mapr/services/hive/__init__.py rename to sahara_plugin_mapr/plugins/mapr/services/hive/__init__.py diff --git a/sahara/plugins/mapr/services/hive/hive.py b/sahara_plugin_mapr/plugins/mapr/services/hive/hive.py similarity index 88% rename from sahara/plugins/mapr/services/hive/hive.py rename to sahara_plugin_mapr/plugins/mapr/services/hive/hive.py index 6023c9f..b9bc642 100644 --- a/sahara/plugins/mapr/services/hive/hive.py +++ b/sahara_plugin_mapr/plugins/mapr/services/hive/hive.py @@ -16,13 +16,13 @@ from oslo_log import log as logging from sahara.i18n import _ -import sahara.plugins.mapr.domain.configuration_file as bcf -import sahara.plugins.mapr.domain.node_process as np -import sahara.plugins.mapr.domain.service as s -import sahara.plugins.mapr.services.sentry.sentry as sentry -import sahara.plugins.mapr.util.event_log as el -import sahara.plugins.mapr.util.validation_utils as vu import sahara.utils.files as files +import sahara_plugin_mapr.plugins.mapr.domain.configuration_file as bcf +import sahara_plugin_mapr.plugins.mapr.domain.node_process as np +import sahara_plugin_mapr.plugins.mapr.domain.service as s +import sahara_plugin_mapr.plugins.mapr.services.sentry.sentry as sentry +import sahara_plugin_mapr.plugins.mapr.util.event_log as el +import sahara_plugin_mapr.plugins.mapr.util.validation_utils as vu LOG = logging.getLogger(__name__) @@ -58,7 +58,8 @@ class Hive(s.Service): hive_site.remote_path = self.conf_dir(cluster_context) if instance: hive_site.fetch(instance) - hive_site.parse(files.get_file_text(hive_default)) + hive_site.parse(files.get_file_text(hive_default, + 'sahara_plugin_mapr')) hive_site.add_properties(self._get_hive_site_props(cluster_context)) sentry_host = cluster_context.get_instance(sentry.SENTRY) if sentry_host: @@ -74,19 +75,21 @@ class Hive(s.Service): 'sentry-default.xml' sentry_db = \ 'plugins/mapr/services/hive/resources/sentry-db.xml' - hive_site.parse(files.get_file_text(sentry_default)) + hive_site.parse(files.get_file_text(sentry_default, + 'sahara_plugin_mapr')) hive_site.add_property('hive.sentry.conf.url', 'file://%s/sentry-site.xml' % sentry_service.conf_dir( cluster_context)) if sentry_mode == sentry.DB_STORAGE_SENTRY_MODE: - hive_site.parse(files.get_file_text(sentry_db)) + hive_site.parse(files.get_file_text(sentry_db, + 'sahara_plugin_mapr')) return [hive_site] def _get_hive_site_props(self, cluster_context): # Import here to resolve circular dependency - from sahara.plugins.mapr.services.mysql import mysql + from sahara_plugin_mapr.plugins.mapr.services.mysql import mysql zookeepers = cluster_context.get_zookeeper_nodes_ip() metastore_specs = mysql.MySQL.METASTORE_SPECS @@ -104,7 +107,7 @@ class Hive(s.Service): def _get_jdbc_uri(self, cluster_context): # Import here to resolve circular dependency - from sahara.plugins.mapr.services.mysql import mysql + from sahara_plugin_mapr.plugins.mapr.services.mysql import mysql jdbc_uri = ('jdbc:mysql://%(db_host)s:%(db_port)s/%(db_name)s?' 'createDatabaseIfNotExist=true') @@ -122,7 +125,7 @@ class Hive(s.Service): def post_start(self, cluster_context, instances): # Import here to resolve circular dependency - import sahara.plugins.mapr.services.maprfs.maprfs as mfs + import sahara_plugin_mapr.plugins.mapr.services.maprfs.maprfs as mfs create_path = lambda p: 'sudo -u mapr hadoop fs -mkdir %s' % p check_path = 'sudo -u mapr hadoop fs -ls %s' diff --git a/sahara/plugins/mapr/services/hive/resources/hive-default.xml b/sahara_plugin_mapr/plugins/mapr/services/hive/resources/hive-default.xml similarity index 100% rename from sahara/plugins/mapr/services/hive/resources/hive-default.xml rename to sahara_plugin_mapr/plugins/mapr/services/hive/resources/hive-default.xml diff --git a/sahara/plugins/mapr/services/hive/resources/sentry-db.xml b/sahara_plugin_mapr/plugins/mapr/services/hive/resources/sentry-db.xml similarity index 100% rename from sahara/plugins/mapr/services/hive/resources/sentry-db.xml rename to sahara_plugin_mapr/plugins/mapr/services/hive/resources/sentry-db.xml diff --git a/sahara/plugins/mapr/services/hive/resources/sentry-default.xml b/sahara_plugin_mapr/plugins/mapr/services/hive/resources/sentry-default.xml similarity index 100% rename from sahara/plugins/mapr/services/hive/resources/sentry-default.xml rename to sahara_plugin_mapr/plugins/mapr/services/hive/resources/sentry-default.xml diff --git a/sahara/plugins/mapr/services/httpfs/__init__.py b/sahara_plugin_mapr/plugins/mapr/services/httpfs/__init__.py similarity index 100% rename from sahara/plugins/mapr/services/httpfs/__init__.py rename to sahara_plugin_mapr/plugins/mapr/services/httpfs/__init__.py diff --git a/sahara/plugins/mapr/services/httpfs/httpfs.py b/sahara_plugin_mapr/plugins/mapr/services/httpfs/httpfs.py similarity index 85% rename from sahara/plugins/mapr/services/httpfs/httpfs.py rename to sahara_plugin_mapr/plugins/mapr/services/httpfs/httpfs.py index 015e969..ff67e06 100644 --- a/sahara/plugins/mapr/services/httpfs/httpfs.py +++ b/sahara_plugin_mapr/plugins/mapr/services/httpfs/httpfs.py @@ -13,9 +13,9 @@ # under the License. -import sahara.plugins.mapr.domain.node_process as np -import sahara.plugins.mapr.domain.service as s -import sahara.plugins.mapr.util.validation_utils as vu +import sahara_plugin_mapr.plugins.mapr.domain.node_process as np +import sahara_plugin_mapr.plugins.mapr.domain.service as s +import sahara_plugin_mapr.plugins.mapr.util.validation_utils as vu HTTP_FS = np.NodeProcess( diff --git a/sahara/plugins/mapr/services/httpfs/resources/httpfs-default.json b/sahara_plugin_mapr/plugins/mapr/services/httpfs/resources/httpfs-default.json similarity index 100% rename from sahara/plugins/mapr/services/httpfs/resources/httpfs-default.json rename to sahara_plugin_mapr/plugins/mapr/services/httpfs/resources/httpfs-default.json diff --git a/sahara/plugins/mapr/services/hue/__init__.py b/sahara_plugin_mapr/plugins/mapr/services/hue/__init__.py similarity index 100% rename from sahara/plugins/mapr/services/hue/__init__.py rename to sahara_plugin_mapr/plugins/mapr/services/hue/__init__.py diff --git a/sahara/plugins/mapr/services/hue/hue.py b/sahara_plugin_mapr/plugins/mapr/services/hue/hue.py similarity index 87% rename from sahara/plugins/mapr/services/hue/hue.py rename to sahara_plugin_mapr/plugins/mapr/services/hue/hue.py index eb6febf..0f0fafc 100644 --- a/sahara/plugins/mapr/services/hue/hue.py +++ b/sahara_plugin_mapr/plugins/mapr/services/hue/hue.py @@ -19,27 +19,27 @@ import string from oslo_log import log as logging -from sahara.i18n import _ -import sahara.plugins.mapr.domain.configuration_file as bcf -import sahara.plugins.mapr.domain.node_process as np -import sahara.plugins.mapr.domain.service as s -import sahara.plugins.mapr.services.hbase.hbase as hbase -import sahara.plugins.mapr.services.hive.hive as hive -import sahara.plugins.mapr.services.httpfs.httpfs as httpfs -import sahara.plugins.mapr.services.impala.impala as impala -import sahara.plugins.mapr.services.mapreduce.mapreduce as mr -import sahara.plugins.mapr.services.mysql.mysql as mysql -import sahara.plugins.mapr.services.oozie.oozie as oozie -import sahara.plugins.mapr.services.sentry.sentry as sentry -import sahara.plugins.mapr.services.spark.spark as spark -import sahara.plugins.mapr.services.sqoop.sqoop2 as sqoop -import sahara.plugins.mapr.services.yarn.yarn as yarn -import sahara.plugins.mapr.util.event_log as el -import sahara.plugins.mapr.util.general as g -import sahara.plugins.mapr.util.password_utils as pu -import sahara.plugins.mapr.util.validation_utils as vu import sahara.plugins.provisioning as p -import sahara.utils.files as files +import sahara.plugins.utils as utils +from sahara_plugin_mapr.i18n import _ +import sahara_plugin_mapr.plugins.mapr.domain.configuration_file as bcf +import sahara_plugin_mapr.plugins.mapr.domain.node_process as np +import sahara_plugin_mapr.plugins.mapr.domain.service as s +import sahara_plugin_mapr.plugins.mapr.services.hbase.hbase as hbase +import sahara_plugin_mapr.plugins.mapr.services.hive.hive as hive +import sahara_plugin_mapr.plugins.mapr.services.httpfs.httpfs as httpfs +import sahara_plugin_mapr.plugins.mapr.services.impala.impala as impala +import sahara_plugin_mapr.plugins.mapr.services.mapreduce.mapreduce as mr +import sahara_plugin_mapr.plugins.mapr.services.mysql.mysql as mysql +import sahara_plugin_mapr.plugins.mapr.services.oozie.oozie as oozie +import sahara_plugin_mapr.plugins.mapr.services.sentry.sentry as sentry +import sahara_plugin_mapr.plugins.mapr.services.spark.spark as spark +import sahara_plugin_mapr.plugins.mapr.services.sqoop.sqoop2 as sqoop +import sahara_plugin_mapr.plugins.mapr.services.yarn.yarn as yarn +import sahara_plugin_mapr.plugins.mapr.util.event_log as el +import sahara_plugin_mapr.plugins.mapr.util.general as g +import sahara_plugin_mapr.plugins.mapr.util.password_utils as pu +import sahara_plugin_mapr.plugins.mapr.util.validation_utils as vu LOG = logging.getLogger(__name__) @@ -102,7 +102,8 @@ class Hue(s.Service): # hue.ini hue_ini = bcf.TemplateFile("hue.ini") hue_ini.remote_path = self.conf_dir(cluster_context) - hue_ini.parse(files.get_file_text(template % self.version)) + hue_ini.parse(utils.get_file_text(template % self.version, + 'sahara_plugin_mapr')) hue_ini.add_properties(self._get_hue_ini_props(cluster_context)) hue_ini.add_property("thrift_version", configs[self.THRIFT_VERSION.name]) @@ -112,7 +113,8 @@ class Hue(s.Service): 'resources/hue_sh_%s.template' hue_sh = bcf.TemplateFile("hue.sh") hue_sh.remote_path = self.home_dir(cluster_context) + '/bin' - hue_sh.parse(files.get_file_text(hue_sh_template % self.version)) + hue_sh.parse(utils.get_file_text(hue_sh_template % self.version, + 'sahara_plugin_mapr')) hue_sh.add_property('hadoop_version', cluster_context.hadoop_version) hue_sh.mode = 777 @@ -339,14 +341,16 @@ class HueLivyV310(s.Service): livy_conf_template = 'plugins/mapr/services/hue/' \ 'resources/livy_conf_%s.template' livy_conf = bcf.TemplateFile("livy.conf") - livy_conf.parse(files.get_file_text(livy_conf_template % self.version)) + livy_conf.parse(utils.get_file_text(livy_conf_template % self.version, + 'sahara_plugin_mapr')) livy_conf.remote_path = self.home_dir(cluster_context) + '/conf' livy_sh_template = 'plugins/mapr/services/hue/' \ 'resources/livy_sh_%s.template' livy_sh = bcf.TemplateFile("livy-env.sh") livy_sh.remote_path = self.home_dir(cluster_context) + '/conf' - livy_sh.parse(files.get_file_text(livy_sh_template % self.version)) + livy_sh.parse(utils.get_file_text(livy_sh_template % self.version, + 'sahara_plugin_mapr')) livy_sh.add_property('hadoop_version', cluster_context.hadoop_version) livy_sh.add_property('spark_version', spark.SparkOnYarnV201().version) livy_sh.mode = 777 diff --git a/sahara/plugins/mapr/services/hue/resources/hue_3.10.0.template b/sahara_plugin_mapr/plugins/mapr/services/hue/resources/hue_3.10.0.template similarity index 100% rename from sahara/plugins/mapr/services/hue/resources/hue_3.10.0.template rename to sahara_plugin_mapr/plugins/mapr/services/hue/resources/hue_3.10.0.template diff --git a/sahara/plugins/mapr/services/hue/resources/hue_3.6.0.template b/sahara_plugin_mapr/plugins/mapr/services/hue/resources/hue_3.6.0.template similarity index 100% rename from sahara/plugins/mapr/services/hue/resources/hue_3.6.0.template rename to sahara_plugin_mapr/plugins/mapr/services/hue/resources/hue_3.6.0.template diff --git a/sahara/plugins/mapr/services/hue/resources/hue_3.7.0.template b/sahara_plugin_mapr/plugins/mapr/services/hue/resources/hue_3.7.0.template similarity index 100% rename from sahara/plugins/mapr/services/hue/resources/hue_3.7.0.template rename to sahara_plugin_mapr/plugins/mapr/services/hue/resources/hue_3.7.0.template diff --git a/sahara/plugins/mapr/services/hue/resources/hue_3.8.1.template b/sahara_plugin_mapr/plugins/mapr/services/hue/resources/hue_3.8.1.template similarity index 100% rename from sahara/plugins/mapr/services/hue/resources/hue_3.8.1.template rename to sahara_plugin_mapr/plugins/mapr/services/hue/resources/hue_3.8.1.template diff --git a/sahara/plugins/mapr/services/hue/resources/hue_3.9.0.template b/sahara_plugin_mapr/plugins/mapr/services/hue/resources/hue_3.9.0.template similarity index 100% rename from sahara/plugins/mapr/services/hue/resources/hue_3.9.0.template rename to sahara_plugin_mapr/plugins/mapr/services/hue/resources/hue_3.9.0.template diff --git a/sahara/plugins/mapr/services/hue/resources/hue_sh_3.10.0.template b/sahara_plugin_mapr/plugins/mapr/services/hue/resources/hue_sh_3.10.0.template similarity index 100% rename from sahara/plugins/mapr/services/hue/resources/hue_sh_3.10.0.template rename to sahara_plugin_mapr/plugins/mapr/services/hue/resources/hue_sh_3.10.0.template diff --git a/sahara/plugins/mapr/services/hue/resources/hue_sh_3.7.0.template b/sahara_plugin_mapr/plugins/mapr/services/hue/resources/hue_sh_3.7.0.template similarity index 100% rename from sahara/plugins/mapr/services/hue/resources/hue_sh_3.7.0.template rename to sahara_plugin_mapr/plugins/mapr/services/hue/resources/hue_sh_3.7.0.template diff --git a/sahara/plugins/mapr/services/hue/resources/hue_sh_3.8.1.template b/sahara_plugin_mapr/plugins/mapr/services/hue/resources/hue_sh_3.8.1.template similarity index 100% rename from sahara/plugins/mapr/services/hue/resources/hue_sh_3.8.1.template rename to sahara_plugin_mapr/plugins/mapr/services/hue/resources/hue_sh_3.8.1.template diff --git a/sahara/plugins/mapr/services/hue/resources/hue_sh_3.9.0.template b/sahara_plugin_mapr/plugins/mapr/services/hue/resources/hue_sh_3.9.0.template similarity index 100% rename from sahara/plugins/mapr/services/hue/resources/hue_sh_3.9.0.template rename to sahara_plugin_mapr/plugins/mapr/services/hue/resources/hue_sh_3.9.0.template diff --git a/sahara/plugins/mapr/services/hue/resources/livy_conf_3.10.0.template b/sahara_plugin_mapr/plugins/mapr/services/hue/resources/livy_conf_3.10.0.template similarity index 100% rename from sahara/plugins/mapr/services/hue/resources/livy_conf_3.10.0.template rename to sahara_plugin_mapr/plugins/mapr/services/hue/resources/livy_conf_3.10.0.template diff --git a/sahara/plugins/mapr/services/hue/resources/livy_sh_3.10.0.template b/sahara_plugin_mapr/plugins/mapr/services/hue/resources/livy_sh_3.10.0.template similarity index 100% rename from sahara/plugins/mapr/services/hue/resources/livy_sh_3.10.0.template rename to sahara_plugin_mapr/plugins/mapr/services/hue/resources/livy_sh_3.10.0.template diff --git a/sahara/plugins/mapr/services/impala/__init__.py b/sahara_plugin_mapr/plugins/mapr/services/impala/__init__.py similarity index 100% rename from sahara/plugins/mapr/services/impala/__init__.py rename to sahara_plugin_mapr/plugins/mapr/services/impala/__init__.py diff --git a/sahara/plugins/mapr/services/impala/impala.py b/sahara_plugin_mapr/plugins/mapr/services/impala/impala.py similarity index 91% rename from sahara/plugins/mapr/services/impala/impala.py rename to sahara_plugin_mapr/plugins/mapr/services/impala/impala.py index 57673f6..f2ec88c 100644 --- a/sahara/plugins/mapr/services/impala/impala.py +++ b/sahara_plugin_mapr/plugins/mapr/services/impala/impala.py @@ -13,15 +13,15 @@ # under the License. -import sahara.plugins.mapr.domain.configuration_file as bcf -import sahara.plugins.mapr.domain.node_process as np -import sahara.plugins.mapr.domain.service as s -import sahara.plugins.mapr.services.hbase.hbase as hbase -import sahara.plugins.mapr.services.hive.hive as hive -import sahara.plugins.mapr.services.sentry.sentry as sentry -import sahara.plugins.mapr.util.general as g -import sahara.plugins.mapr.util.validation_utils as vu -import sahara.utils.files as files +import sahara.plugins.utils as utils +import sahara_plugin_mapr.plugins.mapr.domain.configuration_file as bcf +import sahara_plugin_mapr.plugins.mapr.domain.node_process as np +import sahara_plugin_mapr.plugins.mapr.domain.service as s +import sahara_plugin_mapr.plugins.mapr.services.hbase.hbase as hbase +import sahara_plugin_mapr.plugins.mapr.services.hive.hive as hive +import sahara_plugin_mapr.plugins.mapr.services.sentry.sentry as sentry +import sahara_plugin_mapr.plugins.mapr.util.general as g +import sahara_plugin_mapr.plugins.mapr.util.validation_utils as vu IMPALA_SERVER = np.NodeProcess( name='impalaserver', @@ -82,7 +82,7 @@ class Impala(s.Service): impala_env.remote_path = self.conf_dir(cluster_context) if instance: impala_env.fetch(instance) - impala_env.parse(files.get_file_text(defaults)) + impala_env.parse(utils.get_file_text(defaults, 'sahara_plugin_mapr')) impala_env.add_properties(self._get_impala_env_props(cluster_context)) sentry_host = cluster_context.get_instance(sentry.SENTRY) if sentry_host: diff --git a/sahara/plugins/mapr/services/impala/resources/impala-env.sh.j2 b/sahara_plugin_mapr/plugins/mapr/services/impala/resources/impala-env.sh.j2 similarity index 100% rename from sahara/plugins/mapr/services/impala/resources/impala-env.sh.j2 rename to sahara_plugin_mapr/plugins/mapr/services/impala/resources/impala-env.sh.j2 diff --git a/sahara/plugins/mapr/services/kafka/__init__.py b/sahara_plugin_mapr/plugins/mapr/services/kafka/__init__.py similarity index 100% rename from sahara/plugins/mapr/services/kafka/__init__.py rename to sahara_plugin_mapr/plugins/mapr/services/kafka/__init__.py diff --git a/sahara/plugins/mapr/services/kafka/kafka.py b/sahara_plugin_mapr/plugins/mapr/services/kafka/kafka.py similarity index 91% rename from sahara/plugins/mapr/services/kafka/kafka.py rename to sahara_plugin_mapr/plugins/mapr/services/kafka/kafka.py index a97c7fe..192d707 100644 --- a/sahara/plugins/mapr/services/kafka/kafka.py +++ b/sahara_plugin_mapr/plugins/mapr/services/kafka/kafka.py @@ -13,9 +13,9 @@ # under the License. -import sahara.plugins.mapr.domain.node_process as np -import sahara.plugins.mapr.domain.service as s -import sahara.plugins.mapr.util.validation_utils as vu +import sahara_plugin_mapr.plugins.mapr.domain.node_process as np +import sahara_plugin_mapr.plugins.mapr.domain.service as s +import sahara_plugin_mapr.plugins.mapr.util.validation_utils as vu KAFKA = np.NodeProcess( name='kafka', diff --git a/sahara/plugins/mapr/services/mahout/__init__.py b/sahara_plugin_mapr/plugins/mapr/services/mahout/__init__.py similarity index 100% rename from sahara/plugins/mapr/services/mahout/__init__.py rename to sahara_plugin_mapr/plugins/mapr/services/mahout/__init__.py diff --git a/sahara/plugins/mapr/services/mahout/mahout.py b/sahara_plugin_mapr/plugins/mapr/services/mahout/mahout.py similarity index 88% rename from sahara/plugins/mapr/services/mahout/mahout.py rename to sahara_plugin_mapr/plugins/mapr/services/mahout/mahout.py index 9069fa4..1d76914 100644 --- a/sahara/plugins/mapr/services/mahout/mahout.py +++ b/sahara_plugin_mapr/plugins/mapr/services/mahout/mahout.py @@ -13,9 +13,9 @@ # under the License. -import sahara.plugins.mapr.domain.node_process as np -import sahara.plugins.mapr.domain.service as s -import sahara.plugins.mapr.util.validation_utils as vu +import sahara_plugin_mapr.plugins.mapr.domain.node_process as np +import sahara_plugin_mapr.plugins.mapr.domain.service as s +import sahara_plugin_mapr.plugins.mapr.util.validation_utils as vu MAHOUT = np.NodeProcess( name='mahout', diff --git a/sahara/plugins/mapr/services/management/__init__.py b/sahara_plugin_mapr/plugins/mapr/services/management/__init__.py similarity index 100% rename from sahara/plugins/mapr/services/management/__init__.py rename to sahara_plugin_mapr/plugins/mapr/services/management/__init__.py diff --git a/sahara/plugins/mapr/services/management/management.py b/sahara_plugin_mapr/plugins/mapr/services/management/management.py similarity index 87% rename from sahara/plugins/mapr/services/management/management.py rename to sahara_plugin_mapr/plugins/mapr/services/management/management.py index c933acb..a536b19 100644 --- a/sahara/plugins/mapr/services/management/management.py +++ b/sahara_plugin_mapr/plugins/mapr/services/management/management.py @@ -13,10 +13,10 @@ # under the License. -import sahara.plugins.mapr.domain.node_process as np -import sahara.plugins.mapr.domain.service as s -import sahara.plugins.mapr.util.password_utils as pu -import sahara.plugins.mapr.util.validation_utils as vu +import sahara_plugin_mapr.plugins.mapr.domain.node_process as np +import sahara_plugin_mapr.plugins.mapr.domain.service as s +import sahara_plugin_mapr.plugins.mapr.util.password_utils as pu +import sahara_plugin_mapr.plugins.mapr.util.validation_utils as vu ZK_CLIENT_PORT = 5181 diff --git a/sahara/plugins/mapr/services/mapreduce/__init__.py b/sahara_plugin_mapr/plugins/mapr/services/mapreduce/__init__.py similarity index 100% rename from sahara/plugins/mapr/services/mapreduce/__init__.py rename to sahara_plugin_mapr/plugins/mapr/services/mapreduce/__init__.py diff --git a/sahara/plugins/mapr/services/mapreduce/mapreduce.py b/sahara_plugin_mapr/plugins/mapr/services/mapreduce/mapreduce.py similarity index 92% rename from sahara/plugins/mapr/services/mapreduce/mapreduce.py rename to sahara_plugin_mapr/plugins/mapr/services/mapreduce/mapreduce.py index feb35a4..c288847 100644 --- a/sahara/plugins/mapr/services/mapreduce/mapreduce.py +++ b/sahara_plugin_mapr/plugins/mapr/services/mapreduce/mapreduce.py @@ -12,12 +12,12 @@ # License for the specific language governing permissions and limitations # under the License. -import sahara.plugins.mapr.domain.configuration_file as bcf -import sahara.plugins.mapr.domain.node_process as np -import sahara.plugins.mapr.domain.service as s -import sahara.plugins.mapr.util.validation_utils as vu -from sahara.swift import swift_helper -from sahara.topology import topology_helper as topo +from sahara.plugins import swift_helper +from sahara.plugins import topology_helper as topo +import sahara_plugin_mapr.plugins.mapr.domain.configuration_file as bcf +import sahara_plugin_mapr.plugins.mapr.domain.node_process as np +import sahara_plugin_mapr.plugins.mapr.domain.service as s +import sahara_plugin_mapr.plugins.mapr.util.validation_utils as vu JOB_TRACKER = np.NodeProcess( name='jobtracker', diff --git a/sahara/plugins/mapr/services/mapreduce/resources/mapreduce-cluster.json b/sahara_plugin_mapr/plugins/mapr/services/mapreduce/resources/mapreduce-cluster.json similarity index 100% rename from sahara/plugins/mapr/services/mapreduce/resources/mapreduce-cluster.json rename to sahara_plugin_mapr/plugins/mapr/services/mapreduce/resources/mapreduce-cluster.json diff --git a/sahara/plugins/mapr/services/maprfs/__init__.py b/sahara_plugin_mapr/plugins/mapr/services/maprfs/__init__.py similarity index 100% rename from sahara/plugins/mapr/services/maprfs/__init__.py rename to sahara_plugin_mapr/plugins/mapr/services/maprfs/__init__.py diff --git a/sahara/plugins/mapr/services/maprfs/maprfs.py b/sahara_plugin_mapr/plugins/mapr/services/maprfs/maprfs.py similarity index 85% rename from sahara/plugins/mapr/services/maprfs/maprfs.py rename to sahara_plugin_mapr/plugins/mapr/services/maprfs/maprfs.py index d78ac2d..00071b8 100644 --- a/sahara/plugins/mapr/services/maprfs/maprfs.py +++ b/sahara_plugin_mapr/plugins/mapr/services/maprfs/maprfs.py @@ -15,17 +15,16 @@ from oslo_log import log as logging -from sahara import context -from sahara.i18n import _ -import sahara.plugins.mapr.domain.configuration_file as bcf -import sahara.plugins.mapr.domain.node_process as np -import sahara.plugins.mapr.domain.service as s -import sahara.plugins.mapr.util.event_log as el -import sahara.plugins.mapr.util.general as g -import sahara.plugins.mapr.util.validation_utils as vu +from sahara.plugins import context import sahara.plugins.provisioning as p -from sahara.utils import cluster_progress_ops as cpo -from sahara.utils import files +from sahara.plugins import utils +from sahara_plugin_mapr.i18n import _ +import sahara_plugin_mapr.plugins.mapr.domain.configuration_file as bcf +import sahara_plugin_mapr.plugins.mapr.domain.node_process as np +import sahara_plugin_mapr.plugins.mapr.domain.service as s +import sahara_plugin_mapr.plugins.mapr.util.event_log as el +import sahara_plugin_mapr.plugins.mapr.util.general as g +import sahara_plugin_mapr.plugins.mapr.util.validation_utils as vu LOG = logging.getLogger(__name__) @@ -106,9 +105,10 @@ class MapRFS(s.Service): LOG.debug('Initializing MapR FS') instances = instances or cluster_context.get_instances() file_servers = cluster_context.filter_instances(instances, FILE_SERVER) - cpo.add_provisioning_step(cluster_context.cluster.id, - _("Initializing MapR-FS"), len(file_servers)) - with context.ThreadGroup() as tg: + utils.add_provisioning_step(cluster_context.cluster.id, + _("Initializing MapR-FS"), + len(file_servers)) + with context.PluginsThreadGroup() as tg: for instance in file_servers: tg.spawn('init-mfs-%s' % instance.id, self._init_mfs_instance, instance) @@ -139,7 +139,8 @@ class MapRFS(s.Service): cldb_conf.remote_path = self.conf_dir(cluster_context) if instance: cldb_conf.fetch(instance) - cldb_conf.parse(files.get_file_text(default_path)) + cldb_conf.parse(utils.get_file_text(default_path, + 'sahara_plugin_mapr')) cldb_conf.add_properties(self._get_cldb_conf_props(cluster_context)) warden_conf = bcf.PropertiesFile("warden.conf") diff --git a/sahara/plugins/mapr/services/maprfs/resources/cldb.conf b/sahara_plugin_mapr/plugins/mapr/services/maprfs/resources/cldb.conf similarity index 100% rename from sahara/plugins/mapr/services/maprfs/resources/cldb.conf rename to sahara_plugin_mapr/plugins/mapr/services/maprfs/resources/cldb.conf diff --git a/sahara/plugins/mapr/services/mysql/__init__.py b/sahara_plugin_mapr/plugins/mapr/services/mysql/__init__.py similarity index 100% rename from sahara/plugins/mapr/services/mysql/__init__.py rename to sahara_plugin_mapr/plugins/mapr/services/mysql/__init__.py diff --git a/sahara/plugins/mapr/services/mysql/mysql.py b/sahara_plugin_mapr/plugins/mapr/services/mysql/mysql.py similarity index 96% rename from sahara/plugins/mapr/services/mysql/mysql.py rename to sahara_plugin_mapr/plugins/mapr/services/mysql/mysql.py index d2ab783..a6cbf75 100644 --- a/sahara/plugins/mapr/services/mysql/mysql.py +++ b/sahara_plugin_mapr/plugins/mapr/services/mysql/mysql.py @@ -18,10 +18,10 @@ import collections as c from oslo_log import log as logging import six -import sahara.plugins.mapr.domain.configuration_file as cf -import sahara.plugins.mapr.domain.service as s -import sahara.plugins.mapr.util.general as g -import sahara.utils.files as f +import sahara.plugins.utils as u +import sahara_plugin_mapr.plugins.mapr.domain.configuration_file as cf +import sahara_plugin_mapr.plugins.mapr.domain.service as s +import sahara_plugin_mapr.plugins.mapr.util.general as g LOG = logging.getLogger(__name__) @@ -183,8 +183,9 @@ class MySQL(s.Service): def _create_script_obj(filename, template, **kwargs): script = cf.TemplateFile(filename) script.remote_path = '/tmp/' - script.parse(f.get_file_text( - 'plugins/mapr/services/mysql/resources/%s' % template)) + script.parse(u.get_file_text( + 'plugins/mapr/services/mysql/resources/%s' % template, + 'sahara_plugin_mapr')) for k, v in six.iteritems(kwargs): script.add_property(k, v) return script diff --git a/sahara/plugins/mapr/services/mysql/resources/create_database.sql b/sahara_plugin_mapr/plugins/mapr/services/mysql/resources/create_database.sql similarity index 100% rename from sahara/plugins/mapr/services/mysql/resources/create_database.sql rename to sahara_plugin_mapr/plugins/mapr/services/mysql/resources/create_database.sql diff --git a/sahara/plugins/mapr/services/mysql/resources/grant_access.sql b/sahara_plugin_mapr/plugins/mapr/services/mysql/resources/grant_access.sql similarity index 100% rename from sahara/plugins/mapr/services/mysql/resources/grant_access.sql rename to sahara_plugin_mapr/plugins/mapr/services/mysql/resources/grant_access.sql diff --git a/sahara/plugins/mapr/services/mysql/resources/hive_schema.sql b/sahara_plugin_mapr/plugins/mapr/services/mysql/resources/hive_schema.sql similarity index 100% rename from sahara/plugins/mapr/services/mysql/resources/hive_schema.sql rename to sahara_plugin_mapr/plugins/mapr/services/mysql/resources/hive_schema.sql diff --git a/sahara/plugins/mapr/services/oozie/__init__.py b/sahara_plugin_mapr/plugins/mapr/services/oozie/__init__.py similarity index 100% rename from sahara/plugins/mapr/services/oozie/__init__.py rename to sahara_plugin_mapr/plugins/mapr/services/oozie/__init__.py diff --git a/sahara/plugins/mapr/services/oozie/oozie.py b/sahara_plugin_mapr/plugins/mapr/services/oozie/oozie.py similarity index 92% rename from sahara/plugins/mapr/services/oozie/oozie.py rename to sahara_plugin_mapr/plugins/mapr/services/oozie/oozie.py index 691a196..e0fdeea 100644 --- a/sahara/plugins/mapr/services/oozie/oozie.py +++ b/sahara_plugin_mapr/plugins/mapr/services/oozie/oozie.py @@ -14,15 +14,15 @@ from oslo_log import log as logging -import sahara.context as context -from sahara.i18n import _ -import sahara.plugins.mapr.domain.configuration_file as bcf -import sahara.plugins.mapr.domain.node_process as np -import sahara.plugins.mapr.domain.service as s -import sahara.plugins.mapr.services.mysql.mysql as mysql -import sahara.plugins.mapr.util.event_log as el -import sahara.plugins.mapr.util.general as g -import sahara.plugins.mapr.util.validation_utils as vu +import sahara.plugins.context as context +from sahara_plugin_mapr.i18n import _ +import sahara_plugin_mapr.plugins.mapr.domain.configuration_file as bcf +import sahara_plugin_mapr.plugins.mapr.domain.node_process as np +import sahara_plugin_mapr.plugins.mapr.domain.service as s +import sahara_plugin_mapr.plugins.mapr.services.mysql.mysql as mysql +import sahara_plugin_mapr.plugins.mapr.util.event_log as el +import sahara_plugin_mapr.plugins.mapr.util.general as g +import sahara_plugin_mapr.plugins.mapr.util.validation_utils as vu LOG = logging.getLogger(__name__) OOZIE_START_DELAY = 30 diff --git a/sahara/plugins/mapr/services/oozie/resources/oozie-default.json b/sahara_plugin_mapr/plugins/mapr/services/oozie/resources/oozie-default.json similarity index 100% rename from sahara/plugins/mapr/services/oozie/resources/oozie-default.json rename to sahara_plugin_mapr/plugins/mapr/services/oozie/resources/oozie-default.json diff --git a/sahara/plugins/mapr/services/pig/__init__.py b/sahara_plugin_mapr/plugins/mapr/services/pig/__init__.py similarity index 100% rename from sahara/plugins/mapr/services/pig/__init__.py rename to sahara_plugin_mapr/plugins/mapr/services/pig/__init__.py diff --git a/sahara/plugins/mapr/services/pig/pig.py b/sahara_plugin_mapr/plugins/mapr/services/pig/pig.py similarity index 87% rename from sahara/plugins/mapr/services/pig/pig.py rename to sahara_plugin_mapr/plugins/mapr/services/pig/pig.py index 8044cdb..2015b0b 100644 --- a/sahara/plugins/mapr/services/pig/pig.py +++ b/sahara_plugin_mapr/plugins/mapr/services/pig/pig.py @@ -13,9 +13,9 @@ # under the License. -import sahara.plugins.mapr.domain.node_process as np -import sahara.plugins.mapr.domain.service as s -import sahara.plugins.mapr.util.validation_utils as vu +import sahara_plugin_mapr.plugins.mapr.domain.node_process as np +import sahara_plugin_mapr.plugins.mapr.domain.service as s +import sahara_plugin_mapr.plugins.mapr.util.validation_utils as vu PIG = np.NodeProcess( name='pig', diff --git a/sahara/plugins/mapr/services/sentry/__init__.py b/sahara_plugin_mapr/plugins/mapr/services/sentry/__init__.py similarity index 100% rename from sahara/plugins/mapr/services/sentry/__init__.py rename to sahara_plugin_mapr/plugins/mapr/services/sentry/__init__.py diff --git a/sahara/plugins/mapr/services/sentry/resources/global-policy.ini b/sahara_plugin_mapr/plugins/mapr/services/sentry/resources/global-policy.ini similarity index 100% rename from sahara/plugins/mapr/services/sentry/resources/global-policy.ini rename to sahara_plugin_mapr/plugins/mapr/services/sentry/resources/global-policy.ini diff --git a/sahara/plugins/mapr/services/sentry/resources/sentry-default.xml b/sahara_plugin_mapr/plugins/mapr/services/sentry/resources/sentry-default.xml similarity index 100% rename from sahara/plugins/mapr/services/sentry/resources/sentry-default.xml rename to sahara_plugin_mapr/plugins/mapr/services/sentry/resources/sentry-default.xml diff --git a/sahara/plugins/mapr/services/sentry/sentry.py b/sahara_plugin_mapr/plugins/mapr/services/sentry/sentry.py similarity index 89% rename from sahara/plugins/mapr/services/sentry/sentry.py rename to sahara_plugin_mapr/plugins/mapr/services/sentry/sentry.py index f38c7e9..b7d2918 100644 --- a/sahara/plugins/mapr/services/sentry/sentry.py +++ b/sahara_plugin_mapr/plugins/mapr/services/sentry/sentry.py @@ -12,15 +12,15 @@ # License for the specific language governing permissions and limitations # under the License. -from sahara.i18n import _ -import sahara.plugins.mapr.domain.configuration_file as cf -import sahara.plugins.mapr.domain.node_process as np -import sahara.plugins.mapr.domain.service as s -import sahara.plugins.mapr.services.mysql.mysql as mysql -import sahara.plugins.mapr.util.event_log as el -import sahara.plugins.mapr.util.maprfs_helper as mfs import sahara.plugins.provisioning as p -import sahara.utils.files as files +import sahara.plugins.utils as utils +from sahara_plugin_mapr.i18n import _ +import sahara_plugin_mapr.plugins.mapr.domain.configuration_file as cf +import sahara_plugin_mapr.plugins.mapr.domain.node_process as np +import sahara_plugin_mapr.plugins.mapr.domain.service as s +import sahara_plugin_mapr.plugins.mapr.services.mysql.mysql as mysql +import sahara_plugin_mapr.plugins.mapr.util.event_log as el +import sahara_plugin_mapr.plugins.mapr.util.maprfs_helper as mfs SENTRY = np.NodeProcess( name='sentry', @@ -68,12 +68,14 @@ class Sentry(s.Service): sentry_site.fetch(instance) sentry_site.load_properties(configs) sentry_mode = configs[self.SENTRY_STORAGE_MODE.name] - sentry_site.parse(files.get_file_text(sentry_default)) + sentry_site.parse(utils.get_file_text(sentry_default, + 'sahara_plugin_mapr')) sentry_site.add_properties( self._get_sentry_site_props(cluster_context, sentry_mode)) global_policy = cf.TemplateFile('global-policy.ini') global_policy.remote_path = self.conf_dir(cluster_context) - global_policy.parse(files.get_file_text(global_policy_template)) + global_policy.parse(utils.get_file_text(global_policy_template, + 'sahara_plugin_mapr')) return [sentry_site, global_policy] def _get_jdbc_uri(self, cluster_context): diff --git a/sahara/plugins/mapr/services/spark/__init__.py b/sahara_plugin_mapr/plugins/mapr/services/spark/__init__.py similarity index 100% rename from sahara/plugins/mapr/services/spark/__init__.py rename to sahara_plugin_mapr/plugins/mapr/services/spark/__init__.py diff --git a/sahara/plugins/mapr/services/spark/resources/spark-env.template b/sahara_plugin_mapr/plugins/mapr/services/spark/resources/spark-env.template similarity index 100% rename from sahara/plugins/mapr/services/spark/resources/spark-env.template rename to sahara_plugin_mapr/plugins/mapr/services/spark/resources/spark-env.template diff --git a/sahara/plugins/mapr/services/spark/spark.py b/sahara_plugin_mapr/plugins/mapr/services/spark/spark.py similarity index 94% rename from sahara/plugins/mapr/services/spark/spark.py rename to sahara_plugin_mapr/plugins/mapr/services/spark/spark.py index c83aaaa..98cbab3 100644 --- a/sahara/plugins/mapr/services/spark/spark.py +++ b/sahara_plugin_mapr/plugins/mapr/services/spark/spark.py @@ -13,15 +13,15 @@ # under the License. -import sahara.plugins.mapr.domain.configuration_file as bcf -import sahara.plugins.mapr.domain.node_process as np -import sahara.plugins.mapr.domain.service as s -import sahara.plugins.mapr.services.hbase.hbase as hbase -import sahara.plugins.mapr.services.hive.hive as hive -import sahara.plugins.mapr.util.general as g -import sahara.plugins.mapr.util.maprfs_helper as mfs -import sahara.plugins.mapr.util.validation_utils as vu -import sahara.utils.files as files +import sahara.plugins.utils as utils +import sahara_plugin_mapr.plugins.mapr.domain.configuration_file as bcf +import sahara_plugin_mapr.plugins.mapr.domain.node_process as np +import sahara_plugin_mapr.plugins.mapr.domain.service as s +import sahara_plugin_mapr.plugins.mapr.services.hbase.hbase as hbase +import sahara_plugin_mapr.plugins.mapr.services.hive.hive as hive +import sahara_plugin_mapr.plugins.mapr.util.general as g +import sahara_plugin_mapr.plugins.mapr.util.maprfs_helper as mfs +import sahara_plugin_mapr.plugins.mapr.util.validation_utils as vu SPARK_SLAVE_UI_PORT = 8081 SPARK_HS_UI_PORT = 18080 @@ -85,7 +85,7 @@ class SparkOnYarn(s.Service): 'spark/resources/spark-env.template' env_sh = bcf.TemplateFile('spark-env.sh') env_sh.remote_path = self.conf_dir(cluster_context) - env_sh.parse(files.get_file_text(template)) + env_sh.parse(utils.get_file_text(template, 'sahara_plugin_mapr')) env_sh.add_property('version', self.version) env_sh.add_property('servlet_api_jar', self.SERVLET_JAR) @@ -284,7 +284,7 @@ class SparkOnYarnV201(SparkOnYarn): 'spark/resources/spark-env.template' env_sh = bcf.TemplateFile('spark-env.sh') env_sh.remote_path = self.conf_dir(cluster_context) - env_sh.parse(files.get_file_text(template)) + env_sh.parse(utils.get_file_text(template, 'sahara_plugin_mapr')) env_sh.add_property('version', self.version) # spark-defaults diff --git a/sahara/plugins/mapr/services/sqoop/__init__.py b/sahara_plugin_mapr/plugins/mapr/services/sqoop/__init__.py similarity index 100% rename from sahara/plugins/mapr/services/sqoop/__init__.py rename to sahara_plugin_mapr/plugins/mapr/services/sqoop/__init__.py diff --git a/sahara/plugins/mapr/services/sqoop/sqoop2.py b/sahara_plugin_mapr/plugins/mapr/services/sqoop/sqoop2.py similarity index 87% rename from sahara/plugins/mapr/services/sqoop/sqoop2.py rename to sahara_plugin_mapr/plugins/mapr/services/sqoop/sqoop2.py index 32c2718..1818081 100644 --- a/sahara/plugins/mapr/services/sqoop/sqoop2.py +++ b/sahara_plugin_mapr/plugins/mapr/services/sqoop/sqoop2.py @@ -13,9 +13,9 @@ # under the License. -import sahara.plugins.mapr.domain.node_process as np -import sahara.plugins.mapr.domain.service as s -import sahara.plugins.mapr.util.validation_utils as vu +import sahara_plugin_mapr.plugins.mapr.domain.node_process as np +import sahara_plugin_mapr.plugins.mapr.domain.service as s +import sahara_plugin_mapr.plugins.mapr.util.validation_utils as vu SQOOP_2_SERVER = np.NodeProcess( diff --git a/sahara/plugins/mapr/services/swift/__init__.py b/sahara_plugin_mapr/plugins/mapr/services/swift/__init__.py similarity index 100% rename from sahara/plugins/mapr/services/swift/__init__.py rename to sahara_plugin_mapr/plugins/mapr/services/swift/__init__.py diff --git a/sahara/plugins/mapr/services/swift/resources/hadoop-swift-latest.jar b/sahara_plugin_mapr/plugins/mapr/services/swift/resources/hadoop-swift-latest.jar similarity index 100% rename from sahara/plugins/mapr/services/swift/resources/hadoop-swift-latest.jar rename to sahara_plugin_mapr/plugins/mapr/services/swift/resources/hadoop-swift-latest.jar diff --git a/sahara/plugins/mapr/services/swift/resources/swift-default.json b/sahara_plugin_mapr/plugins/mapr/services/swift/resources/swift-default.json similarity index 100% rename from sahara/plugins/mapr/services/swift/resources/swift-default.json rename to sahara_plugin_mapr/plugins/mapr/services/swift/resources/swift-default.json diff --git a/sahara/plugins/mapr/services/swift/swift.py b/sahara_plugin_mapr/plugins/mapr/services/swift/swift.py similarity index 86% rename from sahara/plugins/mapr/services/swift/swift.py rename to sahara_plugin_mapr/plugins/mapr/services/swift/swift.py index 388280c..a480c9b 100644 --- a/sahara/plugins/mapr/services/swift/swift.py +++ b/sahara_plugin_mapr/plugins/mapr/services/swift/swift.py @@ -15,10 +15,10 @@ from oslo_log import log as logging -import sahara.plugins.mapr.domain.service as s -import sahara.plugins.mapr.services.maprfs.maprfs as maprfs -import sahara.plugins.mapr.util.event_log as el -import sahara.utils.files as f +import sahara.plugins.utils as u +import sahara_plugin_mapr.plugins.mapr.domain.service as s +import sahara_plugin_mapr.plugins.mapr.services.maprfs.maprfs as maprfs +import sahara_plugin_mapr.plugins.mapr.util.event_log as el LOG = logging.getLogger(__name__) @@ -46,7 +46,7 @@ class Swift(s.Service): @el.provision_step("Install Swift service") def _install_swift_jar(self, cluster_context, instances): LOG.debug('Installing Swift jar') - jar = f.get_file_text(Swift.HADOOP_SWIFT_JAR) + jar = u.get_file_text(Swift.HADOOP_SWIFT_JAR, 'sahara_plugin_mapr') path = '%s/swift.jar' % cluster_context.hadoop_lib @el.provision_event() diff --git a/sahara/plugins/mapr/services/yarn/__init__.py b/sahara_plugin_mapr/plugins/mapr/services/yarn/__init__.py similarity index 100% rename from sahara/plugins/mapr/services/yarn/__init__.py rename to sahara_plugin_mapr/plugins/mapr/services/yarn/__init__.py diff --git a/sahara/plugins/mapr/services/yarn/resources/yarn-cluster.json b/sahara_plugin_mapr/plugins/mapr/services/yarn/resources/yarn-cluster.json similarity index 100% rename from sahara/plugins/mapr/services/yarn/resources/yarn-cluster.json rename to sahara_plugin_mapr/plugins/mapr/services/yarn/resources/yarn-cluster.json diff --git a/sahara/plugins/mapr/services/yarn/resources/yarn-node.json b/sahara_plugin_mapr/plugins/mapr/services/yarn/resources/yarn-node.json similarity index 100% rename from sahara/plugins/mapr/services/yarn/resources/yarn-node.json rename to sahara_plugin_mapr/plugins/mapr/services/yarn/resources/yarn-node.json diff --git a/sahara/plugins/mapr/services/yarn/yarn.py b/sahara_plugin_mapr/plugins/mapr/services/yarn/yarn.py similarity index 92% rename from sahara/plugins/mapr/services/yarn/yarn.py rename to sahara_plugin_mapr/plugins/mapr/services/yarn/yarn.py index 879e972..b1f810c 100644 --- a/sahara/plugins/mapr/services/yarn/yarn.py +++ b/sahara_plugin_mapr/plugins/mapr/services/yarn/yarn.py @@ -13,12 +13,12 @@ # under the License. -import sahara.plugins.mapr.domain.configuration_file as bcf -import sahara.plugins.mapr.domain.node_process as np -import sahara.plugins.mapr.domain.service as s -import sahara.plugins.mapr.util.validation_utils as vu -from sahara.swift import swift_helper -from sahara.topology import topology_helper as topo +from sahara.plugins import swift_helper +from sahara.plugins import topology_helper as topo +import sahara_plugin_mapr.plugins.mapr.domain.configuration_file as bcf +import sahara_plugin_mapr.plugins.mapr.domain.node_process as np +import sahara_plugin_mapr.plugins.mapr.domain.service as s +import sahara_plugin_mapr.plugins.mapr.util.validation_utils as vu RESOURCE_MANAGER = np.NodeProcess( diff --git a/sahara/plugins/mapr/util/__init__.py b/sahara_plugin_mapr/plugins/mapr/util/__init__.py similarity index 100% rename from sahara/plugins/mapr/util/__init__.py rename to sahara_plugin_mapr/plugins/mapr/util/__init__.py diff --git a/sahara/plugins/mapr/util/commands.py b/sahara_plugin_mapr/plugins/mapr/util/commands.py similarity index 100% rename from sahara/plugins/mapr/util/commands.py rename to sahara_plugin_mapr/plugins/mapr/util/commands.py diff --git a/sahara/plugins/mapr/util/event_log.py b/sahara_plugin_mapr/plugins/mapr/util/event_log.py similarity index 82% rename from sahara/plugins/mapr/util/event_log.py rename to sahara_plugin_mapr/plugins/mapr/util/event_log.py index 922e540..0d18a14 100644 --- a/sahara/plugins/mapr/util/event_log.py +++ b/sahara_plugin_mapr/plugins/mapr/util/event_log.py @@ -13,7 +13,7 @@ # under the License. -from sahara.utils import cluster_progress_ops as cpo +from sahara.plugins import utils def provision_step(name, cluster_context_reference=1, instances_reference=2): @@ -26,7 +26,7 @@ def provision_step(name, cluster_context_reference=1, instances_reference=2): cluster_id = cluster_context.cluster.id instance_count = len(instances) - cpo.add_provisioning_step(cluster_id, name, instance_count) + utils.add_provisioning_step(cluster_id, name, instance_count) return function(*args, **kwargs) @@ -41,14 +41,14 @@ def provision_event(instance_reference=0, name=None, instance=None): event_instance = instance or _find_argument(instance_reference, *args, **kwargs) if name: - cpo.add_provisioning_step(event_instance.node_group.cluster.id, - name, 1) + utils.add_provisioning_step( + event_instance.node_group.cluster.id, name, 1) try: result = function(*args, **kwargs) - cpo.add_successful_event(event_instance) + utils.add_successful_event(event_instance) return result except Exception as exception: - cpo.add_fail_event(event_instance, exception) + utils.add_fail_event(event_instance, exception) raise exception return wrapped diff --git a/sahara/plugins/mapr/util/general.py b/sahara_plugin_mapr/plugins/mapr/util/general.py similarity index 95% rename from sahara/plugins/mapr/util/general.py rename to sahara_plugin_mapr/plugins/mapr/util/general.py index f09408a..e99ed8d 100644 --- a/sahara/plugins/mapr/util/general.py +++ b/sahara_plugin_mapr/plugins/mapr/util/general.py @@ -15,9 +15,9 @@ from oslo_utils import uuidutils -from sahara.conductor import objects -from sahara import context -import sahara.utils.files as files +from sahara.plugins import context +from sahara.plugins import objects +import sahara.plugins.utils as utils def unique_list(iterable, mapper=lambda i: i): @@ -92,7 +92,7 @@ def copy(s_path, s_instance, d_path, d_instance, run_as=None): def run_script(instance, script, run_as=None, *args, **kwargs): with instance.remote() as r: path = '/tmp/%s.sh' % uuidutils.generate_uuid() - script = files.get_file_text(script) % kwargs + script = utils.get_file_text(script, 'sahara_plugin_mapr') % kwargs r.write_file_to(path, script, run_as_root=(run_as == 'root')) r.execute_command(_run_as(run_as, 'chmod +x %s' % path)) r.execute_command(_run_as(run_as, '%s %s' % (path, ' '.join(args))), @@ -102,7 +102,7 @@ def run_script(instance, script, run_as=None, *args, **kwargs): def execute_on_instances(instances, function, *args, **kwargs): - with context.ThreadGroup() as tg: + with context.PluginsThreadGroup() as tg: for instance in instances: t_name = '%s-execution' % function.__name__ tg.spawn(t_name, function, instance, *args, **kwargs) @@ -116,7 +116,7 @@ def remote_command(position): def wrap(func): def wrapped(*args, **kwargs): target = args[position] - if isinstance(target, objects.Instance): + if objects.is_object_instance(target): with target.remote() as remote: return func(*_replace(args, position, remote), **kwargs) return func(*args, **kwargs) diff --git a/sahara/plugins/mapr/util/maprfs_helper.py b/sahara_plugin_mapr/plugins/mapr/util/maprfs_helper.py similarity index 98% rename from sahara/plugins/mapr/util/maprfs_helper.py rename to sahara_plugin_mapr/plugins/mapr/util/maprfs_helper.py index 74627ce..af62f0d 100644 --- a/sahara/plugins/mapr/util/maprfs_helper.py +++ b/sahara_plugin_mapr/plugins/mapr/util/maprfs_helper.py @@ -17,7 +17,7 @@ import os from oslo_utils import uuidutils -import sahara.plugins.mapr.util.general as g +import sahara_plugin_mapr.plugins.mapr.util.general as g MV_TO_MAPRFS_CMD = ('sudo -u %(user)s' ' hadoop fs -copyFromLocal %(source)s %(target)s' diff --git a/sahara/plugins/mapr/util/password_utils.py b/sahara_plugin_mapr/plugins/mapr/util/password_utils.py similarity index 93% rename from sahara/plugins/mapr/util/password_utils.py rename to sahara_plugin_mapr/plugins/mapr/util/password_utils.py index 8f443b2..1df3bef 100644 --- a/sahara/plugins/mapr/util/password_utils.py +++ b/sahara_plugin_mapr/plugins/mapr/util/password_utils.py @@ -16,14 +16,12 @@ from oslo_utils import uuidutils -from sahara import conductor -from sahara import context -from sahara.service.castellan import utils as key_manager +from sahara.plugins import castellan_utils as key_manager +from sahara.plugins import conductor +from sahara.plugins import context MAPR_USER_PASSWORD = 'mapr_password' -conductor = conductor.API - def delete_password(cluster, pw_name): """delete the named password from the key manager diff --git a/sahara/plugins/mapr/util/service_utils.py b/sahara_plugin_mapr/plugins/mapr/util/service_utils.py similarity index 93% rename from sahara/plugins/mapr/util/service_utils.py rename to sahara_plugin_mapr/plugins/mapr/util/service_utils.py index 15528b6..cc02243 100644 --- a/sahara/plugins/mapr/util/service_utils.py +++ b/sahara_plugin_mapr/plugins/mapr/util/service_utils.py @@ -15,12 +15,12 @@ import six -from sahara.i18n import _ +from sahara_plugin_mapr.i18n import _ def get_node_process_name(node_process): # This import is placed here to avoid circular imports - from sahara.plugins.mapr.domain import node_process as np # noqa + from sahara_plugin_mapr.plugins.mapr.domain import node_process as np # noqa if isinstance(node_process, np.NodeProcess): return node_process.ui_name diff --git a/sahara/plugins/mapr/util/validation_utils.py b/sahara_plugin_mapr/plugins/mapr/util/validation_utils.py similarity index 94% rename from sahara/plugins/mapr/util/validation_utils.py rename to sahara_plugin_mapr/plugins/mapr/util/validation_utils.py index 2ff1f90..2ee56b7 100644 --- a/sahara/plugins/mapr/util/validation_utils.py +++ b/sahara_plugin_mapr/plugins/mapr/util/validation_utils.py @@ -15,12 +15,11 @@ import functools as ft -from sahara.conductor import resource as r -import sahara.exceptions as ex -from sahara.i18n import _ import sahara.plugins.exceptions as e -from sahara.service.api import v10 as api -import sahara.utils.openstack.nova as nova +from sahara.plugins import resource as r +from sahara.plugins import service_api as api +import sahara.plugins.utils as utils +from sahara_plugin_mapr.i18n import _ class LessThanCountException(e.InvalidComponentCountException): @@ -55,7 +54,7 @@ class MoreThanCountException(e.InvalidComponentCountException): self.message = MoreThanCountException.MESSAGE % args -class EvenCountException(ex.SaharaException): +class EvenCountException(e.SaharaPluginException): MESSAGE = _("Hadoop cluster should contain odd number of %(component)s" " but %(actual_count)s found.") @@ -81,7 +80,7 @@ class NodeRequiredServiceMissingException(e.RequiredServiceMissingException): NodeRequiredServiceMissingException.REQUIRED_MSG % args) -class NodeServiceConflictException(ex.SaharaException): +class NodeServiceConflictException(e.SaharaPluginException): MESSAGE = _('%(service)s service cannot be installed alongside' ' %(package)s package') ERROR_CODE = "NODE_PROCESS_CONFLICT" @@ -96,7 +95,7 @@ class NodeServiceConflictException(ex.SaharaException): self.code = NodeServiceConflictException.ERROR_CODE -class NoVolumesException(ex.SaharaException): +class NoVolumesException(e.SaharaPluginException): MESSAGE = _('%s must have at least 1 volume or ephemeral drive') ERROR_CODE = "NO_VOLUMES" @@ -106,7 +105,7 @@ class NoVolumesException(ex.SaharaException): self.code = NoVolumesException.ERROR_CODE -class NotRequiredImageException(ex.SaharaException): +class NotRequiredImageException(e.SaharaPluginException): MESSAGE = _('Service %(service)s requires %(os)s OS.' ' Use %(os)s image and add "%(os)s" tag to it.') ERROR_CODE = "INVALID_IMAGE" @@ -229,7 +228,7 @@ def create_fake_cluster(cluster, existing, additional): ng_dict = node_group.to_dict() count = counts[node_group.id] ng_dict.update(dict(count=count)) - return r.NodeGroupResource(ng_dict) + return r.create_node_group_resource(ng_dict) def need_upd(node_group): return node_group.id in counts and counts[node_group.id] > 0 @@ -240,12 +239,12 @@ def create_fake_cluster(cluster, existing, additional): cluster.node_groups) cluster_dict = cluster.to_dict() cluster_dict.update({'node_groups': updated + not_updated}) - fake = r.ClusterResource(cluster_dict) + fake = r.create_cluster_resource(cluster_dict) return fake def get_ephemeral(node_group): - return nova.get_flavor(id=node_group.flavor_id).ephemeral + return utils.get_flavor(id=node_group.flavor_id).ephemeral def has_volumes(): diff --git a/sahara/plugins/mapr/versions/__init__.py b/sahara_plugin_mapr/plugins/mapr/versions/__init__.py similarity index 100% rename from sahara/plugins/mapr/versions/__init__.py rename to sahara_plugin_mapr/plugins/mapr/versions/__init__.py diff --git a/sahara/plugins/mapr/versions/v5_2_0_mrv2/__init__.py b/sahara_plugin_mapr/plugins/mapr/versions/v5_2_0_mrv2/__init__.py similarity index 100% rename from sahara/plugins/mapr/versions/v5_2_0_mrv2/__init__.py rename to sahara_plugin_mapr/plugins/mapr/versions/v5_2_0_mrv2/__init__.py diff --git a/sahara/plugins/mapr/versions/v5_2_0_mrv2/context.py b/sahara_plugin_mapr/plugins/mapr/versions/v5_2_0_mrv2/context.py similarity index 94% rename from sahara/plugins/mapr/versions/v5_2_0_mrv2/context.py rename to sahara_plugin_mapr/plugins/mapr/versions/v5_2_0_mrv2/context.py index 664b294..c4c1264 100644 --- a/sahara/plugins/mapr/versions/v5_2_0_mrv2/context.py +++ b/sahara_plugin_mapr/plugins/mapr/versions/v5_2_0_mrv2/context.py @@ -13,8 +13,8 @@ # under the License. -import sahara.plugins.mapr.base.base_cluster_context as bc -import sahara.plugins.mapr.services.yarn.yarn as yarn +import sahara_plugin_mapr.plugins.mapr.base.base_cluster_context as bc +import sahara_plugin_mapr.plugins.mapr.services.yarn.yarn as yarn class Context(bc.BaseClusterContext): diff --git a/sahara/plugins/mapr/versions/v5_2_0_mrv2/version_handler.py b/sahara_plugin_mapr/plugins/mapr/versions/v5_2_0_mrv2/version_handler.py similarity index 54% rename from sahara/plugins/mapr/versions/v5_2_0_mrv2/version_handler.py rename to sahara_plugin_mapr/plugins/mapr/versions/v5_2_0_mrv2/version_handler.py index f947db7..1e346de 100644 --- a/sahara/plugins/mapr/versions/v5_2_0_mrv2/version_handler.py +++ b/sahara_plugin_mapr/plugins/mapr/versions/v5_2_0_mrv2/version_handler.py @@ -13,26 +13,26 @@ # under the License. -import sahara.plugins.mapr.base.base_version_handler as bvh -from sahara.plugins.mapr.services.drill import drill -from sahara.plugins.mapr.services.flume import flume -from sahara.plugins.mapr.services.hbase import hbase -from sahara.plugins.mapr.services.hive import hive -from sahara.plugins.mapr.services.httpfs import httpfs -from sahara.plugins.mapr.services.hue import hue -from sahara.plugins.mapr.services.impala import impala -from sahara.plugins.mapr.services.kafka import kafka -from sahara.plugins.mapr.services.mahout import mahout -from sahara.plugins.mapr.services.management import management as mng -from sahara.plugins.mapr.services.maprfs import maprfs -from sahara.plugins.mapr.services.oozie import oozie -from sahara.plugins.mapr.services.pig import pig -from sahara.plugins.mapr.services.sentry import sentry -from sahara.plugins.mapr.services.spark import spark -from sahara.plugins.mapr.services.sqoop import sqoop2 -from sahara.plugins.mapr.services.swift import swift -from sahara.plugins.mapr.services.yarn import yarn -import sahara.plugins.mapr.versions.v5_2_0_mrv2.context as c +import sahara_plugin_mapr.plugins.mapr.base.base_version_handler as bvh +from sahara_plugin_mapr.plugins.mapr.services.drill import drill +from sahara_plugin_mapr.plugins.mapr.services.flume import flume +from sahara_plugin_mapr.plugins.mapr.services.hbase import hbase +from sahara_plugin_mapr.plugins.mapr.services.hive import hive +from sahara_plugin_mapr.plugins.mapr.services.httpfs import httpfs +from sahara_plugin_mapr.plugins.mapr.services.hue import hue +from sahara_plugin_mapr.plugins.mapr.services.impala import impala +from sahara_plugin_mapr.plugins.mapr.services.kafka import kafka +from sahara_plugin_mapr.plugins.mapr.services.mahout import mahout +from sahara_plugin_mapr.plugins.mapr.services.management import management +from sahara_plugin_mapr.plugins.mapr.services.maprfs import maprfs +from sahara_plugin_mapr.plugins.mapr.services.oozie import oozie +from sahara_plugin_mapr.plugins.mapr.services.pig import pig +from sahara_plugin_mapr.plugins.mapr.services.sentry import sentry +from sahara_plugin_mapr.plugins.mapr.services.spark import spark +from sahara_plugin_mapr.plugins.mapr.services.sqoop import sqoop2 +from sahara_plugin_mapr.plugins.mapr.services.swift import swift +from sahara_plugin_mapr.plugins.mapr.services.yarn import yarn +import sahara_plugin_mapr.plugins.mapr.versions.v5_2_0_mrv2.context as c version = "5.2.0.mrv2" @@ -44,7 +44,7 @@ class VersionHandler(bvh.BaseVersionHandler): self._required_services = [ yarn.YARNv270(), maprfs.MapRFS(), - mng.Management(), + management.Management(), oozie.Oozie(), ] self._services = [ @@ -61,7 +61,7 @@ class VersionHandler(bvh.BaseVersionHandler): drill.DrillV19(), yarn.YARNv270(), maprfs.MapRFS(), - mng.Management(), + management.Management(), httpfs.HttpFS(), swift.Swift(), sentry.SentryV16(), diff --git a/sahara/plugins/mapr/versions/version_handler_factory.py b/sahara_plugin_mapr/plugins/mapr/versions/version_handler_factory.py similarity index 95% rename from sahara/plugins/mapr/versions/version_handler_factory.py rename to sahara_plugin_mapr/plugins/mapr/versions/version_handler_factory.py index 7949a9a..caadfbd 100644 --- a/sahara/plugins/mapr/versions/version_handler_factory.py +++ b/sahara_plugin_mapr/plugins/mapr/versions/version_handler_factory.py @@ -18,7 +18,7 @@ import os def _load_versions(): d_name = os.path.dirname(__file__) - m_template = 'sahara.plugins.mapr.versions.%s.version_handler' + m_template = 'sahara_plugin_mapr.plugins.mapr.versions.%s.version_handler' def predicate(v_dir): if v_dir != "__pycache__": diff --git a/sahara/tests/__init__.py b/sahara_plugin_mapr/tests/__init__.py similarity index 94% rename from sahara/tests/__init__.py rename to sahara_plugin_mapr/tests/__init__.py index 90fe6c0..957889f 100644 --- a/sahara/tests/__init__.py +++ b/sahara_plugin_mapr/tests/__init__.py @@ -13,7 +13,7 @@ # See the License for the specific language governing permissions and # limitations under the License. -from sahara.utils import patches +from sahara_plugin_mapr.utils import patches patches.patch_all() import oslo_i18n diff --git a/sahara/tests/unit/__init__.py b/sahara_plugin_mapr/tests/unit/__init__.py similarity index 100% rename from sahara/tests/unit/__init__.py rename to sahara_plugin_mapr/tests/unit/__init__.py diff --git a/sahara/tests/unit/base.py b/sahara_plugin_mapr/tests/unit/base.py similarity index 67% rename from sahara/tests/unit/base.py rename to sahara_plugin_mapr/tests/unit/base.py index 962f220..364f04a 100644 --- a/sahara/tests/unit/base.py +++ b/sahara_plugin_mapr/tests/unit/base.py @@ -13,13 +13,12 @@ # See the License for the specific language governing permissions and # limitations under the License. -import mock from oslotest import base -from sahara import context -from sahara.db import api as db_api -from sahara import main -from sahara.utils import rpc +from sahara.plugins import context +from sahara.plugins import db as db_api +from sahara.plugins import main +from sahara.plugins import utils class SaharaTestCase(base.BaseTestCase): @@ -27,7 +26,7 @@ class SaharaTestCase(base.BaseTestCase): def setUp(self): super(SaharaTestCase, self).setUp() self.setup_context() - rpc.setup('all-in-one') + utils.rpc_setup('all-in-one') def setup_context(self, username="test_user", tenant_id="tenant_1", auth_token="test_auth_token", tenant_name='test_tenant', @@ -35,14 +34,14 @@ class SaharaTestCase(base.BaseTestCase): self.addCleanup(context.set_ctx, context.ctx() if context.has_ctx() else None) - context.set_ctx(context.Context( + context.set_ctx(context.PluginsContext( username=username, tenant_id=tenant_id, auth_token=auth_token, service_catalog=service_catalog or {}, tenant_name=tenant_name, **kwargs)) def override_config(self, name, override, group=None): - main.CONF.set_override(name, override, group) - self.addCleanup(main.CONF.clear_override, name, group) + main.set_override(name, override, group) + self.addCleanup(main.clear_override, name, group) class SaharaWithDbTestCase(SaharaTestCase): @@ -52,22 +51,3 @@ class SaharaWithDbTestCase(SaharaTestCase): self.override_config('connection', "sqlite://", group='database') db_api.setup_db() self.addCleanup(db_api.drop_db) - - -class _ConsecutiveThreadGroup(context.ThreadGroup): - def __init__(self, _thread_pool_size=1000): - pass - - def spawn(self, thread_description, func, *args, **kwargs): - func(*args, **kwargs) - - def __enter__(self): - return self - - def __exit__(self, *ex): - pass - - -def mock_thread_group(func): - return mock.patch('sahara.context.ThreadGroup', - new=_ConsecutiveThreadGroup)(func) diff --git a/sahara/tests/unit/plugins/__init__.py b/sahara_plugin_mapr/tests/unit/plugins/__init__.py similarity index 100% rename from sahara/tests/unit/plugins/__init__.py rename to sahara_plugin_mapr/tests/unit/plugins/__init__.py diff --git a/sahara/tests/unit/plugins/mapr/__init__.py b/sahara_plugin_mapr/tests/unit/plugins/mapr/__init__.py similarity index 100% rename from sahara/tests/unit/plugins/mapr/__init__.py rename to sahara_plugin_mapr/tests/unit/plugins/mapr/__init__.py diff --git a/sahara/tests/unit/plugins/mapr/test_base_handler.py b/sahara_plugin_mapr/tests/unit/plugins/mapr/test_base_handler.py similarity index 82% rename from sahara/tests/unit/plugins/mapr/test_base_handler.py rename to sahara_plugin_mapr/tests/unit/plugins/mapr/test_base_handler.py index f412edb..199d292 100644 --- a/sahara/tests/unit/plugins/mapr/test_base_handler.py +++ b/sahara_plugin_mapr/tests/unit/plugins/mapr/test_base_handler.py @@ -14,22 +14,23 @@ import six -import sahara.plugins.mapr.abstract.cluster_context as abstract_context -from sahara.plugins.mapr.domain import service as bs -from sahara.plugins.mapr.services.management import management -from sahara.plugins.mapr.services.maprfs import maprfs -from sahara.plugins.mapr.services.oozie import oozie -from sahara.plugins.mapr.services.yarn import yarn -import sahara.plugins.mapr.versions.v5_2_0_mrv2.version_handler as handler from sahara.plugins import provisioning as p -from sahara.tests.unit import base as b -from sahara.tests.unit import testutils as tu +from sahara.plugins import testutils as tu +import sahara_plugin_mapr.plugins.mapr.abstract.cluster_context as a_context +from sahara_plugin_mapr.plugins.mapr.domain import service as bs +from sahara_plugin_mapr.plugins.mapr.services.management import management +from sahara_plugin_mapr.plugins.mapr.services.maprfs import maprfs +from sahara_plugin_mapr.plugins.mapr.services.oozie import oozie +from sahara_plugin_mapr.plugins.mapr.services.yarn import yarn +from sahara_plugin_mapr.plugins.mapr.versions.v5_2_0_mrv2 import \ + version_handler +from sahara_plugin_mapr.tests.unit import base as b class TestHandler(b.SaharaTestCase): def __init__(self, *args, **kwds): super(TestHandler, self).__init__(*args, **kwds) - self.handler = handler.VersionHandler() + self.handler = version_handler.VersionHandler() self.cluster = self._get_cluster() def _get_cluster(self): @@ -54,7 +55,7 @@ class TestHandler(b.SaharaTestCase): cluster = tu.create_cluster( name='test_cluster', - tenant='large', + project='large', plugin='mapr', version='5.2.0.mrv2', node_groups=[master_ng], @@ -66,7 +67,7 @@ class TestHandler(b.SaharaTestCase): def test_get_context(self): self.assertIsInstance(self.handler.get_context(self._get_cluster()), - abstract_context.AbstractClusterContext) + a_context.AbstractClusterContext) def test_get_services(self): services = self.handler.get_services() diff --git a/sahara/tests/unit/plugins/mapr/test_cluster_context.py b/sahara_plugin_mapr/tests/unit/plugins/mapr/test_cluster_context.py similarity index 92% rename from sahara/tests/unit/plugins/mapr/test_cluster_context.py rename to sahara_plugin_mapr/tests/unit/plugins/mapr/test_cluster_context.py index f0128a9..ef724b5 100644 --- a/sahara/tests/unit/plugins/mapr/test_cluster_context.py +++ b/sahara_plugin_mapr/tests/unit/plugins/mapr/test_cluster_context.py @@ -14,18 +14,19 @@ import testtools -import sahara.exceptions as e -from sahara.plugins.mapr.domain import node_process as np -from sahara.plugins.mapr.services.management import management -from sahara.plugins.mapr.services.maprfs import maprfs -from sahara.plugins.mapr.services.oozie import oozie -from sahara.plugins.mapr.services.swift import swift -from sahara.plugins.mapr.services.yarn import yarn -import sahara.plugins.mapr.versions.v5_2_0_mrv2.context as cc -import sahara.plugins.mapr.versions.v5_2_0_mrv2.version_handler as handler +import sahara.plugins.exceptions as e from sahara.plugins import provisioning as p -from sahara.tests.unit import base as b -from sahara.tests.unit import testutils as tu +from sahara.plugins import testutils as tu +from sahara_plugin_mapr.plugins.mapr.domain import node_process as np +from sahara_plugin_mapr.plugins.mapr.services.management import management +from sahara_plugin_mapr.plugins.mapr.services.maprfs import maprfs +from sahara_plugin_mapr.plugins.mapr.services.oozie import oozie +from sahara_plugin_mapr.plugins.mapr.services.swift import swift +from sahara_plugin_mapr.plugins.mapr.services.yarn import yarn +import sahara_plugin_mapr.plugins.mapr.versions.v5_2_0_mrv2.context as cc +from sahara_plugin_mapr.plugins.mapr.versions.v5_2_0_mrv2 import \ + version_handler +from sahara_plugin_mapr.tests.unit import base as b MANAGEMENT_IP = '1.1.1.1' @@ -61,7 +62,7 @@ class TestClusterContext(b.SaharaTestCase): } cluster = tu.create_cluster( name='test_cluster', - tenant='large', + project='large', plugin='mapr', version='5.2.0.mrv2', node_groups=[master_ng], @@ -69,7 +70,7 @@ class TestClusterContext(b.SaharaTestCase): ) self.ng = cluster.node_groups[0] self.instance = self.ng.instances[0] - return cc.Context(cluster, handler.VersionHandler()) + return cc.Context(cluster, version_handler.VersionHandler()) def test_get_oozie_server_uri(self): ctx = self._get_context() @@ -201,7 +202,7 @@ class TestClusterContext(b.SaharaTestCase): ctx = self._get_context() service = ctx.get_service(yarn.HISTORY_SERVER) self.assertEqual(yarn.YARN().ui_name, service.ui_name) - with testtools.ExpectedException(e.InvalidDataException): + with testtools.ExpectedException(e.PluginInvalidDataException): ctx.get_service(self.fake_np) def test_get_service_name_by_node_process(self): diff --git a/sahara/tests/unit/plugins/mapr/test_config_files.py b/sahara_plugin_mapr/tests/unit/plugins/mapr/test_config_files.py similarity index 98% rename from sahara/tests/unit/plugins/mapr/test_config_files.py rename to sahara_plugin_mapr/tests/unit/plugins/mapr/test_config_files.py index 647b744..16c41a8 100644 --- a/sahara/tests/unit/plugins/mapr/test_config_files.py +++ b/sahara_plugin_mapr/tests/unit/plugins/mapr/test_config_files.py @@ -13,8 +13,8 @@ # under the License. -import sahara.plugins.mapr.domain.configuration_file as conf_f -import sahara.tests.unit.base as b +import sahara_plugin_mapr.plugins.mapr.domain.configuration_file as conf_f +import sahara_plugin_mapr.tests.unit.base as b class TestHadoopXML(b.SaharaTestCase): diff --git a/sahara/tests/unit/plugins/mapr/test_distro.py b/sahara_plugin_mapr/tests/unit/plugins/mapr/test_distro.py similarity index 92% rename from sahara/tests/unit/plugins/mapr/test_distro.py rename to sahara_plugin_mapr/tests/unit/plugins/mapr/test_distro.py index 8432d0b..7013c09 100644 --- a/sahara/tests/unit/plugins/mapr/test_distro.py +++ b/sahara_plugin_mapr/tests/unit/plugins/mapr/test_distro.py @@ -12,8 +12,8 @@ # License for the specific language governing permissions and limitations # under the License. -from sahara.plugins.mapr.domain import distro -import sahara.tests.unit.base as b +from sahara_plugin_mapr.plugins.mapr.domain import distro +import sahara_plugin_mapr.tests.unit.base as b class TestDistro(b.SaharaTestCase): diff --git a/sahara/utils/__init__.py b/sahara_plugin_mapr/utils/__init__.py similarity index 100% rename from sahara/utils/__init__.py rename to sahara_plugin_mapr/utils/__init__.py diff --git a/sahara/utils/patches.py b/sahara_plugin_mapr/utils/patches.py similarity index 100% rename from sahara/utils/patches.py rename to sahara_plugin_mapr/utils/patches.py diff --git a/setup.cfg b/setup.cfg index c75e0a2..3c462c1 100644 --- a/setup.cfg +++ b/setup.cfg @@ -1,6 +1,6 @@ [metadata] -name = sahara -summary = Sahara project +name = sahara_plugin_mapr +summary = Mapr Plugin for Sahara Project description-file = README.rst license = Apache Software License classifiers = @@ -23,80 +23,17 @@ setup-hooks = pbr.hooks.setup_hook [files] packages = - sahara - -data_files = - etc/sahara = - etc/sahara/api-paste.ini - etc/sahara/rootwrap.conf - etc/sahara/rootwrap.d = etc/sahara/rootwrap.d/* + sahara_plugin_mapr [entry_points] -console_scripts = - sahara-all = sahara.cli.sahara_all:main - sahara-api = sahara.cli.sahara_api:main - sahara-engine = sahara.cli.sahara_engine:main - sahara-db-manage = sahara.db.migration.cli:main - sahara-rootwrap = oslo_rootwrap.cmd:main - _sahara-subprocess = sahara.cli.sahara_subprocess:main - sahara-templates = sahara.db.templates.cli:main - sahara-image-pack = sahara.cli.image_pack.cli:main - sahara-status = sahara.cli.sahara_status:main - -wsgi_scripts = - sahara-wsgi-api = sahara.cli.sahara_api:setup_api - sahara.cluster.plugins = - vanilla = sahara.plugins.vanilla.plugin:VanillaProvider - ambari = sahara.plugins.ambari.plugin:AmbariPluginProvider - mapr = sahara.plugins.mapr.plugin:MapRPlugin - cdh = sahara.plugins.cdh.plugin:CDHPluginProvider - fake = sahara.plugins.fake.plugin:FakePluginProvider - spark = sahara.plugins.spark.plugin:SparkProvider - storm = sahara.plugins.storm.plugin:StormProvider + mapr = sahara_plugin_mapr.plugins.mapr.plugin:MapRPlugin -sahara.data_source.types = - hdfs = sahara.service.edp.data_sources.hdfs.implementation:HDFSType - manila = sahara.service.edp.data_sources.manila.implementation:ManilaType - maprfs = sahara.service.edp.data_sources.maprfs.implementation:MapRFSType - swift = sahara.service.edp.data_sources.swift.implementation:SwiftType - s3 = sahara.service.edp.data_sources.s3.implementation:S3Type +[build_sphinx] +all_files = 1 +build-dir = doc/build +source-dir = doc/source +warning-is-error = 1 -sahara.job_binary.types = - internal-db = sahara.service.edp.job_binaries.internal_db.implementation:InternalDBType - manila = sahara.service.edp.job_binaries.manila.implementation:ManilaType - swift = sahara.service.edp.job_binaries.swift.implementation:SwiftType - s3 = sahara.service.edp.job_binaries.s3.implementation:S3Type - -sahara.infrastructure.engine = - heat = sahara.service.heat.heat_engine:HeatEngine - -sahara.remote = - ssh = sahara.utils.ssh_remote:SshRemoteDriver - -sahara.run.mode = - all-in-one = sahara.service.ops:LocalOps - distributed = sahara.service.ops:RemoteOps - -oslo.config.opts = - sahara.config = sahara.config:list_opts - -oslo.config.opts.defaults = - sahara.config = sahara.common.config:set_cors_middleware_defaults - -oslo.policy.policies = - sahara = sahara.common.policies:list_rules - -[extract_messages] -keywords = _ gettext ngettext l_ lazy_gettext -mapping_file = babel.cfg -output_file = sahara/locale/sahara.pot - -[compile_catalog] -directory = sahara/locale -domain = sahara - -[update_catalog] -domain = sahara -output_dir = sahara/locale -input_file = sahara/locale/sahara.pot +[wheel] +universal = 1 diff --git a/tox.ini b/tox.ini index 5c16d20..cfd8f6c 100644 --- a/tox.ini +++ b/tox.ini @@ -1,6 +1,6 @@ [tox] -envlist = py36,py35,py27,pep8,genpolicy -minversion = 1.6 +envlist = py36,py35,py27,pep8 +minversion = 2.0 skipsdist = True [testenv] @@ -8,7 +8,7 @@ usedevelop = True install_command = pip install {opts} {packages} setenv = VIRTUAL_ENV={envdir} - DISCOVER_DIRECTORY=sahara/tests/unit + DISCOVER_DIRECTORY=sahara_plugin_mapr/tests/unit deps = -c{env:UPPER_CONSTRAINTS_FILE:https://git.openstack.org/cgit/openstack/requirements/plain/upper-constraints.txt} -r{toxinidir}/requirements.txt @@ -16,19 +16,13 @@ deps = commands = stestr run {posargs} passenv = http_proxy HTTP_PROXY https_proxy HTTPS_PROXY no_proxy NO_PROXY -[testenv:cover] -basepython = python3 -setenv = - PACKAGE_NAME=sahara -commands = {toxinidir}/tools/cover.sh {posargs} - [testenv:debug-py27] basepython = python2.7 -commands = oslo_debug_helper -t sahara/tests/unit {posargs} +commands = oslo_debug_helper -t sahara_plugin_mapr/tests/unit {posargs} [testenv:debug-py35] basepython = python3.5 -commands = oslo_debug_helper -t sahara/tests/unit {posargs} +commands = oslo_debug_helper -t sahara_plugin_mapr/tests/unit {posargs} [testenv:pep8] basepython = python3 @@ -41,23 +35,12 @@ commands = flake8 {posargs} doc8 doc/source # Run bashate checks - bash -c "find sahara -iname '*.sh' -print0 | xargs -0 bashate -v" - bash -c "find devstack -not -name \*.template -and -not -name README.rst -and -not -name \*.json -type f -print0 | xargs -0 bashate -v" - # Run security linter - bandit -c bandit.yaml -r sahara -n5 -p sahara_default -x tests - -[testenv:genpolicy] -basepython = python3 -commands = oslopolicy-sample-generator --config-file tools/config/sahara-policy-generator.conf + bash -c "find sahara_plugin_mapr -iname '*.sh' -print0 | xargs -0 bashate -v" [testenv:venv] basepython = python3 commands = {posargs} -[testenv:images] -sitepackages = True -commands = {posargs} - [testenv:docs] basepython = python3 deps = @@ -65,35 +48,9 @@ deps = -r{toxinidir}/doc/requirements.txt commands = rm -rf doc/html doc/build - rm -rf api-ref/build api-ref/html - rm -rf doc/source/apidoc doc/source/api sphinx-build -W -b html doc/source doc/build/html - sphinx-build -W -b html -d api-ref/build/doctrees api-ref/source api-ref/build/html - whereto doc/source/_extra/.htaccess doc/test/redirect-tests.txt -whitelist_externals = rm - -[testenv:api-ref] -basepython = python3 -deps = - -c{env:UPPER_CONSTRAINTS_FILE:https://git.openstack.org/cgit/openstack/requirements/plain/upper-constraints.txt} - -r{toxinidir}/doc/requirements.txt -install_command = pip install -U --force-reinstall {opts} {packages} -commands = - rm -rf api-ref/build api-ref/html - sphinx-build -W -b html -d api-ref/build/doctrees api-ref/source api-ref/build/html -whitelist_externals = rm - - -[testenv:pylint] -basepython = python3 -setenv = VIRTUAL_ENV={envdir} -commands = bash tools/lintstack.sh - -[testenv:genconfig] -basepython = python3 -commands = - oslo-config-generator --config-file tools/config/config-generator.sahara.conf \ - --output-file etc/sahara/sahara.conf.sample +whitelist_externals = + rm [testenv:releasenotes] basepython = python3 @@ -107,16 +64,11 @@ whitelist_externals = rm [testenv:debug] basepython = python3 -# It runs tests from the specified dir (default is sahara/tests) +# It runs tests from the specified dir (default is sahara_plugin_mapr/tests) # in interactive mode, so, you could use pbr for tests debug. -# Example usage: tox -e debug -- -t sahara/tests/unit some.test.path +# Example usage: tox -e debug -- -t sahara_plugin_mapr/tests/unit some.test.path # https://docs.openstack.org/oslotest/latest/features.html#debugging-with-oslo-debug-helper -commands = oslo_debug_helper -t sahara/tests/unit {posargs} - -[testenv:bandit] -basepython = python3 -deps = -r{toxinidir}/test-requirements-bandit.txt -commands = bandit -c bandit.yaml -r sahara -n5 -p sahara_default -x tests +commands = oslo_debug_helper -t sahara_plugin_mapr/tests/unit {posargs} [flake8] show-source = true @@ -135,19 +87,6 @@ enable-extensions=H904,H106,H203,H204,H205 # [W503] Line break occurred before a binary operator ignore=E123,E226,E402,E731,W503 -[hacking] -import_exceptions = sahara.i18n -local-check-factory = sahara.utils.hacking.checks.factory - -[testenv:bindep] -basepython = python3 -# Do not install any requirements. We want this to be fast and work even if -# system dependencies are missing, since it's used to tell you what system -# dependencies are missing! This also means that bindep must be installed -# separately, outside of the requirements files. -deps = bindep -commands = bindep test - [testenv:lower-constraints] basepython = python3 deps =