Retire Sahara: remove repo content
Sahara project is retiring - https://review.opendev.org/c/openstack/governance/+/919374 this commit remove the content of this project repo Depends-On: https://review.opendev.org/c/openstack/project-config/+/919376 Change-Id: Ie38207822e0cc65aa30b114a401a9943a64e5bf2
This commit is contained in:
parent
f187fff27e
commit
b7c783cb1b
.gitignore.stestr.conf.zuul.yamlCONTRIBUTING.rstLICENSEREADME.rstbabel.cfgrequirements.txt
doc
requirements.txt
errors.txtsource
releasenotes
notes
source
sahara_plugin_vanilla
__init__.pyi18n.py
locale
de/LC_MESSAGES
en_GB/LC_MESSAGES
id/LC_MESSAGES
ne/LC_MESSAGES
plugins
__init__.py
vanilla
__init__.pyabstractversionhandler.pyconfighints_helper.pyedp_engine.py
hadoop2
__init__.pyconfig.pyconfig_helper.pyedp_engine.pykeypairs.pyoozie_helper.pyrecommendations_utils.py
plugin.pyutils.pyresources
create_oozie_db.sqlpost_conf.templatespark-cleanup.crontmp-cleanup.sh.templatetopology.shzoo_sample.cfg
run_scripts.pyscaling.pystarting_scripts.pyutils.pyvalidation.pyv2_7_1
__init__.pyconfig_helper.pyedp_engine.py
resources
README.rstcore-default.xmlcreate_hive_db.sqlhdfs-default.xmlhive-default.xmlmapred-default.xmloozie-default.xmlyarn-default.xml
versionhandler.pyv2_7_5
__init__.pyconfig_helper.pyedp_engine.py
resources
README.rstcore-default.xmlcreate_hive_db.sqlhdfs-default.xmlhive-default.xmlmapred-default.xmloozie-default.xmlyarn-default.xml
versionhandler.pyv2_8_2
__init__.pyconfig_helper.pyedp_engine.py
versionfactory.pyresources
README.rstcore-default.xmlcreate_hive_db.sqlhdfs-default.xmlhive-default.xmlmapred-default.xmloozie-default.xmlyarn-default.xml
versionhandler.py
30
.gitignore
vendored
30
.gitignore
vendored
@ -1,30 +0,0 @@
|
||||
*.egg-info
|
||||
*.egg[s]
|
||||
*.log
|
||||
*.py[co]
|
||||
.coverage
|
||||
.testrepository
|
||||
.tox
|
||||
.stestr
|
||||
.venv
|
||||
.idea
|
||||
AUTHORS
|
||||
ChangeLog
|
||||
build
|
||||
cover
|
||||
develop-eggs
|
||||
dist
|
||||
doc/build
|
||||
doc/html
|
||||
eggs
|
||||
etc/sahara.conf
|
||||
etc/sahara/*.conf
|
||||
etc/sahara/*.topology
|
||||
sdist
|
||||
target
|
||||
tools/lintstack.head.py
|
||||
tools/pylint_exceptions
|
||||
doc/source/sample.config
|
||||
|
||||
# Files created by releasenotes build
|
||||
releasenotes/build
|
@ -1,3 +0,0 @@
|
||||
[DEFAULT]
|
||||
test_path=./sahara_plugin_vanilla/tests/unit
|
||||
top_dir=./
|
@ -1,6 +0,0 @@
|
||||
- project:
|
||||
templates:
|
||||
- check-requirements
|
||||
- openstack-python3-zed-jobs
|
||||
- publish-openstack-docs-pti
|
||||
- release-notes-jobs-python3
|
@ -1,19 +0,0 @@
|
||||
The source repository for this project can be found at:
|
||||
|
||||
https://opendev.org/openstack/sahara-plugin-vanilla
|
||||
|
||||
Pull requests submitted through GitHub are not monitored.
|
||||
|
||||
To start contributing to OpenStack, follow the steps in the contribution guide
|
||||
to set up and use Gerrit:
|
||||
|
||||
https://docs.openstack.org/contributors/code-and-documentation/quick-start.html
|
||||
|
||||
Bugs should be filed on Storyboard:
|
||||
|
||||
https://storyboard.openstack.org/#!/project/openstack/sahara-plugin-vanilla
|
||||
|
||||
For more specific information about contributing to this repository, see the
|
||||
sahara-plugin-vanilla contributor guide:
|
||||
|
||||
https://docs.openstack.org/sahara-plugin-vanilla/latest/contributor/contributing.html
|
175
LICENSE
175
LICENSE
@ -1,175 +0,0 @@
|
||||
|
||||
Apache License
|
||||
Version 2.0, January 2004
|
||||
http://www.apache.org/licenses/
|
||||
|
||||
TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
|
||||
|
||||
1. Definitions.
|
||||
|
||||
"License" shall mean the terms and conditions for use, reproduction,
|
||||
and distribution as defined by Sections 1 through 9 of this document.
|
||||
|
||||
"Licensor" shall mean the copyright owner or entity authorized by
|
||||
the copyright owner that is granting the License.
|
||||
|
||||
"Legal Entity" shall mean the union of the acting entity and all
|
||||
other entities that control, are controlled by, or are under common
|
||||
control with that entity. For the purposes of this definition,
|
||||
"control" means (i) the power, direct or indirect, to cause the
|
||||
direction or management of such entity, whether by contract or
|
||||
otherwise, or (ii) ownership of fifty percent (50%) or more of the
|
||||
outstanding shares, or (iii) beneficial ownership of such entity.
|
||||
|
||||
"You" (or "Your") shall mean an individual or Legal Entity
|
||||
exercising permissions granted by this License.
|
||||
|
||||
"Source" form shall mean the preferred form for making modifications,
|
||||
including but not limited to software source code, documentation
|
||||
source, and configuration files.
|
||||
|
||||
"Object" form shall mean any form resulting from mechanical
|
||||
transformation or translation of a Source form, including but
|
||||
not limited to compiled object code, generated documentation,
|
||||
and conversions to other media types.
|
||||
|
||||
"Work" shall mean the work of authorship, whether in Source or
|
||||
Object form, made available under the License, as indicated by a
|
||||
copyright notice that is included in or attached to the work
|
||||
(an example is provided in the Appendix below).
|
||||
|
||||
"Derivative Works" shall mean any work, whether in Source or Object
|
||||
form, that is based on (or derived from) the Work and for which the
|
||||
editorial revisions, annotations, elaborations, or other modifications
|
||||
represent, as a whole, an original work of authorship. For the purposes
|
||||
of this License, Derivative Works shall not include works that remain
|
||||
separable from, or merely link (or bind by name) to the interfaces of,
|
||||
the Work and Derivative Works thereof.
|
||||
|
||||
"Contribution" shall mean any work of authorship, including
|
||||
the original version of the Work and any modifications or additions
|
||||
to that Work or Derivative Works thereof, that is intentionally
|
||||
submitted to Licensor for inclusion in the Work by the copyright owner
|
||||
or by an individual or Legal Entity authorized to submit on behalf of
|
||||
the copyright owner. For the purposes of this definition, "submitted"
|
||||
means any form of electronic, verbal, or written communication sent
|
||||
to the Licensor or its representatives, including but not limited to
|
||||
communication on electronic mailing lists, source code control systems,
|
||||
and issue tracking systems that are managed by, or on behalf of, the
|
||||
Licensor for the purpose of discussing and improving the Work, but
|
||||
excluding communication that is conspicuously marked or otherwise
|
||||
designated in writing by the copyright owner as "Not a Contribution."
|
||||
|
||||
"Contributor" shall mean Licensor and any individual or Legal Entity
|
||||
on behalf of whom a Contribution has been received by Licensor and
|
||||
subsequently incorporated within the Work.
|
||||
|
||||
2. Grant of Copyright License. Subject to the terms and conditions of
|
||||
this License, each Contributor hereby grants to You a perpetual,
|
||||
worldwide, non-exclusive, no-charge, royalty-free, irrevocable
|
||||
copyright license to reproduce, prepare Derivative Works of,
|
||||
publicly display, publicly perform, sublicense, and distribute the
|
||||
Work and such Derivative Works in Source or Object form.
|
||||
|
||||
3. Grant of Patent License. Subject to the terms and conditions of
|
||||
this License, each Contributor hereby grants to You a perpetual,
|
||||
worldwide, non-exclusive, no-charge, royalty-free, irrevocable
|
||||
(except as stated in this section) patent license to make, have made,
|
||||
use, offer to sell, sell, import, and otherwise transfer the Work,
|
||||
where such license applies only to those patent claims licensable
|
||||
by such Contributor that are necessarily infringed by their
|
||||
Contribution(s) alone or by combination of their Contribution(s)
|
||||
with the Work to which such Contribution(s) was submitted. If You
|
||||
institute patent litigation against any entity (including a
|
||||
cross-claim or counterclaim in a lawsuit) alleging that the Work
|
||||
or a Contribution incorporated within the Work constitutes direct
|
||||
or contributory patent infringement, then any patent licenses
|
||||
granted to You under this License for that Work shall terminate
|
||||
as of the date such litigation is filed.
|
||||
|
||||
4. Redistribution. You may reproduce and distribute copies of the
|
||||
Work or Derivative Works thereof in any medium, with or without
|
||||
modifications, and in Source or Object form, provided that You
|
||||
meet the following conditions:
|
||||
|
||||
(a) You must give any other recipients of the Work or
|
||||
Derivative Works a copy of this License; and
|
||||
|
||||
(b) You must cause any modified files to carry prominent notices
|
||||
stating that You changed the files; and
|
||||
|
||||
(c) You must retain, in the Source form of any Derivative Works
|
||||
that You distribute, all copyright, patent, trademark, and
|
||||
attribution notices from the Source form of the Work,
|
||||
excluding those notices that do not pertain to any part of
|
||||
the Derivative Works; and
|
||||
|
||||
(d) If the Work includes a "NOTICE" text file as part of its
|
||||
distribution, then any Derivative Works that You distribute must
|
||||
include a readable copy of the attribution notices contained
|
||||
within such NOTICE file, excluding those notices that do not
|
||||
pertain to any part of the Derivative Works, in at least one
|
||||
of the following places: within a NOTICE text file distributed
|
||||
as part of the Derivative Works; within the Source form or
|
||||
documentation, if provided along with the Derivative Works; or,
|
||||
within a display generated by the Derivative Works, if and
|
||||
wherever such third-party notices normally appear. The contents
|
||||
of the NOTICE file are for informational purposes only and
|
||||
do not modify the License. You may add Your own attribution
|
||||
notices within Derivative Works that You distribute, alongside
|
||||
or as an addendum to the NOTICE text from the Work, provided
|
||||
that such additional attribution notices cannot be construed
|
||||
as modifying the License.
|
||||
|
||||
You may add Your own copyright statement to Your modifications and
|
||||
may provide additional or different license terms and conditions
|
||||
for use, reproduction, or distribution of Your modifications, or
|
||||
for any such Derivative Works as a whole, provided Your use,
|
||||
reproduction, and distribution of the Work otherwise complies with
|
||||
the conditions stated in this License.
|
||||
|
||||
5. Submission of Contributions. Unless You explicitly state otherwise,
|
||||
any Contribution intentionally submitted for inclusion in the Work
|
||||
by You to the Licensor shall be under the terms and conditions of
|
||||
this License, without any additional terms or conditions.
|
||||
Notwithstanding the above, nothing herein shall supersede or modify
|
||||
the terms of any separate license agreement you may have executed
|
||||
with Licensor regarding such Contributions.
|
||||
|
||||
6. Trademarks. This License does not grant permission to use the trade
|
||||
names, trademarks, service marks, or product names of the Licensor,
|
||||
except as required for reasonable and customary use in describing the
|
||||
origin of the Work and reproducing the content of the NOTICE file.
|
||||
|
||||
7. Disclaimer of Warranty. Unless required by applicable law or
|
||||
agreed to in writing, Licensor provides the Work (and each
|
||||
Contributor provides its Contributions) on an "AS IS" BASIS,
|
||||
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
|
||||
implied, including, without limitation, any warranties or conditions
|
||||
of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
|
||||
PARTICULAR PURPOSE. You are solely responsible for determining the
|
||||
appropriateness of using or redistributing the Work and assume any
|
||||
risks associated with Your exercise of permissions under this License.
|
||||
|
||||
8. Limitation of Liability. In no event and under no legal theory,
|
||||
whether in tort (including negligence), contract, or otherwise,
|
||||
unless required by applicable law (such as deliberate and grossly
|
||||
negligent acts) or agreed to in writing, shall any Contributor be
|
||||
liable to You for damages, including any direct, indirect, special,
|
||||
incidental, or consequential damages of any character arising as a
|
||||
result of this License or out of the use or inability to use the
|
||||
Work (including but not limited to damages for loss of goodwill,
|
||||
work stoppage, computer failure or malfunction, or any and all
|
||||
other commercial damages or losses), even if such Contributor
|
||||
has been advised of the possibility of such damages.
|
||||
|
||||
9. Accepting Warranty or Additional Liability. While redistributing
|
||||
the Work or Derivative Works thereof, You may choose to offer,
|
||||
and charge a fee for, acceptance of support, warranty, indemnity,
|
||||
or other liability obligations and/or rights consistent with this
|
||||
License. However, in accepting such obligations, You may act only
|
||||
on Your own behalf and on Your sole responsibility, not on behalf
|
||||
of any other Contributor, and only if You agree to indemnify,
|
||||
defend, and hold each Contributor harmless for any liability
|
||||
incurred by, or claims asserted against, such Contributor by reason
|
||||
of your accepting any such warranty or additional liability.
|
44
README.rst
44
README.rst
@ -1,38 +1,10 @@
|
||||
========================
|
||||
Team and repository tags
|
||||
========================
|
||||
This project is no longer maintained.
|
||||
|
||||
.. image:: https://governance.openstack.org/tc/badges/sahara.svg
|
||||
:target: https://governance.openstack.org/tc/reference/tags/index.html
|
||||
|
||||
.. Change things from this point on
|
||||
|
||||
OpenStack Data Processing ("Sahara") Vanilla Plugin
|
||||
====================================================
|
||||
|
||||
OpenStack Sahara Vanilla Plugin provides the users the option to
|
||||
start Vanilla clusters on OpenStack Sahara.
|
||||
Check out OpenStack Sahara documentation to see how to deploy the
|
||||
Vanilla Plugin.
|
||||
|
||||
Sahara at wiki.openstack.org: https://wiki.openstack.org/wiki/Sahara
|
||||
|
||||
Storyboard project: https://storyboard.openstack.org/#!/project/openstack/sahara-plugin-vanilla
|
||||
|
||||
Sahara docs site: https://docs.openstack.org/sahara/latest/
|
||||
|
||||
Quickstart guide: https://docs.openstack.org/sahara/latest/user/quickstart.html
|
||||
|
||||
How to participate: https://docs.openstack.org/sahara/latest/contributor/how-to-participate.html
|
||||
|
||||
Source: https://opendev.org/openstack/sahara-plugin-vanilla
|
||||
|
||||
Bugs and feature requests: https://storyboard.openstack.org/#!/project/openstack/sahara-plugin-vanilla
|
||||
|
||||
Release notes: https://docs.openstack.org/releasenotes/sahara-plugin-vanilla/
|
||||
|
||||
License
|
||||
-------
|
||||
|
||||
Apache License Version 2.0 http://www.apache.org/licenses/LICENSE-2.0
|
||||
The contents of this repository are still available in the Git
|
||||
source code management system. To see the contents of this
|
||||
repository before it reached its end of life, please check out the
|
||||
previous commit with "git checkout HEAD^1".
|
||||
|
||||
For any further questions, please email
|
||||
openstack-discuss@lists.openstack.org or join #openstack-dev on
|
||||
OFTC.
|
||||
|
@ -1 +0,0 @@
|
||||
[python: **.py]
|
@ -1,9 +0,0 @@
|
||||
# The order of packages is significant, because pip processes them in the order
|
||||
# of appearance. Changing the order has an impact on the overall integration
|
||||
# process, which may cause wedges in the gate later.
|
||||
openstackdocstheme>=2.2.1 # Apache-2.0
|
||||
os-api-ref>=1.4.0 # Apache-2.0
|
||||
reno>=3.1.0 # Apache-2.0
|
||||
sphinx>=2.0.0,!=2.1.0 # BSD
|
||||
sphinxcontrib-httpdomain>=1.3.0 # BSD
|
||||
whereto>=0.3.0 # Apache-2.0
|
@ -1,214 +0,0 @@
|
||||
# -*- coding: utf-8 -*-
|
||||
#
|
||||
# sahara-plugin-vanilla documentation build configuration file.
|
||||
#
|
||||
# -- General configuration -----------------------------------------------------
|
||||
|
||||
# If your documentation needs a minimal Sphinx version, state it here.
|
||||
#needs_sphinx = '1.0'
|
||||
|
||||
# Add any Sphinx extension module names here, as strings. They can be extensions
|
||||
# coming with Sphinx (named 'sphinx.ext.*') or your custom ones.
|
||||
extensions = [
|
||||
'reno.sphinxext',
|
||||
'openstackdocstheme',
|
||||
]
|
||||
|
||||
# openstackdocstheme options
|
||||
openstackdocs_repo_name = 'openstack/sahara-plugin-vanilla'
|
||||
openstackdocs_pdf_link = True
|
||||
openstackdocs_use_storyboard = True
|
||||
|
||||
openstackdocs_projects = [
|
||||
'sahara'
|
||||
]
|
||||
|
||||
# Add any paths that contain templates here, relative to this directory.
|
||||
templates_path = ['_templates']
|
||||
|
||||
# The suffix of source filenames.
|
||||
source_suffix = '.rst'
|
||||
|
||||
# The encoding of source files.
|
||||
#source_encoding = 'utf-8-sig'
|
||||
|
||||
# The master toctree document.
|
||||
master_doc = 'index'
|
||||
|
||||
# General information about the project.
|
||||
copyright = u'2015, Sahara team'
|
||||
|
||||
# The language for content autogenerated by Sphinx. Refer to documentation
|
||||
# for a list of supported languages.
|
||||
#language = None
|
||||
|
||||
# There are two options for replacing |today|: either, you set today to some
|
||||
# non-false value, then it is used:
|
||||
#today = ''
|
||||
# Else, today_fmt is used as the format for a strftime call.
|
||||
#today_fmt = '%B %d, %Y'
|
||||
|
||||
# List of patterns, relative to source directory, that match files and
|
||||
# directories to ignore when looking for source files.
|
||||
exclude_patterns = []
|
||||
|
||||
# The reST default role (used for this markup: `text`) to use for all documents.
|
||||
#default_role = None
|
||||
|
||||
# If true, '()' will be appended to :func: etc. cross-reference text.
|
||||
#add_function_parentheses = True
|
||||
|
||||
# If true, the current module name will be prepended to all description
|
||||
# unit titles (such as .. function::).
|
||||
#add_module_names = True
|
||||
|
||||
# If true, sectionauthor and moduleauthor directives will be shown in the
|
||||
# output. They are ignored by default.
|
||||
#show_authors = False
|
||||
|
||||
# The name of the Pygments (syntax highlighting) style to use.
|
||||
pygments_style = 'native'
|
||||
|
||||
# A list of ignored prefixes for module index sorting.
|
||||
#modindex_common_prefix = []
|
||||
|
||||
|
||||
# -- Options for HTML output ---------------------------------------------------
|
||||
|
||||
# The theme to use for HTML and HTML Help pages. See the documentation for
|
||||
# a list of builtin themes.
|
||||
html_theme = 'openstackdocs'
|
||||
|
||||
# Theme options are theme-specific and customize the look and feel of a theme
|
||||
# further. For a list of options available for each theme, see the
|
||||
# documentation.
|
||||
#html_theme_options = {}
|
||||
|
||||
# Add any paths that contain custom themes here, relative to this directory.
|
||||
#html_theme_path = []
|
||||
|
||||
# The name for this set of Sphinx documents. If None, it defaults to
|
||||
# "<project> v<release> documentation".
|
||||
#html_title = None
|
||||
|
||||
# A shorter title for the navigation bar. Default is the same as html_title.
|
||||
#html_short_title = None
|
||||
|
||||
# The name of an image file (relative to this directory) to place at the top
|
||||
# of the sidebar.
|
||||
#html_logo = None
|
||||
|
||||
# The name of an image file (within the static path) to use as favicon of the
|
||||
# docs. This file should be a Windows icon file (.ico) being 16x16 or 32x32
|
||||
# pixels large.
|
||||
#html_favicon = None
|
||||
|
||||
# Add any paths that contain custom static files (such as style sheets) here,
|
||||
# relative to this directory. They are copied after the builtin static files,
|
||||
# so a file named "default.css" will overwrite the builtin "default.css".
|
||||
#html_static_path = ['_static']
|
||||
|
||||
# If not '', a 'Last updated on:' timestamp is inserted at every page bottom,
|
||||
# using the given strftime format.
|
||||
#html_last_updated_fmt = '%b %d, %Y'
|
||||
|
||||
# Custom sidebar templates, maps document names to template names.
|
||||
#html_sidebars = {}
|
||||
|
||||
# Additional templates that should be rendered to pages, maps page names to
|
||||
# template names.
|
||||
#html_additional_pages = {}
|
||||
|
||||
# If false, no module index is generated.
|
||||
#html_domain_indices = True
|
||||
|
||||
# If false, no index is generated.
|
||||
#html_use_index = True
|
||||
|
||||
# If true, the index is split into individual pages for each letter.
|
||||
#html_split_index = False
|
||||
|
||||
# If true, links to the reST sources are added to the pages.
|
||||
#html_show_sourcelink = True
|
||||
|
||||
# If true, "Created using Sphinx" is shown in the HTML footer. Default is True.
|
||||
#html_show_sphinx = True
|
||||
|
||||
# If true, "(C) Copyright ..." is shown in the HTML footer. Default is True.
|
||||
#html_show_copyright = True
|
||||
|
||||
# If true, an OpenSearch description file will be output, and all pages will
|
||||
# contain a <link> tag referring to it. The value of this option must be the
|
||||
# base URL from which the finished HTML is served.
|
||||
#html_use_opensearch = ''
|
||||
|
||||
# This is the file name suffix for HTML files (e.g. ".xhtml").
|
||||
#html_file_suffix = None
|
||||
|
||||
# Output file base name for HTML help builder.
|
||||
htmlhelp_basename = 'saharavanillaplugin-testsdoc'
|
||||
|
||||
|
||||
# -- Options for LaTeX output --------------------------------------------------
|
||||
|
||||
# Grouping the document tree into LaTeX files. List of tuples
|
||||
# (source start file, target name, title, author, documentclass [howto/manual]).
|
||||
latex_documents = [
|
||||
('index', 'doc-sahara-plugin-vanilla.tex', u'Sahara Vanilla Plugin Documentation',
|
||||
u'Sahara team', 'manual'),
|
||||
]
|
||||
|
||||
# The name of an image file (relative to this directory) to place at the top of
|
||||
# the title page.
|
||||
#latex_logo = None
|
||||
|
||||
# For "manual" documents, if this is true, then toplevel headings are parts,
|
||||
# not chapters.
|
||||
#latex_use_parts = False
|
||||
|
||||
# If true, show page references after internal links.
|
||||
#latex_show_pagerefs = False
|
||||
|
||||
# If true, show URL addresses after external links.
|
||||
#latex_show_urls = False
|
||||
|
||||
# Documents to append as an appendix to all manuals.
|
||||
#latex_appendices = []
|
||||
|
||||
# If false, no module index is generated.
|
||||
#latex_domain_indices = True
|
||||
|
||||
smartquotes_excludes = {'builders': ['latex']}
|
||||
|
||||
# -- Options for manual page output --------------------------------------------
|
||||
|
||||
# One entry per manual page. List of tuples
|
||||
# (source start file, name, description, authors, manual section).
|
||||
man_pages = [
|
||||
('index', 'sahara-plugin-vanilla', u'sahara-plugin-vanilla Documentation',
|
||||
[u'Sahara team'], 1)
|
||||
]
|
||||
|
||||
# If true, show URL addresses after external links.
|
||||
#man_show_urls = False
|
||||
|
||||
|
||||
# -- Options for Texinfo output ------------------------------------------------
|
||||
|
||||
# Grouping the document tree into Texinfo files. List of tuples
|
||||
# (source start file, target name, title, author,
|
||||
# dir menu entry, description, category)
|
||||
texinfo_documents = [
|
||||
('index', 'sahara-plugin-vanilla', u'sahara-plugin-vanilla Documentation',
|
||||
u'Sahara team', 'sahara-plugin-vanilla', 'One line description of project.',
|
||||
'Miscellaneous'),
|
||||
]
|
||||
|
||||
# Documents to append as an appendix to all manuals.
|
||||
#texinfo_appendices = []
|
||||
|
||||
# If false, no module index is generated.
|
||||
#texinfo_domain_indices = True
|
||||
|
||||
# How to display URL addresses: 'footnote', 'no', or 'inline'.
|
||||
#texinfo_show_urls = 'footnote'
|
@ -1,14 +0,0 @@
|
||||
============================
|
||||
So You Want to Contribute...
|
||||
============================
|
||||
|
||||
For general information on contributing to OpenStack, please check out the
|
||||
`contributor guide <https://docs.openstack.org/contributors/>`_ to get started.
|
||||
It covers all the basics that are common to all OpenStack projects: the
|
||||
accounts you need, the basics of interacting with our Gerrit review system, how
|
||||
we communicate as a community, etc.
|
||||
|
||||
sahara-plugin-vanilla is maintained by the OpenStack Sahara project.
|
||||
To understand our development process and how you can contribute to it, please
|
||||
look at the Sahara project's general contributor's page:
|
||||
http://docs.openstack.org/sahara/latest/contributor/contributing.html
|
@ -1,8 +0,0 @@
|
||||
=================
|
||||
Contributor Guide
|
||||
=================
|
||||
|
||||
.. toctree::
|
||||
:maxdepth: 2
|
||||
|
||||
contributing
|
@ -1,8 +0,0 @@
|
||||
Vanilla plugin for Sahara
|
||||
=========================
|
||||
|
||||
.. toctree::
|
||||
:maxdepth: 2
|
||||
|
||||
user/index
|
||||
contributor/index
|
@ -1,9 +0,0 @@
|
||||
==========
|
||||
User Guide
|
||||
==========
|
||||
|
||||
.. toctree::
|
||||
:maxdepth: 2
|
||||
|
||||
vanilla-plugin
|
||||
|
@ -1,114 +0,0 @@
|
||||
Vanilla Plugin
|
||||
==============
|
||||
|
||||
The vanilla plugin is a reference implementation which allows users to operate
|
||||
a cluster with Apache Hadoop.
|
||||
|
||||
Since the Newton release Spark is integrated into the Vanilla plugin so you
|
||||
can launch Spark jobs on a Vanilla cluster.
|
||||
|
||||
Images
|
||||
------
|
||||
|
||||
For cluster provisioning, prepared images should be used.
|
||||
|
||||
.. list-table:: Support matrix for the `vanilla` plugin
|
||||
:widths: 15 15 20 15 35
|
||||
:header-rows: 1
|
||||
|
||||
* - Version
|
||||
(image tag)
|
||||
- Distribution
|
||||
- Build method
|
||||
- Version
|
||||
(build parameter)
|
||||
- Notes
|
||||
|
||||
* - 2.8.2
|
||||
- Ubuntu 16.04, CentOS 7
|
||||
- sahara-image-create
|
||||
- 2.8.2
|
||||
- Hive 2.3.2, Oozie 4.3.0
|
||||
|
||||
* - 2.7.5
|
||||
- Ubuntu 16.04, CentOS 7
|
||||
- sahara-image-create
|
||||
- 2.7.5
|
||||
- Hive 2.3.2, Oozie 4.3.0
|
||||
|
||||
* - 2.7.1
|
||||
- Ubuntu 16.04, CentOS 7
|
||||
- sahara-image-create
|
||||
- 2.7.1
|
||||
- Hive 0.11.0, Oozie 4.2.0
|
||||
|
||||
For more information about building image, refer to
|
||||
:sahara-doc:`Sahara documentation <user/building-guest-images.html>`.
|
||||
|
||||
Vanilla plugin requires an image to be tagged in Sahara Image Registry with
|
||||
two tags: 'vanilla' and '<hadoop version>' (e.g. '2.7.1').
|
||||
|
||||
The image requires a username. For more information, refer to the
|
||||
:sahara-doc:`registering image <user/registering-image.html>` section
|
||||
of the Sahara documentation.
|
||||
|
||||
Build settings
|
||||
~~~~~~~~~~~~~~
|
||||
|
||||
When ``sahara-image-create`` is used, you can override few settings
|
||||
by exporting the corresponding environment variables
|
||||
before starting the build command:
|
||||
|
||||
* ``DIB_HADOOP_VERSION`` - version of Hadoop to install
|
||||
* ``HIVE_VERSION`` - version of Hive to install
|
||||
* ``OOZIE_DOWNLOAD_URL`` - download link for Oozie (we have built
|
||||
Oozie libs here: https://tarballs.openstack.org/sahara-extra/dist/oozie/)
|
||||
* ``SPARK_DOWNLOAD_URL`` - download link for Spark
|
||||
|
||||
Vanilla Plugin Requirements
|
||||
---------------------------
|
||||
|
||||
The image building tools described in
|
||||
:sahara-doc:`Building guest images <user/building-guest-images.html>`
|
||||
add the required software to the image and their usage is strongly suggested.
|
||||
Nevertheless, here are listed the software that should be pre-loaded
|
||||
on the guest image so that it can be used to create Vanilla clusters:
|
||||
|
||||
* ssh-client installed
|
||||
* Java (version >= 7)
|
||||
* Apache Hadoop installed
|
||||
* 'hadoop' user created
|
||||
|
||||
See :sahara-doc:`Swift Integration <user/hadoop-swift.html>` for information
|
||||
on using Swift with your sahara cluster (for EDP support Swift integration
|
||||
is currently required).
|
||||
|
||||
To support EDP, the following components must also be installed on the guest:
|
||||
|
||||
* Oozie version 4 or higher
|
||||
* mysql/mariadb
|
||||
* hive
|
||||
|
||||
Cluster Validation
|
||||
------------------
|
||||
|
||||
When user creates or scales a Hadoop cluster using a Vanilla plugin,
|
||||
the cluster topology requested by user is verified for consistency.
|
||||
|
||||
Currently there are the following limitations in cluster topology for Vanilla
|
||||
plugin:
|
||||
|
||||
For Vanilla Hadoop version 2.x.x:
|
||||
|
||||
+ Cluster must contain exactly one namenode
|
||||
+ Cluster can contain at most one resourcemanager
|
||||
+ Cluster can contain at most one secondary namenode
|
||||
+ Cluster can contain at most one historyserver
|
||||
+ Cluster can contain at most one oozie and this process is also required
|
||||
for EDP
|
||||
+ Cluster can't contain oozie without resourcemanager and without
|
||||
historyserver
|
||||
+ Cluster can't have nodemanager nodes if it doesn't have resourcemanager
|
||||
+ Cluster can have at most one hiveserver node.
|
||||
+ Cluster can have at most one spark history server and this process is also
|
||||
required for Spark EDP (Spark is available since the Newton release).
|
34
errors.txt
34
errors.txt
@ -1,34 +0,0 @@
|
||||
py27 develop-inst-nodeps: /home/tenobreg/coding/upstream/sahara/sahara
|
||||
py27 installed: alabaster==0.7.11,alembic==1.0.0,amqp==2.3.2,appdirs==1.4.3,asn1crypto==0.24.0,astroid==1.3.8,Babel==2.6.0,bandit==1.5.0,bashate==0.6.0,bcrypt==3.1.4,botocore==1.10.62,cachetools==2.1.0,castellan==0.18.0,certifi==2018.4.16,cffi==1.11.5,chardet==3.0.4,click==6.7,cliff==2.13.0,cmd2==0.8.8,contextlib2==0.5.5,coverage==4.5.1,cryptography==2.3,debtcollector==1.20.0,decorator==4.3.0,deprecation==2.0.5,doc8==0.8.0,docutils==0.14,dogpile.cache==0.6.6,dulwich==0.19.5,enum-compat==0.0.2,enum34==1.1.6,eventlet==0.20.0,extras==1.0.0,fasteners==0.14.1,fixtures==3.0.0,flake8==2.5.5,Flask==1.0.2,funcsigs==1.0.2,functools32==3.2.3.post2,future==0.16.0,futures==3.2.0,futurist==1.7.0,gitdb2==2.0.4,GitPython==2.1.11,greenlet==0.4.13,hacking==0.12.0,idna==2.7,imagesize==1.0.0,ipaddress==1.0.22,iso8601==0.1.12,itsdangerous==0.24,Jinja2==2.10,jmespath==0.9.3,jsonpatch==1.23,jsonpointer==2.0,jsonschema==2.6.0,keystoneauth1==3.10.0,keystonemiddleware==5.2.0,kombu==4.2.1,linecache2==1.0.0,logilab-common==1.4.2,Mako==1.0.7,MarkupSafe==1.0,mccabe==0.2.1,mock==2.0.0,monotonic==1.5,mox3==0.26.0,msgpack==0.5.6,munch==2.3.2,netaddr==0.7.19,netifaces==0.10.7,openstackdocstheme==1.22.0,openstacksdk==0.17.2,os-api-ref==1.5.0,os-client-config==1.31.2,os-service-types==1.3.0,os-testr==1.0.0,osc-lib==1.11.1,oslo.cache==1.30.1,oslo.concurrency==3.27.0,oslo.config==6.4.0,oslo.context==2.21.0,oslo.db==4.40.0,oslo.i18n==3.21.0,oslo.log==3.39.0,oslo.messaging==8.1.0,oslo.middleware==3.36.0,oslo.policy==1.38.1,oslo.rootwrap==5.14.1,oslo.serialization==2.27.0,oslo.service==1.31.3,oslo.utils==3.36.4,oslotest==3.6.0,packaging==17.1,paramiko==2.4.1,Paste==2.0.3,PasteDeploy==1.5.2,pbr==4.2.0,pep8==1.5.7,prettytable==0.7.2,psycopg2==2.7.5,pyasn1==0.4.3,pycadf==2.8.0,pycparser==2.18,pyflakes==0.8.1,Pygments==2.2.0,pyinotify==0.9.6,pylint==1.4.5,PyMySQL==0.9.2,PyNaCl==1.2.1,pyOpenSSL==18.0.0,pyparsing==2.2.0,pyperclip==1.6.4,python-barbicanclient==4.7.0,python-cinderclient==4.0.1,python-dateutil==2.7.3,python-editor==1.0.3,python-glanceclient==2.12.1,python-heatclient==1.16.1,python-keystoneclient==3.17.0,python-manilaclient==1.24.1,python-mimeparse==1.6.0,python-neutronclient==6.9.0,python-novaclient==11.0.0,python-openstackclient==3.16.0,python-saharaclient==2.0.0,python-subunit==1.3.0,python-swiftclient==3.6.0,pytz==2018.5,PyYAML==3.13,reno==2.9.2,repoze.lru==0.7,requests==2.19.1,requestsexceptions==1.4.0,restructuredtext-lint==1.1.3,rfc3986==1.1.0,Routes==2.4.1,-e git+https://github.com/openstack/sahara.git@efb05b3624044f307168d0b5da888132f51aebb7#egg=sahara,simplejson==3.16.0,six==1.11.0,smmap2==2.0.4,snowballstemmer==1.2.1,Sphinx==1.7.6,sphinxcontrib-httpdomain==1.7.0,sphinxcontrib-websupport==1.1.0,SQLAlchemy==1.2.10,sqlalchemy-migrate==0.11.0,sqlparse==0.2.4,statsd==3.2.2,stestr==2.1.0,stevedore==1.29.0,subprocess32==3.5.2,Tempita==0.5.2,tenacity==4.12.0,testresources==2.0.1,testscenarios==0.5.0,testtools==2.3.0,tooz==1.62.0,traceback2==1.4.0,typing==3.6.4,unicodecsv==0.14.1,unittest2==1.1.0,urllib3==1.23,vine==1.1.4,voluptuous==0.11.1,warlock==1.3.0,wcwidth==0.1.7,WebOb==1.8.2,Werkzeug==0.14.1,wrapt==1.10.11
|
||||
py27 runtests: PYTHONHASHSEED='839100177'
|
||||
py27 runtests: commands[0] | ostestr
|
||||
|
||||
=========================
|
||||
Failures during discovery
|
||||
=========================
|
||||
--- import errors ---
|
||||
Failed to import test module: sahara.tests.unit.service.edp.spark.test_shell
|
||||
Traceback (most recent call last):
|
||||
File "/home/tenobreg/coding/upstream/sahara/sahara/.tox/py27/lib/python2.7/site-packages/unittest2/loader.py", line 456, in _find_test_path
|
||||
module = self._get_module_from_name(name)
|
||||
File "/home/tenobreg/coding/upstream/sahara/sahara/.tox/py27/lib/python2.7/site-packages/unittest2/loader.py", line 395, in _get_module_from_name
|
||||
__import__(name)
|
||||
File "sahara/tests/unit/service/edp/spark/test_shell.py", line 18, in <module>
|
||||
from sahara.plugins.spark import shell_engine
|
||||
ImportError: No module named spark
|
||||
|
||||
Failed to import test module: sahara.tests.unit.service.edp.spark.test_spark
|
||||
Traceback (most recent call last):
|
||||
File "/home/tenobreg/coding/upstream/sahara/sahara/.tox/py27/lib/python2.7/site-packages/unittest2/loader.py", line 456, in _find_test_path
|
||||
module = self._get_module_from_name(name)
|
||||
File "/home/tenobreg/coding/upstream/sahara/sahara/.tox/py27/lib/python2.7/site-packages/unittest2/loader.py", line 395, in _get_module_from_name
|
||||
__import__(name)
|
||||
File "sahara/tests/unit/service/edp/spark/test_spark.py", line 17, in <module>
|
||||
from sahara.plugins.spark import edp_engine as spark_edp
|
||||
ImportError: No module named spark
|
||||
|
||||
================================================================================
|
||||
The above traceback was encountered during test discovery which imports all the found test modules in the specified test_path.
|
||||
ERROR: InvocationError: '/home/tenobreg/coding/upstream/sahara/sahara/.tox/py27/bin/ostestr'
|
||||
___________________________________ summary ____________________________________
|
||||
ERROR: py27: commands failed
|
@ -1,6 +0,0 @@
|
||||
---
|
||||
upgrade:
|
||||
- |
|
||||
Python 2.7 support has been dropped. Last release of sahara and its plugins
|
||||
to support python 2.7 is OpenStack Train. The minimum version of Python now
|
||||
supported by sahara and its plugins is Python 3.6.
|
@ -1,210 +0,0 @@
|
||||
# -*- coding: utf-8 -*-
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
|
||||
# implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
# Sahara Release Notes documentation build configuration file
|
||||
|
||||
extensions = [
|
||||
'reno.sphinxext',
|
||||
'openstackdocstheme'
|
||||
]
|
||||
|
||||
# openstackdocstheme options
|
||||
openstackdocs_repo_name = 'openstack/sahara-plugin-vanilla'
|
||||
openstackdocs_use_storyboard = True
|
||||
|
||||
# Add any paths that contain templates here, relative to this directory.
|
||||
templates_path = ['_templates']
|
||||
|
||||
# The suffix of source filenames.
|
||||
source_suffix = '.rst'
|
||||
|
||||
# The master toctree document.
|
||||
master_doc = 'index'
|
||||
|
||||
# General information about the project.
|
||||
copyright = u'2015, Sahara Developers'
|
||||
|
||||
# Release do not need a version number in the title, they
|
||||
# cover multiple versions.
|
||||
# The full version, including alpha/beta/rc tags.
|
||||
release = ''
|
||||
# The short X.Y version.
|
||||
version = ''
|
||||
|
||||
# List of patterns, relative to source directory, that match files and
|
||||
# directories to ignore when looking for source files.
|
||||
exclude_patterns = []
|
||||
|
||||
# The name of the Pygments (syntax highlighting) style to use.
|
||||
pygments_style = 'native'
|
||||
|
||||
|
||||
# -- Options for HTML output ----------------------------------------------
|
||||
|
||||
# The theme to use for HTML and HTML Help pages. See the documentation for
|
||||
# a list of builtin themes.
|
||||
html_theme = 'openstackdocs'
|
||||
|
||||
# Theme options are theme-specific and customize the look and feel of a theme
|
||||
# further. For a list of options available for each theme, see the
|
||||
# documentation.
|
||||
# html_theme_options = {}
|
||||
|
||||
# Add any paths that contain custom themes here, relative to this directory.
|
||||
# html_theme_path = []
|
||||
|
||||
# The name for this set of Sphinx documents. If None, it defaults to
|
||||
# "<project> v<release> documentation".
|
||||
# html_title = None
|
||||
|
||||
# A shorter title for the navigation bar. Default is the same as html_title.
|
||||
# html_short_title = None
|
||||
|
||||
# The name of an image file (relative to this directory) to place at the top
|
||||
# of the sidebar.
|
||||
# html_logo = None
|
||||
|
||||
# The name of an image file (within the static path) to use as favicon of the
|
||||
# docs. This file should be a Windows icon file (.ico) being 16x16 or 32x32
|
||||
# pixels large.
|
||||
# html_favicon = None
|
||||
|
||||
# Add any paths that contain custom static files (such as style sheets) here,
|
||||
# relative to this directory. They are copied after the builtin static files,
|
||||
# so a file named "default.css" will overwrite the builtin "default.css".
|
||||
html_static_path = ['_static']
|
||||
|
||||
# Add any extra paths that contain custom files (such as robots.txt or
|
||||
# .htaccess) here, relative to this directory. These files are copied
|
||||
# directly to the root of the documentation.
|
||||
# html_extra_path = []
|
||||
|
||||
# If not '', a 'Last updated on:' timestamp is inserted at every page bottom,
|
||||
# using the given strftime format.
|
||||
# html_last_updated_fmt = '%b %d, %Y'
|
||||
|
||||
# If true, SmartyPants will be used to convert quotes and dashes to
|
||||
# typographically correct entities.
|
||||
# html_use_smartypants = True
|
||||
|
||||
# Custom sidebar templates, maps document names to template names.
|
||||
# html_sidebars = {}
|
||||
|
||||
# Additional templates that should be rendered to pages, maps page names to
|
||||
# template names.
|
||||
# html_additional_pages = {}
|
||||
|
||||
# If false, no module index is generated.
|
||||
# html_domain_indices = True
|
||||
|
||||
# If false, no index is generated.
|
||||
# html_use_index = True
|
||||
|
||||
# If true, the index is split into individual pages for each letter.
|
||||
# html_split_index = False
|
||||
|
||||
# If true, links to the reST sources are added to the pages.
|
||||
# html_show_sourcelink = True
|
||||
|
||||
# If true, "Created using Sphinx" is shown in the HTML footer. Default is True.
|
||||
# html_show_sphinx = True
|
||||
|
||||
# If true, "(C) Copyright ..." is shown in the HTML footer. Default is True.
|
||||
# html_show_copyright = True
|
||||
|
||||
# If true, an OpenSearch description file will be output, and all pages will
|
||||
# contain a <link> tag referring to it. The value of this option must be the
|
||||
# base URL from which the finished HTML is served.
|
||||
# html_use_opensearch = ''
|
||||
|
||||
# This is the file name suffix for HTML files (e.g. ".xhtml").
|
||||
# html_file_suffix = None
|
||||
|
||||
# Output file base name for HTML help builder.
|
||||
htmlhelp_basename = 'SaharaVanillaReleaseNotesdoc'
|
||||
|
||||
|
||||
# -- Options for LaTeX output ---------------------------------------------
|
||||
|
||||
# Grouping the document tree into LaTeX files. List of tuples
|
||||
# (source start file, target name, title,
|
||||
# author, documentclass [howto, manual, or own class]).
|
||||
latex_documents = [
|
||||
('index', 'SaharaVanillaReleaseNotes.tex',
|
||||
u'Sahara Vanilla Plugin Release Notes Documentation',
|
||||
u'Sahara Developers', 'manual'),
|
||||
]
|
||||
|
||||
# The name of an image file (relative to this directory) to place at the top of
|
||||
# the title page.
|
||||
# latex_logo = None
|
||||
|
||||
# For "manual" documents, if this is true, then toplevel headings are parts,
|
||||
# not chapters.
|
||||
# latex_use_parts = False
|
||||
|
||||
# If true, show page references after internal links.
|
||||
# latex_show_pagerefs = False
|
||||
|
||||
# If true, show URL addresses after external links.
|
||||
# latex_show_urls = False
|
||||
|
||||
# Documents to append as an appendix to all manuals.
|
||||
# latex_appendices = []
|
||||
|
||||
# If false, no module index is generated.
|
||||
# latex_domain_indices = True
|
||||
|
||||
|
||||
# -- Options for manual page output ---------------------------------------
|
||||
|
||||
# One entry per manual page. List of tuples
|
||||
# (source start file, name, description, authors, manual section).
|
||||
man_pages = [
|
||||
('index', 'saharavanillareleasenotes',
|
||||
u'Sahara Vanilla Plugin Release Notes Documentation',
|
||||
[u'Sahara Developers'], 1)
|
||||
]
|
||||
|
||||
# If true, show URL addresses after external links.
|
||||
# man_show_urls = False
|
||||
|
||||
|
||||
# -- Options for Texinfo output -------------------------------------------
|
||||
|
||||
# Grouping the document tree into Texinfo files. List of tuples
|
||||
# (source start file, target name, title, author,
|
||||
# dir menu entry, description, category)
|
||||
texinfo_documents = [
|
||||
('index', 'SaharaVanillaReleaseNotes',
|
||||
u'Sahara Vanilla Plugin Release Notes Documentation',
|
||||
u'Sahara Developers', 'SaharaVanillaReleaseNotes',
|
||||
'One line description of project.',
|
||||
'Miscellaneous'),
|
||||
]
|
||||
|
||||
# Documents to append as an appendix to all manuals.
|
||||
# texinfo_appendices = []
|
||||
|
||||
# If false, no module index is generated.
|
||||
# texinfo_domain_indices = True
|
||||
|
||||
# How to display URL addresses: 'footnote', 'no', or 'inline'.
|
||||
# texinfo_show_urls = 'footnote'
|
||||
|
||||
# If true, do not generate a @detailmenu in the "Top" node's menu.
|
||||
# texinfo_no_detailmenu = False
|
||||
|
||||
# -- Options for Internationalization output ------------------------------
|
||||
locale_dirs = ['locale/']
|
@ -1,15 +0,0 @@
|
||||
=====================================
|
||||
Sahara Vanilla Plugin Release Notes
|
||||
=====================================
|
||||
|
||||
.. toctree::
|
||||
:maxdepth: 1
|
||||
|
||||
unreleased
|
||||
yoga
|
||||
xena
|
||||
wallaby
|
||||
victoria
|
||||
ussuri
|
||||
train
|
||||
stein
|
@ -1,44 +0,0 @@
|
||||
# Andreas Jaeger <jaegerandi@gmail.com>, 2019. #zanata
|
||||
# Andreas Jaeger <jaegerandi@gmail.com>, 2020. #zanata
|
||||
msgid ""
|
||||
msgstr ""
|
||||
"Project-Id-Version: sahara-plugin-vanilla\n"
|
||||
"Report-Msgid-Bugs-To: \n"
|
||||
"POT-Creation-Date: 2020-04-24 23:47+0000\n"
|
||||
"MIME-Version: 1.0\n"
|
||||
"Content-Type: text/plain; charset=UTF-8\n"
|
||||
"Content-Transfer-Encoding: 8bit\n"
|
||||
"PO-Revision-Date: 2020-04-25 10:38+0000\n"
|
||||
"Last-Translator: Andreas Jaeger <jaegerandi@gmail.com>\n"
|
||||
"Language-Team: German\n"
|
||||
"Language: de\n"
|
||||
"X-Generator: Zanata 4.3.3\n"
|
||||
"Plural-Forms: nplurals=2; plural=(n != 1)\n"
|
||||
|
||||
msgid "Current Series Release Notes"
|
||||
msgstr "Aktuelle Serie Releasenotes"
|
||||
|
||||
msgid ""
|
||||
"Python 2.7 support has been dropped. Last release of sahara and its plugins "
|
||||
"to support python 2.7 is OpenStack Train. The minimum version of Python now "
|
||||
"supported by sahara and its plugins is Python 3.6."
|
||||
msgstr ""
|
||||
"Python 2.7 Unterstützung wurde beendet. Der letzte Release von Sahara und "
|
||||
"seinen Plugins der Python 2.7 unterstützt ist OpenStack Train. Die minimal "
|
||||
"Python Version welche von Sahara und seinen Plugins unterstützt wird, ist "
|
||||
"Python 3.6."
|
||||
|
||||
msgid "Sahara Vanilla Plugin Release Notes"
|
||||
msgstr "Sahara Vanilla Plugin Releasenotes"
|
||||
|
||||
msgid "Stein Series Release Notes"
|
||||
msgstr "Stein Serie Releasenotes"
|
||||
|
||||
msgid "Train Series Release Notes"
|
||||
msgstr "Train Serie Releasenotes"
|
||||
|
||||
msgid "Upgrade Notes"
|
||||
msgstr "Aktualisierungsnotizen"
|
||||
|
||||
msgid "Ussuri Series Release Notes"
|
||||
msgstr "Ussuri Serie Releasenotes"
|
@ -1,48 +0,0 @@
|
||||
# Andi Chandler <andi@gowling.com>, 2020. #zanata
|
||||
msgid ""
|
||||
msgstr ""
|
||||
"Project-Id-Version: sahara-plugin-vanilla\n"
|
||||
"Report-Msgid-Bugs-To: \n"
|
||||
"POT-Creation-Date: 2020-10-07 22:09+0000\n"
|
||||
"MIME-Version: 1.0\n"
|
||||
"Content-Type: text/plain; charset=UTF-8\n"
|
||||
"Content-Transfer-Encoding: 8bit\n"
|
||||
"PO-Revision-Date: 2020-11-03 10:20+0000\n"
|
||||
"Last-Translator: Andi Chandler <andi@gowling.com>\n"
|
||||
"Language-Team: English (United Kingdom)\n"
|
||||
"Language: en_GB\n"
|
||||
"X-Generator: Zanata 4.3.3\n"
|
||||
"Plural-Forms: nplurals=2; plural=(n != 1)\n"
|
||||
|
||||
msgid "3.0.0"
|
||||
msgstr "3.0.0"
|
||||
|
||||
msgid "Current Series Release Notes"
|
||||
msgstr "Current Series Release Notes"
|
||||
|
||||
msgid ""
|
||||
"Python 2.7 support has been dropped. Last release of sahara and its plugins "
|
||||
"to support python 2.7 is OpenStack Train. The minimum version of Python now "
|
||||
"supported by sahara and its plugins is Python 3.6."
|
||||
msgstr ""
|
||||
"Python 2.7 support has been dropped. Last release of sahara and its plugins "
|
||||
"to support python 2.7 is OpenStack Train. The minimum version of Python now "
|
||||
"supported by Sahara and its plugins is Python 3.6."
|
||||
|
||||
msgid "Sahara Vanilla Plugin Release Notes"
|
||||
msgstr "Sahara Vanilla Plugin Release Notes"
|
||||
|
||||
msgid "Stein Series Release Notes"
|
||||
msgstr "Stein Series Release Notes"
|
||||
|
||||
msgid "Train Series Release Notes"
|
||||
msgstr "Train Series Release Notes"
|
||||
|
||||
msgid "Upgrade Notes"
|
||||
msgstr "Upgrade Notes"
|
||||
|
||||
msgid "Ussuri Series Release Notes"
|
||||
msgstr "Ussuri Series Release Notes"
|
||||
|
||||
msgid "Victoria Series Release Notes"
|
||||
msgstr "Victoria Series Release Notes"
|
@ -1,24 +0,0 @@
|
||||
# Surit Aryal <aryalsurit@gmail.com>, 2019. #zanata
|
||||
msgid ""
|
||||
msgstr ""
|
||||
"Project-Id-Version: sahara-plugin-vanilla\n"
|
||||
"Report-Msgid-Bugs-To: \n"
|
||||
"POT-Creation-Date: 2019-07-23 14:28+0000\n"
|
||||
"MIME-Version: 1.0\n"
|
||||
"Content-Type: text/plain; charset=UTF-8\n"
|
||||
"Content-Transfer-Encoding: 8bit\n"
|
||||
"PO-Revision-Date: 2019-08-02 08:43+0000\n"
|
||||
"Last-Translator: Surit Aryal <aryalsurit@gmail.com>\n"
|
||||
"Language-Team: Nepali\n"
|
||||
"Language: ne\n"
|
||||
"X-Generator: Zanata 4.3.3\n"
|
||||
"Plural-Forms: nplurals=2; plural=(n != 1)\n"
|
||||
|
||||
msgid "Current Series Release Notes"
|
||||
msgstr "Current Series रिलीज नोट्स"
|
||||
|
||||
msgid "Sahara Vanilla Plugin Release Notes"
|
||||
msgstr "Sahara Vanilla प्लगइन रिलीज नोट्स"
|
||||
|
||||
msgid "Stein Series Release Notes"
|
||||
msgstr "Stein Series रिलीज नोट्स"
|
@ -1,6 +0,0 @@
|
||||
===================================
|
||||
Stein Series Release Notes
|
||||
===================================
|
||||
|
||||
.. release-notes::
|
||||
:branch: stable/stein
|
@ -1,6 +0,0 @@
|
||||
==========================
|
||||
Train Series Release Notes
|
||||
==========================
|
||||
|
||||
.. release-notes::
|
||||
:branch: stable/train
|
@ -1,5 +0,0 @@
|
||||
==============================
|
||||
Current Series Release Notes
|
||||
==============================
|
||||
|
||||
.. release-notes::
|
@ -1,6 +0,0 @@
|
||||
===========================
|
||||
Ussuri Series Release Notes
|
||||
===========================
|
||||
|
||||
.. release-notes::
|
||||
:branch: stable/ussuri
|
@ -1,6 +0,0 @@
|
||||
=============================
|
||||
Victoria Series Release Notes
|
||||
=============================
|
||||
|
||||
.. release-notes::
|
||||
:branch: stable/victoria
|
@ -1,6 +0,0 @@
|
||||
============================
|
||||
Wallaby Series Release Notes
|
||||
============================
|
||||
|
||||
.. release-notes::
|
||||
:branch: stable/wallaby
|
@ -1,6 +0,0 @@
|
||||
=========================
|
||||
Xena Series Release Notes
|
||||
=========================
|
||||
|
||||
.. release-notes::
|
||||
:branch: stable/xena
|
@ -1,6 +0,0 @@
|
||||
=========================
|
||||
Yoga Series Release Notes
|
||||
=========================
|
||||
|
||||
.. release-notes::
|
||||
:branch: stable/yoga
|
@ -1,15 +0,0 @@
|
||||
# The order of packages is significant, because pip processes them in the order
|
||||
# of appearance. Changing the order has an impact on the overall integration
|
||||
# process, which may cause wedges in the gate later.
|
||||
|
||||
pbr!=2.1.0,>=2.0.0 # Apache-2.0
|
||||
|
||||
Babel!=2.4.0,>=2.3.4 # BSD
|
||||
eventlet>=0.26.0 # MIT
|
||||
oslo.i18n>=3.15.3 # Apache-2.0
|
||||
oslo.log>=3.36.0 # Apache-2.0
|
||||
oslo.serialization!=2.19.1,>=2.18.0 # Apache-2.0
|
||||
oslo.utils>=3.33.0 # Apache-2.0
|
||||
requests>=2.14.2 # Apache-2.0
|
||||
sahara>=10.0.0.0b1
|
||||
six>=1.10.0 # MIT
|
@ -1,26 +0,0 @@
|
||||
# Copyright (c) 2014 Mirantis Inc.
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
|
||||
# implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
# It's based on oslo.i18n usage in OpenStack Keystone project and
|
||||
# recommendations from https://docs.openstack.org/oslo.i18n/latest/
|
||||
# user/usage.html
|
||||
|
||||
import oslo_i18n
|
||||
|
||||
|
||||
_translators = oslo_i18n.TranslatorFactory(domain='sahara_plugin_vanilla')
|
||||
|
||||
# The primary translation function using the well-known name "_"
|
||||
_ = _translators.primary
|
@ -1,103 +0,0 @@
|
||||
# Andreas Jaeger <jaegerandi@gmail.com>, 2019. #zanata
|
||||
msgid ""
|
||||
msgstr ""
|
||||
"Project-Id-Version: sahara-plugin-vanilla VERSION\n"
|
||||
"Report-Msgid-Bugs-To: https://bugs.launchpad.net/openstack-i18n/\n"
|
||||
"POT-Creation-Date: 2019-09-27 11:37+0000\n"
|
||||
"MIME-Version: 1.0\n"
|
||||
"Content-Type: text/plain; charset=UTF-8\n"
|
||||
"Content-Transfer-Encoding: 8bit\n"
|
||||
"PO-Revision-Date: 2019-09-28 07:59+0000\n"
|
||||
"Last-Translator: Andreas Jaeger <jaegerandi@gmail.com>\n"
|
||||
"Language-Team: German\n"
|
||||
"Language: de\n"
|
||||
"X-Generator: Zanata 4.3.3\n"
|
||||
"Plural-Forms: nplurals=2; plural=(n != 1)\n"
|
||||
|
||||
msgid "0 or 1"
|
||||
msgstr "0 oder 1"
|
||||
|
||||
#, python-format
|
||||
msgid "Await %s start up"
|
||||
msgstr "Warte auf %s starte"
|
||||
|
||||
msgid "Configure instances"
|
||||
msgstr "Konfigurieren Sie Instanzen"
|
||||
|
||||
msgid "Configure topology data"
|
||||
msgstr "Konfigurieren Sie Topologiedaten"
|
||||
|
||||
#, python-format
|
||||
msgid "Decommission %s"
|
||||
msgstr "Außerkraftsetzung %s"
|
||||
|
||||
msgid "Number of datanodes must be not less than dfs.replication."
|
||||
msgstr "Die Anzahl der Daten muss nicht kleiner als dfs.replication sein."
|
||||
|
||||
msgid "Number of zookeeper nodes should be odd."
|
||||
msgstr "Anzahl der Zookeeper-Knoten sollte ungerade sein."
|
||||
|
||||
#, python-format
|
||||
msgid "Refresh %s nodes"
|
||||
msgstr "Aktualisiere die Knoten %s"
|
||||
|
||||
msgid "Spark {base} or higher required to run {type} jobs"
|
||||
msgstr "Spark {base} oder höher erforderlich, um {type} Jobs auszuführen"
|
||||
|
||||
msgid ""
|
||||
"The Apache Vanilla plugin provides the ability to launch upstream Vanilla "
|
||||
"Apache Hadoop cluster without any management consoles. It can also deploy "
|
||||
"the Oozie component."
|
||||
msgstr ""
|
||||
"Das Apache Vanilla-Plugin bietet die Möglichkeit, Upstream-Vanilla-Apache-"
|
||||
"Hadoop-Cluster ohne Verwaltungskonsolen zu starten. Es kann auch die Oozie-"
|
||||
"Komponente bereitstellen."
|
||||
|
||||
#, python-format
|
||||
msgid "Unable to get parameter '%(name)s' from service %(service)s"
|
||||
msgstr ""
|
||||
"Der Parameter '%(name)s' konnte nicht vom Service %(service)s abgerufen "
|
||||
"werden"
|
||||
|
||||
msgid "Update include files"
|
||||
msgstr "Update-Include-Dateien"
|
||||
|
||||
msgid ""
|
||||
"Vanilla plugin cannot scale cluster because it must keep zookeeper service "
|
||||
"in odd."
|
||||
msgstr ""
|
||||
"Das Vanilla-Plugin kann den Cluster nicht skalieren, da der Zoowäklerservice "
|
||||
"in Odd gehalten werden muss."
|
||||
|
||||
msgid ""
|
||||
"Vanilla plugin cannot scale node group with processes which have no master-"
|
||||
"processes run in cluster"
|
||||
msgstr ""
|
||||
"Das Vanilla-Plugin kann Knotengruppen nicht mit Prozessen skalieren, für die "
|
||||
"keine Masterprozesse im Cluster ausgeführt werden"
|
||||
|
||||
#, python-format
|
||||
msgid "Vanilla plugin cannot scale nodegroup with processes: %s"
|
||||
msgstr ""
|
||||
"Das Vanilla-Plugin kann Knotengruppen nicht mit Prozessen skalieren: %s"
|
||||
|
||||
#, python-format
|
||||
msgid ""
|
||||
"Vanilla plugin cannot shrink cluster because it would be not enough nodes "
|
||||
"for replicas (replication factor is %s)"
|
||||
msgstr ""
|
||||
"Das Vanilla-Plugin kann den Cluster nicht verkleinern, da es nicht genug "
|
||||
"Knoten für Replikate geben würde (der Replikationsfaktor ist %s)"
|
||||
|
||||
msgid "Wait for decommissioning"
|
||||
msgstr "Warten Sie auf die Außerbetriebnahme"
|
||||
|
||||
#, python-format
|
||||
msgid "Waiting on %s datanodes to start up"
|
||||
msgstr "Warten auf %s Daten, um zu starten"
|
||||
|
||||
msgid "Waiting on 1 datanodes to start up"
|
||||
msgstr "Warte auf 1 Daten Node zum Starten"
|
||||
|
||||
msgid "odd"
|
||||
msgstr "ungerade"
|
@ -1,100 +0,0 @@
|
||||
# Andi Chandler <andi@gowling.com>, 2020. #zanata
|
||||
msgid ""
|
||||
msgstr ""
|
||||
"Project-Id-Version: sahara-plugin-vanilla VERSION\n"
|
||||
"Report-Msgid-Bugs-To: https://bugs.launchpad.net/openstack-i18n/\n"
|
||||
"POT-Creation-Date: 2020-10-07 22:09+0000\n"
|
||||
"MIME-Version: 1.0\n"
|
||||
"Content-Type: text/plain; charset=UTF-8\n"
|
||||
"Content-Transfer-Encoding: 8bit\n"
|
||||
"PO-Revision-Date: 2020-11-03 10:20+0000\n"
|
||||
"Last-Translator: Andi Chandler <andi@gowling.com>\n"
|
||||
"Language-Team: English (United Kingdom)\n"
|
||||
"Language: en_GB\n"
|
||||
"X-Generator: Zanata 4.3.3\n"
|
||||
"Plural-Forms: nplurals=2; plural=(n != 1)\n"
|
||||
|
||||
msgid "0 or 1"
|
||||
msgstr "0 or 1"
|
||||
|
||||
#, python-format
|
||||
msgid "Await %s start up"
|
||||
msgstr "Await %s start up"
|
||||
|
||||
msgid "Configure instances"
|
||||
msgstr "Configure instances"
|
||||
|
||||
msgid "Configure topology data"
|
||||
msgstr "Configure topology data"
|
||||
|
||||
#, python-format
|
||||
msgid "Decommission %s"
|
||||
msgstr "Decommission %s"
|
||||
|
||||
msgid "Number of datanodes must be not less than dfs.replication."
|
||||
msgstr "Number of datanodes must be not less than dfs.replication."
|
||||
|
||||
msgid "Number of zookeeper nodes should be odd."
|
||||
msgstr "Number of Zookeeper nodes should be odd."
|
||||
|
||||
#, python-format
|
||||
msgid "Refresh %s nodes"
|
||||
msgstr "Refresh %s nodes"
|
||||
|
||||
msgid "Spark {base} or higher required to run {type} jobs"
|
||||
msgstr "Spark {base} or higher required to run {type} jobs"
|
||||
|
||||
msgid ""
|
||||
"The Apache Vanilla plugin provides the ability to launch upstream Vanilla "
|
||||
"Apache Hadoop cluster without any management consoles. It can also deploy "
|
||||
"the Oozie component."
|
||||
msgstr ""
|
||||
"The Apache Vanilla plugin provides the ability to launch upstream Vanilla "
|
||||
"Apache Hadoop cluster without any management consoles. It can also deploy "
|
||||
"the Oozie component."
|
||||
|
||||
#, python-format
|
||||
msgid "Unable to get parameter '%(name)s' from service %(service)s"
|
||||
msgstr "Unable to get parameter '%(name)s' from service %(service)s"
|
||||
|
||||
msgid "Update include files"
|
||||
msgstr "Update include files"
|
||||
|
||||
msgid ""
|
||||
"Vanilla plugin cannot scale cluster because it must keep zookeeper service "
|
||||
"in odd."
|
||||
msgstr ""
|
||||
"Vanilla plugin cannot scale cluster because it must keep Zookeeper service "
|
||||
"in odd."
|
||||
|
||||
msgid ""
|
||||
"Vanilla plugin cannot scale node group with processes which have no master-"
|
||||
"processes run in cluster"
|
||||
msgstr ""
|
||||
"Vanilla plugin cannot scale node group with processes which have no master-"
|
||||
"processes run in cluster"
|
||||
|
||||
#, python-format
|
||||
msgid "Vanilla plugin cannot scale nodegroup with processes: %s"
|
||||
msgstr "Vanilla plugin cannot scale nodegroup with processes: %s"
|
||||
|
||||
#, python-format
|
||||
msgid ""
|
||||
"Vanilla plugin cannot shrink cluster because it would be not enough nodes "
|
||||
"for replicas (replication factor is %s)"
|
||||
msgstr ""
|
||||
"Vanilla plugin cannot shrink cluster because it would be not enough nodes "
|
||||
"for replicas (replication factor is %s)"
|
||||
|
||||
msgid "Wait for decommissioning"
|
||||
msgstr "Wait for decommissioning"
|
||||
|
||||
#, python-format
|
||||
msgid "Waiting on %s datanodes to start up"
|
||||
msgstr "Waiting on %s datanodes to start up"
|
||||
|
||||
msgid "Waiting on 1 datanodes to start up"
|
||||
msgstr "Waiting on 1 datanodes to start up"
|
||||
|
||||
msgid "odd"
|
||||
msgstr "odd"
|
@ -1,101 +0,0 @@
|
||||
# suhartono <cloudsuhartono@gmail.com>, 2019. #zanata
|
||||
msgid ""
|
||||
msgstr ""
|
||||
"Project-Id-Version: sahara-plugin-vanilla VERSION\n"
|
||||
"Report-Msgid-Bugs-To: https://bugs.launchpad.net/openstack-i18n/\n"
|
||||
"POT-Creation-Date: 2019-09-30 09:36+0000\n"
|
||||
"MIME-Version: 1.0\n"
|
||||
"Content-Type: text/plain; charset=UTF-8\n"
|
||||
"Content-Transfer-Encoding: 8bit\n"
|
||||
"PO-Revision-Date: 2019-10-06 03:37+0000\n"
|
||||
"Last-Translator: suhartono <cloudsuhartono@gmail.com>\n"
|
||||
"Language-Team: Indonesian\n"
|
||||
"Language: id\n"
|
||||
"X-Generator: Zanata 4.3.3\n"
|
||||
"Plural-Forms: nplurals=1; plural=0\n"
|
||||
|
||||
msgid "0 or 1"
|
||||
msgstr "0 atau 1"
|
||||
|
||||
#, python-format
|
||||
msgid "Await %s start up"
|
||||
msgstr "Tunggu %s start up (mulai)"
|
||||
|
||||
msgid "Configure instances"
|
||||
msgstr "Konfigurasikan instance"
|
||||
|
||||
msgid "Configure topology data"
|
||||
msgstr "Konfigurasikan data topologi"
|
||||
|
||||
#, python-format
|
||||
msgid "Decommission %s"
|
||||
msgstr "Decommission %s"
|
||||
|
||||
msgid "Number of datanodes must be not less than dfs.replication."
|
||||
msgstr "Jumlah datanoda harus tidak kurang dari dfs.replication."
|
||||
|
||||
msgid "Number of zookeeper nodes should be odd."
|
||||
msgstr "Jumlah node zookeeper harus ganjil."
|
||||
|
||||
#, python-format
|
||||
msgid "Refresh %s nodes"
|
||||
msgstr "Refresh %s nodes"
|
||||
|
||||
msgid "Spark {base} or higher required to run {type} jobs"
|
||||
msgstr ""
|
||||
"Spark {base} atau lebih tinggi diperlukan untuk menjalankan jobs {type}"
|
||||
|
||||
msgid ""
|
||||
"The Apache Vanilla plugin provides the ability to launch upstream Vanilla "
|
||||
"Apache Hadoop cluster without any management consoles. It can also deploy "
|
||||
"the Oozie component."
|
||||
msgstr ""
|
||||
"Plugin Apache Vanilla menyediakan kemampuan untuk meluncurkan cluster "
|
||||
"Vanilla Apache Hadoop hulu tanpa konsol manajemen. Itu juga dapat "
|
||||
"menggunakan komponen Oozie."
|
||||
|
||||
#, python-format
|
||||
msgid "Unable to get parameter '%(name)s' from service %(service)s"
|
||||
msgstr "Tidak dapat memperoleh parameter '%(name)s' dari layanan %(service)s"
|
||||
|
||||
msgid "Update include files"
|
||||
msgstr "Perbarui menyertakan file"
|
||||
|
||||
msgid ""
|
||||
"Vanilla plugin cannot scale cluster because it must keep zookeeper service "
|
||||
"in odd."
|
||||
msgstr ""
|
||||
"Plugin vanilla tidak dapat mengukur cluster karena harus menjaga layanan "
|
||||
"zookeeper dalam keadaan odd."
|
||||
|
||||
msgid ""
|
||||
"Vanilla plugin cannot scale node group with processes which have no master-"
|
||||
"processes run in cluster"
|
||||
msgstr ""
|
||||
"Plugin vanilla tidak dapat menskala node group dengan proses yang tidak "
|
||||
"memiliki proses master berjalan di cluster"
|
||||
|
||||
#, python-format
|
||||
msgid "Vanilla plugin cannot scale nodegroup with processes: %s"
|
||||
msgstr "Plugin vanilla tidak dapat menskala nodegroup dengan proses: %s"
|
||||
|
||||
#, python-format
|
||||
msgid ""
|
||||
"Vanilla plugin cannot shrink cluster because it would be not enough nodes "
|
||||
"for replicas (replication factor is %s)"
|
||||
msgstr ""
|
||||
"Plugin vanilla tidak dapat mengecilkan cluster karena tidak akan cukup node "
|
||||
"untuk replika (faktor replikasi adalah %s)"
|
||||
|
||||
msgid "Wait for decommissioning"
|
||||
msgstr "Tunggu pembongkaran (decommissioning)"
|
||||
|
||||
#, python-format
|
||||
msgid "Waiting on %s datanodes to start up"
|
||||
msgstr "Menunggu datanodes %s untuk memulai"
|
||||
|
||||
msgid "Waiting on 1 datanodes to start up"
|
||||
msgstr "Menunggu 1 datanode untuk memulai"
|
||||
|
||||
msgid "odd"
|
||||
msgstr "ganjil"
|
@ -1,95 +0,0 @@
|
||||
# Surit Aryal <aryalsurit@gmail.com>, 2019. #zanata
|
||||
msgid ""
|
||||
msgstr ""
|
||||
"Project-Id-Version: sahara-plugin-vanilla VERSION\n"
|
||||
"Report-Msgid-Bugs-To: https://bugs.launchpad.net/openstack-i18n/\n"
|
||||
"POT-Creation-Date: 2019-09-27 11:37+0000\n"
|
||||
"MIME-Version: 1.0\n"
|
||||
"Content-Type: text/plain; charset=UTF-8\n"
|
||||
"Content-Transfer-Encoding: 8bit\n"
|
||||
"PO-Revision-Date: 2019-08-02 09:08+0000\n"
|
||||
"Last-Translator: Surit Aryal <aryalsurit@gmail.com>\n"
|
||||
"Language-Team: Nepali\n"
|
||||
"Language: ne\n"
|
||||
"X-Generator: Zanata 4.3.3\n"
|
||||
"Plural-Forms: nplurals=2; plural=(n != 1)\n"
|
||||
|
||||
msgid "0 or 1"
|
||||
msgstr "० वा १"
|
||||
|
||||
#, python-format
|
||||
msgid "Await %s start up"
|
||||
msgstr "पर्खिरहेका %s शुरू"
|
||||
|
||||
msgid "Configure instances"
|
||||
msgstr "instances कन्फिगर गर्नुहोस्"
|
||||
|
||||
msgid "Configure topology data"
|
||||
msgstr "टोपोलजी डाटा कन्फिगर गर्नुहोस्"
|
||||
|
||||
#, python-format
|
||||
msgid "Decommission %s"
|
||||
msgstr "Decommission %s"
|
||||
|
||||
msgid "Number of datanodes must be not less than dfs.replication."
|
||||
msgstr "datanodes संख्या dfs.replication भन्दा कम हुनुपर्छ।"
|
||||
|
||||
#, python-format
|
||||
msgid "Refresh %s nodes"
|
||||
msgstr "%s नोडहरू ताजा गर्नुहोस्"
|
||||
|
||||
msgid "Spark {base} or higher required to run {type} jobs"
|
||||
msgstr "Spark {base} or higher required to run {type} jobs"
|
||||
|
||||
msgid ""
|
||||
"The Apache Vanilla plugin provides the ability to launch upstream Vanilla "
|
||||
"Apache Hadoop cluster without any management consoles. It can also deploy "
|
||||
"the Oozie component."
|
||||
msgstr ""
|
||||
"Apache Vanilla प्लगइनले कुनै व्यवस्थापन कन्सोल बिना अपस्ट्रिम Vanilla Apache Hadoop "
|
||||
"क्लस्टर सुरूवात गर्न क्षमता प्रदान गर्दछ। यसले Oozie कम्पोनेन्ट पनि डिप्लोय गर्न सक्दछ।"
|
||||
|
||||
#, python-format
|
||||
msgid "Unable to get parameter '%(name)s' from service %(service)s"
|
||||
msgstr "सेवा %(service)s बाट प्यारामिटर '%(name)s' प्राप्त गर्न असमर्थ"
|
||||
|
||||
msgid "Update include files"
|
||||
msgstr "अपडेटमा फाईलहरू समावेश छन्"
|
||||
|
||||
msgid ""
|
||||
"Vanilla plugin cannot scale cluster because it must keep zookeeper service "
|
||||
"in odd."
|
||||
msgstr ""
|
||||
"Vanilla pluginले क्लस्टर मापन गर्न सक्दैन किनकि यसले जूटरकीपर सेवा बिजोर राख्नु पर्छ।"
|
||||
|
||||
msgid ""
|
||||
"Vanilla plugin cannot scale node group with processes which have no master-"
|
||||
"processes run in cluster"
|
||||
msgstr ""
|
||||
"Vanilla pluginले नोड समूहलाई प्रक्रियाहरूसँग मापन गर्न सक्दैन जुन मास्टर-प्रक्रियाहरू "
|
||||
"क्लस्टरमा चल्दैनन्"
|
||||
|
||||
#, python-format
|
||||
msgid "Vanilla plugin cannot scale nodegroup with processes: %s"
|
||||
msgstr "Vanilla plugin प्रक्रियासँग नोड ग्रुप मापन गर्न सक्दैन: %s"
|
||||
|
||||
#, python-format
|
||||
msgid ""
|
||||
"Vanilla plugin cannot shrink cluster because it would be not enough nodes "
|
||||
"for replicas (replication factor is %s)"
|
||||
msgstr ""
|
||||
"Vanilla plugin क्लस्टर सिको गर्न सक्दैन किनकि यो प्रतिकृतिको लागि पर्याप्त नोडहरू हुनेछैन "
|
||||
"(प्रतिकृति कारक %s हो)"
|
||||
|
||||
msgid "Wait for decommissioning"
|
||||
msgstr "डिकमिमिशनको लागि कुर्नुहोस्"
|
||||
|
||||
#, python-format
|
||||
msgid "Waiting on %s datanodes to start up"
|
||||
msgstr "सुरू गर्न %s डाटाटोड पर्खँदै"
|
||||
|
||||
msgid "Waiting on 1 datanodes to start up"
|
||||
msgstr "सुरू गर्न १ डेटानोडमा पर्खँदै"
|
||||
|
||||
msgid "odd"
|
||||
msgstr "बिजोर"
|
@ -1,71 +0,0 @@
|
||||
# Copyright (c) 2014 Mirantis, Inc.
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
|
||||
# implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
import abc
|
||||
|
||||
import six
|
||||
|
||||
|
||||
@six.add_metaclass(abc.ABCMeta)
|
||||
class AbstractVersionHandler(object):
|
||||
|
||||
@abc.abstractmethod
|
||||
def get_node_processes(self):
|
||||
return
|
||||
|
||||
@abc.abstractmethod
|
||||
def get_plugin_configs(self):
|
||||
return
|
||||
|
||||
@abc.abstractmethod
|
||||
def configure_cluster(self, cluster):
|
||||
return
|
||||
|
||||
@abc.abstractmethod
|
||||
def start_cluster(self, cluster):
|
||||
return
|
||||
|
||||
@abc.abstractmethod
|
||||
def validate(self, cluster):
|
||||
return
|
||||
|
||||
@abc.abstractmethod
|
||||
def scale_cluster(self, cluster, instances):
|
||||
return
|
||||
|
||||
@abc.abstractmethod
|
||||
def decommission_nodes(self, cluster, instances):
|
||||
return
|
||||
|
||||
@abc.abstractmethod
|
||||
def validate_scaling(self, cluster, existing, additional):
|
||||
return
|
||||
|
||||
@abc.abstractmethod
|
||||
def get_edp_engine(self, cluster, job_type):
|
||||
return
|
||||
|
||||
def get_edp_job_types(self):
|
||||
return []
|
||||
|
||||
def get_edp_config_hints(self, job_type):
|
||||
return {}
|
||||
|
||||
@abc.abstractmethod
|
||||
def get_open_ports(self, node_group):
|
||||
return
|
||||
|
||||
def on_terminate_cluster(self, cluster):
|
||||
pass
|
@ -1,47 +0,0 @@
|
||||
# Copyright (c) 2015 Red Hat, Inc.
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
|
||||
# implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
from sahara.plugins import edp
|
||||
from sahara.plugins import utils
|
||||
|
||||
|
||||
def get_possible_hive_config_from(file_name):
|
||||
'''Return the possible configs, args, params for a Hive job.'''
|
||||
config = {
|
||||
'configs': utils.load_hadoop_xml_defaults(file_name,
|
||||
'sahara_plugin_vanilla'),
|
||||
'params': {}
|
||||
}
|
||||
return config
|
||||
|
||||
|
||||
def get_possible_mapreduce_config_from(file_name):
|
||||
'''Return the possible configs, args, params for a MapReduce job.'''
|
||||
config = {
|
||||
'configs': get_possible_pig_config_from(file_name).get('configs')
|
||||
}
|
||||
config['configs'] += edp.get_possible_mapreduce_configs()
|
||||
return config
|
||||
|
||||
|
||||
def get_possible_pig_config_from(file_name):
|
||||
'''Return the possible configs, args, params for a Pig job.'''
|
||||
config = {
|
||||
'configs': utils.load_hadoop_xml_defaults(file_name,
|
||||
'sahara_plugin_vanilla'),
|
||||
'args': [],
|
||||
'params': {}
|
||||
}
|
||||
return config
|
@ -1,40 +0,0 @@
|
||||
# Copyright (c) 2014 Mirantis Inc.
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
|
||||
# implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
from sahara.plugins import edp
|
||||
from sahara.plugins import exceptions as ex
|
||||
from sahara.plugins import utils as u
|
||||
from sahara_plugin_vanilla.plugins.vanilla import utils as vu
|
||||
|
||||
|
||||
class EdpOozieEngine(edp.PluginsOozieJobEngine):
|
||||
def get_hdfs_user(self):
|
||||
return 'hadoop'
|
||||
|
||||
def get_name_node_uri(self, cluster):
|
||||
return cluster['info']['HDFS']['NameNode']
|
||||
|
||||
def get_oozie_server_uri(self, cluster):
|
||||
return cluster['info']['JobFlow']['Oozie'] + "/oozie/"
|
||||
|
||||
def get_oozie_server(self, cluster):
|
||||
return vu.get_oozie(cluster)
|
||||
|
||||
def validate_job_execution(self, cluster, job, data):
|
||||
oo_count = u.get_instances_count(cluster, 'oozie')
|
||||
if oo_count != 1:
|
||||
raise ex.InvalidComponentCountException('oozie', '1', oo_count)
|
||||
|
||||
super(EdpOozieEngine, self).validate_job_execution(cluster, job, data)
|
@ -1,475 +0,0 @@
|
||||
# Copyright (c) 2014 Mirantis Inc.
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
|
||||
# implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
import os
|
||||
|
||||
from oslo_config import cfg
|
||||
from oslo_log import log as logging
|
||||
import six
|
||||
|
||||
from sahara.plugins import castellan_utils as key_manager
|
||||
from sahara.plugins import context
|
||||
from sahara.plugins import swift_helper as swift
|
||||
from sahara.plugins import topology_helper as th
|
||||
from sahara.plugins import utils
|
||||
from sahara_plugin_vanilla.i18n import _
|
||||
from sahara_plugin_vanilla.plugins.vanilla.hadoop2 import config_helper
|
||||
from sahara_plugin_vanilla.plugins.vanilla.hadoop2 import oozie_helper
|
||||
from sahara_plugin_vanilla.plugins.vanilla.hadoop2 import utils as u
|
||||
from sahara_plugin_vanilla.plugins.vanilla import utils as vu
|
||||
|
||||
CONF = cfg.CONF
|
||||
LOG = logging.getLogger(__name__)
|
||||
|
||||
HADOOP_CONF_DIR = '/opt/hadoop/etc/hadoop'
|
||||
OOZIE_CONF_DIR = '/opt/oozie/conf'
|
||||
HIVE_CONF_DIR = '/opt/hive/conf'
|
||||
HADOOP_USER = 'hadoop'
|
||||
HADOOP_GROUP = 'hadoop'
|
||||
|
||||
PORTS_MAP = {
|
||||
"namenode": [50070, 9000],
|
||||
"secondarynamenode": [50090],
|
||||
"resourcemanager": [8088, 8032],
|
||||
"historyserver": [19888],
|
||||
"datanode": [50010, 50075, 50020],
|
||||
"nodemanager": [8042],
|
||||
"oozie": [11000],
|
||||
"hiveserver": [9999, 10000],
|
||||
"spark history server": [18080],
|
||||
"zookeeper": [2181, 2888, 3888]
|
||||
}
|
||||
|
||||
|
||||
def configure_cluster(pctx, cluster):
|
||||
LOG.debug("Configuring cluster")
|
||||
if (CONF.use_identity_api_v3 and CONF.use_domain_for_proxy_users and
|
||||
vu.get_hiveserver(cluster) and
|
||||
config_helper.is_swift_enabled(pctx, cluster)):
|
||||
cluster = utils.create_proxy_user_for_cluster(cluster)
|
||||
|
||||
instances = utils.get_instances(cluster)
|
||||
configure_instances(pctx, instances)
|
||||
configure_topology_data(pctx, cluster)
|
||||
configure_zookeeper(cluster)
|
||||
configure_spark(cluster)
|
||||
|
||||
|
||||
def configure_zookeeper(cluster, instances=None):
|
||||
zk_servers = vu.get_zk_servers(cluster)
|
||||
if zk_servers:
|
||||
zk_conf = config_helper.generate_zk_basic_config(cluster)
|
||||
zk_conf += _form_zk_servers_to_quorum(cluster, instances)
|
||||
_push_zk_configs_to_nodes(cluster, zk_conf, instances)
|
||||
|
||||
|
||||
def _form_zk_servers_to_quorum(cluster, to_delete_instances=None):
|
||||
quorum = []
|
||||
instances = map(vu.get_instance_hostname, vu.get_zk_servers(cluster))
|
||||
if to_delete_instances:
|
||||
delete_instances = map(vu.get_instance_hostname, to_delete_instances)
|
||||
reserve_instances = list(set(instances) - set(delete_instances))
|
||||
# keep the original order of instances
|
||||
reserve_instances.sort(key=instances.index)
|
||||
else:
|
||||
reserve_instances = instances
|
||||
for index, instance in enumerate(reserve_instances):
|
||||
quorum.append("server.%s=%s:2888:3888" % (index, instance))
|
||||
return '\n'.join(quorum)
|
||||
|
||||
|
||||
def _push_zk_configs_to_nodes(cluster, zk_conf, to_delete_instances=None):
|
||||
instances = vu.get_zk_servers(cluster)
|
||||
if to_delete_instances:
|
||||
for instance in to_delete_instances:
|
||||
if instance in instances:
|
||||
instances.remove(instance)
|
||||
for index, instance in enumerate(instances):
|
||||
with instance.remote() as r:
|
||||
r.write_file_to('/opt/zookeeper/conf/zoo.cfg', zk_conf,
|
||||
run_as_root=True)
|
||||
r.execute_command(
|
||||
'sudo su - -c "echo %s > /var/zookeeper/myid" hadoop' % index)
|
||||
|
||||
|
||||
def configure_spark(cluster):
|
||||
spark_servers = vu.get_spark_history_server(cluster)
|
||||
if spark_servers:
|
||||
extra = _extract_spark_configs_to_extra(cluster)
|
||||
_push_spark_configs_to_node(cluster, extra)
|
||||
|
||||
|
||||
def _push_spark_configs_to_node(cluster, extra):
|
||||
spark_master = vu.get_spark_history_server(cluster)
|
||||
if spark_master:
|
||||
_push_spark_configs_to_existing_node(spark_master, cluster, extra)
|
||||
_push_cleanup_job(spark_master, extra)
|
||||
with spark_master.remote() as r:
|
||||
r.execute_command('sudo su - -c "mkdir /tmp/spark-events" hadoop')
|
||||
|
||||
|
||||
def _push_spark_configs_to_existing_node(spark_master, cluster, extra):
|
||||
|
||||
sp_home = config_helper.get_spark_home(cluster)
|
||||
files = {
|
||||
os.path.join(sp_home,
|
||||
'conf/spark-env.sh'): extra['sp_master'],
|
||||
os.path.join(
|
||||
sp_home,
|
||||
'conf/spark-defaults.conf'): extra['sp_defaults']
|
||||
}
|
||||
|
||||
with spark_master.remote() as r:
|
||||
r.write_files_to(files, run_as_root=True)
|
||||
|
||||
|
||||
def _push_cleanup_job(sp_master, extra):
|
||||
with sp_master.remote() as r:
|
||||
if extra['job_cleanup']['valid']:
|
||||
r.write_file_to('/opt/hadoop/tmp-cleanup.sh',
|
||||
extra['job_cleanup']['script'],
|
||||
run_as_root=True)
|
||||
r.execute_command("sudo chmod 755 /opt/hadoop/tmp-cleanup.sh")
|
||||
cmd = 'sudo sh -c \'echo "%s" > /etc/cron.d/spark-cleanup\''
|
||||
r.execute_command(cmd % extra['job_cleanup']['cron'])
|
||||
else:
|
||||
r.execute_command("sudo rm -f /opt/hadoop/tmp-cleanup.sh")
|
||||
r.execute_command("sudo rm -f /etc/cron.d/spark-cleanup")
|
||||
|
||||
|
||||
def _extract_spark_configs_to_extra(cluster):
|
||||
sp_master = utils.get_instance(cluster, "spark history server")
|
||||
|
||||
extra = dict()
|
||||
|
||||
config_master = ''
|
||||
if sp_master is not None:
|
||||
config_master = config_helper.generate_spark_env_configs(cluster)
|
||||
|
||||
# Any node that might be used to run spark-submit will need
|
||||
# these libs for swift integration
|
||||
config_defaults = config_helper.generate_spark_executor_classpath(cluster)
|
||||
|
||||
extra['job_cleanup'] = config_helper.generate_job_cleanup_config(cluster)
|
||||
extra['sp_master'] = config_master
|
||||
extra['sp_defaults'] = config_defaults
|
||||
|
||||
return extra
|
||||
|
||||
|
||||
def configure_instances(pctx, instances):
|
||||
if len(instances) == 0:
|
||||
return
|
||||
|
||||
utils.add_provisioning_step(
|
||||
instances[0].cluster_id, _("Configure instances"), len(instances))
|
||||
|
||||
for instance in instances:
|
||||
with context.set_current_instance_id(instance.instance_id):
|
||||
_configure_instance(pctx, instance)
|
||||
|
||||
|
||||
@utils.event_wrapper(True)
|
||||
def _configure_instance(pctx, instance):
|
||||
_provisioning_configs(pctx, instance)
|
||||
_post_configuration(pctx, instance)
|
||||
|
||||
|
||||
def _provisioning_configs(pctx, instance):
|
||||
xmls, env = _generate_configs(pctx, instance)
|
||||
_push_xml_configs(instance, xmls)
|
||||
_push_env_configs(instance, env)
|
||||
|
||||
|
||||
def _generate_configs(pctx, instance):
|
||||
hadoop_xml_confs = _get_hadoop_configs(pctx, instance)
|
||||
user_xml_confs, user_env_confs = _get_user_configs(
|
||||
pctx, instance.node_group)
|
||||
xml_confs = utils.merge_configs(user_xml_confs, hadoop_xml_confs)
|
||||
env_confs = utils.merge_configs(pctx['env_confs'], user_env_confs)
|
||||
|
||||
return xml_confs, env_confs
|
||||
|
||||
|
||||
def _get_hadoop_configs(pctx, instance):
|
||||
cluster = instance.node_group.cluster
|
||||
nn_hostname = vu.get_instance_hostname(vu.get_namenode(cluster))
|
||||
dirs = _get_hadoop_dirs(instance)
|
||||
confs = {
|
||||
'Hadoop': {
|
||||
'fs.defaultFS': 'hdfs://%s:9000' % nn_hostname
|
||||
},
|
||||
'HDFS': {
|
||||
'dfs.namenode.name.dir': ','.join(dirs['hadoop_name_dirs']),
|
||||
'dfs.datanode.data.dir': ','.join(dirs['hadoop_data_dirs']),
|
||||
'dfs.hosts': '%s/dn-include' % HADOOP_CONF_DIR,
|
||||
'dfs.hosts.exclude': '%s/dn-exclude' % HADOOP_CONF_DIR
|
||||
}
|
||||
}
|
||||
|
||||
res_hostname = vu.get_instance_hostname(vu.get_resourcemanager(cluster))
|
||||
if res_hostname:
|
||||
confs['YARN'] = {
|
||||
'yarn.nodemanager.aux-services': 'mapreduce_shuffle',
|
||||
'yarn.resourcemanager.hostname': '%s' % res_hostname,
|
||||
'yarn.resourcemanager.nodes.include-path': '%s/nm-include' % (
|
||||
HADOOP_CONF_DIR),
|
||||
'yarn.resourcemanager.nodes.exclude-path': '%s/nm-exclude' % (
|
||||
HADOOP_CONF_DIR)
|
||||
}
|
||||
confs['MapReduce'] = {
|
||||
'mapreduce.framework.name': 'yarn'
|
||||
}
|
||||
hs_hostname = vu.get_instance_hostname(vu.get_historyserver(cluster))
|
||||
if hs_hostname:
|
||||
confs['MapReduce']['mapreduce.jobhistory.address'] = (
|
||||
"%s:10020" % hs_hostname)
|
||||
|
||||
oozie = vu.get_oozie(cluster)
|
||||
if oozie:
|
||||
hadoop_cfg = {
|
||||
'hadoop.proxyuser.hadoop.hosts': '*',
|
||||
'hadoop.proxyuser.hadoop.groups': 'hadoop'
|
||||
}
|
||||
confs['Hadoop'].update(hadoop_cfg)
|
||||
|
||||
oozie_cfg = oozie_helper.get_oozie_required_xml_configs(
|
||||
HADOOP_CONF_DIR)
|
||||
if config_helper.is_mysql_enabled(pctx, cluster):
|
||||
oozie_cfg.update(oozie_helper.get_oozie_mysql_configs(cluster))
|
||||
|
||||
confs['JobFlow'] = oozie_cfg
|
||||
|
||||
if config_helper.is_swift_enabled(pctx, cluster):
|
||||
swift_configs = {}
|
||||
for config in swift.get_swift_configs():
|
||||
swift_configs[config['name']] = config['value']
|
||||
|
||||
confs['Hadoop'].update(swift_configs)
|
||||
|
||||
if config_helper.is_data_locality_enabled(pctx, cluster):
|
||||
confs['Hadoop'].update(th.TOPOLOGY_CONFIG)
|
||||
confs['Hadoop'].update({"topology.script.file.name":
|
||||
HADOOP_CONF_DIR + "/topology.sh"})
|
||||
|
||||
hive_hostname = vu.get_instance_hostname(vu.get_hiveserver(cluster))
|
||||
if hive_hostname:
|
||||
hive_pass = u.get_hive_password(cluster)
|
||||
|
||||
hive_cfg = {
|
||||
'hive.warehouse.subdir.inherit.perms': True,
|
||||
'javax.jdo.option.ConnectionURL':
|
||||
'jdbc:derby:;databaseName=/opt/hive/metastore_db;create=true'
|
||||
}
|
||||
|
||||
if config_helper.is_mysql_enabled(pctx, cluster):
|
||||
hive_cfg.update({
|
||||
'javax.jdo.option.ConnectionURL':
|
||||
'jdbc:mysql://%s/metastore' % hive_hostname,
|
||||
'javax.jdo.option.ConnectionDriverName':
|
||||
'com.mysql.jdbc.Driver',
|
||||
'javax.jdo.option.ConnectionUserName': 'hive',
|
||||
'javax.jdo.option.ConnectionPassword': hive_pass,
|
||||
'datanucleus.autoCreateSchema': 'false',
|
||||
'datanucleus.fixedDatastore': 'true',
|
||||
'hive.metastore.uris': 'thrift://%s:9083' % hive_hostname,
|
||||
})
|
||||
|
||||
proxy_configs = cluster.cluster_configs.get('proxy_configs')
|
||||
if proxy_configs and config_helper.is_swift_enabled(pctx, cluster):
|
||||
hive_cfg.update({
|
||||
swift.HADOOP_SWIFT_USERNAME: proxy_configs['proxy_username'],
|
||||
swift.HADOOP_SWIFT_PASSWORD: key_manager.get_secret(
|
||||
proxy_configs['proxy_password']),
|
||||
swift.HADOOP_SWIFT_TRUST_ID: proxy_configs['proxy_trust_id'],
|
||||
swift.HADOOP_SWIFT_DOMAIN_NAME: CONF.proxy_user_domain_name
|
||||
})
|
||||
|
||||
confs['Hive'] = hive_cfg
|
||||
|
||||
return confs
|
||||
|
||||
|
||||
def _get_user_configs(pctx, node_group):
|
||||
ng_xml_confs, ng_env_confs = _separate_configs(node_group.node_configs,
|
||||
pctx['env_confs'])
|
||||
cl_xml_confs, cl_env_confs = _separate_configs(
|
||||
node_group.cluster.cluster_configs, pctx['env_confs'])
|
||||
|
||||
xml_confs = utils.merge_configs(cl_xml_confs, ng_xml_confs)
|
||||
env_confs = utils.merge_configs(cl_env_confs, ng_env_confs)
|
||||
return xml_confs, env_confs
|
||||
|
||||
|
||||
def _separate_configs(configs, all_env_configs):
|
||||
xml_configs = {}
|
||||
env_configs = {}
|
||||
for service, params in six.iteritems(configs):
|
||||
for param, value in six.iteritems(params):
|
||||
if all_env_configs.get(service, {}).get(param):
|
||||
if not env_configs.get(service):
|
||||
env_configs[service] = {}
|
||||
env_configs[service][param] = value
|
||||
else:
|
||||
if not xml_configs.get(service):
|
||||
xml_configs[service] = {}
|
||||
xml_configs[service][param] = value
|
||||
|
||||
return xml_configs, env_configs
|
||||
|
||||
|
||||
def _generate_xml(configs):
|
||||
xml_confs = {}
|
||||
for service, confs in six.iteritems(configs):
|
||||
xml_confs[service] = utils.create_hadoop_xml(confs)
|
||||
|
||||
return xml_confs
|
||||
|
||||
|
||||
def _push_env_configs(instance, configs):
|
||||
nn_heap = configs['HDFS']['NameNode Heap Size']
|
||||
snn_heap = configs['HDFS']['SecondaryNameNode Heap Size']
|
||||
dn_heap = configs['HDFS']['DataNode Heap Size']
|
||||
rm_heap = configs['YARN']['ResourceManager Heap Size']
|
||||
nm_heap = configs['YARN']['NodeManager Heap Size']
|
||||
hs_heap = configs['MapReduce']['JobHistoryServer Heap Size']
|
||||
|
||||
with instance.remote() as r:
|
||||
r.replace_remote_string(
|
||||
'%s/hadoop-env.sh' % HADOOP_CONF_DIR,
|
||||
'export HADOOP_NAMENODE_OPTS=.*',
|
||||
'export HADOOP_NAMENODE_OPTS="-Xmx%dm"' % nn_heap)
|
||||
r.replace_remote_string(
|
||||
'%s/hadoop-env.sh' % HADOOP_CONF_DIR,
|
||||
'export HADOOP_SECONDARYNAMENODE_OPTS=.*',
|
||||
'export HADOOP_SECONDARYNAMENODE_OPTS="-Xmx%dm"' % snn_heap)
|
||||
r.replace_remote_string(
|
||||
'%s/hadoop-env.sh' % HADOOP_CONF_DIR,
|
||||
'export HADOOP_DATANODE_OPTS=.*',
|
||||
'export HADOOP_DATANODE_OPTS="-Xmx%dm"' % dn_heap)
|
||||
r.replace_remote_string(
|
||||
'%s/yarn-env.sh' % HADOOP_CONF_DIR,
|
||||
'\\#export YARN_RESOURCEMANAGER_HEAPSIZE=.*',
|
||||
'export YARN_RESOURCEMANAGER_HEAPSIZE=%d' % rm_heap)
|
||||
r.replace_remote_string(
|
||||
'%s/yarn-env.sh' % HADOOP_CONF_DIR,
|
||||
'\\#export YARN_NODEMANAGER_HEAPSIZE=.*',
|
||||
'export YARN_NODEMANAGER_HEAPSIZE=%d' % nm_heap)
|
||||
r.replace_remote_string(
|
||||
'%s/mapred-env.sh' % HADOOP_CONF_DIR,
|
||||
'export HADOOP_JOB_HISTORYSERVER_HEAPSIZE=.*',
|
||||
'export HADOOP_JOB_HISTORYSERVER_HEAPSIZE=%d' % hs_heap)
|
||||
|
||||
|
||||
def _push_xml_configs(instance, configs):
|
||||
xmls = _generate_xml(configs)
|
||||
service_to_conf_map = {
|
||||
'Hadoop': '%s/core-site.xml' % HADOOP_CONF_DIR,
|
||||
'HDFS': '%s/hdfs-site.xml' % HADOOP_CONF_DIR,
|
||||
'YARN': '%s/yarn-site.xml' % HADOOP_CONF_DIR,
|
||||
'MapReduce': '%s/mapred-site.xml' % HADOOP_CONF_DIR,
|
||||
'JobFlow': '%s/oozie-site.xml' % OOZIE_CONF_DIR,
|
||||
'Hive': '%s/hive-site.xml' % HIVE_CONF_DIR
|
||||
}
|
||||
xml_confs = {}
|
||||
for service, confs in six.iteritems(xmls):
|
||||
if service not in service_to_conf_map.keys():
|
||||
continue
|
||||
|
||||
xml_confs[service_to_conf_map[service]] = confs
|
||||
|
||||
_push_configs_to_instance(instance, xml_confs)
|
||||
|
||||
|
||||
def _push_configs_to_instance(instance, configs):
|
||||
LOG.debug("Push configs to instance {instance}".format(
|
||||
instance=instance.instance_name))
|
||||
with instance.remote() as r:
|
||||
for fl, data in six.iteritems(configs):
|
||||
r.write_file_to(fl, data, run_as_root=True)
|
||||
|
||||
|
||||
def _post_configuration(pctx, instance):
|
||||
dirs = _get_hadoop_dirs(instance)
|
||||
args = {
|
||||
'hadoop_user': HADOOP_USER,
|
||||
'hadoop_group': HADOOP_GROUP,
|
||||
'hadoop_conf_dir': HADOOP_CONF_DIR,
|
||||
'oozie_conf_dir': OOZIE_CONF_DIR,
|
||||
'hadoop_name_dirs': " ".join(dirs['hadoop_name_dirs']),
|
||||
'hadoop_data_dirs': " ".join(dirs['hadoop_data_dirs']),
|
||||
'hadoop_log_dir': dirs['hadoop_log_dir'],
|
||||
'hadoop_secure_dn_log_dir': dirs['hadoop_secure_dn_log_dir'],
|
||||
'yarn_log_dir': dirs['yarn_log_dir']
|
||||
}
|
||||
post_conf_script = utils.get_file_text(
|
||||
'plugins/vanilla/hadoop2/resources/post_conf.template',
|
||||
'sahara_plugin_vanilla')
|
||||
post_conf_script = post_conf_script.format(**args)
|
||||
|
||||
with instance.remote() as r:
|
||||
r.write_file_to('/tmp/post_conf.sh', post_conf_script)
|
||||
r.execute_command('chmod +x /tmp/post_conf.sh')
|
||||
r.execute_command('sudo /tmp/post_conf.sh')
|
||||
|
||||
if config_helper.is_data_locality_enabled(pctx, instance.cluster):
|
||||
t_script = HADOOP_CONF_DIR + '/topology.sh'
|
||||
r.write_file_to(t_script, utils.get_file_text(
|
||||
'plugins/vanilla/hadoop2/resources/topology.sh',
|
||||
'sahara_plugin_vanilla'), run_as_root=True)
|
||||
r.execute_command('chmod +x ' + t_script, run_as_root=True)
|
||||
|
||||
|
||||
def _get_hadoop_dirs(instance):
|
||||
dirs = {}
|
||||
storage_paths = instance.storage_paths()
|
||||
dirs['hadoop_name_dirs'] = _make_hadoop_paths(
|
||||
storage_paths, '/hdfs/namenode')
|
||||
dirs['hadoop_data_dirs'] = _make_hadoop_paths(
|
||||
storage_paths, '/hdfs/datanode')
|
||||
dirs['hadoop_log_dir'] = _make_hadoop_paths(
|
||||
storage_paths, '/hadoop/logs')[0]
|
||||
dirs['hadoop_secure_dn_log_dir'] = _make_hadoop_paths(
|
||||
storage_paths, '/hadoop/logs/secure')[0]
|
||||
dirs['yarn_log_dir'] = _make_hadoop_paths(
|
||||
storage_paths, '/yarn/logs')[0]
|
||||
|
||||
return dirs
|
||||
|
||||
|
||||
def _make_hadoop_paths(paths, hadoop_dir):
|
||||
return [path + hadoop_dir for path in paths]
|
||||
|
||||
|
||||
@utils.event_wrapper(
|
||||
True, step=_("Configure topology data"), param=('cluster', 1))
|
||||
def configure_topology_data(pctx, cluster):
|
||||
if config_helper.is_data_locality_enabled(pctx, cluster):
|
||||
LOG.warning("Node group awareness is not implemented in YARN yet "
|
||||
"so enable_hypervisor_awareness set to False explicitly")
|
||||
tpl_map = th.generate_topology_map(cluster, is_node_awareness=False)
|
||||
topology_data = "\n".join(
|
||||
[k + " " + v for k, v in tpl_map.items()]) + "\n"
|
||||
for ng in cluster.node_groups:
|
||||
for i in ng.instances:
|
||||
i.remote().write_file_to(HADOOP_CONF_DIR + "/topology.data",
|
||||
topology_data, run_as_root=True)
|
||||
|
||||
|
||||
def get_open_ports(node_group):
|
||||
ports = []
|
||||
for key in PORTS_MAP:
|
||||
if key in node_group.node_processes:
|
||||
ports += PORTS_MAP[key]
|
||||
return ports
|
@ -1,331 +0,0 @@
|
||||
# Copyright (c) 2014 Mirantis, Inc.
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
|
||||
# implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
from oslo_config import cfg
|
||||
import six
|
||||
|
||||
from sahara.plugins import exceptions as ex
|
||||
from sahara.plugins import provisioning as p
|
||||
from sahara.plugins import utils
|
||||
from sahara_plugin_vanilla.i18n import _
|
||||
|
||||
CONF = cfg.CONF
|
||||
CONF.import_opt("enable_data_locality", "sahara.topology.topology_helper")
|
||||
|
||||
HIDDEN_CONFS = [
|
||||
'dfs.hosts',
|
||||
'dfs.hosts.exclude',
|
||||
'dfs.namenode.data.dir',
|
||||
'dfs.namenode.name.dir',
|
||||
'fs.default.name',
|
||||
'fs.defaultFS',
|
||||
'fs.swift.impl',
|
||||
'hadoop.proxyuser.hadoop.groups',
|
||||
'hadoop.proxyuser.hadoop.hosts',
|
||||
'mapreduce.framework.name',
|
||||
'mapreduce.jobhistory.address',
|
||||
'mapreduce.jobhistory.done.dir',
|
||||
'mapreduce.jobhistory.intermediate-done-dir',
|
||||
'mapreduce.jobhistory.webapp.address',
|
||||
'yarn.nodemanager.aux-services',
|
||||
'yarn.resourcemanager.address',
|
||||
'yarn.resourcemanager.admin.address',
|
||||
'yarn.resourcemanager.hostname',
|
||||
'yarn.resourcemanager.nodes.exclude-path',
|
||||
'yarn.resourcemanager.nodes.include-path',
|
||||
'yarn.resourcemanager.resource-tracker.address',
|
||||
'yarn.resourcemanager.scheduler.address',
|
||||
'yarn.resourcemanager.webapp.address'
|
||||
]
|
||||
|
||||
CLUSTER_WIDE_CONFS = [
|
||||
'dfs.blocksize', 'dfs.namenode.replication.min', 'dfs.permissions.enabled',
|
||||
'dfs.replication', 'dfs.replication.max', 'io.compression.codecs',
|
||||
'io.file.buffer.size', 'mapreduce.job.counters.max',
|
||||
'mapreduce.map.output.compress.codec',
|
||||
'mapreduce.output.fileoutputformat.compress.codec',
|
||||
'mapreduce.output.fileoutputformat.compress.type',
|
||||
'mapredude.map.output.compress',
|
||||
'mapredude.output.fileoutputformat.compress'
|
||||
]
|
||||
|
||||
PRIORITY_1_CONFS = [
|
||||
'dfs.datanode.du.reserved',
|
||||
'dfs.datanode.failed.volumes.tolerated',
|
||||
'dfs.datanode.handler.count',
|
||||
'dfs.datanode.max.transfer.threads',
|
||||
'dfs.namenode.handler.count',
|
||||
'mapred.child.java.opts',
|
||||
'mapred.jobtracker.maxtasks.per.job',
|
||||
'mapreduce.jobtracker.handler.count',
|
||||
'mapreduce.map.java.opts',
|
||||
'mapreduce.reduce.java.opts',
|
||||
'mapreduce.task.io.sort.mb',
|
||||
'mapreduce.tasktracker.map.tasks.maximum',
|
||||
'mapreduce.tasktracker.reduce.tasks.maximum',
|
||||
'yarn.nodemanager.resource.cpu-vcores',
|
||||
'yarn.nodemanager.resource.memory-mb',
|
||||
'yarn.scheduler.maximum-allocation-mb',
|
||||
'yarn.scheduler.maximum-allocation-vcores',
|
||||
'yarn.scheduler.minimum-allocation-mb',
|
||||
'yarn.scheduler.minimum-allocation-vcores'
|
||||
]
|
||||
|
||||
SPARK_CONFS = {
|
||||
'Spark': {
|
||||
"OPTIONS": [
|
||||
{
|
||||
'name': 'Spark home',
|
||||
'description': 'The location of the spark installation'
|
||||
' (default: /opt/spark)',
|
||||
'default': '/opt/spark',
|
||||
'priority': 2,
|
||||
},
|
||||
{
|
||||
'name': 'Minimum cleanup seconds',
|
||||
'description': 'Job data will never be purged before this'
|
||||
' amount of time elapses (default: 86400 = 1 day)',
|
||||
'default': '86400',
|
||||
'priority': 2,
|
||||
},
|
||||
{
|
||||
'name': 'Maximum cleanup seconds',
|
||||
'description': 'Job data will always be purged after this'
|
||||
' amount of time elapses (default: 1209600 = 14 days)',
|
||||
'default': '1209600',
|
||||
'priority': 2,
|
||||
},
|
||||
{
|
||||
'name': 'Minimum cleanup megabytes',
|
||||
'description': 'No job data will be purged unless the total'
|
||||
' job data exceeds this size (default: 4096 = 4GB)',
|
||||
'default': '4096',
|
||||
'priority': 2,
|
||||
},
|
||||
]
|
||||
}
|
||||
}
|
||||
|
||||
ZOOKEEPER_CONFS = {
|
||||
"ZooKeeper": {
|
||||
"OPTIONS": [
|
||||
{
|
||||
'name': 'tickTime',
|
||||
'description': 'The number of milliseconds of each tick',
|
||||
'default': 2000,
|
||||
'priority': 2,
|
||||
},
|
||||
{
|
||||
'name': 'initLimit',
|
||||
'description': 'The number of ticks that the initial'
|
||||
' synchronization phase can take',
|
||||
'default': 10,
|
||||
'priority': 2,
|
||||
},
|
||||
{
|
||||
'name': 'syncLimit',
|
||||
'description': 'The number of ticks that can pass between'
|
||||
' sending a request and getting an acknowledgement',
|
||||
'default': 5,
|
||||
'priority': 2,
|
||||
},
|
||||
]
|
||||
}
|
||||
}
|
||||
|
||||
# for now we have not so many cluster-wide configs
|
||||
# lets consider all of them having high priority
|
||||
PRIORITY_1_CONFS += CLUSTER_WIDE_CONFS
|
||||
|
||||
|
||||
def init_xml_configs(xml_confs):
|
||||
configs = []
|
||||
for service, config_lists in six.iteritems(xml_confs):
|
||||
for config_list in config_lists:
|
||||
for config in config_list:
|
||||
if config['name'] not in HIDDEN_CONFS:
|
||||
cfg = p.Config(config['name'], service, "node",
|
||||
is_optional=True, config_type="string",
|
||||
default_value=str(config['value']),
|
||||
description=config['description'])
|
||||
if cfg.default_value in ["true", "false"]:
|
||||
cfg.config_type = "bool"
|
||||
cfg.default_value = (cfg.default_value == 'true')
|
||||
elif utils.is_int(cfg.default_value):
|
||||
cfg.config_type = "int"
|
||||
cfg.default_value = int(cfg.default_value)
|
||||
if config['name'] in CLUSTER_WIDE_CONFS:
|
||||
cfg.scope = 'cluster'
|
||||
if config['name'] in PRIORITY_1_CONFS:
|
||||
cfg.priority = 1
|
||||
configs.append(cfg)
|
||||
|
||||
return configs
|
||||
|
||||
|
||||
ENABLE_SWIFT = p.Config('Enable Swift', 'general', 'cluster',
|
||||
config_type="bool", priority=1,
|
||||
default_value=True, is_optional=False)
|
||||
|
||||
ENABLE_MYSQL = p.Config('Enable MySQL', 'general', 'cluster',
|
||||
config_type="bool", priority=1,
|
||||
default_value=True, is_optional=True)
|
||||
|
||||
ENABLE_DATA_LOCALITY = p.Config('Enable Data Locality', 'general', 'cluster',
|
||||
config_type="bool", priority=1,
|
||||
default_value=True, is_optional=True)
|
||||
|
||||
|
||||
DATANODES_DECOMMISSIONING_TIMEOUT = p.Config(
|
||||
'DataNodes decommissioning timeout', 'general',
|
||||
'cluster', config_type='int', priority=1,
|
||||
default_value=3600 * 4, is_optional=True,
|
||||
description='Timeout for datanode decommissioning operation'
|
||||
' during scaling, in seconds')
|
||||
|
||||
|
||||
NODEMANAGERS_DECOMMISSIONING_TIMEOUT = p.Config(
|
||||
'NodeManagers decommissioning timeout', 'general',
|
||||
'cluster', config_type='int', priority=1,
|
||||
default_value=300, is_optional=True,
|
||||
description='Timeout for NodeManager decommissioning operation'
|
||||
' during scaling, in seconds')
|
||||
|
||||
|
||||
DATANODES_STARTUP_TIMEOUT = p.Config(
|
||||
'DataNodes startup timeout', 'general', 'cluster', config_type='int',
|
||||
priority=1, default_value=10800, is_optional=True,
|
||||
description='Timeout for DataNodes startup, in seconds')
|
||||
|
||||
|
||||
def init_env_configs(env_confs):
|
||||
configs = []
|
||||
for service, config_items in six.iteritems(env_confs):
|
||||
for name, value in six.iteritems(config_items):
|
||||
configs.append(p.Config(name, service, "node",
|
||||
default_value=value, priority=1,
|
||||
config_type="int"))
|
||||
|
||||
return configs
|
||||
|
||||
|
||||
def _init_general_configs():
|
||||
configs = [ENABLE_SWIFT, ENABLE_MYSQL, DATANODES_STARTUP_TIMEOUT,
|
||||
DATANODES_DECOMMISSIONING_TIMEOUT,
|
||||
NODEMANAGERS_DECOMMISSIONING_TIMEOUT]
|
||||
if CONF.enable_data_locality:
|
||||
configs.append(ENABLE_DATA_LOCALITY)
|
||||
return configs
|
||||
|
||||
|
||||
PLUGIN_GENERAL_CONFIGS = _init_general_configs()
|
||||
|
||||
|
||||
def get_config_value(pctx, service, name, cluster=None):
|
||||
if cluster:
|
||||
for ng in cluster.node_groups:
|
||||
cl_param = ng.configuration().get(service, {}).get(name)
|
||||
if cl_param is not None:
|
||||
return cl_param
|
||||
|
||||
for c in pctx['all_confs']:
|
||||
if c.applicable_target == service and c.name == name:
|
||||
return c.default_value
|
||||
|
||||
raise ex.PluginNotFoundException(
|
||||
{"name": name, "service": service},
|
||||
_("Unable to get parameter '%(name)s' from service %(service)s"))
|
||||
|
||||
|
||||
def is_swift_enabled(pctx, cluster):
|
||||
return get_config_value(pctx, ENABLE_SWIFT.applicable_target,
|
||||
ENABLE_SWIFT.name, cluster)
|
||||
|
||||
|
||||
def is_mysql_enabled(pctx, cluster):
|
||||
return get_config_value(
|
||||
pctx, ENABLE_MYSQL.applicable_target, ENABLE_MYSQL.name, cluster)
|
||||
|
||||
|
||||
def is_data_locality_enabled(pctx, cluster):
|
||||
if not CONF.enable_data_locality:
|
||||
return False
|
||||
return get_config_value(pctx, ENABLE_DATA_LOCALITY.applicable_target,
|
||||
ENABLE_DATA_LOCALITY.name, cluster)
|
||||
|
||||
|
||||
def generate_spark_env_configs(cluster):
|
||||
configs = []
|
||||
|
||||
# point to the hadoop conf dir so that Spark can read things
|
||||
# like the swift configuration without having to copy core-site
|
||||
# to /opt/spark/conf
|
||||
HADOOP_CONF_DIR = '/opt/hadoop/etc/hadoop'
|
||||
configs.append('HADOOP_CONF_DIR=' + HADOOP_CONF_DIR)
|
||||
|
||||
# Hadoop and YARN configs there are in one folder
|
||||
configs.append('YARN_CONF_DIR=' + HADOOP_CONF_DIR)
|
||||
|
||||
return '\n'.join(configs)
|
||||
|
||||
|
||||
def generate_spark_executor_classpath(cluster):
|
||||
cp = utils.get_config_value_or_default(
|
||||
"Spark", "Executor extra classpath", cluster)
|
||||
if cp:
|
||||
return "spark.executor.extraClassPath " + cp
|
||||
return "\n"
|
||||
|
||||
|
||||
def generate_job_cleanup_config(cluster):
|
||||
args = {
|
||||
'minimum_cleanup_megabytes': utils.get_config_value_or_default(
|
||||
"Spark", "Minimum cleanup megabytes", cluster),
|
||||
'minimum_cleanup_seconds': utils.get_config_value_or_default(
|
||||
"Spark", "Minimum cleanup seconds", cluster),
|
||||
'maximum_cleanup_seconds': utils.get_config_value_or_default(
|
||||
"Spark", "Maximum cleanup seconds", cluster)
|
||||
}
|
||||
job_conf = {'valid': (args['maximum_cleanup_seconds'] > 0 and
|
||||
(args['minimum_cleanup_megabytes'] > 0
|
||||
and args['minimum_cleanup_seconds'] > 0))}
|
||||
if job_conf['valid']:
|
||||
job_conf['cron'] = utils.get_file_text(
|
||||
'plugins/vanilla/hadoop2/resources/spark-cleanup.cron',
|
||||
'sahara_plugin_vanilla'),
|
||||
job_cleanup_script = utils.get_file_text(
|
||||
'plugins/vanilla/hadoop2/resources/tmp-cleanup.sh.template',
|
||||
'sahara_plugin_vanilla')
|
||||
job_conf['script'] = job_cleanup_script.format(**args)
|
||||
return job_conf
|
||||
|
||||
|
||||
def get_spark_home(cluster):
|
||||
return utils.get_config_value_or_default("Spark", "Spark home", cluster)
|
||||
|
||||
|
||||
def generate_zk_basic_config(cluster):
|
||||
args = {
|
||||
'ticktime': utils.get_config_value_or_default(
|
||||
"ZooKeeper", "tickTime", cluster),
|
||||
'initlimit': utils.get_config_value_or_default(
|
||||
"ZooKeeper", "initLimit", cluster),
|
||||
'synclimit': utils.get_config_value_or_default(
|
||||
"ZooKeeper", "syncLimit", cluster)
|
||||
}
|
||||
zoo_cfg = utils.get_file_text(
|
||||
'plugins/vanilla/hadoop2/resources/zoo_sample.cfg',
|
||||
'sahara_plugin_vanilla')
|
||||
return zoo_cfg.format(**args)
|
@ -1,26 +0,0 @@
|
||||
# Copyright (c) 2014 Mirantis Inc.
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
|
||||
# implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
from sahara.plugins import edp
|
||||
from sahara_plugin_vanilla.plugins.vanilla import edp_engine
|
||||
|
||||
|
||||
class EdpOozieEngine(edp_engine.EdpOozieEngine):
|
||||
|
||||
def create_hdfs_dir(self, remote, dir_name):
|
||||
edp.create_dir_hadoop2(remote, dir_name, self.get_hdfs_user())
|
||||
|
||||
def get_resource_manager_uri(self, cluster):
|
||||
return cluster['info']['YARN']['ResourceManager']
|
@ -1,92 +0,0 @@
|
||||
# Copyright (c) 2016 Mirantis Inc.
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
|
||||
# implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
from castellan.common.objects import passphrase
|
||||
from castellan import key_manager
|
||||
from oslo_log import log as logging
|
||||
|
||||
|
||||
from sahara.plugins import conductor
|
||||
from sahara.plugins import context
|
||||
from sahara.plugins import utils
|
||||
|
||||
LOG = logging.getLogger(__name__)
|
||||
|
||||
|
||||
def _provision_key(instance, keypair):
|
||||
def append_to(remote, file, *args, **kwargs):
|
||||
kwargs['run_as_root'] = True
|
||||
path = "/home/hadoop/.ssh/%s" % file
|
||||
remote.append_to_file(path, *args, **kwargs)
|
||||
public, private = keypair['public'], keypair['private']
|
||||
folder = '/home/hadoop/.ssh'
|
||||
with context.set_current_instance_id(instance_id=instance.instance_id):
|
||||
with instance.remote() as r:
|
||||
r.execute_command('sudo mkdir -p %s' % folder)
|
||||
append_to(r, 'authorized_keys', public)
|
||||
append_to(r, 'id_rsa', private)
|
||||
append_to(r, 'id_rsa.pub', public)
|
||||
r.execute_command('sudo chown -R hadoop %s' % folder)
|
||||
r.execute_command("sudo chmod 600 %s/id_rsa" % folder)
|
||||
LOG.debug("Passwordless ssh enabled")
|
||||
|
||||
|
||||
def _get_secret(secret):
|
||||
key = key_manager.API().get(context.current(), secret)
|
||||
return key.get_encoded()
|
||||
|
||||
|
||||
def _store_secret(secret):
|
||||
key = passphrase.Passphrase(secret)
|
||||
password = key_manager.API().store(context.current(), key)
|
||||
return password
|
||||
|
||||
|
||||
def _remove_secret(secret):
|
||||
key_manager.API().delete(context.current(), secret)
|
||||
|
||||
|
||||
def provision_keypairs(cluster, instances=None):
|
||||
extra = cluster.extra.to_dict() if cluster.extra else {}
|
||||
# use same keypair for scaling
|
||||
keypair = extra.get('vanilla_keypair')
|
||||
if not instances:
|
||||
instances = utils.get_instances(cluster)
|
||||
else:
|
||||
# scaling
|
||||
if not keypair:
|
||||
# cluster created before mitaka, skipping provisioning
|
||||
return
|
||||
if not keypair:
|
||||
private, public = utils.generate_key_pair()
|
||||
keypair = {'public': public, 'private': private}
|
||||
extra['vanilla_keypair'] = keypair
|
||||
extra['vanilla_keypair']['private'] = _store_secret(
|
||||
keypair['private'])
|
||||
conductor.cluster_update(context.ctx(), cluster, {'extra': extra})
|
||||
else:
|
||||
keypair['private'] = _get_secret(keypair['private'])
|
||||
with context.PluginsThreadGroup() as tg:
|
||||
for instance in instances:
|
||||
tg.spawn(
|
||||
'provision-key-%s' % instance.instance_name,
|
||||
_provision_key, instance, keypair)
|
||||
|
||||
|
||||
def drop_key(cluster):
|
||||
extra = cluster.extra.to_dict() if cluster.extra else {}
|
||||
keypair = extra.get('vanilla_keypair')
|
||||
if keypair:
|
||||
_remove_secret(keypair['private'])
|
@ -1,51 +0,0 @@
|
||||
# Copyright (c) 2014 Mirantis Inc.
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
|
||||
# implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
from sahara_plugin_vanilla.plugins.vanilla.hadoop2 import utils as u
|
||||
|
||||
|
||||
def get_oozie_required_xml_configs(hadoop_conf_dir):
|
||||
"""Following configs differ from default configs in oozie-default.xml."""
|
||||
return {
|
||||
'oozie.service.ActionService.executor.ext.classes':
|
||||
'org.apache.oozie.action.email.EmailActionExecutor,'
|
||||
'org.apache.oozie.action.hadoop.HiveActionExecutor,'
|
||||
'org.apache.oozie.action.hadoop.ShellActionExecutor,'
|
||||
'org.apache.oozie.action.hadoop.SqoopActionExecutor,'
|
||||
'org.apache.oozie.action.hadoop.DistcpActionExecutor',
|
||||
|
||||
'oozie.service.SchemaService.wf.ext.schemas':
|
||||
'shell-action-0.1.xsd,shell-action-0.2.xsd,shell-action-0.3.xsd,'
|
||||
'email-action-0.1.xsd,hive-action-0.2.xsd,hive-action-0.3.xsd,'
|
||||
'hive-action-0.4.xsd,hive-action-0.5.xsd,sqoop-action-0.2.xsd,'
|
||||
'sqoop-action-0.3.xsd,sqoop-action-0.4.xsd,ssh-action-0.1.xsd,'
|
||||
'ssh-action-0.2.xsd,distcp-action-0.1.xsd,distcp-action-0.2.xsd,'
|
||||
'oozie-sla-0.1.xsd,oozie-sla-0.2.xsd',
|
||||
|
||||
'oozie.service.JPAService.create.db.schema': 'false',
|
||||
'oozie.service.HadoopAccessorService.hadoop.configurations': '*=%s' % (
|
||||
hadoop_conf_dir)
|
||||
}
|
||||
|
||||
|
||||
def get_oozie_mysql_configs(cluster):
|
||||
return {
|
||||
'oozie.service.JPAService.jdbc.driver':
|
||||
'com.mysql.jdbc.Driver',
|
||||
'oozie.service.JPAService.jdbc.url':
|
||||
'jdbc:mysql://localhost:3306/oozie',
|
||||
'oozie.service.JPAService.jdbc.username': 'oozie',
|
||||
'oozie.service.JPAService.jdbc.password': u.get_oozie_password(
|
||||
cluster)
|
||||
}
|
@ -1,48 +0,0 @@
|
||||
# Copyright (c) 2015 Mirantis Inc.
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
|
||||
# implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
from sahara.plugins import recommendations_utils as ru
|
||||
|
||||
|
||||
def recommend_configs(cluster, plugin_configs, scaling):
|
||||
yarn_configs = [
|
||||
'yarn.nodemanager.resource.memory-mb',
|
||||
'yarn.scheduler.minimum-allocation-mb',
|
||||
'yarn.scheduler.maximum-allocation-mb',
|
||||
'yarn.nodemanager.vmem-check-enabled',
|
||||
]
|
||||
mapred_configs = [
|
||||
'yarn.app.mapreduce.am.resource.mb',
|
||||
'yarn.app.mapreduce.am.command-opts',
|
||||
'mapreduce.map.memory.mb',
|
||||
'mapreduce.reduce.memory.mb',
|
||||
'mapreduce.map.java.opts',
|
||||
'mapreduce.reduce.java.opts',
|
||||
'mapreduce.task.io.sort.mb',
|
||||
]
|
||||
configs_to_configure = {
|
||||
'cluster_configs': {
|
||||
'dfs.replication': ('HDFS', 'dfs.replication')
|
||||
},
|
||||
'node_configs': {
|
||||
}
|
||||
}
|
||||
for mapr in mapred_configs:
|
||||
configs_to_configure['node_configs'][mapr] = ("MapReduce", mapr)
|
||||
for yarn in yarn_configs:
|
||||
configs_to_configure['node_configs'][yarn] = ('YARN', yarn)
|
||||
provider = ru.HadoopAutoConfigsProvider(
|
||||
configs_to_configure, plugin_configs, cluster, scaling)
|
||||
provider.apply_recommended_configs()
|
@ -1,5 +0,0 @@
|
||||
create database oozie;
|
||||
grant all privileges on oozie.* to 'oozie'@'localhost' identified by 'password';
|
||||
grant all privileges on oozie.* to 'oozie'@'%' identified by 'password';
|
||||
flush privileges;
|
||||
exit
|
@ -1,33 +0,0 @@
|
||||
#!/bin/bash
|
||||
|
||||
# change permission to config
|
||||
hadoop_configs=('core-site.xml' 'hdfs-site.xml' 'mapred-site.xml' 'yarn-site.xml')
|
||||
for conf in "${{hadoop_configs[@]}}"
|
||||
do
|
||||
chown -R {hadoop_group}:{hadoop_user} {hadoop_conf_dir}/$conf
|
||||
done
|
||||
|
||||
chown -R {hadoop_group}:{hadoop_user} {oozie_conf_dir}/oozie-site.xml
|
||||
|
||||
# create dirs for hdfs and mapreduce service
|
||||
dirs=({hadoop_name_dirs} {hadoop_data_dirs} {hadoop_log_dir} {hadoop_secure_dn_log_dir} {yarn_log_dir})
|
||||
for dir in "${{dirs[@]}}"
|
||||
do
|
||||
mkdir -p $dir
|
||||
chown -R {hadoop_group}:{hadoop_user} $dir
|
||||
done
|
||||
|
||||
# change hadoop log dir
|
||||
sed -i "s,\#export HADOOP_LOG_DIR=.*,export HADOOP_LOG_DIR={hadoop_log_dir}," {hadoop_conf_dir}/hadoop-env.sh
|
||||
sed -i "s,export HADOOP_SECURE_DN_LOG_DIR=.*,export HADOOP_SECURE_DN_LOG_DIR={hadoop_secure_dn_log_dir}," {hadoop_conf_dir}/hadoop-env.sh
|
||||
|
||||
# change yarn log dir
|
||||
sed -i "s,YARN_LOG_DIR=.*,YARN_LOG_DIR={yarn_log_dir}," {hadoop_conf_dir}/yarn-env.sh
|
||||
|
||||
# prepare scaling files
|
||||
sc_all_files=('dn-include' 'nm-include' 'dn-exclude' 'nm-exclude')
|
||||
for file in "${{sc_all_files[@]}}"
|
||||
do
|
||||
touch {hadoop_conf_dir}/$file
|
||||
chown {hadoop_group}:{hadoop_user} {hadoop_conf_dir}/$file
|
||||
done
|
@ -1,2 +0,0 @@
|
||||
# Cleans up old Spark job directories once per hour.
|
||||
0 * * * * root /etc/hadoop/tmp-cleanup.sh
|
@ -1,48 +0,0 @@
|
||||
#!/bin/sh
|
||||
|
||||
MINIMUM_CLEANUP_MEGABYTES={minimum_cleanup_megabytes}
|
||||
MINIMUM_CLEANUP_SECONDS={minimum_cleanup_seconds}
|
||||
MAXIMUM_CLEANUP_SECONDS={maximum_cleanup_seconds}
|
||||
|
||||
CURRENT_TIMESTAMP=`date +%s`
|
||||
POSSIBLE_CLEANUP_THRESHOLD=$(($CURRENT_TIMESTAMP - $MINIMUM_CLEANUP_SECONDS))
|
||||
DEFINITE_CLEANUP_THRESHOLD=$(($CURRENT_TIMESTAMP - $MAXIMUM_CLEANUP_SECONDS))
|
||||
|
||||
unset MAY_DELETE
|
||||
unset WILL_DELETE
|
||||
|
||||
if [ ! -d /tmp/spark-edp ]
|
||||
then
|
||||
exit 0
|
||||
fi
|
||||
|
||||
cd /tmp/spark-edp
|
||||
for JOB in $(find . -maxdepth 1 -mindepth 1 -type d -printf '%f\n')
|
||||
do
|
||||
for EXECUTION in $(find $JOB -maxdepth 1 -mindepth 1 -type d -printf '%f\n')
|
||||
do
|
||||
TIMESTAMP=`stat $JOB/$EXECUTION --printf '%Y'`
|
||||
if [[ $TIMESTAMP -lt $DEFINITE_CLEANUP_THRESHOLD ]]
|
||||
then
|
||||
WILL_DELETE="$WILL_DELETE $JOB/$EXECUTION"
|
||||
else
|
||||
if [[ $TIMESTAMP -lt $POSSIBLE_CLEANUP_THRESHOLD ]]
|
||||
then
|
||||
MAY_DELETE="$MAY_DELETE $JOB/$EXECUTION"
|
||||
fi
|
||||
fi
|
||||
done
|
||||
done
|
||||
|
||||
for EXECUTION in $WILL_DELETE
|
||||
do
|
||||
rm -Rf $EXECUTION
|
||||
done
|
||||
|
||||
for EXECUTION in $(ls $MAY_DELETE -trd)
|
||||
do
|
||||
if [[ `du -s -BM | grep -o '[0-9]\+'` -le $MINIMUM_CLEANUP_MEGABYTES ]]; then
|
||||
break
|
||||
fi
|
||||
rm -Rf $EXECUTION
|
||||
done
|
@ -1,21 +0,0 @@
|
||||
#!/bin/bash
|
||||
HADOOP_CONF=/opt/hadoop/etc/hadoop
|
||||
|
||||
while [ $# -gt 0 ] ; do
|
||||
nodeArg=$1
|
||||
exec< ${HADOOP_CONF}/topology.data
|
||||
result=""
|
||||
while read line ; do
|
||||
ar=( $line )
|
||||
if [ "${ar[0]}" = "$nodeArg" ] ; then
|
||||
result="${ar[1]}"
|
||||
fi
|
||||
done
|
||||
shift
|
||||
if [ -z "$result" ] ; then
|
||||
echo -n "/default/rack "
|
||||
else
|
||||
echo -n "$result "
|
||||
fi
|
||||
done
|
||||
|
@ -1,29 +0,0 @@
|
||||
# The number of milliseconds of each tick
|
||||
tickTime={ticktime}
|
||||
# The number of ticks that the initial
|
||||
# synchronization phase can take
|
||||
initLimit={initlimit}
|
||||
# The number of ticks that can pass between
|
||||
# sending a request and getting an acknowledgement
|
||||
syncLimit={synclimit}
|
||||
# the directory where the snapshot is stored.
|
||||
# do not use /tmp for storage, /tmp here is just
|
||||
# example sakes.
|
||||
dataDir=/var/zookeeper
|
||||
# the port at which the clients will connect
|
||||
clientPort=2181
|
||||
# the maximum number of client connections.
|
||||
# increase this if you need to handle more clients
|
||||
#maxClientCnxns=60
|
||||
#
|
||||
# Be sure to read the maintenance section of the
|
||||
# administrator guide before turning on autopurge.
|
||||
#
|
||||
# http://zookeeper.apache.org/doc/current/zookeeperAdmin.html#sc_maintenance
|
||||
#
|
||||
# The number of snapshots to retain in dataDir
|
||||
#autopurge.snapRetainCount=3
|
||||
# Purge task interval in hours
|
||||
# Set to "0" to disable auto purge feature
|
||||
#autopurge.purgeInterval=1
|
||||
|
@ -1,293 +0,0 @@
|
||||
# Copyright (c) 2014 Mirantis Inc.
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
|
||||
# implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
import os
|
||||
|
||||
from oslo_log import log as logging
|
||||
|
||||
from sahara.plugins import context
|
||||
from sahara.plugins import edp
|
||||
from sahara.plugins import utils
|
||||
from sahara_plugin_vanilla.i18n import _
|
||||
from sahara_plugin_vanilla.plugins.vanilla.hadoop2 import config_helper
|
||||
from sahara_plugin_vanilla.plugins.vanilla.hadoop2 import oozie_helper
|
||||
from sahara_plugin_vanilla.plugins.vanilla.hadoop2 import utils as u
|
||||
from sahara_plugin_vanilla.plugins.vanilla import utils as vu
|
||||
|
||||
LOG = logging.getLogger(__name__)
|
||||
|
||||
|
||||
def start_dn_nm_processes(instances):
|
||||
filternames = ['datanode', 'nodemanager']
|
||||
instances = utils.instances_with_services(instances, filternames)
|
||||
|
||||
if len(instances) == 0:
|
||||
return
|
||||
|
||||
utils.add_provisioning_step(
|
||||
instances[0].cluster_id,
|
||||
utils.start_process_event_message("DataNodes, NodeManagers"),
|
||||
len(instances))
|
||||
|
||||
with context.PluginsThreadGroup() as tg:
|
||||
for instance in instances:
|
||||
with context.set_current_instance_id(instance.instance_id):
|
||||
processes = set(instance.node_group.node_processes)
|
||||
processes = processes.intersection(filternames)
|
||||
tg.spawn('vanilla-start-processes-%s' % instance.instance_name,
|
||||
_start_processes, instance, list(processes))
|
||||
|
||||
|
||||
@utils.event_wrapper(True)
|
||||
def _start_processes(instance, processes):
|
||||
with instance.remote() as r:
|
||||
if 'datanode' in processes:
|
||||
r.execute_command(
|
||||
'sudo su - -c "hadoop-daemon.sh start datanode" hadoop')
|
||||
if 'nodemanager' in processes:
|
||||
r.execute_command(
|
||||
'sudo su - -c "yarn-daemon.sh start nodemanager" hadoop')
|
||||
|
||||
|
||||
def start_hadoop_process(instance, process):
|
||||
instance.remote().execute_command(
|
||||
'sudo su - -c "hadoop-daemon.sh start %s" hadoop' % process)
|
||||
|
||||
|
||||
def start_yarn_process(instance, process):
|
||||
instance.remote().execute_command(
|
||||
'sudo su - -c "yarn-daemon.sh start %s" hadoop' % process)
|
||||
|
||||
|
||||
@utils.event_wrapper(
|
||||
True, step=utils.start_process_event_message("HistoryServer"))
|
||||
def start_historyserver(instance):
|
||||
instance.remote().execute_command(
|
||||
'sudo su - -c "mr-jobhistory-daemon.sh start historyserver" hadoop')
|
||||
|
||||
|
||||
@utils.event_wrapper(True, step=utils.start_process_event_message("Oozie"))
|
||||
def start_oozie_process(pctx, instance):
|
||||
with context.set_current_instance_id(instance.instance_id):
|
||||
with instance.remote() as r:
|
||||
if config_helper.is_mysql_enabled(pctx, instance.cluster):
|
||||
_start_mysql(r)
|
||||
LOG.debug("Creating Oozie DB Schema")
|
||||
sql_script = utils.get_file_text(
|
||||
'plugins/vanilla/hadoop2/resources/create_oozie_db.sql',
|
||||
'sahara_plugin_vanilla')
|
||||
|
||||
password = oozie_helper.get_oozie_mysql_configs(
|
||||
instance.cluster)[
|
||||
'oozie.service.JPAService.jdbc.password']
|
||||
sql_script = sql_script.replace("password", password)
|
||||
|
||||
script_location = "create_oozie_db.sql"
|
||||
r.write_file_to(script_location, sql_script)
|
||||
r.execute_command('mysql -u root < %(script_location)s && '
|
||||
'rm %(script_location)s' %
|
||||
{"script_location": script_location})
|
||||
|
||||
_oozie_share_lib(r)
|
||||
_start_oozie(r)
|
||||
|
||||
|
||||
@utils.event_wrapper(
|
||||
True, step=utils.start_process_event_message("Spark History Server"))
|
||||
def start_spark_history_server(master):
|
||||
sp_home = config_helper.get_spark_home(master.cluster)
|
||||
with context.set_current_instance_id(master.instance_id):
|
||||
with master.remote() as r:
|
||||
r.execute_command('sudo su - -c "bash %s" hadoop' % os.path.join(
|
||||
sp_home, 'sbin/start-history-server.sh'))
|
||||
|
||||
|
||||
def start_zk_server(instances):
|
||||
utils.add_provisioning_step(
|
||||
instances[0].cluster_id,
|
||||
utils.start_process_event_message("ZooKeeper"),
|
||||
len(instances))
|
||||
|
||||
with context.PluginsThreadGroup() as tg:
|
||||
for instance in instances:
|
||||
with context.set_current_instance_id(instance.instance_id):
|
||||
tg.spawn('ZK-start-processes-%s' % instance.instance_name,
|
||||
_start_zk_processes, instance, 'start')
|
||||
|
||||
|
||||
def refresh_zk_servers(cluster, to_delete_instances=None):
|
||||
instances = vu.get_zk_servers(cluster)
|
||||
if to_delete_instances:
|
||||
for instance in to_delete_instances:
|
||||
if instance in instances:
|
||||
instances.remove(instance)
|
||||
|
||||
utils.add_provisioning_step(
|
||||
cluster.id,
|
||||
utils.start_process_event_message("ZooKeeper"),
|
||||
len(instances))
|
||||
|
||||
with context.PluginsThreadGroup() as tg:
|
||||
for instance in instances:
|
||||
with context.set_current_instance_id(instance.instance_id):
|
||||
tg.spawn('ZK-restart-processes-%s' % instance.instance_name,
|
||||
_start_zk_processes, instance, 'restart')
|
||||
|
||||
|
||||
@utils.event_wrapper(True)
|
||||
def _start_zk_processes(instance, operation):
|
||||
with instance.remote() as r:
|
||||
r.execute_command(
|
||||
'sudo su - -c "bash /opt/zookeeper/bin/zkServer.sh %s"'
|
||||
' hadoop' % operation)
|
||||
|
||||
|
||||
def format_namenode(instance):
|
||||
instance.remote().execute_command(
|
||||
'sudo su - -c "hdfs namenode -format" hadoop')
|
||||
|
||||
|
||||
@utils.event_wrapper(
|
||||
True,
|
||||
step=utils.start_process_event_message("Oozie"), param=('cluster', 0))
|
||||
def refresh_hadoop_nodes(cluster):
|
||||
nn = vu.get_namenode(cluster)
|
||||
nn.remote().execute_command(
|
||||
'sudo su - -c "hdfs dfsadmin -refreshNodes" hadoop')
|
||||
|
||||
|
||||
@utils.event_wrapper(
|
||||
True, step=_("Refresh %s nodes") % "YARN", param=('cluster', 0))
|
||||
def refresh_yarn_nodes(cluster):
|
||||
rm = vu.get_resourcemanager(cluster)
|
||||
rm.remote().execute_command(
|
||||
'sudo su - -c "yarn rmadmin -refreshNodes" hadoop')
|
||||
|
||||
|
||||
def _oozie_share_lib(remote):
|
||||
LOG.debug("Sharing Oozie libs")
|
||||
# remote.execute_command('sudo su - -c "/opt/oozie/bin/oozie-setup.sh '
|
||||
# 'sharelib create -fs hdfs://%s:8020" hadoop'
|
||||
# % nn_hostname)
|
||||
|
||||
# TODO(alazarev) return 'oozie-setup.sh sharelib create' back
|
||||
# when #1262023 is resolved
|
||||
|
||||
remote.execute_command(
|
||||
'sudo su - -c "mkdir /tmp/oozielib && '
|
||||
'tar zxf /opt/oozie/oozie-sharelib-*.tar.gz -C '
|
||||
'/tmp/oozielib && '
|
||||
'hadoop fs -mkdir /user && '
|
||||
'hadoop fs -mkdir /user/hadoop && '
|
||||
'hadoop fs -put /tmp/oozielib/share /user/hadoop/ && '
|
||||
'rm -rf /tmp/oozielib" hadoop')
|
||||
|
||||
LOG.debug("Creating sqlfile for Oozie")
|
||||
remote.execute_command('sudo su - -c "/opt/oozie/bin/ooziedb.sh '
|
||||
'create -sqlfile oozie.sql '
|
||||
'-run Validate DB Connection" hadoop')
|
||||
|
||||
|
||||
def _start_mysql(remote):
|
||||
LOG.debug("Starting mysql")
|
||||
remote.execute_command('/opt/start-mysql.sh')
|
||||
|
||||
|
||||
def _start_oozie(remote):
|
||||
remote.execute_command(
|
||||
'sudo su - -c "/opt/oozie/bin/oozied.sh start" hadoop')
|
||||
|
||||
|
||||
@utils.event_wrapper(
|
||||
True, step=_("Await %s start up") % "DataNodes", param=('cluster', 0))
|
||||
def await_datanodes(cluster):
|
||||
datanodes_count = len(vu.get_datanodes(cluster))
|
||||
if datanodes_count < 1:
|
||||
return
|
||||
|
||||
l_message = _("Waiting on %s datanodes to start up") % datanodes_count
|
||||
with vu.get_namenode(cluster).remote() as r:
|
||||
utils.plugin_option_poll(
|
||||
cluster, _check_datanodes_count,
|
||||
config_helper.DATANODES_STARTUP_TIMEOUT, l_message, 1, {
|
||||
'remote': r, 'count': datanodes_count})
|
||||
|
||||
|
||||
def _check_datanodes_count(remote, count):
|
||||
if count < 1:
|
||||
return True
|
||||
|
||||
LOG.debug("Checking datanode count")
|
||||
exit_code, stdout = remote.execute_command(
|
||||
'sudo su -lc "hdfs dfsadmin -report" hadoop | '
|
||||
r'grep \'Live datanodes\|Datanodes available:\' | '
|
||||
r'grep -o \'[0-9]\+\' | head -n 1')
|
||||
LOG.debug("Datanode count='{count}'".format(count=stdout.rstrip()))
|
||||
|
||||
return exit_code == 0 and stdout and int(stdout) == count
|
||||
|
||||
|
||||
def _hive_create_warehouse_dir(remote):
|
||||
LOG.debug("Creating Hive warehouse dir")
|
||||
remote.execute_command("sudo su - -c 'hadoop fs -mkdir -p "
|
||||
"/user/hive/warehouse' hadoop")
|
||||
|
||||
|
||||
def _hive_copy_shared_conf(remote, dest):
|
||||
LOG.debug("Copying shared Hive conf")
|
||||
dirname, filename = os.path.split(dest)
|
||||
remote.execute_command(
|
||||
"sudo su - -c 'hadoop fs -mkdir -p %s && "
|
||||
"hadoop fs -put /opt/hive/conf/hive-site.xml "
|
||||
"%s' hadoop" % (dirname, dest))
|
||||
|
||||
|
||||
def _hive_create_db(remote):
|
||||
LOG.debug("Creating Hive metastore db")
|
||||
remote.execute_command("mysql -u root < /tmp/create_hive_db.sql")
|
||||
|
||||
|
||||
def _hive_metastore_start(remote):
|
||||
LOG.debug("Starting Hive Metastore Server")
|
||||
remote.execute_command("sudo su - -c 'nohup /opt/hive/bin/hive"
|
||||
" --service metastore > /dev/null &' hadoop")
|
||||
|
||||
|
||||
@utils.event_wrapper(
|
||||
True, step=utils.start_process_event_message("HiveServer"))
|
||||
def start_hiveserver_process(pctx, instance):
|
||||
with context.set_current_instance_id(instance.instance_id):
|
||||
with instance.remote() as r:
|
||||
_hive_create_warehouse_dir(r)
|
||||
_hive_copy_shared_conf(
|
||||
r, edp.get_hive_shared_conf_path('hadoop'))
|
||||
|
||||
if config_helper.is_mysql_enabled(pctx, instance.cluster):
|
||||
oozie = vu.get_oozie(instance.node_group.cluster)
|
||||
if not oozie or instance.hostname() != oozie.hostname():
|
||||
_start_mysql(r)
|
||||
|
||||
version = instance.cluster.hadoop_version
|
||||
sql_script = utils.get_file_text(
|
||||
'plugins/vanilla/v{}/resources/create_hive_db.sql'.format(
|
||||
version.replace('.', '_')), 'sahara_plugin_vanilla')
|
||||
|
||||
sql_script = sql_script.replace(
|
||||
'{{password}}', u.get_hive_password(instance.cluster))
|
||||
r.write_file_to('/tmp/create_hive_db.sql', sql_script)
|
||||
_hive_create_db(r)
|
||||
_hive_metastore_start(r)
|
||||
LOG.info("Hive Metastore server at {host} has been "
|
||||
"started".format(host=instance.hostname()))
|
@ -1,145 +0,0 @@
|
||||
# Copyright (c) 2014 Mirantis Inc.
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
|
||||
# implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
from sahara.plugins import swift_helper
|
||||
from sahara.plugins import utils
|
||||
from sahara_plugin_vanilla.i18n import _
|
||||
from sahara_plugin_vanilla.plugins.vanilla.hadoop2 import config
|
||||
from sahara_plugin_vanilla.plugins.vanilla.hadoop2 import config_helper
|
||||
from sahara_plugin_vanilla.plugins.vanilla.hadoop2 import run_scripts as run
|
||||
from sahara_plugin_vanilla.plugins.vanilla.hadoop2 import utils as pu
|
||||
from sahara_plugin_vanilla.plugins.vanilla import utils as vu
|
||||
|
||||
HADOOP_CONF_DIR = config.HADOOP_CONF_DIR
|
||||
|
||||
|
||||
def scale_cluster(pctx, cluster, instances):
|
||||
config.configure_instances(pctx, instances)
|
||||
_update_include_files(cluster)
|
||||
run.refresh_hadoop_nodes(cluster)
|
||||
rm = vu.get_resourcemanager(cluster)
|
||||
if rm:
|
||||
run.refresh_yarn_nodes(cluster)
|
||||
|
||||
config.configure_topology_data(pctx, cluster)
|
||||
run.start_dn_nm_processes(instances)
|
||||
swift_helper.install_ssl_certs(instances)
|
||||
config.configure_zookeeper(cluster)
|
||||
run.refresh_zk_servers(cluster)
|
||||
|
||||
|
||||
def _get_instances_with_service(instances, service):
|
||||
return [instance for instance in instances
|
||||
if service in instance.node_group.node_processes]
|
||||
|
||||
|
||||
@utils.event_wrapper(
|
||||
True, step=_("Update include files"), param=('cluster', 0))
|
||||
def _update_include_files(cluster, dec_instances=None):
|
||||
dec_instances = dec_instances or []
|
||||
dec_instances_ids = [instance.id for instance in dec_instances]
|
||||
|
||||
instances = utils.get_instances(cluster)
|
||||
|
||||
inst_filter = lambda inst: inst.id not in dec_instances_ids
|
||||
|
||||
datanodes = filter(inst_filter, vu.get_datanodes(cluster))
|
||||
nodemanagers = filter(inst_filter, vu.get_nodemanagers(cluster))
|
||||
dn_hosts = utils.generate_fqdn_host_names(datanodes)
|
||||
nm_hosts = utils.generate_fqdn_host_names(nodemanagers)
|
||||
for instance in instances:
|
||||
with instance.remote() as r:
|
||||
r.execute_command(
|
||||
'sudo su - -c "echo \'%s\' > %s/dn-include" hadoop' % (
|
||||
dn_hosts, HADOOP_CONF_DIR))
|
||||
r.execute_command(
|
||||
'sudo su - -c "echo \'%s\' > %s/nm-include" hadoop' % (
|
||||
nm_hosts, HADOOP_CONF_DIR))
|
||||
|
||||
|
||||
def decommission_nodes(pctx, cluster, instances):
|
||||
datanodes = _get_instances_with_service(instances, 'datanode')
|
||||
nodemanagers = _get_instances_with_service(instances, 'nodemanager')
|
||||
_update_exclude_files(cluster, instances)
|
||||
|
||||
run.refresh_hadoop_nodes(cluster)
|
||||
rm = vu.get_resourcemanager(cluster)
|
||||
if rm:
|
||||
run.refresh_yarn_nodes(cluster)
|
||||
|
||||
_check_nodemanagers_decommission(cluster, nodemanagers)
|
||||
_check_datanodes_decommission(cluster, datanodes)
|
||||
|
||||
_update_include_files(cluster, instances)
|
||||
_clear_exclude_files(cluster)
|
||||
run.refresh_hadoop_nodes(cluster)
|
||||
|
||||
config.configure_topology_data(pctx, cluster)
|
||||
config.configure_zookeeper(cluster, instances)
|
||||
# TODO(shuyingya):should invent a way to lastly restart the leader node
|
||||
run.refresh_zk_servers(cluster, instances)
|
||||
|
||||
|
||||
def _update_exclude_files(cluster, instances):
|
||||
datanodes = _get_instances_with_service(instances, 'datanode')
|
||||
nodemanagers = _get_instances_with_service(instances, 'nodemanager')
|
||||
dn_hosts = utils.generate_fqdn_host_names(datanodes)
|
||||
nm_hosts = utils.generate_fqdn_host_names(nodemanagers)
|
||||
for instance in utils.get_instances(cluster):
|
||||
with instance.remote() as r:
|
||||
r.execute_command(
|
||||
'sudo su - -c "echo \'%s\' > %s/dn-exclude" hadoop' % (
|
||||
dn_hosts, HADOOP_CONF_DIR))
|
||||
r.execute_command(
|
||||
'sudo su - -c "echo \'%s\' > %s/nm-exclude" hadoop' % (
|
||||
nm_hosts, HADOOP_CONF_DIR))
|
||||
|
||||
|
||||
def _clear_exclude_files(cluster):
|
||||
for instance in utils.get_instances(cluster):
|
||||
with instance.remote() as r:
|
||||
r.execute_command(
|
||||
'sudo su - -c "echo > %s/dn-exclude" hadoop' % HADOOP_CONF_DIR)
|
||||
r.execute_command(
|
||||
'sudo su - -c "echo > %s/nm-exclude" hadoop' % HADOOP_CONF_DIR)
|
||||
|
||||
|
||||
def is_decommissioned(cluster, check_func, instances):
|
||||
statuses = check_func(cluster)
|
||||
for instance in instances:
|
||||
if statuses[instance.fqdn()] != 'decommissioned':
|
||||
return False
|
||||
return True
|
||||
|
||||
|
||||
def _check_decommission(cluster, instances, check_func, option):
|
||||
utils.plugin_option_poll(
|
||||
cluster, is_decommissioned, option, _("Wait for decommissioning"),
|
||||
5, {'cluster': cluster, 'check_func': check_func,
|
||||
'instances': instances})
|
||||
|
||||
|
||||
@utils.event_wrapper(
|
||||
True, step=_("Decommission %s") % "NodeManagers", param=('cluster', 0))
|
||||
def _check_nodemanagers_decommission(cluster, instances):
|
||||
_check_decommission(cluster, instances, pu.get_nodemanagers_status,
|
||||
config_helper.NODEMANAGERS_DECOMMISSIONING_TIMEOUT)
|
||||
|
||||
|
||||
@utils.event_wrapper(
|
||||
True, step=_("Decommission %s") % "DataNodes", param=('cluster', 0))
|
||||
def _check_datanodes_decommission(cluster, instances):
|
||||
_check_decommission(cluster, instances, pu.get_datanodes_status,
|
||||
config_helper.DATANODES_DECOMMISSIONING_TIMEOUT)
|
@ -1,84 +0,0 @@
|
||||
# Copyright (c) 2015 Mirantis Inc.
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
|
||||
# implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
from sahara.plugins import utils
|
||||
from sahara_plugin_vanilla.plugins.vanilla.hadoop2 import run_scripts as run
|
||||
from sahara_plugin_vanilla.plugins.vanilla import utils as vu
|
||||
|
||||
|
||||
def start_namenode(cluster):
|
||||
nn = vu.get_namenode(cluster)
|
||||
_start_namenode(nn)
|
||||
|
||||
|
||||
@utils.event_wrapper(
|
||||
True, step=utils.start_process_event_message('NameNode'))
|
||||
def _start_namenode(nn):
|
||||
run.format_namenode(nn)
|
||||
run.start_hadoop_process(nn, 'namenode')
|
||||
|
||||
|
||||
def start_secondarynamenode(cluster):
|
||||
snn = vu.get_secondarynamenode(cluster)
|
||||
if snn:
|
||||
_start_secondarynamenode(snn)
|
||||
|
||||
|
||||
@utils.event_wrapper(
|
||||
True, step=utils.start_process_event_message("SecondaryNameNodes"))
|
||||
def _start_secondarynamenode(snn):
|
||||
run.start_hadoop_process(snn, 'secondarynamenode')
|
||||
|
||||
|
||||
def start_resourcemanager(cluster):
|
||||
rm = vu.get_resourcemanager(cluster)
|
||||
if rm:
|
||||
_start_resourcemanager(rm)
|
||||
|
||||
|
||||
@utils.event_wrapper(
|
||||
True, step=utils.start_process_event_message('ResourceManager'))
|
||||
def _start_resourcemanager(snn):
|
||||
run.start_yarn_process(snn, 'resourcemanager')
|
||||
|
||||
|
||||
def start_historyserver(cluster):
|
||||
hs = vu.get_historyserver(cluster)
|
||||
if hs:
|
||||
run.start_historyserver(hs)
|
||||
|
||||
|
||||
def start_oozie(pctx, cluster):
|
||||
oo = vu.get_oozie(cluster)
|
||||
if oo:
|
||||
run.start_oozie_process(pctx, oo)
|
||||
|
||||
|
||||
def start_hiveserver(pctx, cluster):
|
||||
hiveserver = vu.get_hiveserver(cluster)
|
||||
if hiveserver:
|
||||
run.start_hiveserver_process(pctx, hiveserver)
|
||||
|
||||
|
||||
def start_spark(cluster):
|
||||
spark = vu.get_spark_history_server(cluster)
|
||||
if spark:
|
||||
run.start_spark_history_server(spark)
|
||||
|
||||
|
||||
def start_zookeeper(cluster):
|
||||
zk_servers = vu.get_zk_servers(cluster)
|
||||
if zk_servers:
|
||||
run.start_zk_server(zk_servers)
|
@ -1,88 +0,0 @@
|
||||
# Copyright (c) 2014 Mirantis Inc.
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
|
||||
# implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
import re
|
||||
|
||||
from oslo_log import log as logging
|
||||
|
||||
from sahara.plugins import castellan_utils as castellan
|
||||
from sahara.plugins import conductor
|
||||
from sahara.plugins import context
|
||||
from sahara_plugin_vanilla.plugins.vanilla import utils as u
|
||||
|
||||
|
||||
LOG = logging.getLogger(__name__)
|
||||
|
||||
|
||||
def get_datanodes_status(cluster):
|
||||
statuses = {}
|
||||
namenode = u.get_namenode(cluster)
|
||||
status_regexp = r'^Hostname: (.*)\nDecommission Status : (.*)$'
|
||||
matcher = re.compile(status_regexp, re.MULTILINE)
|
||||
dfs_report = namenode.remote().execute_command(
|
||||
'sudo su - -c "hdfs dfsadmin -report" hadoop')[1]
|
||||
|
||||
for host, status in matcher.findall(dfs_report):
|
||||
statuses[host] = status.lower()
|
||||
|
||||
return statuses
|
||||
|
||||
|
||||
def get_nodemanagers_status(cluster):
|
||||
statuses = {}
|
||||
resourcemanager = u.get_resourcemanager(cluster)
|
||||
status_regexp = r'^(\S+):\d+\s+(\w+)'
|
||||
matcher = re.compile(status_regexp, re.MULTILINE)
|
||||
yarn_report = resourcemanager.remote().execute_command(
|
||||
'sudo su - -c "yarn node -all -list" hadoop')[1]
|
||||
|
||||
for host, status in matcher.findall(yarn_report):
|
||||
statuses[host] = status.lower()
|
||||
|
||||
return statuses
|
||||
|
||||
|
||||
def get_oozie_password(cluster):
|
||||
cluster = conductor.cluster_get(context.ctx(), cluster)
|
||||
extra = cluster.extra.to_dict()
|
||||
if 'oozie_pass_id' not in extra:
|
||||
extra['oozie_pass_id'] = u.generate_random_password()
|
||||
conductor.cluster_update(context.ctx(), cluster, {'extra': extra})
|
||||
return castellan.get_secret(extra['oozie_pass_id'])
|
||||
|
||||
|
||||
def delete_oozie_password(cluster):
|
||||
extra = cluster.extra.to_dict()
|
||||
if 'oozie_pass_id' in extra:
|
||||
castellan.delete_secret(extra['oozie_pass_id'])
|
||||
else:
|
||||
LOG.warning("Cluster hasn't Oozie password")
|
||||
|
||||
|
||||
def get_hive_password(cluster):
|
||||
cluster = conductor.cluster_get(context.ctx(), cluster)
|
||||
extra = cluster.extra.to_dict()
|
||||
if 'hive_pass_id' not in extra:
|
||||
extra['hive_pass_id'] = u.generate_random_password()
|
||||
conductor.cluster_update(context.ctx(), cluster, {'extra': extra})
|
||||
return castellan.get_secret(extra['hive_pass_id'])
|
||||
|
||||
|
||||
def delete_hive_password(cluster):
|
||||
extra = cluster.extra.to_dict()
|
||||
if 'hive_pass_id' in extra:
|
||||
castellan.delete_secret(extra['hive_pass_id'])
|
||||
else:
|
||||
LOG.warning("Cluster hasn't hive password")
|
@ -1,157 +0,0 @@
|
||||
# Copyright (c) 2014 Mirantis Inc.
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
|
||||
# implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
from sahara.plugins import exceptions as ex
|
||||
from sahara.plugins import utils as u
|
||||
from sahara_plugin_vanilla.i18n import _
|
||||
from sahara_plugin_vanilla.plugins.vanilla.hadoop2 import config_helper as cu
|
||||
from sahara_plugin_vanilla.plugins.vanilla import utils as vu
|
||||
|
||||
|
||||
def validate_cluster_creating(pctx, cluster):
|
||||
nn_count = _get_inst_count(cluster, 'namenode')
|
||||
if nn_count != 1:
|
||||
raise ex.InvalidComponentCountException('namenode', 1, nn_count)
|
||||
|
||||
snn_count = _get_inst_count(cluster, 'secondarynamenode')
|
||||
if snn_count > 1:
|
||||
raise ex.InvalidComponentCountException('secondarynamenode',
|
||||
_('0 or 1'), snn_count)
|
||||
|
||||
rm_count = _get_inst_count(cluster, 'resourcemanager')
|
||||
if rm_count > 1:
|
||||
raise ex.InvalidComponentCountException('resourcemanager', _('0 or 1'),
|
||||
rm_count)
|
||||
|
||||
hs_count = _get_inst_count(cluster, 'historyserver')
|
||||
if hs_count > 1:
|
||||
raise ex.InvalidComponentCountException('historyserver', _('0 or 1'),
|
||||
hs_count)
|
||||
|
||||
nm_count = _get_inst_count(cluster, 'nodemanager')
|
||||
if rm_count == 0:
|
||||
if nm_count > 0:
|
||||
raise ex.RequiredServiceMissingException('resourcemanager',
|
||||
required_by='nodemanager')
|
||||
|
||||
oo_count = _get_inst_count(cluster, 'oozie')
|
||||
dn_count = _get_inst_count(cluster, 'datanode')
|
||||
if oo_count > 1:
|
||||
raise ex.InvalidComponentCountException('oozie', _('0 or 1'), oo_count)
|
||||
|
||||
if oo_count == 1:
|
||||
if dn_count < 1:
|
||||
raise ex.RequiredServiceMissingException('datanode',
|
||||
required_by='oozie')
|
||||
|
||||
if nm_count < 1:
|
||||
raise ex.RequiredServiceMissingException('nodemanager',
|
||||
required_by='oozie')
|
||||
|
||||
if hs_count != 1:
|
||||
raise ex.RequiredServiceMissingException('historyserver',
|
||||
required_by='oozie')
|
||||
|
||||
spark_hist_count = _get_inst_count(cluster, 'spark history server')
|
||||
if spark_hist_count > 1:
|
||||
raise ex.InvalidComponentCountException('spark history server',
|
||||
_('0 or 1'),
|
||||
spark_hist_count)
|
||||
|
||||
rep_factor = cu.get_config_value(pctx, 'HDFS', 'dfs.replication', cluster)
|
||||
if dn_count < rep_factor:
|
||||
raise ex.InvalidComponentCountException(
|
||||
'datanode', rep_factor, dn_count, _('Number of datanodes must be '
|
||||
'not less than '
|
||||
'dfs.replication.'))
|
||||
|
||||
hive_count = _get_inst_count(cluster, 'hiveserver')
|
||||
if hive_count > 1:
|
||||
raise ex.InvalidComponentCountException('hive', _('0 or 1'),
|
||||
hive_count)
|
||||
|
||||
zk_count = _get_inst_count(cluster, 'zookeeper')
|
||||
if zk_count > 0 and (zk_count % 2) != 1:
|
||||
raise ex.InvalidComponentCountException(
|
||||
'zookeeper', _('odd'), zk_count, _('Number of zookeeper nodes '
|
||||
'should be odd.'))
|
||||
|
||||
|
||||
def validate_additional_ng_scaling(cluster, additional):
|
||||
rm = vu.get_resourcemanager(cluster)
|
||||
scalable_processes = _get_scalable_processes()
|
||||
|
||||
for ng_id in additional:
|
||||
ng = u.get_by_id(cluster.node_groups, ng_id)
|
||||
if not set(ng.node_processes).issubset(scalable_processes):
|
||||
msg = _("Vanilla plugin cannot scale nodegroup with processes: %s")
|
||||
raise ex.NodeGroupCannotBeScaled(ng.name,
|
||||
msg % ' '.join(ng.node_processes))
|
||||
|
||||
if not rm and 'nodemanager' in ng.node_processes:
|
||||
msg = _("Vanilla plugin cannot scale node group with processes "
|
||||
"which have no master-processes run in cluster")
|
||||
raise ex.NodeGroupCannotBeScaled(ng.name, msg)
|
||||
|
||||
|
||||
def validate_existing_ng_scaling(pctx, cluster, existing):
|
||||
scalable_processes = _get_scalable_processes()
|
||||
dn_to_delete = 0
|
||||
for ng in cluster.node_groups:
|
||||
if ng.id in existing:
|
||||
if ng.count > existing[ng.id] and "datanode" in ng.node_processes:
|
||||
dn_to_delete += ng.count - existing[ng.id]
|
||||
|
||||
if not set(ng.node_processes).issubset(scalable_processes):
|
||||
msg = _("Vanilla plugin cannot scale nodegroup "
|
||||
"with processes: %s")
|
||||
raise ex.NodeGroupCannotBeScaled(
|
||||
ng.name, msg % ' '.join(ng.node_processes))
|
||||
|
||||
dn_amount = len(vu.get_datanodes(cluster))
|
||||
rep_factor = cu.get_config_value(pctx, 'HDFS', 'dfs.replication', cluster)
|
||||
|
||||
if dn_to_delete > 0 and dn_amount - dn_to_delete < rep_factor:
|
||||
msg = _("Vanilla plugin cannot shrink cluster because it would be "
|
||||
"not enough nodes for replicas (replication factor is %s)")
|
||||
raise ex.ClusterCannotBeScaled(
|
||||
cluster.name, msg % rep_factor)
|
||||
|
||||
|
||||
def validate_zookeeper_node_count(zk_ng, existing, additional):
|
||||
zk_amount = 0
|
||||
for ng in zk_ng:
|
||||
if ng.id in existing:
|
||||
zk_amount += existing[ng.id]
|
||||
else:
|
||||
zk_amount += ng.count
|
||||
|
||||
for ng_id in additional:
|
||||
ng = u.get_by_id(zk_ng, ng_id)
|
||||
if "zookeeper" in ng.node_processes:
|
||||
zk_amount += ng.count
|
||||
|
||||
if (zk_amount % 2) != 1:
|
||||
msg = _("Vanilla plugin cannot scale cluster because it must keep"
|
||||
" zookeeper service in odd.")
|
||||
raise ex.ClusterCannotBeScaled(zk_ng[0].cluster.name, msg)
|
||||
|
||||
|
||||
def _get_scalable_processes():
|
||||
return ['datanode', 'nodemanager', 'zookeeper']
|
||||
|
||||
|
||||
def _get_inst_count(cluster, process):
|
||||
return sum([ng.count for ng in u.get_node_groups(cluster, process)])
|
@ -1,108 +0,0 @@
|
||||
# Copyright (c) 2013 Mirantis Inc.
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
|
||||
# implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
import copy
|
||||
|
||||
from sahara.plugins import provisioning as p
|
||||
from sahara_plugin_vanilla.i18n import _
|
||||
from sahara_plugin_vanilla.plugins.vanilla import versionfactory as vhf
|
||||
|
||||
|
||||
class VanillaProvider(p.ProvisioningPluginBase):
|
||||
def __init__(self):
|
||||
self.version_factory = vhf.VersionFactory.get_instance()
|
||||
|
||||
def get_description(self):
|
||||
return _('The Apache Vanilla plugin provides the ability to launch '
|
||||
'upstream Vanilla Apache Hadoop cluster without any '
|
||||
'management consoles. It can also deploy the Oozie '
|
||||
'component.')
|
||||
|
||||
def _get_version_handler(self, hadoop_version):
|
||||
return self.version_factory.get_version_handler(hadoop_version)
|
||||
|
||||
def get_node_processes(self, hadoop_version):
|
||||
return self._get_version_handler(hadoop_version).get_node_processes()
|
||||
|
||||
def get_labels(self):
|
||||
default = {'enabled': {'status': True}, 'stable': {'status': True}}
|
||||
result = {'plugin_labels': copy.deepcopy(default)}
|
||||
result['version_labels'] = {
|
||||
version: copy.deepcopy(default) for version in self.get_versions()
|
||||
}
|
||||
return result
|
||||
|
||||
def get_versions(self):
|
||||
return self.version_factory.get_versions()
|
||||
|
||||
def get_title(self):
|
||||
return "Vanilla Apache Hadoop"
|
||||
|
||||
def get_configs(self, hadoop_version):
|
||||
return self._get_version_handler(hadoop_version).get_plugin_configs()
|
||||
|
||||
def configure_cluster(self, cluster):
|
||||
return self._get_version_handler(
|
||||
cluster.hadoop_version).configure_cluster(cluster)
|
||||
|
||||
def start_cluster(self, cluster):
|
||||
return self._get_version_handler(
|
||||
cluster.hadoop_version).start_cluster(cluster)
|
||||
|
||||
def validate(self, cluster):
|
||||
return self._get_version_handler(
|
||||
cluster.hadoop_version).validate(cluster)
|
||||
|
||||
def scale_cluster(self, cluster, instances):
|
||||
return self._get_version_handler(
|
||||
cluster.hadoop_version).scale_cluster(cluster, instances)
|
||||
|
||||
def decommission_nodes(self, cluster, instances):
|
||||
return self._get_version_handler(
|
||||
cluster.hadoop_version).decommission_nodes(cluster, instances)
|
||||
|
||||
def validate_scaling(self, cluster, existing, additional):
|
||||
return self._get_version_handler(
|
||||
cluster.hadoop_version).validate_scaling(cluster, existing,
|
||||
additional)
|
||||
|
||||
def get_edp_engine(self, cluster, job_type):
|
||||
return self._get_version_handler(
|
||||
cluster.hadoop_version).get_edp_engine(cluster, job_type)
|
||||
|
||||
def get_edp_job_types(self, versions=None):
|
||||
res = {}
|
||||
for vers in self.version_factory.get_versions():
|
||||
if not versions or vers in versions:
|
||||
vh = self.version_factory.get_version_handler(vers)
|
||||
res[vers] = vh.get_edp_job_types()
|
||||
return res
|
||||
|
||||
def get_edp_config_hints(self, job_type, version):
|
||||
version_handler = (
|
||||
self.version_factory.get_version_handler(version))
|
||||
return version_handler.get_edp_config_hints(job_type)
|
||||
|
||||
def get_open_ports(self, node_group):
|
||||
return self._get_version_handler(
|
||||
node_group.cluster.hadoop_version).get_open_ports(node_group)
|
||||
|
||||
def on_terminate_cluster(self, cluster):
|
||||
return self._get_version_handler(
|
||||
cluster.hadoop_version).on_terminate_cluster(cluster)
|
||||
|
||||
def recommend_configs(self, cluster, scaling=False):
|
||||
return self._get_version_handler(
|
||||
cluster.hadoop_version).recommend_configs(cluster, scaling)
|
@ -1,69 +0,0 @@
|
||||
# Copyright (c) 2014 Mirantis Inc.
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
|
||||
# implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
|
||||
from oslo_utils import uuidutils
|
||||
|
||||
from sahara.plugins import castellan_utils as castellan
|
||||
from sahara.plugins import utils as u
|
||||
|
||||
|
||||
def get_namenode(cluster):
|
||||
return u.get_instance(cluster, "namenode")
|
||||
|
||||
|
||||
def get_resourcemanager(cluster):
|
||||
return u.get_instance(cluster, 'resourcemanager')
|
||||
|
||||
|
||||
def get_nodemanagers(cluster):
|
||||
return u.get_instances(cluster, 'nodemanager')
|
||||
|
||||
|
||||
def get_oozie(cluster):
|
||||
return u.get_instance(cluster, "oozie")
|
||||
|
||||
|
||||
def get_spark_history_server(cluster):
|
||||
return u.get_instance(cluster, "spark history server")
|
||||
|
||||
|
||||
def get_hiveserver(cluster):
|
||||
return u.get_instance(cluster, "hiveserver")
|
||||
|
||||
|
||||
def get_datanodes(cluster):
|
||||
return u.get_instances(cluster, 'datanode')
|
||||
|
||||
|
||||
def get_secondarynamenode(cluster):
|
||||
return u.get_instance(cluster, 'secondarynamenode')
|
||||
|
||||
|
||||
def get_historyserver(cluster):
|
||||
return u.get_instance(cluster, 'historyserver')
|
||||
|
||||
|
||||
def get_instance_hostname(instance):
|
||||
return instance.hostname() if instance else None
|
||||
|
||||
|
||||
def get_zk_servers(cluster):
|
||||
return u.get_instances(cluster, 'zookeeper')
|
||||
|
||||
|
||||
def generate_random_password():
|
||||
password = uuidutils.generate_uuid()
|
||||
return castellan.store_secret(password)
|
@ -1,157 +0,0 @@
|
||||
# Copyright (c) 2015 Mirantis Inc.
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
|
||||
# implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
import copy
|
||||
|
||||
from oslo_config import cfg
|
||||
import six
|
||||
|
||||
from sahara.plugins import provisioning as p
|
||||
from sahara.plugins import utils
|
||||
from sahara_plugin_vanilla.plugins.vanilla.hadoop2 import config_helper
|
||||
|
||||
CONF = cfg.CONF
|
||||
CONF.import_opt("enable_data_locality", "sahara.topology.topology_helper")
|
||||
|
||||
CORE_DEFAULT = utils.load_hadoop_xml_defaults(
|
||||
'plugins/vanilla/v2_7_1/resources/core-default.xml',
|
||||
'sahara_plugin_vanilla')
|
||||
|
||||
HDFS_DEFAULT = utils.load_hadoop_xml_defaults(
|
||||
'plugins/vanilla/v2_7_1/resources/hdfs-default.xml',
|
||||
'sahara_plugin_vanilla')
|
||||
|
||||
MAPRED_DEFAULT = utils.load_hadoop_xml_defaults(
|
||||
'plugins/vanilla/v2_7_1/resources/mapred-default.xml',
|
||||
'sahara_plugin_vanilla')
|
||||
|
||||
YARN_DEFAULT = utils.load_hadoop_xml_defaults(
|
||||
'plugins/vanilla/v2_7_1/resources/yarn-default.xml',
|
||||
'sahara_plugin_vanilla')
|
||||
|
||||
OOZIE_DEFAULT = utils.load_hadoop_xml_defaults(
|
||||
'plugins/vanilla/v2_7_1/resources/oozie-default.xml',
|
||||
'sahara_plugin_vanilla')
|
||||
|
||||
HIVE_DEFAULT = utils.load_hadoop_xml_defaults(
|
||||
'plugins/vanilla/v2_7_1/resources/hive-default.xml',
|
||||
'sahara_plugin_vanilla')
|
||||
|
||||
_default_executor_classpath = ":".join(
|
||||
['/opt/hadoop/share/hadoop/tools/lib/hadoop-openstack-2.7.1.jar'])
|
||||
|
||||
SPARK_CONFS = copy.deepcopy(config_helper.SPARK_CONFS)
|
||||
|
||||
SPARK_CONFS['Spark']['OPTIONS'].append(
|
||||
{
|
||||
'name': 'Executor extra classpath',
|
||||
'description': 'Value for spark.executor.extraClassPath'
|
||||
' in spark-defaults.conf'
|
||||
' (default: %s)' % _default_executor_classpath,
|
||||
'default': '%s' % _default_executor_classpath,
|
||||
'priority': 2,
|
||||
}
|
||||
)
|
||||
|
||||
XML_CONFS = {
|
||||
"Hadoop": [CORE_DEFAULT],
|
||||
"HDFS": [HDFS_DEFAULT],
|
||||
"YARN": [YARN_DEFAULT],
|
||||
"MapReduce": [MAPRED_DEFAULT],
|
||||
"JobFlow": [OOZIE_DEFAULT],
|
||||
"Hive": [HIVE_DEFAULT]
|
||||
}
|
||||
|
||||
ENV_CONFS = {
|
||||
"YARN": {
|
||||
'ResourceManager Heap Size': 1024,
|
||||
'NodeManager Heap Size': 1024
|
||||
},
|
||||
"HDFS": {
|
||||
'NameNode Heap Size': 1024,
|
||||
'SecondaryNameNode Heap Size': 1024,
|
||||
'DataNode Heap Size': 1024
|
||||
},
|
||||
"MapReduce": {
|
||||
'JobHistoryServer Heap Size': 1024
|
||||
},
|
||||
"JobFlow": {
|
||||
'Oozie Heap Size': 1024
|
||||
}
|
||||
}
|
||||
|
||||
# Initialise plugin Hadoop configurations
|
||||
PLUGIN_XML_CONFIGS = config_helper.init_xml_configs(XML_CONFS)
|
||||
PLUGIN_ENV_CONFIGS = config_helper.init_env_configs(ENV_CONFS)
|
||||
|
||||
|
||||
def _init_all_configs():
|
||||
configs = []
|
||||
configs.extend(PLUGIN_XML_CONFIGS)
|
||||
configs.extend(PLUGIN_ENV_CONFIGS)
|
||||
configs.extend(config_helper.PLUGIN_GENERAL_CONFIGS)
|
||||
configs.extend(_get_spark_configs())
|
||||
configs.extend(_get_zookeeper_configs())
|
||||
return configs
|
||||
|
||||
|
||||
def _get_spark_opt_default(opt_name):
|
||||
for opt in SPARK_CONFS["Spark"]["OPTIONS"]:
|
||||
if opt_name == opt["name"]:
|
||||
return opt["default"]
|
||||
return None
|
||||
|
||||
|
||||
def _get_spark_configs():
|
||||
spark_configs = []
|
||||
for service, config_items in six.iteritems(SPARK_CONFS):
|
||||
for item in config_items['OPTIONS']:
|
||||
cfg = p.Config(name=item["name"],
|
||||
description=item["description"],
|
||||
default_value=item["default"],
|
||||
applicable_target=service,
|
||||
scope="cluster", is_optional=True,
|
||||
priority=item["priority"])
|
||||
spark_configs.append(cfg)
|
||||
return spark_configs
|
||||
|
||||
|
||||
def _get_zookeeper_configs():
|
||||
zk_configs = []
|
||||
for service, config_items in six.iteritems(config_helper.ZOOKEEPER_CONFS):
|
||||
for item in config_items['OPTIONS']:
|
||||
cfg = p.Config(name=item["name"],
|
||||
description=item["description"],
|
||||
default_value=item["default"],
|
||||
applicable_target=service,
|
||||
scope="cluster", is_optional=True,
|
||||
priority=item["priority"])
|
||||
zk_configs.append(cfg)
|
||||
return zk_configs
|
||||
|
||||
|
||||
PLUGIN_CONFIGS = _init_all_configs()
|
||||
|
||||
|
||||
def get_plugin_configs():
|
||||
return PLUGIN_CONFIGS
|
||||
|
||||
|
||||
def get_xml_configs():
|
||||
return PLUGIN_XML_CONFIGS
|
||||
|
||||
|
||||
def get_env_configs():
|
||||
return ENV_CONFS
|
@ -1,80 +0,0 @@
|
||||
# Copyright (c) 2015 Red Hat, Inc.
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
|
||||
# implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
import os
|
||||
|
||||
from sahara.plugins import edp
|
||||
from sahara.plugins import exceptions as ex
|
||||
from sahara.plugins import utils as plugin_utils
|
||||
from sahara_plugin_vanilla.i18n import _
|
||||
from sahara_plugin_vanilla.plugins.vanilla import confighints_helper as chh
|
||||
from sahara_plugin_vanilla.plugins.vanilla.hadoop2 import edp_engine
|
||||
from sahara_plugin_vanilla.plugins.vanilla import utils as v_utils
|
||||
|
||||
|
||||
class EdpOozieEngine(edp_engine.EdpOozieEngine):
|
||||
@staticmethod
|
||||
def get_possible_job_config(job_type):
|
||||
if edp.compare_job_type(job_type, edp.JOB_TYPE_HIVE):
|
||||
return {'job_config': chh.get_possible_hive_config_from(
|
||||
'plugins/vanilla/v2_7_1/resources/hive-default.xml')}
|
||||
if edp.compare_job_type(job_type,
|
||||
edp.JOB_TYPE_MAPREDUCE,
|
||||
edp.JOB_TYPE_MAPREDUCE_STREAMING):
|
||||
return {'job_config': chh.get_possible_mapreduce_config_from(
|
||||
'plugins/vanilla/v2_7_1/resources/mapred-default.xml')}
|
||||
if edp.compare_job_type(job_type, edp.JOB_TYPE_PIG):
|
||||
return {'job_config': chh.get_possible_pig_config_from(
|
||||
'plugins/vanilla/v2_7_1/resources/mapred-default.xml')}
|
||||
return edp_engine.EdpOozieEngine.get_possible_job_config(job_type)
|
||||
|
||||
|
||||
class EdpSparkEngine(edp.PluginsSparkJobEngine):
|
||||
|
||||
edp_base_version = "2.7.1"
|
||||
|
||||
def __init__(self, cluster):
|
||||
super(EdpSparkEngine, self).__init__(cluster)
|
||||
self.master = plugin_utils.get_instance(cluster,
|
||||
"spark history server")
|
||||
self.plugin_params["spark-user"] = "sudo -u hadoop "
|
||||
self.plugin_params["spark-submit"] = os.path.join(
|
||||
plugin_utils.get_config_value_or_default(
|
||||
"Spark", "Spark home", self.cluster),
|
||||
"bin/spark-submit")
|
||||
self.plugin_params["deploy-mode"] = "cluster"
|
||||
self.plugin_params["master"] = "yarn"
|
||||
|
||||
driver_cp = plugin_utils.get_config_value_or_default(
|
||||
"Spark", "Executor extra classpath", self.cluster)
|
||||
self.plugin_params["driver-class-path"] = driver_cp
|
||||
|
||||
@staticmethod
|
||||
def edp_supported(version):
|
||||
return version >= EdpSparkEngine.edp_base_version
|
||||
|
||||
@staticmethod
|
||||
def job_type_supported(job_type):
|
||||
return (job_type in
|
||||
edp.PluginsSparkJobEngine.get_supported_job_types())
|
||||
|
||||
def validate_job_execution(self, cluster, job, data):
|
||||
if (not self.edp_supported(cluster.hadoop_version) or
|
||||
not v_utils.get_spark_history_server(cluster)):
|
||||
|
||||
raise ex.PluginInvalidDataException(
|
||||
_('Spark {base} or higher required to run {type} jobs').format(
|
||||
base=EdpSparkEngine.edp_base_version, type=job.type))
|
||||
|
||||
super(EdpSparkEngine, self).validate_job_execution(cluster, job, data)
|
@ -1,28 +0,0 @@
|
||||
Apache Hadoop Configurations for Sahara
|
||||
=======================================
|
||||
|
||||
This directory contains default XML configuration files:
|
||||
|
||||
* core-default.xml
|
||||
* hdfs-default.xml
|
||||
* mapred-default.xml
|
||||
* yarn-default.xml
|
||||
* oozie-default.xml
|
||||
* hive-default.xml
|
||||
|
||||
These files are applied for Sahara's plugin of Apache Hadoop version 2.7.1
|
||||
|
||||
|
||||
Files were taken from here:
|
||||
|
||||
* `core-default.xml <https://github.com/apache/hadoop/blob/release-2.7.1/hadoop-common-project/hadoop-common/src/main/resources/core-default.xml>`_
|
||||
* `hdfs-default.xml <https://github.com/apache/hadoop/blob/release-2.7.1/hadoop-hdfs-project/hadoop-hdfs/src/main/resources/hdfs-default.xml>`_
|
||||
* `yarn-default.xml <https://github.com/apache/hadoop/blob/release-2.7.1/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml>`_
|
||||
* `mapred-default.xml <https://github.com/apache/hadoop/blob/release-2.7.1/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/resources/mapred-default.xml>`_
|
||||
* `oozie-default.xml <https://github.com/apache/oozie/blob/release-4.2.0/core/src/main/resources/oozie-default.xml>`_
|
||||
* `hive-default.xml <https://github.com/apache/hive/blob/release-0.11.0/conf/hive-default.xml.template>`_
|
||||
|
||||
XML configs are used to expose default Hadoop configurations to the users
|
||||
through Sahara's REST API. It allows users to override some config values which
|
||||
will be pushed to the provisioned VMs running Hadoop services as part of
|
||||
appropriate xml config.
|
File diff suppressed because it is too large
Load Diff
@ -1,9 +0,0 @@
|
||||
CREATE DATABASE metastore;
|
||||
USE metastore;
|
||||
SOURCE /opt/hive/scripts/metastore/upgrade/mysql/hive-schema-0.10.0.mysql.sql;
|
||||
CREATE USER 'hive'@'localhost' IDENTIFIED BY '{{password}}';
|
||||
REVOKE ALL PRIVILEGES, GRANT OPTION FROM 'hive'@'localhost';
|
||||
GRANT ALL PRIVILEGES ON metastore.* TO 'hive'@'localhost' IDENTIFIED BY '{{password}}';
|
||||
GRANT ALL PRIVILEGES ON metastore.* TO 'hive'@'%' IDENTIFIED BY '{{password}}';
|
||||
FLUSH PRIVILEGES;
|
||||
exit
|
File diff suppressed because it is too large
Load Diff
File diff suppressed because it is too large
Load Diff
File diff suppressed because it is too large
Load Diff
File diff suppressed because it is too large
Load Diff
File diff suppressed because it is too large
Load Diff
@ -1,170 +0,0 @@
|
||||
# Copyright (c) 2015 Mirantis Inc.
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
|
||||
# implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
from oslo_config import cfg
|
||||
|
||||
from sahara.plugins import conductor
|
||||
from sahara.plugins import context
|
||||
from sahara.plugins import swift_helper
|
||||
from sahara.plugins import utils
|
||||
from sahara_plugin_vanilla.plugins.vanilla import abstractversionhandler as avm
|
||||
from sahara_plugin_vanilla.plugins.vanilla.hadoop2 import config as c
|
||||
from sahara_plugin_vanilla.plugins.vanilla.hadoop2 import keypairs
|
||||
from sahara_plugin_vanilla.plugins.vanilla.hadoop2 import recommendations_utils
|
||||
from sahara_plugin_vanilla.plugins.vanilla.hadoop2 import run_scripts as run
|
||||
from sahara_plugin_vanilla.plugins.vanilla.hadoop2 import scaling as sc
|
||||
from sahara_plugin_vanilla.plugins.vanilla.hadoop2 import starting_scripts
|
||||
from sahara_plugin_vanilla.plugins.vanilla.hadoop2 import utils as u
|
||||
from sahara_plugin_vanilla.plugins.vanilla.hadoop2 import validation as vl
|
||||
from sahara_plugin_vanilla.plugins.vanilla import utils as vu
|
||||
from sahara_plugin_vanilla.plugins.vanilla.v2_7_1 import config_helper
|
||||
from sahara_plugin_vanilla.plugins.vanilla.v2_7_1 import edp_engine
|
||||
|
||||
|
||||
CONF = cfg.CONF
|
||||
|
||||
|
||||
class VersionHandler(avm.AbstractVersionHandler):
|
||||
def __init__(self):
|
||||
self.pctx = {
|
||||
'env_confs': config_helper.get_env_configs(),
|
||||
'all_confs': config_helper.get_plugin_configs()
|
||||
}
|
||||
|
||||
def get_plugin_configs(self):
|
||||
return self.pctx['all_confs']
|
||||
|
||||
def get_node_processes(self):
|
||||
return {
|
||||
"Hadoop": [],
|
||||
"MapReduce": ["historyserver"],
|
||||
"HDFS": ["namenode", "datanode", "secondarynamenode"],
|
||||
"YARN": ["resourcemanager", "nodemanager"],
|
||||
"JobFlow": ["oozie"],
|
||||
"Hive": ["hiveserver"],
|
||||
"Spark": ["spark history server"],
|
||||
"ZooKeeper": ["zookeeper"]
|
||||
}
|
||||
|
||||
def validate(self, cluster):
|
||||
vl.validate_cluster_creating(self.pctx, cluster)
|
||||
|
||||
def update_infra(self, cluster):
|
||||
pass
|
||||
|
||||
def configure_cluster(self, cluster):
|
||||
c.configure_cluster(self.pctx, cluster)
|
||||
|
||||
def start_cluster(self, cluster):
|
||||
keypairs.provision_keypairs(cluster)
|
||||
|
||||
starting_scripts.start_namenode(cluster)
|
||||
starting_scripts.start_secondarynamenode(cluster)
|
||||
starting_scripts.start_resourcemanager(cluster)
|
||||
|
||||
run.start_dn_nm_processes(utils.get_instances(cluster))
|
||||
run.await_datanodes(cluster)
|
||||
|
||||
starting_scripts.start_historyserver(cluster)
|
||||
starting_scripts.start_oozie(self.pctx, cluster)
|
||||
starting_scripts.start_hiveserver(self.pctx, cluster)
|
||||
starting_scripts.start_zookeeper(cluster)
|
||||
|
||||
swift_helper.install_ssl_certs(utils.get_instances(cluster))
|
||||
|
||||
self._set_cluster_info(cluster)
|
||||
starting_scripts.start_spark(cluster)
|
||||
|
||||
def decommission_nodes(self, cluster, instances):
|
||||
sc.decommission_nodes(self.pctx, cluster, instances)
|
||||
|
||||
def validate_scaling(self, cluster, existing, additional):
|
||||
vl.validate_additional_ng_scaling(cluster, additional)
|
||||
vl.validate_existing_ng_scaling(self.pctx, cluster, existing)
|
||||
zk_ng = utils.get_node_groups(cluster, "zookeeper")
|
||||
if zk_ng:
|
||||
vl.validate_zookeeper_node_count(zk_ng, existing, additional)
|
||||
|
||||
def scale_cluster(self, cluster, instances):
|
||||
keypairs.provision_keypairs(cluster, instances)
|
||||
sc.scale_cluster(self.pctx, cluster, instances)
|
||||
|
||||
def _set_cluster_info(self, cluster):
|
||||
nn = vu.get_namenode(cluster)
|
||||
rm = vu.get_resourcemanager(cluster)
|
||||
hs = vu.get_historyserver(cluster)
|
||||
oo = vu.get_oozie(cluster)
|
||||
sp = vu.get_spark_history_server(cluster)
|
||||
info = {}
|
||||
|
||||
if rm:
|
||||
info['YARN'] = {
|
||||
'Web UI': 'http://%s:%s' % (rm.get_ip_or_dns_name(), '8088'),
|
||||
'ResourceManager': 'http://%s:%s' % (
|
||||
rm.get_ip_or_dns_name(), '8032')
|
||||
}
|
||||
|
||||
if nn:
|
||||
info['HDFS'] = {
|
||||
'Web UI': 'http://%s:%s' % (nn.get_ip_or_dns_name(), '50070'),
|
||||
'NameNode': 'hdfs://%s:%s' % (nn.hostname(), '9000')
|
||||
}
|
||||
|
||||
if oo:
|
||||
info['JobFlow'] = {
|
||||
'Oozie': 'http://%s:%s' % (oo.get_ip_or_dns_name(), '11000')
|
||||
}
|
||||
|
||||
if hs:
|
||||
info['MapReduce JobHistory Server'] = {
|
||||
'Web UI': 'http://%s:%s' % (hs.get_ip_or_dns_name(), '19888')
|
||||
}
|
||||
|
||||
if sp:
|
||||
info['Apache Spark'] = {
|
||||
'Spark UI': 'http://%s:%s' % (sp.management_ip, '4040'),
|
||||
'Spark History Server UI':
|
||||
'http://%s:%s' % (sp.management_ip, '18080')
|
||||
}
|
||||
|
||||
ctx = context.ctx()
|
||||
conductor.cluster_update(ctx, cluster, {'info': info})
|
||||
|
||||
def get_edp_engine(self, cluster, job_type):
|
||||
if job_type in edp_engine.EdpOozieEngine.get_supported_job_types():
|
||||
return edp_engine.EdpOozieEngine(cluster)
|
||||
if job_type in edp_engine.EdpSparkEngine.get_supported_job_types():
|
||||
return edp_engine.EdpSparkEngine(cluster)
|
||||
|
||||
return None
|
||||
|
||||
def get_edp_job_types(self):
|
||||
return (edp_engine.EdpOozieEngine.get_supported_job_types() +
|
||||
edp_engine.EdpSparkEngine.get_supported_job_types())
|
||||
|
||||
def get_edp_config_hints(self, job_type):
|
||||
return edp_engine.EdpOozieEngine.get_possible_job_config(job_type)
|
||||
|
||||
def on_terminate_cluster(self, cluster):
|
||||
u.delete_oozie_password(cluster)
|
||||
keypairs.drop_key(cluster)
|
||||
|
||||
def get_open_ports(self, node_group):
|
||||
return c.get_open_ports(node_group)
|
||||
|
||||
def recommend_configs(self, cluster, scaling):
|
||||
recommendations_utils.recommend_configs(cluster,
|
||||
self.get_plugin_configs(),
|
||||
scaling)
|
@ -1,157 +0,0 @@
|
||||
# Copyright (c) 2015 Mirantis Inc.
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
|
||||
# implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
import copy
|
||||
|
||||
from oslo_config import cfg
|
||||
import six
|
||||
|
||||
from sahara.plugins import provisioning as p
|
||||
from sahara.plugins import utils
|
||||
from sahara_plugin_vanilla.plugins.vanilla.hadoop2 import config_helper
|
||||
|
||||
CONF = cfg.CONF
|
||||
CONF.import_opt("enable_data_locality", "sahara.topology.topology_helper")
|
||||
|
||||
CORE_DEFAULT = utils.load_hadoop_xml_defaults(
|
||||
'plugins/vanilla/v2_7_5/resources/core-default.xml',
|
||||
'sahara_plugin_vanilla')
|
||||
|
||||
HDFS_DEFAULT = utils.load_hadoop_xml_defaults(
|
||||
'plugins/vanilla/v2_7_5/resources/hdfs-default.xml',
|
||||
'sahara_plugin_vanilla')
|
||||
|
||||
MAPRED_DEFAULT = utils.load_hadoop_xml_defaults(
|
||||
'plugins/vanilla/v2_7_5/resources/mapred-default.xml',
|
||||
'sahara_plugin_vanilla')
|
||||
|
||||
YARN_DEFAULT = utils.load_hadoop_xml_defaults(
|
||||
'plugins/vanilla/v2_7_5/resources/yarn-default.xml',
|
||||
'sahara_plugin_vanilla')
|
||||
|
||||
OOZIE_DEFAULT = utils.load_hadoop_xml_defaults(
|
||||
'plugins/vanilla/v2_7_5/resources/oozie-default.xml',
|
||||
'sahara_plugin_vanilla')
|
||||
|
||||
HIVE_DEFAULT = utils.load_hadoop_xml_defaults(
|
||||
'plugins/vanilla/v2_7_5/resources/hive-default.xml',
|
||||
'sahara_plugin_vanilla')
|
||||
|
||||
_default_executor_classpath = ":".join(
|
||||
['/opt/hadoop/share/hadoop/tools/lib/hadoop-openstack-2.7.5.jar'])
|
||||
|
||||
SPARK_CONFS = copy.deepcopy(config_helper.SPARK_CONFS)
|
||||
|
||||
SPARK_CONFS['Spark']['OPTIONS'].append(
|
||||
{
|
||||
'name': 'Executor extra classpath',
|
||||
'description': 'Value for spark.executor.extraClassPath'
|
||||
' in spark-defaults.conf'
|
||||
' (default: %s)' % _default_executor_classpath,
|
||||
'default': '%s' % _default_executor_classpath,
|
||||
'priority': 2,
|
||||
}
|
||||
)
|
||||
|
||||
XML_CONFS = {
|
||||
"Hadoop": [CORE_DEFAULT],
|
||||
"HDFS": [HDFS_DEFAULT],
|
||||
"YARN": [YARN_DEFAULT],
|
||||
"MapReduce": [MAPRED_DEFAULT],
|
||||
"JobFlow": [OOZIE_DEFAULT],
|
||||
"Hive": [HIVE_DEFAULT]
|
||||
}
|
||||
|
||||
ENV_CONFS = {
|
||||
"YARN": {
|
||||
'ResourceManager Heap Size': 1024,
|
||||
'NodeManager Heap Size': 1024
|
||||
},
|
||||
"HDFS": {
|
||||
'NameNode Heap Size': 1024,
|
||||
'SecondaryNameNode Heap Size': 1024,
|
||||
'DataNode Heap Size': 1024
|
||||
},
|
||||
"MapReduce": {
|
||||
'JobHistoryServer Heap Size': 1024
|
||||
},
|
||||
"JobFlow": {
|
||||
'Oozie Heap Size': 1024
|
||||
}
|
||||
}
|
||||
|
||||
# Initialise plugin Hadoop configurations
|
||||
PLUGIN_XML_CONFIGS = config_helper.init_xml_configs(XML_CONFS)
|
||||
PLUGIN_ENV_CONFIGS = config_helper.init_env_configs(ENV_CONFS)
|
||||
|
||||
|
||||
def _init_all_configs():
|
||||
configs = []
|
||||
configs.extend(PLUGIN_XML_CONFIGS)
|
||||
configs.extend(PLUGIN_ENV_CONFIGS)
|
||||
configs.extend(config_helper.PLUGIN_GENERAL_CONFIGS)
|
||||
configs.extend(_get_spark_configs())
|
||||
configs.extend(_get_zookeeper_configs())
|
||||
return configs
|
||||
|
||||
|
||||
def _get_spark_opt_default(opt_name):
|
||||
for opt in SPARK_CONFS["Spark"]["OPTIONS"]:
|
||||
if opt_name == opt["name"]:
|
||||
return opt["default"]
|
||||
return None
|
||||
|
||||
|
||||
def _get_spark_configs():
|
||||
spark_configs = []
|
||||
for service, config_items in six.iteritems(SPARK_CONFS):
|
||||
for item in config_items['OPTIONS']:
|
||||
cfg = p.Config(name=item["name"],
|
||||
description=item["description"],
|
||||
default_value=item["default"],
|
||||
applicable_target=service,
|
||||
scope="cluster", is_optional=True,
|
||||
priority=item["priority"])
|
||||
spark_configs.append(cfg)
|
||||
return spark_configs
|
||||
|
||||
|
||||
def _get_zookeeper_configs():
|
||||
zk_configs = []
|
||||
for service, config_items in six.iteritems(config_helper.ZOOKEEPER_CONFS):
|
||||
for item in config_items['OPTIONS']:
|
||||
cfg = p.Config(name=item["name"],
|
||||
description=item["description"],
|
||||
default_value=item["default"],
|
||||
applicable_target=service,
|
||||
scope="cluster", is_optional=True,
|
||||
priority=item["priority"])
|
||||
zk_configs.append(cfg)
|
||||
return zk_configs
|
||||
|
||||
|
||||
PLUGIN_CONFIGS = _init_all_configs()
|
||||
|
||||
|
||||
def get_plugin_configs():
|
||||
return PLUGIN_CONFIGS
|
||||
|
||||
|
||||
def get_xml_configs():
|
||||
return PLUGIN_XML_CONFIGS
|
||||
|
||||
|
||||
def get_env_configs():
|
||||
return ENV_CONFS
|
@ -1,84 +0,0 @@
|
||||
# Copyright (c) 2015 Red Hat, Inc.
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
|
||||
# implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
import os
|
||||
|
||||
from sahara.plugins import edp
|
||||
from sahara.plugins import exceptions as ex
|
||||
from sahara.plugins import utils as plugin_utils
|
||||
from sahara_plugin_vanilla.i18n import _
|
||||
from sahara_plugin_vanilla.plugins.vanilla import confighints_helper
|
||||
from sahara_plugin_vanilla.plugins.vanilla.hadoop2 import edp_engine
|
||||
from sahara_plugin_vanilla.plugins.vanilla import utils as v_utils
|
||||
|
||||
|
||||
class EdpOozieEngine(edp_engine.EdpOozieEngine):
|
||||
@staticmethod
|
||||
def get_possible_job_config(job_type):
|
||||
if edp.compare_job_type(job_type, edp.JOB_TYPE_HIVE):
|
||||
return {
|
||||
'job_config': confighints_helper.get_possible_hive_config_from(
|
||||
'plugins/vanilla/v2_7_5/resources/hive-default.xml')}
|
||||
if edp.compare_job_type(job_type,
|
||||
edp.JOB_TYPE_MAPREDUCE,
|
||||
edp.JOB_TYPE_MAPREDUCE_STREAMING):
|
||||
return {
|
||||
'job_config':
|
||||
confighints_helper.get_possible_mapreduce_config_from(
|
||||
'plugins/vanilla/v2_7_5/resources/mapred-default.xml')}
|
||||
if edp.compare_job_type(job_type, edp.JOB_TYPE_PIG):
|
||||
return {
|
||||
'job_config': confighints_helper.get_possible_pig_config_from(
|
||||
'plugins/vanilla/v2_7_5/resources/mapred-default.xml')}
|
||||
return edp_engine.EdpOozieEngine.get_possible_job_config(job_type)
|
||||
|
||||
|
||||
class EdpSparkEngine(edp.PluginsSparkJobEngine):
|
||||
|
||||
edp_base_version = "2.7.5"
|
||||
|
||||
def __init__(self, cluster):
|
||||
super(EdpSparkEngine, self).__init__(cluster)
|
||||
self.master = plugin_utils.get_instance(cluster,
|
||||
"spark history server")
|
||||
self.plugin_params["spark-user"] = "sudo -u hadoop "
|
||||
self.plugin_params["spark-submit"] = os.path.join(
|
||||
plugin_utils.get_config_value_or_default(
|
||||
"Spark", "Spark home", self.cluster),
|
||||
"bin/spark-submit")
|
||||
self.plugin_params["deploy-mode"] = "cluster"
|
||||
self.plugin_params["master"] = "yarn"
|
||||
|
||||
driver_cp = plugin_utils.get_config_value_or_default(
|
||||
"Spark", "Executor extra classpath", self.cluster)
|
||||
self.plugin_params["driver-class-path"] = driver_cp
|
||||
|
||||
@staticmethod
|
||||
def edp_supported(version):
|
||||
return version >= EdpSparkEngine.edp_base_version
|
||||
|
||||
@staticmethod
|
||||
def job_type_supported(job_type):
|
||||
return (job_type in
|
||||
edp.PluginsSparkJobEngine.get_supported_job_types())
|
||||
|
||||
def validate_job_execution(self, cluster, job, data):
|
||||
if (not self.edp_supported(cluster.hadoop_version) or
|
||||
not v_utils.get_spark_history_server(cluster)):
|
||||
|
||||
raise ex.InvalidDataException(
|
||||
_('Spark {base} or higher required to run {type} jobs').format(
|
||||
base=EdpSparkEngine.edp_base_version, type=job.type))
|
||||
|
||||
super(EdpSparkEngine, self).validate_job_execution(cluster, job, data)
|
@ -1,27 +0,0 @@
|
||||
Apache Hadoop Configurations for Sahara
|
||||
=======================================
|
||||
|
||||
This directory contains default XML configuration files:
|
||||
|
||||
* core-default.xml
|
||||
* hdfs-default.xml
|
||||
* mapred-default.xml
|
||||
* yarn-default.xml
|
||||
* oozie-default.xml
|
||||
* hive-default.xml
|
||||
|
||||
These files are applied for Sahara's plugin of Apache Hadoop version 2.7.5
|
||||
|
||||
|
||||
Files were taken from here:
|
||||
|
||||
* `core-default.xml <https://github.com/apache/hadoop/tree/branch-2.7.5/hadoop-common-project/hadoop-common/src/main/resources/core-default.xml>`_
|
||||
* `hdfs-default.xml <https://github.com/apache/hadoop/tree/branch-2.7.5/hadoop-hdfs-project/hadoop-hdfs/src/main/resources/hdfs-default.xml>`_
|
||||
* `yarn-default.xml <https://github.com/apache/hadoop/tree/branch-2.7.5/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml>`_
|
||||
* `mapred-default.xml <https://github.com/apache/hadoop/tree/branch-2.7.5/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/resources/mapred-default.xml>`_
|
||||
* `oozie-default.xml <https://github.com/apache/oozie/blob/release-4.3.0/core/src/main/resources/oozie-default.xml>`_
|
||||
|
||||
XML configs are used to expose default Hadoop configurations to the users
|
||||
through Sahara's REST API. It allows users to override some config values which
|
||||
will be pushed to the provisioned VMs running Hadoop services as part of
|
||||
appropriate xml config.
|
File diff suppressed because it is too large
Load Diff
@ -1,9 +0,0 @@
|
||||
CREATE DATABASE metastore;
|
||||
USE metastore;
|
||||
SOURCE /opt/hive/scripts/metastore/upgrade/mysql/hive-schema-2.3.0.mysql.sql;
|
||||
CREATE USER 'hive'@'localhost' IDENTIFIED BY '{{password}}';
|
||||
REVOKE ALL PRIVILEGES, GRANT OPTION FROM 'hive'@'localhost';
|
||||
GRANT ALL PRIVILEGES ON metastore.* TO 'hive'@'localhost' IDENTIFIED BY '{{password}}';
|
||||
GRANT ALL PRIVILEGES ON metastore.* TO 'hive'@'%' IDENTIFIED BY '{{password}}';
|
||||
FLUSH PRIVILEGES;
|
||||
exit
|
File diff suppressed because it is too large
Load Diff
File diff suppressed because it is too large
Load Diff
File diff suppressed because it is too large
Load Diff
File diff suppressed because it is too large
Load Diff
File diff suppressed because it is too large
Load Diff
@ -1,170 +0,0 @@
|
||||
# Copyright (c) 2015 Mirantis Inc.
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
|
||||
# implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
from oslo_config import cfg
|
||||
|
||||
from sahara.plugins import conductor
|
||||
from sahara.plugins import context
|
||||
from sahara.plugins import swift_helper
|
||||
from sahara.plugins import utils
|
||||
from sahara_plugin_vanilla.plugins.vanilla import abstractversionhandler as avm
|
||||
from sahara_plugin_vanilla.plugins.vanilla.hadoop2 import config as c
|
||||
from sahara_plugin_vanilla.plugins.vanilla.hadoop2 import keypairs
|
||||
from sahara_plugin_vanilla.plugins.vanilla.hadoop2 import recommendations_utils
|
||||
from sahara_plugin_vanilla.plugins.vanilla.hadoop2 import run_scripts as run
|
||||
from sahara_plugin_vanilla.plugins.vanilla.hadoop2 import scaling as sc
|
||||
from sahara_plugin_vanilla.plugins.vanilla.hadoop2 import starting_scripts
|
||||
from sahara_plugin_vanilla.plugins.vanilla.hadoop2 import utils as u
|
||||
from sahara_plugin_vanilla.plugins.vanilla.hadoop2 import validation as vl
|
||||
from sahara_plugin_vanilla.plugins.vanilla import utils as vu
|
||||
from sahara_plugin_vanilla.plugins.vanilla.v2_7_5 import config_helper
|
||||
from sahara_plugin_vanilla.plugins.vanilla.v2_7_5 import edp_engine
|
||||
|
||||
|
||||
CONF = cfg.CONF
|
||||
|
||||
|
||||
class VersionHandler(avm.AbstractVersionHandler):
|
||||
def __init__(self):
|
||||
self.pctx = {
|
||||
'env_confs': config_helper.get_env_configs(),
|
||||
'all_confs': config_helper.get_plugin_configs()
|
||||
}
|
||||
|
||||
def get_plugin_configs(self):
|
||||
return self.pctx['all_confs']
|
||||
|
||||
def get_node_processes(self):
|
||||
return {
|
||||
"Hadoop": [],
|
||||
"MapReduce": ["historyserver"],
|
||||
"HDFS": ["namenode", "datanode", "secondarynamenode"],
|
||||
"YARN": ["resourcemanager", "nodemanager"],
|
||||
"JobFlow": ["oozie"],
|
||||
"Hive": ["hiveserver"],
|
||||
"Spark": ["spark history server"],
|
||||
"ZooKeeper": ["zookeeper"]
|
||||
}
|
||||
|
||||
def validate(self, cluster):
|
||||
vl.validate_cluster_creating(self.pctx, cluster)
|
||||
|
||||
def update_infra(self, cluster):
|
||||
pass
|
||||
|
||||
def configure_cluster(self, cluster):
|
||||
c.configure_cluster(self.pctx, cluster)
|
||||
|
||||
def start_cluster(self, cluster):
|
||||
keypairs.provision_keypairs(cluster)
|
||||
|
||||
starting_scripts.start_namenode(cluster)
|
||||
starting_scripts.start_secondarynamenode(cluster)
|
||||
starting_scripts.start_resourcemanager(cluster)
|
||||
|
||||
run.start_dn_nm_processes(utils.get_instances(cluster))
|
||||
run.await_datanodes(cluster)
|
||||
|
||||
starting_scripts.start_historyserver(cluster)
|
||||
starting_scripts.start_oozie(self.pctx, cluster)
|
||||
starting_scripts.start_hiveserver(self.pctx, cluster)
|
||||
starting_scripts.start_zookeeper(cluster)
|
||||
|
||||
swift_helper.install_ssl_certs(utils.get_instances(cluster))
|
||||
|
||||
self._set_cluster_info(cluster)
|
||||
starting_scripts.start_spark(cluster)
|
||||
|
||||
def decommission_nodes(self, cluster, instances):
|
||||
sc.decommission_nodes(self.pctx, cluster, instances)
|
||||
|
||||
def validate_scaling(self, cluster, existing, additional):
|
||||
vl.validate_additional_ng_scaling(cluster, additional)
|
||||
vl.validate_existing_ng_scaling(self.pctx, cluster, existing)
|
||||
zk_ng = utils.get_node_groups(cluster, "zookeeper")
|
||||
if zk_ng:
|
||||
vl.validate_zookeeper_node_count(zk_ng, existing, additional)
|
||||
|
||||
def scale_cluster(self, cluster, instances):
|
||||
keypairs.provision_keypairs(cluster, instances)
|
||||
sc.scale_cluster(self.pctx, cluster, instances)
|
||||
|
||||
def _set_cluster_info(self, cluster):
|
||||
nn = vu.get_namenode(cluster)
|
||||
rm = vu.get_resourcemanager(cluster)
|
||||
hs = vu.get_historyserver(cluster)
|
||||
oo = vu.get_oozie(cluster)
|
||||
sp = vu.get_spark_history_server(cluster)
|
||||
info = {}
|
||||
|
||||
if rm:
|
||||
info['YARN'] = {
|
||||
'Web UI': 'http://%s:%s' % (rm.get_ip_or_dns_name(), '8088'),
|
||||
'ResourceManager': 'http://%s:%s' % (
|
||||
rm.get_ip_or_dns_name(), '8032')
|
||||
}
|
||||
|
||||
if nn:
|
||||
info['HDFS'] = {
|
||||
'Web UI': 'http://%s:%s' % (nn.get_ip_or_dns_name(), '50070'),
|
||||
'NameNode': 'hdfs://%s:%s' % (nn.hostname(), '9000')
|
||||
}
|
||||
|
||||
if oo:
|
||||
info['JobFlow'] = {
|
||||
'Oozie': 'http://%s:%s' % (oo.get_ip_or_dns_name(), '11000')
|
||||
}
|
||||
|
||||
if hs:
|
||||
info['MapReduce JobHistory Server'] = {
|
||||
'Web UI': 'http://%s:%s' % (hs.get_ip_or_dns_name(), '19888')
|
||||
}
|
||||
|
||||
if sp:
|
||||
info['Apache Spark'] = {
|
||||
'Spark UI': 'http://%s:%s' % (sp.management_ip, '4040'),
|
||||
'Spark History Server UI':
|
||||
'http://%s:%s' % (sp.management_ip, '18080')
|
||||
}
|
||||
|
||||
ctx = context.ctx()
|
||||
conductor.cluster_update(ctx, cluster, {'info': info})
|
||||
|
||||
def get_edp_engine(self, cluster, job_type):
|
||||
if job_type in edp_engine.EdpOozieEngine.get_supported_job_types():
|
||||
return edp_engine.EdpOozieEngine(cluster)
|
||||
if job_type in edp_engine.EdpSparkEngine.get_supported_job_types():
|
||||
return edp_engine.EdpSparkEngine(cluster)
|
||||
|
||||
return None
|
||||
|
||||
def get_edp_job_types(self):
|
||||
return (edp_engine.EdpOozieEngine.get_supported_job_types() +
|
||||
edp_engine.EdpSparkEngine.get_supported_job_types())
|
||||
|
||||
def get_edp_config_hints(self, job_type):
|
||||
return edp_engine.EdpOozieEngine.get_possible_job_config(job_type)
|
||||
|
||||
def on_terminate_cluster(self, cluster):
|
||||
u.delete_oozie_password(cluster)
|
||||
keypairs.drop_key(cluster)
|
||||
|
||||
def get_open_ports(self, node_group):
|
||||
return c.get_open_ports(node_group)
|
||||
|
||||
def recommend_configs(self, cluster, scaling):
|
||||
recommendations_utils.recommend_configs(cluster,
|
||||
self.get_plugin_configs(),
|
||||
scaling)
|
@ -1,157 +0,0 @@
|
||||
# Copyright (c) 2015 Mirantis Inc.
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
|
||||
# implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
import copy
|
||||
|
||||
from oslo_config import cfg
|
||||
import six
|
||||
|
||||
from sahara.plugins import provisioning as p
|
||||
from sahara.plugins import utils
|
||||
from sahara_plugin_vanilla.plugins.vanilla.hadoop2 import config_helper
|
||||
|
||||
CONF = cfg.CONF
|
||||
CONF.import_opt("enable_data_locality", "sahara.topology.topology_helper")
|
||||
|
||||
CORE_DEFAULT = utils.load_hadoop_xml_defaults(
|
||||
'plugins/vanilla/v2_8_2/resources/core-default.xml',
|
||||
'sahara_plugin_vanilla')
|
||||
|
||||
HDFS_DEFAULT = utils.load_hadoop_xml_defaults(
|
||||
'plugins/vanilla/v2_8_2/resources/hdfs-default.xml',
|
||||
'sahara_plugin_vanilla')
|
||||
|
||||
MAPRED_DEFAULT = utils.load_hadoop_xml_defaults(
|
||||
'plugins/vanilla/v2_8_2/resources/mapred-default.xml',
|
||||
'sahara_plugin_vanilla')
|
||||
|
||||
YARN_DEFAULT = utils.load_hadoop_xml_defaults(
|
||||
'plugins/vanilla/v2_8_2/resources/yarn-default.xml',
|
||||
'sahara_plugin_vanilla')
|
||||
|
||||
OOZIE_DEFAULT = utils.load_hadoop_xml_defaults(
|
||||
'plugins/vanilla/v2_8_2/resources/oozie-default.xml',
|
||||
'sahara_plugin_vanilla')
|
||||
|
||||
HIVE_DEFAULT = utils.load_hadoop_xml_defaults(
|
||||
'plugins/vanilla/v2_8_2/resources/hive-default.xml',
|
||||
'sahara_plugin_vanilla')
|
||||
|
||||
_default_executor_classpath = ":".join(
|
||||
['/opt/hadoop/share/hadoop/tools/lib/hadoop-openstack-2.8.2.jar'])
|
||||
|
||||
SPARK_CONFS = copy.deepcopy(config_helper.SPARK_CONFS)
|
||||
|
||||
SPARK_CONFS['Spark']['OPTIONS'].append(
|
||||
{
|
||||
'name': 'Executor extra classpath',
|
||||
'description': 'Value for spark.executor.extraClassPath'
|
||||
' in spark-defaults.conf'
|
||||
' (default: %s)' % _default_executor_classpath,
|
||||
'default': '%s' % _default_executor_classpath,
|
||||
'priority': 2,
|
||||
}
|
||||
)
|
||||
|
||||
XML_CONFS = {
|
||||
"Hadoop": [CORE_DEFAULT],
|
||||
"HDFS": [HDFS_DEFAULT],
|
||||
"YARN": [YARN_DEFAULT],
|
||||
"MapReduce": [MAPRED_DEFAULT],
|
||||
"JobFlow": [OOZIE_DEFAULT],
|
||||
"Hive": [HIVE_DEFAULT]
|
||||
}
|
||||
|
||||
ENV_CONFS = {
|
||||
"YARN": {
|
||||
'ResourceManager Heap Size': 1024,
|
||||
'NodeManager Heap Size': 1024
|
||||
},
|
||||
"HDFS": {
|
||||
'NameNode Heap Size': 1024,
|
||||
'SecondaryNameNode Heap Size': 1024,
|
||||
'DataNode Heap Size': 1024
|
||||
},
|
||||
"MapReduce": {
|
||||
'JobHistoryServer Heap Size': 1024
|
||||
},
|
||||
"JobFlow": {
|
||||
'Oozie Heap Size': 1024
|
||||
}
|
||||
}
|
||||
|
||||
# Initialise plugin Hadoop configurations
|
||||
PLUGIN_XML_CONFIGS = config_helper.init_xml_configs(XML_CONFS)
|
||||
PLUGIN_ENV_CONFIGS = config_helper.init_env_configs(ENV_CONFS)
|
||||
|
||||
|
||||
def _init_all_configs():
|
||||
configs = []
|
||||
configs.extend(PLUGIN_XML_CONFIGS)
|
||||
configs.extend(PLUGIN_ENV_CONFIGS)
|
||||
configs.extend(config_helper.PLUGIN_GENERAL_CONFIGS)
|
||||
configs.extend(_get_spark_configs())
|
||||
configs.extend(_get_zookeeper_configs())
|
||||
return configs
|
||||
|
||||
|
||||
def _get_spark_opt_default(opt_name):
|
||||
for opt in SPARK_CONFS["Spark"]["OPTIONS"]:
|
||||
if opt_name == opt["name"]:
|
||||
return opt["default"]
|
||||
return None
|
||||
|
||||
|
||||
def _get_spark_configs():
|
||||
spark_configs = []
|
||||
for service, config_items in six.iteritems(SPARK_CONFS):
|
||||
for item in config_items['OPTIONS']:
|
||||
cfg = p.Config(name=item["name"],
|
||||
description=item["description"],
|
||||
default_value=item["default"],
|
||||
applicable_target=service,
|
||||
scope="cluster", is_optional=True,
|
||||
priority=item["priority"])
|
||||
spark_configs.append(cfg)
|
||||
return spark_configs
|
||||
|
||||
|
||||
def _get_zookeeper_configs():
|
||||
zk_configs = []
|
||||
for service, config_items in six.iteritems(config_helper.ZOOKEEPER_CONFS):
|
||||
for item in config_items['OPTIONS']:
|
||||
cfg = p.Config(name=item["name"],
|
||||
description=item["description"],
|
||||
default_value=item["default"],
|
||||
applicable_target=service,
|
||||
scope="cluster", is_optional=True,
|
||||
priority=item["priority"])
|
||||
zk_configs.append(cfg)
|
||||
return zk_configs
|
||||
|
||||
|
||||
PLUGIN_CONFIGS = _init_all_configs()
|
||||
|
||||
|
||||
def get_plugin_configs():
|
||||
return PLUGIN_CONFIGS
|
||||
|
||||
|
||||
def get_xml_configs():
|
||||
return PLUGIN_XML_CONFIGS
|
||||
|
||||
|
||||
def get_env_configs():
|
||||
return ENV_CONFS
|
@ -1,84 +0,0 @@
|
||||
# Copyright (c) 2015 Red Hat, Inc.
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
|
||||
# implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
import os
|
||||
|
||||
from sahara.plugins import edp
|
||||
from sahara.plugins import exceptions as ex
|
||||
from sahara.plugins import utils as plugin_utils
|
||||
from sahara_plugin_vanilla.i18n import _
|
||||
from sahara_plugin_vanilla.plugins.vanilla import confighints_helper
|
||||
from sahara_plugin_vanilla.plugins.vanilla.hadoop2 import edp_engine
|
||||
from sahara_plugin_vanilla.plugins.vanilla import utils as v_utils
|
||||
|
||||
|
||||
class EdpOozieEngine(edp_engine.EdpOozieEngine):
|
||||
@staticmethod
|
||||
def get_possible_job_config(job_type):
|
||||
if edp.compare_job_type(job_type, edp.JOB_TYPE_HIVE):
|
||||
return {
|
||||
'job_config': confighints_helper.get_possible_hive_config_from(
|
||||
'plugins/vanilla/v2_8_2/resources/hive-default.xml')}
|
||||
if edp.compare_job_type(job_type,
|
||||
edp.JOB_TYPE_MAPREDUCE,
|
||||
edp.JOB_TYPE_MAPREDUCE_STREAMING):
|
||||
return {
|
||||
'job_config':
|
||||
confighints_helper.get_possible_mapreduce_config_from(
|
||||
'plugins/vanilla/v2_8_2/resources/mapred-default.xml')}
|
||||
if edp.compare_job_type(job_type, edp.JOB_TYPE_PIG):
|
||||
return {
|
||||
'job_config': confighints_helper.get_possible_pig_config_from(
|
||||
'plugins/vanilla/v2_8_2/resources/mapred-default.xml')}
|
||||
return edp_engine.EdpOozieEngine.get_possible_job_config(job_type)
|
||||
|
||||
|
||||
class EdpSparkEngine(edp.PluginsSparkJobEngine):
|
||||
|
||||
edp_base_version = "2.8.2"
|
||||
|
||||
def __init__(self, cluster):
|
||||
super(EdpSparkEngine, self).__init__(cluster)
|
||||
self.master = plugin_utils.get_instance(cluster,
|
||||
"spark history server")
|
||||
self.plugin_params["spark-user"] = "sudo -u hadoop "
|
||||
self.plugin_params["spark-submit"] = os.path.join(
|
||||
plugin_utils.get_config_value_or_default(
|
||||
"Spark", "Spark home", self.cluster),
|
||||
"bin/spark-submit")
|
||||
self.plugin_params["deploy-mode"] = "cluster"
|
||||
self.plugin_params["master"] = "yarn"
|
||||
|
||||
driver_cp = plugin_utils.get_config_value_or_default(
|
||||
"Spark", "Executor extra classpath", self.cluster)
|
||||
self.plugin_params["driver-class-path"] = driver_cp
|
||||
|
||||
@staticmethod
|
||||
def edp_supported(version):
|
||||
return version >= EdpSparkEngine.edp_base_version
|
||||
|
||||
@staticmethod
|
||||
def job_type_supported(job_type):
|
||||
return (job_type in
|
||||
edp.PluginsSparkJobEngine.get_supported_job_types())
|
||||
|
||||
def validate_job_execution(self, cluster, job, data):
|
||||
if (not self.edp_supported(cluster.hadoop_version) or
|
||||
not v_utils.get_spark_history_server(cluster)):
|
||||
|
||||
raise ex.PluginInvalidDataException(
|
||||
_('Spark {base} or higher required to run {type} jobs').format(
|
||||
base=EdpSparkEngine.edp_base_version, type=job.type))
|
||||
|
||||
super(EdpSparkEngine, self).validate_job_execution(cluster, job, data)
|
@ -1,27 +0,0 @@
|
||||
Apache Hadoop Configurations for Sahara
|
||||
=======================================
|
||||
|
||||
This directory contains default XML configuration files:
|
||||
|
||||
* core-default.xml
|
||||
* hdfs-default.xml
|
||||
* mapred-default.xml
|
||||
* yarn-default.xml
|
||||
* oozie-default.xml
|
||||
* hive-default.xml
|
||||
|
||||
These files are applied for Sahara's plugin of Apache Hadoop version 2.8.2
|
||||
|
||||
|
||||
Files were taken from here:
|
||||
|
||||
* `core-default.xml <https://github.com/apache/hadoop/blob/branch-2.8.2/hadoop-common-project/hadoop-common/src/main/resources/core-default.xml>`_
|
||||
* `hdfs-default.xml <https://github.com/apache/hadoop/blob/branch-2.8.2/hadoop-hdfs-project/hadoop-hdfs/src/main/resources/hdfs-default.xml>`_
|
||||
* `yarn-default.xml <https://github.com/apache/hadoop/blob/branch-2.8.2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml>`_
|
||||
* `mapred-default.xml <https://github.com/apache/hadoop/blob/branch-2.8.2/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/resources/mapred-default.xml>`_
|
||||
* `oozie-default.xml <https://github.com/apache/oozie/blob/release-4.3.0/core/src/main/resources/oozie-default.xml>`_
|
||||
|
||||
XML configs are used to expose default Hadoop configurations to the users
|
||||
through Sahara's REST API. It allows users to override some config values which
|
||||
will be pushed to the provisioned VMs running Hadoop services as part of
|
||||
appropriate xml config.
|
File diff suppressed because it is too large
Load Diff
@ -1,9 +0,0 @@
|
||||
CREATE DATABASE metastore;
|
||||
USE metastore;
|
||||
SOURCE /opt/hive/scripts/metastore/upgrade/mysql/hive-schema-2.3.0.mysql.sql;
|
||||
CREATE USER 'hive'@'localhost' IDENTIFIED BY '{{password}}';
|
||||
REVOKE ALL PRIVILEGES, GRANT OPTION FROM 'hive'@'localhost';
|
||||
GRANT ALL PRIVILEGES ON metastore.* TO 'hive'@'localhost' IDENTIFIED BY '{{password}}';
|
||||
GRANT ALL PRIVILEGES ON metastore.* TO 'hive'@'%' IDENTIFIED BY '{{password}}';
|
||||
FLUSH PRIVILEGES;
|
||||
exit
|
File diff suppressed because it is too large
Load Diff
File diff suppressed because it is too large
Load Diff
File diff suppressed because it is too large
Load Diff
File diff suppressed because it is too large
Load Diff
File diff suppressed because it is too large
Load Diff
@ -1,170 +0,0 @@
|
||||
# Copyright (c) 2015 Mirantis Inc.
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
|
||||
# implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
from oslo_config import cfg
|
||||
|
||||
from sahara.plugins import conductor
|
||||
from sahara.plugins import context
|
||||
from sahara.plugins import swift_helper
|
||||
from sahara.plugins import utils
|
||||
from sahara_plugin_vanilla.plugins.vanilla import abstractversionhandler as avm
|
||||
from sahara_plugin_vanilla.plugins.vanilla.hadoop2 import config as c
|
||||
from sahara_plugin_vanilla.plugins.vanilla.hadoop2 import keypairs
|
||||
from sahara_plugin_vanilla.plugins.vanilla.hadoop2 import recommendations_utils
|
||||
from sahara_plugin_vanilla.plugins.vanilla.hadoop2 import run_scripts as run
|
||||
from sahara_plugin_vanilla.plugins.vanilla.hadoop2 import scaling as sc
|
||||
from sahara_plugin_vanilla.plugins.vanilla.hadoop2 import starting_scripts
|
||||
from sahara_plugin_vanilla.plugins.vanilla.hadoop2 import utils as u
|
||||
from sahara_plugin_vanilla.plugins.vanilla.hadoop2 import validation as vl
|
||||
from sahara_plugin_vanilla.plugins.vanilla import utils as vu
|
||||
from sahara_plugin_vanilla.plugins.vanilla.v2_8_2 import config_helper
|
||||
from sahara_plugin_vanilla.plugins.vanilla.v2_8_2 import edp_engine
|
||||
|
||||
|
||||
CONF = cfg.CONF
|
||||
|
||||
|
||||
class VersionHandler(avm.AbstractVersionHandler):
|
||||
def __init__(self):
|
||||
self.pctx = {
|
||||
'env_confs': config_helper.get_env_configs(),
|
||||
'all_confs': config_helper.get_plugin_configs()
|
||||
}
|
||||
|
||||
def get_plugin_configs(self):
|
||||
return self.pctx['all_confs']
|
||||
|
||||
def get_node_processes(self):
|
||||
return {
|
||||
"Hadoop": [],
|
||||
"MapReduce": ["historyserver"],
|
||||
"HDFS": ["namenode", "datanode", "secondarynamenode"],
|
||||
"YARN": ["resourcemanager", "nodemanager"],
|
||||
"JobFlow": ["oozie"],
|
||||
"Hive": ["hiveserver"],
|
||||
"Spark": ["spark history server"],
|
||||
"ZooKeeper": ["zookeeper"]
|
||||
}
|
||||
|
||||
def validate(self, cluster):
|
||||
vl.validate_cluster_creating(self.pctx, cluster)
|
||||
|
||||
def update_infra(self, cluster):
|
||||
pass
|
||||
|
||||
def configure_cluster(self, cluster):
|
||||
c.configure_cluster(self.pctx, cluster)
|
||||
|
||||
def start_cluster(self, cluster):
|
||||
keypairs.provision_keypairs(cluster)
|
||||
|
||||
starting_scripts.start_namenode(cluster)
|
||||
starting_scripts.start_secondarynamenode(cluster)
|
||||
starting_scripts.start_resourcemanager(cluster)
|
||||
|
||||
run.start_dn_nm_processes(utils.get_instances(cluster))
|
||||
run.await_datanodes(cluster)
|
||||
|
||||
starting_scripts.start_historyserver(cluster)
|
||||
starting_scripts.start_oozie(self.pctx, cluster)
|
||||
starting_scripts.start_hiveserver(self.pctx, cluster)
|
||||
starting_scripts.start_zookeeper(cluster)
|
||||
|
||||
swift_helper.install_ssl_certs(utils.get_instances(cluster))
|
||||
|
||||
self._set_cluster_info(cluster)
|
||||
starting_scripts.start_spark(cluster)
|
||||
|
||||
def decommission_nodes(self, cluster, instances):
|
||||
sc.decommission_nodes(self.pctx, cluster, instances)
|
||||
|
||||
def validate_scaling(self, cluster, existing, additional):
|
||||
vl.validate_additional_ng_scaling(cluster, additional)
|
||||
vl.validate_existing_ng_scaling(self.pctx, cluster, existing)
|
||||
zk_ng = utils.get_node_groups(cluster, "zookeeper")
|
||||
if zk_ng:
|
||||
vl.validate_zookeeper_node_count(zk_ng, existing, additional)
|
||||
|
||||
def scale_cluster(self, cluster, instances):
|
||||
keypairs.provision_keypairs(cluster, instances)
|
||||
sc.scale_cluster(self.pctx, cluster, instances)
|
||||
|
||||
def _set_cluster_info(self, cluster):
|
||||
nn = vu.get_namenode(cluster)
|
||||
rm = vu.get_resourcemanager(cluster)
|
||||
hs = vu.get_historyserver(cluster)
|
||||
oo = vu.get_oozie(cluster)
|
||||
sp = vu.get_spark_history_server(cluster)
|
||||
info = {}
|
||||
|
||||
if rm:
|
||||
info['YARN'] = {
|
||||
'Web UI': 'http://%s:%s' % (rm.get_ip_or_dns_name(), '8088'),
|
||||
'ResourceManager': 'http://%s:%s' % (
|
||||
rm.get_ip_or_dns_name(), '8032')
|
||||
}
|
||||
|
||||
if nn:
|
||||
info['HDFS'] = {
|
||||
'Web UI': 'http://%s:%s' % (nn.get_ip_or_dns_name(), '50070'),
|
||||
'NameNode': 'hdfs://%s:%s' % (nn.hostname(), '9000')
|
||||
}
|
||||
|
||||
if oo:
|
||||
info['JobFlow'] = {
|
||||
'Oozie': 'http://%s:%s' % (oo.get_ip_or_dns_name(), '11000')
|
||||
}
|
||||
|
||||
if hs:
|
||||
info['MapReduce JobHistory Server'] = {
|
||||
'Web UI': 'http://%s:%s' % (hs.get_ip_or_dns_name(), '19888')
|
||||
}
|
||||
|
||||
if sp:
|
||||
info['Apache Spark'] = {
|
||||
'Spark UI': 'http://%s:%s' % (sp.management_ip, '4040'),
|
||||
'Spark History Server UI':
|
||||
'http://%s:%s' % (sp.management_ip, '18080')
|
||||
}
|
||||
|
||||
ctx = context.ctx()
|
||||
conductor.cluster_update(ctx, cluster, {'info': info})
|
||||
|
||||
def get_edp_engine(self, cluster, job_type):
|
||||
if job_type in edp_engine.EdpOozieEngine.get_supported_job_types():
|
||||
return edp_engine.EdpOozieEngine(cluster)
|
||||
if job_type in edp_engine.EdpSparkEngine.get_supported_job_types():
|
||||
return edp_engine.EdpSparkEngine(cluster)
|
||||
|
||||
return None
|
||||
|
||||
def get_edp_job_types(self):
|
||||
return (edp_engine.EdpOozieEngine.get_supported_job_types() +
|
||||
edp_engine.EdpSparkEngine.get_supported_job_types())
|
||||
|
||||
def get_edp_config_hints(self, job_type):
|
||||
return edp_engine.EdpOozieEngine.get_possible_job_config(job_type)
|
||||
|
||||
def on_terminate_cluster(self, cluster):
|
||||
u.delete_oozie_password(cluster)
|
||||
keypairs.drop_key(cluster)
|
||||
|
||||
def get_open_ports(self, node_group):
|
||||
return c.get_open_ports(node_group)
|
||||
|
||||
def recommend_configs(self, cluster, scaling):
|
||||
recommendations_utils.recommend_configs(cluster,
|
||||
self.get_plugin_configs(),
|
||||
scaling)
|
@ -1,57 +0,0 @@
|
||||
# Copyright (c) 2014 Mirantis, Inc.
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
|
||||
# implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
import os
|
||||
import re
|
||||
|
||||
from sahara.plugins import utils
|
||||
|
||||
|
||||
class VersionFactory(object):
|
||||
versions = None
|
||||
modules = None
|
||||
initialized = False
|
||||
|
||||
@staticmethod
|
||||
def get_instance():
|
||||
if not VersionFactory.initialized:
|
||||
src_dir = os.path.join(os.path.dirname(__file__), '')
|
||||
versions = (
|
||||
[name[1:].replace('_', '.')
|
||||
for name in os.listdir(src_dir)
|
||||
if (os.path.isdir(os.path.join(src_dir, name))
|
||||
and re.match(r'^v\d+_\d+_\d+$', name))])
|
||||
versions.sort(key=utils.natural_sort_key)
|
||||
VersionFactory.versions = versions
|
||||
|
||||
VersionFactory.modules = {}
|
||||
for version in VersionFactory.versions:
|
||||
module_name = ('sahara_plugin_vanilla.plugins.vanilla.v%s.'
|
||||
'versionhandler' % (version.replace('.', '_')))
|
||||
module_class = getattr(
|
||||
__import__(module_name, fromlist=['sahara']),
|
||||
'VersionHandler')
|
||||
module = module_class()
|
||||
VersionFactory.modules[version] = module
|
||||
|
||||
VersionFactory.initialized = True
|
||||
|
||||
return VersionFactory()
|
||||
|
||||
def get_versions(self):
|
||||
return VersionFactory.versions
|
||||
|
||||
def get_version_handler(self, version):
|
||||
return VersionFactory.modules[version]
|
Some files were not shown because too many files have changed in this diff Show More
Loading…
x
Reference in New Issue
Block a user