Data Sources panel for Sahara

The Data Sources panel allows for the creation
of data sources that can eventually be used
as input and output for Elastic Data Processing
jobs.
This code was originally from:
https://github.com/openstack/sahara-dashboard

Change-Id: I61f5522755de6adcdef4c2f3ce61101918e2768a
Partial-Implements: blueprint merge-sahara-dashboard
Co-Authored-By: Nikita Konovalov <nkonovalov@mirantis.com>
Co-Authored-By: Dmitry Mescheryakov <dmescheryakov@mirantis.com>
This commit is contained in:
Chad Roberts 2014-04-29 11:35:00 -04:00
parent d38189c0c9
commit 991882ca4a
17 changed files with 534 additions and 1 deletions

View File

@ -64,7 +64,8 @@ class DataProcessingPanels(horizon.PanelGroup):
'data_processing.data_image_registry',
'data_processing.nodegroup_templates',
'data_processing.cluster_templates',
'data_processing.clusters', )
'data_processing.clusters',
'data_processing.data_sources', )
class Project(horizon.Dashboard):

View File

@ -0,0 +1,27 @@
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
# implied.
# See the License for the specific language governing permissions and
# limitations under the License.
from django.utils.translation import ugettext_lazy as _
import horizon
from openstack_dashboard.dashboards.project import dashboard
class DataSourcesPanel(horizon.Panel):
name = _("Data Sources")
slug = 'data_processing.data_sources'
permissions = ('openstack.services.data_processing',)
dashboard.Project.register(DataSourcesPanel)

View File

@ -0,0 +1,58 @@
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
# implied.
# See the License for the specific language governing permissions and
# limitations under the License.
import logging
from django.utils.translation import ugettext_lazy as _
from horizon import tables
from openstack_dashboard.api import sahara as saharaclient
LOG = logging.getLogger(__name__)
class CreateDataSource(tables.LinkAction):
name = "create data source"
verbose_name = _("Create Data Source")
url = "horizon:project:data_processing.data_sources:create-data-source"
classes = ("btn-launch", "ajax-modal")
class DeleteDataSource(tables.BatchAction):
name = "delete"
action_present = _("Delete")
action_past = _("Deleted")
data_type_singular = _("Data source")
data_type_plural = _("Data sources")
classes = ('btn-danger', 'btn-terminate')
def action(self, request, obj_id):
saharaclient.data_source_delete(request, obj_id)
class DataSourcesTable(tables.DataTable):
name = tables.Column("name",
verbose_name=_("Name"),
link=("horizon:project:data_processing.data_sources:details"))
type = tables.Column("type",
verbose_name=_("Type"))
description = tables.Column("description",
verbose_name=_("Description"))
class Meta:
name = "data_sources"
verbose_name = _("Data Sources")
table_actions = (CreateDataSource,
DeleteDataSource)
row_actions = (DeleteDataSource,)

View File

@ -0,0 +1,47 @@
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
# implied.
# See the License for the specific language governing permissions and
# limitations under the License.
import logging
from django.utils.translation import ugettext_lazy as _
from horizon import exceptions
from horizon import tabs
from openstack_dashboard.api import sahara as saharaclient
LOG = logging.getLogger(__name__)
class GeneralTab(tabs.Tab):
name = _("General Info")
slug = "data_source_details_tab"
template_name = ("project/data_processing.data_sources/_details.html")
def get_context_data(self, request):
data_source_id = self.tab_group.kwargs['data_source_id']
try:
data_source = saharaclient.data_source_get(request,
data_source_id)
except Exception:
exceptions.handle(self.tab_group.request,
_("Unable to retreive data source details"))
data_source = {}
return {"data_source": data_source}
class DataSourceDetailsTabs(tabs.TabGroup):
slug = "data_source_details"
tabs = (GeneralTab,)
sticky = True

View File

@ -0,0 +1,15 @@
{% load i18n horizon %}
<div class="well">
<p>
{% blocktrans %}Create a Data Source with a specified name.{% endblocktrans %}
</p>
<p>
{% blocktrans %}Select the type of your Data Source.{% endblocktrans %}
</p>
<p>
{% blocktrans %}You may need to enter the username and password for your Data Source.{% endblocktrans %}
</p>
<p>
{% blocktrans %}You may also enter an optional description for your Data Source.{% endblocktrans %}
</p>
</div>

View File

@ -0,0 +1,20 @@
{% load i18n sizeformat %}
<h3>{% trans "Data Source Overview" %}</h3>
<div class="status row-fluid detail">
<dl>
<dt>{% trans "Name" %}</dt>
<dd>{{ data_source.name }}</dd>
<dt>{% trans "ID" %}</dt>
<dd>{{ data_source.id }}</dd>
<dt>{% trans "Type" %}</dt>
<dd>{{ data_source.type }}</dd>
<dt>{% trans "URL" %}</dt>
<dd>{{ data_source.url }}</dd>
<dt>{% trans "Tenant id" %}</dt>
<dd>{{ data_source.tenant_id }}</dd>
<dt>{% trans "Description" %}</dt>
<dd>{{ data_source.description|default:"None" }}</dd>
<dt>{% trans "Create time" %}</dt>
<dd>{{ data_source.created_at }}</dd>
</dl>
</div>

View File

@ -0,0 +1,11 @@
{% extends 'base.html' %}
{% load i18n %}
{% block title %}{% trans "Create Data Source" %}{% endblock %}
{% block page_header %}
{% include "horizon/common/_page_header.html" with title=_("Create Data Source") %}
{% endblock page_header %}
{% block main %}
{% include 'horizon/common/_workflow.html' %}
{% endblock %}

View File

@ -0,0 +1,21 @@
{% extends 'base.html' %}
{% load i18n %}
{% block title %}{% trans "Data Processing" %}{% endblock %}
{% block page_header %}
{% include "horizon/common/_page_header.html" with title=_("Data Sources") %}
{% endblock page_header %}
{% block main %}
<style type="text/css">
#id_data_source_url {
width: 250px !important; }
</style>
<div class="data_sources">
{{ data_sources_table.render }}
</div>
{% include "project/data_processing.data_sources/data_sources_form_script.html" %}
{% endblock %}

View File

@ -0,0 +1,25 @@
<script type="text/javascript">
addHorizonLoadEvent(function () {
horizon.modals.addModalInitFunction(function (modal) {
$("#id_data_source_type").change(function() {
var label = $("#id_data_source_url_label");
var username = $("#id_data_source_credential_user").closest(".control-group");
var password = $("#id_data_source_credential_pass").closest(".control-group")
switch($(this).val()) {
case "hdfs":
label.html("hdfs://");
username.hide();
password.hide();
break;
case "swift":
username.show();
password.show();
label.html("swift://");
break;
}
});
$("#id_data_source_type").change();
});
});
</script>

View File

@ -0,0 +1,15 @@
{% extends 'base.html' %}
{% load i18n %}
{% block title %}{% trans "Data Source Details" %}{% endblock %}
{% block page_header %}
{% include "horizon/common/_page_header.html" with title=_("Data Source Details") %}
{% endblock page_header %}
{% block main %}
<div class="row-fluid">
<div class="span12">
{{ tab_group.render }}
</div>
</div>
{% endblock %}

View File

@ -0,0 +1,50 @@
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
from django.core.urlresolvers import reverse
from django import http
from mox import IsA # noqa
from openstack_dashboard import api
from openstack_dashboard.test import helpers as test
INDEX_URL = reverse('horizon:project:data_processing.data_sources:index')
DETAILS_URL = reverse(
'horizon:project:data_processing.data_sources:details', args=['id'])
class DataProcessingDataSourceTests(test.TestCase):
@test.create_stubs({api.sahara: ('data_source_list',)})
def test_index(self):
api.sahara.data_source_list(IsA(http.HttpRequest)) \
.AndReturn(self.data_sources.list())
self.mox.ReplayAll()
res = self.client.get(INDEX_URL)
self.assertTemplateUsed(res,
'project/data_processing.data_sources/data_sources.html')
self.assertContains(res, 'Data Sources')
self.assertContains(res, 'Name')
self.assertContains(res, 'sampleOutput')
self.assertContains(res, 'sampleOutput2')
@test.create_stubs({api.sahara: ('data_source_get',)})
def test_details(self):
api.sahara.data_source_get(IsA(http.HttpRequest), IsA(unicode)) \
.AndReturn(self.data_sources.list()[0])
self.mox.ReplayAll()
res = self.client.get(DETAILS_URL)
self.assertTemplateUsed(res,
'project/data_processing.data_sources/details.html')
self.assertContains(res, 'sampleOutput')
self.assertContains(res, 'Data Source Details')

View File

@ -0,0 +1,32 @@
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
# implied.
# See the License for the specific language governing permissions and
# limitations under the License.
from django.conf.urls import patterns # noqa
from django.conf.urls import url # noqa
import openstack_dashboard.dashboards.project.data_processing. \
data_sources.views as views
urlpatterns = patterns('',
url(r'^$', views.DataSourcesView.as_view(),
name='index'),
url(r'^$', views.DataSourcesView.as_view(),
name='data-sources'),
url(r'^create-data-source$',
views.CreateDataSourceView.as_view(),
name='create-data-source'),
url(r'^(?P<data_source_id>[^/]+)$',
views.DataSourceDetailsView.as_view(),
name='details'))

View File

@ -0,0 +1,59 @@
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
# implied.
# See the License for the specific language governing permissions and
# limitations under the License.
import logging
from django.utils.translation import ugettext_lazy as _
from horizon import exceptions
from horizon import tables
from horizon import tabs
from horizon import workflows
from openstack_dashboard.api import sahara as saharaclient
import openstack_dashboard.dashboards.project.data_processing.data_sources. \
tables as ds_tables
import openstack_dashboard.dashboards.project.data_processing.data_sources. \
tabs as _tabs
import openstack_dashboard.dashboards.project.data_processing.data_sources. \
workflows.create as create_flow
LOG = logging.getLogger(__name__)
class DataSourcesView(tables.DataTableView):
table_class = ds_tables.DataSourcesTable
template_name = 'project/data_processing.data_sources/data_sources.html'
def get_data(self):
try:
data_sources = saharaclient.data_source_list(self.request)
except Exception:
data_sources = []
exceptions.handle(self.request,
_("Unable to fetch data sources."))
return data_sources
class CreateDataSourceView(workflows.WorkflowView):
workflow_class = create_flow.CreateDataSource
success_url = \
"horizon:project:data_processing.data-sources:create-data-source"
classes = ("ajax-modal")
template_name = "project/data_processing.data_sources/create.html"
class DataSourceDetailsView(tabs.TabView):
tab_group_class = _tabs.DataSourceDetailsTabs
template_name = 'project/data_processing.data_sources/details.html'

View File

@ -0,0 +1,120 @@
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
# implied.
# See the License for the specific language governing permissions and
# limitations under the License.
import logging
from django.forms import util
from django.forms import widgets
from django.utils.safestring import mark_safe
from django.utils.translation import ugettext_lazy as _
from horizon import exceptions
from horizon import forms
from horizon import workflows
from openstack_dashboard.api import sahara as saharaclient
LOG = logging.getLogger(__name__)
class LabeledInput(widgets.Input):
def render(self, name, values, attrs=None):
final_attrs = self.build_attrs(attrs, type=self.input_type, name=name)
output = "<span id='%s'>%s</span>%s" %\
("id_%s_label" % name,
"swift://",
('<input%s />' % util.flatatt(final_attrs)))
return mark_safe(output)
class GeneralConfigAction(workflows.Action):
data_source_name = forms.CharField(label=_("Name"))
data_source_type = forms.ChoiceField(
label=_("Data Source Type"),
choices=[("swift", "Swift"), ("hdfs", "HDFS")],
widget=forms.Select(attrs={"class": "data_source_type_choice"}))
data_source_url = forms.CharField(label=_("URL"),
widget=LabeledInput())
data_source_credential_user = forms.CharField(label=_("Source username"))
data_source_credential_pass = forms.CharField(
widget=forms.PasswordInput(attrs={'autocomplete': 'off'}),
label=_("Source password"))
data_source_description = forms.CharField(
label=_("Description"),
required=False,
widget=forms.Textarea)
def clean(self):
cleaned_data = super(workflows.Action, self).clean()
ds_type = cleaned_data.get("data_source_type", "")
if ds_type == "hdfs":
if "data_source_credential_user" in self._errors:
del self._errors["data_source_credential_user"]
if "data_source_credential_pass" in self._errors:
del self._errors["data_source_credential_pass"]
return cleaned_data
def __init__(self, request, *args, **kwargs):
super(GeneralConfigAction, self).__init__(request, *args, **kwargs)
class Meta:
name = _("Create Data Source")
help_text_template = ("project/data_processing.data_sources/"
"_create_data_source_help.html")
class GeneralConfig(workflows.Step):
action_class = GeneralConfigAction
def contribute(self, data, context):
for k, v in data.items():
context["general_" + k] = v
context["source_url"] = "%s://%s" % \
(context["general_data_source_type"],
context["general_data_source_url"])
return context
class CreateDataSource(workflows.Workflow):
slug = "create_data_source"
name = _("Create Data Source")
finalize_button_name = _("Create")
success_message = _("Data source created")
failure_message = _("Could not create data source")
success_url = "horizon:project:data_processing.data_sources:index"
default_steps = (GeneralConfig, )
def handle(self, request, context):
try:
saharaclient.data_source_create(
request,
context["general_data_source_name"],
context["general_data_source_description"],
context["general_data_source_type"],
context["source_url"],
context.get("general_data_source_credential_user", None),
context.get("general_data_source_credential_pass", None))
return True
except Exception:
exceptions.handle(request)
return False

View File

@ -14,6 +14,7 @@ from openstack_dashboard.test.test_data import utils
from saharaclient.api import cluster_templates
from saharaclient.api import clusters
from saharaclient.api import data_sources
from saharaclient.api import node_group_templates
from saharaclient.api import plugins
@ -23,6 +24,7 @@ def data(TEST):
TEST.nodegroup_templates = utils.TestDataContainer()
TEST.cluster_templates = utils.TestDataContainer()
TEST.clusters = utils.TestDataContainer()
TEST.data_sources = utils.TestDataContainer()
plugin1_dict = {
"description": "vanilla plugin",
@ -229,3 +231,33 @@ def data(TEST):
cluster1 = clusters.Cluster(
clusters.ClusterManager(None), cluster1_dict)
TEST.clusters.add(cluster1)
#Data Sources
data_source1_dict = {
"created_at": "2014-06-04 14:01:10.371562",
"description": "sample output",
"id": "426fb01c-5c7e-472d-bba2-b1f0fe7e0ede",
"name": "sampleOutput",
"tenant_id": "429ad8447c2d47bc8e0382d244e1d1df",
"type": "swift",
"updated_at": None,
"url": "swift://example.sahara/output"
}
data_source2_dict = {
"created_at": "2014-06-05 15:01:12.331361",
"description": "second sample output",
"id": "ab3413-adfb-bba2-123456785675",
"name": "sampleOutput2",
"tenant_id": "429ad8447c2d47bc8e0382d244e1d1df",
"type": "hdfs",
"updated_at": None,
"url": "hdfs://example.sahara/output"
}
data_source1 = data_sources.DataSources(
data_sources.DataSourceManager(None), data_source1_dict)
data_source2 = data_sources.DataSources(
data_sources.DataSourceManager(None), data_source2_dict)
TEST.data_sources.add(data_source1)
TEST.data_sources.add(data_source2)