diff --git a/sahara_dashboard/api/sahara.py b/sahara_dashboard/api/sahara.py index fc05179..5d0e4cd 100644 --- a/sahara_dashboard/api/sahara.py +++ b/sahara_dashboard/api/sahara.py @@ -15,12 +15,17 @@ import logging from django.conf import settings from saharaclient.api.base import APIException +from saharaclient.api.base import Page from saharaclient import client as api_client from horizon import exceptions +from horizon.utils import functions from horizon.utils.memoized import memoized # noqa from openstack_dashboard.api import base +from sahara_dashboard import utils as u + + LOG = logging.getLogger(__name__) # "type" of Sahara service registered in keystone @@ -40,6 +45,25 @@ VERSIONS = base.APIVersionManager( VERSIONS.load_supported_version(1.1, {"client": api_client, "version": 1.1}) +SAHARA_PAGE_SIZE = 15 + + +def get_page_size(request=None): + if request: + return functions.get_page_size(request) + else: + return SAHARA_PAGE_SIZE + + +def _get_marker(request): + return request.GET["marker"] if 'marker' in request.GET else None + + +def _update_pagination_params(marker, limit, request=None): + marker = _get_marker(request) if marker is None else marker + limit = get_page_size(request) if limit is None else limit + return marker, limit + def safe_call(func, *args, **kwargs): """Call a function ignoring Not Found error @@ -166,8 +190,11 @@ def nodegroup_template_create(request, name, plugin_name, hadoop_version, is_protected=is_protected) -def nodegroup_template_list(request, search_opts=None): - return client(request).node_group_templates.list(search_opts=search_opts) +def nodegroup_template_list(request, search_opts=None, + marker=None, limit=None): + marker, limit = _update_pagination_params(marker, limit, request) + return client(request).node_group_templates.list( + search_opts=search_opts, limit=limit, marker=marker) def nodegroup_template_get(request, ngt_id): @@ -251,8 +278,12 @@ def cluster_template_create(request, name, plugin_name, hadoop_version, is_protected=is_protected) -def cluster_template_list(request, search_opts=None): - return client(request).cluster_templates.list(search_opts=search_opts) +def cluster_template_list(request, search_opts=None, marker=None, limit=None): + marker, limit = _update_pagination_params(marker, limit, request) + return client(request).cluster_templates.list( + search_opts=search_opts, + limit=limit, + marker=marker) def cluster_template_get(request, ct_id): @@ -328,8 +359,14 @@ def cluster_scale(request, cluster_id, scale_object): scale_object=scale_object) -def cluster_list(request, search_opts=None): - return client(request).clusters.list(search_opts=search_opts) +def cluster_list(request, search_opts=None, marker=None, limit=None): + marker, limit = _update_pagination_params(marker, limit, request) + return client(request).clusters.list( + search_opts=search_opts, limit=limit, marker=marker) + + +def _cluster_list(request): + return client(request).clusters.list() def cluster_get(request, cluster_id, show_progress=False): @@ -376,8 +413,12 @@ def data_source_create(request, name, description, ds_type, url, is_protected=is_protected) -def data_source_list(request, search_opts=None): - return client(request).data_sources.list(search_opts=search_opts) +def data_source_list(request, search_opts=None, limit=None, marker=None): + marker, limit = _update_pagination_params(marker, limit, request) + return client(request).data_sources.list( + search_opts=search_opts, + limit=limit, + marker=marker) def data_source_get(request, ds_id): @@ -404,8 +445,12 @@ def job_binary_create(request, name, url, description, extra, ) -def job_binary_list(request, search_opts=None): - return client(request).job_binaries.list(search_opts=search_opts) +def job_binary_list(request, search_opts=None, marker=None, limit=None): + marker, limit = _update_pagination_params(marker, limit, request) + return client(request).job_binaries.list( + search_opts=search_opts, + limit=limit, + marker=marker) def job_binary_get(request, jb_id): @@ -430,8 +475,13 @@ def job_binary_internal_create(request, name, data): data=data) -def job_binary_internal_list(request, search_opts=None): - return client(request).job_binary_internals.list(search_opts=search_opts) +def job_binary_internal_list(request, search_opts=None, + marker=None, limit=None): + marker, limit = _update_pagination_params(marker, limit, request) + return client(request).job_binary_internals.list( + search_opts=search_opts, + limit=limit, + marker=marker) def job_binary_internal_get(request, jbi_id): @@ -464,8 +514,16 @@ def job_update(request, job_id, is_public=None, is_protected=None): job_id=job_id, **prepare_acl_update_dict(is_public, is_protected)) -def job_list(request, search_opts=None): - return client(request).jobs.list(search_opts=search_opts) +def job_list(request, search_opts=None, marker=None, limit=None): + marker, limit = _update_pagination_params(marker, limit, request) + return client(request).jobs.list( + search_opts=search_opts, + limit=limit, + marker=marker) + + +def _job_list(request): + return client(request).jobs.list() def job_get(request, job_id): @@ -519,11 +577,17 @@ def _resolve_job_execution_names(job_execution, cluster=None, return job_execution -def job_execution_list(request, search_opts=None): +def job_execution_list(request, search_opts=None, marker=None, limit=None): + marker, limit = _update_pagination_params(marker, limit, request) job_execution_list = client(request).job_executions.list( - search_opts=search_opts) - job_dict = dict((j.id, j) for j in job_list(request)) - cluster_dict = dict((c.id, c) for c in cluster_list(request)) + search_opts=search_opts, limit=limit, + marker=marker) + + new_request = u.delete_pagination_params_from_request( + request, save_limit=False) + + job_dict = dict((j.id, j) for j in _job_list(new_request)) + cluster_dict = dict((c.id, c) for c in _cluster_list(new_request)) resolved_job_execution_list = [ _resolve_job_execution_names( @@ -533,7 +597,8 @@ def job_execution_list(request, search_opts=None): for job_execution in job_execution_list ] - return resolved_job_execution_list + return Page(resolved_job_execution_list, job_execution_list.prev, + job_execution_list.next, job_execution_list.limit) def job_execution_get(request, jex_id): diff --git a/sahara_dashboard/content/data_processing/clusters/cluster_templates/tables.py b/sahara_dashboard/content/data_processing/clusters/cluster_templates/tables.py index 9436ba1..0e4cbc4 100644 --- a/sahara_dashboard/content/data_processing/clusters/cluster_templates/tables.py +++ b/sahara_dashboard/content/data_processing/clusters/cluster_templates/tables.py @@ -18,8 +18,11 @@ from django.utils.translation import ugettext_lazy as _ from django.utils.translation import ungettext_lazy from horizon import tables +from horizon.tabs import base as tabs_base from sahara_dashboard.api import sahara as saharaclient +from sahara_dashboard.content.data_processing \ + import tables as sahara_table from sahara_dashboard.content.data_processing.utils \ import acl as acl_utils @@ -132,7 +135,9 @@ class MakeUnProtected(acl_utils.MakeUnProtected): request, datum_id, **update_kwargs) -class ClusterTemplatesTable(tables.DataTable): +class ClusterTemplatesTable(sahara_table.SaharaPaginateTabbedTable): + tab_name = 'cluster_tabs%sclusters_templates_tab' % tabs_base.SEPARATOR + name = tables.Column("name", verbose_name=_("Name"), link=("horizon:project:data_processing." diff --git a/sahara_dashboard/content/data_processing/clusters/clusters/tables.py b/sahara_dashboard/content/data_processing/clusters/clusters/tables.py index 2cac57e..45fc1c3 100644 --- a/sahara_dashboard/content/data_processing/clusters/clusters/tables.py +++ b/sahara_dashboard/content/data_processing/clusters/clusters/tables.py @@ -20,8 +20,11 @@ from saharaclient.api import base as api_base from horizon import messages from horizon import tables from horizon.tables import base as tables_base +from horizon.tabs import base as tabs_base from sahara_dashboard.api import sahara as saharaclient +from sahara_dashboard.content.data_processing \ + import tables as sahara_table from sahara_dashboard.content.data_processing.utils \ import acl as acl_utils from sahara_dashboard.content.data_processing.utils import helpers @@ -216,7 +219,9 @@ def get_health_filter(health): {'status': health, 'label': label}) -class ClustersTable(tables.DataTable): +class ClustersTable(sahara_table.SaharaPaginateTabbedTable): + + tab_name = 'cluster_tabs%sclusters_tab' % tabs_base.SEPARATOR class UptimeColumn(tables.Column): def get_data(self, cluster): diff --git a/sahara_dashboard/content/data_processing/clusters/nodegroup_templates/tables.py b/sahara_dashboard/content/data_processing/clusters/nodegroup_templates/tables.py index ffc9aca..612e2b1 100644 --- a/sahara_dashboard/content/data_processing/clusters/nodegroup_templates/tables.py +++ b/sahara_dashboard/content/data_processing/clusters/nodegroup_templates/tables.py @@ -16,8 +16,11 @@ from django.utils.translation import ugettext_lazy as _ from django.utils.translation import ungettext_lazy from horizon import tables +from horizon.tabs import base as tabs_base from sahara_dashboard.api import sahara as saharaclient +from sahara_dashboard.content.data_processing \ + import tables as sahara_table from sahara_dashboard.content.data_processing.utils \ import acl as acl_utils @@ -107,7 +110,10 @@ class MakeUnProtected(acl_utils.MakeUnProtected): request, datum_id, **update_kwargs) -class NodegroupTemplatesTable(tables.DataTable): +class NodegroupTemplatesTable(sahara_table.SaharaPaginateTabbedTable): + + tab_name = 'cluster_tabs%snode_group_templates_tab' % tabs_base.SEPARATOR + name = tables.Column( "name", verbose_name=_("Name"), diff --git a/sahara_dashboard/content/data_processing/clusters/templates/nodegroup_templates/nodegroup_templates.html b/sahara_dashboard/content/data_processing/clusters/templates/nodegroup_templates/nodegroup_templates.html index b40ed06..c214aab 100644 --- a/sahara_dashboard/content/data_processing/clusters/templates/nodegroup_templates/nodegroup_templates.html +++ b/sahara_dashboard/content/data_processing/clusters/templates/nodegroup_templates/nodegroup_templates.html @@ -8,4 +8,4 @@ {{ nodegroup_templates_table.render }} -{% endblock %} +{% endblock %} \ No newline at end of file diff --git a/sahara_dashboard/content/data_processing/clusters/views.py b/sahara_dashboard/content/data_processing/clusters/views.py index 66bd621..fbe075f 100644 --- a/sahara_dashboard/content/data_processing/clusters/views.py +++ b/sahara_dashboard/content/data_processing/clusters/views.py @@ -23,9 +23,11 @@ from sahara_dashboard.content.data_processing.clusters.image_registry \ import tabs as image_registry_tabs from sahara_dashboard.content.data_processing.clusters.nodegroup_templates \ import tabs as node_group_templates_tabs +from sahara_dashboard.content.data_processing.tabs \ + import PaginationFriendlyTabGroup -class ClusterTabs(tabs.TabGroup): +class ClusterTabs(PaginationFriendlyTabGroup): slug = "cluster_tabs" tabs = (clusters_tabs.ClustersTab, cluster_templates_tabs.ClusterTemplatesTab, diff --git a/sahara_dashboard/content/data_processing/jobs/data_sources/tables.py b/sahara_dashboard/content/data_processing/jobs/data_sources/tables.py index 2983e38..851488c 100644 --- a/sahara_dashboard/content/data_processing/jobs/data_sources/tables.py +++ b/sahara_dashboard/content/data_processing/jobs/data_sources/tables.py @@ -15,8 +15,11 @@ from django.utils.translation import ugettext_lazy as _ from django.utils.translation import ungettext_lazy from horizon import tables +from horizon.tabs import base as tabs_base from sahara_dashboard.api import sahara as saharaclient +from sahara_dashboard.content.data_processing \ + import tables as sahara_table from sahara_dashboard.content.data_processing.utils \ import acl as acl_utils @@ -81,7 +84,8 @@ class MakeUnProtected(acl_utils.MakeUnProtected): request, datum_id, update_kwargs) -class DataSourcesTable(tables.DataTable): +class DataSourcesTable(sahara_table.SaharaPaginateTabbedTable): + tab_name = 'job_tabs%sdata_sources_tab' % tabs_base.SEPARATOR name = tables.Column("name", verbose_name=_("Name"), link=("horizon:project:data_processing." diff --git a/sahara_dashboard/content/data_processing/jobs/job_binaries/tables.py b/sahara_dashboard/content/data_processing/jobs/job_binaries/tables.py index 48761c0..a075c18 100644 --- a/sahara_dashboard/content/data_processing/jobs/job_binaries/tables.py +++ b/sahara_dashboard/content/data_processing/jobs/job_binaries/tables.py @@ -16,8 +16,11 @@ from django.utils.translation import ungettext_lazy from saharaclient.api import base as api_base from horizon import tables +from horizon.tabs import base as tabs_base from sahara_dashboard.api import sahara as saharaclient +from sahara_dashboard.content.data_processing \ + import tables as sahara_table from sahara_dashboard.content.data_processing.utils \ import acl as acl_utils @@ -100,7 +103,8 @@ class MakeUnProtected(acl_utils.MakeUnProtected): saharaclient.job_binary_update(request, datum_id, update_kwargs) -class JobBinariesTable(tables.DataTable): +class JobBinariesTable(sahara_table.SaharaPaginateTabbedTable): + tab_name = 'job_tabs%sjob_binaries_tab' % tabs_base.SEPARATOR name = tables.Column( "name", verbose_name=_("Name"), diff --git a/sahara_dashboard/content/data_processing/jobs/job_templates/tables.py b/sahara_dashboard/content/data_processing/jobs/job_templates/tables.py index a676763..4144af1 100644 --- a/sahara_dashboard/content/data_processing/jobs/job_templates/tables.py +++ b/sahara_dashboard/content/data_processing/jobs/job_templates/tables.py @@ -17,8 +17,11 @@ from django.utils.translation import ugettext_lazy as _ from django.utils.translation import ungettext_lazy from horizon import tables +from horizon.tabs import base as tabs_base from sahara_dashboard.api import sahara as saharaclient +from sahara_dashboard.content.data_processing \ + import tables as sahara_table from sahara_dashboard.content.data_processing.utils \ import acl as acl_utils @@ -118,7 +121,8 @@ class MakeUnProtected(acl_utils.MakeUnProtected): saharaclient.job_update(request, datum_id, **update_kwargs) -class JobTemplatesTable(tables.DataTable): +class JobTemplatesTable(sahara_table.SaharaPaginateTabbedTable): + tab_name = 'job_tabs%sjob_templates_tab' % tabs_base.SEPARATOR name = tables.Column("name", verbose_name=_("Name"), link="horizon:project:data_processing.jobs:" diff --git a/sahara_dashboard/content/data_processing/jobs/jobs/tables.py b/sahara_dashboard/content/data_processing/jobs/jobs/tables.py index 918e9b1..a65f919 100644 --- a/sahara_dashboard/content/data_processing/jobs/jobs/tables.py +++ b/sahara_dashboard/content/data_processing/jobs/jobs/tables.py @@ -21,10 +21,13 @@ from saharaclient.api import base as api_base from horizon import messages from horizon import tables +from horizon.tabs import base as tabs_base from sahara_dashboard.api import sahara as saharaclient from sahara_dashboard.content.data_processing.jobs.job_templates \ import tables as j_t +from sahara_dashboard.content.data_processing \ + import tables as sahara_table from sahara_dashboard.content.data_processing.utils \ import acl as acl_utils from sahara_dashboard.content.data_processing.utils import helpers @@ -150,7 +153,9 @@ class MakeUnProtected(acl_utils.MakeUnProtected): saharaclient.job_execution_update(request, datum_id, **update_kwargs) -class JobsTable(tables.DataTable): +class JobsTable(sahara_table.SaharaPaginateTabbedTable): + tab_name = 'job_tabs%sjobs_tab' % tabs_base.SEPARATOR + class StatusColumn(tables.Column): def get_raw_data(self, datum): return datum.info['status'] diff --git a/sahara_dashboard/content/data_processing/jobs/views.py b/sahara_dashboard/content/data_processing/jobs/views.py index 5ccf730..c0cb9f4 100644 --- a/sahara_dashboard/content/data_processing/jobs/views.py +++ b/sahara_dashboard/content/data_processing/jobs/views.py @@ -24,9 +24,11 @@ from sahara_dashboard.content.data_processing.jobs.job_templates \ import tabs as job_template_tabs from sahara_dashboard.content.data_processing.jobs.jobs \ import tabs as job_tabs +from sahara_dashboard.content.data_processing.tabs \ + import PaginationFriendlyTabGroup -class JobTabs(tabs.TabGroup): +class JobTabs(PaginationFriendlyTabGroup): slug = "job_tabs" tabs = (job_tabs.JobsTab, job_template_tabs.JobTemplatesTab, diff --git a/sahara_dashboard/content/data_processing/tables.py b/sahara_dashboard/content/data_processing/tables.py new file mode 100644 index 0000000..2a75412 --- /dev/null +++ b/sahara_dashboard/content/data_processing/tables.py @@ -0,0 +1,50 @@ +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +# implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from horizon import tables + +import sahara_dashboard.api.sahara as saharaclient + + +class SaharaPaginateTabbedTable(tables.DataTable): + def get_pagination_string(self): + return "tab=%s&limit=%s&marker=%s" % ( + self.tab_name, + saharaclient.get_page_size(self.request), + self.data.next + ) + + def get_prev_pagination_string(self): + if self.data.prev is None: + return "tab=%s&limit=%s" % ( + self.tab_name, + saharaclient.get_page_size(self.request) + ) + return "tab=%s&limit=%s&marker=%s" % ( + self.tab_name, + saharaclient.get_page_size(self.request), + self.data.prev + ) + + def has_more_data(self): + return hasattr(self.data, 'next') and self.data.next is not None + + def has_prev_data(self): + if hasattr(self.data, 'prev'): + if 'tab' in self.request.GET: + if self.tab_name == self.request.GET['tab']: + if 'marker' not in self.request.GET: + return False + else: + return True + return False diff --git a/sahara_dashboard/content/data_processing/tabs.py b/sahara_dashboard/content/data_processing/tabs.py index df81956..77ba501 100644 --- a/sahara_dashboard/content/data_processing/tabs.py +++ b/sahara_dashboard/content/data_processing/tabs.py @@ -11,7 +11,11 @@ # See the License for the specific language governing permissions and # limitations under the License. +from horizon import exceptions from horizon import tabs +from horizon.tabs import base + +from sahara_dashboard import utils as u class SaharaTableTab(tabs.TableTab): @@ -42,3 +46,32 @@ class SaharaTableTab(tabs.TableTab): 'changed': changed } return filter_info + + +class PaginationFriendlyTabGroup(tabs.TabGroup): + def load_tab_data(self): + """Preload all data that for the tabs that will be displayed.""" + request_without_marker = u.delete_pagination_params_from_request( + self.request, save_limit=True) + for tab in self._tabs.values(): + current_tab_id = tab.slug + + tab_request = self.request.GET.get('tab') + request_tab_id = None + + if tab_request: + request_tab_id = tab_request.split(base.SEPARATOR)[1] + if request_tab_id and current_tab_id != request_tab_id: + try: + tab.request = request_without_marker + tab._data = tab.get_context_data(request_without_marker) + except Exception: + tab._data = False + exceptions.handle(request_without_marker) + + if tab.load and not tab.data_loaded: + try: + tab._data = tab.get_context_data(self.request) + except Exception: + tab._data = False + exceptions.handle(self.request) diff --git a/sahara_dashboard/utils.py b/sahara_dashboard/utils.py index c8729be..c0fc1a1 100644 --- a/sahara_dashboard/utils.py +++ b/sahara_dashboard/utils.py @@ -14,7 +14,9 @@ # limitations under the License. import base64 +import copy import six +from six.moves.urllib import parse def serialize(obj): @@ -41,3 +43,29 @@ def deserialize(obj): if six.PY3: result = result.decode() return result + + +def delete_pagination_params_from_request(request, save_limit=None): + """Delete marker and limit parameters from GET requests + :param request: instance of GET request + :param save_limit: if True, 'limit' will not be deleted + :return: instance of GET request without marker or limit + """ + request = copy.copy(request) + request.GET = request.GET.copy() + + params = ['marker'] + if not save_limit: + params.append('limit') + + for param in ['marker', 'limit']: + if param in request.GET: + del(request.GET[param]) + query_string = request.META.get('QUERY_STRING', '') + query_dict = parse.parse_qs(query_string) + if param in query_dict: + del(query_dict[param]) + + query_string = parse.urlencode(query_dict, doseq=True) + request.META['QUERY_STRING'] = query_string + return request