openstack/deb-murano-dashboard
Code Issues Proposed changes

Refactor Packages and AppCatalog pagination

Browse Source 
Move function for getting list of Application packages to a separate
module - muranodashboard.api.packages, so both Packages and AppCatalog
views can reuse it. That `package_list()` function obtains packages
generator with a call to `muranoclient.packages.filter()` and paginate
the results, adding `has_more_data` flag (as it is done for Glance,
see http://tinyurl.com/lkbnsed).

Also move muranoclient-specific functions from
muranodashboard.environments.api to muranodashboard.api module.

Implements: blueprint app-catalog-pagination
Change-Id: Ic3a433c8467fda94ac7dd3800c29fc1c2b9f3464
This commit is contained in:
Timur Sufiev
2014-05-16 15:38:45 +04:00
parent 17f41b4e75
commit 7be866fcfd
11 changed files with 216 additions and 149 deletions
Download Patch File Download Diff File Expand all files Collapse all files

93
muranodashboard/api/__init__.py Normal file
View File

@@ -0,0 +1,93 @@
# Copyright (c) 2014 Mirantis, Inc.
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
import contextlib
import logging
from django.conf import settings
from django.contrib.messages import api as msg_api
from django.utils.encoding import force_unicode
from django.utils.translation import ugettext_lazy as _
from horizon import exceptions
from openstack_dashboard.api import base
import muranoclient.client as client
from muranoclient.common import exceptions as exc
LOG = logging.getLogger(__name__)
def _handle_message(request, message):
def horizon_message_already_queued(_message):
_message = force_unicode(_message)
if request.is_ajax():
for tag, msg, extra in request.horizon['async_messages']:
if _message == msg:
return True
else:
for msg in msg_api.get_messages(request):
if msg.message == _message:
return True
return False
if horizon_message_already_queued(message):
exceptions.handle(request, ignore=True)
else:
exceptions.handle(request, message=message)
@contextlib.contextmanager
def handled_exceptions(request):
"""Handles all murano-api specific exceptions."""
try:
yield
except exc.CommunicationError:
msg = _('Unable to communicate to murano-api server.')
LOG.exception(msg)
_handle_message(request, msg)
except exc.Unauthorized:
msg = _('Check Keystone configuration of murano-api server.')
LOG.exception(msg)
_handle_message(request, msg)
except exc.Forbidden:
msg = _('Operation is forbidden by murano-api server.')
LOG.exception(msg)
_handle_message(request, msg)
def _get_endpoint(request):
#prefer location specified in settings for dev purposes
endpoint = getattr(settings, 'MURANO_API_URL', None)
if not endpoint:
try:
endpoint = base.url_for(request, 'application_catalog')
except exceptions.ServiceCatalogException:
endpoint = 'http://localhost:8082'
LOG.warning('Murano API location could not be found in Service '
'Catalog, using default: {0}'.format(endpoint))
return endpoint
def muranoclient(request):
endpoint = _get_endpoint(request)
insecure = getattr(settings, 'MURANO_API_INSECURE', False)
token_id = request.user.token.id
LOG.debug('Murano::Client <Url: {0}, '
'TokenId: {1}>'.format(endpoint, token_id))
return client.Client(1, endpoint=endpoint, token=token_id,
insecure=insecure)

49
muranodashboard/api/packages.py Normal file
View File

@@ -0,0 +1,49 @@
# Copyright (c) 2014 Mirantis, Inc.
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
import itertools
from django.conf import settings
from muranodashboard import api
def package_list(request, marker=None, filters=None, paginate=False,
page_size=20):
limit = getattr(settings, 'PACKAGES_LIMIT', 100)
filters = filters or {}
if paginate:
request_size = page_size + 1
else:
request_size = limit
if marker:
filters['marker'] = marker
client = api.muranoclient(request)
packages_iter = client.packages.filter(page_size=request_size,
limit=limit,
**filters)
has_more_data = False
if paginate:
packages = list(itertools.islice(packages_iter, request_size))
if len(packages) > page_size:
packages.pop()
has_more_data = True
else:
packages = list(packages_iter)
return packages, has_more_data

28
muranodashboard/catalog/views.py
View File

@@ -36,11 +36,13 @@ from horizon import messages
from horizon import tabs
from muranoclient.common import exceptions as exc
from muranodashboard import api
from muranodashboard.api import packages as pkg_api
from muranodashboard.catalog import tabs as catalog_tabs
from muranodashboard.common import cache
from muranodashboard.dynamic_ui import helpers
from muranodashboard.dynamic_ui import services
from muranodashboard.environments import api
from muranodashboard.environments import api as env_api
from muranodashboard.environments import consts
from muranodashboard import utils
@@ -58,7 +60,7 @@ class DictToObj(object):
def get_available_environments(request):
envs = []
for env in api.environments_list(request):
for env in env_api.environments_list(request):
obj = DictToObj(id=env.id, name=env.name, status=env.status)
envs.append(obj)
@@ -105,14 +107,14 @@ def create_quick_environment(request):
match = re.match(quick_env_re, env.name)
return int(match.group(1)) if match else 0
numbers = [parse_number(e) for e in api.environments_list(request)]
numbers = [parse_number(e) for e in env_api.environments_list(request)]
new_env_number = 1
if numbers:
numbers.sort()
new_env_number = numbers[-1] + 1
params = {'name': quick_env_prefix + str(new_env_number)}
return api.environment_create(request, params)
return env_api.environment_create(request, params)
def update_latest_apps(func):
@@ -142,7 +144,7 @@ def quick_deploy(request, app_id):
return view(request, app_id=app_id, environment_id=env.id,
do_redirect=True, drop_wm_form=True)
except Exception:
api.environment_delete(request, env.id)
env_api.environment_delete(request, env.id)
raise
@@ -235,7 +237,8 @@ class Wizard(views.ModalFormMixin, LazyWizard):
fail_url = reverse("horizon:murano:environments:index")
try:
srv = api.service_create(self.request, environment_id, attributes)
srv = env_api.service_create(
self.request, environment_id, attributes)
except exc.HTTPForbidden:
msg = _("Sorry, you can't add application right now. "
"The environment is deploying.")
@@ -316,15 +319,14 @@ class IndexView(list_view.ListView):
if category != ALL_CATEGORY_NAME:
query_params['category'] = category
pkgs, self.mappings = [], {}
packages = []
with api.handled_exceptions(self.request):
client = api.muranoclient(self.request)
pkgs = client.packages.filter(**query_params)
packages, has_more = pkg_api.package_list(
self.request, filters=query_params, paginate=True,
page_size=self.paginate_by)
for pkg in pkgs:
self.mappings[pkg.id] = pkg
return pkgs
self.mappings = dict((pkg.id, pkg) for pkg in packages)
return packages
def get_template_names(self):
return ['catalog/index.html']

4
muranodashboard/dynamic_ui/fields.py
View File

@@ -18,6 +18,7 @@ import logging
import netaddr
import re
from django.core.urlresolvers import reverse
from django.core import validators as django_validator
from django import forms
from django.template import defaultfilters
@@ -36,7 +37,6 @@ import yaql
from muranoclient.common import exceptions as muranoclient_exc
from muranodashboard.environments import api
from django.core import urlresolvers as urls
LOG = logging.getLogger(__name__)
@@ -675,7 +675,7 @@ def make_select_cls(fqn):
msg = "Application with FQN='{0}' doesn't exist"
raise KeyError(msg.format(_fqn))
args = (environment_id, _app.id, False, True)
return _app.name, urls.reverse(ns_url, args=args)
return _app.name, reverse(ns_url, args=args)
return json.dumps(
[_reverse(cls) for cls in api.split_classes(fqn)])

4
muranodashboard/dynamic_ui/services.py
View File

@@ -18,6 +18,7 @@ import re
import yaml
import yaql
from muranodashboard import api
from muranodashboard.catalog import forms as catalog_forms
from muranodashboard.common import cache
from muranodashboard.dynamic_ui import helpers
@@ -163,8 +164,6 @@ def make_loader_cls():
def import_app(request, app_id):
from muranodashboard.environments import api
if not request.session.get('apps'):
request.session['apps'] = {}
services = request.session['apps']
@@ -209,7 +208,6 @@ def service_type_from_id(service_id):
def get_service_name(request, app_id):
from muranodashboard.environments import api
app = api.muranoclient(request).packages.get(app_id)
return app.name

150
muranodashboard/environments/api.py
View File

@@ -12,18 +12,10 @@
# License for the specific language governing permissions and limitations
# under the License.
import contextlib
import logging
from django.conf import settings
from django.contrib.messages import api as msg_api
from django.utils.encoding import force_unicode
from django.utils.translation import ugettext_lazy as _
from horizon import exceptions
from openstack_dashboard.api import base
import muranoclient.client as client
from muranoclient.common import exceptions as exc
from muranodashboard import api
from muranodashboard.common import utils
from muranodashboard.environments import consts
from muranodashboard.environments import topology
@@ -32,82 +24,17 @@ from muranodashboard.environments import topology
LOG = logging.getLogger(__name__)
def _handle_message(request, message):
def horizon_message_already_queued(_message):
_message = force_unicode(_message)
if request.is_ajax():
for tag, msg, extra in request.horizon['async_messages']:
if _message == msg:
return True
else:
for msg in msg_api.get_messages(request):
if msg.message == _message:
return True
return False
if horizon_message_already_queued(message):
exceptions.handle(request, ignore=True)
else:
exceptions.handle(request, message=message)
@contextlib.contextmanager
def handled_exceptions(request):
"""Handles all murano-api specific exceptions."""
try:
yield
except exc.CommunicationError:
msg = _('Unable to communicate to murano-api server.')
LOG.exception(msg)
_handle_message(request, msg)
except exc.Unauthorized:
msg = _('Check Keystone configuration of murano-api server.')
LOG.exception(msg)
_handle_message(request, msg)
except exc.Forbidden:
msg = _('Operation is forbidden by murano-api server.')
LOG.exception(msg)
_handle_message(request, msg)
def get_endpoint(request):
#prefer location specified in settings for dev purposes
endpoint = getattr(settings, 'MURANO_API_URL', None)
if not endpoint:
try:
endpoint = base.url_for(request, 'application_catalog')
except exceptions.ServiceCatalogException:
endpoint = 'http://localhost:8082'
LOG.warning('Murano API location could not be found in Service '
'Catalog, using default: {0}'.format(endpoint))
return endpoint
def muranoclient(request):
endpoint = get_endpoint(request)
insecure = getattr(settings, 'MURANO_API_INSECURE', False)
token_id = request.user.token.id
LOG.debug('Murano::Client <Url: {0}, '
'TokenId: {1}>'.format(endpoint, token_id))
return client.Client(
1, endpoint=endpoint, token=token_id, insecure=insecure)
def get_status_messages_for_service(request, service_id, environment_id):
deployments = muranoclient(request).deployments. \
list(environment_id)
client = api.muranoclient(request)
deployments = client.deployments.list(environment_id)
LOG.debug('Deployment::List {0}'.format(deployments))
result = '\n'
# TODO(efedorova): Add updated time to logs
if deployments:
for deployment in reversed(deployments):
reports = muranoclient(request).deployments.reports(environment_id,
deployment.id,
service_id)
reports = client.deployments.reports(
environment_id, deployment.id, service_id)
for report in reports:
result += report.created.replace('T', ' ') + ' - ' + str(
@@ -132,7 +59,8 @@ class Session(object):
id = sessions[environment_id]
else:
id = muranoclient(request).sessions.configure(environment_id).id
client = api.muranoclient(request)
id = client.sessions.configure(environment_id).id
sessions[environment_id] = id
request.session['sessions'] = sessions
@@ -151,9 +79,10 @@ class Session(object):
"""
sessions = request.session.get('sessions', {})
client = api.muranoclient(request)
def create_session(request, environment_id):
id = muranoclient(request).sessions.configure(environment_id).id
id = client.sessions.configure(environment_id).id
sessions[environment_id] = id
request.session['sessions'] = sessions
return id
@@ -161,8 +90,7 @@ class Session(object):
if environment_id in sessions:
id = sessions[environment_id]
try:
session_data = \
muranoclient(request).sessions.get(environment_id, id)
session_data = client.sessions.get(environment_id, id)
except exc.HTTPForbidden:
del sessions[environment_id]
LOG.debug("The environment is deploying by other user."
@@ -207,8 +135,9 @@ class Session(object):
def environments_list(request):
environments = []
with handled_exceptions(request):
environments = muranoclient(request).environments.list()
client = api.muranoclient(request)
with api.handled_exceptions(request):
environments = client.environments.list()
LOG.debug('Environment::List {0}'.format(environments))
for index, env in enumerate(environments):
environments[index].has_services = False
@@ -226,21 +155,22 @@ def environments_list(request):
def environment_create(request, parameters):
#name is required param
body = {'name': parameters['name']}
env = muranoclient(request).environments.create(body)
env = api.muranoclient(request).environments.create(body)
LOG.debug('Environment::Create {0}'.format(env))
return env
def environment_delete(request, environment_id):
LOG.debug('Environment::Delete <Id: {0}>'.format(environment_id))
return muranoclient(request).environments.delete(environment_id)
return api.muranoclient(request).environments.delete(environment_id)
def environment_get(request, environment_id):
session_id = Session.get(request, environment_id)
LOG.debug('Environment::Get <Id: {0}, SessionId: {1}>'.
format(environment_id, session_id))
env = muranoclient(request).environments.get(environment_id, session_id)
client = api.muranoclient(request)
env = client.environments.get(environment_id, session_id)
LOG.debug('Environment::Get {0}'.format(env))
return env
@@ -248,14 +178,14 @@ def environment_get(request, environment_id):
def environment_deploy(request, environment_id):
session_id = Session.get(request, environment_id)
LOG.debug('Session::Get <Id: {0}>'.format(session_id))
env = muranoclient(request).sessions.deploy(environment_id, session_id)
env = api.muranoclient(request).sessions.deploy(environment_id, session_id)
LOG.debug('Environment::Deploy <EnvId: {0}, SessionId: {1}>'
''.format(environment_id, session_id))
return env
def environment_update(request, environment_id, name):
return muranoclient(request).environments.update(environment_id, name)
return api.muranoclient(request).environments.update(environment_id, name)
def services_list(request, environment_id):
@@ -266,11 +196,11 @@ def services_list(request, environment_id):
# need to create new session to see services deployed be other user
session_id = Session.get_or_create(request, environment_id)
get_environment = muranoclient(request).environments.get
get_environment = api.muranoclient(request).environments.get
environment = get_environment(environment_id, session_id)
try:
reports = muranoclient(request).environments.last_status(
environment_id, session_id)
client = api.muranoclient(request)
reports = client.environments.last_status(environment_id, session_id)
except exc.HTTPNotFound:
reports = {}
@@ -299,14 +229,12 @@ def services_list(request, environment_id):
return [utils.Bunch(**service) for service in services]
def app_id_by_fqn(request, fqn):
apps = muranoclient(request).packages.filter(fqn=fqn)
return apps[0].id if apps else None
def app_by_fqn(request, fqn):
apps = muranoclient(request).packages.filter(fqn=fqn)
return apps[0] if apps else None
apps = api.muranoclient(request).packages.filter(fqn=fqn)
try:
return apps.next()
except StopIteration:
return None
def split_classes(classes_str):
@@ -325,18 +253,18 @@ def service_create(request, environment_id, parameters):
# if we what add new services to this environment
session_id = Session.get_or_create_or_delete(request, environment_id)
LOG.debug('Service::Create {0}'.format(parameters['?']['type']))
return muranoclient(request).services.post(environment_id,
path='/',
data=parameters,
session_id=session_id)
return api.muranoclient(request).services.post(environment_id,
path='/',
data=parameters,
session_id=session_id)
def service_delete(request, environment_id, service_id):
LOG.debug('Service::Delete <SrvId: {0}>'.format(service_id))
session_id = Session.get_or_create_or_delete(request, environment_id)
return muranoclient(request).services.delete(environment_id,
'/' + service_id,
session_id)
return api.muranoclient(request).services.delete(environment_id,
'/' + service_id,
session_id)
def service_get(request, environment_id, service_id):
@@ -349,7 +277,7 @@ def service_get(request, environment_id, service_id):
def deployments_list(request, environment_id):
LOG.debug('Deployments::List')
deployments = muranoclient(request).deployments.list(environment_id)
deployments = api.muranoclient(request).deployments.list(environment_id)
for deployment in deployments:
if deployment.started:
deployment.started = deployment.started.replace('T', ' ')
@@ -362,14 +290,14 @@ def deployments_list(request, environment_id):
def deployment_reports(request, environment_id, deployment_id):
LOG.debug('Deployment::Reports::List')
reports = muranoclient(request).deployments.reports(environment_id,
deployment_id)
reports = api.muranoclient(request).deployments.reports(environment_id,
deployment_id)
LOG.debug('Deployment::Reports::List {0}'.format(reports))
return reports
def get_deployment_start(request, environment_id, deployment_id):
deployments = muranoclient(request).deployments.list(environment_id)
deployments = api.muranoclient(request).deployments.list(environment_id)
LOG.debug('Get deployment start time')
for deployment in deployments:
if deployment.id == deployment_id:
@@ -378,7 +306,7 @@ def get_deployment_start(request, environment_id, deployment_id):
def get_deployment_descr(request, environment_id, deployment_id):
deployments = muranoclient(request).deployments.list(environment_id)
deployments = api.muranoclient(request).deployments.list(environment_id)
LOG.debug('Get deployment description')
for deployment in deployments:
if deployment.id == deployment_id:

2
muranodashboard/packages/forms.py
View File

@@ -24,8 +24,8 @@ from horizon import forms as horizon_forms
from horizon import messages
from muranoclient.common import exceptions as exc
from muranodashboard import api
from muranodashboard.catalog import views
from muranodashboard.environments import api
LOG = logging.getLogger(__name__)

2
muranodashboard/packages/tables.py
View File

@@ -23,7 +23,7 @@ from horizon import messages
from horizon import tables
from muranoclient.common import exceptions as exc
from muranodashboard.environments import api
from muranodashboard import api
LOG = logging.getLogger(__name__)

23
muranodashboard/packages/views.py
View File

@@ -17,8 +17,10 @@ import logging
from django.core.urlresolvers import reverse_lazy
from horizon.forms import views
from horizon import tables as horizon_tables
from horizon.utils import functions as utils
from muranodashboard.environments import api
from muranodashboard import api
from muranodashboard.api import packages as pkg_api
from muranodashboard.packages import forms
from muranodashboard.packages import tables
@@ -28,7 +30,6 @@ LOG = logging.getLogger(__name__)
class PackageDefinitionsView(horizon_tables.DataTableView):
table_class = tables.PackageDefinitionsTable
template_name = 'packages/index.html'
PACKAGES_LIMIT = 10
def has_more_data(self, table):
return self._more
@@ -36,23 +37,19 @@ class PackageDefinitionsView(horizon_tables.DataTableView):
def get_data(self):
opts = {
'include_disabled': True,
'owned': True,
'limit': self.PACKAGES_LIMIT + 1,
'owned': True
}
marker = self.request.GET.get(
tables.PackageDefinitionsTable._meta.pagination_param, None)
if marker:
opts['marker'] = marker
packages = []
self._more = False
page_size = utils.get_page_size(self.request)
with api.handled_exceptions(self.request):
packages = api.muranoclient(self.request).packages.filter(**opts)
# The client doesn't return any info about whether there are
# more packages, so we need to try to get one extra, and
# determine whether to show the More link ourselves.
if len(packages) == self.PACKAGES_LIMIT + 1:
self._more = True
packages = packages[:self.PACKAGES_LIMIT]
packages, self._more = pkg_api.package_list(
self.request, marker=marker, filters=opts, paginate=True,
page_size=page_size)
return packages

7
muranodashboard/stats/models.py
View File

@@ -14,8 +14,9 @@
import logging
from muranodashboard import api
from muranodashboard.common import utils
from muranodashboard.environments import api
from muranodashboard.environments import api as env_api
from muranodashboard.environments import consts
LOG = logging.getLogger(__name__)
@@ -36,7 +37,7 @@ class BillingStats(object):
return st_list
def get_all(self, request):
env_list = api.environments_list(request)
env_list = env_api.environments_list(request)
stats = []
LOG.debug('Got env list: {0}'.format(env_list))
for env in env_list:
@@ -80,7 +81,7 @@ class BillingStats(object):
return ids
def build_service_list(self, request, env):
serv_list = api.services_list(request, env.id)
serv_list = env_api.services_list(request, env.id)
LOG.debug('Got Service List: {0}'.format(serv_list))
id_list = {}
for service in serv_list:

3
requirements.txt
View File

@@ -5,5 +5,4 @@ six>=1.5.2
PyYAML>=3.1.0
django-floppyforms>=1.1
yaql>=0.2.2,<0.3
python-muranoclient>=0.5.1
python-muranoclient>=0.5.2