From 93af461e40fca76eae58066cc1f9de83d7b716ea Mon Sep 17 00:00:00 2001 From: Timur Sufiev Date: Tue, 17 May 2016 13:16:15 +0300 Subject: [PATCH] [Django] Allow to upload the image directly to Glance service Since large Glance images even temporarily stored on dashboard side tend to fill up Web Server filesystem, it is desirable to route image payload directly to Glance service (which usually streams it to storage backend, which in turn has plenty of space). To make it possible we need to trick Django into thinking that a file was selected inside FileInput, while its contents are not actually transferred to Django server. Then, once image is created client-side code needs to know the exact url the image payload needs to be transferred to. Both tasks are solved via using ExternalFileField / ExternalUploadMeta classes which allow to work around the usual Django form processing workflow with minimal changes to CreateImage form business logic. The client-side code relies on CORS being enabled for Glance service (otherwise browser would forbid the PUT request to a location different from the one form content came from). In a Devstack setup you'll need to edit [cors] section of glance-api.conf file, setting `allowed_origin` setting to the full hostname of the web server (say, http:///dashboard) and restart glance-api process. A progress bar is implemented to track the progress of a file upload, in case a really huge image is transferred. The new machinery could be easily switched on/off with a single setting `HORIZON_IMAGES_UPLOAD_MODE` set to 'direct' / 'legacy'. Related-Bug: #1467890 Closes-Bug: #1403129 Implements blueprint: horizon-glance-large-image-upload Change-Id: I01d02f75268186b43066df6fd966aa01c08e01d7 --- horizon/forms/__init__.py | 2 + horizon/forms/fields.py | 51 +++++++ horizon/forms/views.py | 5 + horizon/middleware/base.py | 9 ++ horizon/static/horizon/js/horizon.modals.js | 137 ++++++++++++++---- .../static/horizon/js/horizon.templates.js | 3 +- horizon/templates/bootstrap/progress_bar.html | 11 +- .../horizon/client_side/_progress.html | 19 +++ .../horizon/client_side/templates.html | 1 + horizon/templatetags/bootstrap.py | 2 +- .../dashboards/project/images/images/forms.py | 22 ++- .../scss/components/_progress_bars.scss | 23 ++- .../test/test_data/keystone_data.py | 11 +- ...e-large-image-upload-c987dc86bab38761.yaml | 4 +- 14 files changed, 257 insertions(+), 43 deletions(-) create mode 100644 horizon/templates/horizon/client_side/_progress.html diff --git a/horizon/forms/__init__.py b/horizon/forms/__init__.py index 997199ea33..8a0df8af8a 100644 --- a/horizon/forms/__init__.py +++ b/horizon/forms/__init__.py @@ -28,6 +28,8 @@ from horizon.forms.base import SelfHandlingForm # noqa from horizon.forms.base import SelfHandlingMixin # noqa from horizon.forms.fields import DynamicChoiceField # noqa from horizon.forms.fields import DynamicTypedChoiceField # noqa +from horizon.forms.fields import ExternalFileField # noqa +from horizon.forms.fields import ExternalUploadMeta # noqa from horizon.forms.fields import IPField # noqa from horizon.forms.fields import IPv4 # noqa from horizon.forms.fields import IPv6 # noqa diff --git a/horizon/forms/fields.py b/horizon/forms/fields.py index c928b5387f..9c6564b66c 100644 --- a/horizon/forms/fields.py +++ b/horizon/forms/fields.py @@ -22,6 +22,7 @@ import uuid from django.core.exceptions import ValidationError # noqa from django.core import urlresolvers from django.forms import fields +from django.forms import forms from django.forms.utils import flatatt # noqa from django.forms import widgets from django.template import Context # noqa @@ -380,3 +381,53 @@ class ThemableCheckboxFieldRenderer(widgets.CheckboxFieldRenderer): class ThemableCheckboxSelectMultiple(widgets.CheckboxSelectMultiple): renderer = ThemableCheckboxFieldRenderer _empty_value = [] + + +class ExternalFileField(fields.FileField): + """A special flavor of FileField which is meant to be used in cases when + instead of uploading file to Django it should be uploaded to some external + location, while the form validation is done as usual. Should be paired + with ExternalUploadMeta metaclass embedded into the Form class. + """ + def __init__(self, *args, **kwargs): + super(ExternalFileField, self).__init__(*args, **kwargs) + self.widget.attrs.update({'data-external-upload': 'true'}) + + +class ExternalUploadMeta(forms.DeclarativeFieldsMetaclass): + """Set this class as the metaclass of a form that contains + ExternalFileField in order to process ExternalFileField fields in a + specific way. A hidden CharField twin of FieldField is created which + contains just the filename (if any file was selected on browser side) and + a special `clean` method for FileField is defined which extracts just file + name. This allows to avoid actual file upload to Django server, yet + process form clean() phase as usual. Actual file upload happens entirely + on client-side. + """ + def __new__(mcs, name, bases, attrs): + def get_double_name(name): + suffix = '__hidden' + slen = len(suffix) + return name[:-slen] if name.endswith(suffix) else name + suffix + + def make_clean_method(field_name): + def _clean_method(self): + value = self.cleaned_data[field_name] + if value: + self.cleaned_data[get_double_name(field_name)] = value + return value + return _clean_method + + new_attrs = {} + for attr_name, attr in attrs.items(): + new_attrs[attr_name] = attr + if isinstance(attr, ExternalFileField): + hidden_field = fields.CharField(widget=fields.HiddenInput, + required=False) + hidden_field.creation_counter = attr.creation_counter + 1000 + new_attr_name = get_double_name(attr_name) + new_attrs[new_attr_name] = hidden_field + meth_name = 'clean_' + new_attr_name + new_attrs[meth_name] = make_clean_method(new_attr_name) + return super(ExternalUploadMeta, mcs).__new__( + mcs, name, bases, new_attrs) diff --git a/horizon/forms/views.py b/horizon/forms/views.py index 3cc7263ae4..0f7316cdc4 100644 --- a/horizon/forms/views.py +++ b/horizon/forms/views.py @@ -191,6 +191,11 @@ class ModalFormView(ModalFormMixin, views.HorizonFormView): else: success_url = self.get_success_url() response = http.HttpResponseRedirect(success_url) + if hasattr(handled, 'to_dict'): + obj_dict = handled.to_dict() + if 'upload_url' in obj_dict: + response['X-File-Upload-URL'] = obj_dict['upload_url'] + response['X-Auth-Token'] = obj_dict['token_id'] # TODO(gabriel): This is not a long-term solution to how # AJAX should be handled, but it's an expedient solution # until the blueprint for AJAX handling is architected diff --git a/horizon/middleware/base.py b/horizon/middleware/base.py index f0ba39436e..1d60e1c210 100644 --- a/horizon/middleware/base.py +++ b/horizon/middleware/base.py @@ -149,6 +149,11 @@ class HorizonMiddleware(object): # the user *on* the login form... return shortcuts.redirect(exception.location) + @staticmethod + def copy_headers(src, dst, headers): + for header in headers: + dst[header] = src[header] + def process_response(self, request, response): """Convert HttpResponseRedirect to HttpResponse if request is via ajax to allow ajax request to redirect url @@ -183,6 +188,10 @@ class HorizonMiddleware(object): redirect_response.set_cookie( cookie_name, cookie.value, **cookie_kwargs) redirect_response['X-Horizon-Location'] = response['location'] + upload_url_key = 'X-File-Upload-URL' + if upload_url_key in response: + self.copy_headers(response, redirect_response, + (upload_url_key, 'X-Auth-Token')) return redirect_response if queued_msgs: # TODO(gabriel): When we have an async connection to the diff --git a/horizon/static/horizon/js/horizon.modals.js b/horizon/static/horizon/js/horizon.modals.js index 558bb0ed3e..1574e1afeb 100644 --- a/horizon/static/horizon/js/horizon.modals.js +++ b/horizon/static/horizon/js/horizon.modals.js @@ -14,6 +14,7 @@ horizon.modals = { // Storage for our current jqXHR object. _request: null, spinner: null, + progress_bar: null, _init_functions: [] }; @@ -61,6 +62,21 @@ horizon.modals.modal_spinner = function (text) { horizon.modals.spinner.find(".modal-body").spin(horizon.conf.spinner_options.modal); }; +horizon.modals.progress_bar = function (text) { + var template = horizon.templates.compiled_templates["#progress-modal"]; + horizon.modals.bar = $(template.render({text: text})) + .appendTo("#modal_wrapper"); + horizon.modals.bar.modal({backdrop: 'static'}); + + var $progress_bar = horizon.modals.bar.find('.progress-bar'); + horizon.modals.progress_bar.update = function(fraction) { + var percent = Math.round(100 * fraction) + '%'; + $progress_bar + .css('width', Math.round(100 * fraction) + '%') + .parents('.progress-text').find('.progress-bar-text').text(percent); + }; +}; + horizon.modals.init_wizard = function () { // If workflow is in wizard mode, initialize wizard. var _max_visited_step = 0; @@ -176,6 +192,54 @@ horizon.modals.init_wizard = function () { }); }; +horizon.modals.getUploadUrl = function(jqXHR) { + return jqXHR.getResponseHeader("X-File-Upload-URL"); +}; + +horizon.modals.fileUpload = function(url, file, jqXHR) { + var token = jqXHR.getResponseHeader('X-Auth-Token'); + + horizon.modals.progress_bar(gettext("Uploading image")); + return $.ajax({ + type: 'PUT', + url: url, + xhrFields: { + withCredentials: true + }, + headers: { + 'X-Auth-Token': token + }, + data: file, + processData: false, // tell jQuery not to process the data + contentType: 'application/octet-stream', + xhr: function() { + var xhr = new window.XMLHttpRequest(); + xhr.upload.addEventListener('progress', function(evt) { + if (evt.lengthComputable) { + horizon.modals.progress_bar.update(evt.loaded / evt.total); + } + }, false); + return xhr; + } + }); +}; + +horizon.modals.prepareFileUpload = function($form) { + var $elem = $form.find('input[data-external-upload]'); + if (!$elem.length) { + return undefined; + } + var file = $elem.get(0).files[0]; + var $hiddenPseudoFile = $form.find('input[name="' + $elem.attr('name') + '__hidden"]'); + if (file) { + $hiddenPseudoFile.val(file.name); + $elem.remove(); + return file; + } else { + $hiddenPseudoFile.val(''); + return undefined; + } +}; horizon.addInitFunction(horizon.modals.init = function() { @@ -200,7 +264,7 @@ horizon.addInitFunction(horizon.modals.init = function() { update_field_id = $form.attr("data-add-to-field"), headers = {}, modalFileUpload = $form.attr("enctype") === "multipart/form-data", - formData, ajaxOpts, featureFileList, featureFormData; + formData, ajaxOpts, featureFileList, featureFormData, file; if (modalFileUpload) { featureFileList = $("").get(0).files !== undefined; @@ -213,6 +277,7 @@ horizon.addInitFunction(horizon.modals.init = function() { // modal forms won't work in them (namely, IE9). return; } else { + file = horizon.modals.prepareFileUpload($form); formData = new window.FormData(form); } } else { @@ -227,6 +292,38 @@ horizon.addInitFunction(horizon.modals.init = function() { headers["X-Horizon-Add-To-Field"] = update_field_id; } + function processServerSuccess(data, textStatus, jqXHR) { + var redirect_header = jqXHR.getResponseHeader("X-Horizon-Location"), + add_to_field_header = jqXHR.getResponseHeader("X-Horizon-Add-To-Field"), + json_data, field_to_update; + if (redirect_header === null) { + $('.ajax-modal, .dropdown-toggle').removeAttr("disabled"); + } + + if (redirect_header) { + location.href = redirect_header; + } else if (add_to_field_header) { + json_data = $.parseJSON(data); + field_to_update = $("#" + add_to_field_header); + field_to_update.append(""); + field_to_update.change(); + field_to_update.val(json_data[0]); + } else { + horizon.modals.success(data, textStatus, jqXHR); + } + } + + function processServerError(jqXHR, textStatus, errorThrown, $formElement) { + $formElement = $formElement || $form; + if (jqXHR.getResponseHeader('logout')) { + location.href = jqXHR.getResponseHeader("X-Horizon-Location"); + } else { + $('.ajax-modal, .dropdown-toggle').removeAttr("disabled"); + $formElement.closest(".modal").modal("hide"); + horizon.alert("danger", gettext("There was an error submitting the form. Please try again.")); + } + } + ajaxOpts = { type: "POST", url: $form.attr('action'), @@ -246,35 +343,25 @@ horizon.addInitFunction(horizon.modals.init = function() { $button.prop("disabled", false); }, success: function (data, textStatus, jqXHR) { - var redirect_header = jqXHR.getResponseHeader("X-Horizon-Location"), - add_to_field_header = jqXHR.getResponseHeader("X-Horizon-Add-To-Field"), - json_data, field_to_update; - if (redirect_header === null) { - $('.ajax-modal, .dropdown-toggle').removeAttr("disabled"); - } + var promise; + var uploadUrl = horizon.modals.getUploadUrl(jqXHR); $form.closest(".modal").modal("hide"); - if (redirect_header) { - location.href = redirect_header; + if (uploadUrl) { + promise = horizon.modals.fileUpload(uploadUrl, file, jqXHR); } - else if (add_to_field_header) { - json_data = $.parseJSON(data); - field_to_update = $("#" + add_to_field_header); - field_to_update.append(""); - field_to_update.change(); - field_to_update.val(json_data[0]); + if (promise) { + promise.then(function() { + // ignore data resolved in asyncUpload promise + processServerSuccess(data, textStatus, jqXHR); + }, function(jqXHR, statusText, errorThrown) { + var $progressBar = horizon.modals.bar.find('.progress-bar'); + processServerError(jqXHR, statusText, errorThrown, $progressBar); + }); } else { - horizon.modals.success(data, textStatus, jqXHR); + processServerSuccess(data, textStatus, jqXHR); } }, - error: function (jqXHR) { - if (jqXHR.getResponseHeader('logout')) { - location.href = jqXHR.getResponseHeader("X-Horizon-Location"); - } else { - $('.ajax-modal, .dropdown-toggle').removeAttr("disabled"); - $form.closest(".modal").modal("hide"); - horizon.alert("danger", gettext("There was an error submitting the form. Please try again.")); - } - } + error: processServerError }; if (modalFileUpload) { diff --git a/horizon/static/horizon/js/horizon.templates.js b/horizon/static/horizon/js/horizon.templates.js index 244c618fe5..5796cc4dec 100644 --- a/horizon/static/horizon/js/horizon.templates.js +++ b/horizon/static/horizon/js/horizon.templates.js @@ -7,7 +7,8 @@ horizon.templates = { "#alert_message_template", "#spinner-modal", "#membership_template", - "#confirm_modal" + "#confirm_modal", + "#progress-modal" ], compiled_templates: {} }; diff --git a/horizon/templates/bootstrap/progress_bar.html b/horizon/templates/bootstrap/progress_bar.html index 05be8c333e..8bc5e718b5 100644 --- a/horizon/templates/bootstrap/progress_bar.html +++ b/horizon/templates/bootstrap/progress_bar.html @@ -1,6 +1,10 @@ {% load horizon %} {% minifyspace %} +{% if text %} +
+{% endif %} +
{% for this_bar in bars %}
+ style="width: {{ this_bar.percent }}%;"> {% if not text %} {{ this_bar.percent }}% @@ -26,4 +30,9 @@
{% endfor %}
+ +{% if text %} + {{ text }} +
+{% endif %} {% endminifyspace %} diff --git a/horizon/templates/horizon/client_side/_progress.html b/horizon/templates/horizon/client_side/_progress.html new file mode 100644 index 0000000000..e059440a65 --- /dev/null +++ b/horizon/templates/horizon/client_side/_progress.html @@ -0,0 +1,19 @@ +{% extends "horizon/client_side/template.html" %} +{% load i18n horizon bootstrap %} + +{% block id %}progress-modal{% endblock %} + +{% block template %}{% spaceless %}{% jstemplate %} +
+
+
+
+
+ {% bs_progress_bar 0 text="0%" %} +
[[text]]
+
+
+
+
+
+{% endjstemplate %}{% endspaceless %}{% endblock %} diff --git a/horizon/templates/horizon/client_side/templates.html b/horizon/templates/horizon/client_side/templates.html index e738200e62..32547cbc80 100644 --- a/horizon/templates/horizon/client_side/templates.html +++ b/horizon/templates/horizon/client_side/templates.html @@ -4,3 +4,4 @@ {% include "horizon/client_side/_loading.html" %} {% include "horizon/client_side/_membership.html" %} {% include "horizon/client_side/_confirm.html" %} +{% include "horizon/client_side/_progress.html" %} \ No newline at end of file diff --git a/horizon/templatetags/bootstrap.py b/horizon/templatetags/bootstrap.py index 4c9005949d..311a3da0c1 100644 --- a/horizon/templatetags/bootstrap.py +++ b/horizon/templatetags/bootstrap.py @@ -29,7 +29,7 @@ def bs_progress_bar(*args, **kwargs): param args (Array of Numbers: 0-100): Percent of Progress Bars param context (String): Adds 'progress-bar-{context} to the class attribute param contexts (Array of Strings): Cycles through contexts for stacked bars - param text (Boolean): True: shows value within the bar, False: uses sr span + param text (String): True: shows value within the bar, False: uses sr span param striped (Boolean): Adds 'progress-bar-striped' to the class attribute param animated (Boolean): Adds 'active' to the class attribute if striped param min_val (0): Used for the aria-min value diff --git a/openstack_dashboard/dashboards/project/images/images/forms.py b/openstack_dashboard/dashboards/project/images/images/forms.py index b4f3db4af0..03cd4cf1f4 100644 --- a/openstack_dashboard/dashboards/project/images/images/forms.py +++ b/openstack_dashboard/dashboards/project/images/images/forms.py @@ -26,6 +26,7 @@ from django.forms import ValidationError # noqa from django.forms.widgets import HiddenInput # noqa from django.template import defaultfilters from django.utils.translation import ugettext_lazy as _ +import six from horizon import exceptions from horizon import forms @@ -86,7 +87,16 @@ def create_image_metadata(data): return meta -class CreateImageForm(forms.SelfHandlingForm): +if api.glance.get_image_upload_mode() == 'direct': + FileField = forms.ExternalFileField + CreateParent = six.with_metaclass(forms.ExternalUploadMeta, + forms.SelfHandlingForm) +else: + FileField = forms.FileField + CreateParent = forms.SelfHandlingForm + + +class CreateImageForm(CreateParent): name = forms.CharField(max_length=255, label=_("Name")) description = forms.CharField( max_length=255, @@ -121,10 +131,10 @@ class CreateImageForm(forms.SelfHandlingForm): 'ng-change': 'ctrl.selectImageFormat(ctrl.imageFile.name)', 'image-file-on-change': None } - image_file = forms.FileField(label=_("Image File"), - help_text=_("A local image to upload."), - widget=forms.FileInput(attrs=image_attrs), - required=False) + image_file = FileField(label=_("Image File"), + help_text=_("A local image to upload."), + widget=forms.FileInput(attrs=image_attrs), + required=False) kernel = forms.ChoiceField( label=_('Kernel'), required=False, @@ -274,7 +284,7 @@ class CreateImageForm(forms.SelfHandlingForm): if (api.glance.get_image_upload_mode() != 'off' and policy.check((("image", "upload_image"),), request) and data.get('image_file', None)): - meta['data'] = self.files['image_file'] + meta['data'] = data['image_file'] elif data['is_copying']: meta['copy_from'] = data['image_url'] else: diff --git a/openstack_dashboard/static/dashboard/scss/components/_progress_bars.scss b/openstack_dashboard/static/dashboard/scss/components/_progress_bars.scss index 5e11391f10..58abc923b2 100644 --- a/openstack_dashboard/static/dashboard/scss/components/_progress_bars.scss +++ b/openstack_dashboard/static/dashboard/scss/components/_progress_bars.scss @@ -8,7 +8,7 @@ } .progress-bar-text { - bottom: 0; + top: 0; line-height: 1.5em; position: absolute; z-index: 1; @@ -18,7 +18,6 @@ text-align: center; display: inline-block; width: 100%; - height: 100%; @include text-overflow(); } } @@ -38,4 +37,24 @@ .progress-bar { width: 100%; } +} + +.modal-progress-loader { + display: flex; + flex-direction: column; + height: 100%; + + .progress-text { + flex: 1 0 auto; + position: relative; + + .progress, + .progress-bar-text { + top: 50%; + } + + .progress { + position: relative; + } + } } \ No newline at end of file diff --git a/openstack_dashboard/test/test_data/keystone_data.py b/openstack_dashboard/test/test_data/keystone_data.py index 3b9cd1517a..b125c152f0 100644 --- a/openstack_dashboard/test/test_data/keystone_data.py +++ b/openstack_dashboard/test/test_data/keystone_data.py @@ -141,10 +141,10 @@ def data(TEST): admin_role_dict = {'id': '1', 'name': 'admin'} - admin_role = roles.Role(roles.RoleManager, admin_role_dict) + admin_role = roles.Role(roles.RoleManager, admin_role_dict, loaded=True) member_role_dict = {'id': "2", 'name': settings.OPENSTACK_KEYSTONE_DEFAULT_ROLE} - member_role = roles.Role(roles.RoleManager, member_role_dict) + member_role = roles.Role(roles.RoleManager, member_role_dict, loaded=True) TEST.roles.add(admin_role, member_role) TEST.roles.admin = admin_role TEST.roles.member = member_role @@ -370,14 +370,14 @@ def data(TEST): 'remote_ids': ['rid_1', 'rid_2']} idp_1 = identity_providers.IdentityProvider( identity_providers.IdentityProviderManager, - idp_dict_1) + idp_dict_1, loaded=True) idp_dict_2 = {'id': 'idp_2', 'description': 'identiy provider 2', 'enabled': True, 'remote_ids': ['rid_3', 'rid_4']} idp_2 = identity_providers.IdentityProvider( identity_providers.IdentityProviderManager, - idp_dict_2) + idp_dict_2, loaded=True) TEST.identity_providers.add(idp_1, idp_2) idp_mapping_dict = { @@ -420,5 +420,6 @@ def data(TEST): 'mapping_id': 'mapping_1'} idp_protocol = protocols.Protocol( protocols.ProtocolManager, - idp_protocol_dict_1) + idp_protocol_dict_1, + loaded=True) TEST.idp_protocols.add(idp_protocol) diff --git a/releasenotes/notes/bp-horizon-glance-large-image-upload-c987dc86bab38761.yaml b/releasenotes/notes/bp-horizon-glance-large-image-upload-c987dc86bab38761.yaml index 97d26b33c8..151eff9758 100644 --- a/releasenotes/notes/bp-horizon-glance-large-image-upload-c987dc86bab38761.yaml +++ b/releasenotes/notes/bp-horizon-glance-large-image-upload-c987dc86bab38761.yaml @@ -1,7 +1,7 @@ --- features: - - Create from a local file feature is added to the Angular - Create Image workflow. It works either in a 'legacy' mode + - Create from a local file feature is added to both Angular and Django + Create Image workflows. It works either in a 'legacy' mode which proxies an image upload through Django, or in a new 'direct' mode, which in turn implements [`blueprint horizon-glance-large-image-upload