[Django] Allow to upload the image directly to Glance service

Since large Glance images even temporarily stored on dashboard side
tend to fill up Web Server filesystem, it is desirable to route image
payload directly to Glance service (which usually streams it to
storage backend, which in turn has plenty of space).

To make it possible we need to trick Django into thinking that a file
was selected inside FileInput, while its contents are not actually
transferred to Django server. Then, once image is created client-side
code needs to know the exact url the image payload needs to be
transferred to. Both tasks are solved via using ExternalFileField /
ExternalUploadMeta classes which allow to work around the usual Django
form processing workflow with minimal changes to CreateImage form
business logic.

The client-side code relies on CORS being enabled for Glance service
(otherwise browser would forbid the PUT request to a location
different from the one form content came from). In a Devstack setup
you'll need to edit [cors] section of glance-api.conf file, setting
`allowed_origin` setting to the full hostname of the web server (say,
http://<HOST_IP>/dashboard) and restart glance-api process.

A progress bar is implemented to track the progress of a file upload,
in case a really huge image is transferred.

The new machinery could be easily switched on/off with a single
setting `HORIZON_IMAGES_UPLOAD_MODE` set to 'direct' / 'legacy'.

Related-Bug: #1467890
Closes-Bug: #1403129
Implements blueprint: horizon-glance-large-image-upload
Change-Id: I01d02f75268186b43066df6fd966aa01c08e01d7
This commit is contained in:
Timur Sufiev 2016-05-17 13:16:15 +03:00
parent 450baeed4f
commit 93af461e40
14 changed files with 257 additions and 43 deletions

View File

@ -28,6 +28,8 @@ from horizon.forms.base import SelfHandlingForm # noqa
from horizon.forms.base import SelfHandlingMixin # noqa
from horizon.forms.fields import DynamicChoiceField # noqa
from horizon.forms.fields import DynamicTypedChoiceField # noqa
from horizon.forms.fields import ExternalFileField # noqa
from horizon.forms.fields import ExternalUploadMeta # noqa
from horizon.forms.fields import IPField # noqa
from horizon.forms.fields import IPv4 # noqa
from horizon.forms.fields import IPv6 # noqa

View File

@ -22,6 +22,7 @@ import uuid
from django.core.exceptions import ValidationError # noqa
from django.core import urlresolvers
from django.forms import fields
from django.forms import forms
from django.forms.utils import flatatt # noqa
from django.forms import widgets
from django.template import Context # noqa
@ -380,3 +381,53 @@ class ThemableCheckboxFieldRenderer(widgets.CheckboxFieldRenderer):
class ThemableCheckboxSelectMultiple(widgets.CheckboxSelectMultiple):
renderer = ThemableCheckboxFieldRenderer
_empty_value = []
class ExternalFileField(fields.FileField):
"""A special flavor of FileField which is meant to be used in cases when
instead of uploading file to Django it should be uploaded to some external
location, while the form validation is done as usual. Should be paired
with ExternalUploadMeta metaclass embedded into the Form class.
"""
def __init__(self, *args, **kwargs):
super(ExternalFileField, self).__init__(*args, **kwargs)
self.widget.attrs.update({'data-external-upload': 'true'})
class ExternalUploadMeta(forms.DeclarativeFieldsMetaclass):
"""Set this class as the metaclass of a form that contains
ExternalFileField in order to process ExternalFileField fields in a
specific way. A hidden CharField twin of FieldField is created which
contains just the filename (if any file was selected on browser side) and
a special `clean` method for FileField is defined which extracts just file
name. This allows to avoid actual file upload to Django server, yet
process form clean() phase as usual. Actual file upload happens entirely
on client-side.
"""
def __new__(mcs, name, bases, attrs):
def get_double_name(name):
suffix = '__hidden'
slen = len(suffix)
return name[:-slen] if name.endswith(suffix) else name + suffix
def make_clean_method(field_name):
def _clean_method(self):
value = self.cleaned_data[field_name]
if value:
self.cleaned_data[get_double_name(field_name)] = value
return value
return _clean_method
new_attrs = {}
for attr_name, attr in attrs.items():
new_attrs[attr_name] = attr
if isinstance(attr, ExternalFileField):
hidden_field = fields.CharField(widget=fields.HiddenInput,
required=False)
hidden_field.creation_counter = attr.creation_counter + 1000
new_attr_name = get_double_name(attr_name)
new_attrs[new_attr_name] = hidden_field
meth_name = 'clean_' + new_attr_name
new_attrs[meth_name] = make_clean_method(new_attr_name)
return super(ExternalUploadMeta, mcs).__new__(
mcs, name, bases, new_attrs)

View File

@ -191,6 +191,11 @@ class ModalFormView(ModalFormMixin, views.HorizonFormView):
else:
success_url = self.get_success_url()
response = http.HttpResponseRedirect(success_url)
if hasattr(handled, 'to_dict'):
obj_dict = handled.to_dict()
if 'upload_url' in obj_dict:
response['X-File-Upload-URL'] = obj_dict['upload_url']
response['X-Auth-Token'] = obj_dict['token_id']
# TODO(gabriel): This is not a long-term solution to how
# AJAX should be handled, but it's an expedient solution
# until the blueprint for AJAX handling is architected

View File

@ -149,6 +149,11 @@ class HorizonMiddleware(object):
# the user *on* the login form...
return shortcuts.redirect(exception.location)
@staticmethod
def copy_headers(src, dst, headers):
for header in headers:
dst[header] = src[header]
def process_response(self, request, response):
"""Convert HttpResponseRedirect to HttpResponse if request is via ajax
to allow ajax request to redirect url
@ -183,6 +188,10 @@ class HorizonMiddleware(object):
redirect_response.set_cookie(
cookie_name, cookie.value, **cookie_kwargs)
redirect_response['X-Horizon-Location'] = response['location']
upload_url_key = 'X-File-Upload-URL'
if upload_url_key in response:
self.copy_headers(response, redirect_response,
(upload_url_key, 'X-Auth-Token'))
return redirect_response
if queued_msgs:
# TODO(gabriel): When we have an async connection to the

View File

@ -14,6 +14,7 @@ horizon.modals = {
// Storage for our current jqXHR object.
_request: null,
spinner: null,
progress_bar: null,
_init_functions: []
};
@ -61,6 +62,21 @@ horizon.modals.modal_spinner = function (text) {
horizon.modals.spinner.find(".modal-body").spin(horizon.conf.spinner_options.modal);
};
horizon.modals.progress_bar = function (text) {
var template = horizon.templates.compiled_templates["#progress-modal"];
horizon.modals.bar = $(template.render({text: text}))
.appendTo("#modal_wrapper");
horizon.modals.bar.modal({backdrop: 'static'});
var $progress_bar = horizon.modals.bar.find('.progress-bar');
horizon.modals.progress_bar.update = function(fraction) {
var percent = Math.round(100 * fraction) + '%';
$progress_bar
.css('width', Math.round(100 * fraction) + '%')
.parents('.progress-text').find('.progress-bar-text').text(percent);
};
};
horizon.modals.init_wizard = function () {
// If workflow is in wizard mode, initialize wizard.
var _max_visited_step = 0;
@ -176,6 +192,54 @@ horizon.modals.init_wizard = function () {
});
};
horizon.modals.getUploadUrl = function(jqXHR) {
return jqXHR.getResponseHeader("X-File-Upload-URL");
};
horizon.modals.fileUpload = function(url, file, jqXHR) {
var token = jqXHR.getResponseHeader('X-Auth-Token');
horizon.modals.progress_bar(gettext("Uploading image"));
return $.ajax({
type: 'PUT',
url: url,
xhrFields: {
withCredentials: true
},
headers: {
'X-Auth-Token': token
},
data: file,
processData: false, // tell jQuery not to process the data
contentType: 'application/octet-stream',
xhr: function() {
var xhr = new window.XMLHttpRequest();
xhr.upload.addEventListener('progress', function(evt) {
if (evt.lengthComputable) {
horizon.modals.progress_bar.update(evt.loaded / evt.total);
}
}, false);
return xhr;
}
});
};
horizon.modals.prepareFileUpload = function($form) {
var $elem = $form.find('input[data-external-upload]');
if (!$elem.length) {
return undefined;
}
var file = $elem.get(0).files[0];
var $hiddenPseudoFile = $form.find('input[name="' + $elem.attr('name') + '__hidden"]');
if (file) {
$hiddenPseudoFile.val(file.name);
$elem.remove();
return file;
} else {
$hiddenPseudoFile.val('');
return undefined;
}
};
horizon.addInitFunction(horizon.modals.init = function() {
@ -200,7 +264,7 @@ horizon.addInitFunction(horizon.modals.init = function() {
update_field_id = $form.attr("data-add-to-field"),
headers = {},
modalFileUpload = $form.attr("enctype") === "multipart/form-data",
formData, ajaxOpts, featureFileList, featureFormData;
formData, ajaxOpts, featureFileList, featureFormData, file;
if (modalFileUpload) {
featureFileList = $("<input type='file'/>").get(0).files !== undefined;
@ -213,6 +277,7 @@ horizon.addInitFunction(horizon.modals.init = function() {
// modal forms won't work in them (namely, IE9).
return;
} else {
file = horizon.modals.prepareFileUpload($form);
formData = new window.FormData(form);
}
} else {
@ -227,6 +292,38 @@ horizon.addInitFunction(horizon.modals.init = function() {
headers["X-Horizon-Add-To-Field"] = update_field_id;
}
function processServerSuccess(data, textStatus, jqXHR) {
var redirect_header = jqXHR.getResponseHeader("X-Horizon-Location"),
add_to_field_header = jqXHR.getResponseHeader("X-Horizon-Add-To-Field"),
json_data, field_to_update;
if (redirect_header === null) {
$('.ajax-modal, .dropdown-toggle').removeAttr("disabled");
}
if (redirect_header) {
location.href = redirect_header;
} else if (add_to_field_header) {
json_data = $.parseJSON(data);
field_to_update = $("#" + add_to_field_header);
field_to_update.append("<option value='" + json_data[0] + "'>" + json_data[1] + "</option>");
field_to_update.change();
field_to_update.val(json_data[0]);
} else {
horizon.modals.success(data, textStatus, jqXHR);
}
}
function processServerError(jqXHR, textStatus, errorThrown, $formElement) {
$formElement = $formElement || $form;
if (jqXHR.getResponseHeader('logout')) {
location.href = jqXHR.getResponseHeader("X-Horizon-Location");
} else {
$('.ajax-modal, .dropdown-toggle').removeAttr("disabled");
$formElement.closest(".modal").modal("hide");
horizon.alert("danger", gettext("There was an error submitting the form. Please try again."));
}
}
ajaxOpts = {
type: "POST",
url: $form.attr('action'),
@ -246,35 +343,25 @@ horizon.addInitFunction(horizon.modals.init = function() {
$button.prop("disabled", false);
},
success: function (data, textStatus, jqXHR) {
var redirect_header = jqXHR.getResponseHeader("X-Horizon-Location"),
add_to_field_header = jqXHR.getResponseHeader("X-Horizon-Add-To-Field"),
json_data, field_to_update;
if (redirect_header === null) {
$('.ajax-modal, .dropdown-toggle').removeAttr("disabled");
}
var promise;
var uploadUrl = horizon.modals.getUploadUrl(jqXHR);
$form.closest(".modal").modal("hide");
if (redirect_header) {
location.href = redirect_header;
if (uploadUrl) {
promise = horizon.modals.fileUpload(uploadUrl, file, jqXHR);
}
else if (add_to_field_header) {
json_data = $.parseJSON(data);
field_to_update = $("#" + add_to_field_header);
field_to_update.append("<option value='" + json_data[0] + "'>" + json_data[1] + "</option>");
field_to_update.change();
field_to_update.val(json_data[0]);
if (promise) {
promise.then(function() {
// ignore data resolved in asyncUpload promise
processServerSuccess(data, textStatus, jqXHR);
}, function(jqXHR, statusText, errorThrown) {
var $progressBar = horizon.modals.bar.find('.progress-bar');
processServerError(jqXHR, statusText, errorThrown, $progressBar);
});
} else {
horizon.modals.success(data, textStatus, jqXHR);
processServerSuccess(data, textStatus, jqXHR);
}
},
error: function (jqXHR) {
if (jqXHR.getResponseHeader('logout')) {
location.href = jqXHR.getResponseHeader("X-Horizon-Location");
} else {
$('.ajax-modal, .dropdown-toggle').removeAttr("disabled");
$form.closest(".modal").modal("hide");
horizon.alert("danger", gettext("There was an error submitting the form. Please try again."));
}
}
error: processServerError
};
if (modalFileUpload) {

View File

@ -7,7 +7,8 @@ horizon.templates = {
"#alert_message_template",
"#spinner-modal",
"#membership_template",
"#confirm_modal"
"#confirm_modal",
"#progress-modal"
],
compiled_templates: {}
};

View File

@ -1,6 +1,10 @@
{% load horizon %}
{% minifyspace %}
{% if text %}
<div class="progress-text">
{% endif %}
<div class="progress">
{% for this_bar in bars %}
<div class="progress-bar
@ -17,7 +21,7 @@
aria-valuenow="{{ this_bar.percent }}"
aria-valuemin="{{ min_val }}"
aria-valuemax="{{ max_val }}"
style="width: {{ this_bar.percent }}%;{% if text %} min-width: 2em;{% endif %}">
style="width: {{ this_bar.percent }}%;">
{% if not text %}
<span class="sr-only">
{{ this_bar.percent }}%
@ -26,4 +30,9 @@
</div>
{% endfor %}
</div>
{% if text %}
<span class="progress-bar-text">{{ text }}</span>
</div>
{% endif %}
{% endminifyspace %}

View File

@ -0,0 +1,19 @@
{% extends "horizon/client_side/template.html" %}
{% load i18n horizon bootstrap %}
{% block id %}progress-modal{% endblock %}
{% block template %}{% spaceless %}{% jstemplate %}
<div class="modal loading">
<div class="modal-dialog modal-sm">
<div class="modal-content">
<div class="modal-body">
<div class="modal-progress-loader">
{% bs_progress_bar 0 text="0%" %}
<div class="progress-label text-center h4">[[text]]</div>
</div>
</div>
</div>
</div>
</div>
{% endjstemplate %}{% endspaceless %}{% endblock %}

View File

@ -4,3 +4,4 @@
{% include "horizon/client_side/_loading.html" %}
{% include "horizon/client_side/_membership.html" %}
{% include "horizon/client_side/_confirm.html" %}
{% include "horizon/client_side/_progress.html" %}

View File

@ -29,7 +29,7 @@ def bs_progress_bar(*args, **kwargs):
param args (Array of Numbers: 0-100): Percent of Progress Bars
param context (String): Adds 'progress-bar-{context} to the class attribute
param contexts (Array of Strings): Cycles through contexts for stacked bars
param text (Boolean): True: shows value within the bar, False: uses sr span
param text (String): True: shows value within the bar, False: uses sr span
param striped (Boolean): Adds 'progress-bar-striped' to the class attribute
param animated (Boolean): Adds 'active' to the class attribute if striped
param min_val (0): Used for the aria-min value

View File

@ -26,6 +26,7 @@ from django.forms import ValidationError # noqa
from django.forms.widgets import HiddenInput # noqa
from django.template import defaultfilters
from django.utils.translation import ugettext_lazy as _
import six
from horizon import exceptions
from horizon import forms
@ -86,7 +87,16 @@ def create_image_metadata(data):
return meta
class CreateImageForm(forms.SelfHandlingForm):
if api.glance.get_image_upload_mode() == 'direct':
FileField = forms.ExternalFileField
CreateParent = six.with_metaclass(forms.ExternalUploadMeta,
forms.SelfHandlingForm)
else:
FileField = forms.FileField
CreateParent = forms.SelfHandlingForm
class CreateImageForm(CreateParent):
name = forms.CharField(max_length=255, label=_("Name"))
description = forms.CharField(
max_length=255,
@ -121,10 +131,10 @@ class CreateImageForm(forms.SelfHandlingForm):
'ng-change': 'ctrl.selectImageFormat(ctrl.imageFile.name)',
'image-file-on-change': None
}
image_file = forms.FileField(label=_("Image File"),
help_text=_("A local image to upload."),
widget=forms.FileInput(attrs=image_attrs),
required=False)
image_file = FileField(label=_("Image File"),
help_text=_("A local image to upload."),
widget=forms.FileInput(attrs=image_attrs),
required=False)
kernel = forms.ChoiceField(
label=_('Kernel'),
required=False,
@ -274,7 +284,7 @@ class CreateImageForm(forms.SelfHandlingForm):
if (api.glance.get_image_upload_mode() != 'off' and
policy.check((("image", "upload_image"),), request) and
data.get('image_file', None)):
meta['data'] = self.files['image_file']
meta['data'] = data['image_file']
elif data['is_copying']:
meta['copy_from'] = data['image_url']
else:

View File

@ -8,7 +8,7 @@
}
.progress-bar-text {
bottom: 0;
top: 0;
line-height: 1.5em;
position: absolute;
z-index: 1;
@ -18,7 +18,6 @@
text-align: center;
display: inline-block;
width: 100%;
height: 100%;
@include text-overflow();
}
}
@ -38,4 +37,24 @@
.progress-bar {
width: 100%;
}
}
.modal-progress-loader {
display: flex;
flex-direction: column;
height: 100%;
.progress-text {
flex: 1 0 auto;
position: relative;
.progress,
.progress-bar-text {
top: 50%;
}
.progress {
position: relative;
}
}
}

View File

@ -141,10 +141,10 @@ def data(TEST):
admin_role_dict = {'id': '1',
'name': 'admin'}
admin_role = roles.Role(roles.RoleManager, admin_role_dict)
admin_role = roles.Role(roles.RoleManager, admin_role_dict, loaded=True)
member_role_dict = {'id': "2",
'name': settings.OPENSTACK_KEYSTONE_DEFAULT_ROLE}
member_role = roles.Role(roles.RoleManager, member_role_dict)
member_role = roles.Role(roles.RoleManager, member_role_dict, loaded=True)
TEST.roles.add(admin_role, member_role)
TEST.roles.admin = admin_role
TEST.roles.member = member_role
@ -370,14 +370,14 @@ def data(TEST):
'remote_ids': ['rid_1', 'rid_2']}
idp_1 = identity_providers.IdentityProvider(
identity_providers.IdentityProviderManager,
idp_dict_1)
idp_dict_1, loaded=True)
idp_dict_2 = {'id': 'idp_2',
'description': 'identiy provider 2',
'enabled': True,
'remote_ids': ['rid_3', 'rid_4']}
idp_2 = identity_providers.IdentityProvider(
identity_providers.IdentityProviderManager,
idp_dict_2)
idp_dict_2, loaded=True)
TEST.identity_providers.add(idp_1, idp_2)
idp_mapping_dict = {
@ -420,5 +420,6 @@ def data(TEST):
'mapping_id': 'mapping_1'}
idp_protocol = protocols.Protocol(
protocols.ProtocolManager,
idp_protocol_dict_1)
idp_protocol_dict_1,
loaded=True)
TEST.idp_protocols.add(idp_protocol)

View File

@ -1,7 +1,7 @@
---
features:
- Create from a local file feature is added to the Angular
Create Image workflow. It works either in a 'legacy' mode
- Create from a local file feature is added to both Angular and Django
Create Image workflows. It works either in a 'legacy' mode
which proxies an image upload through Django, or in a new
'direct' mode, which in turn implements
[`blueprint horizon-glance-large-image-upload