openstack/glance
Code Issues Proposed changes

Merge "blueprint progressbar-upload-image"

Browse Source
This commit is contained in:
Jenkins 2012-02-22 18:35:06 +00:00 committed by Gerrit Code Review
commit 76fec1476b
8 changed files with 348 additions and 11 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
Authors
View File

@ -40,6 +40,7 @@ Pavan Kumar Sunkara <pavan.sss1991@gmail.com>
Pete Zaitcev <zaitcev@kotori.zaitcev.us>
Rick Clark <rick@openstack.org>
Rick Harris <rconradharris@gmail.com>
Reynolds Chin <benzwt@gmail.com>
Russell Bryant <rbryant@redhat.com>
Soren Hansen <soren.hansen@rackspace.com>
Stuart McLaren <stuart.mclaren@hp.com>

19
bin/glance
View File

@ -762,11 +762,16 @@ def get_client(options):
creds['auth_url'] is not None and
creds['auth_url'].find('https') != -1))
return glance_client.Client(host=options.host, port=options.port,
use_ssl=use_ssl,
auth_tok=options.auth_token or \
os.getenv('OS_TOKEN'),
creds=creds, insecure=options.insecure)
client = (glance_client.ProgressClient if not options.is_silent_upload
else glance_client.Client)
return client(host=options.host,
port=options.port,
use_ssl=use_ssl,
auth_tok=options.auth_token or
os.getenv('OS_TOKEN'),
creds=creds,
insecure=options.insecure)
def create_options(parser):
@ -776,6 +781,10 @@ def create_options(parser):
:param parser: The option parser
"""
parser.add_option('--silent-upload', default=False, action="store_true",
dest="is_silent_upload",
help="disable progress bar animation and information "
"during upload")
parser.add_option('-v', '--verbose', default=False, action="store_true",
help="Print more verbose output")
parser.add_option('-d', '--debug', default=False, action="store_true",

55
glance/client.py
View File

@ -20,11 +20,15 @@ Client classes for callers of a Glance system
"""
import errno
import httplib
import json
import logging
import os
import socket
import sys
import glance.api.v1
from glance.common import animation
from glance.common import client as base_client
from glance.common import exception
from glance.common import utils
@ -329,4 +333,55 @@ class V1Client(base_client.BaseClient):
return True
class ProgressIteratorWrapper(object):
def __init__(self, wrapped, transfer_info):
self.wrapped = wrapped
self.transfer_info = transfer_info
self.prev_len = 0L
def __iter__(self):
for chunk in self.wrapped:
if self.prev_len:
self.transfer_info['so_far'] += self.prev_len
self.prev_len = len(chunk)
yield chunk
# report final chunk
self.transfer_info['so_far'] += self.prev_len
class ProgressClient(V1Client):
"""
Specialized class that adds progress bar output/interaction into the
TTY of the calling client
"""
def image_iterator(self, connection, headers, body):
wrapped = super(ProgressClient, self).image_iterator(connection,
headers,
body)
try:
# spawn the animation thread if the connection is good
connection.connect()
return ProgressIteratorWrapper(wrapped,
self.start_animation(headers))
except (httplib.HTTPResponse, socket.error):
# the connection is out, just "pass"
# and let the "glance add" fail with [Errno 111] Connection refused
pass
def start_animation(self, headers):
transfer_info = {
'so_far': 0L,
'size': headers.get('x-image-meta-size', 0L)
}
pg = animation.UploadProgressStatus(transfer_info)
if transfer_info['size'] == 0L:
sys.stdout.write("The progressbar doesn't show-up because "
"the headers[x-meta-size] is zero or missing\n")
sys.stdout.write("Uploading image '%s'\n" %
headers.get('x-image-meta-name', ''))
pg.start()
return transfer_info
Client = V1Client

203
glance/common/animation.py Normal file
View File

@ -0,0 +1,203 @@
# vim: tabstop=4 shiftwidth=4 softtabstop=4
# Copyright 2012 OpenStack LLC.
# All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
import sys
import threading
import time
from glance.common import exception
from glance.common import utils
class UploadProgressStatus(threading.Thread):
"""
A class for showing:
1. progress;
2. rate;
3. ETA;
4. and status, e.g. active or stalled.
In order to sample the rate as closely as possible, this
implementation uses two FIFO buffers (times and bytes)
to record fine-grain transfer rate over a period of time.
"""
NUM_OF_ELEMENTS = 20 # number of element in FIFO
TIME_TO_STALL = 5 # if no data transfer longer this
# time(secs), the network will be
# considered as stalled.
REFRESH_STATUS_INTERVAL = 0.1 # time interval to refresh screen
MIN_SAMPLING_INTERVAL = 0.15 # Minimum sampling time for data
CALC_ETA_WITH_AVE_RATE = True # calc eta with average rate
def __init__(self, transfer_info):
self.current_size = 0L
self.size = transfer_info['size']
self.last_start = 0.0
self.last_bytes = 0L
self.elapsed_time = 0.0
self.total_times = 0.0
self.total_bytes = 0L
self.nelements = self.NUM_OF_ELEMENTS
self.times = [0.0, ] * self.nelements
self.bytes = [0L, ] * self.nelements
self.index = 0
self.stalled = False
self.transfer_info = transfer_info
threading.Thread.__init__(self)
def run(self):
self.start = self.last_start = time.time()
while self.current_size != self.size:
time.sleep(self.REFRESH_STATUS_INTERVAL)
bytes = self.transfer_info['so_far']
self.sampling(bytes, time.time())
self.render()
sys.stdout.write("\n")
sys.stdout.flush()
def _reset_buffer(self):
self.times = [0.0, ] * self.nelements
self.bytes = [0L, ] * self.nelements
def _update(self, bytes, time):
self.elapsed_time = time - self.last_start
transferred_bytes = bytes - self.current_size
self.last_bytes += transferred_bytes
self.current_size += transferred_bytes
if self.elapsed_time < self.MIN_SAMPLING_INTERVAL:
return False
if transferred_bytes == 0:
if self.elapsed_time > self.TIME_TO_STALL:
self.stalled = True
self._reset_buffer()
self.last_bytes = 0
return False
if self.stalled:
self.stalled = False
self.elapsed_time = 1.0
return True
def render(self):
fraction = self.current_size / float(self.size)
percentage = fraction * 100.0
str_percent = "[%3d%%]" % percentage
try:
height, width = utils.get_terminal_size()
sys.stdout.write("\b" * width)
eta = self._calc_eta()
rate = self._get_speed()
width -= len(eta)
width -= len(rate)
bar = ('=' * int((width - len(str_percent)) * fraction)
+ str_percent)
padding = ' ' * (width - len(bar))
sys.stdout.write(bar + padding + rate + eta)
sys.stdout.flush()
except (exception.DataInvalid, NotImplementedError):
sys.stdout.write("\b" * 6) # use the len of [%3d%%]
percent = (str_percent + ' '
if self.current_size == self.size else str_percent)
percent += ' ' + self._get_speed() + 'ETA ' + self._calc_eta()
sys.stdout.write(percent)
sys.stdout.write("\b" * len(percent))
sys.stdout.flush()
def _get_speed(self):
speed, unit = self._calc_speed()
if speed > 0.0:
if speed >= 99.95:
rate = "%4f" % speed
elif speed >= 9.995:
rate = "%4.1f" % speed
else:
rate = "%4.2f" % speed
return " " + rate + unit + ", "
else:
return " ?B/s "
def _calc_eta(self):
if self.stalled or self.current_size < self.size * 0.01:
return "ETA ??h ??m ??s"
if self.CALC_ETA_WITH_AVE_RATE:
eta = ((self.size - self.current_size)
* (time.time() - self.start) / self.current_size)
else:
eta = (((self.size - self.current_size)
* self.total_times) / (self.total_bytes))
eta = int(eta)
hrs = mins = secs = 0
hrs = eta / 3600
secs = eta - hrs * 3600
if secs >= 60:
mins = secs / 60
secs = secs % 60
return "ETA %dh %2dm %2ds" % (hrs, mins, secs)
def _calc_speed(self):
idx = 0
units = ('B/s', 'K/s', 'M/s', 'G/s')
total_times = self.total_times + time.time() - self.last_start
total_bytes = self.total_bytes + self.last_bytes
if self.stalled or total_times == 0:
return None, None
speed = total_bytes / float(total_times)
if speed < 1024:
idx = 0
elif speed < 1048576.: # 1024*1024
idx = 1
speed /= 1024.
elif speed < 1073741824.: # 1024*1024*1024
idx = 2
speed /= 1048576.
else:
idx = 3
speed /= 1073741824.
return speed, units[idx]
def sampling(self, bytes, time):
if not self._update(bytes, time):
return
self.total_times -= self.times[self.index]
self.total_bytes -= self.bytes[self.index]
self.times[self.index] = self.elapsed_time
self.bytes[self.index] = self.last_bytes
self.total_times += self.elapsed_time
self.total_bytes += self.last_bytes
self.last_start = time
self.last_bytes = 0L
self.index += 1
if self.index == self.nelements:
self.index = 0

2
glance/common/client.py
View File

@ -125,7 +125,7 @@ class SendFileIterator:
def __init__(self, len):
self.len = len
def __len__():
def __len__(self):
return self.len
while self.sending:

65
glance/common/utils.py
View File

@ -25,6 +25,7 @@ import errno
import inspect
import logging
import os
import platform
import random
import subprocess
import socket
@ -292,3 +293,67 @@ class PrettyTable(object):
justified = clipped_data.ljust(width)
return justified
def get_terminal_size():
def _get_terminal_size_posix():
import fcntl
import struct
import termios
height_width = None
try:
height_width = struct.unpack('hh', fcntl.ioctl(sys.stderr.fileno(),
termios.TIOCGWINSZ,
struct.pack('HH', 0, 0)))
except:
pass
if not height_width:
try:
p = subprocess.Popen(['stty', 'size'],
shell=false,
stdout=subprocess.PIPE)
return tuple(int(x) for x in p.communicate()[0].split())
except:
pass
return height_width
def _get_terminal_size_win32():
try:
from ctypes import windll, create_string_buffer
handle = windll.kernel32.GetStdHandle(-12)
csbi = create_string_buffer(22)
res = windll.kernel32.GetConsoleScreenBufferInfo(handle, csbi)
except:
return None
if res:
import struct
unpack_tmp = struct.unpack("hhhhHhhhhhh", csbi.raw)
(bufx, bufy, curx, cury, wattr,
left, top, right, bottom, maxx, maxy) = unpack_tmp
height = bottom - top + 1
width = right - left + 1
return (height, width)
else:
return None
def _get_terminal_size_unknownOS():
raise NotImplementedError
func = {'posix': _get_terminal_size_posix,
'win32': _get_terminal_size_win32}
height_width = func.get(platform.os.name, _get_terminal_size_unknownOS)()
if height_width == None:
raise exception.DataInvalid()
for i in height_width:
if not isinstance(i, int) or i <= 0:
raise exception.DataInvalid()
return height_width[0], height_width[1]

10
glance/tests/functional/test_bin_glance.py
View File

@ -163,7 +163,9 @@ class TestBinGlance(functional.FunctionalTest):
exitcode, out, err = execute(cmd)
self.assertEqual(0, exitcode)
self.assertTrue(out.strip().startswith('Added new image with ID:'))
msg = out.split("\n")
self.assertTrue(msg[0].startswith('Uploading image'))
self.assertTrue(msg[1].startswith('Added new image with ID:'))
# 2. Verify image added as public image
cmd = "bin/glance --port=%d index" % api_port
@ -239,7 +241,8 @@ class TestBinGlance(functional.FunctionalTest):
exitcode, out, err = execute(cmd)
self.assertEqual(0, exitcode)
self.assertTrue(out.strip().startswith('Added new image with ID:'))
msg = out.split('\n')
self.assertTrue(msg[0].startswith('Added new image with ID:'))
image_id = out.strip().split(':')[1].strip()
@ -791,7 +794,8 @@ class TestBinGlance(functional.FunctionalTest):
exitcode, out, err = execute(cmd)
self.assertEqual(0, exitcode)
self.assertTrue(out.strip().startswith('Added new image with ID:'))
msg = out.split("\n")
self.assertTrue(msg[3].startswith('Added new image with ID:'))
# 2. Verify image added as public image
cmd = "bin/glance --port=%d index" % api_port

4
glance/tests/functional/test_private_images.py
View File

@ -760,11 +760,11 @@ class TestPrivateImagesCli(keystone_utils.KeystoneTests):
self.start_servers()
# Add a non-public image using the given glance command line
exitcode, out, err = execute("echo testdata | %s" % cmd)
exitcode, out, err = execute("echo testdata | %s" % cmd +
" --silent-upload")
self.assertEqual(0, exitcode)
image_id = out.strip()[25:]
# Verify the attributes of the image
headers = {'X-Auth-Token': keystone_utils.pattieblack_token}
path = "http://%s:%d/v1/images/%s" % ("0.0.0.0", self.api_port,