openstack/python-glanceclient
Code Issues Proposed changes

Add support for multi-store import

Browse Source 
This change adds support for providing multiple target
stores where image can be imported.

Co-authored-by: Erno Kuvaja <jokke@usr.fi>
Co-authored-by: Abhishek Kekane <akekane@redhat.com>

bp: import-multi-stores
Change-Id: I8730364263f1afd5d11fd56939851bda73a892bb
This commit is contained in:
Erno Kuvaja 2019-12-18 10:30:54 +00:00 committed by Abhishek Kekane
parent 23fb691dfb
commit c23d86738f
4 changed files with 277 additions and 11 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

170
glanceclient/tests/unit/v2/test_shell_v2.py
View File

@ -870,6 +870,90 @@ class ShellV2Test(testtools.TestCase):
pass
mock_utils_exit.assert_called_once_with(expected_msg)
@mock.patch('glanceclient.common.utils.exit')
def test_neg_image_create_via_import_stores_all_stores_specified(
self, mock_utils_exit):
expected_msg = ('Only one of --store, --stores and --all-stores can '
'be provided')
mock_utils_exit.side_effect = self._mock_utils_exit
my_args = self.base_args.copy()
my_args.update(
{'id': 'IMG-01', 'import_method': 'glance-direct',
'stores': 'file1,file2', 'os_all_stores': True,
'file': 'some.mufile',
'disk_format': 'raw',
'container_format': 'bare',
})
args = self._make_args(my_args)
with mock.patch.object(self.gc.images,
'get_import_info') as mocked_info:
mocked_info.return_value = self.import_info_response
try:
test_shell.do_image_create_via_import(self.gc, args)
self.fail("utils.exit should have been called")
except SystemExit:
pass
mock_utils_exit.assert_called_once_with(expected_msg)
@mock.patch('glanceclient.common.utils.exit')
@mock.patch('sys.stdin', autospec=True)
def test_neg_image_create_via_import_stores_without_file(
self, mock_stdin, mock_utils_exit):
expected_msg = ('--stores option should only be provided with --file '
'option or stdin for the glance-direct import method.')
mock_utils_exit.side_effect = self._mock_utils_exit
mock_stdin.isatty = lambda: True
my_args = self.base_args.copy()
my_args.update(
{'id': 'IMG-01', 'import_method': 'glance-direct',
'stores': 'file1,file2',
'disk_format': 'raw',
'container_format': 'bare',
})
args = self._make_args(my_args)
with mock.patch.object(self.gc.images,
'get_import_info') as mocked_info:
with mock.patch.object(self.gc.images,
'get_stores_info') as mocked_stores_info:
mocked_stores_info.return_value = self.stores_info_response
mocked_info.return_value = self.import_info_response
try:
test_shell.do_image_create_via_import(self.gc, args)
self.fail("utils.exit should have been called")
except SystemExit:
pass
mock_utils_exit.assert_called_once_with(expected_msg)
@mock.patch('glanceclient.common.utils.exit')
@mock.patch('sys.stdin', autospec=True)
def test_neg_image_create_via_import_all_stores_without_file(
self, mock_stdin, mock_utils_exit):
expected_msg = ('--all-stores option should only be provided with '
'--file option or stdin for the glance-direct import '
'method.')
mock_utils_exit.side_effect = self._mock_utils_exit
mock_stdin.isatty = lambda: True
my_args = self.base_args.copy()
my_args.update(
{'id': 'IMG-01', 'import_method': 'glance-direct',
'os_all_stores': True,
'disk_format': 'raw',
'container_format': 'bare',
})
args = self._make_args(my_args)
with mock.patch.object(self.gc.images,
'get_import_info') as mocked_info:
mocked_info.return_value = self.import_info_response
try:
test_shell.do_image_create_via_import(self.gc, args)
self.fail("utils.exit should have been called")
except SystemExit:
pass
mock_utils_exit.assert_called_once_with(expected_msg)
@mock.patch('glanceclient.common.utils.exit')
@mock.patch('os.access')
@mock.patch('sys.stdin', autospec=True)
@ -1083,6 +1167,60 @@ class ShellV2Test(testtools.TestCase):
pass
mock_utils_exit.assert_called_once_with(expected_msg)
@mock.patch('glanceclient.common.utils.exit')
def test_neg_image_create_via_import_stores_without_uri(
self, mock_utils_exit):
expected_msg = ('--stores option should only be provided with --uri '
'option for the web-download import method.')
mock_utils_exit.side_effect = self._mock_utils_exit
my_args = self.base_args.copy()
my_args.update(
{'id': 'IMG-01', 'import_method': 'web-download',
'stores': 'file1,file2',
'disk_format': 'raw',
'container_format': 'bare',
})
args = self._make_args(my_args)
with mock.patch.object(self.gc.images,
'get_import_info') as mocked_info:
with mock.patch.object(self.gc.images,
'get_stores_info') as mocked_stores_info:
mocked_stores_info.return_value = self.stores_info_response
mocked_info.return_value = self.import_info_response
try:
test_shell.do_image_create_via_import(self.gc, args)
self.fail("utils.exit should have been called")
except SystemExit:
pass
mock_utils_exit.assert_called_once_with(expected_msg)
@mock.patch('glanceclient.common.utils.exit')
def test_neg_image_create_via_import_all_stores_without_uri(
self, mock_utils_exit):
expected_msg = ('--all-stores option should only be provided with '
'--uri option for the web-download import '
'method.')
mock_utils_exit.side_effect = self._mock_utils_exit
my_args = self.base_args.copy()
my_args.update(
{'id': 'IMG-01', 'import_method': 'web-download',
'os_all_stores': True,
'disk_format': 'raw',
'container_format': 'bare',
})
args = self._make_args(my_args)
with mock.patch.object(self.gc.images,
'get_import_info') as mocked_info:
mocked_info.return_value = self.import_info_response
try:
test_shell.do_image_create_via_import(self.gc, args)
self.fail("utils.exit should have been called")
except SystemExit:
pass
mock_utils_exit.assert_called_once_with(expected_msg)
@mock.patch('glanceclient.common.utils.exit')
@mock.patch('sys.stdin', autospec=True)
def test_neg_image_create_via_import_web_download_no_uri_with_file(
@ -1785,7 +1923,8 @@ class ShellV2Test(testtools.TestCase):
mock_import.return_value = None
test_shell.do_image_import(self.gc, args)
mock_import.assert_called_once_with(
'IMG-01', 'glance-direct', None, backend=None)
'IMG-01', 'glance-direct', None, backend=None,
all_stores=None, allow_failure=True, stores=None)
def test_image_import_web_download(self):
args = self._make_args(
@ -1803,7 +1942,9 @@ class ShellV2Test(testtools.TestCase):
test_shell.do_image_import(self.gc, args)
mock_import.assert_called_once_with(
'IMG-01', 'web-download',
'http://example.com/image.qcow', backend=None)
'http://example.com/image.qcow',
all_stores=None, allow_failure=True,
backend=None, stores=None)
@mock.patch('glanceclient.common.utils.print_image')
def test_image_import_no_print_image(self, mocked_utils_print_image):
@ -1821,9 +1962,32 @@ class ShellV2Test(testtools.TestCase):
mock_import.return_value = None
test_shell.do_image_import(self.gc, args)
mock_import.assert_called_once_with(
'IMG-02', 'glance-direct', None, backend=None)
'IMG-02', 'glance-direct', None, stores=None,
all_stores=None, allow_failure=True, backend=None)
mocked_utils_print_image.assert_not_called()
@mock.patch('glanceclient.common.utils.print_image')
@mock.patch('glanceclient.v2.shell._validate_backend')
def test_image_import_multiple_stores(self, mocked_utils_print_image,
msvb):
args = self._make_args(
{'id': 'IMG-02', 'uri': None, 'import_method': 'glance-direct',
'from_create': False, 'stores': 'site1,site2'})
with mock.patch.object(self.gc.images, 'image_import') as mock_import:
with mock.patch.object(self.gc.images, 'get') as mocked_get:
with mock.patch.object(self.gc.images,
'get_import_info') as mocked_info:
mocked_get.return_value = {'status': 'uploading',
'container_format': 'bare',
'disk_format': 'raw'}
mocked_info.return_value = self.import_info_response
mock_import.return_value = None
test_shell.do_image_import(self.gc, args)
mock_import.assert_called_once_with(
'IMG-02', 'glance-direct', None, all_stores=None,
allow_failure=True, stores=['site1', 'site2'],
backend=None)
def test_image_download(self):
args = self._make_args(
{'id': 'IMG-01', 'file': 'test', 'progress': True,

11
glanceclient/v2/images.py
View File

@ -318,13 +318,22 @@ class Controller(object):
@utils.add_req_id_to_object()
def image_import(self, image_id, method='glance-direct', uri=None,
backend=None):
backend=None, stores=None, allow_failure=True,
all_stores=None):
"""Import Image via method."""
headers = {}
url = '/v2/images/%s/import' % image_id
data = {'method': {'name': method}}
if stores:
data['stores'] = stores
if allow_failure:
data['all_stores_must_succeed'] = 'false'
if backend is not None:
headers['x-image-meta-store'] = backend
if all_stores:
data['all_stores'] = 'true'
if allow_failure:
data['all_stores_must_succeed'] = 'false'
if uri:
if method == 'web-download':

102
glanceclient/v2/shell.py
View File

@ -150,6 +150,28 @@ def do_image_create(gc, args):
@utils.arg('--store', metavar='<STORE>',
default=utils.env('OS_IMAGE_STORE', default=None),
help='Backend store to upload image to.')
@utils.arg('--stores', metavar='<STORES>',
default=utils.env('OS_IMAGE_STORES', default=None),
help=_('Stores to upload image to if multi-stores import '
'available. Comma separated list. Available stores can be '
'listed with "stores-info" call.'))
@utils.arg('--all-stores', type=strutils.bool_from_string,
metavar='[True|False]',
default=None,
dest='os_all_stores',
help=_('"all-stores" can be ued instead of "stores"-list to '
'indicate that image should be imported into all available '
'stores.'))
@utils.arg('--allow-failure', type=strutils.bool_from_string,
metavar='[True|False]',
dest='os_allow_failure',
default=utils.env('OS_IMAGE_ALLOW_FAILURE', default=True),
help=_('Indicator if all stores listed (or available) must '
'succeed. "True" by default meaning that we allow some '
'stores to fail and the status can be monitored from the '
'image metadata. If this is set to "False" the import will '
'be reverted should any of the uploads fail. Only usable '
'with "stores" or "all-stores".'))
@utils.on_data_require_fields(DATA_FIELDS)
def do_image_create_via_import(gc, args):
"""EXPERIMENTAL: Create a new image via image import.
@ -198,9 +220,21 @@ def do_image_create_via_import(gc, args):
# determine if backend is valid
backend = None
if args.store:
stores = getattr(args, "stores", None)
all_stores = getattr(args, "os_all_stores", None)
if (args.store and (stores or all_stores)) or (stores and all_stores):
utils.exit("Only one of --store, --stores and --all-stores can be "
"provided")
elif args.store:
backend = args.store
# determine if backend is valid
_validate_backend(backend, gc)
elif stores:
stores = str(stores).split(',')
for store in stores:
# determine if backend is valid
_validate_backend(store, gc)
# make sure we have all and only correct inputs for the requested method
if args.import_method is None:
@ -211,6 +245,14 @@ def do_image_create_via_import(gc, args):
if backend and not (file_name or using_stdin):
utils.exit("--store option should only be provided with --file "
"option or stdin for the glance-direct import method.")
if stores and not (file_name or using_stdin):
utils.exit("--stores option should only be provided with --file "
"option or stdin for the glance-direct import method.")
if all_stores and not (file_name or using_stdin):
utils.exit("--all-stores option should only be provided with "
"--file option or stdin for the glance-direct import "
"method.")
if args.uri:
utils.exit("You cannot specify a --uri with the glance-direct "
"import method.")
@ -227,6 +269,12 @@ def do_image_create_via_import(gc, args):
if backend and not args.uri:
utils.exit("--store option should only be provided with --uri "
"option for the web-download import method.")
if stores and not args.uri:
utils.exit("--stores option should only be provided with --uri "
"option for the web-download import method.")
if all_stores and not args.uri:
utils.exit("--all-stores option should only be provided with "
"--uri option for the web-download import method.")
if not args.uri:
utils.exit("URI is required for web-download import method. "
"Please use '--uri <uri>'.")
@ -246,6 +294,7 @@ def do_image_create_via_import(gc, args):
args.size = None
do_image_stage(gc, args)
args.from_create = True
args.stores = stores
do_image_import(gc, args)
image = gc.images.get(args.id)
finally:
@ -617,19 +666,56 @@ def do_image_stage(gc, args):
@utils.arg('--store', metavar='<STORE>',
default=utils.env('OS_IMAGE_STORE', default=None),
help='Backend store to upload image to.')
@utils.arg('--stores', metavar='<STORES>',
default=utils.env('OS_IMAGE_STORES', default=None),
help='Stores to upload image to if multi-stores import available.')
@utils.arg('--all-stores', type=strutils.bool_from_string,
metavar='[True|False]',
default=None,
dest='os_all_stores',
help=_('"all-stores" can be ued instead of "stores"-list to '
'indicate that image should be imported all available '
'stores.'))
@utils.arg('--allow-failure', type=strutils.bool_from_string,
metavar='[True|False]',
dest='os_allow_failure',
default=utils.env('OS_IMAGE_ALLOW_FAILURE', default=True),
help=_('Indicator if all stores listed (or available) must '
'succeed. "True" by default meaning that we allow some '
'stores to fail and the status can be monitored from the '
'image metadata. If this is set to "False" the import will '
'be reverted should any of the uploads fail. Only usable '
'with "stores" or "all-stores".'))
def do_image_import(gc, args):
"""Initiate the image import taskflow."""
backend = None
if args.store:
backend = args.store
backend = getattr(args, "store", None)
stores = getattr(args, "stores", None)
all_stores = getattr(args, "os_all_stores", None)
allow_failure = getattr(args, "os_allow_failure", True)
if not getattr(args, 'from_create', False):
if (args.store and (stores or all_stores)) or (stores and all_stores):
utils.exit("Only one of --store, --stores and --all-stores can be "
"provided")
elif args.store:
backend = args.store
# determine if backend is valid
_validate_backend(backend, gc)
elif stores:
stores = str(stores).split(',')
# determine if backend is valid
_validate_backend(backend, gc)
if stores:
for store in stores:
_validate_backend(store, gc)
if getattr(args, 'from_create', False):
# this command is being called "internally" so we can skip
# validation -- just do the import and get out of here
gc.images.image_import(args.id, args.import_method, args.uri,
backend=backend)
backend=backend,
stores=stores, all_stores=all_stores,
allow_failure=allow_failure)
return
# do input validation
@ -669,7 +755,9 @@ def do_image_import(gc, args):
# finally, do the import
gc.images.image_import(args.id, args.import_method, args.uri,
backend=backend)
backend=backend,
stores=stores, all_stores=all_stores,
allow_failure=allow_failure)
image = gc.images.get(args.id)
utils.print_image(image)

5
releasenotes/notes/multi-store-import-45d05a6193ef2c04.yaml Normal file
View File

@ -0,0 +1,5 @@
---
features:
- |
Adds support for multi-store import where user can import
image into multiple backend stores with single command.