Adding Job Binaries support to CLI

Adding Job Binaries commands to Sahara
OpenstackClient plugin:

$ dataprocessing job binary create
$ dataprocessing job binary list
$ dataprocessing job binary show
$ dataprocessing job binary update
$ dataprocessing job binary delete
$ dataprocessing job binary download

Partially implements: blueprint cli-as-openstackclient-plugin

Change-Id: Ia452b22ac27677a911b5b2f38a0250d4962be447
This commit is contained in:
Andrey Pavlov 2015-10-23 17:27:32 +03:00 committed by Vitaly Gridnev
parent 5f3f704ad0
commit b3e5eb893c
3 changed files with 754 additions and 0 deletions

View File

@ -0,0 +1,423 @@
# Copyright (c) 2015 Mirantis Inc.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
# implied.
# See the License for the specific language governing permissions and
# limitations under the License.
from os import path
from cliff import command
from cliff import lister
from cliff import show
from openstackclient.common import exceptions
from openstackclient.common import utils as osc_utils
from oslo_log import log as logging
from oslo_serialization import jsonutils
from saharaclient.api import base
from saharaclient.osc.v1 import utils
JOB_BINARY_FIELDS = ['name', 'id', 'url', 'description', 'is_public',
'is_protected']
class CreateJobBinary(show.ShowOne):
"""Creates job binary"""
log = logging.getLogger(__name__ + ".CreateJobBinary")
def get_parser(self, prog_name):
parser = super(CreateJobBinary, self).get_parser(prog_name)
parser.add_argument(
'--name',
metavar="<name>",
help="Name of the job binary [REQUIRED if JSON is not provided]",
)
creation_type = parser.add_mutually_exclusive_group()
creation_type.add_argument(
'--data',
metavar='<file>',
help='File that will be stored in the internal DB [REQUIRED if '
'JSON and URL are not provided]'
)
creation_type.add_argument(
'--url',
metavar='<url>',
help='URL for the job binary [REQUIRED if JSON and file are '
'not provided]'
)
parser.add_argument(
'--description',
metavar="<description>",
help="Description of the job binary"
)
parser.add_argument(
'--username',
metavar='<username>',
help='Username for accessing the job binary URL',
)
password = parser.add_mutually_exclusive_group()
password.add_argument(
'--password',
metavar='<password>',
help='Password for accessing the job binary URL',
)
password.add_argument(
'--password-prompt',
dest="password_prompt",
action="store_true",
help='Prompt interactively for password',
)
parser.add_argument(
'--public',
action='store_true',
default=False,
help='Make the job binary public',
)
parser.add_argument(
'--protected',
action='store_true',
default=False,
help='Make the job binary protected',
)
parser.add_argument(
'--json',
metavar='<filename>',
help='JSON representation of the job binary. Other '
'arguments will not be taken into account if this one is '
'provided'
)
return parser
def take_action(self, parsed_args):
self.log.debug("take_action(%s)" % parsed_args)
client = self.app.client_manager.data_processing
if parsed_args.json:
blob = osc_utils.read_blob_file_contents(parsed_args.json)
try:
template = jsonutils.loads(blob)
except ValueError as e:
raise exceptions.CommandError(
'An error occurred when reading '
'template from file %s: %s' % (parsed_args.json, e))
data = client.job_binaries.create(**template).to_dict()
else:
if parsed_args.data:
data = open(parsed_args.data).read()
jbi_id = client.job_binary_internals.create(
parsed_args.name, data).id
parsed_args.url = 'internal-db://' + jbi_id
if parsed_args.password_prompt:
parsed_args.password = osc_utils.get_password(
self.app.stdin, confirm=False)
if parsed_args.password and not parsed_args.username:
raise exceptions.CommandError(
'Username via --username should be provided with password')
if parsed_args.username and not parsed_args.password:
raise exceptions.CommandError(
'Password should be provided via --password or entered '
'interactively with --password-prompt')
if parsed_args.password and parsed_args.username:
extra = {
'user': parsed_args.username,
'password': parsed_args.password
}
else:
extra = None
data = client.job_binaries.create(
name=parsed_args.name, url=parsed_args.url,
description=parsed_args.description, extra=extra,
is_public=parsed_args.public,
is_protected=parsed_args.protected).to_dict()
data = utils.prepare_data(data, JOB_BINARY_FIELDS)
return self.dict2columns(data)
class ListJobBinaries(lister.Lister):
"""Lists job binaries"""
log = logging.getLogger(__name__ + ".ListJobBinaries")
def get_parser(self, prog_name):
parser = super(ListJobBinaries, self).get_parser(prog_name)
parser.add_argument(
'--long',
action='store_true',
default=False,
help='List additional fields in output',
)
parser.add_argument(
'--name',
metavar="<name-substring>",
help="List job binaries with specific substring in the "
"name"
)
return parser
def take_action(self, parsed_args):
self.log.debug("take_action(%s)" % parsed_args)
client = self.app.client_manager.data_processing
data = client.job_binaries.list()
if parsed_args.name:
data = utils.get_by_name_substring(data, parsed_args.name)
if parsed_args.long:
columns = ('name', 'id', 'url', 'description', 'is_public',
'is_protected')
column_headers = utils.prepare_column_headers(columns)
else:
columns = ('name', 'id', 'url')
column_headers = utils.prepare_column_headers(columns)
return (
column_headers,
(osc_utils.get_item_properties(
s,
columns
) for s in data)
)
class ShowJobBinary(show.ShowOne):
"""Display job binary details"""
log = logging.getLogger(__name__ + ".ShowJobBinary")
def get_parser(self, prog_name):
parser = super(ShowJobBinary, self).get_parser(prog_name)
parser.add_argument(
"job_binary",
metavar="<job-binary>",
help="Name or ID of the job binary to display",
)
return parser
def take_action(self, parsed_args):
self.log.debug("take_action(%s)" % parsed_args)
client = self.app.client_manager.data_processing
data = utils.get_resource(
client.job_binaries, parsed_args.job_binary).to_dict()
data = utils.prepare_data(data, JOB_BINARY_FIELDS)
return self.dict2columns(data)
class DeleteJobBinary(command.Command):
"""Deletes job binary"""
log = logging.getLogger(__name__ + ".DeleteJobBinary")
def get_parser(self, prog_name):
parser = super(DeleteJobBinary, self).get_parser(prog_name)
parser.add_argument(
"job_binary",
metavar="<job-binary>",
nargs="+",
help="Name(s) or id(s) of the job binary(ies) to delete",
)
return parser
def take_action(self, parsed_args):
self.log.debug("take_action(%s)" % parsed_args)
client = self.app.client_manager.data_processing
for jb in parsed_args.job_binary:
jb = utils.get_resource(client.job_binaries, jb)
if jb.url.startswith("internal-db"):
jbi_id = jb.url.replace('internal-db://', '')
try:
client.job_binary_internals.delete(jbi_id)
except base.APIException as ex:
# check if job binary internal was already deleted for
# some reasons
if not ex.error_code == '404':
raise
client.job_binaries.delete(jb.id)
class UpdateJobBinary(show.ShowOne):
"""Updates job binary"""
log = logging.getLogger(__name__ + ".UpdateJobBinary")
def get_parser(self, prog_name):
parser = super(UpdateJobBinary, self).get_parser(prog_name)
parser.add_argument(
'job_binary',
metavar="<job-binary>",
help="Name or ID of the job binary",
)
parser.add_argument(
'--name',
metavar="<name>",
help="New name of the job binary",
)
parser.add_argument(
'--url',
metavar='<url>',
help='URL for the job binary [Internal DB URL can not be updated]'
)
parser.add_argument(
'--description',
metavar="<description>",
help='Description of the job binary'
)
parser.add_argument(
'--username',
metavar='<username>',
help='Username for accessing the job binary URL',
)
password = parser.add_mutually_exclusive_group()
password.add_argument(
'--password',
metavar='<password>',
help='Password for accessing the job binary URL',
)
password.add_argument(
'--password-prompt',
dest="password_prompt",
action="store_true",
help='Prompt interactively for password',
)
public = parser.add_mutually_exclusive_group()
public.add_argument(
'--public',
action='store_true',
help='Make the job binary public (Visible from other tenants)',
dest='is_public'
)
public.add_argument(
'--private',
action='store_false',
help='Make the job binary private (Visible only from this tenant)',
dest='is_public'
)
protected = parser.add_mutually_exclusive_group()
protected.add_argument(
'--protected',
action='store_true',
help='Make the job binary protected',
dest='is_protected'
)
protected.add_argument(
'--unprotected',
action='store_false',
help='Make the job binary unprotected',
dest='is_protected'
)
parser.add_argument(
'--json',
metavar='<filename>',
help='JSON representation of the update object. Other '
'arguments will not be taken into account if this one is '
'provided'
)
parser.set_defaults(is_public=None, is_protected=None)
return parser
def take_action(self, parsed_args):
self.log.debug("take_action(%s)" % parsed_args)
client = self.app.client_manager.data_processing
jb = utils.get_resource(client.job_binaries, parsed_args.job_binary)
if parsed_args.json:
blob = osc_utils.read_blob_file_contents(parsed_args.json)
try:
template = jsonutils.loads(blob)
except ValueError as e:
raise exceptions.CommandError(
'An error occurred when reading '
'template from file %s: %s' % (parsed_args.json, e))
data = client.job_binaries.update(jb.id, template).to_dict()
else:
if parsed_args.password_prompt:
parsed_args.password = osc_utils.get_password(
self.app.stdin, confirm=False)
extra = {}
if parsed_args.password:
extra['password'] = parsed_args.password
if parsed_args.username:
extra['user'] = parsed_args.username
if not extra:
extra = None
update_fields = utils.create_dict_from_kwargs(
name=parsed_args.name, url=parsed_args.url,
description=parsed_args.description,
extra=extra, is_public=parsed_args.is_public,
is_protected=parsed_args.is_protected
)
data = client.job_binaries.update(
jb.id, update_fields).to_dict()
data = utils.prepare_data(data, JOB_BINARY_FIELDS)
return self.dict2columns(data)
class DownloadJobBinary(command.Command):
"""Downloads job binary"""
log = logging.getLogger(__name__ + ".DownloadJobBinary")
def get_parser(self, prog_name):
parser = super(DownloadJobBinary, self).get_parser(prog_name)
parser.add_argument(
"job_binary",
metavar="<job-binary>",
help="Name or ID of the job binary to download",
)
parser.add_argument(
'--file',
metavar="<file>",
help='Destination file (defaults to job binary name)',
)
return parser
def take_action(self, parsed_args):
self.log.debug("take_action(%s)" % parsed_args)
client = self.app.client_manager.data_processing
if not parsed_args.file:
parsed_args.file = parsed_args.job_binary
jb_id = utils.get_resource(
client.job_binaries, parsed_args.job_binary).id
data = client.job_binaries.get_file(jb_id)
if path.exists(parsed_args.file):
self.log.error('File "%s" already exists. Chose another one with '
'--file argument.' % parsed_args.file)
else:
with open(parsed_args.file, 'w') as f:
f.write(data)

View File

@ -0,0 +1,324 @@
# Copyright (c) 2015 Mirantis Inc.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
# implied.
# See the License for the specific language governing permissions and
# limitations under the License.
import mock
from saharaclient.api import job_binaries as api_jb
from saharaclient.osc.v1 import job_binaries as osc_jb
from saharaclient.tests.unit.osc.v1 import fakes
JOB_BINARY_INFO = {
"name": 'job-binary',
"description": 'descr',
"id": 'jb_id',
"is_protected": False,
"is_public": False,
"url": 'swift://cont/test'
}
class TestJobBinaries(fakes.TestDataProcessing):
def setUp(self):
super(TestJobBinaries, self).setUp()
self.jb_mock = self.app.client_manager.data_processing.job_binaries
self.jb_mock.reset_mock()
class TestCreateJobBinary(TestJobBinaries):
# TODO(apavlov): check for creation with --json
def setUp(self):
super(TestCreateJobBinary, self).setUp()
self.jb_mock.create.return_value = api_jb.JobBinaries(
None, JOB_BINARY_INFO)
self.jbi_mock = (self.app.client_manager.
data_processing.job_binary_internals)
self.jbi_mock.create.return_value = mock.Mock(id='jbi_id')
self.jbi_mock.reset_mock()
# Command to test
self.cmd = osc_jb.CreateJobBinary(self.app, None)
def test_job_binary_create_swift_public_protected(self):
arglist = ['--name', 'job-binary', '--url', 'swift://cont/test',
'--username', 'user', '--password', 'pass', '--public',
'--protected']
verifylist = [('name', 'job-binary'), ('url', 'swift://cont/test'),
('username', 'user'), ('password', 'pass'),
('public', True), ('protected', True)]
parsed_args = self.check_parser(self.cmd, arglist, verifylist)
columns, data = self.cmd.take_action(parsed_args)
# Check that correct arguments were passed
self.jb_mock.create.assert_called_once_with(
description=None, extra={'password': 'pass', 'user': 'user'},
is_protected=True, is_public=True, name='job-binary',
url='swift://cont/test')
# Check that columns are correct
expected_columns = ('Description', 'Id', 'Is protected', 'Is public',
'Name', 'Url')
self.assertEqual(expected_columns, columns)
# Check that data is correct
expected_data = ('descr', 'jb_id', False, False, 'job-binary',
'swift://cont/test')
self.assertEqual(expected_data, data)
def test_job_binary_create_internal(self):
m_open = mock.mock_open()
with mock.patch('six.moves.builtins.open', m_open, create=True):
arglist = ['--name', 'job-binary', '--data', 'filepath']
verifylist = [('name', 'job-binary'), ('data', 'filepath')]
parsed_args = self.check_parser(self.cmd, arglist, verifylist)
self.cmd.take_action(parsed_args)
# Check that correct arguments were passed
self.jb_mock.create.assert_called_once_with(
description=None, extra=None, is_protected=False,
is_public=False, name='job-binary', url='internal-db://jbi_id')
self.jbi_mock.create.assert_called_once_with('job-binary', '')
class TestListJobBinaries(TestJobBinaries):
def setUp(self):
super(TestListJobBinaries, self).setUp()
self.jb_mock.list.return_value = [api_jb.JobBinaries(
None, JOB_BINARY_INFO)]
# Command to test
self.cmd = osc_jb.ListJobBinaries(self.app, None)
def test_job_binary_list_no_options(self):
arglist = []
verifylist = []
parsed_args = self.check_parser(self.cmd, arglist, verifylist)
columns, data = self.cmd.take_action(parsed_args)
# Check that columns are correct
expected_columns = ['Name', 'Id', 'Url']
self.assertEqual(expected_columns, columns)
# Check that data is correct
expected_data = [('job-binary', 'jb_id', 'swift://cont/test')]
self.assertEqual(expected_data, list(data))
def test_job_binary_list_long(self):
arglist = ['--long']
verifylist = [('long', True)]
parsed_args = self.check_parser(self.cmd, arglist, verifylist)
columns, data = self.cmd.take_action(parsed_args)
# Check that columns are correct
expected_columns = ['Name', 'Id', 'Url', 'Description', 'Is public',
'Is protected']
self.assertEqual(expected_columns, columns)
# Check that data is correct
expected_data = [('job-binary', 'jb_id', 'swift://cont/test', 'descr',
False, False)]
self.assertEqual(expected_data, list(data))
def test_job_binary_list_extra_search_opts(self):
arglist = ['--name', 'bin']
verifylist = [('name', 'bin')]
parsed_args = self.check_parser(self.cmd, arglist, verifylist)
columns, data = self.cmd.take_action(parsed_args)
# Check that columns are correct
expected_columns = ['Name', 'Id', 'Url']
self.assertEqual(expected_columns, columns)
# Check that data is correct
expected_data = [('job-binary', 'jb_id', 'swift://cont/test')]
self.assertEqual(expected_data, list(data))
class TestShowJobBinary(TestJobBinaries):
def setUp(self):
super(TestShowJobBinary, self).setUp()
self.jb_mock.find_unique.return_value = api_jb.JobBinaries(
None, JOB_BINARY_INFO)
# Command to test
self.cmd = osc_jb.ShowJobBinary(self.app, None)
def test_job_binary_show(self):
arglist = ['job-binary']
verifylist = [('job_binary', 'job-binary')]
parsed_args = self.check_parser(self.cmd, arglist, verifylist)
columns, data = self.cmd.take_action(parsed_args)
# Check that correct arguments were passed
self.jb_mock.find_unique.assert_called_once_with(name='job-binary')
# Check that columns are correct
expected_columns = ('Description', 'Id', 'Is protected', 'Is public',
'Name', 'Url')
self.assertEqual(expected_columns, columns)
# Check that data is correct
expected_data = ('descr', 'jb_id', False, False, 'job-binary',
'swift://cont/test')
self.assertEqual(expected_data, data)
class TestDeleteJobBinary(TestJobBinaries):
def setUp(self):
super(TestDeleteJobBinary, self).setUp()
self.jb_mock.find_unique.return_value = api_jb.JobBinaries(
None, JOB_BINARY_INFO)
# Command to test
self.cmd = osc_jb.DeleteJobBinary(self.app, None)
def test_job_binary_delete(self):
arglist = ['job-binary']
verifylist = [('job_binary', ['job-binary'])]
parsed_args = self.check_parser(self.cmd, arglist, verifylist)
self.cmd.take_action(parsed_args)
# Check that correct arguments were passed
self.jb_mock.delete.assert_called_once_with('jb_id')
class TestUpdateJobBinary(TestJobBinaries):
def setUp(self):
super(TestUpdateJobBinary, self).setUp()
self.jb_mock.find_unique.return_value = api_jb.JobBinaries(
None, JOB_BINARY_INFO)
self.jb_mock.update.return_value = api_jb.JobBinaries(
None, JOB_BINARY_INFO)
# Command to test
self.cmd = osc_jb.UpdateJobBinary(self.app, None)
def test_job_binary_update_all_options(self):
arglist = ['job-binary', '--name', 'job-binary', '--description',
'descr', '--username', 'user', '--password', 'pass',
'--public', '--protected']
verifylist = [('job_binary', 'job-binary'), ('name', 'job-binary'),
('description', 'descr'), ('username', 'user'),
('password', 'pass'), ('is_public', True),
('is_protected', True)]
parsed_args = self.check_parser(self.cmd, arglist, verifylist)
columns, data = self.cmd.take_action(parsed_args)
# Check that correct arguments were passed
self.jb_mock.update.assert_called_once_with(
'jb_id',
{'is_public': True, 'description': 'descr', 'is_protected': True,
'name': 'job-binary',
'extra': {'password': 'pass', 'user': 'user'}})
# Check that columns are correct
expected_columns = ('Description', 'Id', 'Is protected', 'Is public',
'Name', 'Url')
self.assertEqual(expected_columns, columns)
# Check that data is correct
expected_data = ('descr', 'jb_id', False, False, 'job-binary',
'swift://cont/test')
self.assertEqual(expected_data, data)
def test_job_binary_update_private_unprotected(self):
arglist = ['job-binary', '--private', '--unprotected']
verifylist = [('job_binary', 'job-binary'), ('is_public', False),
('is_protected', False)]
parsed_args = self.check_parser(self.cmd, arglist, verifylist)
columns, data = self.cmd.take_action(parsed_args)
# Check that correct arguments were passed
self.jb_mock.update.assert_called_once_with(
'jb_id', {'is_public': False, 'is_protected': False})
def test_job_binary_update_nothing_updated(self):
arglist = ['job-binary']
verifylist = [('job_binary', 'job-binary')]
parsed_args = self.check_parser(self.cmd, arglist, verifylist)
columns, data = self.cmd.take_action(parsed_args)
# Check that correct arguments were passed
self.jb_mock.update.assert_called_once_with(
'jb_id', {})
class TestDownloadJobBinary(TestJobBinaries):
def setUp(self):
super(TestDownloadJobBinary, self).setUp()
self.jb_mock.get_file.return_value = 'data'
self.jb_mock.find_unique.return_value = api_jb.JobBinaries(
None, JOB_BINARY_INFO)
# Command to test
self.cmd = osc_jb.DownloadJobBinary(self.app, None)
def test_download_job_binary_default_file(self):
m_open = mock.mock_open()
with mock.patch('six.moves.builtins.open', m_open, create=True):
arglist = ['job-binary']
verifylist = [('job_binary', 'job-binary')]
parsed_args = self.check_parser(self.cmd, arglist, verifylist)
self.cmd.take_action(parsed_args)
# Check that correct arguments was passed
self.jb_mock.get_file.assert_called_once_with(
'jb_id')
# Check that data will be saved to the right file
self.assertEqual('job-binary', m_open.call_args[0][0])
def test_download_job_binary_specified_file(self):
m_open = mock.mock_open()
with mock.patch('six.moves.builtins.open', m_open, create=True):
arglist = ['job-binary', '--file', 'test']
verifylist = [('job_binary', 'job-binary'), ('file', 'test')]
parsed_args = self.check_parser(self.cmd, arglist, verifylist)
self.cmd.take_action(parsed_args)
# Check that correct arguments was passed
self.jb_mock.get_file.assert_called_once_with(
'jb_id')
# Check that data will be saved to the right file
self.assertEqual('test', m_open.call_args[0][0])

View File

@ -88,6 +88,13 @@ openstack.data_processing.v1 =
dataprocessing_job_update = saharaclient.osc.v1.jobs:UpdateJob
dataprocessing_job_delete = saharaclient.osc.v1.jobs:DeleteJob
dataprocessing_job_binary_create = saharaclient.osc.v1.job_binaries:CreateJobBinary
dataprocessing_job_binary_list = saharaclient.osc.v1.job_binaries:ListJobBinaries
dataprocessing_job_binary_show = saharaclient.osc.v1.job_binaries:ShowJobBinary
dataprocessing_job_binary_update = saharaclient.osc.v1.job_binaries:UpdateJobBinary
dataprocessing_job_binary_delete = saharaclient.osc.v1.job_binaries:DeleteJobBinary
dataprocessing_job_binary_download = saharaclient.osc.v1.job_binaries:DownloadJobBinary
[build_sphinx]
all_files = 1
build-dir = doc/build