Support clearing collections of configdocs

Adds an option to create configdocs as an empty colleciton. This is done
as an explicit flag (--empty-collection) on the command as opposed to
using empty files to prevent accidental emtpy file loads leading to
frustration.

Since this introduced a new flag value for the CLI, the CLIs using flag
values were updated to use the standard is_flag=True instead of the
flag_value=True or some other value when a boolean flag is expected.

Minor updates to CLI tests due to moving to responses 0.10.2

Depends-On: https://review.openstack.org/#/c/614421/

Change-Id: I489b0e1183335cbfbaa2014c1458a84dadf6bb0b
This commit is contained in:
Bryan Strassner 2018-10-17 17:33:40 -05:00
parent 03d7269b6a
commit 667a538330
15 changed files with 389 additions and 126 deletions

View File

@ -158,10 +158,9 @@ Deckhand
POST /v1.0/configdocs/{collection_id} POST /v1.0/configdocs/{collection_id}
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
Ingests a collection of documents. Synchronous. POSTing an empty body Ingests a collection of documents. Synchronous. If a POST to the
indicates that the specified collection should be deleted when the commitconfigdocs is already in progress, this POST should be rejected with a
Shipyard Buffer is committed. If a POST to the commitconfigdocs is in 409 error.
progress, this POST should be rejected with a 409 error.
.. note:: .. note::
@ -183,6 +182,10 @@ Query Parameters
- replace: Clear the Shipyard Buffer before adding the specified - replace: Clear the Shipyard Buffer before adding the specified
collection. collection.
- empty-collection: Set to true to indicate that this collection should be
made empty and effectively deleted when the Shipyard Buffer is committed.
If this parameter is specified, the POST body will be ignored.
Responses Responses
''''''''' '''''''''
201 Created 201 Created

View File

@ -293,7 +293,7 @@ or one or more directory options must be specified.
shipyard create configdocs shipyard create configdocs
<collection> <collection>
[--append | --replace] [--append | --replace] [--empty-collection]
--filename=<filename> (repeatable) --filename=<filename> (repeatable)
| |
--directory=<directory> (repeatable) --directory=<directory> (repeatable)
@ -308,8 +308,8 @@ or one or more directory options must be specified.
.. note:: .. note::
Either --filename or --directory must be specified, but both may not be --filename and/or --directory must be specified unless --empty-collection
specified for the same invocation of shipyard. is used.
<collection> <collection>
The collection to load. The collection to load.
@ -321,17 +321,22 @@ or one or more directory options must be specified.
\--replace \--replace
Clear the shipyard buffer and replace it with the specified contents. Clear the shipyard buffer and replace it with the specified contents.
\--empty-collection
Indicate to Shipyard that the named collection should be made empty (contain
no documents). If --empty-collection is specified, the files named by
--filename or --directory will be ignored.
\--filename=<filename> \--filename=<filename>
The file name to use as the contents of the collection. (repeatable) If The file name to use as the contents of the collection. (repeatable) If
any documents specified fail basic validation, all of the documents will any documents specified fail basic validation, all of the documents will
be rejected. Use of filename parameters may not be used in conjunction be rejected. Use of ``filename`` parameters may not be used in conjunction
with the directory parameter. with the directory parameter.
\--directory=<directory> \--directory=<directory>
A directory containing documents that will be joined and loaded as a A directory containing documents that will be joined and loaded as a
collection. (Repeatable) Any documents that fail basic validation will reject the collection. (Repeatable) Any documents that fail basic validation will
whole set. Use of the directory parameter may not be used with the reject the whole set. Use of the ``directory`` parameter may not be used
filename parameter. with the ``filename`` parameter.
\--recurse \--recurse
Recursively search through all directories for sub-directories that Recursively search through all directories for sub-directories that

View File

@ -14,6 +14,8 @@
""" """
Resources representing the configdocs API for shipyard Resources representing the configdocs API for shipyard
""" """
import logging
import falcon import falcon
from oslo_config import cfg from oslo_config import cfg
@ -26,6 +28,7 @@ from shipyard_airflow.control.helpers.configdocs_helper import (
from shipyard_airflow.errors import ApiError from shipyard_airflow.errors import ApiError
CONF = cfg.CONF CONF = cfg.CONF
LOG = logging.getLogger(__name__)
VERSION_VALUES = ['buffer', VERSION_VALUES = ['buffer',
'committed', 'committed',
'last_site_action', 'last_site_action',
@ -59,23 +62,17 @@ class ConfigDocsResource(BaseResource):
""" """
Ingests a collection of documents Ingests a collection of documents
""" """
content_length = req.content_length or 0 # Determine if this request is clearing the collection's contents.
if (content_length == 0): empty_coll = req.get_param_as_bool('empty-collection') or False
raise ApiError( if empty_coll:
title=('Content-Length is a required header'), document_data = ""
description='Content Length is 0 or not specified', LOG.debug("Collection %s is being emptied", collection_id)
status=falcon.HTTP_400, else:
error_list=[{ # Note, a newline in a prior header can trigger subsequent
'message': ( # headers to be "missing" (and hence cause this code to think
'The Content-Length specified is 0 or not set. Check ' # that the content length is missing)
'that a valid payload is included with this request ' content_length = self.validate_content_length(req.content_length)
'and that your client is properly including a ' document_data = req.stream.read(content_length)
'Content-Length header. Note that a newline character '
'in a prior header can trigger subsequent headers to '
'be ignored and trigger this failure.')
}],
retry=False, )
document_data = req.stream.read(content_length)
buffer_mode = req.get_param('buffermode') buffer_mode = req.get_param('buffermode')
@ -84,7 +81,8 @@ class ConfigDocsResource(BaseResource):
helper=helper, helper=helper,
collection_id=collection_id, collection_id=collection_id,
document_data=document_data, document_data=document_data,
buffer_mode_param=buffer_mode) buffer_mode_param=buffer_mode,
empty_collection=empty_coll)
resp.status = falcon.HTTP_201 resp.status = falcon.HTTP_201
if validations and validations['status'] == 'Success': if validations and validations['status'] == 'Success':
@ -92,6 +90,30 @@ class ConfigDocsResource(BaseResource):
resp.location = '/api/v1.0/configdocs/{}'.format(collection_id) resp.location = '/api/v1.0/configdocs/{}'.format(collection_id)
resp.body = self.to_json(validations) resp.body = self.to_json(validations)
def validate_content_length(self, content_length):
"""Validates that the content length header is valid
:param content_length: the value of the content-length header.
:returns: the validate content length value
"""
content_length = content_length or 0
if (content_length == 0):
raise ApiError(
title=('Content-Length is a required header'),
description='Content Length is 0 or not specified',
status=falcon.HTTP_400,
error_list=[{
'message': (
"The Content-Length specified is 0 or not set. To "
"clear a collection's contents, please specify "
"the query parameter 'empty-collection=true'."
"Otherwise, a non-zero length payload and "
"matching Content-Length header is required to "
"post a collection.")
}],
retry=False, )
return content_length
@policy.ApiEnforcer(policy.GET_CONFIGDOCS) @policy.ApiEnforcer(policy.GET_CONFIGDOCS)
def on_get(self, req, resp, collection_id): def on_get(self, req, resp, collection_id):
""" """
@ -132,7 +154,8 @@ class ConfigDocsResource(BaseResource):
helper, helper,
collection_id, collection_id,
document_data, document_data,
buffer_mode_param=None): buffer_mode_param=None,
empty_collection=False):
""" """
Ingest the collection after checking preconditions Ingest the collection after checking preconditions
""" """
@ -141,23 +164,28 @@ class ConfigDocsResource(BaseResource):
if helper.is_buffer_valid_for_bucket(collection_id, buffer_mode): if helper.is_buffer_valid_for_bucket(collection_id, buffer_mode):
buffer_revision = helper.add_collection(collection_id, buffer_revision = helper.add_collection(collection_id,
document_data) document_data)
if helper.is_collection_in_buffer(collection_id): if not (empty_collection or helper.is_collection_in_buffer(
return helper.get_deckhand_validation_status(buffer_revision) collection_id)):
else: # raise an error if adding the collection resulted in no new
# revision (meaning it was unchanged) and we're not explicitly
# clearing the collection
raise ApiError( raise ApiError(
title=('Collection {} not added to Shipyard ' title=('Collection {} not added to Shipyard '
'buffer'.format(collection_id)), 'buffer'.format(collection_id)),
description='Collection empty or resulted in no revision', description='Collection created no new revision',
status=falcon.HTTP_400, status=falcon.HTTP_400,
error_list=[{ error_list=[{
'message': ( 'message':
'Empty collections are not supported. After ' ('The collection {} added no new revision, and has '
'processing, the collection {} added no new ' 'been rejected as invalid input. This likely '
'revision, and has been rejected as invalid ' 'means that the collection already exists and '
'input'.format(collection_id)) 'was reloaded with the same contents'.format(
collection_id))
}], }],
retry=False, retry=False,
) )
else:
return helper.get_deckhand_validation_status(buffer_revision)
else: else:
raise ApiError( raise ApiError(
title='Invalid collection specified for buffer', title='Invalid collection specified for buffer',

View File

@ -105,8 +105,23 @@ class ConfigdocsHelper(object):
return BufferMode.REJECTONCONTENTS return BufferMode.REJECTONCONTENTS
def is_buffer_empty(self): def is_buffer_empty(self):
""" Check if the buffer is empty. """ """ Check if the buffer is empty.
return self._get_revision(BUFFER) is None
This can occur if there is no buffer revision, or if the buffer
revision is unchanged since the last committed version (or version 0)
"""
if self._get_revision(BUFFER) is None:
return True
# Get the "diff" of the collctions for Buffer vs. Committed (or 0)
collections = self.get_configdocs_status()
# If there are no collections or they are all unmodified, return True
# Deleted, created, or modified means there's something in the buffer.
if not collections:
return True
for c in collections:
if c['new_status'] != 'unmodified':
return False
return True
def is_collection_in_buffer(self, collection_id): def is_collection_in_buffer(self, collection_id):
""" """

View File

@ -1,7 +1,7 @@
# Testing # Testing
pytest==3.4 pytest==3.4
pytest-cov==2.5.1 pytest-cov==2.5.1
responses==0.8.1 responses==0.10.2
testfixtures==5.1.1 testfixtures==5.1.1
apache-airflow[crypto,celery,postgres,hive,hdfs,jdbc]==1.10.0 apache-airflow[crypto,celery,postgres,hive,hdfs,jdbc]==1.10.0

View File

@ -61,6 +61,8 @@ REV_BUFFER_DICT = {
} }
DIFF_BUFFER_DICT = {'mop': 'unmodified', 'chum': 'created', 'slop': 'deleted'} DIFF_BUFFER_DICT = {'mop': 'unmodified', 'chum': 'created', 'slop': 'deleted'}
UNMOD_BUFFER_DICT = {'mop': 'unmodified', 'chum': 'unmodified'}
EMPTY_BUFFER_DICT = {}
ORDERED_VER = ['committed', 'buffer'] ORDERED_VER = ['committed', 'buffer']
REV_NAME_ID = ('committed', 'buffer', 3, 5) REV_NAME_ID = ('committed', 'buffer', 3, 5)
@ -183,21 +185,41 @@ def test_get_buffer_mode():
assert ConfigdocsHelper.get_buffer_mode('hippopotomus') is None assert ConfigdocsHelper.get_buffer_mode('hippopotomus') is None
def test_is_buffer_emtpy(): def test_is_buffer_empty():
""" """
Test the method to check if the configdocs buffer is empty Test the method to check if the configdocs buffer is empty
""" """
helper = ConfigdocsHelper(CTX) helper = ConfigdocsHelper(CTX)
helper._get_revision_dict = lambda: REV_BUFFER_DICT
assert not helper.is_buffer_empty()
# BUFFER revision is none, short circuit case (no buffer revision)
# buffer is empty.
helper._get_revision_dict = lambda: REV_BUFF_EMPTY_DICT helper._get_revision_dict = lambda: REV_BUFF_EMPTY_DICT
assert helper.is_buffer_empty() assert helper.is_buffer_empty()
helper._get_revision_dict = lambda: REV_NO_COMMIT_DICT # BUFFER revision is none, also a short circuit case (no revisions at all)
# buffer is empty
helper._get_revision_dict = lambda: REV_EMPTY_DICT
assert helper.is_buffer_empty()
# BUFFER revision is not none, collections have been modified
# buffer is NOT empty.
helper._get_revision_dict = lambda: REV_BUFFER_DICT
helper.deckhand.get_diff = (
lambda old_revision_id, new_revision_id: DIFF_BUFFER_DICT)
assert not helper.is_buffer_empty() assert not helper.is_buffer_empty()
helper._get_revision_dict = lambda: REV_EMPTY_DICT # BUFFER revision is not none, all collections unmodified
# buffer is empty.
helper._get_revision_dict = lambda: REV_NO_COMMIT_DICT
helper.deckhand.get_diff = (
lambda old_revision_id, new_revision_id: UNMOD_BUFFER_DICT)
assert helper.is_buffer_empty()
# BUFFER revision is not none, no collections listed (deleted, rollback 0)
# buffer is empty.
helper._get_revision_dict = lambda: REV_NO_COMMIT_DICT
helper.deckhand.get_diff = (
lambda old_revision_id, new_revision_id: EMPTY_BUFFER_DICT)
assert helper.is_buffer_empty() assert helper.is_buffer_empty()

View File

@ -52,16 +52,21 @@ class ShipyardClient(BaseClient):
def post_configdocs(self, def post_configdocs(self,
collection_id=None, collection_id=None,
buffer_mode='rejectoncontents', buffer_mode='rejectoncontents',
empty_collection=False,
document_data=None): document_data=None):
""" """
Ingests a collection of documents Ingests a collection of documents
:param str collection_id: identifies a collection of docs.Bucket_id :param str collection_id: identifies a collection of docs.Bucket_id
:param str buffermode: append|replace|rejectOnContents :param str buffermode: append|replace|rejectOnContents
:param empty_collection: True if the collection is empty. Document
data will be ignored if this flag is set to True. Default: False
:param str document_data: data in a format understood by Deckhand(YAML) :param str document_data: data in a format understood by Deckhand(YAML)
:returns: diff from last committed revision to new revision :returns: diff from last committed revision to new revision
:rtype: Response object :rtype: Response object
""" """
query_params = {"buffermode": buffer_mode} query_params = {"buffermode": buffer_mode}
if empty_collection:
query_params['empty-collection'] = True
url = ApiPaths.POST_GET_CONFIG.value.format( url = ApiPaths.POST_GET_CONFIG.value.format(
self.get_endpoint(), self.get_endpoint(),
collection_id collection_id

View File

@ -47,11 +47,11 @@ SHORT_DESC_CONFIGDOCS = ("Attempts to commit the Shipyard Buffer documents, "
@click.option( @click.option(
'--force', '--force',
'-f', '-f',
flag_value=True, is_flag=True,
help='Force the commit to occur, even if validations fail.') help='Force the commit to occur, even if validations fail.')
@click.option( @click.option(
'--dryrun', '--dryrun',
flag_value=True, is_flag=True,
help='Retrieve validation status for the contents of the buffer without ' help='Retrieve validation status for the contents of the buffer without '
'committing.') 'committing.')
@click.pass_context @click.pass_context

View File

@ -22,8 +22,8 @@ class CreateAction(CliAction):
super().__init__(ctx) super().__init__(ctx)
self.logger.debug( self.logger.debug(
"CreateAction action initialized with action command " "CreateAction action initialized with action command "
"%s, parameters %s and allow-intermediate-commits=%s", "%s, parameters %s and allow-intermediate-commits=%s", action_name,
action_name, param, allow_intermediate_commits) param, allow_intermediate_commits)
self.action_name = action_name self.action_name = action_name
self.param = param self.param = param
self.allow_intermediate_commits = allow_intermediate_commits self.allow_intermediate_commits = allow_intermediate_commits
@ -57,27 +57,34 @@ class CreateAction(CliAction):
class CreateConfigdocs(CliAction): class CreateConfigdocs(CliAction):
"""Action to Create Configdocs""" """Action to Create Configdocs"""
def __init__(self, ctx, collection, buffer, data, filename): def __init__(self, ctx, collection, buffer_mode, empty_collection, data,
filenames):
"""Sets parameters.""" """Sets parameters."""
super().__init__(ctx) super().__init__(ctx)
self.logger.debug("CreateConfigdocs action initialized with " self.logger.debug(
"collection=%s,buffer=%s, " "CreateConfigdocs action initialized with collection: %s, "
"Processed Files=" % (collection, buffer)) "buffer mode: %s, empty collection: %s, data length: %s. "
for file in filename: "Processed Files:", collection, buffer_mode, empty_collection,
len(data))
for file in filenames:
self.logger.debug(file) self.logger.debug(file)
self.logger.debug("data=%s" % str(data))
self.collection = collection self.collection = collection
self.buffer = buffer self.buffer_mode = buffer_mode
self.empty_collection = empty_collection
self.data = data self.data = data
def invoke(self): def invoke(self):
"""Calls API Client and formats response from API Client""" """Calls API Client and formats response from API Client"""
self.logger.debug("Calling API Client post_configdocs.") self.logger.debug("Calling API Client post_configdocs.")
# Only send data payload if not empty_collection
data_to_send = "" if self.empty_collection else self.data
return self.get_api_client().post_configdocs( return self.get_api_client().post_configdocs(
collection_id=self.collection, collection_id=self.collection,
buffer_mode=self.buffer, buffer_mode=self.buffer_mode,
document_data=self.data empty_collection=self.empty_collection,
) document_data=data_to_send)
# Handle 409 with default error handler for cli. # Handle 409 with default error handler for cli.
cli_handled_err_resp_codes = [409] cli_handled_err_resp_codes = [409]
@ -94,5 +101,4 @@ class CreateConfigdocs(CliAction):
""" """
outfmt_string = "Configuration documents added.\n{}" outfmt_string = "Configuration documents added.\n{}"
return outfmt_string.format( return outfmt_string.format(
format_utils.cli_format_status_handler(response) format_utils.cli_format_status_handler(response))
)

View File

@ -59,7 +59,7 @@ SHORT_DESC_ACTION = (
@click.option( @click.option(
'--allow-intermediate-commits', '--allow-intermediate-commits',
'allow_intermediate_commits', 'allow_intermediate_commits',
flag_value=True, is_flag=True,
help="Allow site action to go through even though there are prior commits " help="Allow site action to go through even though there are prior commits "
"that have not been used as part of a site action.") "that have not been used as part of a site action.")
@click.pass_context @click.pass_context
@ -82,6 +82,7 @@ DESC_CONFIGDOCS = """
COMMAND: configdocs \n COMMAND: configdocs \n
DESCRIPTION: Load documents into the Shipyard Buffer. \n DESCRIPTION: Load documents into the Shipyard Buffer. \n
FORMAT: shipyard create configdocs <collection> [--append | --replace] FORMAT: shipyard create configdocs <collection> [--append | --replace]
[--empty-collection]
[--filename=<filename> (repeatable) | --directory=<directory>] (repeatable) [--filename=<filename> (repeatable) | --directory=<directory>] (repeatable)
--recurse\n --recurse\n
EXAMPLE: shipyard create configdocs design --append EXAMPLE: shipyard create configdocs design --append
@ -96,15 +97,16 @@ SHORT_DESC_CONFIGDOCS = "Load documents into the Shipyard Buffer."
@click.argument('collection') @click.argument('collection')
@click.option( @click.option(
'--append', '--append',
flag_value=True, is_flag=True,
help='Add the collection to the Shipyard Buffer. ') help='Add the collection to the Shipyard Buffer. ')
@click.option( @click.option(
'--replace', '--replace',
flag_value=True, is_flag=True,
help='Clear the Shipyard Buffer and replace it with the specified ' help='Clear the Shipyard Buffer and replace it with the specified '
'contents. ') 'contents. ')
@click.option( @click.option(
'--filename', '--filename',
'filenames',
multiple=True, multiple=True,
type=click.Path(exists=True), type=click.Path(exists=True),
help='The file name to use as the contents of the collection. ' help='The file name to use as the contents of the collection. '
@ -117,59 +119,89 @@ SHORT_DESC_CONFIGDOCS = "Load documents into the Shipyard Buffer."
'a collection. (Repeatable).') 'a collection. (Repeatable).')
@click.option( @click.option(
'--recurse', '--recurse',
flag_value=True, is_flag=True,
help='Recursively search through directories for yaml files.' help='Recursively search through directories for yaml files.'
) )
# The --empty-collection flag is explicit to prevent a user from accidentally
# loading an empty file and deleting things. This requires the user to clearly
# state their intention.
@click.option(
'--empty-collection',
is_flag=True,
help='Creates a version of the specified collection with no contents. '
'This option is the method by which a collection can be effectively '
'deleted. Any file and directory parameters will be ignored if this '
'option is used.'
)
@click.pass_context @click.pass_context
def create_configdocs(ctx, collection, filename, directory, append, def create_configdocs(ctx, collection, filenames, directory, append, replace,
replace, recurse): recurse, empty_collection):
if (append and replace): if (append and replace):
ctx.fail('Either append or replace may be selected but not both') ctx.fail('Either append or replace may be selected but not both')
if (not filename and not directory) or (filename and directory):
ctx.fail('Please specify one or more filenames using '
'--filename="<filename>" OR one or more directories using '
'--directory="<directory>"')
if append: if append:
create_buffer = 'append' buffer_mode = 'append'
elif replace: elif replace:
create_buffer = 'replace' buffer_mode = 'replace'
else: else:
create_buffer = None buffer_mode = None
if directory: if empty_collection:
for dir in directory: # Use an empty string as the document payload, and indicate no files.
if recurse: data = ""
for path, dirs, files in os.walk(dir): filenames = []
filename += tuple( else:
[os.path.join(path, name) for name in files # Validate that appropriate file/directory params were specified.
if name.endswith('.yaml')]) if (not filenames and not directory) or (filenames and directory):
else: ctx.fail('Please specify one or more filenames using '
filename += tuple( '--filename="<filename>" OR one or more directories '
[os.path.join(dir, each) for each in os.listdir(dir) 'using --directory="<directory>"')
if each.endswith('.yaml')]) # Scan and parse the input directories and files
if directory:
for _dir in directory:
if recurse:
for path, dirs, files in os.walk(_dir):
filenames += tuple([
os.path.join(path, name) for name in files
if is_yaml(name)
])
else:
filenames += tuple([
os.path.join(_dir, each) for each in os.listdir(_dir)
if is_yaml(each)
])
if not filename: if not filenames:
# None or empty list should raise this error # None or empty list should raise this error
ctx.fail('The directory does not contain any YAML files. ' ctx.fail('The directory does not contain any YAML files. '
'Please enter one or more YAML files or a ' 'Please enter one or more YAML files or a '
'directory that contains one or more YAML files.') 'directory that contains one or more YAML files.')
docs = [] docs = []
for file in filename: for _file in filenames:
with open(file, 'r') as stream: with open(_file, 'r') as stream:
if file.endswith(".yaml"): if is_yaml(_file):
try: try:
docs += list(yaml.safe_load_all(stream)) docs += list(yaml.safe_load_all(stream))
except yaml.YAMLError as exc: except yaml.YAMLError as exc:
ctx.fail('YAML file {} is invalid because {}' ctx.fail('YAML file {} is invalid because {}'.format(
.format(file, exc)) _file, exc))
else: else:
ctx.fail('The file {} is not a YAML file. Please enter ' ctx.fail('The file {} is not a YAML file. Please enter '
'only YAML files.'.format(file)) 'only YAML files.'.format(_file))
data = yaml.safe_dump_all(docs) data = yaml.safe_dump_all(docs)
click.echo( click.echo(
CreateConfigdocs(ctx, collection, create_buffer, data, filename) CreateConfigdocs(
.invoke_and_return_resp()) ctx=ctx,
collection=collection,
buffer_mode=buffer_mode,
empty_collection=empty_collection,
data=data,
filenames=filenames).invoke_and_return_resp())
def is_yaml(filename):
"""Test if the filename should be regarded as a yaml file"""
return filename.endswith(".yaml") or filename.endswith(".yml")

View File

@ -77,25 +77,25 @@ SHORT_DESC_CONFIGDOCS = ("Retrieve documents loaded into Shipyard, either "
@click.option( @click.option(
'--committed', '--committed',
'-c', '-c',
flag_value='committed', is_flag=True,
help='Retrieve the documents that have last been committed for this ' help='Retrieve the documents that have last been committed for this '
'collection') 'collection')
@click.option( @click.option(
'--buffer', '--buffer',
'-b', '-b',
flag_value='buffer', is_flag=True,
help='Retrieve the documents that have been loaded into Shipyard since ' help='Retrieve the documents that have been loaded into Shipyard since '
'the prior commit. If no documents have been loaded into the buffer for ' 'the prior commit. If no documents have been loaded into the buffer for '
'this collection, this will return an empty response (default)') 'this collection, this will return an empty response (default)')
@click.option( @click.option(
'--last-site-action', '--last-site-action',
'-l', '-l',
flag_value='last_site_action', is_flag=True,
help='Holds the revision information for the most recent site action') help='Holds the revision information for the most recent site action')
@click.option( @click.option(
'--successful-site-action', '--successful-site-action',
'-s', '-s',
flag_value='successful_site_action', is_flag=True,
help='Holds the revision information for the most recent successfully ' help='Holds the revision information for the most recent successfully '
'executed site action.') 'executed site action.')
@click.option( @click.option(
@ -150,23 +150,23 @@ SHORT_DESC_RENDEREDCONFIGDOCS = (
@click.option( @click.option(
'--committed', '--committed',
'-c', '-c',
flag_value='committed', is_flag=True,
help='Retrieve the documents that have last been committed.') help='Retrieve the documents that have last been committed.')
@click.option( @click.option(
'--buffer', '--buffer',
'-b', '-b',
flag_value='buffer', is_flag=True,
help='Retrieve the documents that have been loaded into Shipyard since the' help='Retrieve the documents that have been loaded into Shipyard since the'
' prior commit. (default)') ' prior commit. (default)')
@click.option( @click.option(
'--last-site-action', '--last-site-action',
'-l', '-l',
flag_value='last_site_action', is_flag=True,
help='Holds the revision information for the most recent site action') help='Holds the revision information for the most recent site action')
@click.option( @click.option(
'--successful-site-action', '--successful-site-action',
'-s', '-s',
flag_value='successful_site_action', is_flag=True,
help='Holds the revision information for the most recent successfully ' help='Holds the revision information for the most recent successfully '
'executed site action.') 'executed site action.')
@click.option( @click.option(

View File

@ -1,7 +1,7 @@
# Testing # Testing
pytest==3.4 pytest==3.4
pytest-cov==2.5.1 pytest-cov==2.5.1
responses==0.8.1 responses==0.10.2
testfixtures==5.1.1 testfixtures==5.1.1
# Linting # Linting

View File

@ -19,13 +19,18 @@ from shipyard_client.api_client.base_client import BaseClient
from shipyard_client.cli.commit.actions import CommitConfigdocs from shipyard_client.cli.commit.actions import CommitConfigdocs
from tests.unit.cli import stubs from tests.unit.cli import stubs
# TODO: refactor these tests to use responses callbacks (or other features)
# so that query parameter passing can be validated.
# moving to responses > 0.8 (e.g. 0.10.2) changed how URLS for responses
# seem to operate.
@responses.activate @responses.activate
@mock.patch.object(BaseClient, 'get_endpoint', lambda x: 'http://shiptest') @mock.patch.object(BaseClient, 'get_endpoint', lambda x: 'http://shiptest')
@mock.patch.object(BaseClient, 'get_token', lambda x: 'abc') @mock.patch.object(BaseClient, 'get_token', lambda x: 'abc')
def test_commit_configdocs(*args): def test_commit_configdocs(*args):
responses.add(responses.POST, responses.add(responses.POST,
'http://shiptest/commitconfigdocs?force=false', 'http://shiptest/commitconfigdocs',
body=None, body=None,
status=200) status=200)
response = CommitConfigdocs(stubs.StubCliContext(), response = CommitConfigdocs(stubs.StubCliContext(),
@ -44,7 +49,7 @@ def test_commit_configdocs_409(*args):
reason='Conflicts reason', reason='Conflicts reason',
code=409) code=409)
responses.add(responses.POST, responses.add(responses.POST,
'http://shiptest/commitconfigdocs?force=false', 'http://shiptest/commitconfigdocs',
body=api_resp, body=api_resp,
status=409) status=409)
response = CommitConfigdocs(stubs.StubCliContext(), response = CommitConfigdocs(stubs.StubCliContext(),
@ -65,7 +70,7 @@ def test_commit_configdocs_forced(*args):
reason='Conflicts reason', reason='Conflicts reason',
code=200) code=200)
responses.add(responses.POST, responses.add(responses.POST,
'http://shiptest/commitconfigdocs?force=true', 'http://shiptest/commitconfigdocs',
body=api_resp, body=api_resp,
status=200) status=200)
response = CommitConfigdocs(stubs.StubCliContext(), response = CommitConfigdocs(stubs.StubCliContext(),
@ -80,7 +85,7 @@ def test_commit_configdocs_forced(*args):
@mock.patch.object(BaseClient, 'get_token', lambda x: 'abc') @mock.patch.object(BaseClient, 'get_token', lambda x: 'abc')
def test_commit_configdocs_dryrun(*args): def test_commit_configdocs_dryrun(*args):
responses.add(responses.POST, responses.add(responses.POST,
'http://shiptest/commitconfigdocs?force=false', 'http://shiptest/commitconfigdocs',
body=None, body=None,
status=200) status=200)
response = CommitConfigdocs(stubs.StubCliContext(), response = CommitConfigdocs(stubs.StubCliContext(),

View File

@ -116,6 +116,7 @@ def test_create_configdocs(*args):
response = CreateConfigdocs(stubs.StubCliContext(), response = CreateConfigdocs(stubs.StubCliContext(),
'design', 'design',
'append', 'append',
False,
document_data, document_data,
file_list).invoke_and_return_resp() file_list).invoke_and_return_resp()
assert 'Configuration documents added.' assert 'Configuration documents added.'
@ -145,6 +146,7 @@ def test_create_configdocs_201_with_val_fails(*args):
response = CreateConfigdocs(stubs.StubCliContext(), response = CreateConfigdocs(stubs.StubCliContext(),
'design', 'design',
'append', 'append',
False,
document_data, document_data,
file_list).invoke_and_return_resp() file_list).invoke_and_return_resp()
assert 'Configuration documents added.' in response assert 'Configuration documents added.' in response
@ -175,8 +177,51 @@ def test_create_configdocs_409(*args):
response = CreateConfigdocs(stubs.StubCliContext(), response = CreateConfigdocs(stubs.StubCliContext(),
'design', 'design',
'append', 'append',
False,
document_data, document_data,
file_list).invoke_and_return_resp() file_list).invoke_and_return_resp()
assert 'Error: Invalid collection' in response assert 'Error: Invalid collection' in response
assert 'Reason: Buffermode : append' in response assert 'Reason: Buffermode : append' in response
assert 'Buffer is either not...' in response assert 'Buffer is either not...' in response
@responses.activate
@mock.patch.object(BaseClient, 'get_endpoint', lambda x: 'http://shiptest')
@mock.patch.object(BaseClient, 'get_token', lambda x: 'abc')
def test_create_configdocs_empty(*args):
def validating_callback(request):
# a request that has empty_collection should have no body.
assert request.body is None
resp_body = stubs.gen_err_resp(
message='Validations succeeded',
sub_error_count=0,
sub_info_count=0,
reason='Validation',
code=200)
return (201, {}, resp_body)
responses.add_callback(
responses.POST,
'http://shiptest/configdocs/design',
callback=validating_callback,
content_type='application/json')
filename = 'tests/unit/cli/create/sample_yaml/sample.yaml'
document_data = yaml.dump_all(filename)
file_list = (filename, )
# pass data and empty_collection = True - should init with data, but
# not send the data on invoke
action = CreateConfigdocs(
stubs.StubCliContext(),
collection='design',
buffer_mode='append',
empty_collection=True,
data=document_data,
filenames=file_list)
assert action.data == document_data
assert action.empty_collection == True
response = action.invoke_and_return_resp()
assert response.startswith("Configuration documents added.")

View File

@ -66,8 +66,93 @@ def test_create_configdocs():
auth_vars, 'create', 'configdocs', collection, '--' + append, auth_vars, 'create', 'configdocs', collection, '--' + append,
'--filename=' + filename '--filename=' + filename
]) ])
mock_method.assert_called_once_with(ANY, collection, 'append', mock_method.assert_called_once_with(ctx=ANY, collection=collection,
ANY, file_list) buffer_mode='append', empty_collection=False, data=ANY,
filenames=file_list)
def test_create_configdocs_empty():
"""test create configdocs with the --empty-collection flag"""
collection = 'design'
filename = 'tests/unit/cli/create/sample_yaml/sample.yaml'
directory = 'tests/unit/cli/create/sample_yaml'
runner = CliRunner()
tests = [
{
# replace mode, no file, no data, empty collection
'kwargs': {
'buffer_mode': 'replace',
'empty_collection': True,
'filenames': [],
'data': ""
},
'args': [
'--replace',
'--empty-collection',
],
},
{
# Append mode, no file, no data, empty collection
'kwargs': {
'buffer_mode': 'append',
'empty_collection': True,
'filenames': [],
'data': ""
},
'args': [
'--append',
'--empty-collection',
],
},
{
# No buffer mode specified, empty collection
'kwargs': {
'buffer_mode': None,
'empty_collection': True,
'filenames': [],
'data': ""
},
'args': [
'--empty-collection',
],
},
{
# Filename should be ignored and not passed, empty collection
'kwargs': {
'buffer_mode': None,
'empty_collection': True,
'filenames': [],
'data': ""
},
'args': [
'--empty-collection',
'--filename={}'.format(filename)
],
},
{
# Directory should be ignored and not passed, empty collection
'kwargs': {
'buffer_mode': None,
'empty_collection': True,
'filenames': [],
'data': ""
},
'args': [
'--empty-collection',
'--directory={}'.format(directory)
],
},
]
for tc in tests:
with patch.object(CreateConfigdocs, '__init__') as mock_method:
runner.invoke(shipyard, [
auth_vars, 'create', 'configdocs', collection, *tc['args']
])
mock_method.assert_called_once_with(ctx=ANY, collection=collection,
**tc['kwargs'])
def test_create_configdocs_directory(): def test_create_configdocs_directory():
@ -82,7 +167,11 @@ def test_create_configdocs_directory():
auth_vars, 'create', 'configdocs', collection, '--' + append, auth_vars, 'create', 'configdocs', collection, '--' + append,
'--directory=' + directory '--directory=' + directory
]) ])
mock_method.assert_called_once_with(ANY, collection, 'append', ANY, ANY) # TODO(bryan-strassner) Make this test useful to show directory parsing
# happened.
mock_method.assert_called_once_with(ctx=ANY, collection=collection,
buffer_mode='append', empty_collection=False, data=ANY,
filenames=ANY)
def test_create_configdocs_directory_empty(): def test_create_configdocs_directory_empty():
@ -114,11 +203,15 @@ def test_create_configdocs_multi_directory():
auth_vars, 'create', 'configdocs', collection, '--' + append, auth_vars, 'create', 'configdocs', collection, '--' + append,
'--directory=' + dir1, '--directory=' + dir2 '--directory=' + dir1, '--directory=' + dir2
]) ])
mock_method.assert_called_once_with(ANY, collection, 'append', ANY, ANY) # TODO(bryan-strassner) Make this test useful to show multiple directories
# were actually traversed.
mock_method.assert_called_once_with(ctx=ANY, collection=collection,
buffer_mode='append', empty_collection=False, data=ANY,
filenames=ANY)
def test_create_configdocs_multi_directory_recurse(): def test_create_configdocs_multi_directory_recurse():
"""test create configdocs with multiple directories""" """test create configdocs with multiple directories recursively"""
collection = 'design' collection = 'design'
dir1 = 'tests/unit/cli/create/sample_yaml/' dir1 = 'tests/unit/cli/create/sample_yaml/'
@ -130,7 +223,11 @@ def test_create_configdocs_multi_directory_recurse():
auth_vars, 'create', 'configdocs', collection, '--' + append, auth_vars, 'create', 'configdocs', collection, '--' + append,
'--directory=' + dir1, '--directory=' + dir2, '--recurse' '--directory=' + dir1, '--directory=' + dir2, '--recurse'
]) ])
mock_method.assert_called_once_with(ANY, collection, 'append', ANY, ANY) # TODO(bryan-strassner) Make this test useful to show multiple directories
# were actually traversed and recursed.
mock_method.assert_called_once_with(ctx=ANY, collection=collection,
buffer_mode='append', empty_collection=False, data=ANY,
filenames=ANY)
def test_create_configdocs_negative(): def test_create_configdocs_negative():