Browse Source

Support clearing collections of configdocs

Adds an option to create configdocs as an empty colleciton. This is done
as an explicit flag (--empty-collection) on the command as opposed to
using empty files to prevent accidental emtpy file loads leading to
frustration.

Since this introduced a new flag value for the CLI, the CLIs using flag
values were updated to use the standard is_flag=True instead of the
flag_value=True or some other value when a boolean flag is expected.

Minor updates to CLI tests due to moving to responses 0.10.2

Depends-On: https://review.openstack.org/#/c/614421/

Change-Id: I489b0e1183335cbfbaa2014c1458a84dadf6bb0b
changes/57/611457/10
Bryan Strassner 9 months ago
parent
commit
667a538330

+ 7
- 4
doc/source/API.rst View File

@@ -158,10 +158,9 @@ Deckhand
158 158
 
159 159
 POST /v1.0/configdocs/{collection_id}
160 160
 ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
161
-Ingests a collection of documents. Synchronous. POSTing an empty body
162
-indicates that the specified collection should be deleted when the
163
-Shipyard Buffer is committed. If a POST to the commitconfigdocs is in
164
-progress, this POST should be rejected with a 409 error.
161
+Ingests a collection of documents. Synchronous. If a POST to the
162
+commitconfigdocs is already in progress, this POST should be rejected with a
163
+409 error.
165 164
 
166 165
 .. note::
167 166
 
@@ -183,6 +182,10 @@ Query Parameters
183 182
    -  replace: Clear the Shipyard Buffer before adding the specified
184 183
       collection.
185 184
 
185
+-  empty-collection: Set to true to indicate that this collection should be
186
+   made empty and effectively deleted when the Shipyard Buffer is committed.
187
+   If this parameter is specified, the POST body will be ignored.
188
+
186 189
 Responses
187 190
 '''''''''
188 191
 201 Created

+ 12
- 7
doc/source/CLI.rst View File

@@ -293,7 +293,7 @@ or one or more directory options must be specified.
293 293
 
294 294
     shipyard create configdocs
295 295
         <collection>
296
-        [--append | --replace]
296
+        [--append | --replace] [--empty-collection]
297 297
         --filename=<filename>    (repeatable)
298 298
             |
299 299
         --directory=<directory>  (repeatable)
@@ -308,8 +308,8 @@ or one or more directory options must be specified.
308 308
 
309 309
 .. note::
310 310
 
311
-  Either --filename or --directory must be specified, but both may not be
312
-  specified for the same invocation of shipyard.
311
+  --filename and/or --directory must be specified unless --empty-collection
312
+  is used.
313 313
 
314 314
 <collection>
315 315
   The collection to load.
@@ -321,17 +321,22 @@ or one or more directory options must be specified.
321 321
 \--replace
322 322
   Clear the shipyard buffer and replace it with the specified contents.
323 323
 
324
+\--empty-collection
325
+  Indicate to Shipyard that the named collection should be made empty (contain
326
+  no documents). If --empty-collection is specified, the files named by
327
+  --filename or --directory will be ignored.
328
+
324 329
 \--filename=<filename>
325 330
   The file name to use as the contents of the collection. (repeatable) If
326 331
   any documents specified fail basic validation, all of the documents will
327
-  be rejected. Use of filename parameters may not be used in conjunction
332
+  be rejected. Use of ``filename`` parameters may not be used in conjunction
328 333
   with the directory parameter.
329 334
 
330 335
 \--directory=<directory>
331 336
   A directory containing documents that will be joined and loaded as a
332
-  collection. (Repeatable) Any documents that fail basic validation will reject the
333
-  whole set. Use of the directory parameter may not be used with the
334
-  filename parameter.
337
+  collection. (Repeatable) Any documents that fail basic validation will
338
+  reject the whole set. Use of the ``directory`` parameter may not be used
339
+  with the ``filename`` parameter.
335 340
 
336 341
 \--recurse
337 342
   Recursively search through all directories for sub-directories that

+ 56
- 28
src/bin/shipyard_airflow/shipyard_airflow/control/configdocs/configdocs_api.py View File

@@ -14,6 +14,8 @@
14 14
 """
15 15
 Resources representing the configdocs API for shipyard
16 16
 """
17
+import logging
18
+
17 19
 import falcon
18 20
 from oslo_config import cfg
19 21
 
@@ -26,6 +28,7 @@ from shipyard_airflow.control.helpers.configdocs_helper import (
26 28
 from shipyard_airflow.errors import ApiError
27 29
 
28 30
 CONF = cfg.CONF
31
+LOG = logging.getLogger(__name__)
29 32
 VERSION_VALUES = ['buffer',
30 33
                   'committed',
31 34
                   'last_site_action',
@@ -59,23 +62,17 @@ class ConfigDocsResource(BaseResource):
59 62
         """
60 63
         Ingests a collection of documents
61 64
         """
62
-        content_length = req.content_length or 0
63
-        if (content_length == 0):
64
-            raise ApiError(
65
-                title=('Content-Length is a required header'),
66
-                description='Content Length is 0 or not specified',
67
-                status=falcon.HTTP_400,
68
-                error_list=[{
69
-                    'message': (
70
-                        'The Content-Length specified is 0 or not set. Check '
71
-                        'that a valid payload is included with this request '
72
-                        'and that your client is properly including a '
73
-                        'Content-Length header. Note that a newline character '
74
-                        'in a prior header can trigger subsequent headers to '
75
-                        'be ignored and trigger this failure.')
76
-                }],
77
-                retry=False, )
78
-        document_data = req.stream.read(content_length)
65
+        # Determine if this request is clearing the collection's contents.
66
+        empty_coll = req.get_param_as_bool('empty-collection') or False
67
+        if empty_coll:
68
+            document_data = ""
69
+            LOG.debug("Collection %s is being emptied", collection_id)
70
+        else:
71
+            # Note, a newline in a prior header can trigger subsequent
72
+            # headers to be "missing" (and hence cause this code to think
73
+            # that the content length is missing)
74
+            content_length = self.validate_content_length(req.content_length)
75
+            document_data = req.stream.read(content_length)
79 76
 
80 77
         buffer_mode = req.get_param('buffermode')
81 78
 
@@ -84,7 +81,8 @@ class ConfigDocsResource(BaseResource):
84 81
             helper=helper,
85 82
             collection_id=collection_id,
86 83
             document_data=document_data,
87
-            buffer_mode_param=buffer_mode)
84
+            buffer_mode_param=buffer_mode,
85
+            empty_collection=empty_coll)
88 86
 
89 87
         resp.status = falcon.HTTP_201
90 88
         if validations and validations['status'] == 'Success':
@@ -92,6 +90,30 @@ class ConfigDocsResource(BaseResource):
92 90
         resp.location = '/api/v1.0/configdocs/{}'.format(collection_id)
93 91
         resp.body = self.to_json(validations)
94 92
 
93
+    def validate_content_length(self, content_length):
94
+        """Validates that the content length header is valid
95
+
96
+        :param content_length: the value of the content-length header.
97
+        :returns: the validate content length value
98
+        """
99
+        content_length = content_length or 0
100
+        if (content_length == 0):
101
+            raise ApiError(
102
+                title=('Content-Length is a required header'),
103
+                description='Content Length is 0 or not specified',
104
+                status=falcon.HTTP_400,
105
+                error_list=[{
106
+                    'message': (
107
+                        "The Content-Length specified is 0 or not set. To "
108
+                        "clear a collection's contents, please specify "
109
+                        "the query parameter 'empty-collection=true'."
110
+                        "Otherwise, a non-zero length payload and "
111
+                        "matching Content-Length header is required to "
112
+                        "post a collection.")
113
+                }],
114
+                retry=False, )
115
+        return content_length
116
+
95 117
     @policy.ApiEnforcer(policy.GET_CONFIGDOCS)
96 118
     def on_get(self, req, resp, collection_id):
97 119
         """
@@ -132,7 +154,8 @@ class ConfigDocsResource(BaseResource):
132 154
                         helper,
133 155
                         collection_id,
134 156
                         document_data,
135
-                        buffer_mode_param=None):
157
+                        buffer_mode_param=None,
158
+                        empty_collection=False):
136 159
         """
137 160
         Ingest the collection after checking preconditions
138 161
         """
@@ -141,23 +164,28 @@ class ConfigDocsResource(BaseResource):
141 164
         if helper.is_buffer_valid_for_bucket(collection_id, buffer_mode):
142 165
             buffer_revision = helper.add_collection(collection_id,
143 166
                                                     document_data)
144
-            if helper.is_collection_in_buffer(collection_id):
145
-                return helper.get_deckhand_validation_status(buffer_revision)
146
-            else:
167
+            if not (empty_collection or helper.is_collection_in_buffer(
168
+                    collection_id)):
169
+                # raise an error if adding the collection resulted in no new
170
+                # revision (meaning it was unchanged) and we're not explicitly
171
+                # clearing the collection
147 172
                 raise ApiError(
148 173
                     title=('Collection {} not added to Shipyard '
149 174
                            'buffer'.format(collection_id)),
150
-                    description='Collection empty or resulted in no revision',
175
+                    description='Collection created no new revision',
151 176
                     status=falcon.HTTP_400,
152 177
                     error_list=[{
153
-                        'message': (
154
-                            'Empty collections are not supported. After '
155
-                            'processing, the collection {} added no new '
156
-                            'revision, and has been rejected as invalid '
157
-                            'input'.format(collection_id))
178
+                        'message':
179
+                        ('The collection {} added no new revision, and has '
180
+                         'been rejected as invalid input. This likely '
181
+                         'means that the collection already exists and '
182
+                         'was reloaded with the same contents'.format(
183
+                             collection_id))
158 184
                     }],
159 185
                     retry=False,
160 186
                 )
187
+            else:
188
+                return helper.get_deckhand_validation_status(buffer_revision)
161 189
         else:
162 190
             raise ApiError(
163 191
                 title='Invalid collection specified for buffer',

+ 17
- 2
src/bin/shipyard_airflow/shipyard_airflow/control/helpers/configdocs_helper.py View File

@@ -105,8 +105,23 @@ class ConfigdocsHelper(object):
105 105
         return BufferMode.REJECTONCONTENTS
106 106
 
107 107
     def is_buffer_empty(self):
108
-        """ Check if the buffer is empty. """
109
-        return self._get_revision(BUFFER) is None
108
+        """ Check if the buffer is empty.
109
+
110
+        This can occur if there is no buffer revision, or if the buffer
111
+        revision is unchanged since the last committed version (or version 0)
112
+        """
113
+        if self._get_revision(BUFFER) is None:
114
+            return True
115
+        # Get the "diff" of the collctions for Buffer vs. Committed (or 0)
116
+        collections = self.get_configdocs_status()
117
+        # If there are no collections or they are all unmodified, return True
118
+        # Deleted, created, or modified means there's something in the buffer.
119
+        if not collections:
120
+            return True
121
+        for c in collections:
122
+            if c['new_status'] != 'unmodified':
123
+                return False
124
+        return True
110 125
 
111 126
     def is_collection_in_buffer(self, collection_id):
112 127
         """

+ 1
- 1
src/bin/shipyard_airflow/test-requirements.txt View File

@@ -1,7 +1,7 @@
1 1
 # Testing
2 2
 pytest==3.4
3 3
 pytest-cov==2.5.1
4
-responses==0.8.1
4
+responses==0.10.2
5 5
 testfixtures==5.1.1
6 6
 apache-airflow[crypto,celery,postgres,hive,hdfs,jdbc]==1.10.0
7 7
 

+ 27
- 5
src/bin/shipyard_airflow/tests/unit/control/test_configdocs_helper.py View File

@@ -61,6 +61,8 @@ REV_BUFFER_DICT = {
61 61
 }
62 62
 
63 63
 DIFF_BUFFER_DICT = {'mop': 'unmodified', 'chum': 'created', 'slop': 'deleted'}
64
+UNMOD_BUFFER_DICT = {'mop': 'unmodified', 'chum': 'unmodified'}
65
+EMPTY_BUFFER_DICT = {}
64 66
 
65 67
 ORDERED_VER = ['committed', 'buffer']
66 68
 REV_NAME_ID = ('committed', 'buffer', 3, 5)
@@ -183,21 +185,41 @@ def test_get_buffer_mode():
183 185
     assert ConfigdocsHelper.get_buffer_mode('hippopotomus') is None
184 186
 
185 187
 
186
-def test_is_buffer_emtpy():
188
+def test_is_buffer_empty():
187 189
     """
188 190
     Test the method to check if the configdocs buffer is empty
189 191
     """
190 192
     helper = ConfigdocsHelper(CTX)
191
-    helper._get_revision_dict = lambda: REV_BUFFER_DICT
192
-    assert not helper.is_buffer_empty()
193 193
 
194
+    # BUFFER revision is none, short circuit case (no buffer revision)
195
+    # buffer is empty.
194 196
     helper._get_revision_dict = lambda: REV_BUFF_EMPTY_DICT
195 197
     assert helper.is_buffer_empty()
196 198
 
197
-    helper._get_revision_dict = lambda: REV_NO_COMMIT_DICT
199
+    # BUFFER revision is none, also a short circuit case (no revisions at all)
200
+    # buffer is empty
201
+    helper._get_revision_dict = lambda: REV_EMPTY_DICT
202
+    assert helper.is_buffer_empty()
203
+
204
+    # BUFFER revision is not none, collections have been modified
205
+    # buffer is NOT empty.
206
+    helper._get_revision_dict = lambda: REV_BUFFER_DICT
207
+    helper.deckhand.get_diff = (
208
+        lambda old_revision_id, new_revision_id: DIFF_BUFFER_DICT)
198 209
     assert not helper.is_buffer_empty()
199 210
 
200
-    helper._get_revision_dict = lambda: REV_EMPTY_DICT
211
+    # BUFFER revision is not none, all collections unmodified
212
+    # buffer is empty.
213
+    helper._get_revision_dict = lambda: REV_NO_COMMIT_DICT
214
+    helper.deckhand.get_diff = (
215
+        lambda old_revision_id, new_revision_id: UNMOD_BUFFER_DICT)
216
+    assert helper.is_buffer_empty()
217
+
218
+    # BUFFER revision is not none, no collections listed (deleted, rollback 0)
219
+    # buffer is empty.
220
+    helper._get_revision_dict = lambda: REV_NO_COMMIT_DICT
221
+    helper.deckhand.get_diff = (
222
+        lambda old_revision_id, new_revision_id: EMPTY_BUFFER_DICT)
201 223
     assert helper.is_buffer_empty()
202 224
 
203 225
 

+ 5
- 0
src/bin/shipyard_client/shipyard_client/api_client/shipyard_api_client.py View File

@@ -52,16 +52,21 @@ class ShipyardClient(BaseClient):
52 52
     def post_configdocs(self,
53 53
                         collection_id=None,
54 54
                         buffer_mode='rejectoncontents',
55
+                        empty_collection=False,
55 56
                         document_data=None):
56 57
         """
57 58
         Ingests a collection of documents
58 59
         :param str collection_id: identifies a collection of docs.Bucket_id
59 60
         :param str buffermode: append|replace|rejectOnContents
61
+        :param empty_collection: True if the collection is empty. Document
62
+            data will be ignored if this flag is set to True. Default: False
60 63
         :param str document_data: data in a format understood by Deckhand(YAML)
61 64
         :returns: diff from last committed revision to new revision
62 65
         :rtype: Response object
63 66
         """
64 67
         query_params = {"buffermode": buffer_mode}
68
+        if empty_collection:
69
+            query_params['empty-collection'] = True
65 70
         url = ApiPaths.POST_GET_CONFIG.value.format(
66 71
             self.get_endpoint(),
67 72
             collection_id

+ 2
- 2
src/bin/shipyard_client/shipyard_client/cli/commit/commands.py View File

@@ -47,11 +47,11 @@ SHORT_DESC_CONFIGDOCS = ("Attempts to commit the Shipyard Buffer documents, "
47 47
 @click.option(
48 48
     '--force',
49 49
     '-f',
50
-    flag_value=True,
50
+    is_flag=True,
51 51
     help='Force the commit to occur, even if validations fail.')
52 52
 @click.option(
53 53
     '--dryrun',
54
-    flag_value=True,
54
+    is_flag=True,
55 55
     help='Retrieve validation status for the contents of the buffer without '
56 56
     'committing.')
57 57
 @click.pass_context

+ 20
- 14
src/bin/shipyard_client/shipyard_client/cli/create/actions.py View File

@@ -22,8 +22,8 @@ class CreateAction(CliAction):
22 22
         super().__init__(ctx)
23 23
         self.logger.debug(
24 24
             "CreateAction action initialized with action command "
25
-            "%s, parameters %s and allow-intermediate-commits=%s",
26
-            action_name, param, allow_intermediate_commits)
25
+            "%s, parameters %s and allow-intermediate-commits=%s", action_name,
26
+            param, allow_intermediate_commits)
27 27
         self.action_name = action_name
28 28
         self.param = param
29 29
         self.allow_intermediate_commits = allow_intermediate_commits
@@ -57,27 +57,34 @@ class CreateAction(CliAction):
57 57
 class CreateConfigdocs(CliAction):
58 58
     """Action to Create Configdocs"""
59 59
 
60
-    def __init__(self, ctx, collection, buffer, data, filename):
60
+    def __init__(self, ctx, collection, buffer_mode, empty_collection, data,
61
+                 filenames):
61 62
         """Sets parameters."""
62 63
         super().__init__(ctx)
63
-        self.logger.debug("CreateConfigdocs action initialized with "
64
-                          "collection=%s,buffer=%s, "
65
-                          "Processed Files=" % (collection, buffer))
66
-        for file in filename:
64
+        self.logger.debug(
65
+            "CreateConfigdocs action initialized with collection: %s, "
66
+            "buffer mode: %s, empty collection: %s, data length: %s. "
67
+            "Processed Files:", collection, buffer_mode, empty_collection,
68
+            len(data))
69
+        for file in filenames:
67 70
             self.logger.debug(file)
68
-        self.logger.debug("data=%s" % str(data))
69 71
         self.collection = collection
70
-        self.buffer = buffer
72
+        self.buffer_mode = buffer_mode
73
+        self.empty_collection = empty_collection
71 74
         self.data = data
72 75
 
73 76
     def invoke(self):
74 77
         """Calls API Client and formats response from API Client"""
75 78
         self.logger.debug("Calling API Client post_configdocs.")
79
+
80
+        # Only send data payload if not empty_collection
81
+        data_to_send = "" if self.empty_collection else self.data
82
+
76 83
         return self.get_api_client().post_configdocs(
77 84
             collection_id=self.collection,
78
-            buffer_mode=self.buffer,
79
-            document_data=self.data
80
-        )
85
+            buffer_mode=self.buffer_mode,
86
+            empty_collection=self.empty_collection,
87
+            document_data=data_to_send)
81 88
 
82 89
     # Handle 409 with default error handler for cli.
83 90
     cli_handled_err_resp_codes = [409]
@@ -94,5 +101,4 @@ class CreateConfigdocs(CliAction):
94 101
         """
95 102
         outfmt_string = "Configuration documents added.\n{}"
96 103
         return outfmt_string.format(
97
-            format_utils.cli_format_status_handler(response)
98
-        )
104
+            format_utils.cli_format_status_handler(response))

+ 80
- 48
src/bin/shipyard_client/shipyard_client/cli/create/commands.py View File

@@ -59,7 +59,7 @@ SHORT_DESC_ACTION = (
59 59
 @click.option(
60 60
     '--allow-intermediate-commits',
61 61
     'allow_intermediate_commits',
62
-    flag_value=True,
62
+    is_flag=True,
63 63
     help="Allow site action to go through even though there are prior commits "
64 64
     "that have not been used as part of a site action.")
65 65
 @click.pass_context
@@ -82,6 +82,7 @@ DESC_CONFIGDOCS = """
82 82
 COMMAND: configdocs \n
83 83
 DESCRIPTION: Load documents into the Shipyard Buffer. \n
84 84
 FORMAT: shipyard create configdocs <collection> [--append | --replace]
85
+[--empty-collection]
85 86
 [--filename=<filename> (repeatable) | --directory=<directory>] (repeatable)
86 87
  --recurse\n
87 88
 EXAMPLE: shipyard create configdocs design --append
@@ -96,15 +97,16 @@ SHORT_DESC_CONFIGDOCS = "Load documents into the Shipyard Buffer."
96 97
 @click.argument('collection')
97 98
 @click.option(
98 99
     '--append',
99
-    flag_value=True,
100
+    is_flag=True,
100 101
     help='Add the collection to the Shipyard Buffer. ')
101 102
 @click.option(
102 103
     '--replace',
103
-    flag_value=True,
104
+    is_flag=True,
104 105
     help='Clear the Shipyard Buffer and replace it with the specified '
105 106
     'contents. ')
106 107
 @click.option(
107 108
     '--filename',
109
+    'filenames',
108 110
     multiple=True,
109 111
     type=click.Path(exists=True),
110 112
     help='The file name to use as the contents of the collection. '
@@ -117,59 +119,89 @@ SHORT_DESC_CONFIGDOCS = "Load documents into the Shipyard Buffer."
117 119
     'a collection. (Repeatable).')
118 120
 @click.option(
119 121
     '--recurse',
120
-    flag_value=True,
122
+    is_flag=True,
121 123
     help='Recursively search through directories for yaml files.'
122 124
 )
125
+# The --empty-collection flag is explicit to prevent a user from accidentally
126
+# loading an empty file and deleting things. This requires the user to clearly
127
+# state their intention.
128
+@click.option(
129
+    '--empty-collection',
130
+    is_flag=True,
131
+    help='Creates a version of the specified collection with no contents. '
132
+    'This option is the method by which a collection can be effectively '
133
+    'deleted. Any file and directory parameters will be ignored if this '
134
+    'option is used.'
135
+)
123 136
 @click.pass_context
124
-def create_configdocs(ctx, collection, filename, directory, append,
125
-                      replace, recurse):
137
+def create_configdocs(ctx, collection, filenames, directory, append, replace,
138
+                      recurse, empty_collection):
126 139
     if (append and replace):
127 140
         ctx.fail('Either append or replace may be selected but not both')
128
-    if (not filename and not directory) or (filename and directory):
129
-        ctx.fail('Please specify one or more filenames using '
130
-                 '--filename="<filename>" OR one or more directories using '
131
-                 '--directory="<directory>"')
132 141
 
133 142
     if append:
134
-        create_buffer = 'append'
143
+        buffer_mode = 'append'
135 144
     elif replace:
136
-        create_buffer = 'replace'
145
+        buffer_mode = 'replace'
146
+    else:
147
+        buffer_mode = None
148
+
149
+    if empty_collection:
150
+        # Use an empty string as the document payload, and indicate no files.
151
+        data = ""
152
+        filenames = []
137 153
     else:
138
-        create_buffer = None
139
-
140
-    if directory:
141
-        for dir in directory:
142
-            if recurse:
143
-                for path, dirs, files in os.walk(dir):
144
-                    filename += tuple(
145
-                        [os.path.join(path, name) for name in files
146
-                         if name.endswith('.yaml')])
147
-            else:
148
-                filename += tuple(
149
-                    [os.path.join(dir, each) for each in os.listdir(dir)
150
-                     if each.endswith('.yaml')])
151
-
152
-        if not filename:
153
-            # None or empty list should raise this error
154
-            ctx.fail('The directory does not contain any YAML files. '
155
-                     'Please enter one or more YAML files or a '
156
-                     'directory that contains one or more YAML files.')
157
-
158
-    docs = []
159
-    for file in filename:
160
-        with open(file, 'r') as stream:
161
-            if file.endswith(".yaml"):
162
-                try:
163
-                    docs += list(yaml.safe_load_all(stream))
164
-                except yaml.YAMLError as exc:
165
-                    ctx.fail('YAML file {} is invalid because {}'
166
-                             .format(file, exc))
167
-            else:
168
-                ctx.fail('The file {} is not a YAML file.  Please enter '
169
-                         'only YAML files.'.format(file))
170
-
171
-    data = yaml.safe_dump_all(docs)
154
+        # Validate that appropriate file/directory params were specified.
155
+        if (not filenames and not directory) or (filenames and directory):
156
+            ctx.fail('Please specify one or more filenames using '
157
+                     '--filename="<filename>" OR one or more directories '
158
+                     'using --directory="<directory>"')
159
+        # Scan and parse the input directories and files
160
+        if directory:
161
+            for _dir in directory:
162
+                if recurse:
163
+                    for path, dirs, files in os.walk(_dir):
164
+                        filenames += tuple([
165
+                            os.path.join(path, name) for name in files
166
+                            if is_yaml(name)
167
+                        ])
168
+                else:
169
+                    filenames += tuple([
170
+                        os.path.join(_dir, each) for each in os.listdir(_dir)
171
+                        if is_yaml(each)
172
+                    ])
173
+
174
+            if not filenames:
175
+                # None or empty list should raise this error
176
+                ctx.fail('The directory does not contain any YAML files. '
177
+                         'Please enter one or more YAML files or a '
178
+                         'directory that contains one or more YAML files.')
179
+
180
+        docs = []
181
+        for _file in filenames:
182
+            with open(_file, 'r') as stream:
183
+                if is_yaml(_file):
184
+                    try:
185
+                        docs += list(yaml.safe_load_all(stream))
186
+                    except yaml.YAMLError as exc:
187
+                        ctx.fail('YAML file {} is invalid because {}'.format(
188
+                            _file, exc))
189
+                else:
190
+                    ctx.fail('The file {} is not a YAML file.  Please enter '
191
+                             'only YAML files.'.format(_file))
192
+
193
+        data = yaml.safe_dump_all(docs)
172 194
 
173 195
     click.echo(
174
-        CreateConfigdocs(ctx, collection, create_buffer, data, filename)
175
-        .invoke_and_return_resp())
196
+        CreateConfigdocs(
197
+            ctx=ctx,
198
+            collection=collection,
199
+            buffer_mode=buffer_mode,
200
+            empty_collection=empty_collection,
201
+            data=data,
202
+            filenames=filenames).invoke_and_return_resp())
203
+
204
+
205
+def is_yaml(filename):
206
+    """Test if the filename should be regarded as a yaml file"""
207
+    return filename.endswith(".yaml") or filename.endswith(".yml")

+ 8
- 8
src/bin/shipyard_client/shipyard_client/cli/get/commands.py View File

@@ -77,25 +77,25 @@ SHORT_DESC_CONFIGDOCS = ("Retrieve documents loaded into Shipyard, either "
77 77
 @click.option(
78 78
     '--committed',
79 79
     '-c',
80
-    flag_value='committed',
80
+    is_flag=True,
81 81
     help='Retrieve the documents that have last been committed for this '
82 82
     'collection')
83 83
 @click.option(
84 84
     '--buffer',
85 85
     '-b',
86
-    flag_value='buffer',
86
+    is_flag=True,
87 87
     help='Retrieve the documents that have been loaded into Shipyard since '
88 88
     'the prior commit. If no documents have been loaded into the buffer for '
89 89
     'this collection, this will return an empty response (default)')
90 90
 @click.option(
91 91
     '--last-site-action',
92 92
     '-l',
93
-    flag_value='last_site_action',
93
+    is_flag=True,
94 94
     help='Holds the revision information for the most recent site action')
95 95
 @click.option(
96 96
     '--successful-site-action',
97 97
     '-s',
98
-    flag_value='successful_site_action',
98
+    is_flag=True,
99 99
     help='Holds the revision information for the most recent successfully '
100 100
     'executed site action.')
101 101
 @click.option(
@@ -150,23 +150,23 @@ SHORT_DESC_RENDEREDCONFIGDOCS = (
150 150
 @click.option(
151 151
     '--committed',
152 152
     '-c',
153
-    flag_value='committed',
153
+    is_flag=True,
154 154
     help='Retrieve the documents that have last been committed.')
155 155
 @click.option(
156 156
     '--buffer',
157 157
     '-b',
158
-    flag_value='buffer',
158
+    is_flag=True,
159 159
     help='Retrieve the documents that have been loaded into Shipyard since the'
160 160
     ' prior commit. (default)')
161 161
 @click.option(
162 162
     '--last-site-action',
163 163
     '-l',
164
-    flag_value='last_site_action',
164
+    is_flag=True,
165 165
     help='Holds the revision information for the most recent site action')
166 166
 @click.option(
167 167
     '--successful-site-action',
168 168
     '-s',
169
-    flag_value='successful_site_action',
169
+    is_flag=True,
170 170
     help='Holds the revision information for the most recent successfully '
171 171
     'executed site action.')
172 172
 @click.option(

+ 1
- 1
src/bin/shipyard_client/test-requirements.txt View File

@@ -1,7 +1,7 @@
1 1
 # Testing
2 2
 pytest==3.4
3 3
 pytest-cov==2.5.1
4
-responses==0.8.1
4
+responses==0.10.2
5 5
 testfixtures==5.1.1
6 6
 
7 7
 # Linting

+ 9
- 4
src/bin/shipyard_client/tests/unit/cli/commit/test_commit_actions.py View File

@@ -19,13 +19,18 @@ from shipyard_client.api_client.base_client import BaseClient
19 19
 from shipyard_client.cli.commit.actions import CommitConfigdocs
20 20
 from tests.unit.cli import stubs
21 21
 
22
+# TODO: refactor these tests to use responses callbacks (or other features)
23
+#     so that query parameter passing can be validated.
24
+#     moving to responses > 0.8 (e.g. 0.10.2) changed how URLS for responses
25
+#     seem to operate.
26
+
22 27
 
23 28
 @responses.activate
24 29
 @mock.patch.object(BaseClient, 'get_endpoint', lambda x: 'http://shiptest')
25 30
 @mock.patch.object(BaseClient, 'get_token', lambda x: 'abc')
26 31
 def test_commit_configdocs(*args):
27 32
     responses.add(responses.POST,
28
-                  'http://shiptest/commitconfigdocs?force=false',
33
+                  'http://shiptest/commitconfigdocs',
29 34
                   body=None,
30 35
                   status=200)
31 36
     response = CommitConfigdocs(stubs.StubCliContext(),
@@ -44,7 +49,7 @@ def test_commit_configdocs_409(*args):
44 49
                                   reason='Conflicts reason',
45 50
                                   code=409)
46 51
     responses.add(responses.POST,
47
-                  'http://shiptest/commitconfigdocs?force=false',
52
+                  'http://shiptest/commitconfigdocs',
48 53
                   body=api_resp,
49 54
                   status=409)
50 55
     response = CommitConfigdocs(stubs.StubCliContext(),
@@ -65,7 +70,7 @@ def test_commit_configdocs_forced(*args):
65 70
                                   reason='Conflicts reason',
66 71
                                   code=200)
67 72
     responses.add(responses.POST,
68
-                  'http://shiptest/commitconfigdocs?force=true',
73
+                  'http://shiptest/commitconfigdocs',
69 74
                   body=api_resp,
70 75
                   status=200)
71 76
     response = CommitConfigdocs(stubs.StubCliContext(),
@@ -80,7 +85,7 @@ def test_commit_configdocs_forced(*args):
80 85
 @mock.patch.object(BaseClient, 'get_token', lambda x: 'abc')
81 86
 def test_commit_configdocs_dryrun(*args):
82 87
     responses.add(responses.POST,
83
-                  'http://shiptest/commitconfigdocs?force=false',
88
+                  'http://shiptest/commitconfigdocs',
84 89
                   body=None,
85 90
                   status=200)
86 91
     response = CommitConfigdocs(stubs.StubCliContext(),

+ 45
- 0
src/bin/shipyard_client/tests/unit/cli/create/test_create_actions.py View File

@@ -116,6 +116,7 @@ def test_create_configdocs(*args):
116 116
     response = CreateConfigdocs(stubs.StubCliContext(),
117 117
                                 'design',
118 118
                                 'append',
119
+                                False,
119 120
                                 document_data,
120 121
                                 file_list).invoke_and_return_resp()
121 122
     assert 'Configuration documents added.'
@@ -145,6 +146,7 @@ def test_create_configdocs_201_with_val_fails(*args):
145 146
     response = CreateConfigdocs(stubs.StubCliContext(),
146 147
                                 'design',
147 148
                                 'append',
149
+                                False,
148 150
                                 document_data,
149 151
                                 file_list).invoke_and_return_resp()
150 152
     assert 'Configuration documents added.' in response
@@ -175,8 +177,51 @@ def test_create_configdocs_409(*args):
175 177
     response = CreateConfigdocs(stubs.StubCliContext(),
176 178
                                 'design',
177 179
                                 'append',
180
+                                False,
178 181
                                 document_data,
179 182
                                 file_list).invoke_and_return_resp()
180 183
     assert 'Error: Invalid collection' in response
181 184
     assert 'Reason: Buffermode : append' in response
182 185
     assert 'Buffer is either not...' in response
186
+
187
+
188
+@responses.activate
189
+@mock.patch.object(BaseClient, 'get_endpoint', lambda x: 'http://shiptest')
190
+@mock.patch.object(BaseClient, 'get_token', lambda x: 'abc')
191
+def test_create_configdocs_empty(*args):
192
+    def validating_callback(request):
193
+        # a request that has empty_collection should have no body.
194
+        assert request.body is None
195
+        resp_body = stubs.gen_err_resp(
196
+            message='Validations succeeded',
197
+            sub_error_count=0,
198
+            sub_info_count=0,
199
+            reason='Validation',
200
+            code=200)
201
+        return (201, {}, resp_body)
202
+
203
+    responses.add_callback(
204
+        responses.POST,
205
+        'http://shiptest/configdocs/design',
206
+        callback=validating_callback,
207
+        content_type='application/json')
208
+
209
+    filename = 'tests/unit/cli/create/sample_yaml/sample.yaml'
210
+    document_data = yaml.dump_all(filename)
211
+    file_list = (filename, )
212
+
213
+    # pass data and empty_collection = True - should init with data, but
214
+    # not send the data on invoke
215
+    action = CreateConfigdocs(
216
+        stubs.StubCliContext(),
217
+        collection='design',
218
+        buffer_mode='append',
219
+        empty_collection=True,
220
+        data=document_data,
221
+        filenames=file_list)
222
+
223
+    assert action.data == document_data
224
+    assert action.empty_collection == True
225
+
226
+    response = action.invoke_and_return_resp()
227
+    assert response.startswith("Configuration documents added.")

+ 103
- 6
src/bin/shipyard_client/tests/unit/cli/create/test_create_commands.py View File

@@ -66,8 +66,93 @@ def test_create_configdocs():
66 66
             auth_vars, 'create', 'configdocs', collection, '--' + append,
67 67
             '--filename=' + filename
68 68
         ])
69
-    mock_method.assert_called_once_with(ANY, collection, 'append',
70
-                                        ANY, file_list)
69
+    mock_method.assert_called_once_with(ctx=ANY, collection=collection,
70
+        buffer_mode='append', empty_collection=False, data=ANY,
71
+        filenames=file_list)
72
+
73
+
74
+def test_create_configdocs_empty():
75
+    """test create configdocs with the --empty-collection flag"""
76
+
77
+    collection = 'design'
78
+    filename = 'tests/unit/cli/create/sample_yaml/sample.yaml'
79
+    directory = 'tests/unit/cli/create/sample_yaml'
80
+    runner = CliRunner()
81
+    tests = [
82
+        {
83
+            # replace mode, no file, no data, empty collection
84
+            'kwargs': {
85
+                'buffer_mode': 'replace',
86
+                'empty_collection': True,
87
+                'filenames': [],
88
+                'data': ""
89
+            },
90
+            'args': [
91
+                '--replace',
92
+                '--empty-collection',
93
+            ],
94
+        },
95
+        {
96
+            # Append mode, no file, no data, empty collection
97
+            'kwargs': {
98
+                'buffer_mode': 'append',
99
+                'empty_collection': True,
100
+                'filenames': [],
101
+                'data': ""
102
+            },
103
+            'args': [
104
+                '--append',
105
+                '--empty-collection',
106
+            ],
107
+        },
108
+        {
109
+            # No buffer mode specified, empty collection
110
+            'kwargs': {
111
+                'buffer_mode': None,
112
+                'empty_collection': True,
113
+                'filenames': [],
114
+                'data': ""
115
+            },
116
+            'args': [
117
+                '--empty-collection',
118
+            ],
119
+        },
120
+        {
121
+            # Filename should be ignored and not passed, empty collection
122
+            'kwargs': {
123
+                'buffer_mode': None,
124
+                'empty_collection': True,
125
+                'filenames': [],
126
+                'data': ""
127
+            },
128
+            'args': [
129
+                '--empty-collection',
130
+                '--filename={}'.format(filename)
131
+            ],
132
+        },
133
+        {
134
+            # Directory should be ignored and not passed, empty collection
135
+            'kwargs': {
136
+                'buffer_mode': None,
137
+                'empty_collection': True,
138
+                'filenames': [],
139
+                'data': ""
140
+            },
141
+            'args': [
142
+                '--empty-collection',
143
+                '--directory={}'.format(directory)
144
+            ],
145
+        },
146
+    ]
147
+
148
+    for tc in tests:
149
+        with patch.object(CreateConfigdocs, '__init__') as mock_method:
150
+            runner.invoke(shipyard, [
151
+                auth_vars, 'create', 'configdocs', collection, *tc['args']
152
+            ])
153
+
154
+        mock_method.assert_called_once_with(ctx=ANY, collection=collection,
155
+            **tc['kwargs'])
71 156
 
72 157
 
73 158
 def test_create_configdocs_directory():
@@ -82,7 +167,11 @@ def test_create_configdocs_directory():
82 167
             auth_vars, 'create', 'configdocs', collection, '--' + append,
83 168
             '--directory=' + directory
84 169
         ])
85
-    mock_method.assert_called_once_with(ANY, collection, 'append', ANY, ANY)
170
+    # TODO(bryan-strassner) Make this test useful to show directory parsing
171
+    #     happened.
172
+    mock_method.assert_called_once_with(ctx=ANY, collection=collection,
173
+        buffer_mode='append', empty_collection=False, data=ANY,
174
+        filenames=ANY)
86 175
 
87 176
 
88 177
 def test_create_configdocs_directory_empty():
@@ -114,11 +203,15 @@ def test_create_configdocs_multi_directory():
114 203
             auth_vars, 'create', 'configdocs', collection, '--' + append,
115 204
             '--directory=' + dir1, '--directory=' + dir2
116 205
         ])
117
-    mock_method.assert_called_once_with(ANY, collection, 'append', ANY, ANY)
206
+    # TODO(bryan-strassner) Make this test useful to show multiple directories
207
+    #     were actually traversed.
208
+    mock_method.assert_called_once_with(ctx=ANY, collection=collection,
209
+        buffer_mode='append', empty_collection=False, data=ANY,
210
+        filenames=ANY)
118 211
 
119 212
 
120 213
 def test_create_configdocs_multi_directory_recurse():
121
-    """test create configdocs with multiple directories"""
214
+    """test create configdocs with multiple directories recursively"""
122 215
 
123 216
     collection = 'design'
124 217
     dir1 = 'tests/unit/cli/create/sample_yaml/'
@@ -130,7 +223,11 @@ def test_create_configdocs_multi_directory_recurse():
130 223
             auth_vars, 'create', 'configdocs', collection, '--' + append,
131 224
             '--directory=' + dir1, '--directory=' + dir2, '--recurse'
132 225
         ])
133
-    mock_method.assert_called_once_with(ANY, collection, 'append', ANY, ANY)
226
+    # TODO(bryan-strassner) Make this test useful to show multiple directories
227
+    #     were actually traversed and recursed.
228
+    mock_method.assert_called_once_with(ctx=ANY, collection=collection,
229
+        buffer_mode='append', empty_collection=False, data=ANY,
230
+        filenames=ANY)
134 231
 
135 232
 
136 233
 def test_create_configdocs_negative():

Loading…
Cancel
Save