Extended whitelist with task name

Change-Id: Id832b0c66256b8808ac17c657c7077b6d2164803

README.md

@@ -151,7 +151,7 @@ This command will run audit, check the changes and will enforce configuration, i
 Since fuel-library contains non-idempotent tasks, that contain Puppet resources, which will be
 triggered on each deployment run, this extension provides the operator the ability to filter such changes out.
 
-A whitelist rule is a string, that is included into a Puppet report line for the whitelisted resource change, e.g. for
+A whitelist rule is a pair of strings. The first one is a fuel task name to match. The second one is what should be included into a Puppet report line for the whitelisted resource change, e.g. for
 ```
 Openstack_tasks::Swift::Proxy_storage/Package[mc]/ensure
 ```
@@ -159,14 +159,25 @@ the whitelist rule could be
 ```
 Package[mc]/ensure
 ```
+A rule with empty fuel_task filter will match to all tasks.
+
 Whitelist rules for an environment can be listed by
 ```
 fuel2 audit whitelist show <env-id>
 ```
 These rules can be managed by following commands:
 ```
-fuel2 audit whitelist add <env-id> <rule>
+fuel2 audit whitelist add <env-id> --task <fuel-task> --rule <rule>
 fuel2 audit whitelist delete <rule-id>
+fuel2 audit whitelist load fromfile <env-id> <path-to-yaml>
+```
+
+Example YAML file with whitelist rules:
+```
+- fuel_task: netconfig
+  rule: L23_stored_configs
+- fuel_task: top-role-compute
+  rule: Service[nova-compute]/ensure
 ```
 
 ### REST API

doc/api.md

@@ -54,11 +54,12 @@ Input data schema:
 "type": "object",
 "properties": {
     "rule": {"type": "string"},
+    "fuel_task": {"type": "string"},
 }
 ```
 Example
 ```
-curl -H "X-Auth-Token: $(fuel token)" -X PUT http://localhost:8000/api/v1/clusters/changes-whitelist/1 -d '{"rule": "new-rule-string"}'
+curl -H "X-Auth-Token: $(fuel token)" -X PUT http://localhost:8000/api/v1/clusters/changes-whitelist/1 -d '{"rule": "new-rule-string", "fuel_task": "fuel-task-id"}'
 ```
 
 #### DELETE /clusters/changes-whitelist/(obj_id)
@@ -83,10 +84,11 @@ Input data schema:
 "description": "Serialized ChangesWhitelistRule collection",
 "type": "object",
 "items": {
-    "rule": {"type": "string"}
+    "rule": {"type": "string"},
+    "fuel_task": {"fuel_task": "string"},
 }
 ```
 Example
 ```
-curl -H "X-Auth-Token: $(fuel token)" -X POST http://localhost:8000/api/v1/clusters/1/changes-whitelist/ -d '[{"rule": "new-rule-string"}, {"rule": "new-rule-2"}]'
+curl -H "X-Auth-Token: $(fuel token)" -X POST http://localhost:8000/api/v1/clusters/1/changes-whitelist/ -d '[{"rule": "new-rule-string", "fuel_task": "task1"}, {"rule": "new-rule-2", "fuel_task": ""}]'
 ```

doc/cli.md

@@ -77,10 +77,15 @@ fuel2 audit whitelist show <env-id>
 
 To add a rule:
 ```
-fuel2 audit whitelist add <env-id> <rule>
+fuel2 audit whitelist add <env-id> --task <fuel-task> --rule <rule>
 ```
 
 To delete a rule:
 ```
 fuel2 audit whitelist delete <rule-id>
 ```
+
+To add rules from YAML file:
+```
+fuel2 audit whitelist load fromfile <env-id> <path-to-yaml>
+```

fuel_external_git/fuelclient_audit.py

@@ -17,6 +17,7 @@ import time
 
 from cliff import command
 from cliff import lister
+import yaml
 
 from fuelclient import client
 if hasattr(client, 'DefaultAPIClient'):
@@ -76,14 +77,27 @@ class Audit(lister.Lister, command.Command):
         for task in changed_tasks:
             name = task['task_name']
             for item in task['summary']['raw_report']:
-                if 'Would have triggered' not in item['message'] and \
+                if item['source'].startswith('/Stage[main]/'):
-                    'Finished catalog run' not in item['message']:
                     short_item = item['source'].replace('/Stage[main]/', '')
                     changes.append({'task_id': name,
                                     'resource': short_item,
                                     'node_id': task['node_id']})
         return changes
 
+    @staticmethod
+    def filter_changes(changes, env_id):
+        wl = fc_client.get_request(
+            '/clusters/{env}/changes-whitelist/'.format(env=env_id)
+        )
+
+        changes = filter(lambda c:
+                         len(filter(lambda w: w['rule'] in c['resource'] and
+                                    (w['fuel_task'] == c['task_id'] or
+                                     w['fuel_task'] == ''), wl) == 0),
+                         changes)
+
+        return changes
+
     def get_parser(self, prog_name):
         parser = super(Audit, self).get_parser(prog_name)
         group = parser.add_mutually_exclusive_group(required=True)
@@ -209,7 +223,10 @@ class OutOfSyncResources(lister.Lister, command.Command):
         else:
             fuel_task = Task(task_id)
 
+        env_id = fuel_task.data['cluster']
+
         changes = Audit.get_outofsync(fuel_task)
+        changes = Audit.filter_changes(changes, env_id)
 
         data = data_utils.get_display_data_multi(self.columns, changes)
         return (self.columns, data)
@@ -218,6 +235,7 @@ class OutOfSyncResources(lister.Lister, command.Command):
 class WhitelistRulesShow(lister.Lister, command.Command):
     columns = (
         'id',
+        'fuel_task',
         'rule'
     )
 
@@ -243,6 +261,7 @@ class WhitelistRulesShow(lister.Lister, command.Command):
 class WhitelistRuleAdd(lister.Lister, command.Command):
     columns = (
         'id',
+        'fuel_task',
         'rule'
     )
 
@@ -251,15 +270,24 @@ class WhitelistRuleAdd(lister.Lister, command.Command):
         parser.add_argument('env',
                             type=int,
                             help='Environment to add whitelist rules to')
-        parser.add_argument('rule',
+        parser.add_argument('--rule', '-r',
                             type=str,
+                            required=True,
                             help='Rule to add')
+        parser.add_argument('--task', '-t',
+                            type=str,
+                            required=False,
+                            help=('Fuel task for the rule. Omitting this will'
+                                  ' result in matching all Fuel tasks.'))
         return parser
 
     def take_action(self, parsed_args):
         env_id = parsed_args.env
         rule = parsed_args.rule
+        task = parsed_args.task
         data = {'rule': rule}
+        if task:
+            data['fuel_task'] = task
 
         ret = fc_client.post_request(
             '/clusters/{env}/changes-whitelist/'.format(env=env_id),
@@ -288,3 +316,36 @@ class WhitelistRuleDelete(command.Command):
         )
 
         return ((), {})
+
+
+class WhitelistRuleAddFromFile(lister.Lister, command.Command):
+    columns = (
+        'id',
+        'fuel_task',
+        'rule'
+    )
+
+    def get_parser(self, prog_name):
+        parser = super(WhitelistRuleAddFromFile, self).get_parser(prog_name)
+        parser.add_argument('env',
+                            type=int,
+                            help='Environment to add whitelist rules to')
+        parser.add_argument('file_name',
+                            type=str,
+                            help='YAML file to load rules from')
+        return parser
+
+    def take_action(self, parsed_args):
+        env_id = parsed_args.env
+        file_name = parsed_args.file_name
+
+        with open(file_name, 'r') as f:
+            data = yaml.load(f)
+
+        ret = fc_client.post_request(
+            '/clusters/{env}/changes-whitelist/'.format(env=env_id),
+            data
+        )
+        ret = data_utils.get_display_data_multi(self.columns, ret)
+
+        return (self.columns, ret)

fuel_external_git/json_schema.py

@@ -42,6 +42,7 @@ changeswhitelistrule_single_schema = {
         "id": {"type": "number"},
         "env_id": {"type": "number"},
         "rule": {"type": "string"},
+        "fuel_task": {"type": "string"},
     }
 }
 

fuel_external_git/migrations/versions/fc4f164a7b6c_extended_whitelist_with_task_name.py

@@ -0,0 +1,42 @@
+# Licensed under the Apache License, Version 2.0 (the "License"); you may
+# not use this file except in compliance with the License. You may obtain
+# a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+# License for the specific language governing permissions and limitations
+# under the License.
+
+"""extended whitelist with task name
+
+Revision ID: fc4f164a7b6c
+Revises: 8736ad38ca31
+Create Date: 2016-11-15 09:09:46.300987
+
+"""
+
+# revision identifiers, used by Alembic.
+revision = 'fc4f164a7b6c'
+down_revision = '8736ad38ca31'
+branch_labels = None
+depends_on = None
+
+from alembic import context
+from alembic import op
+import sqlalchemy as sa
+
+
+def upgrade():
+    table_prefix = context.config.get_main_option('table_prefix')
+    op.add_column(
+        table_prefix + 'changes_whitelist',
+        sa.Column('fuel_task', sa.String, server_default='', nullable=False)
+    )
+
+
+def downgrade():
+    table_prefix = context.config.get_main_option('table_prefix')
+    op.drop_column(table_prefix + 'changes_whitelist', 'fuel_task')

fuel_external_git/models.py

@@ -37,3 +37,4 @@ class ChangesWhitelistRule(Base):
     id = Column(Integer, primary_key=True)
     env_id = Column(Integer, nullable=False)
     rule = Column(String(255), server_default='', nullable=False)
+    fuel_task = Column(String(255), server_default='', nullable=False)

fuel_external_git/objects.py

@@ -50,7 +50,8 @@ class ChangesWhitelistRuleSerializer(BasicSerializer):
     fields = (
         "id",
         "env_id",
-        "rule"
+        "rule",
+        "fuel_task"
     )
 
 

setup.cfg

@@ -35,4 +35,5 @@ fuelclient:
     audit_list_outofsync = fuel_external_git.fuelclient_audit:OutOfSyncResources
     audit_whitelist_show = fuel_external_git.fuelclient_audit:WhitelistRulesShow
     audit_whitelist_add = fuel_external_git.fuelclient_audit:WhitelistRuleAdd
+    audit_whitelist_load_fromfile = fuel_external_git.fuelclient_audit:WhitelistRuleAddFromFile
     audit_whitelist_delete = fuel_external_git.fuelclient_audit:WhitelistRuleDelete