This patch is used to delete batch of resources

1. Add a method delete in gnocchi/rest/__init__.py to
   accept an attribute filter to delete matched resources.
2. Add method delete_resources in gnocchi/indexer/sqlalchemy.py
   to delete related data in db.
3. Add a new rule in policy.
4. Add document to descrbe the functions

The HTTP request for deleting a batch of resources by ids
is looks like:

    DELETE /v1/resource/<resource_type>
        Content-Type: application/json

                {"in": {"id":[xx_id,yy_id...]}}.

The HTTP request for deleting a batch of resources filter
by resources started_data is looks like:

    DELETE /v1/resource/<resource_type>
        Content-Type: application/json

                {">=": {"started_data": "2016-08-24"}}

Or even more complicated for deleing a batch of resources:

    DELETE /v1/resource/<resource_type>
        Content-Type: application/json
                {
                  "and": [
                     {">=": {"started_data":"2016-08-06"}},
                     {"=":{"id":"xxxx_id"}}
                        ]
                }

TODO:
    An corresponding gnocchi client CLI needs to
    be added later.

Partial-Bug: #1585262

Co-Authored-By: Mehdi Abaakouk <sileht@redhat.com>

Change-Id: I2a21c9e76fe08819b60e1a198335213c3b32e96f

doc/source/rest.j2

@@ -292,14 +292,29 @@ And to retrieve its modification history:
 
 {{ scenarios['get-patched-instance-history']['doc'] }}
 
-It possible to delete a resource altogether:
+It is possible to delete a resource altogether:
 
 {{ scenarios['delete-resource-generic']['doc'] }}
 
+It is also possible to delete a batch of resources based on attribute values, and
+returns a number of deleted resources.
+
+To delete resources based on ids:
+
+{{ scenarios['delete-resources-by-ids']['doc'] }}
+
+or delete resources based on time:
+
+{{ scenarios['delete-resources-by-time']['doc']}}
+
 .. IMPORTANT::
 
-   When a resource is deleted, all its associated metrics are deleted at the
+  When a resource is deleted, all its associated metrics are deleted at the
-   same time.
+  same time.
+
+  When a batch of resources are deleted, an attribute filter is required to
+  avoid deletion of the entire database.
+
 
 All resources can be listed, either by using the `generic` type that will list
 all types of resources, or by filtering on their resource type:

doc/source/rest.yaml

@@ -501,6 +501,97 @@
 - name: delete-resource-generic
   request: DELETE /v1/resource/generic/{{ scenarios['create-resource-generic']['response'].json['id'] }} HTTP/1.1
 
+- name: create-resources-a
+  request: |
+    POST /v1/resource/generic HTTP/1.1
+    Content-Type: application/json
+
+    {
+      "id": "340102AA-AA19-BBE0-E1E2-2D3JDC7D289R",
+      "user_id": "BD3A1E52-KKKC-2123-BGLH-WWUUD88CD7WZ",
+      "project_id": "BD3A1E52-KKKC-2123-BGLH-WWUUD88CD7WZ"
+    }
+
+- name: create-resources-b
+  request: |
+    POST /v1/resource/generic HTTP/1.1
+    Content-Type: application/json
+
+    {
+      "id": "340102AA-AAEF-AA90-E1E2-2D3JDC7D289R",
+      "user_id": "BD3A1E52-KKKC-2123-BGLH-WWUUD88CD7WZ",
+      "project_id": "BD3A1E52-KKKC-2123-BGLH-WWUUD88CD7WZ"
+    }
+
+- name: create-resources-c
+  request: |
+    POST /v1/resource/generic HTTP/1.1
+    Content-Type: application/json
+
+    {
+      "id": "340102AA-AAEF-BCEF-E112-2D3JDC7D289R",
+      "user_id": "BD3A1E52-KKKC-2123-BGLH-WWUUD88CD7WZ",
+      "project_id": "BD3A1E52-KKKC-2123-BGLH-WWUUD88CD7WZ"
+    }
+
+- name: create-resources-d
+  request: |
+    POST /v1/resource/generic HTTP/1.1
+    Content-Type: application/json
+
+    {
+      "id": "340102AA-AAEF-BCEF-E112-2D15DC7D289R",
+      "user_id": "BD3A1E52-KKKC-2123-BGLH-WWUUD88CD7WZ",
+      "project_id": "BD3A1E52-KKKC-2123-BGLH-WWUUD88CD7WZ"
+    }
+
+- name: create-resources-e
+  request: |
+    POST /v1/resource/generic HTTP/1.1
+    Content-Type: application/json
+
+    {
+      "id": "340102AA-AAEF-BCEF-E112-2D3JDC30289R",
+      "user_id": "BD3A1E52-KKKC-2123-BGLH-WWUUD88CD7WZ",
+      "project_id": "BD3A1E52-KKKC-2123-BGLH-WWUUD88CD7WZ"
+    }
+
+- name: create-resources-f
+  request: |
+    POST /v1/resource/generic HTTP/1.1
+    Content-Type: application/json
+
+    {
+      "id": "340102AA-AAEF-BCEF-E112-2D15349D109R",
+      "user_id": "BD3A1E52-KKKC-2123-BGLH-WWUUD88CD7WZ",
+      "project_id": "BD3A1E52-KKKC-2123-BGLH-WWUUD88CD7WZ"
+    }
+
+- name: delete-resources-by-ids
+  request: |
+    DELETE /v1/resource/generic HTTP/1.1
+    Content-Type: application/json
+
+    {
+      "in": {
+        "id": [
+          "{{ scenarios['create-resources-a']['response'].json['id'] }}",
+          "{{ scenarios['create-resources-b']['response'].json['id'] }}",
+          "{{ scenarios['create-resources-c']['response'].json['id'] }}"
+        ]
+      }
+    }
+
+- name: delete-resources-by-time
+  request: |
+    DELETE /v1/resource/generic HTTP/1.1
+    Content-Type: application/json
+
+    {
+      ">=": {"started_at": "{{ scenarios['create-resources-f']['response'].json['started_at'] }}"}
+    }
+
+
 - name: get-resource-named-metrics-measures
   request: GET /v1/resource/generic/{{ scenarios['create-resource-instance-with-metrics']['response'].json['id'] }}/metric/cpu.util/measures?start=2014-10-06T14:34 HTTP/1.1
 

etc/gnocchi/policy.json

@@ -9,6 +9,7 @@
     "get resource": "rule:admin_or_creator or rule:resource_owner",
     "update resource": "rule:admin_or_creator",
     "delete resource": "rule:admin_or_creator",
+    "delete resources": "rule:admin_or_creator",
     "list resource": "rule:admin_or_creator or rule:resource_owner",
     "search resource": "rule:admin_or_creator or rule:resource_owner",
 

gnocchi/indexer/__init__.py

@@ -362,6 +362,11 @@ class IndexerDriver(object):
     def delete_resource(uuid):
         raise exceptions.NotImplementedError
 
+    @staticmethod
+    def delete_resources(resource_type='generic',
+                         attribute_filter=None):
+        raise exceptions.NotImplementedError
+
     @staticmethod
     def delete_metric(id):
         raise exceptions.NotImplementedError

gnocchi/indexer/sqlalchemy.py

@@ -866,6 +866,37 @@ class SQLAlchemyIndexer(indexer.IndexerDriver):
                     Resource.id == resource_id).delete() == 0:
                 raise indexer.NoSuchResource(resource_id)
 
+    @retry_on_deadlock
+    def delete_resources(self, resource_type='generic',
+                         attribute_filter=None):
+        if not attribute_filter:
+            raise ValueError("attribute_filter must be set")
+
+        with self.facade.writer() as session:
+            target_cls = self._resource_type_to_mappers(
+                session, resource_type)["resource"]
+
+            q = session.query(target_cls.id)
+
+            engine = session.connection()
+            try:
+                f = QueryTransformer.build_filter(engine.dialect.name,
+                                                  target_cls,
+                                                  attribute_filter)
+            except indexer.QueryAttributeError as e:
+                # NOTE(jd) The QueryAttributeError does not know about
+                # resource_type, so convert it
+                raise indexer.ResourceAttributeError(resource_type,
+                                                     e.attribute)
+
+            q = q.filter(f)
+
+            session.query(Metric).filter(
+                Metric.resource_id.in_(q)
+            ).update({"status": "delete"},
+                     synchronize_session=False)
+            return q.delete(synchronize_session=False)
+
     @retry_on_deadlock
     def get_resource(self, resource_type, resource_id, with_metrics=False):
         with self.facade.independent_reader() as session:

gnocchi/rest/__init__.py

@@ -1059,6 +1059,34 @@ class ResourcesController(rest.RestController):
         except indexer.IndexerException as e:
             abort(400, e)
 
+    @pecan.expose('json')
+    def delete(self, **kwargs):
+        # NOTE(sileht): Don't allow empty filter, this is going to delete
+        # the entire database.
+        attr_filter = deserialize_and_validate(
+            SearchResourceTypeController.ResourceSearchSchema)
+
+        # the voluptuous checks everything, but it is better to
+        # have this here.
+        if not attr_filter:
+            abort(400, "caution: the query can not be empty, or it will \
+                  delete entire database")
+
+        user, project = get_user_and_project()
+        policy_filter = _get_list_resource_policy_filter(
+            "delete resources", self._resource_type, user, project)
+
+        if policy_filter:
+            attr_filter = {"and": [policy_filter, attr_filter]}
+
+        try:
+            delete_num = pecan.request.indexer.delete_resources(
+                self._resource_type, attribute_filter=attr_filter)
+        except indexer.IndexerException as e:
+            abort(400, e)
+
+        return {"deleted": delete_num}
+
 
 class ResourcesByTypeController(rest.RestController):
     @pecan.expose('json')
@@ -1104,7 +1132,9 @@ class SearchResourceTypeController(rest.RestController):
                     u"and", u"∨",
                     u"or", u"∧",
                     u"not",
-                ): [_ResourceSearchSchema],
+                ): voluptuous.All(
+                    [_ResourceSearchSchema], voluptuous.Length(min=1)
+                )
             }
         )
     )

gnocchi/tests/gabbi/gabbits/resource.yaml

@@ -826,3 +826,308 @@ tests:
           x-user-id: 0fbb231484614b1a80131fc22f6afc9c
           x-project-id: f3d41b770cc14f0bb94a1d5be9c0e3ea
       status: 404
+
+# Delete a batch of resources by attributes filter
+
+    - name: create resource one
+      desc: before test batch delete, create some resources
+      POST: /v1/resource/generic
+      request_headers:
+          x-user-id: 0fbb231484614b1a80131fc22f6afc9c
+          x-project-id: f3d41b770cc14f0bb94a1d5be9c0e3ea
+          content-type: application/json
+      data:
+          id: f93450f2-aaaa-4d67-9985-02511241e7d1
+          started_at: "2014-01-03T02:02:02.000000"
+          user_id: 0fbb231484614b1a80131fc22f6afc9c
+          project_id: f3d41b770cc14f0bb94a1d5be9c0e3ea
+      status: 201
+
+    - name: create resource two
+      desc: before test batch delete, create some resources
+      POST: $LAST_URL
+      request_headers:
+          x-user-id: 0fbb231484614b1a80131fc22f6afc9c
+          x-project-id: f3d41b770cc14f0bb94a1d5be9c0e3ea
+          content-type: application/json
+      data:
+          id: f93450f2-bbbb-4d67-9985-02511241e7d1
+          started_at: "2014-01-03T02:02:02.000000"
+          user_id: 0fbb231484614b1a80131fc22f6afc9c
+          project_id: f3d41b770cc14f0bb94a1d5be9c0e3ea
+      status: 201
+
+    - name: create resource three
+      desc: before test batch delete, create some resources
+      POST: $LAST_URL
+      request_headers:
+          x-user-id: 0fbb231484614b1a80131fc22f6afc9c
+          x-project-id: f3d41b770cc14f0bb94a1d5be9c0e3ea
+          content-type: application/json
+      data:
+          id: f93450f2-cccc-4d67-9985-02511241e7d1
+          started_at: "2014-08-04T00:00:00.000000"
+          user_id: 0fbb231484614b1a80131fc22f6afc9c
+          project_id: f3d41b770cc14f0bb94a1d5be9c0e3ea
+      status: 201
+
+    - name: create resource four
+      desc: before test batch delete, create some resources
+      POST: $LAST_URL
+      request_headers:
+          x-user-id: 0fbb231484614b1a80131fc22f6afc9c
+          x-project-id: f3d41b770cc14f0bb94a1d5be9c0e3ea
+          content-type: application/json
+      data:
+          id: f93450f2-dddd-4d67-9985-02511241e7d1
+          started_at: "2014-08-04T00:00:00.000000"
+          user_id: 0fbb231484614b1a80131fc22f6afc9c
+          project_id: f3d41b770cc14f0bb94a1d5be9c0e3ea
+      status: 201
+
+    - name: create resource five
+      desc: before test batch delete, create some resources
+      POST: $LAST_URL
+      request_headers:
+          x-user-id: 0fbb231484614b1a80131fc22f6afc9c
+          x-project-id: f3d41b770cc14f0bb94a1d5be9c0e3ea
+          content-type: application/json
+      data:
+          id: f93450f2-eeee-4d67-9985-02511241e7d1
+          started_at: "2015-08-14T00:00:00.000000"
+          user_id: 0fbb231484614b1a80131fc22f6afc9c
+          project_id: f3d41b770cc14f0bb94a1d5be9c0e3ea
+      status: 201
+
+    - name: create resource six
+      desc: before test batch delete, create some resources
+      POST: $LAST_URL
+      request_headers:
+          x-user-id: 0fbb231484614b1a80131fc22f6afc9c
+          x-project-id: f3d41b770cc14f0bb94a1d5be9c0e3ea
+          content-type: application/json
+      data:
+          id: f93450f2-ffff-4d67-9985-02511241e7d1
+          started_at: "2015-08-14T00:00:00.000000"
+          user_id: 0fbb231484614b1a80131fc22f6afc9c
+          project_id: f3d41b770cc14f0bb94a1d5be9c0e3ea
+      status: 201
+
+    - name: get resource one
+      desc: ensure the resources exists
+      GET: /v1/resource/generic/f93450f2-aaaa-4d67-9985-02511241e7d1
+      request_headers:
+          x-user-id: 0fbb231484614b1a80131fc22f6afc9c
+          x-project-id: f3d41b770cc14f0bb94a1d5be9c0e3ea
+          content-type: application/json
+      status: 200
+
+    - name: get resource two
+      desc: ensure the resources exists
+      GET: /v1/resource/generic/f93450f2-bbbb-4d67-9985-02511241e7d1
+      request_headers:
+          x-user-id: 0fbb231484614b1a80131fc22f6afc9c
+          x-project-id: f3d41b770cc14f0bb94a1d5be9c0e3ea
+          content-type: application/json
+      status: 200
+
+    - name: get resource three
+      desc: ensure the resources exists
+      GET: /v1/resource/generic/f93450f2-cccc-4d67-9985-02511241e7d1
+      request_headers:
+          x-user-id: 0fbb231484614b1a80131fc22f6afc9c
+          x-project-id: f3d41b770cc14f0bb94a1d5be9c0e3ea
+          content-type: application/json
+      status: 200
+
+    - name: get resource four
+      desc: ensure the resources exists
+      GET: /v1/resource/generic/f93450f2-dddd-4d67-9985-02511241e7d1
+      request_headers:
+          x-user-id: 0fbb231484614b1a80131fc22f6afc9c
+          x-project-id: f3d41b770cc14f0bb94a1d5be9c0e3ea
+          content-type: application/json
+      status: 200
+
+    - name: get resource five
+      desc: ensure the resources exists
+      GET: /v1/resource/generic/f93450f2-eeee-4d67-9985-02511241e7d1
+      request_headers:
+          x-user-id: 0fbb231484614b1a80131fc22f6afc9c
+          x-project-id: f3d41b770cc14f0bb94a1d5be9c0e3ea
+          content-type: application/json
+      status: 200
+
+    - name: get resource six
+      desc: ensure the resources exists
+      GET: /v1/resource/generic/f93450f2-ffff-4d67-9985-02511241e7d1
+      request_headers:
+          x-user-id: 0fbb231484614b1a80131fc22f6afc9c
+          x-project-id: f3d41b770cc14f0bb94a1d5be9c0e3ea
+          content-type: application/json
+      status: 200
+
+    - name: delete random data structure
+      desc: delete a empty list test
+      DELETE: /v1/resource/generic
+      request_headers:
+          x-user-id: 0fbb231484614b1a80131fc22f6afc9c
+          x-project-id: f3d41b770cc14f0bb94a1d5be9c0e3ea
+          content-type: application/json
+      data:
+          resource_ids:
+              []
+          attrs:
+              test
+      status: 400
+
+    - name: delete something empty
+      desc: use  empty filter for delete
+      DELETE: $LAST_URL
+      request_headers:
+          x-user-id: 0fbb231484614b1a80131fc22f6afc9c
+          x-project-id: f3d41b770cc14f0bb94a1d5be9c0e3ea
+          content-type: application/json
+      data: ""
+      status: 400
+
+    - name: delete something empty a
+      desc: use  empty filter for delete
+      DELETE: $LAST_URL
+      request_headers:
+          x-user-id: 0fbb231484614b1a80131fc22f6afc9c
+          x-project-id: f3d41b770cc14f0bb94a1d5be9c0e3ea
+          content-type: application/json
+      data:
+          in:
+             id: []
+      status: 200
+      response_json_paths:
+        $.deleted: 0
+
+    - name: delete something empty b
+      desc: use  empty filter for delete
+      DELETE: $LAST_URL
+      request_headers:
+          x-user-id: 0fbb231484614b1a80131fc22f6afc9c
+          x-project-id: f3d41b770cc14f0bb94a1d5be9c0e3ea
+          content-type: application/json
+      data:
+          in: {}
+      status: 400
+
+    - name: delete something empty c
+      desc: use  empty filter for delete
+      DELETE: $LAST_URL
+      request_headers:
+          x-user-id: 0fbb231484614b1a80131fc22f6afc9c
+          x-project-id: f3d41b770cc14f0bb94a1d5be9c0e3ea
+          content-type: application/json
+      data:
+          in:
+              and: []
+      status: 400
+
+    - name: delete something empty d
+      desc: use  empty filter for delete
+      DELETE: $LAST_URL
+      request_headers:
+          x-user-id: 0fbb231484614b1a80131fc22f6afc9c
+          x-project-id: f3d41b770cc14f0bb94a1d5be9c0e3ea
+          content-type: application/json
+      data:
+          in:
+              and:
+                  - or: []
+                  - id:
+                      =: ""
+      status: 400
+
+    - name: delete something empty e
+      desc: use  empty filter for delete
+      DELETE: $LAST_URL
+      request_headers:
+          x-user-id: 0fbb231484614b1a80131fc22f6afc9c
+          x-project-id: f3d41b770cc14f0bb94a1d5be9c0e3ea
+          content-type: application/json
+      data:
+          and: []
+      status: 400
+
+    - name: delete something empty f
+      desc: use  empty filter for delete
+      DELETE: $LAST_URL
+      request_headers:
+          x-user-id: 0fbb231484614b1a80131fc22f6afc9c
+          x-project-id: f3d41b770cc14f0bb94a1d5be9c0e3ea
+          content-type: application/json
+      data:
+          and:
+              - in:
+                  id: []
+              - started_at: ""
+      status: 400
+
+    - name: delete batch of resources filter by started_at
+      desc: delete the created resources
+      DELETE: /v1/resource/generic
+      request_headers:
+          x-user-id: 0fbb231484614b1a80131fc22f6afc9c
+          x-project-id: f3d41b770cc14f0bb94a1d5be9c0e3ea
+          content-type: application/json
+      data:
+          eq:
+            started_at: "2014-08-04"
+      status: 200
+      response_json_paths:
+        $.deleted: 2
+
+    - name: delete batch of resources filter by mutliple ids
+      desc: delete the created resources
+      DELETE: /v1/resource/generic
+      request_headers:
+          x-user-id: 0fbb231484614b1a80131fc22f6afc9c
+          x-project-id: f3d41b770cc14f0bb94a1d5be9c0e3ea
+          content-type: application/json
+      data:
+          in:
+            id:
+              - f93450f2-aaaa-4d67-9985-02511241e7d1
+              - f93450f2-bbbb-4d67-9985-02511241e7d1
+      status: 200
+      response_json_paths:
+        $.deleted: 2
+
+    - name: delete both existent and non-existent data
+      desc: delete exits and non-exist data
+      DELETE: $LAST_URL
+      request_headers:
+          x-user-id: 0fbb231484614b1a80131fc22f6afc9c
+          x-project-id: f3d41b770cc14f0bb94a1d5be9c0e3ea
+          content-type: application/json
+      data:
+          in:
+            id:
+              - f93450f2-eeee-4d67-9985-02511241e7d1
+              - f93450f2-ffff-4d67-9985-02511241e7d1
+              - f93450f2-yyyy-4d67-9985-02511241e7d1
+              - f93450f2-xxxx-4d67-9985-02511241e7d1
+      status: 200
+      response_json_paths:
+        $.deleted: 2
+
+    - name: delete multiple non-existent resources
+      desc: delete a batch of non-existent resources
+      DELETE: $LAST_URL
+      request_headers:
+          x-user-id: 0fbb231484614b1a80131fc22f6afc9c
+          x-project-id: f3d41b770cc14f0bb94a1d5be9c0e3ea
+          content-type: application/json
+      data:
+          in:
+            id:
+              - f93450f2-zzzz-4d67-9985-02511241e7d1
+              - f93450f2-kkkk-4d67-9985-02511241e7d1
+      status: 200
+      response_json_paths:
+        $.deleted: 0

gnocchi/tests/test_indexer.py

@@ -950,6 +950,40 @@ class TestIndexerDriver(tests_base.TestCase):
             })
         self.assertEqual(0, len(resources))
 
+    def test_deletes_resources(self):
+        r1 = uuid.uuid4()
+        r2 = uuid.uuid4()
+        user = str(uuid.uuid4())
+        project = str(uuid.uuid4())
+        metrics = {'foo': {'archive_policy_name': 'medium'}}
+        g1 = self.index.create_resource('generic', r1, user, project,
+                                        user, project, metrics=metrics)
+        g2 = self.index.create_resource('generic', r2, user, project,
+                                        user, project, metrics=metrics)
+
+        metrics = self.index.list_metrics(ids=[g1['metrics'][0]['id'],
+                                               g2['metrics'][0]['id']])
+        self.assertEqual(2, len(metrics))
+        for m in metrics:
+            self.assertEqual('active', m['status'])
+
+        deleted = self.index.delete_resources(
+            'generic',
+            attribute_filter={"=": {"user_id": user}})
+        self.assertEqual(2, deleted)
+
+        resources = self.index.list_resources(
+            'generic',
+            attribute_filter={"=": {"user_id": user}})
+        self.assertEqual(0, len(resources))
+
+        metrics = self.index.list_metrics(ids=[g1['metrics'][0]['id'],
+                                               g2['metrics'][0]['id']],
+                                          status='delete')
+        self.assertEqual(2, len(metrics))
+        for m in metrics:
+            self.assertEqual('delete', m['status'])
+
     def test_get_metric(self):
         e1 = uuid.uuid4()
         user = str(uuid.uuid4())

releasenotes/notes/delete-resources-f10d21fc02f53f16.yaml

@@ -0,0 +1,3 @@
+---
+feature:
+  - A new REST API call is provided to delete multiple resources at once using a search filter.