Merge "Add regression test for bug 1670627"

nova/tests/functional/api/client.py

@@ -393,3 +393,6 @@ class TestOpenStackClient(object):
 
     def delete_aggregate(self, aggregate_id):
         self.api_delete('/os-aggregates/%s' % aggregate_id)
+
+    def get_limits(self):
+        return self.api_get('/limits').body['limits']

nova/tests/functional/regressions/test_bug_1670627.py

@@ -0,0 +1,143 @@
+# Copyright 2017 IBM Corp.
+#
+# Licensed under the Apache License, Version 2.0 (the "License"); you may
+# not use this file except in compliance with the License. You may obtain
+# a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+# License for the specific language governing permissions and limitations
+# under the License.
+
+import time
+
+from nova import test
+from nova.tests import fixtures as nova_fixtures
+from nova.tests.functional.api import client
+from nova.tests.unit import cast_as_call
+import nova.tests.unit.image.fake
+from nova.tests.unit import policy_fixture
+
+
+class TestDeleteFromCell0CheckQuota(test.TestCase):
+    """This tests a regression introduced in the Ocata release.
+
+    In Ocata we started building instances in conductor for cells v2. If we
+    can't schedule the instance, it gets put into ERROR state, the instance
+    record is created in the cell0 database and the BuildRequest is deleted.
+
+    In the API:
+
+    1) quota.reserve creates a reservation record and updates the resource
+       usage record to increment 'reserved' which is counted as part of
+       usage.
+    2) quota.commit deletes the reservation record and updates the resource
+       usage record to decrement 'reserved' and increment 'in_use' which is
+       counted as part of usage
+
+    When the user deletes the instance, the API code sees that the instance is
+    living in cell0 and deletes it from cell0. The original quota reservation
+    was made in the cell (nova) database and usage was not decremented.
+
+    So a user that has several failed instance builds eventually runs out of
+    quota even though they successfully deleted their servers.
+    """
+
+    def setUp(self):
+        super(TestDeleteFromCell0CheckQuota, self).setUp()
+        self.useFixture(policy_fixture.RealPolicyFixture())
+        self.useFixture(nova_fixtures.NeutronFixture(self))
+        api_fixture = self.useFixture(nova_fixtures.OSAPIFixture(
+            api_version='v2.1'))
+        self.api = api_fixture.api
+
+        # the image fake backend needed for image discovery
+        nova.tests.unit.image.fake.stub_out_image_service(self)
+
+        self.start_service('conductor')
+        self.start_service('scheduler')
+
+        # We don't actually start a compute service; this way we don't have any
+        # compute hosts to schedule the instance to and will go into error and
+        # be put into cell0.
+
+        self.useFixture(cast_as_call.CastAsCall(self.stubs))
+
+        self.image_id = self.api.get_images()[0]['id']
+        self.flavor_id = self.api.get_flavors()[0]['id']
+
+    def _wait_for_instance_status(self, server_id, status):
+        timeout = 0.0
+        server = self.api.get_server(server_id)
+        while server['status'] != status and timeout < 10.0:
+            time.sleep(.1)
+            timeout += .1
+            server = self.api.get_server(server_id)
+        if server['status'] != status:
+            self.fail('Timed out waiting for server %s to have status: %s.' %
+                      (server_id, status))
+        return server
+
+    def _wait_for_instance_delete(self, server_id):
+        timeout = 0.0
+        while timeout < 10.0:
+            try:
+                server = self.api.get_server(server_id)
+            except client.OpenStackApiNotFoundException:
+                # the instance is gone so we're happy
+                return
+            else:
+                time.sleep(.1)
+                timeout += .1
+
+        self.fail('Timed out waiting for server %s to be deleted. '
+                  'Current vm_state: %s. Current task_state: %s' %
+                  (server_id, server['OS-EXT-STS:vm_state'],
+                   server['OS-EXT-STS:task_state']))
+
+    def _delete_server(self, server_id):
+        try:
+            self.api.delete_server(server_id)
+        except client.OpenStackApiNotFoundException:
+            pass
+
+    def test_delete_error_instance_in_cell0_and_check_quota(self):
+        """Tests deleting the error instance in cell0 and quota.
+
+        This test will create the server which will fail to schedule because
+        there is no compute to send it to. This will trigger the conductor
+        manager to put the instance into ERROR state and create it in cell0.
+
+        The test asserts that quota was decremented. Then it deletes the
+        instance and will check quota again after the instance is gone.
+        """
+        # Get the current quota usage
+        starting_usage = self.api.get_limits()
+
+        # Create the server which we expect to go into ERROR state.
+        server = dict(
+            name='cell0-quota-test',
+            imageRef=self.image_id,
+            flavorRef=self.flavor_id)
+        server = self.api.post_server({'server': server})
+        self.addCleanup(self._delete_server, server['id'])
+        self._wait_for_instance_status(server['id'], 'ERROR')
+
+        # Check quota to see we've incremented usage by 1.
+        current_usage = self.api.get_limits()
+        self.assertEqual(starting_usage['absolute']['totalInstancesUsed'] + 1,
+                         current_usage['absolute']['totalInstancesUsed'])
+
+        # Now delete the server and wait for it to be gone.
+        self._delete_server(server['id'])
+        self._wait_for_instance_delete(server['id'])
+
+        # Now check the quota again. Because we have not fixed the bug, the
+        # quota is still going to be showing a usage for instances. When the
+        # bug is fixed, ending usage should be current usage - 1.
+        ending_usage = self.api.get_limits()
+        self.assertEqual(current_usage['absolute']['totalInstancesUsed'],
+                         ending_usage['absolute']['totalInstancesUsed'])