From 2f8b54be747a49617c6fd07f0039a05a89d15633 Mon Sep 17 00:00:00 2001 From: Andrew Melton Date: Tue, 18 Jun 2013 15:52:27 -0400 Subject: [PATCH] Using JSONBridge in reconciler --- stacktach/models.py | 7 + stacktach/reconciler.py | 117 --------- stacktach/reconciler/__init__.py | 107 ++++++++ stacktach/reconciler/exceptions.py | 3 + stacktach/reconciler/nova.py | 54 ++++ stacktach/reconciler/utils.py | 9 + tests/unit/test_reconciler.py | 381 ++++++++++++----------------- 7 files changed, 337 insertions(+), 341 deletions(-) delete mode 100644 stacktach/reconciler.py create mode 100644 stacktach/reconciler/__init__.py create mode 100644 stacktach/reconciler/exceptions.py create mode 100644 stacktach/reconciler/nova.py create mode 100644 stacktach/reconciler/utils.py diff --git a/stacktach/models.py b/stacktach/models.py index 00d233c..906b170 100644 --- a/stacktach/models.py +++ b/stacktach/models.py @@ -89,6 +89,13 @@ class InstanceUsage(models.Model): tenant = models.CharField(max_length=50, null=True, blank=True, db_index=True) + def deployment(self): + raws = RawData.objects.filter(request_id=self.request_id) + if raws.count() == 0: + return False + raw = raws[0] + return raw.deployment + class InstanceDeletes(models.Model): instance = models.CharField(max_length=50, null=True, blank=True, db_index=True) diff --git a/stacktach/reconciler.py b/stacktach/reconciler.py deleted file mode 100644 index 1c3645f..0000000 --- a/stacktach/reconciler.py +++ /dev/null @@ -1,117 +0,0 @@ -# Copyright (c) 2013 - Rackspace Inc. -# -# Permission is hereby granted, free of charge, to any person obtaining a copy -# of this software and associated documentation files (the "Software"), to -# deal in the Software without restriction, including without limitation the -# rights to use, copy, modify, merge, publish, distribute, sublicense, and/or -# sell copies of the Software, and to permit persons to whom the Software is -# furnished to do so, subject to the following conditions: -# -# The above copyright notice and this permission notice shall be included in -# all copies or substantial portions of the Software. -# -# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING -# FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS -# IN THE SOFTWARE. - -import json - -from novaclient.exceptions import NotFound -from novaclient.v1_1 import client - -from stacktach import models -from stacktach import utils - -TERMINATED_AT_KEY = 'OS-INST-USG:terminated_at' - - -class Reconciler(object): - - def __init__(self, config, region_mapping=None): - self.config = config - self.region_mapping = (region_mapping or - Reconciler._load_region_mapping(config)) - self.nova_clients = {} - - @classmethod - def _load_region_mapping(cls, config): - with open(config['region_mapping_loc']) as f: - return json.load(f) - - def _get_nova(self, region): - if region in self.nova_clients: - return self.nova_clients[region] - - region_cfg = self.config['nova'][region] - region_auth_system = region_cfg.get('auth_system', 'keystone') - - nova = client.Client(region_cfg['username'], region_cfg['api_key'], - region_cfg['project_id'], - auth_url=region_cfg['auth_url'], - auth_system=region_auth_system) - - self.nova_clients[region] = nova - return nova - - def _region_for_launch(self, launch): - request = launch.request_id - raws = models.RawData.objects.filter(request_id=request) - if raws.count() == 0: - return False - raw = raws[0] - deployment_name = str(raw.deployment.name) - if deployment_name in self.region_mapping: - return self.region_mapping[deployment_name] - else: - return False - - def _reconcile_from_api(self, launch, server): - terminated_at = server._info[TERMINATED_AT_KEY] - terminated_at = utils.str_time_to_unix(terminated_at) - values = { - 'instance': server.id, - 'launched_at': launch.launched_at, - 'deleted_at': terminated_at, - 'instance_type_id': launch.instance_type_id, - 'source': 'reconciler:nova_api', - } - models.InstanceReconcile(**values).save() - - def _reconcile_from_api_not_found(self, launch): - values = { - 'instance': launch.instance, - 'launched_at': launch.launched_at, - 'deleted_at': 1, - 'instance_type_id': launch.instance_type_id, - 'source': 'reconciler:nova_api:not_found', - } - models.InstanceReconcile(**values).save() - - def missing_exists_for_instance(self, launched_id, - period_beginning): - reconciled = False - launch = models.InstanceUsage.objects.get(id=launched_id) - region = self._region_for_launch(launch) - nova = self._get_nova(region) - try: - server = nova.servers.get(launch.instance) - if (server.status == 'DELETED' and - TERMINATED_AT_KEY in server._info): - # Check to see if instance has been deleted - terminated_at = server._info[TERMINATED_AT_KEY] - terminated_at = utils.str_time_to_unix(terminated_at) - - if terminated_at < period_beginning: - # Check to see if instance was deleted before period. - # If so, we shouldn't expect an exists. - self._reconcile_from_api(launch, server) - reconciled = True - except NotFound: - self._reconcile_from_api_not_found(launch) - reconciled = True - - return reconciled diff --git a/stacktach/reconciler/__init__.py b/stacktach/reconciler/__init__.py new file mode 100644 index 0000000..24852ad --- /dev/null +++ b/stacktach/reconciler/__init__.py @@ -0,0 +1,107 @@ +# Copyright (c) 2013 - Rackspace Inc. +# +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the "Software"), to +# deal in the Software without restriction, including without limitation the +# rights to use, copy, modify, merge, publish, distribute, sublicense, and/or +# sell copies of the Software, and to permit persons to whom the Software is +# furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in +# all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +# FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS +# IN THE SOFTWARE. + +import json + +from stacktach import models +from stacktach.reconciler import exceptions +from stacktach.reconciler import nova + +DEFAULT_CLIENT = nova.JSONBridgeClient + +CONFIG = { + 'client_class': 'JSONBridgeClient', + 'client': { + 'url': 'http://stack.dev.ramielrowe.com:8080/query/', + 'username': '', + 'password': '', + 'databases': { + 'RegionOne': 'nova', + } + }, + 'region_mapping_loc': '/etc/stacktach/region_mapping.json' +} + + +class Reconciler(object): + + def __init__(self, config, client=None, region_mapping=None): + self.config = config + self.client = (client or Reconciler._load_client(config)) + self.region_mapping = (region_mapping or + Reconciler._load_region_mapping(config)) + + @classmethod + def load_client(cls, config): + client_class = config.get('client_class') + if client_class == 'JSONBridgeClient': + return nova.JSONBridgeClient(config['client']) + else: + return DEFAULT_CLIENT(config['client']) + + @classmethod + def load_region_mapping(cls, config): + with open(config['region_mapping_loc']) as f: + return json.load(f) + + def _region_for_launch(self, launch): + deployment = launch.deployment() + if deployment: + deployment_name = str(deployment.name) + if deployment_name in self.region_mapping: + return self.region_mapping[deployment_name] + else: + return False + else: + return False + + def _reconcile_instance(self, launch, src, + launched_at=None, deleted_at=None, + instance_type_id=None): + values = { + 'instance': launch.instance, + 'launched_at': (launched_at or launch.launched_at), + 'deleted_at': deleted_at, + 'instance_type_id': (instance_type_id or launch.instance_type_id), + 'source': 'reconciler:%s' % src, + } + models.InstanceReconcile(**values).save() + + def missing_exists_for_instance(self, launched_id, + period_beginning): + reconciled = False + launch = models.InstanceUsage.objects.get(id=launched_id) + region = self._region_for_launch(launch) + try: + instance = self.client.get_instance(region, launch.instance) + if instance['deleted'] and instance['deleted_at'] is not None: + # Check to see if instance has been deleted + deleted_at = instance['deleted_at'] + + if deleted_at < period_beginning: + # Check to see if instance was deleted before period. + # If so, we shouldn't expect an exists. + self._reconcile_instance(launch, self.client.src_str, + deleted_at=instance['deleted_at']) + reconciled = True + except exceptions.NotFound: + reconciled = False + + return reconciled diff --git a/stacktach/reconciler/exceptions.py b/stacktach/reconciler/exceptions.py new file mode 100644 index 0000000..e48f49a --- /dev/null +++ b/stacktach/reconciler/exceptions.py @@ -0,0 +1,3 @@ +class NotFound(Exception): + def __init__(self, message="NotFound"): + self.message = message diff --git a/stacktach/reconciler/nova.py b/stacktach/reconciler/nova.py new file mode 100644 index 0000000..aae40b2 --- /dev/null +++ b/stacktach/reconciler/nova.py @@ -0,0 +1,54 @@ +import json + +import requests + +from stacktach import utils as stackutils +from stacktach.reconciler import exceptions +from stacktach.reconciler.utils import empty_reconciler_instance + + +GET_INSTANCE_QUERY = "SELECT * FROM instances where uuid ='%s';" + + +class JSONBridgeClient(object): + src_str = 'json_bridge:nova_db' + + def __init__(self, config): + self.config = config + + def _url_for_region(self, region): + return self.config['url'] + self.config['regions'][region] + + def _do_query(self, region, query): + data = {'sql': query} + credentials = (self.config['username'], self.config['password']) + return requests.post(self._url_for_region(region), data, + verify=False, auth=credentials).json() + + def _to_reconciler_instance(self, instance): + r_instance = empty_reconciler_instance() + r_instance.update({ + 'id': instance['uuid'], + 'instance_type_id': instance['instance_type_id'], + }) + + if instance['launched_at'] is not None: + launched_at = stackutils.str_time_to_unix(instance['launched_at']) + r_instance['launched_at'] = launched_at + + if instance['terminated_at'] is not None: + deleted_at = stackutils.str_time_to_unix(instance['terminated_at']) + r_instance['deleted_at'] = deleted_at + + if instance['deleted'] != 0: + r_instance['deleted'] = True + + return r_instance + + def get_instance(self, region, uuid): + results = self._do_query(region, GET_INSTANCE_QUERY % uuid)['result'] + if len(results) > 0: + return self._to_reconciler_instance(results[0]) + else: + msg = "Couldn't find instance (%s) using JSON Bridge in region (%s)" + raise exceptions.NotFound(msg % (uuid, region)) \ No newline at end of file diff --git a/stacktach/reconciler/utils.py b/stacktach/reconciler/utils.py new file mode 100644 index 0000000..b835d25 --- /dev/null +++ b/stacktach/reconciler/utils.py @@ -0,0 +1,9 @@ +def empty_reconciler_instance(): + r_instance = { + 'id': None, + 'launched_at': None, + 'deleted': False, + 'deleted_at': None, + 'instance_type_ud': None + } + return r_instance diff --git a/tests/unit/test_reconciler.py b/tests/unit/test_reconciler.py index 994c5c3..7ba5d0d 100644 --- a/tests/unit/test_reconciler.py +++ b/tests/unit/test_reconciler.py @@ -22,14 +22,15 @@ import datetime import unittest import mox -from novaclient.exceptions import NotFound -from novaclient.v1_1 import client as nova_client +import requests from stacktach import models from stacktach import reconciler -import utils -from utils import INSTANCE_ID_1 -from utils import REQUEST_ID_1 +from stacktach import utils as stackutils +from stacktach.reconciler import exceptions +from stacktach.reconciler import nova +from tests.unit import utils +from tests.unit.utils import INSTANCE_ID_1 config = { @@ -61,9 +62,12 @@ region_mapping = { class ReconcilerTestCase(unittest.TestCase): def setUp(self): - self.reconciler = reconciler.Reconciler(config, - region_mapping=region_mapping) self.mox = mox.Mox() + self.client = self.mox.CreateMockAnything() + self.client.src_str = 'mocked_client' + self.reconciler = reconciler.Reconciler(config, + client=self.client, + region_mapping=region_mapping) self.mox.StubOutWithMock(models, 'RawData', use_mock_anything=True) models.RawData.objects = self.mox.CreateMockAnything() self.mox.StubOutWithMock(models, 'Deployment', use_mock_anything=True) @@ -89,27 +93,50 @@ class ReconcilerTestCase(unittest.TestCase): models.InstanceExists.objects = self.mox.CreateMockAnything() self.mox.StubOutWithMock(models, 'JsonReport', use_mock_anything=True) models.JsonReport.objects = self.mox.CreateMockAnything() - self.mox.StubOutWithMock(nova_client, 'Client', use_mock_anything=True) def tearDown(self): self.mox.UnsetStubs() - def _mocked_nova_client(self): - nova = self.mox.CreateMockAnything() - nova.servers = self.mox.CreateMockAnything() - return nova + def _fake_reconciler_instance(self, uuid=INSTANCE_ID_1, launched_at=None, + deleted_at=None, deleted=False, + instance_type_id=1): + return { + 'id': uuid, + 'launched_at': launched_at, + 'deleted_at': deleted_at, + 'deleted': deleted, + 'instance_type_id': instance_type_id + } + + def test_load_client_json_bridge(self): + mock_config = self.mox.CreateMockAnything() + config = {'client_class': 'JSONBridgeClient', 'client': mock_config} + nova.JSONBridgeClient(mock_config) + self.mox.ReplayAll() + reconciler.Reconciler.load_client(config) + self.mox.VerifyAll() + + def test_load_client_no_class_loads_default_class(self): + mock_config = self.mox.CreateMockAnything() + config = {'client': mock_config} + nova.JSONBridgeClient(mock_config) + self.mox.ReplayAll() + reconciler.Reconciler.load_client(config) + self.mox.VerifyAll() + + def test_load_client_incorrect_class_loads_default_class(self): + mock_config = self.mox.CreateMockAnything() + config = {'client_class': 'BadConfigValue', 'client': mock_config} + nova.JSONBridgeClient(mock_config) + self.mox.ReplayAll() + reconciler.Reconciler.load_client(config) + self.mox.VerifyAll() def test_region_for_launch(self): launch = self.mox.CreateMockAnything() - launch.request_id = REQUEST_ID_1 - result = self.mox.CreateMockAnything() - models.RawData.objects.filter(request_id=REQUEST_ID_1)\ - .AndReturn(result) - result.count().AndReturn(1) - raw = self.mox.CreateMockAnything() - raw.deployment = self.mox.CreateMockAnything() - raw.deployment.name = 'RegionOne.prod.cell1' - result[0].AndReturn(raw) + deployment = self.mox.CreateMockAnything() + deployment.name = 'RegionOne.prod.cell1' + launch.deployment().AndReturn(deployment) self.mox.ReplayAll() region = self.reconciler._region_for_launch(launch) self.assertEqual('RegionOne', region) @@ -117,15 +144,9 @@ class ReconcilerTestCase(unittest.TestCase): def test_region_for_launch_no_mapping(self): launch = self.mox.CreateMockAnything() - launch.request_id = REQUEST_ID_1 - result = self.mox.CreateMockAnything() - models.RawData.objects.filter(request_id=REQUEST_ID_1)\ - .AndReturn(result) - result.count().AndReturn(1) - raw = self.mox.CreateMockAnything() - raw.deployment = self.mox.CreateMockAnything() - raw.deployment.name = 'RegionOne.prod.cell2' - result[0].AndReturn(raw) + deployment = self.mox.CreateMockAnything() + deployment.name = 'RegionOne.prod.cell2' + launch.deployment().AndReturn(deployment) self.mox.ReplayAll() region = self.reconciler._region_for_launch(launch) self.assertFalse(region) @@ -133,215 +154,127 @@ class ReconcilerTestCase(unittest.TestCase): def test_region_for_launch_no_raws(self): launch = self.mox.CreateMockAnything() - launch.request_id = REQUEST_ID_1 - result = self.mox.CreateMockAnything() - models.RawData.objects.filter(request_id=REQUEST_ID_1)\ - .AndReturn(result) - result.count().AndReturn(0) + launch.deployment() self.mox.ReplayAll() region = self.reconciler._region_for_launch(launch) self.assertFalse(region) self.mox.VerifyAll() - def test_get_nova(self): - expected_client = self._mocked_nova_client - nova_client.Client('demo', 'some_key', '111111', - auth_url='https://identity.example.com/v2.0', - auth_system='keystone').AndReturn(expected_client) - self.mox.ReplayAll() - client = self.reconciler._get_nova('RegionOne') - self.assertEqual(expected_client, client) - self.mox.VerifyAll() - - def test_get_nova_already_created(self): - expected_client = self.mox.CreateMockAnything() - nova_client.Client('demo', 'some_key', '111111', - auth_url='https://identity.example.com/v2.0', - auth_system='keystone').AndReturn(expected_client) - self.mox.ReplayAll() - self.reconciler._get_nova('RegionOne') - client = self.reconciler._get_nova('RegionOne') - self.assertEqual(expected_client, client) - self.mox.VerifyAll() - - def test_reconcile_from_api(self): - deleted_at = datetime.datetime.utcnow() - launched_at = deleted_at - datetime.timedelta(hours=4) - launch = self.mox.CreateMockAnything() - launch.instance = INSTANCE_ID_1 - launch.launched_at = utils.decimal_utc(launched_at) - launch.instance_type_id = 1 - server = self.mox.CreateMockAnything() - server.id = INSTANCE_ID_1 - server._info = { - 'OS-INST-USG:terminated_at': str(deleted_at), - } - values = { - 'instance': INSTANCE_ID_1, - 'instance_type_id': 1, - 'launched_at': utils.decimal_utc(launched_at), - 'deleted_at': utils.decimal_utc(deleted_at), - 'source': 'reconciler:nova_api' - } - result = self.mox.CreateMockAnything() - models.InstanceReconcile(**values).AndReturn(result) - result.save() - self.mox.ReplayAll() - self.reconciler._reconcile_from_api(launch, server) - self.mox.VerifyAll() - - def test_reconcile_from_api_not_found(self): - deleted_at = datetime.datetime.utcnow() - launched_at = deleted_at - datetime.timedelta(hours=4) - launch = self.mox.CreateMockAnything() - launch.instance = INSTANCE_ID_1 - launch.launched_at = utils.decimal_utc(launched_at) - launch.instance_type_id = 1 - values = { - 'instance': INSTANCE_ID_1, - 'instance_type_id': 1, - 'launched_at': utils.decimal_utc(launched_at), - 'deleted_at': 1, - 'source': 'reconciler:nova_api:not_found' - } - result = self.mox.CreateMockAnything() - models.InstanceReconcile(**values).AndReturn(result) - result.save() - self.mox.ReplayAll() - self.reconciler._reconcile_from_api_not_found(launch) - self.mox.VerifyAll() - def test_missing_exists_for_instance(self): - now = datetime.datetime.utcnow() - deleted_at_dt = now - datetime.timedelta(days=2) - beginning_dt = now - datetime.timedelta(days=1) - beginning_dec = utils.decimal_utc(beginning_dt) - + launch_id = 1 + beginning_d = utils.decimal_utc() launch = self.mox.CreateMockAnything() launch.instance = INSTANCE_ID_1 - models.InstanceUsage.objects.get(id=1).AndReturn(launch) - self.mox.StubOutWithMock(self.reconciler, '_region_for_launch') - self.reconciler._region_for_launch(launch).AndReturn('RegionOne') - - self.mox.StubOutWithMock(self.reconciler, '_get_nova') - nova = self._mocked_nova_client() - self.reconciler._get_nova('RegionOne').AndReturn(nova) - server = self.mox.CreateMockAnything() - server.status = 'DELETED' - server._info = { - 'OS-INST-USG:terminated_at': str(deleted_at_dt), + launch.launched_at = beginning_d - (60*60) + launch.instance_type_id = 1 + models.InstanceUsage.objects.get(id=launch_id).AndReturn(launch) + deployment = self.mox.CreateMockAnything() + launch.deployment().AndReturn(deployment) + deployment.name = 'RegionOne.prod.cell1' + deleted_at = beginning_d - (60*30) + rec_inst = self._fake_reconciler_instance(deleted=True, + deleted_at=deleted_at) + self.client.get_instance('RegionOne', INSTANCE_ID_1).AndReturn(rec_inst) + reconcile_vals = { + 'instance': launch.instance, + 'launched_at': launch.launched_at, + 'deleted_at': deleted_at, + 'instance_type_id': launch.instance_type_id, + 'source': 'reconciler:mocked_client' } - nova.servers.get(INSTANCE_ID_1).AndReturn(server) - - self.mox.StubOutWithMock(self.reconciler, '_reconcile_from_api') - self.reconciler._reconcile_from_api(launch, server) - + result = self.mox.CreateMockAnything() + models.InstanceReconcile(**reconcile_vals).AndReturn(result) + result.save() self.mox.ReplayAll() - result = self.reconciler.missing_exists_for_instance(1, beginning_dec) + result = self.reconciler.missing_exists_for_instance(launch_id, + beginning_d) self.assertTrue(result) self.mox.VerifyAll() - def test_missing_exists_for_instance_non_deleted_status(self): - now = datetime.datetime.utcnow() - beginning_dt = now - datetime.timedelta(days=1) - beginning_dec = utils.decimal_utc(beginning_dt) - - launch = self.mox.CreateMockAnything() - launch.instance = INSTANCE_ID_1 - models.InstanceUsage.objects.get(id=1).AndReturn(launch) - self.mox.StubOutWithMock(self.reconciler, '_region_for_launch') - self.reconciler._region_for_launch(launch).AndReturn('RegionOne') - - self.mox.StubOutWithMock(self.reconciler, '_get_nova') - nova = self._mocked_nova_client() - self.reconciler._get_nova('RegionOne').AndReturn(nova) - server = self.mox.CreateMockAnything() - server.status = 'ACTIVE' - server._info = { - 'OS-INST-USG:terminated_at': None, - } - nova.servers.get(INSTANCE_ID_1).AndReturn(server) - - self.mox.ReplayAll() - result = self.reconciler.missing_exists_for_instance(1, beginning_dec) - self.assertFalse(result) - self.mox.VerifyAll() - - def test_missing_exists_for_instance_deleted_too_soon(self): - now = datetime.datetime.utcnow() - deleted_at_dt = now - datetime.timedelta(hours=4) - beginning_dt = now - datetime.timedelta(days=1) - beginning_dec = utils.decimal_utc(beginning_dt) - - launch = self.mox.CreateMockAnything() - launch.instance = INSTANCE_ID_1 - models.InstanceUsage.objects.get(id=1).AndReturn(launch) - self.mox.StubOutWithMock(self.reconciler, '_region_for_launch') - self.reconciler._region_for_launch(launch).AndReturn('RegionOne') - - self.mox.StubOutWithMock(self.reconciler, '_get_nova') - nova = self._mocked_nova_client() - self.reconciler._get_nova('RegionOne').AndReturn(nova) - server = self.mox.CreateMockAnything() - server._info = { - 'OS-INST-USG:terminated_at': str(deleted_at_dt), - } - nova.servers.get(INSTANCE_ID_1).AndReturn(server) - - self.mox.StubOutWithMock(self.reconciler, '_reconcile_from_api') - - self.mox.ReplayAll() - result = self.reconciler.missing_exists_for_instance(1, beginning_dec) - self.assertFalse(result) - self.mox.VerifyAll() - - def test_missing_exists_for_instance_not_deleted(self): - now = datetime.datetime.utcnow() - beginning_dt = now - datetime.timedelta(days=1) - beginning_dec = utils.decimal_utc(beginning_dt) - - launch = self.mox.CreateMockAnything() - launch.instance = INSTANCE_ID_1 - models.InstanceUsage.objects.get(id=1).AndReturn(launch) - self.mox.StubOutWithMock(self.reconciler, '_region_for_launch') - self.reconciler._region_for_launch(launch).AndReturn('RegionOne') - - self.mox.StubOutWithMock(self.reconciler, '_get_nova') - nova = self._mocked_nova_client() - self.reconciler._get_nova('RegionOne').AndReturn(nova) - server = self.mox.CreateMockAnything() - server._info = {} - nova.servers.get(INSTANCE_ID_1).AndReturn(server) - - self.mox.StubOutWithMock(self.reconciler, '_reconcile_from_api') - - self.mox.ReplayAll() - result = self.reconciler.missing_exists_for_instance(1, beginning_dec) - self.assertFalse(result) - self.mox.VerifyAll() - def test_missing_exists_for_instance_not_found(self): - now = datetime.datetime.utcnow() - beginning_dt = now - datetime.timedelta(days=1) - beginning_dec = utils.decimal_utc(beginning_dt) - + launch_id = 1 + beginning_d = utils.decimal_utc() launch = self.mox.CreateMockAnything() launch.instance = INSTANCE_ID_1 - models.InstanceUsage.objects.get(id=1).AndReturn(launch) - self.mox.StubOutWithMock(self.reconciler, '_region_for_launch') - self.reconciler._region_for_launch(launch).AndReturn('RegionOne') - - self.mox.StubOutWithMock(self.reconciler, '_get_nova') - nova = self._mocked_nova_client() - self.reconciler._get_nova('RegionOne').AndReturn(nova) - - nova.servers.get(INSTANCE_ID_1).AndRaise(NotFound(404)) - - self.mox.StubOutWithMock(self.reconciler, - '_reconcile_from_api_not_found') - self.reconciler._reconcile_from_api_not_found(launch) - + launch.launched_at = beginning_d - (60*60) + launch.instance_type_id = 1 + models.InstanceUsage.objects.get(id=launch_id).AndReturn(launch) + deployment = self.mox.CreateMockAnything() + launch.deployment().AndReturn(deployment) + deployment.name = 'RegionOne.prod.cell1' + ex = exceptions.NotFound() + self.client.get_instance('RegionOne', INSTANCE_ID_1).AndRaise(ex) self.mox.ReplayAll() - result = self.reconciler.missing_exists_for_instance(1, beginning_dec) - self.assertTrue(result) + result = self.reconciler.missing_exists_for_instance(launch_id, + beginning_d) + self.assertFalse(result) + self.mox.VerifyAll() + + +json_bridge_config = { + 'url': 'http://json_bridge.example.com/query/', + 'username': 'user', + 'password': 'pass', + 'regions': { + 'RegionOne': 'nova', + } +} + + +class NovaJSONBridgeClientTestCase(unittest.TestCase): + def setUp(self): + self.mox = mox.Mox() + self.client = nova.JSONBridgeClient(json_bridge_config) + self.mox.StubOutWithMock(requests, 'post') + + def tearDown(self): + self.mox.UnsetStubs() + + def mock_for_query(self, database, query, results): + url = json_bridge_config['url'] + database + data = {'sql': query} + auth = (json_bridge_config['username'], json_bridge_config['password']) + result = {'result': results} + response = self.mox.CreateMockAnything() + requests.post(url, data, auth=auth, verify=False)\ + .AndReturn(response) + response.json().AndReturn(result) + + def _fake_instance(self, uuid=INSTANCE_ID_1, launched_at=None, + terminated_at=None, deleted=0, instance_type_id=1): + return { + 'uuid': uuid, + 'launched_at': launched_at, + 'terminated_at': terminated_at, + 'deleted': deleted, + 'instance_type_id': instance_type_id + } + + def test_get_instance(self): + launched_at = datetime.datetime.utcnow() - datetime.timedelta(minutes=5) + launched_at = str(launched_at) + terminated_at = str(datetime.datetime.utcnow()) + results = [self._fake_instance(launched_at=launched_at, + terminated_at=terminated_at, + deleted=True)] + self.mock_for_query('nova', nova.GET_INSTANCE_QUERY % INSTANCE_ID_1, + results) + self.mox.ReplayAll() + instance = self.client.get_instance('RegionOne', INSTANCE_ID_1) + self.assertIsNotNone(instance) + self.assertEqual(instance['id'], INSTANCE_ID_1) + self.assertEqual(instance['instance_type_id'], 1) + launched_at_dec = stackutils.str_time_to_unix(launched_at) + self.assertEqual(instance['launched_at'], launched_at_dec) + terminated_at_dec = stackutils.str_time_to_unix(terminated_at) + self.assertEqual(instance['deleted_at'], terminated_at_dec) + self.assertTrue(instance['deleted']) + self.mox.VerifyAll() + + def test_get_instance_not_found(self): + self.mock_for_query('nova', nova.GET_INSTANCE_QUERY % INSTANCE_ID_1, + []) + self.mox.ReplayAll() + self.assertRaises(exceptions.NotFound, self.client.get_instance, + 'RegionOne', INSTANCE_ID_1) self.mox.VerifyAll()