Adding reconciler to auditor

This commit is contained in:
Andrew Melton 2013-05-13 17:39:45 -04:00
parent 6ec0ae3015
commit cb65528d95
5 changed files with 108 additions and 27 deletions

View File

@ -0,0 +1,12 @@
{
"nova": {
"RegionOne": {
"username": "admin",
"project_id": "admin",
"api_key": "some_key",
"auth_url": "http://identity.example.com:5000/v2.0",
"auth_system": "keystone"
}
},
"region_mapping_loc": "/etc/stacktach/region_mapping.json"
}

View File

@ -0,0 +1,6 @@
{
"RegionOne.dev.global": "RegionOne",
"RegionOne.dev.cell1": "RegionOne",
"RegionTwo.dev.global": "RegionTwo",
"RegionTwo.dev.cell1": "RegionTwo"
}

View File

@ -30,12 +30,21 @@ from django.db.models import F
from stacktach import datetime_to_decimal as dt
from stacktach import models
from stacktach.reconciler import Reconciler
OLD_LAUNCHES_QUERY = "select * from stacktach_instanceusage " \
"where launched_at is not null and " \
"launched_at < %s and instance not in " \
"(select distinct(instance) " \
"from stacktach_instancedeletes where deleted_at < %s)"
OLD_LAUNCHES_QUERY = """
select * from stacktach_instanceusage where
launched_at is not null and
launched_at < %s and
instance not in
(select distinct(instance)
from stacktach_instancedeletes where
deleted_at < %s union
select distinct(instance)
from stacktach_instancereconcile where
deleted_at < %s);"""
reconciler = None
def _get_new_launches(beginning, ending):
@ -63,25 +72,34 @@ def _get_exists(beginning, ending):
return models.InstanceExists.objects.filter(**filters)
def _audit_launches_to_exists(launches, exists):
def _audit_launches_to_exists(launches, exists, beginning):
fails = []
for (instance, launches) in launches.items():
if instance in exists:
for launch1 in launches:
for expected in launches:
found = False
for launch2 in exists[instance]:
if int(launch1['launched_at']) == int(launch2['launched_at']):
for actual in exists[instance]:
if int(expected['launched_at']) == \
int(actual['launched_at']):
# HACK (apmelton): Truncate the decimal because we may not
# have the milliseconds.
found = True
if not found:
rec = False
if reconciler:
args = (expected['id'], beginning)
rec = reconciler.missing_exists_for_instance(*args)
msg = "Couldn't find exists for launch (%s, %s)"
msg = msg % (instance, launch1['launched_at'])
fails.append(['Launch', launch1['id'], msg])
msg = msg % (instance, expected['launched_at'])
fails.append(['Launch', expected['id'], msg, 'Y' if rec else 'N'])
else:
rec = False
if reconciler:
args = (launches[0]['id'], beginning)
rec = reconciler.missing_exists_for_instance(*args)
msg = "No exists for instance (%s)" % instance
fails.append(['Launch', '-', msg])
fails.append(['Launch', '-', msg, 'Y' if rec else 'N'])
return fails
@ -175,8 +193,13 @@ def _launch_audit_for_period(beginning, ending):
else:
launches_dict[instance] = [l, ]
old_launches = models.InstanceUsage.objects.raw(OLD_LAUNCHES_QUERY,
[beginning, beginning])
# NOTE (apmelton)
# Django's safe substitution doesn't allow dict substitution...
# Thus, we send it 'beginning' three times...
old_launches = models.InstanceUsage.objects\
.raw(OLD_LAUNCHES_QUERY,
[beginning, beginning, beginning])
old_launches_dict = {}
for launch in old_launches:
instance = launch.instance
@ -205,7 +228,8 @@ def _launch_audit_for_period(beginning, ending):
exists_dict[instance] = [e, ]
launch_to_exists_fails = _audit_launches_to_exists(launches_dict,
exists_dict)
exists_dict,
beginning)
return launch_to_exists_fails, new_launches.count(), len(old_launches_dict)
@ -222,11 +246,11 @@ def audit_for_period(beginning, ending):
summary = {
'verifier': verify_summary,
'launch_fails': {
'total_failures': len(detail),
'launch_summary': {
'new_launches': new_count,
'old_launches': old_count
}
'old_launches': old_count,
'failures': len(detail)
},
}
details = {
@ -276,7 +300,7 @@ def store_results(start, end, summary, details):
def make_json_report(summary, details):
report = [{'summary': summary},
['Object', 'ID', 'Error Description']]
['Object', 'ID', 'Error Description', 'Reconciled?']]
report.extend(details['exist_fails'])
report.extend(details['launch_fails'])
return json.dumps(report)
@ -302,8 +326,20 @@ if __name__ == '__main__':
help="If set to true, report will be stored. "
"Otherwise, it will just be printed",
type=bool, default=False)
parser.add_argument('--reconcile',
help="Enabled reconciliation",
type=bool, default=False)
parser.add_argument('--reconciler_config',
help="Location of the reconciler config file",
type=str,
default='/etc/stacktach/reconciler-config.json')
args = parser.parse_args()
if args.reconcile:
with open(args.reconciler_config) as f:
reconciler_config = json.load(f)
reconciler = Reconciler(reconciler_config)
if args.utcdatetime is not None:
time = args.utcdatetime
else:

View File

@ -94,12 +94,13 @@ class Reconciler(object):
def missing_exists_for_instance(self, launched_id,
period_beginning):
reconciled = False
launch = models.InstanceUsage.objects.get(launched_id)
launch = models.InstanceUsage.objects.get(id=launched_id)
region = self._region_for_launch(launch)
nova = self._get_nova(region)
try:
server = nova.servers.get(launch.instance)
if TERMINATED_AT_KEY in server._info:
if (server.status == 'DELETED' and
TERMINATED_AT_KEY in server._info):
# Check to see if instance has been deleted
terminated_at = server._info[TERMINATED_AT_KEY]
terminated_at = utils.str_time_to_unix(terminated_at)

View File

@ -51,7 +51,6 @@ config = {
},
'region_mapping_loc': '/etc/stacktach/region_mapping.json',
'flavor_mapping_loc': '/etc/stacktach/flavor_mapping.json',
}
region_mapping = {
@ -220,7 +219,7 @@ class ReconcilerTestCase(unittest.TestCase):
launch = self.mox.CreateMockAnything()
launch.instance = INSTANCE_ID_1
models.InstanceUsage.objects.get(1).AndReturn(launch)
models.InstanceUsage.objects.get(id=1).AndReturn(launch)
self.mox.StubOutWithMock(self.reconciler, '_region_for_launch')
self.reconciler._region_for_launch(launch).AndReturn('RegionOne')
@ -228,6 +227,7 @@ class ReconcilerTestCase(unittest.TestCase):
nova = self._mocked_nova_client()
self.reconciler._get_nova('RegionOne').AndReturn(nova)
server = self.mox.CreateMockAnything()
server.status = 'DELETED'
server._info = {
'OS-INST-USG:terminated_at': str(deleted_at_dt),
}
@ -241,6 +241,32 @@ class ReconcilerTestCase(unittest.TestCase):
self.assertTrue(result)
self.mox.VerifyAll()
def test_missing_exists_for_instance_non_deleted_status(self):
now = datetime.datetime.utcnow()
beginning_dt = now - datetime.timedelta(days=1)
beginning_dec = utils.decimal_utc(beginning_dt)
launch = self.mox.CreateMockAnything()
launch.instance = INSTANCE_ID_1
models.InstanceUsage.objects.get(id=1).AndReturn(launch)
self.mox.StubOutWithMock(self.reconciler, '_region_for_launch')
self.reconciler._region_for_launch(launch).AndReturn('RegionOne')
self.mox.StubOutWithMock(self.reconciler, '_get_nova')
nova = self._mocked_nova_client()
self.reconciler._get_nova('RegionOne').AndReturn(nova)
server = self.mox.CreateMockAnything()
server.status = 'ACTIVE'
server._info = {
'OS-INST-USG:terminated_at': None,
}
nova.servers.get(INSTANCE_ID_1).AndReturn(server)
self.mox.ReplayAll()
result = self.reconciler.missing_exists_for_instance(1, beginning_dec)
self.assertFalse(result)
self.mox.VerifyAll()
def test_missing_exists_for_instance_deleted_too_soon(self):
now = datetime.datetime.utcnow()
deleted_at_dt = now - datetime.timedelta(hours=4)
@ -249,7 +275,7 @@ class ReconcilerTestCase(unittest.TestCase):
launch = self.mox.CreateMockAnything()
launch.instance = INSTANCE_ID_1
models.InstanceUsage.objects.get(1).AndReturn(launch)
models.InstanceUsage.objects.get(id=1).AndReturn(launch)
self.mox.StubOutWithMock(self.reconciler, '_region_for_launch')
self.reconciler._region_for_launch(launch).AndReturn('RegionOne')
@ -276,7 +302,7 @@ class ReconcilerTestCase(unittest.TestCase):
launch = self.mox.CreateMockAnything()
launch.instance = INSTANCE_ID_1
models.InstanceUsage.objects.get(1).AndReturn(launch)
models.InstanceUsage.objects.get(id=1).AndReturn(launch)
self.mox.StubOutWithMock(self.reconciler, '_region_for_launch')
self.reconciler._region_for_launch(launch).AndReturn('RegionOne')
@ -301,7 +327,7 @@ class ReconcilerTestCase(unittest.TestCase):
launch = self.mox.CreateMockAnything()
launch.instance = INSTANCE_ID_1
models.InstanceUsage.objects.get(1).AndReturn(launch)
models.InstanceUsage.objects.get(id=1).AndReturn(launch)
self.mox.StubOutWithMock(self.reconciler, '_region_for_launch')
self.reconciler._region_for_launch(launch).AndReturn('RegionOne')