x/stacktach
Code Issues Proposed changes

Merge pull request #146 from ramielrowe/verifier_reconcile

Browse Source 
Including the reconciler in the verifier.
This commit is contained in:
Andrew Melton
2013-07-12 08:55:32 -07:00
parent 9f99e06e0a d17e7eef88
commit 05ff27d8c2
10 changed files with 1024 additions and 74 deletions
Download Patch File Download Diff File Expand all files Collapse all files

9
reports/nova_usage_audit.py
View File

@@ -105,11 +105,12 @@ def _audit_launches_to_exists(launches, exists, beginning):
def _status_queries(exists_query):
verified = exists_query.filter(status=models.InstanceExists.VERIFIED)
reconciled = exists_query.filter(status=models.InstanceExists.RECONCILED)
fail = exists_query.filter(status=models.InstanceExists.FAILED)
pending = exists_query.filter(status=models.InstanceExists.PENDING)
verifying = exists_query.filter(status=models.InstanceExists.VERIFYING)
return verified, fail, pending, verifying
return verified, reconciled, fail, pending, verifying
def _send_status_queries(exists_query):
@@ -126,7 +127,8 @@ def _send_status_queries(exists_query):
def _audit_for_exists(exists_query):
(verified, fail, pending, verifying) = _status_queries(exists_query)
(verified, reconciled,
fail, pending, verifying) = _status_queries(exists_query)
(success, unsent, redirect,
client_error, server_error) = _send_status_queries(verified)
@@ -134,6 +136,7 @@ def _audit_for_exists(exists_query):
report = {
'count': exists_query.count(),
'verified': verified.count(),
'reconciled': reconciled.count(),
'failed': fail.count(),
'pending': pending.count(),
'verifying': verifying.count(),
@@ -290,7 +293,7 @@ def store_results(start, end, summary, details):
'created': dt.dt_to_decimal(datetime.datetime.utcnow()),
'period_start': start,
'period_end': end,
'version': 3,
'version': 4,
'name': 'nova usage audit'
}

189
stacktach/migrations/0005_tenant_and_image_meta_on_instance_reconcile.py Normal file
View File

@@ -0,0 +1,189 @@
# -*- coding: utf-8 -*-
import datetime
from south.db import db
from south.v2 import SchemaMigration
from django.db import models
class Migration(SchemaMigration):
def forwards(self, orm):
# Adding field 'InstanceReconcile.tenant'
db.add_column(u'stacktach_instancereconcile', 'tenant',
self.gf('django.db.models.fields.CharField')(db_index=True, max_length=50, null=True, blank=True),
keep_default=False)
# Adding field 'InstanceReconcile.os_architecture'
db.add_column(u'stacktach_instancereconcile', 'os_architecture',
self.gf('django.db.models.fields.TextField')(null=True, blank=True),
keep_default=False)
# Adding field 'InstanceReconcile.os_distro'
db.add_column(u'stacktach_instancereconcile', 'os_distro',
self.gf('django.db.models.fields.TextField')(null=True, blank=True),
keep_default=False)
# Adding field 'InstanceReconcile.os_version'
db.add_column(u'stacktach_instancereconcile', 'os_version',
self.gf('django.db.models.fields.TextField')(null=True, blank=True),
keep_default=False)
# Adding field 'InstanceReconcile.rax_options'
db.add_column(u'stacktach_instancereconcile', 'rax_options',
self.gf('django.db.models.fields.TextField')(null=True, blank=True),
keep_default=False)
def backwards(self, orm):
# Deleting field 'InstanceReconcile.tenant'
db.delete_column(u'stacktach_instancereconcile', 'tenant')
# Deleting field 'InstanceReconcile.os_architecture'
db.delete_column(u'stacktach_instancereconcile', 'os_architecture')
# Deleting field 'InstanceReconcile.os_distro'
db.delete_column(u'stacktach_instancereconcile', 'os_distro')
# Deleting field 'InstanceReconcile.os_version'
db.delete_column(u'stacktach_instancereconcile', 'os_version')
# Deleting field 'InstanceReconcile.rax_options'
db.delete_column(u'stacktach_instancereconcile', 'rax_options')
models = {
u'stacktach.deployment': {
'Meta': {'object_name': 'Deployment'},
u'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
'name': ('django.db.models.fields.CharField', [], {'max_length': '50'})
},
u'stacktach.instancedeletes': {
'Meta': {'object_name': 'InstanceDeletes'},
'deleted_at': ('django.db.models.fields.DecimalField', [], {'null': 'True', 'max_digits': '20', 'decimal_places': '6', 'db_index': 'True'}),
u'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
'instance': ('django.db.models.fields.CharField', [], {'db_index': 'True', 'max_length': '50', 'null': 'True', 'blank': 'True'}),
'launched_at': ('django.db.models.fields.DecimalField', [], {'null': 'True', 'max_digits': '20', 'decimal_places': '6', 'db_index': 'True'}),
'raw': ('django.db.models.fields.related.ForeignKey', [], {'to': u"orm['stacktach.RawData']", 'null': 'True'})
},
u'stacktach.instanceexists': {
'Meta': {'object_name': 'InstanceExists'},
'audit_period_beginning': ('django.db.models.fields.DecimalField', [], {'null': 'True', 'max_digits': '20', 'decimal_places': '6', 'db_index': 'True'}),
'audit_period_ending': ('django.db.models.fields.DecimalField', [], {'null': 'True', 'max_digits': '20', 'decimal_places': '6', 'db_index': 'True'}),
'delete': ('django.db.models.fields.related.ForeignKey', [], {'related_name': "'+'", 'null': 'True', 'to': u"orm['stacktach.InstanceDeletes']"}),
'deleted_at': ('django.db.models.fields.DecimalField', [], {'null': 'True', 'max_digits': '20', 'decimal_places': '6', 'db_index': 'True'}),
'fail_reason': ('django.db.models.fields.CharField', [], {'db_index': 'True', 'max_length': '300', 'null': 'True', 'blank': 'True'}),
u'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
'instance': ('django.db.models.fields.CharField', [], {'db_index': 'True', 'max_length': '50', 'null': 'True', 'blank': 'True'}),
'instance_type_id': ('django.db.models.fields.CharField', [], {'db_index': 'True', 'max_length': '50', 'null': 'True', 'blank': 'True'}),
'launched_at': ('django.db.models.fields.DecimalField', [], {'null': 'True', 'max_digits': '20', 'decimal_places': '6', 'db_index': 'True'}),
'message_id': ('django.db.models.fields.CharField', [], {'db_index': 'True', 'max_length': '50', 'null': 'True', 'blank': 'True'}),
'os_architecture': ('django.db.models.fields.TextField', [], {'null': 'True', 'blank': 'True'}),
'os_distro': ('django.db.models.fields.TextField', [], {'null': 'True', 'blank': 'True'}),
'os_version': ('django.db.models.fields.TextField', [], {'null': 'True', 'blank': 'True'}),
'raw': ('django.db.models.fields.related.ForeignKey', [], {'related_name': "'+'", 'null': 'True', 'to': u"orm['stacktach.RawData']"}),
'rax_options': ('django.db.models.fields.TextField', [], {'null': 'True', 'blank': 'True'}),
'send_status': ('django.db.models.fields.IntegerField', [], {'default': '0', 'null': 'True', 'db_index': 'True'}),
'status': ('django.db.models.fields.CharField', [], {'default': "'pending'", 'max_length': '50', 'db_index': 'True'}),
'tenant': ('django.db.models.fields.CharField', [], {'db_index': 'True', 'max_length': '50', 'null': 'True', 'blank': 'True'}),
'usage': ('django.db.models.fields.related.ForeignKey', [], {'related_name': "'+'", 'null': 'True', 'to': u"orm['stacktach.InstanceUsage']"})
},
u'stacktach.instancereconcile': {
'Meta': {'object_name': 'InstanceReconcile'},
'deleted_at': ('django.db.models.fields.DecimalField', [], {'null': 'True', 'max_digits': '20', 'decimal_places': '6', 'db_index': 'True'}),
u'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
'instance': ('django.db.models.fields.CharField', [], {'db_index': 'True', 'max_length': '50', 'null': 'True', 'blank': 'True'}),
'instance_type_id': ('django.db.models.fields.CharField', [], {'db_index': 'True', 'max_length': '50', 'null': 'True', 'blank': 'True'}),
'launched_at': ('django.db.models.fields.DecimalField', [], {'null': 'True', 'max_digits': '20', 'decimal_places': '6', 'db_index': 'True'}),
'os_architecture': ('django.db.models.fields.TextField', [], {'null': 'True', 'blank': 'True'}),
'os_distro': ('django.db.models.fields.TextField', [], {'null': 'True', 'blank': 'True'}),
'os_version': ('django.db.models.fields.TextField', [], {'null': 'True', 'blank': 'True'}),
'rax_options': ('django.db.models.fields.TextField', [], {'null': 'True', 'blank': 'True'}),
'row_created': ('django.db.models.fields.DateTimeField', [], {'auto_now_add': 'True', 'blank': 'True'}),
'row_updated': ('django.db.models.fields.DateTimeField', [], {'auto_now': 'True', 'blank': 'True'}),
'source': ('django.db.models.fields.CharField', [], {'db_index': 'True', 'max_length': '150', 'null': 'True', 'blank': 'True'}),
'tenant': ('django.db.models.fields.CharField', [], {'db_index': 'True', 'max_length': '50', 'null': 'True', 'blank': 'True'})
},
u'stacktach.instanceusage': {
'Meta': {'object_name': 'InstanceUsage'},
u'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
'instance': ('django.db.models.fields.CharField', [], {'db_index': 'True', 'max_length': '50', 'null': 'True', 'blank': 'True'}),
'instance_type_id': ('django.db.models.fields.CharField', [], {'db_index': 'True', 'max_length': '50', 'null': 'True', 'blank': 'True'}),
'launched_at': ('django.db.models.fields.DecimalField', [], {'null': 'True', 'max_digits': '20', 'decimal_places': '6', 'db_index': 'True'}),
'os_architecture': ('django.db.models.fields.TextField', [], {'null': 'True', 'blank': 'True'}),
'os_distro': ('django.db.models.fields.TextField', [], {'null': 'True', 'blank': 'True'}),
'os_version': ('django.db.models.fields.TextField', [], {'null': 'True', 'blank': 'True'}),
'rax_options': ('django.db.models.fields.TextField', [], {'null': 'True', 'blank': 'True'}),
'request_id': ('django.db.models.fields.CharField', [], {'db_index': 'True', 'max_length': '50', 'null': 'True', 'blank': 'True'}),
'tenant': ('django.db.models.fields.CharField', [], {'db_index': 'True', 'max_length': '50', 'null': 'True', 'blank': 'True'})
},
u'stacktach.jsonreport': {
'Meta': {'object_name': 'JsonReport'},
'created': ('django.db.models.fields.DecimalField', [], {'max_digits': '20', 'decimal_places': '6', 'db_index': 'True'}),
u'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
'json': ('django.db.models.fields.TextField', [], {}),
'name': ('django.db.models.fields.CharField', [], {'max_length': '50', 'db_index': 'True'}),
'period_end': ('django.db.models.fields.DateTimeField', [], {'db_index': 'True'}),
'period_start': ('django.db.models.fields.DateTimeField', [], {'db_index': 'True'}),
'version': ('django.db.models.fields.IntegerField', [], {'default': '1'})
},
u'stacktach.lifecycle': {
'Meta': {'object_name': 'Lifecycle'},
u'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
'instance': ('django.db.models.fields.CharField', [], {'db_index': 'True', 'max_length': '50', 'null': 'True', 'blank': 'True'}),
'last_raw': ('django.db.models.fields.related.ForeignKey', [], {'to': u"orm['stacktach.RawData']", 'null': 'True'}),
'last_state': ('django.db.models.fields.CharField', [], {'db_index': 'True', 'max_length': '50', 'null': 'True', 'blank': 'True'}),
'last_task_state': ('django.db.models.fields.CharField', [], {'db_index': 'True', 'max_length': '50', 'null': 'True', 'blank': 'True'})
},
u'stacktach.rawdata': {
'Meta': {'object_name': 'RawData'},
'deployment': ('django.db.models.fields.related.ForeignKey', [], {'to': u"orm['stacktach.Deployment']"}),
'event': ('django.db.models.fields.CharField', [], {'db_index': 'True', 'max_length': '50', 'null': 'True', 'blank': 'True'}),
'host': ('django.db.models.fields.CharField', [], {'db_index': 'True', 'max_length': '100', 'null': 'True', 'blank': 'True'}),
u'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
'image_type': ('django.db.models.fields.IntegerField', [], {'default': '0', 'null': 'True', 'db_index': 'True'}),
'instance': ('django.db.models.fields.CharField', [], {'db_index': 'True', 'max_length': '50', 'null': 'True', 'blank': 'True'}),
'json': ('django.db.models.fields.TextField', [], {}),
'old_state': ('django.db.models.fields.CharField', [], {'db_index': 'True', 'max_length': '20', 'null': 'True', 'blank': 'True'}),
'old_task': ('django.db.models.fields.CharField', [], {'db_index': 'True', 'max_length': '30', 'null': 'True', 'blank': 'True'}),
'publisher': ('django.db.models.fields.CharField', [], {'db_index': 'True', 'max_length': '100', 'null': 'True', 'blank': 'True'}),
'request_id': ('django.db.models.fields.CharField', [], {'db_index': 'True', 'max_length': '50', 'null': 'True', 'blank': 'True'}),
'routing_key': ('django.db.models.fields.CharField', [], {'db_index': 'True', 'max_length': '50', 'null': 'True', 'blank': 'True'}),
'service': ('django.db.models.fields.CharField', [], {'db_index': 'True', 'max_length': '50', 'null': 'True', 'blank': 'True'}),
'state': ('django.db.models.fields.CharField', [], {'db_index': 'True', 'max_length': '20', 'null': 'True', 'blank': 'True'}),
'task': ('django.db.models.fields.CharField', [], {'db_index': 'True', 'max_length': '30', 'null': 'True', 'blank': 'True'}),
'tenant': ('django.db.models.fields.CharField', [], {'db_index': 'True', 'max_length': '50', 'null': 'True', 'blank': 'True'}),
'when': ('django.db.models.fields.DecimalField', [], {'max_digits': '20', 'decimal_places': '6', 'db_index': 'True'})
},
u'stacktach.rawdataimagemeta': {
'Meta': {'object_name': 'RawDataImageMeta'},
u'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
'os_architecture': ('django.db.models.fields.TextField', [], {'null': 'True', 'blank': 'True'}),
'os_distro': ('django.db.models.fields.TextField', [], {'null': 'True', 'blank': 'True'}),
'os_version': ('django.db.models.fields.TextField', [], {'null': 'True', 'blank': 'True'}),
'raw': ('django.db.models.fields.related.ForeignKey', [], {'to': u"orm['stacktach.RawData']"}),
'rax_options': ('django.db.models.fields.TextField', [], {'null': 'True', 'blank': 'True'})
},
u'stacktach.requesttracker': {
'Meta': {'object_name': 'RequestTracker'},
'completed': ('django.db.models.fields.BooleanField', [], {'default': 'False', 'db_index': 'True'}),
'duration': ('django.db.models.fields.DecimalField', [], {'max_digits': '20', 'decimal_places': '6', 'db_index': 'True'}),
u'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
'last_timing': ('django.db.models.fields.related.ForeignKey', [], {'to': u"orm['stacktach.Timing']", 'null': 'True'}),
'lifecycle': ('django.db.models.fields.related.ForeignKey', [], {'to': u"orm['stacktach.Lifecycle']"}),
'request_id': ('django.db.models.fields.CharField', [], {'max_length': '50', 'db_index': 'True'}),
'start': ('django.db.models.fields.DecimalField', [], {'max_digits': '20', 'decimal_places': '6', 'db_index': 'True'})
},
u'stacktach.timing': {
'Meta': {'object_name': 'Timing'},
'diff': ('django.db.models.fields.DecimalField', [], {'null': 'True', 'max_digits': '20', 'decimal_places': '6', 'db_index': 'True'}),
'end_raw': ('django.db.models.fields.related.ForeignKey', [], {'related_name': "'+'", 'null': 'True', 'to': u"orm['stacktach.RawData']"}),
'end_when': ('django.db.models.fields.DecimalField', [], {'null': 'True', 'max_digits': '20', 'decimal_places': '6'}),
u'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
'lifecycle': ('django.db.models.fields.related.ForeignKey', [], {'to': u"orm['stacktach.Lifecycle']"}),
'name': ('django.db.models.fields.CharField', [], {'max_length': '50', 'db_index': 'True'}),
'start_raw': ('django.db.models.fields.related.ForeignKey', [], {'related_name': "'+'", 'null': 'True', 'to': u"orm['stacktach.RawData']"}),
'start_when': ('django.db.models.fields.DecimalField', [], {'null': 'True', 'max_digits': '20', 'decimal_places': '6'})
}
}
complete_apps = ['stacktach']

13
stacktach/models.py
View File

@@ -108,6 +108,7 @@ class InstanceUsage(models.Model):
raw = raws[0]
return raw.deployment
class InstanceDeletes(models.Model):
instance = models.CharField(max_length=50, null=True,
blank=True, db_index=True)
@@ -117,6 +118,9 @@ class InstanceDeletes(models.Model):
decimal_places=6, db_index=True)
raw = models.ForeignKey(RawData, null=True)
def deployment(self):
return self.raw.deployment
class InstanceReconcile(models.Model):
row_created = models.DateTimeField(auto_now_add=True)
@@ -131,6 +135,12 @@ class InstanceReconcile(models.Model):
null=True,
blank=True,
db_index=True)
tenant = models.CharField(max_length=50, null=True, blank=True,
db_index=True)
os_architecture = models.TextField(null=True, blank=True)
os_distro = models.TextField(null=True, blank=True)
os_version = models.TextField(null=True, blank=True)
rax_options = models.TextField(null=True, blank=True)
source = models.CharField(max_length=150, null=True,
blank=True, db_index=True)
@@ -181,6 +191,9 @@ class InstanceExists(models.Model):
os_version = models.TextField(null=True, blank=True)
rax_options = models.TextField(null=True, blank=True)
def deployment(self):
return self.raw.deployment
class Timing(models.Model):
"""Each Timing record corresponds to a .start/.end event pair

70
stacktach/reconciler/__init__.py
View File

@@ -23,6 +23,7 @@ import json
from stacktach import models
from stacktach.reconciler import exceptions
from stacktach.reconciler import nova
from stacktach import stacklog
DEFAULT_CLIENT = nova.JSONBridgeClient
@@ -61,8 +62,8 @@ class Reconciler(object):
with open(config['region_mapping_loc']) as f:
return json.load(f)
def _region_for_launch(self, launch):
deployment = launch.deployment()
def _region_for_usage(self, usage):
deployment = usage.deployment()
if deployment:
deployment_name = str(deployment.name)
if deployment_name in self.region_mapping:
@@ -72,23 +73,53 @@ class Reconciler(object):
else:
return False
def _reconcile_instance(self, launch, src,
launched_at=None, deleted_at=None,
instance_type_id=None):
def _reconcile_instance(self, usage, src, deleted_at=None):
values = {
'instance': launch.instance,
'launched_at': (launched_at or launch.launched_at),
'instance': usage.instance,
'launched_at': usage.launched_at,
'deleted_at': deleted_at,
'instance_type_id': (instance_type_id or launch.instance_type_id),
'instance_type_id': usage.instance_type_id,
'source': 'reconciler:%s' % src,
'tenant': usage.tenant,
'os_architecture': usage.os_architecture,
'os_distro': usage.os_distro,
'os_version': usage.os_version,
'rax_options': usage.rax_options,
}
models.InstanceReconcile(**values).save()
def _fields_match(self, exists, instance):
match_code = 0
if (exists.launched_at != instance['launched_at'] or
exists.instance_type_id != instance['instance_type_id'] or
exists.tenant != instance['tenant'] or
exists.os_architecture != instance['os_architecture'] or
exists.os_distro != instance['os_distro'] or
exists.os_version != instance['os_version'] or
exists.rax_options != instance['rax_options']):
match_code = 1
if exists.deleted_at is not None:
# Exists says deleted
if (instance['deleted'] and
exists.deleted_at != instance['deleted_at']):
# Nova says deleted, but times don't match
match_code = 2
elif not instance['deleted']:
# Nova says not deleted
match_code = 3
elif exists.deleted_at is None and instance['deleted']:
# Exists says not deleted, but Nova says deleted
match_code = 4
return match_code
def missing_exists_for_instance(self, launched_id,
period_beginning):
reconciled = False
launch = models.InstanceUsage.objects.get(id=launched_id)
region = self._region_for_launch(launch)
region = self._region_for_usage(launch)
try:
instance = self.client.get_instance(region, launch.instance)
if instance['deleted'] and instance['deleted_at'] is not None:
@@ -102,6 +133,25 @@ class Reconciler(object):
deleted_at=instance['deleted_at'])
reconciled = True
except exceptions.NotFound:
reconciled = False
stacklog.info("Couldn't find instance for launch %s" % launched_id)
return reconciled
def failed_validation(self, exists):
reconciled = False
region = self._region_for_usage(exists)
try:
instance = self.client.get_instance(region, exists.instance)
match_code = self._fields_match(exists, instance)
if match_code == 0:
self._reconcile_instance(exists, self.client.src_str,
deleted_at=exists.deleted_at)
reconciled = True
else:
msg = "Exists %s failed reconciliation with code %s"
msg %= (exists.id, match_code)
stacklog.info(msg)
except exceptions.NotFound:
stacklog.info("Couldn't find instance for exists %s" % exists.id)
return reconciled

56
stacktach/reconciler/nova.py
View File

@@ -1,14 +1,34 @@
import json
import requests
from stacktach import utils as stackutils
from stacktach.reconciler import exceptions
from stacktach.reconciler.utils import empty_reconciler_instance
GET_INSTANCE_QUERY = "SELECT * FROM instances where uuid ='%s';"
METADATA_MAPPING = {
'image_org.openstack__1__architecture': 'os_architecture',
'image_org.openstack__1__os_distro': 'os_distro',
'image_org.openstack__1__os_version': 'os_version',
'image_com.rackspace__1__options': 'rax_options',
}
METADATA_FIELDS = ["'%s'" % x for x in METADATA_MAPPING.keys()]
METADATA_FIELDS = ','.join(METADATA_FIELDS)
GET_INSTANCE_SYSTEM_METADATA = """
SELECT * FROM instance_system_metadata
WHERE instance_uuid = '%s' AND
deleted = 0 AND `key` IN (%s);
"""
GET_INSTANCE_SYSTEM_METADATA %= ('%s', METADATA_FIELDS)
def _json(result):
if callable(result.json):
return result.json()
else:
return result.json
class JSONBridgeClient(object):
src_str = 'json_bridge:nova_db'
@@ -22,14 +42,15 @@ class JSONBridgeClient(object):
def _do_query(self, region, query):
data = {'sql': query}
credentials = (self.config['username'], self.config['password'])
return requests.post(self._url_for_region(region), data,
verify=False, auth=credentials).json()
return _json(requests.post(self._url_for_region(region), data,
verify=False, auth=credentials))
def _to_reconciler_instance(self, instance):
def _to_reconciler_instance(self, instance, metadata=None):
r_instance = empty_reconciler_instance()
r_instance.update({
'id': instance['uuid'],
'instance_type_id': instance['instance_type_id'],
'tenant': instance['project_id'],
'instance_type_id': str(instance['instance_type_id']),
})
if instance['launched_at'] is not None:
@@ -43,12 +64,27 @@ class JSONBridgeClient(object):
if instance['deleted'] != 0:
r_instance['deleted'] = True
if metadata is not None:
r_instance.update(metadata)
return r_instance
def get_instance(self, region, uuid):
def _get_instance_meta(self, region, uuid):
results = self._do_query(region, GET_INSTANCE_SYSTEM_METADATA % uuid)
metadata = {}
for result in results['result']:
key = result['key']
if key in METADATA_MAPPING:
metadata[METADATA_MAPPING[key]] = result['value']
return metadata
def get_instance(self, region, uuid, get_metadata=False):
results = self._do_query(region, GET_INSTANCE_QUERY % uuid)['result']
if len(results) > 0:
return self._to_reconciler_instance(results[0])
metadata = None
if get_metadata:
metadata = self._get_instance_meta(region, uuid)
return self._to_reconciler_instance(results[0], metadata=metadata)
else:
msg = "Couldn't find instance (%s) using JSON Bridge in region (%s)"
raise exceptions.NotFound(msg % (uuid, region))
raise exceptions.NotFound(msg % (uuid, region))

7
stacktach/reconciler/utils.py
View File

@@ -1,9 +1,14 @@
def empty_reconciler_instance():
r_instance = {
'id': None,
'tenant': None,
'launched_at': None,
'deleted': False,
'deleted_at': None,
'instance_type_ud': None
'instance_type_id': None,
'os_architecture': '',
'os_distro': '',
'os_version': '',
'rax_options': '',
}
return r_instance

8
stacktach/stacklog.py
View File

@@ -73,4 +73,10 @@ def warn(msg, name=None):
def error(msg, name=None):
if name is None:
name = default_logger_name
get_logger(name=name).error(msg)
get_logger(name=name).error(msg)
def info(msg, name=None):
if name is None:
name = default_logger_name
get_logger(name=name).info(msg)

261
tests/unit/test_reconciler.py
View File

@@ -29,14 +29,21 @@ from stacktach import reconciler
from stacktach import utils as stackutils
from stacktach.reconciler import exceptions
from stacktach.reconciler import nova
from stacktach.reconciler import utils as rec_utils
from tests.unit import utils
from tests.unit.utils import INSTANCE_ID_1
from tests.unit.utils import TENANT_ID_1
region_mapping = {
'RegionOne.prod.cell1': 'RegionOne',
'RegionTwo.prod.cell1': 'RegionTwo',
}
DEFAULT_OS_ARCH = 'os_arch'
DEFAULT_OS_DISTRO = 'os_dist'
DEFAULT_OS_VERSION = "1.1"
DEFAULT_RAX_OPTIONS = "rax_ops"
class ReconcilerTestCase(unittest.TestCase):
def setUp(self):
@@ -75,16 +82,51 @@ class ReconcilerTestCase(unittest.TestCase):
def tearDown(self):
self.mox.UnsetStubs()
def _fake_usage(self, is_exists=False, is_deleted=False,
mock_deployment=False):
usage = self.mox.CreateMockAnything()
usage.id = 1
beginning_d = utils.decimal_utc()
usage.instance = INSTANCE_ID_1
launched_at = beginning_d - (60*60)
usage.launched_at = launched_at
usage.instance_type_id = 1
usage.tenant = TENANT_ID_1
usage.os_architecture = DEFAULT_OS_ARCH
usage.os_distro = DEFAULT_OS_DISTRO
usage.os_version = DEFAULT_OS_VERSION
usage.rax_options = DEFAULT_RAX_OPTIONS
if is_exists:
usage.deleted_at = None
if is_deleted:
usage.deleted_at = beginning_d
if mock_deployment:
deployment = self.mox.CreateMockAnything()
deployment.name = 'RegionOne.prod.cell1'
usage.deployment().AndReturn(deployment)
return usage
def _fake_reconciler_instance(self, uuid=INSTANCE_ID_1, launched_at=None,
deleted_at=None, deleted=False,
instance_type_id=1):
return {
instance_type_id=1, tenant=TENANT_ID_1,
os_arch=DEFAULT_OS_ARCH,
os_distro=DEFAULT_OS_DISTRO,
os_verison=DEFAULT_OS_VERSION,
rax_options=DEFAULT_RAX_OPTIONS):
instance = rec_utils.empty_reconciler_instance()
instance.update({
'id': uuid,
'launched_at': launched_at,
'deleted_at': deleted_at,
'deleted': deleted,
'instance_type_id': instance_type_id
}
'instance_type_id': instance_type_id,
'tenant': tenant,
'os_architecture': os_arch,
'os_distro': os_distro,
'os_version': os_verison,
'rax_options': rax_options,
})
return instance
def test_load_client_json_bridge(self):
mock_config = self.mox.CreateMockAnything()
@@ -116,7 +158,7 @@ class ReconcilerTestCase(unittest.TestCase):
deployment.name = 'RegionOne.prod.cell1'
launch.deployment().AndReturn(deployment)
self.mox.ReplayAll()
region = self.reconciler._region_for_launch(launch)
region = self.reconciler._region_for_usage(launch)
self.assertEqual('RegionOne', region)
self.mox.VerifyAll()
@@ -126,7 +168,7 @@ class ReconcilerTestCase(unittest.TestCase):
deployment.name = 'RegionOne.prod.cell2'
launch.deployment().AndReturn(deployment)
self.mox.ReplayAll()
region = self.reconciler._region_for_launch(launch)
region = self.reconciler._region_for_usage(launch)
self.assertFalse(region)
self.mox.VerifyAll()
@@ -134,22 +176,16 @@ class ReconcilerTestCase(unittest.TestCase):
launch = self.mox.CreateMockAnything()
launch.deployment()
self.mox.ReplayAll()
region = self.reconciler._region_for_launch(launch)
region = self.reconciler._region_for_usage(launch)
self.assertFalse(region)
self.mox.VerifyAll()
def test_missing_exists_for_instance(self):
launch_id = 1
beginning_d = utils.decimal_utc()
launch = self.mox.CreateMockAnything()
launch.instance = INSTANCE_ID_1
launch.launched_at = beginning_d - (60*60)
launch.instance_type_id = 1
models.InstanceUsage.objects.get(id=launch_id).AndReturn(launch)
deployment = self.mox.CreateMockAnything()
launch.deployment().AndReturn(deployment)
deployment.name = 'RegionOne.prod.cell1'
deleted_at = beginning_d - (60*30)
launch = self._fake_usage(mock_deployment=True)
launched_at = launch.launched_at
deleted_at = launched_at + (60*30)
period_beginning = deleted_at + 1
models.InstanceUsage.objects.get(id=launch.id).AndReturn(launch)
rec_inst = self._fake_reconciler_instance(deleted=True,
deleted_at=deleted_at)
self.client.get_instance('RegionOne', INSTANCE_ID_1).AndReturn(rec_inst)
@@ -158,14 +194,19 @@ class ReconcilerTestCase(unittest.TestCase):
'launched_at': launch.launched_at,
'deleted_at': deleted_at,
'instance_type_id': launch.instance_type_id,
'source': 'reconciler:mocked_client'
'source': 'reconciler:mocked_client',
'tenant': TENANT_ID_1,
'os_architecture': DEFAULT_OS_ARCH,
'os_distro': DEFAULT_OS_DISTRO,
'os_version': DEFAULT_OS_VERSION,
'rax_options': DEFAULT_RAX_OPTIONS,
}
result = self.mox.CreateMockAnything()
models.InstanceReconcile(**reconcile_vals).AndReturn(result)
result.save()
self.mox.ReplayAll()
result = self.reconciler.missing_exists_for_instance(launch_id,
beginning_d)
result = self.reconciler.missing_exists_for_instance(launch.id,
period_beginning)
self.assertTrue(result)
self.mox.VerifyAll()
@@ -188,6 +229,176 @@ class ReconcilerTestCase(unittest.TestCase):
self.assertFalse(result)
self.mox.VerifyAll()
def test_failed_validation(self):
exists = self._fake_usage(is_exists=True, mock_deployment=True)
launched_at = exists.launched_at
rec_inst = self._fake_reconciler_instance(launched_at=launched_at)
self.client.get_instance('RegionOne', INSTANCE_ID_1).AndReturn(rec_inst)
reconcile_vals = {
'instance': exists.instance,
'launched_at': exists.launched_at,
'deleted_at': exists.deleted_at,
'instance_type_id': exists.instance_type_id,
'source': 'reconciler:mocked_client',
'tenant': TENANT_ID_1,
'os_architecture': DEFAULT_OS_ARCH,
'os_distro': DEFAULT_OS_DISTRO,
'os_version': DEFAULT_OS_VERSION,
'rax_options': DEFAULT_RAX_OPTIONS,
}
result = self.mox.CreateMockAnything()
models.InstanceReconcile(**reconcile_vals).AndReturn(result)
result.save()
self.mox.ReplayAll()
result = self.reconciler.failed_validation(exists)
self.assertTrue(result)
self.mox.VerifyAll()
def test_failed_validation_deleted(self):
exists = self._fake_usage(is_exists=True, is_deleted=True,
mock_deployment=True)
launched_at = exists.launched_at
deleted_at = exists.deleted_at
rec_inst = self._fake_reconciler_instance(launched_at=launched_at,
deleted=True,
deleted_at=deleted_at)
self.client.get_instance('RegionOne', INSTANCE_ID_1).AndReturn(rec_inst)
reconcile_vals = {
'instance': exists.instance,
'launched_at': exists.launched_at,
'deleted_at': exists.deleted_at,
'instance_type_id': exists.instance_type_id,
'source': 'reconciler:mocked_client',
'tenant': TENANT_ID_1,
'os_architecture': DEFAULT_OS_ARCH,
'os_distro': DEFAULT_OS_DISTRO,
'os_version': DEFAULT_OS_VERSION,
'rax_options': DEFAULT_RAX_OPTIONS,
}
result = self.mox.CreateMockAnything()
models.InstanceReconcile(**reconcile_vals).AndReturn(result)
result.save()
self.mox.ReplayAll()
result = self.reconciler.failed_validation(exists)
self.assertTrue(result)
self.mox.VerifyAll()
def test_failed_validation_deleted_not_matching(self):
beginning_d = utils.decimal_utc()
exists = self.mox.CreateMockAnything()
exists.instance = INSTANCE_ID_1
launched_at = beginning_d - (60*60)
exists.launched_at = launched_at
exists.instance_type_id = 1
exists.deleted_at = beginning_d
deployment = self.mox.CreateMockAnything()
exists.deployment().AndReturn(deployment)
deployment.name = 'RegionOne.prod.cell1'
rec_inst = self._fake_reconciler_instance(launched_at=launched_at,
deleted=True,
deleted_at=beginning_d+1)
self.client.get_instance('RegionOne', INSTANCE_ID_1).AndReturn(rec_inst)
self.mox.ReplayAll()
result = self.reconciler.failed_validation(exists)
self.assertFalse(result)
self.mox.VerifyAll()
def test_failed_validation_deleted_not_deleted_from_client(self):
beginning_d = utils.decimal_utc()
exists = self.mox.CreateMockAnything()
exists.instance = INSTANCE_ID_1
launched_at = beginning_d - (60*60)
exists.launched_at = launched_at
exists.instance_type_id = 1
exists.deleted_at = beginning_d
deployment = self.mox.CreateMockAnything()
exists.deployment().AndReturn(deployment)
deployment.name = 'RegionOne.prod.cell1'
rec_inst = self._fake_reconciler_instance(launched_at=launched_at)
self.client.get_instance('RegionOne', INSTANCE_ID_1).AndReturn(rec_inst)
self.mox.ReplayAll()
result = self.reconciler.failed_validation(exists)
self.assertFalse(result)
self.mox.VerifyAll()
def test_failed_validation_not_found(self):
beginning_d = utils.decimal_utc()
exists = self.mox.CreateMockAnything()
exists.instance = INSTANCE_ID_1
launched_at = beginning_d - (60*60)
exists.launched_at = launched_at
exists.instance_type_id = 1
exists.deleted_at = None
deployment = self.mox.CreateMockAnything()
exists.deployment().AndReturn(deployment)
deployment.name = 'RegionOne.prod.cell1'
ex = exceptions.NotFound()
self.client.get_instance('RegionOne', INSTANCE_ID_1).AndRaise(ex)
self.mox.ReplayAll()
result = self.reconciler.failed_validation(exists)
self.assertFalse(result)
self.mox.VerifyAll()
def test_fields_match(self):
exists = self._fake_usage(is_exists=True)
kwargs = {'launched_at': exists.launched_at}
instance = self._fake_reconciler_instance(**kwargs)
self.mox.ReplayAll()
match_code = self.reconciler._fields_match(exists, instance)
self.assertEqual(match_code, 0)
self.mox.VerifyAll()
def test_fields_match_field_with_deleted(self):
exists = self._fake_usage(is_exists=True, is_deleted=True)
kwargs = {'launched_at': exists.launched_at,
'deleted': True,
'deleted_at': exists.deleted_at}
instance = self._fake_reconciler_instance(**kwargs)
self.mox.ReplayAll()
match_code = self.reconciler._fields_match(exists, instance)
self.assertEqual(match_code, 0)
self.mox.VerifyAll()
def test_fields_match_field_miss_match(self):
exists = self._fake_usage(is_exists=True)
kwargs = {'launched_at': exists.launched_at + 1}
instance = self._fake_reconciler_instance(**kwargs)
self.mox.ReplayAll()
match_code = self.reconciler._fields_match(exists, instance)
self.assertEqual(match_code, 1)
self.mox.VerifyAll()
def test_fields_match_field_with_deleted_miss_match(self):
exists = self._fake_usage(is_exists=True, is_deleted=True)
kwargs = {'launched_at': exists.launched_at,
'deleted': True,
'deleted_at': exists.deleted_at+1}
instance = self._fake_reconciler_instance(**kwargs)
self.mox.ReplayAll()
match_code = self.reconciler._fields_match(exists, instance)
self.assertEqual(match_code, 2)
self.mox.VerifyAll()
def test_fields_match_field_not_deleted_in_nova(self):
exists = self._fake_usage(is_exists=True, is_deleted=True)
kwargs = {'launched_at': exists.launched_at}
instance = self._fake_reconciler_instance(**kwargs)
self.mox.ReplayAll()
match_code = self.reconciler._fields_match(exists, instance)
self.assertEqual(match_code, 3)
self.mox.VerifyAll()
def test_fields_match_field_not_deleted_in_exists(self):
exists = self._fake_usage(is_exists=True)
kwargs = {'launched_at': exists.launched_at,
'deleted': True,
'deleted_at': exists.launched_at + 1}
instance = self._fake_reconciler_instance(**kwargs)
self.mox.ReplayAll()
match_code = self.reconciler._fields_match(exists, instance)
self.assertEqual(match_code, 4)
self.mox.VerifyAll()
json_bridge_config = {
'url': 'http://json_bridge.example.com/query/',
@@ -219,13 +430,15 @@ class NovaJSONBridgeClientTestCase(unittest.TestCase):
response.json().AndReturn(result)
def _fake_instance(self, uuid=INSTANCE_ID_1, launched_at=None,
terminated_at=None, deleted=0, instance_type_id=1):
terminated_at=None, deleted=0, instance_type_id=1,
project_id=TENANT_ID_1):
return {
'uuid': uuid,
'launched_at': launched_at,
'terminated_at': terminated_at,
'deleted': deleted,
'instance_type_id': instance_type_id
'instance_type_id': instance_type_id,
'project_id': project_id
}
def test_get_instance(self):
@@ -241,7 +454,7 @@ class NovaJSONBridgeClientTestCase(unittest.TestCase):
instance = self.client.get_instance('RegionOne', INSTANCE_ID_1)
self.assertIsNotNone(instance)
self.assertEqual(instance['id'], INSTANCE_ID_1)
self.assertEqual(instance['instance_type_id'], 1)
self.assertEqual(instance['instance_type_id'], '1')
launched_at_dec = stackutils.str_time_to_unix(launched_at)
self.assertEqual(instance['launched_at'], launched_at_dec)
terminated_at_dec = stackutils.str_time_to_unix(terminated_at)

425
tests/unit/test_verifier_db.py
View File

@@ -21,9 +21,9 @@
import datetime
import decimal
import json
import time
import unittest
import uuid
import multiprocessing
import kombu.common
import kombu.entity
@@ -90,7 +90,10 @@ class VerifierTestCase(unittest.TestCase):
"enable_notifications": False,
}
self.pool = self.mox.CreateMockAnything()
self.verifier = dbverifier.Verifier(self.config, pool=self.pool)
self.reconciler = self.mox.CreateMockAnything()
self.verifier = dbverifier.Verifier(self.config,
pool=self.pool,
rec=self.reconciler)
self.config_notif = {
"tick_time": 30,
@@ -109,8 +112,10 @@ class VerifierTestCase(unittest.TestCase):
}
}
self.pool_notif = self.mox.CreateMockAnything()
self.reconciler_notif = self.mox.CreateMockAnything()
self.verifier_notif = dbverifier.Verifier(self.config_notif,
pool=self.pool_notif)
pool=self.pool_notif,
rec=self.reconciler)
def tearDown(self):
self.mox.UnsetStubs()
@@ -476,6 +481,157 @@ class VerifierTestCase(unittest.TestCase):
self.assertEqual(fm.actual, decimal.Decimal('6.1'))
self.mox.VerifyAll()
def test_verify_with_reconciled_data(self):
exists = self.mox.CreateMockAnything()
exists.instance = INSTANCE_ID_1
launched_at = decimal.Decimal('1.1')
exists.launched_at = launched_at
results = self.mox.CreateMockAnything()
models.InstanceReconcile.objects.filter(instance=INSTANCE_ID_1)\
.AndReturn(results)
results.count().AndReturn(1)
launched_min = decimal.Decimal('1')
launched_max = decimal.Decimal('1.999999')
filter = {
'instance': INSTANCE_ID_1,
'launched_at__gte': launched_min,
'launched_at__lte': launched_max
}
recs = self.mox.CreateMockAnything()
models.InstanceReconcile.objects.filter(**filter).AndReturn(recs)
recs.count().AndReturn(1)
reconcile = self.mox.CreateMockAnything()
reconcile.deleted_at = None
recs[0].AndReturn(reconcile)
self.mox.StubOutWithMock(dbverifier, '_verify_for_launch')
dbverifier._verify_for_launch(exists, launch=reconcile,
launch_type='InstanceReconcile')
self.mox.StubOutWithMock(dbverifier, '_verify_for_delete')
dbverifier._verify_for_delete(exists, delete=None,
delete_type='InstanceReconcile')
self.mox.ReplayAll()
dbverifier._verify_with_reconciled_data(exists)
self.mox.VerifyAll()
def test_verify_with_reconciled_data_deleted(self):
exists = self.mox.CreateMockAnything()
exists.instance = INSTANCE_ID_1
launched_at = decimal.Decimal('1.1')
deleted_at = decimal.Decimal('2.1')
exists.launched_at = launched_at
exists.deleted_at = deleted_at
results = self.mox.CreateMockAnything()
models.InstanceReconcile.objects.filter(instance=INSTANCE_ID_1)\
.AndReturn(results)
results.count().AndReturn(1)
launched_min = decimal.Decimal('1')
launched_max = decimal.Decimal('1.999999')
filter = {
'instance': INSTANCE_ID_1,
'launched_at__gte': launched_min,
'launched_at__lte': launched_max
}
recs = self.mox.CreateMockAnything()
models.InstanceReconcile.objects.filter(**filter).AndReturn(recs)
recs.count().AndReturn(1)
reconcile = self.mox.CreateMockAnything()
reconcile.deleted_at = deleted_at
recs[0].AndReturn(reconcile)
self.mox.StubOutWithMock(dbverifier, '_verify_for_launch')
dbverifier._verify_for_launch(exists, launch=reconcile,
launch_type='InstanceReconcile')
self.mox.StubOutWithMock(dbverifier, '_verify_for_delete')
dbverifier._verify_for_delete(exists, delete=reconcile,
delete_type='InstanceReconcile')
self.mox.ReplayAll()
dbverifier._verify_with_reconciled_data(exists)
self.mox.VerifyAll()
def test_verify_with_reconciled_data_not_launched(self):
exists = self.mox.CreateMockAnything()
exists.instance = INSTANCE_ID_1
exists.launched_at = None
self.mox.ReplayAll()
with self.assertRaises(VerificationException) as cm:
dbverifier._verify_with_reconciled_data(exists)
exception = cm.exception
self.assertEquals(exception.reason, 'Exists without a launched_at')
self.mox.VerifyAll()
def test_verify_with_reconciled_data_ambiguous_results(self):
exists = self.mox.CreateMockAnything()
exists.instance = INSTANCE_ID_1
launched_at = decimal.Decimal('1.1')
deleted_at = decimal.Decimal('2.1')
exists.launched_at = launched_at
exists.deleted_at = deleted_at
results = self.mox.CreateMockAnything()
models.InstanceReconcile.objects.filter(instance=INSTANCE_ID_1)\
.AndReturn(results)
results.count().AndReturn(1)
launched_min = decimal.Decimal('1')
launched_max = decimal.Decimal('1.999999')
filter = {
'instance': INSTANCE_ID_1,
'launched_at__gte': launched_min,
'launched_at__lte': launched_max
}
recs = self.mox.CreateMockAnything()
models.InstanceReconcile.objects.filter(**filter).AndReturn(recs)
recs.count().AndReturn(2)
self.mox.ReplayAll()
with self.assertRaises(AmbiguousResults) as cm:
dbverifier._verify_with_reconciled_data(exists)
exception = cm.exception
self.assertEquals(exception.object_type, 'InstanceReconcile')
self.mox.VerifyAll()
def test_verify_with_reconciled_data_instance_not_found(self):
exists = self.mox.CreateMockAnything()
exists.instance = INSTANCE_ID_1
launched_at = decimal.Decimal('1.1')
deleted_at = decimal.Decimal('2.1')
exists.launched_at = launched_at
exists.deleted_at = deleted_at
results = self.mox.CreateMockAnything()
models.InstanceReconcile.objects.filter(instance=INSTANCE_ID_1)\
.AndReturn(results)
results.count().AndReturn(0)
self.mox.ReplayAll()
with self.assertRaises(NotFound) as cm:
dbverifier._verify_with_reconciled_data(exists)
exception = cm.exception
self.assertEquals(exception.object_type, 'InstanceReconcile')
self.mox.VerifyAll()
def test_verify_with_reconciled_data_reconcile_not_found(self):
exists = self.mox.CreateMockAnything()
exists.instance = INSTANCE_ID_1
launched_at = decimal.Decimal('1.1')
deleted_at = decimal.Decimal('2.1')
exists.launched_at = launched_at
exists.deleted_at = deleted_at
results = self.mox.CreateMockAnything()
models.InstanceReconcile.objects.filter(instance=INSTANCE_ID_1)\
.AndReturn(results)
results.count().AndReturn(1)
launched_min = decimal.Decimal('1')
launched_max = decimal.Decimal('1.999999')
filter = {
'instance': INSTANCE_ID_1,
'launched_at__gte': launched_min,
'launched_at__lte': launched_max
}
recs = self.mox.CreateMockAnything()
models.InstanceReconcile.objects.filter(**filter).AndReturn(recs)
recs.count().AndReturn(0)
self.mox.ReplayAll()
with self.assertRaises(NotFound) as cm:
dbverifier._verify_with_reconciled_data(exists)
exception = cm.exception
self.assertEquals(exception.object_type, 'InstanceReconcile')
self.mox.VerifyAll()
def test_verify_pass(self):
exist = self.mox.CreateMockAnything()
exist.launched_at = decimal.Decimal('1.1')
@@ -501,13 +657,30 @@ class VerifierTestCase(unittest.TestCase):
dbverifier._mark_exist_failed(exist,
reason="Exists without a launched_at")
self.mox.StubOutWithMock(dbverifier, '_verify_with_reconciled_data')
dbverifier._verify_with_reconciled_data(exist, mox.IgnoreArg())\
dbverifier._verify_with_reconciled_data(exist)\
.AndRaise(NotFound('InstanceReconcile', {}))
self.mox.ReplayAll()
result, exists = dbverifier._verify(exist)
self.assertFalse(result)
self.mox.VerifyAll()
def test_verify_fails_reconciled_verify_uses_second_exception(self):
exist = self.mox.CreateMockAnything()
self.mox.StubOutWithMock(dbverifier, '_verify_for_launch')
ex1 = VerificationException('test1')
dbverifier._verify_for_launch(exist).AndRaise(ex1)
self.mox.StubOutWithMock(dbverifier, '_verify_for_delete')
self.mox.StubOutWithMock(dbverifier, '_mark_exist_failed')
self.mox.StubOutWithMock(dbverifier, '_mark_exist_verified')
self.mox.StubOutWithMock(dbverifier, '_verify_with_reconciled_data')
dbverifier._verify_with_reconciled_data(exist)\
.AndRaise(VerificationException('test2'))
dbverifier._mark_exist_failed(exist, reason='test2')
self.mox.ReplayAll()
result, exists = dbverifier._verify(exist)
self.assertFalse(result)
self.mox.VerifyAll()
def test_verify_launch_fail(self):
exist = self.mox.CreateMockAnything()
exist.launched_at = decimal.Decimal('1.1')
@@ -518,7 +691,7 @@ class VerifierTestCase(unittest.TestCase):
verify_exception = VerificationException('test')
dbverifier._verify_for_launch(exist).AndRaise(verify_exception)
self.mox.StubOutWithMock(dbverifier, '_verify_with_reconciled_data')
dbverifier._verify_with_reconciled_data(exist, verify_exception)\
dbverifier._verify_with_reconciled_data(exist)\
.AndRaise(NotFound('InstanceReconcile', {}))
dbverifier._mark_exist_failed(exist, reason='test')
self.mox.ReplayAll()
@@ -536,8 +709,8 @@ class VerifierTestCase(unittest.TestCase):
verify_exception = VerificationException('test')
dbverifier._verify_for_launch(exist).AndRaise(verify_exception)
self.mox.StubOutWithMock(dbverifier, '_verify_with_reconciled_data')
dbverifier._verify_with_reconciled_data(exist, verify_exception)
dbverifier._mark_exist_verified(exist)
dbverifier._verify_with_reconciled_data(exist)
dbverifier._mark_exist_verified(exist, reconciled=True)
self.mox.ReplayAll()
result, exists = dbverifier._verify(exist)
self.assertTrue(result)
@@ -553,7 +726,7 @@ class VerifierTestCase(unittest.TestCase):
verify_exception = VerificationException('test')
dbverifier._verify_for_launch(exist).AndRaise(verify_exception)
self.mox.StubOutWithMock(dbverifier, '_verify_with_reconciled_data')
dbverifier._verify_with_reconciled_data(exist, verify_exception)\
dbverifier._verify_with_reconciled_data(exist)\
.AndRaise(Exception())
dbverifier._mark_exist_failed(exist, reason='Exception')
self.mox.ReplayAll()
@@ -572,7 +745,7 @@ class VerifierTestCase(unittest.TestCase):
dbverifier._verify_for_launch(exist)
dbverifier._verify_for_delete(exist).AndRaise(verify_exception)
self.mox.StubOutWithMock(dbverifier, '_verify_with_reconciled_data')
dbverifier._verify_with_reconciled_data(exist, verify_exception)\
dbverifier._verify_with_reconciled_data(exist)\
.AndRaise(NotFound('InstanceReconcile', {}))
dbverifier._mark_exist_failed(exist, reason='test')
self.mox.ReplayAll()
@@ -638,9 +811,103 @@ class VerifierTestCase(unittest.TestCase):
self.assertEqual(exist2.status, 'verifying')
self.mox.VerifyAll()
def test_clean_results_full(self):
self.verifier.reconcile = True
result_not_ready = self.mox.CreateMockAnything()
result_not_ready.ready().AndReturn(False)
result_unsuccessful = self.mox.CreateMockAnything()
result_unsuccessful.ready().AndReturn(True)
result_unsuccessful.successful().AndReturn(False)
result_successful = self.mox.CreateMockAnything()
result_successful.ready().AndReturn(True)
result_successful.successful().AndReturn(True)
result_successful.get().AndReturn((True, None))
result_failed_verification = self.mox.CreateMockAnything()
result_failed_verification.ready().AndReturn(True)
result_failed_verification.successful().AndReturn(True)
failed_exists = self.mox.CreateMockAnything()
result_failed_verification.get().AndReturn((False, failed_exists))
self.verifier.results = [result_not_ready,
result_unsuccessful,
result_successful,
result_failed_verification]
self.mox.ReplayAll()
(result_count, success_count, errored) = self.verifier.clean_results()
self.assertEqual(result_count, 1)
self.assertEqual(success_count, 2)
self.assertEqual(errored, 1)
self.assertEqual(len(self.verifier.results), 1)
self.assertEqual(self.verifier.results[0], result_not_ready)
self.assertEqual(len(self.verifier.failed), 1)
self.assertEqual(self.verifier.failed[0], result_failed_verification)
self.mox.VerifyAll()
def test_clean_results_pending(self):
self.verifier.reconcile = True
result_not_ready = self.mox.CreateMockAnything()
result_not_ready.ready().AndReturn(False)
self.verifier.results = [result_not_ready]
self.mox.ReplayAll()
(result_count, success_count, errored) = self.verifier.clean_results()
self.assertEqual(result_count, 1)
self.assertEqual(success_count, 0)
self.assertEqual(errored, 0)
self.assertEqual(len(self.verifier.results), 1)
self.assertEqual(self.verifier.results[0], result_not_ready)
self.assertEqual(len(self.verifier.failed), 0)
self.mox.VerifyAll()
def test_clean_results_successful(self):
self.verifier.reconcile = True
result_successful = self.mox.CreateMockAnything()
result_successful.ready().AndReturn(True)
result_successful.successful().AndReturn(True)
result_successful.get().AndReturn((True, None))
self.verifier.results = [result_successful]
self.mox.ReplayAll()
(result_count, success_count, errored) = self.verifier.clean_results()
self.assertEqual(result_count, 0)
self.assertEqual(success_count, 1)
self.assertEqual(errored, 0)
self.assertEqual(len(self.verifier.results), 0)
self.assertEqual(len(self.verifier.failed), 0)
self.mox.VerifyAll()
def test_clean_results_unsuccessful(self):
self.verifier.reconcile = True
result_unsuccessful = self.mox.CreateMockAnything()
result_unsuccessful.ready().AndReturn(True)
result_unsuccessful.successful().AndReturn(False)
self.verifier.results = [result_unsuccessful]
self.mox.ReplayAll()
(result_count, success_count, errored) = self.verifier.clean_results()
self.assertEqual(result_count, 0)
self.assertEqual(success_count, 0)
self.assertEqual(errored, 1)
self.assertEqual(len(self.verifier.results), 0)
self.assertEqual(len(self.verifier.failed), 0)
self.mox.VerifyAll()
def test_clean_results_fail_verification(self):
self.verifier.reconcile = True
result_failed_verification = self.mox.CreateMockAnything()
result_failed_verification.ready().AndReturn(True)
result_failed_verification.successful().AndReturn(True)
failed_exists = self.mox.CreateMockAnything()
result_failed_verification.get().AndReturn((False, failed_exists))
self.verifier.results = [result_failed_verification]
self.mox.ReplayAll()
(result_count, success_count, errored) = self.verifier.clean_results()
self.assertEqual(result_count, 0)
self.assertEqual(success_count, 1)
self.assertEqual(errored, 0)
self.assertEqual(len(self.verifier.results), 0)
self.assertEqual(len(self.verifier.failed), 1)
self.assertEqual(self.verifier.failed[0], failed_exists)
self.mox.VerifyAll()
def test_verify_for_range_with_callback(self):
callback = self.mox.CreateMockAnything()
pool = self.mox.CreateMockAnything()
when_max = datetime.datetime.utcnow()
results = self.mox.CreateMockAnything()
models.InstanceExists.objects.select_related().AndReturn(results)
@@ -669,6 +936,18 @@ class VerifierTestCase(unittest.TestCase):
self.assertEqual(exist2.status, 'verifying')
self.mox.VerifyAll()
def test_reconcile_failed(self):
self.verifier.reconcile = True
exists1 = self.mox.CreateMockAnything()
exists2 = self.mox.CreateMockAnything()
self.verifier.failed = [exists1, exists2]
self.reconciler.failed_validation(exists1)
self.reconciler.failed_validation(exists2)
self.mox.ReplayAll()
self.verifier.reconcile_failed()
self.assertEqual(len(self.verifier.failed), 0)
self.mox.VerifyAll()
def test_send_verified_notification_default_routing_key(self):
connection = self.mox.CreateMockAnything()
exchange = self.mox.CreateMockAnything()
@@ -750,7 +1029,7 @@ class VerifierTestCase(unittest.TestCase):
dbverifier._create_connection(self.config_notif).AndReturn(conn)
conn.__enter__().AndReturn(conn)
self.mox.StubOutWithMock(self.verifier_notif, '_run')
self.verifier_notif._run(callback=mox.IgnoreArg())
self.verifier_notif._run(callback=mox.Not(mox.Is(None)))
conn.__exit__(None, None, None)
self.mox.ReplayAll()
self.verifier_notif.run()
@@ -766,7 +1045,7 @@ class VerifierTestCase(unittest.TestCase):
dbverifier._create_connection(self.config_notif).AndReturn(conn)
conn.__enter__().AndReturn(conn)
self.mox.StubOutWithMock(self.verifier_notif, '_run')
self.verifier_notif._run(callback=mox.IgnoreArg())
self.verifier_notif._run(callback=mox.Not(mox.Is(None)))
conn.__exit__(None, None, None)
self.mox.ReplayAll()
self.verifier_notif.run()
@@ -789,7 +1068,7 @@ class VerifierTestCase(unittest.TestCase):
dbverifier._create_connection(self.config_notif).AndReturn(conn)
conn.__enter__().AndReturn(conn)
self.mox.StubOutWithMock(self.verifier_notif, '_run_once')
self.verifier_notif._run_once(callback=mox.IgnoreArg())
self.verifier_notif._run_once(callback=mox.Not(mox.Is(None)))
conn.__exit__(None, None, None)
self.mox.ReplayAll()
self.verifier_notif.run_once()
@@ -801,3 +1080,123 @@ class VerifierTestCase(unittest.TestCase):
self.mox.ReplayAll()
self.verifier.run_once()
self.mox.VerifyAll()
def test_run_full_no_notifications(self):
self.verifier.reconcile = True
self.mox.StubOutWithMock(self.verifier, '_keep_running')
self.verifier._keep_running().AndReturn(True)
start = datetime.datetime.utcnow()
self.mox.StubOutWithMock(self.verifier, '_utcnow')
self.verifier._utcnow().AndReturn(start)
settle_time = self.config['settle_time']
settle_units = self.config['settle_units']
settle_offset = {settle_units: settle_time}
ending_max = start - datetime.timedelta(**settle_offset)
self.mox.StubOutWithMock(self.verifier, 'verify_for_range')
self.verifier.verify_for_range(ending_max, callback=None)
self.mox.StubOutWithMock(self.verifier, 'reconcile_failed')
result1 = self.mox.CreateMockAnything()
result2 = self.mox.CreateMockAnything()
self.verifier.results = [result1, result2]
result1.ready().AndReturn(True)
result1.successful().AndReturn(True)
result1.get().AndReturn((True, None))
result2.ready().AndReturn(True)
result2.successful().AndReturn(True)
result2.get().AndReturn((True, None))
self.verifier.reconcile_failed()
self.mox.StubOutWithMock(time, 'sleep', use_mock_anything=True)
time.sleep(self.config['tick_time'])
self.verifier._keep_running().AndReturn(False)
self.mox.ReplayAll()
self.verifier.run()
self.mox.VerifyAll()
def test_run_full(self):
self.verifier_notif.reconcile = True
self.mox.StubOutWithMock(self.verifier_notif, '_keep_running')
self.verifier_notif._keep_running().AndReturn(True)
start = datetime.datetime.utcnow()
self.mox.StubOutWithMock(self.verifier_notif, '_utcnow')
self.verifier_notif._utcnow().AndReturn(start)
settle_time = self.config['settle_time']
settle_units = self.config['settle_units']
settle_offset = {settle_units: settle_time}
ending_max = start - datetime.timedelta(**settle_offset)
self.mox.StubOutWithMock(self.verifier_notif, 'verify_for_range')
self.verifier_notif.verify_for_range(ending_max,
callback=mox.Not(mox.Is(None)))
self.mox.StubOutWithMock(self.verifier_notif, 'reconcile_failed')
result1 = self.mox.CreateMockAnything()
result2 = self.mox.CreateMockAnything()
self.verifier_notif.results = [result1, result2]
result1.ready().AndReturn(True)
result1.successful().AndReturn(True)
result1.get().AndReturn((True, None))
result2.ready().AndReturn(True)
result2.successful().AndReturn(True)
result2.get().AndReturn((True, None))
self.verifier_notif.reconcile_failed()
self.mox.StubOutWithMock(time, 'sleep', use_mock_anything=True)
time.sleep(self.config['tick_time'])
self.verifier_notif._keep_running().AndReturn(False)
self.mox.ReplayAll()
self.verifier_notif.run()
self.mox.VerifyAll()
def test_run_once_full_no_notifications(self):
self.verifier.reconcile = True
start = datetime.datetime.utcnow()
self.mox.StubOutWithMock(self.verifier, '_utcnow')
self.verifier._utcnow().AndReturn(start)
settle_time = self.config['settle_time']
settle_units = self.config['settle_units']
settle_offset = {settle_units: settle_time}
ending_max = start - datetime.timedelta(**settle_offset)
self.mox.StubOutWithMock(self.verifier, 'verify_for_range')
self.verifier.verify_for_range(ending_max, callback=None)
result1 = self.mox.CreateMockAnything()
result2 = self.mox.CreateMockAnything()
self.verifier.results = [result1, result2]
result1.ready().AndReturn(True)
result1.successful().AndReturn(True)
result1.get().AndReturn((True, None))
result2.ready().AndReturn(True)
result2.successful().AndReturn(True)
result2.get().AndReturn((True, None))
self.mox.StubOutWithMock(self.verifier, 'reconcile_failed')
self.verifier.reconcile_failed()
self.mox.StubOutWithMock(time, 'sleep', use_mock_anything=True)
time.sleep(self.config['tick_time'])
self.mox.ReplayAll()
self.verifier.run_once()
self.mox.VerifyAll()
def test_run_once_full(self):
self.verifier_notif.reconcile = True
start = datetime.datetime.utcnow()
self.mox.StubOutWithMock(self.verifier_notif, '_utcnow')
self.verifier_notif._utcnow().AndReturn(start)
settle_time = self.config['settle_time']
settle_units = self.config['settle_units']
settle_offset = {settle_units: settle_time}
ending_max = start - datetime.timedelta(**settle_offset)
self.mox.StubOutWithMock(self.verifier_notif, 'verify_for_range')
self.verifier_notif.verify_for_range(ending_max,
callback=mox.Not(mox.Is(None)))
result1 = self.mox.CreateMockAnything()
result2 = self.mox.CreateMockAnything()
self.verifier_notif.results = [result1, result2]
result1.ready().AndReturn(True)
result1.successful().AndReturn(True)
result1.get().AndReturn((True, None))
result2.ready().AndReturn(True)
result2.successful().AndReturn(True)
result2.get().AndReturn((True, None))
self.mox.StubOutWithMock(self.verifier_notif, 'reconcile_failed')
self.verifier_notif.reconcile_failed()
self.mox.StubOutWithMock(time, 'sleep', use_mock_anything=True)
time.sleep(self.config['tick_time'])
self.mox.ReplayAll()
self.verifier_notif.run_once()
self.mox.VerifyAll()

60
verifier/dbverifier.py
View File

@@ -23,7 +23,7 @@ import datetime
import json
import os
import sys
from time import sleep
import time
import uuid
from django.db import transaction
@@ -44,6 +44,7 @@ LOG = stacklog.get_logger()
from stacktach import models
from stacktach import datetime_to_decimal as dt
from stacktach import reconciler
from verifier import AmbiguousResults
from verifier import FieldMismatch
from verifier import NotFound
@@ -238,7 +239,7 @@ def _verify_for_delete(exist, delete=None, delete_type="InstanceDelete"):
delete.deleted_at)
def _verify_with_reconciled_data(exist, ex):
def _verify_with_reconciled_data(exist):
if not exist.launched_at:
raise VerificationException("Exists without a launched_at")
@@ -259,7 +260,10 @@ def _verify_with_reconciled_data(exist, ex):
_verify_for_launch(exist, launch=reconcile,
launch_type="InstanceReconcile")
_verify_for_delete(exist, delete=reconcile,
delete = None
if reconcile.deleted_at is not None:
delete = reconcile
_verify_for_delete(exist, delete=delete,
delete_type="InstanceReconcile")
@@ -267,9 +271,9 @@ def _attempt_reconciled_verify(exist, orig_e):
verified = False
try:
# Attempt to verify against reconciled data
_verify_with_reconciled_data(exist, orig_e)
_verify_with_reconciled_data(exist)
verified = True
_mark_exist_verified(exist)
_mark_exist_verified(exist, reconciled=True)
except NotFound, rec_e:
# No reconciled data, just mark it failed
_mark_exist_failed(exist, reason=str(orig_e))
@@ -341,10 +345,25 @@ def _create_connection(config):
class Verifier(object):
def __init__(self, config, pool=None):
def __init__(self, config, pool=None, rec=None):
self.config = config
self.pool = pool or multiprocessing.Pool(self.config['pool_size'])
self.reconcile = self.config.get('reconcile', False)
self.reconciler = self._load_reconciler(config, rec=rec)
self.results = []
self.failed = []
def _load_reconciler(self, config, rec=None):
if rec:
return rec
if self.reconcile:
config_loc = config.get('reconciler_config',
'/etc/stacktach/reconciler_config.json')
with open(config_loc, 'r') as rec_config_file:
rec_config = json.load(rec_config_file)
return reconciler.Reconciler(rec_config)
def clean_results(self):
pending = []
@@ -355,6 +374,9 @@ class Verifier(object):
if result.ready():
finished += 1
if result.successful():
(verified, exists) = result.get()
if self.reconcile and not verified:
self.failed.append(exists)
successful += 1
else:
pending.append(result)
@@ -386,22 +408,34 @@ class Verifier(object):
next_update = datetime.datetime.utcnow() + update_interval
return count
def reconcile_failed(self):
for failed_exist in self.failed:
self.reconciler.failed_validation(failed_exist)
self.failed = []
def _keep_running(self):
return True
def _utcnow(self):
return datetime.datetime.utcnow()
def _run(self, callback=None):
tick_time = self.config['tick_time']
settle_units = self.config['settle_units']
settle_time = self.config['settle_time']
while True:
while self._keep_running():
with transaction.commit_on_success():
now = datetime.datetime.utcnow()
now = self._utcnow()
kwargs = {settle_units: settle_time}
ending_max = now - datetime.timedelta(**kwargs)
new = self.verify_for_range(ending_max,
callback=callback)
values = ((new,) + self.clean_results())
if self.reconcile:
self.reconcile_failed()
msg = "N: %s, P: %s, S: %s, E: %s" % values
LOG.info(msg)
sleep(tick_time)
time.sleep(tick_time)
def run(self):
if self.config['enable_notifications']:
@@ -427,7 +461,7 @@ class Verifier(object):
tick_time = self.config['tick_time']
settle_units = self.config['settle_units']
settle_time = self.config['settle_time']
now = datetime.datetime.utcnow()
now = self._utcnow()
kwargs = {settle_units: settle_time}
ending_max = now - datetime.timedelta(**kwargs)
new = self.verify_for_range(ending_max, callback=callback)
@@ -435,7 +469,9 @@ class Verifier(object):
LOG.info("Verifying %s exist events" % new)
while len(self.results) > 0:
LOG.info("P: %s, F: %s, E: %s" % self.clean_results())
sleep(tick_time)
if self.reconcile:
self.reconcile_failed()
time.sleep(tick_time)
def run_once(self):
if self.config['enable_notifications']: