x/ranger
Code Issues Proposed changes

Update resource tables layout structure

Browse Source 
Remove  previously-added foreign key in resource_status
table that causes issue with updating resource status table.

Also update resource_template_data table to include region field
and replace transaction_id with resource_id.

Modify RDS logic accordingly.

Change-Id: I71b61b0cfb76545058ef26c0117f007b6d6784d3
This commit is contained in:
st6218
2019-12-18 16:19:23 -08:00
parent b3541d3a49
commit 19b035bfb6
8 changed files with 73 additions and 63 deletions
Download Patch File Download Diff File Expand all files Collapse all files

14
orm/services/resource_distributor/rds/ordupdate/ord_notifier.py
View File

@@ -159,6 +159,7 @@ def _notify(ord_url,
files=files, files=files,
headers=headers, headers=headers,
cert=conf.ordupdate.cert_path) cert=conf.ordupdate.cert_path)
except requests.exceptions.SSLError: except requests.exceptions.SSLError:
logger.debug('Received an SSL error (is the certificate valid?)') logger.debug('Received an SSL error (is the certificate valid?)')
raise raise
@@ -195,12 +196,13 @@ def _update_audit(lcp_name, application_id, tracking_id, transaction_id,
# new for template data table # new for template data table
def _update_template_data(transaction_id, resource_name, def _update_template_data(resource_id, resource_name, region_id,
resource_template_data): resource_template_data):
data_to_save = dict( data_to_save = dict(
transaction_id=transaction_id, resource_id=resource_id,
stack_name=resource_name, resource_name=resource_name,
region=region_id,
stack_template=resource_template_data.encode("utf-8")) stack_template=resource_template_data.encode("utf-8"))
regionResourceIdStatus.add_update_template_data(data_to_save) regionResourceIdStatus.add_update_template_data(data_to_save)
@@ -293,7 +295,7 @@ def notify_ord(transaction_id,
raise OrdNotFoundError(Exception(message)) raise OrdNotFoundError(Exception(message))
if operation != 'create': if operation != 'create':
record = regionResourceIdStatus.get_template_by_trans_id( record = regionResourceIdStatus.get_template_data(
resource_id, region_id) resource_id, region_id)
resource_template_version = record.template_version + 1 resource_template_version = record.template_version + 1
@@ -336,8 +338,8 @@ def notify_ord(transaction_id,
event_details, status) event_details, status)
if operation in ('create', 'modify'): if operation in ('create', 'modify'):
# add entry to resource_template_data table # add entry to resource_template_data table
_update_template_data(resource_transaction_id, resource_name, _update_template_data(resource_id, resource_name,
resource_template_data) region_id, resource_template_data)
logger.debug( logger.debug(
"Create Resource Requested to ranger agent: region=%s resource_id=%s " "Create Resource Requested to ranger agent: region=%s resource_id=%s "

6
orm/services/resource_distributor/rds/services/model/region_resource_id_status.py
View File

@@ -47,12 +47,14 @@ class ResourceStatusModel(object):
class ResourceTemplateModel(object): class ResourceTemplateModel(object):
def __init__(self, def __init__(self,
transaction_id, resource_id,
resource_name, resource_name,
region,
template_version, template_version,
template_data): template_data):
self.transaction_id = transaction_id self.resource_id = resource_id
self.resource_name = resource_name self.resource_name = resource_name
self.region = region
self.template_version = template_version self.template_version = template_version
self.template_data = template_data self.template_data = template_data

27
orm/services/resource_distributor/rds/services/region_resource_id_status.py
View File

@@ -17,13 +17,14 @@ num_of_miliseconds_in_seconds = 1000
def add_update_template_data(data): def add_update_template_data(data):
logger.debug("update template stack name [{}], and " logger.debug(
"transaction_id [{}] ".format(data['stack_name'], "add/update template stack name [{}] ".format(data['resource_name']))
data['transaction_id']))
try: try:
conn = factory.get_resource_stack_data_connection() conn = factory.get_resource_stack_data_connection()
conn.add_update_template_record(data['transaction_id'], conn.add_update_template_record(data['resource_id'],
data['stack_name'], data['resource_name'],
data['region'],
data['stack_template']) data['stack_template'])
except Exception: except Exception:
@@ -36,15 +37,13 @@ def delete_resource_status_data(resource_id):
pass pass
def get_template_by_trans_id(resource_id, region): def get_template_data(resource_id, region):
logger.debug("get template transaction id for resource %s and " logger.debug("get template data for resource %s and "
"region %s" % (resource_id, region)) "region %s" % (resource_id, region))
try: try:
conn = factory.get_region_resource_id_status_connection()
resource_trans_id = conn.get_resource_region_data(resource_id, region)
conn = factory.get_resource_stack_data_connection() conn = factory.get_resource_stack_data_connection()
record = conn.get_resource_template_data(resource_trans_id) record = conn.get_resource_template_data(resource_id, region)
return record return record
except Exception: except Exception:
@@ -73,15 +72,13 @@ def add_status(data):
conn = factory.get_region_resource_id_status_connection() conn = factory.get_region_resource_id_status_connection()
resource_transaction_id = \
conn.add_update_status_record( conn.add_update_status_record(
data['timestamp'], data['region'], data['status'], data['timestamp'], data['region'], data['status'],
data['transaction_id'], data['resource_id'], data['transaction_id'], data['resource_id'],
data['ord_notifier_id'], data['error_msg'], data['ord_notifier_id'], data['error_msg'],
data['error_code'], data['resource_operation'], data['error_code'], data['resource_operation'],
data.get('resource_extra_metadata')) data.get('resource_extra_metadata'))
return resource_transaction_id
# post_data_to_image(data)
except Error as e: except Error as e:
logger.exception("invalid inputs error") logger.exception("invalid inputs error")
raise raise
@@ -109,9 +106,9 @@ def delete_resource_template_data(resource_id, region):
logger.debug("delete resource_template_data for resource %s and " logger.debug("delete resource_template_data for resource %s and "
"region %s" % (resource_id, region)) "region %s" % (resource_id, region))
conn = factory.get_region_resource_id_status_connection() conn = factory.get_region_resource_id_status_connection()
resource_trans_id = conn.get_resource_region_data(resource_id, region) resource_id, region = conn.get_resource_region_data(resource_id, region)
conn = factory.get_resource_stack_data_connection() conn = factory.get_resource_stack_data_connection()
conn.delete_resource_template(resource_trans_id) conn.delete_resource_template(resource_id, region)
def validate_resource_type(resource_type): def validate_resource_type(resource_type):

44
orm/services/resource_distributor/rds/storage/mysql/region_resource_id_status.py
View File

@@ -68,10 +68,10 @@ class ResourceTemplateRecord(Base):
__tablename__ = 'resource_template_data' __tablename__ = 'resource_template_data'
id = Column(Integer, autoincrement=True, primary_key=True) id = Column(Integer, autoincrement=True, primary_key=True)
transaction_id = Column(Text, resource_id = Column(Text,
ForeignKey('resource_status.transaction_id'),
primary_key=False) primary_key=False)
resource_name = Column(Text, primary_key=False) resource_name = Column(Text, primary_key=False)
region = Column(Text, primary_key=False)
template_version = Column(Integer, primary_key=False) template_version = Column(Integer, primary_key=False)
template_data = Column(BLOB, primary_key=False) template_data = Column(BLOB, primary_key=False)
@@ -173,7 +173,7 @@ class ResStatusConnection(region_resource_id_status.ResourceStatusBase):
logger.exception( logger.exception(
'No resource status record with resource id {} and' 'No resource status record with resource id {} and'
'region {} found'.format(resource_id, region)) 'region {} found'.format(resource_id, region))
return record.transaction_id return record.resource_id, record.region
except Exception as exp: except Exception as exp:
raise raise
@@ -289,19 +289,20 @@ class ResTemplateConnection(region_resource_id_status.ResourceTemplateBase):
self._engine_facade = LegacyEngineFacade(url) self._engine_facade = LegacyEngineFacade(url)
def add_update_template_record(self, def add_update_template_record(self,
transaction_id, resource_id,
resource_name, resource_name,
region_name,
resource_template): resource_template):
logger.debug("Add/Update template record:\ntransaction_id [{}]\n" logger.debug("Add/Update template record:\nresource_id [{}]\n"
"resource name [{}]\n\n".format(transaction_id, "region [{}]\n\n".format(resource_id, region_name))
resource_name))
try: try:
session = self._engine_facade.get_session() session = self._engine_facade.get_session()
with session.begin(): with session.begin():
record = session.query(ResourceTemplateRecord).\ record = session.query(ResourceTemplateRecord).\
filter_by(transaction_id=transaction_id).first() filter_by(resource_id=resource_id,
region=region_name).first()
if record is not None: if record is not None:
logger.debug("Update resource template record") logger.debug("Update resource template record")
record.resource_name = resource_name record.resource_name = resource_name
@@ -311,8 +312,9 @@ class ResTemplateConnection(region_resource_id_status.ResourceTemplateBase):
logger.debug("Add resource template record") logger.debug("Add resource template record")
resource_template_record = \ resource_template_record = \
ResourceTemplateRecord( ResourceTemplateRecord(
transaction_id=transaction_id, resource_id=resource_id,
resource_name=resource_name, resource_name=resource_name,
region=region_name,
template_version=0, template_version=0,
template_data=resource_template) template_data=resource_template)
@@ -321,33 +323,35 @@ class ResTemplateConnection(region_resource_id_status.ResourceTemplateBase):
except oslo_db.exception.DBDuplicateEntry as e: except oslo_db.exception.DBDuplicateEntry as e:
logger.warning("Duplicate entry: {}".format(str(e))) logger.warning("Duplicate entry: {}".format(str(e)))
def delete_resource_template(self, trans_id): def delete_resource_template(self, resource_id, region_name):
try: try:
session = self._engine_facade.get_session() session = self._engine_facade.get_session()
with session.begin(): with session.begin():
record = session.query(ResourceTemplateRecord).\ record = session.query(ResourceTemplateRecord).\
filter_by(transaction_id=trans_id).delete() filter_by(resource_id=resource_id,
region=region_name).delete()
if record is None: if record is None:
logger.exception( logger.exception(
'Template data not found with transaction' 'Template data not found with resource id {} and '
'id {}'.format(trans_id)) 'region name {}'.format(resource_id, region_name))
except Exception as exp: except Exception as exp:
raise raise
def get_resource_template_data(self, transaction_id): def get_resource_template_data(self, resource_id, region):
logger.debug("Get resource template data by transaction " logger.debug("Get resource template data by resource {} and "
"id '{}' ".format(transaction_id)) "region '{}' ".format(resource_id, region))
try: try:
session = self._engine_facade.get_session() session = self._engine_facade.get_session()
with session.begin(): with session.begin():
record = session.query(ResourceTemplateRecord).\ record = session.query(ResourceTemplateRecord).\
filter_by(transaction_id=transaction_id).first() filter_by(resource_id=resource_id,
region=region).first()
if record is None: if record is None:
logger.exception( logger.exception(
'No resource template record found with transaction ' 'No template record found with resource id {} '
'id '.format(transaction_id)) 'and region {} '.format(resource_id, region))
return record return record
except Exception as exp: except Exception as exp:
raise raise

6
orm/services/resource_distributor/rds/storage/region_resource_id_status.py
View File

@@ -29,9 +29,9 @@ class ResourceTemplateBase(object):
pass pass
def add_update_template_record(self, def add_update_template_record(self,
transaction_id, resource_id,
stack_name, stack_name, region,
stacK_template): stack_template):
raise NotImplementedError(Exception("Please Implement this method")) raise NotImplementedError(Exception("Please Implement this method"))
def get_records_by_transaction_id(self, transaction_id): def get_records_by_transaction_id(self, transaction_id):

3
orm/services/resource_distributor/rds/utils/utils.py
View File

@@ -28,6 +28,9 @@ def post_data_to_image(data):
def invoke_delete_region(data): def invoke_delete_region(data):
# ALLOW delete region for the following conditions:
# (1) if 'delete' operation success
# (2) if error = 'ORD_12' (heat stack not found)
if data['resource_operation'] == 'delete' and (data['status'] == 'Success' or data['error_code'] == 'ORD_012'): if data['resource_operation'] == 'delete' and (data['status'] == 'Success' or data['error_code'] == 'ORD_012'):
rds_resource_service_proxy.invoke_resources_region_delete( rds_resource_service_proxy.invoke_resources_region_delete(
resource_type=data['resource_type'], resource_type=data['resource_type'],

13
orm/services/resource_distributor/scripts/db_scripts/create_db.sql
View File

@@ -20,8 +20,8 @@ create table if not exists resource_status
err_code varchar(64), err_code varchar(64),
operation varchar(64), operation varchar(64),
primary key (id), primary key (id),
unique(resource_id, region), unique(resource_id, region)
unique transaction_idx (transaction_id)); );
#***** #*****
@@ -31,12 +31,13 @@ create table if not exists resource_status
create table if not exists resource_template_data create table if not exists resource_template_data
( (
id integer auto_increment not null, id integer auto_increment not null,
transaction_id varchar(64), resource_id varchar(64) not null,
resource_name varchar(64) NOT NULL, resource_name varchar(64) not null,
template_version integer, region varchar(64) not null,
template_version integer not null,
template_data BLOB NOT NULL, template_data BLOB NOT NULL,
primary key (id), primary key (id),
foreign key (transaction_id) references resource_status(transaction_id) ON DELETE CASCADE foreign key (resource_id, region) references resource_status(resource_id, region) ON DELETE CASCADE
); );

9
orm/tests/unit/rds/services/model/test_region_resource_id_status.py
View File

@@ -26,13 +26,14 @@ class TestModel(unittest.TestCase):
class TestResourceTemplateModel(unittest.TestCase): class TestResourceTemplateModel(unittest.TestCase):
def test_model_as_dict(self): def test_model_as_dict(self):
model = region_resource_id_status.ResourceTemplateModel(1, 2, 3, 4) model = region_resource_id_status.ResourceTemplateModel(1, 2, 3, 4, 5)
expected_template_dict = { expected_template_dict = {
'transaction_id': 1, 'resource_id': 1,
'resource_name': 2, 'resource_name': 2,
'template_version': 3, 'region': 3,
'template_data': 4 'template_version': 4,
'template_data': 5
} }
test_dict = model.as_dict() test_dict = model.as_dict()