x/ranger
Code Issues Proposed changes

Refactor RDS resource status retrieval

Browse Source 
Change-Id: I25fba0f687c42d3b5782fd1434a0da6a3816dcf0
This commit is contained in:
Chi Lo 2020-10-22 15:45:21 -07:00
parent d2c95ac8c8
commit 1f67e02206
9 changed files with 105 additions and 81 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
orm/base_config.py
View File

@ -350,6 +350,8 @@ user_domain_name = CONF.keystone_authtoken.user_domain_name
project_domain_name = CONF.keystone_authtoken.project_domain_name
conn = CONF.database.connection
db_connect = conn.replace("mysql+pymysql", "mysql") if conn else None
resource_status_wait_time = CONF.resource_status_wait_time
resource_status_extended_wait_time = CONF.resource_status_extended_wait_time
ssl_verify = CONF.ssl_verify
token_auth_version = '3' if (CONF.keystone_authtoken.auth_version

12
orm/services/resource_distributor/config.py
View File

@ -119,13 +119,13 @@ block_by_status = "Submitted"
allow_region_statuses = ['building', 'functional', 'maintenance']
region_resource_id_status = {
# interval_time_validation in minutes
# interval_time_validation in seconds
'max_interval_time': {
'images': 2,
'tenants': 2,
'flavors': 2,
'users': 2,
'default': 2
'images': config.resource_status_extended_wait_time,
'tenants': config.resource_status_wait_time,
'flavors': config.resource_status_wait_time,
'users': config.resource_status_wait_time,
'default': config.resource_status_wait_time
},
'allowed_status_values': {
'Success',

6
orm/services/resource_distributor/rds/controllers/v1/base.py
View File

@ -47,13 +47,13 @@ class EntityNotFoundError(ClientSideError):
class LockedEntity(ClientSideError):
"""return 409 locked."""
def __init__(self, name):
def __init__(self, msg):
"""init func.
:param name: locked message
"""
super(LockedEntity, self).__init__("Entity {} is "
"locked".format(name),
super(LockedEntity, self).__init__("Entity is "
"locked: {}".format(msg),
status_code=409)

6
orm/services/resource_distributor/rds/controllers/v1/resources/root.py
View File

@ -197,7 +197,7 @@ class CreateNewResource(rest.RestController):
links=Links(site_link))})
return res
except ConflictValue as e:
my_logger.error("the request blocked need to wait "
my_logger.error("The request blocked, need to wait "
"for previous operation to be done ")
raise LockedEntity(str(e))
except Exception as e:
@ -241,7 +241,7 @@ class CreateNewResource(rest.RestController):
links=Links(site_link))})
return res
except ConflictValue as e:
my_logger.error("the request blocked need to wait "
my_logger.error("The request blocked, need to wait "
"for previous operation to be done ")
raise LockedEntity(str(e))
except Exception as e:
@ -277,7 +277,7 @@ class CreateNewResource(rest.RestController):
operation)
return resource_id
except ConflictValue as e:
my_logger.error("the request blocked need to wait"
my_logger.error("The request blocked, need to wait"
" for previous operation to be done ")
raise LockedEntity(str(e))
except Exception as e:

54
orm/services/resource_distributor/rds/services/region_resource_id_status.py
View File

@ -1,7 +1,8 @@
import logging
import sys
from orm.services.resource_distributor.rds.services.base import Error, InputError
from orm.services.resource_distributor.rds.services.base \
import Error, InputError
from orm.services.resource_distributor.rds.storage import factory
logger = logging.getLogger(__name__)
@ -57,18 +58,19 @@ def get_template_data(resource_id, region):
def add_status(data):
logger.debug("add resource status timestamp [{}], region [{}], status [{}] "
", transaction_id [{}] and resource_id [{}], ord_notifier_id [{}], "
"error message [{}], error code [{}] and "
"resource_extra_metadata [{}]".format(data['timestamp'],
data['region'],
data['status'],
data['transaction_id'],
data['resource_id'],
data['ord_notifier_id'],
data['error_msg'],
data['error_code'],
data.get('resource_extra_metadata', None)))
logger.debug("add resource status timestamp [{}], region [{}], "
"status [{}], transaction_id [{}] and resource_id [{}], "
"ord_notifier_id [{}], error message [{}], error code [{}] "
"and resource_extra_metadata [{}]".format(
data['timestamp'],
data['region'],
data['status'],
data['transaction_id'],
data['resource_id'],
data['ord_notifier_id'],
data['error_msg'],
data['error_code'],
data.get('resource_extra_metadata', None)))
try:
validate_status_value(data['status'])
@ -94,43 +96,39 @@ def add_status(data):
def get_status_by_resource_id(resource_id):
logger.debug("get status by resource id %s " % resource_id)
conn = factory.get_region_resource_id_status_connection()
result = conn.get_records_by_resource_id(resource_id)
return result
def get_regions_by_status_resource_id(status, resource_id):
logger.debug("get regions by status %s for resource %s" % (status, resource_id))
def get_regions_by_status_resource_id(status, resource_id, resource_type):
logger.debug("get regions by status %s for %s resource %s" % (
status, resource_type, resource_id))
conn = factory.get_region_resource_id_status_connection()
result = conn.get_records_by_resource_id_and_status(resource_id,
status)
result = conn.get_records_by_resource_id_and_status(status,
resource_id,
resource_type)
return result
def validate_resource_type(resource_type):
allowed_resource_type = config['allowed_resource_type']
if resource_type not in allowed_resource_type:
logger.exception("status value is not valid: {}".format(resource_type))
logger.exception("status value is invalid: {}".format(resource_type))
raise InputError("operation_type", resource_type)
def validate_operation_type(operation_type):
allowed_operation_type = config['allowed_operation_type']
if operation_type not in allowed_operation_type:
logger.exception("status value is not valid: {}".format(operation_type))
logger.exception("status value is invalid: {}".format(operation_type))
raise InputError("operation_type", operation_type)
def validate_status_value(status):
allowed_status_values = config['allowed_status_values']
if status not in allowed_status_values:
logger.exception("status value is not valid: {}".format(status))
logger.exception("status value is invalid: {}".format(status))
raise InputError("status", status)
# def post_data_to_image(data):
# if data['resource_type'] == "image":
# logger.debug("send metadata {} to ims :- {} for region {}".format(
# data['resource_extra_metadata'], data['resource_id'], data['region']))
# # ims_proxy.send_image_metadata(data['resource_extra_metadata'], data['resource_id'], data['region'])
# return

38
orm/services/resource_distributor/rds/services/resource.py
View File

@ -7,11 +7,15 @@ from orm.services.flavor_manager.fms_rest.data.sql_alchemy.data_manager \
import DataManager
from orm.services.resource_distributor.rds.ordupdate.ord_notifier \
import notify_ord, NoTokenError
from orm.services.resource_distributor.rds.services import region_resource_id_status as regionResourceIdStatus
from orm.services.resource_distributor.rds.services import (yaml_customer_builder, yaml_flavor_builder,
yaml_group_builder, yaml_image_builder)
from orm.services.resource_distributor.rds.services.base import ConflictValue, ErrorMessage
from orm.services.resource_distributor.rds.services.model.resource_input import ResourceData as InputData
from orm.services.resource_distributor.rds.services \
import region_resource_id_status as regionResourceIdStatus
from orm.services.resource_distributor.rds.services \
import (yaml_customer_builder, yaml_flavor_builder,
yaml_group_builder, yaml_image_builder)
from orm.services.resource_distributor.rds.services.base \
import ConflictValue, ErrorMessage
from orm.services.resource_distributor.rds.services.model.resource_input \
import ResourceData as InputData
from orm.services.resource_distributor.rds.utils import utils, uuid_utils
@ -238,21 +242,26 @@ def _submit_template_data(uuid, tranid, targetslist):
def _check_resource_status(input_data):
resource_id = input_data.resource_id
status = conf.block_by_status
# check if any of the region creation in pending
# check if any of the region is blocked by "Submitted" state
regions_by_resource = \
regionResourceIdStatus.get_regions_by_status_resource_id(status,
resource_id)
regionResourceIdStatus.get_regions_by_status_resource_id(
status,
input_data.resource_id,
input_data.resource_type)
# if any not ready return 409
if regions_by_resource is not None and regions_by_resource.regions:
raise ConflictValue([region.region for region in regions_by_resource.regions])
regions_in_error = [reg.region for reg in regions_by_resource.regions]
msg = "Previous operation still in %s state for regions: %s " % (
status, regions_in_error)
raise ConflictValue(msg)
def _generate_resource_data(input_data):
"""create resource."""
my_logger.debug("build yaml file for %s id: %s" % (input_data.resource_type,
input_data.resource_id))
my_logger.debug("build yaml file for %s id: %s" % (
input_data.resource_type, input_data.resource_id))
targetslist = _create_template_data(input_data)
my_logger.debug("submit yaml to ranger-agent...")
_submit_template_data(input_data.resource_id,
@ -260,8 +269,9 @@ def _generate_resource_data(input_data):
targetslist)
# User domain is only used in the case that a customer template is being generated,
# as certain builds of heat require user_domain in order to validate roles
# User domain is only used in the case that a customer template is being
# generated, as certain builds of heat require user_domain in order to
# validate roles
def main(jsondata, external_transaction_id, resource_type, operation):
"""main function handle resource operation."""
my_logger.info("got %s for %s resource" % (operation, resource_type))

60
orm/services/resource_distributor/rds/storage/mysql/region_resource_id_status.py
View File

@ -6,10 +6,13 @@ from oslo_db.sqlalchemy.enginefacade import LegacyEngineFacade
from pecan import conf
from orm.common.orm_common.model.models import ResourceStatusModel, StatusModel
from orm.services.resource_distributor.rds.services.model.region_resource_id_status \
import RegionEndPointData
from orm.services.resource_distributor.rds.storage import region_resource_id_status
from sqlalchemy import BigInteger, BLOB, Column, ForeignKey, Integer, String, Text
from orm.services.resource_distributor.rds.storage \
import region_resource_id_status
from sqlalchemy \
import BigInteger, BLOB, Column, ForeignKey, Integer, String, Text
from sqlalchemy.ext.declarative.api import declarative_base
from sqlalchemy.orm import relationship
from sqlalchemy.sql import and_
@ -31,8 +34,8 @@ class ResourceStatusRecord(Base):
err_code = Column(Text, primary_key=False)
err_msg = Column(Text, primary_key=False)
operation = Column(Text, primary_key=False)
resource_extra_metadata = relationship("ImageMetadData",
cascade="all, delete, delete-orphan")
resource_extra_metadata = relationship(
"ImageMetadData", cascade="all, delete, delete-orphan")
class ImageMetadData(Base):
@ -141,17 +144,19 @@ class ResStatusConnection(region_resource_id_status.ResourceStatusBase):
return record.transaction_id
else:
logger.debug("Add record")
resource_status = ResourceStatusRecord(timestamp=timestamp,
region=region,
status=status,
transaction_id=transaction_id,
resource_id=resource_id,
ord_notifier=ord_notifier,
err_msg=err_msg,
err_code=err_code,
operation=operation)
resource_status = ResourceStatusRecord(
timestamp=timestamp,
region=region,
status=status,
transaction_id=transaction_id,
resource_id=resource_id,
ord_notifier=ord_notifier,
err_msg=err_msg,
err_code=err_code,
operation=operation)
if resource_extra_metadata:
resource_status.resource_extra_metadata.append(image_metadata)
resource_status.resource_extra_metadata.append(
image_metadata)
session.add(resource_status)
return transaction_id
@ -230,8 +235,9 @@ class ResStatusConnection(region_resource_id_status.ResourceStatusBase):
return None
def get_records_by_resource_id_and_status(self,
status,
resource_id,
status):
resource_type):
""" This method filters all the records where resource_id is the given
resource_id and status is the given status.
for the matching records check if a time period elapsed and if so,
@ -240,7 +246,8 @@ class ResStatusConnection(region_resource_id_status.ResourceStatusBase):
logger.debug("Get records filtered by resource_id={} "
"and status={}".format(resource_id,
status))
(timestamp, ref_timestamp) = self.get_timestamp_pair()
(timestamp, ref_timestamp) = self.get_timestamp_pair(resource_type)
logger.debug("timestamp=%s, ref_timestamp=%s" % (timestamp, ref_timestamp))
session = self._engine_facade.get_session()
records_model = []
@ -272,12 +279,19 @@ class ResStatusConnection(region_resource_id_status.ResourceStatusBase):
logger.debug("No records found")
return None
def get_timestamp_pair(self):
timestamp = int(time.time()) * 1000
# assume same time period for all resource types
max_interval_time_in_seconds = conf.region_resource_id_status.max_interval_time.default * 60
ref_timestamp = (int(time.time()) - max_interval_time_in_seconds) * 1000
return timestamp, ref_timestamp
def get_timestamp_pair(self, resource_type=None):
if resource_type == "customer":
interval = conf.region_resource_id_status.max_interval_time.tenants
elif resource_type == "flavor":
interval = conf.region_resource_id_status.max_interval_time.flavors
elif resource_type == "image":
interval = conf.region_resource_id_status.max_interval_time.images
else:
interval = conf.region_resource_id_status.max_interval_time.default
timestamp = int(time.time())
ref_timestamp = timestamp - interval
return timestamp * 1000, ref_timestamp * 1000
def get_region_keystone_ep(self, region_name):
"""get keystone url from region record """
@ -297,7 +311,7 @@ class ResStatusConnection(region_resource_id_status.ResourceStatusBase):
return None
except Exception as exp:
logger.exception("DB error RegionEndPoint filtering by region name")
logger.exception("DB error RegionEndPoint filtered by region name")
raise

2
orm/tests/unit/rds/services/test_region_resource_id_status.py
View File

@ -88,7 +88,7 @@ class TestModel(unittest.TestCase):
@mock.patch.object(region_resource_id_status.factory, 'get_region_resource_id_status_connection')
def test_get_regions_by_status_resource_id_sanity(self, mock_factory):
# Make sure that no exception is raised
region_resource_id_status.get_regions_by_status_resource_id(1, 2)
region_resource_id_status.get_regions_by_status_resource_id(1, 2, 3)
@mock.patch.object(region_resource_id_status.factory, 'get_region_resource_id_status_connection')
def test_get_status_by_resource_id_sanity(self, mock_factory):

6
orm/tests/unit/rds/storage/mysql/test_region_resource_id_status.py
View File

@ -187,7 +187,7 @@ class MysqlRegionResourceIdStatusTest(unittest.TestCase):
mock_statusmodel):
"""Test that the function returns None when it got no records."""
my_connection = region_resource_id_status.ResStatusConnection('url')
self.assertIsNone(my_connection.get_records_by_resource_id_and_status('1', '2'))
self.assertIsNone(my_connection.get_records_by_resource_id_and_status('1', '2', '3'))
@mock.patch.object(region_resource_id_status, 'StatusModel')
@patch.object(region_resource_id_status.ResStatusConnection, 'get_timestamp_pair',
@ -200,7 +200,7 @@ class MysqlRegionResourceIdStatusTest(unittest.TestCase):
mock_model,
mock_statusmodel):
my_connection = region_resource_id_status.ResStatusConnection('url')
my_connection.get_records_by_resource_id_and_status('1', '2')
my_connection.get_records_by_resource_id_and_status('1', '2', '3')
@mock.patch.object(region_resource_id_status, 'StatusModel')
@patch.object(region_resource_id_status.ResStatusConnection, 'get_timestamp_pair',
@ -213,4 +213,4 @@ class MysqlRegionResourceIdStatusTest(unittest.TestCase):
mock_model,
mock_statusmodel):
my_connection = region_resource_id_status.ResStatusConnection('url')
my_connection.get_records_by_resource_id_and_status('1', '2')
my_connection.get_records_by_resource_id_and_status('1', '2', '3')