Restructure plugin base
This change makes some mild restructuring and naming changes to the data-extractor plugin base. Data extraction will now be separated into two main workflows: loading raw data and source specific parsing into model objects. Naming of methods was changed to help more accurately reflect their function in these workflows. Change-Id: Ia3bc892994ff96cce5c1672fadf35ef2d1c4164b
This commit is contained in:
Ian H Pittwood
parent
4dd1cea32a
commit
fd3d935941
|
|
@ -116,7 +116,6 @@ def intermediary_processor(plugin_type, **kwargs):
|
|||
# Extract data from plugin data source
|
||||
LOG.info("Extract data from plugin data source")
|
||||
data_extractor = plugin_class(kwargs['site_name'], **kwargs)
|
||||
data_extractor.extract_data()
|
||||
|
||||
# Apply any additional_config provided by user
|
||||
additional_config = kwargs.get('site_configuration', None)
|
||||
|
|
@ -126,17 +125,17 @@ def intermediary_processor(plugin_type, **kwargs):
|
|||
LOG.debug(
|
||||
"Additional config data:\n{}".format(
|
||||
pprint.pformat(additional_config_data)))
|
||||
else:
|
||||
additional_config_data = None
|
||||
|
||||
LOG.info(
|
||||
"Apply additional configuration from:{}".format(additional_config))
|
||||
data_extractor.apply_additional_data(additional_config_data)
|
||||
LOG.debug(pprint.pformat(data_extractor.site_data))
|
||||
# Extract data into data objects
|
||||
data_extractor.get_data(additional_config_data)
|
||||
LOG.debug(pprint.pformat(data_extractor.data.dict_from_class()))
|
||||
|
||||
# Apply design rules to the data
|
||||
LOG.info("Apply design rules to the extracted data")
|
||||
process_input_ob = ProcessDataSource(kwargs['site_name'])
|
||||
process_input_ob.load_extracted_data_from_data_source(
|
||||
data_extractor.site_data)
|
||||
process_input_ob.load_extracted_data_from_data_source(data_extractor.data)
|
||||
return process_input_ob
|
||||
|
||||
|
||||
|
|
|
|||
|
|
@ -27,13 +27,18 @@ class BaseDataSourcePlugin(metaclass=abc.ABCMeta):
|
|||
self.source_type = None
|
||||
self.source_name = None
|
||||
self.region = region
|
||||
self.site_data = None
|
||||
self.raw_data = None
|
||||
self.data = None
|
||||
|
||||
@abc.abstractmethod
|
||||
def get_racks(self, region):
|
||||
def load_raw_data(self):
|
||||
"""Loads raw data from data source"""
|
||||
return
|
||||
|
||||
@abc.abstractmethod
|
||||
def parse_racks(self):
|
||||
"""Return list of racks in the region
|
||||
|
||||
:param string region: Region name
|
||||
:returns: list of Rack objects
|
||||
:rtype: list
|
||||
"""
|
||||
|
|
@ -41,10 +46,9 @@ class BaseDataSourcePlugin(metaclass=abc.ABCMeta):
|
|||
return []
|
||||
|
||||
@abc.abstractmethod
|
||||
def get_hosts(self, region, rack=None):
|
||||
def parse_hosts(self, rack=None):
|
||||
"""Return list of hosts in the region
|
||||
|
||||
:param string region: Region name
|
||||
:param string rack: Rack name
|
||||
:returns: list of Host objects containing a rack's host data
|
||||
:rtype: list of models.Host
|
||||
|
|
@ -53,10 +57,9 @@ class BaseDataSourcePlugin(metaclass=abc.ABCMeta):
|
|||
return []
|
||||
|
||||
@abc.abstractmethod
|
||||
def get_networks(self, region):
|
||||
def parse_networks(self):
|
||||
"""Return list of networks in the region
|
||||
|
||||
:param string region: Region name
|
||||
:returns: list of network data
|
||||
:rtype: list of models.VLANNetworkData
|
||||
"""
|
||||
|
|
@ -66,10 +69,9 @@ class BaseDataSourcePlugin(metaclass=abc.ABCMeta):
|
|||
return []
|
||||
|
||||
@abc.abstractmethod
|
||||
def get_ips(self, region, host):
|
||||
def parse_ips(self, host):
|
||||
"""Return list of IPs on the host
|
||||
|
||||
:param string region: Region name
|
||||
:param string host: Host name
|
||||
:returns: IPs per network on the host
|
||||
:rtype: models.IPList
|
||||
|
|
@ -82,7 +84,7 @@ class BaseDataSourcePlugin(metaclass=abc.ABCMeta):
|
|||
return {}
|
||||
|
||||
@abc.abstractmethod
|
||||
def get_dns_servers(self, region):
|
||||
def parse_dns_servers(self):
|
||||
"""Return the DNS servers
|
||||
|
||||
:param string region: Region name
|
||||
|
|
@ -93,10 +95,9 @@ class BaseDataSourcePlugin(metaclass=abc.ABCMeta):
|
|||
return []
|
||||
|
||||
@abc.abstractmethod
|
||||
def get_ntp_servers(self, region):
|
||||
def parse_ntp_servers(self):
|
||||
"""Return the NTP servers
|
||||
|
||||
:param string region: Region name
|
||||
:returns: NTP servers to be configured on host
|
||||
:rtype: models.ServerList
|
||||
"""
|
||||
|
|
@ -104,10 +105,9 @@ class BaseDataSourcePlugin(metaclass=abc.ABCMeta):
|
|||
return []
|
||||
|
||||
@abc.abstractmethod
|
||||
def get_ldap_information(self, region):
|
||||
def parse_ldap_information(self):
|
||||
"""Return the LDAP server information
|
||||
|
||||
:param string region: Region name
|
||||
:returns: LDAP server information
|
||||
:rtype: dict
|
||||
|
||||
|
|
@ -120,10 +120,9 @@ class BaseDataSourcePlugin(metaclass=abc.ABCMeta):
|
|||
return {}
|
||||
|
||||
@abc.abstractmethod
|
||||
def get_location_information(self, region):
|
||||
def parse_location_information(self):
|
||||
"""Return location information
|
||||
|
||||
:param string region: Region name
|
||||
:returns: Dict of location information
|
||||
:rtype: dict
|
||||
|
||||
|
|
@ -137,10 +136,9 @@ class BaseDataSourcePlugin(metaclass=abc.ABCMeta):
|
|||
return {}
|
||||
|
||||
@abc.abstractmethod
|
||||
def get_domain_name(self, region):
|
||||
def parse_domain_name(self):
|
||||
"""Return the Domain name
|
||||
|
||||
:param string region: Region name
|
||||
:returns: Domain name
|
||||
:rtype: str
|
||||
|
||||
|
|
@ -149,19 +147,7 @@ class BaseDataSourcePlugin(metaclass=abc.ABCMeta):
|
|||
|
||||
return ""
|
||||
|
||||
@abc.abstractmethod
|
||||
def get_site_info(self, region):
|
||||
"""Return site data as a SiteInfo object
|
||||
|
||||
:param region: Region name
|
||||
:return: general site data including location, domain, name, LDAP, NTP,
|
||||
DNS, and site type
|
||||
:rtype: models.SiteInfo
|
||||
"""
|
||||
|
||||
return None
|
||||
|
||||
def extract_baremetal_information(self):
|
||||
def parse_baremetal_information(self):
|
||||
"""Get baremetal information from plugin
|
||||
|
||||
:returns: racks and hosts as a list of Rack objects containing Host
|
||||
|
|
@ -170,9 +156,9 @@ class BaseDataSourcePlugin(metaclass=abc.ABCMeta):
|
|||
"""
|
||||
|
||||
LOG.info("Extract baremetal information from plugin")
|
||||
return self.get_racks(self.region)
|
||||
return self.parse_racks()
|
||||
|
||||
def extract_site_information(self):
|
||||
def parse_site_information(self):
|
||||
"""Get site information from plugin
|
||||
|
||||
:returns: site information including location, dns servers, ntp servers
|
||||
|
|
@ -185,18 +171,18 @@ class BaseDataSourcePlugin(metaclass=abc.ABCMeta):
|
|||
# Extract location information
|
||||
data = {
|
||||
'region_name': self.region,
|
||||
'dns': self.get_dns_servers(self.region),
|
||||
'ntp': self.get_ntp_servers(self.region),
|
||||
'ldap': self.get_ldap_information(self.region),
|
||||
'domain': self.get_domain_name(self.region)
|
||||
'dns': self.parse_dns_servers(),
|
||||
'ntp': self.parse_ntp_servers(),
|
||||
'ldap': self.parse_ldap_information(),
|
||||
'domain': self.parse_domain_name()
|
||||
}
|
||||
data.update(self.get_location_information(self.region) or {})
|
||||
data.update(self.parse_location_information() or {})
|
||||
|
||||
site_info = models.SiteInfo(**data)
|
||||
|
||||
return site_info
|
||||
|
||||
def extract_network_information(self):
|
||||
def parse_network_information(self):
|
||||
"""Get network details from plugin like Subnets, DNS, NTP and LDAP
|
||||
|
||||
:returns: networking data as a Network object
|
||||
|
|
@ -204,7 +190,7 @@ class BaseDataSourcePlugin(metaclass=abc.ABCMeta):
|
|||
"""
|
||||
|
||||
LOG.info("Extract network information from plugin")
|
||||
networks = self.get_networks(self.region)
|
||||
networks = self.parse_networks()
|
||||
|
||||
# We are interested in only the below networks mentioned in
|
||||
# networks_to_scan, so look for these networks from the data
|
||||
|
|
@ -225,21 +211,13 @@ class BaseDataSourcePlugin(metaclass=abc.ABCMeta):
|
|||
|
||||
return models.Network(desired_networks)
|
||||
|
||||
def extract_data(self):
|
||||
"""Extract data from plugin
|
||||
def parse_data_objects(self):
|
||||
"""Parses raw data into SiteDocumentData object"""
|
||||
self.data = models.SiteDocumentData(
|
||||
self.parse_site_information(), self.parse_network_information(),
|
||||
self.parse_baremetal_information())
|
||||
|
||||
Gather data related to baremetal, networks, storage and other site
|
||||
related information from plugin
|
||||
"""
|
||||
|
||||
LOG.info("Extract data from plugin")
|
||||
self.site_data = models.SiteDocumentData(
|
||||
self.extract_site_information(),
|
||||
self.extract_network_information(),
|
||||
self.extract_baremetal_information())
|
||||
return self.site_data
|
||||
|
||||
def apply_additional_data(self, extra_data):
|
||||
def merge_additional_data(self, extra_data):
|
||||
"""Apply any additional inputs from user
|
||||
|
||||
In case plugin does not provide some data, user can specify
|
||||
|
|
@ -250,4 +228,18 @@ class BaseDataSourcePlugin(metaclass=abc.ABCMeta):
|
|||
"""
|
||||
|
||||
LOG.info("Update site data with additional input")
|
||||
self.site_data.merge_additional_data(extra_data)
|
||||
self.data.merge_additional_data(extra_data)
|
||||
|
||||
def get_data(self, extra_data=None):
|
||||
"""Extract and return SiteDocumentData object
|
||||
|
||||
Load raw data, parse data objects, merge additional data if
|
||||
necessary, and return the resulting SiteDocumentData object
|
||||
"""
|
||||
|
||||
LOG.info("Extract data from plugin")
|
||||
self.load_raw_data()
|
||||
self.parse_data_objects()
|
||||
if extra_data:
|
||||
self.merge_additional_data(extra_data)
|
||||
return self.data
|
||||
|
|
|
|||
|
|
@ -86,7 +86,7 @@ def test_intermediary_processor_additional_config(
|
|||
mock_excel_plugin.return_value.site_data = {}
|
||||
result = intermediary_processor(plugin_name, **data)
|
||||
assert type(result) == ProcessDataSource
|
||||
mock_excel_plugin.return_value.apply_additional_data.\
|
||||
mock_excel_plugin.return_value.get_data.\
|
||||
assert_called_once_with(_get_site_config_data())
|
||||
|
||||
|
||||
|
|
|
|||
Loading…
Reference in New Issue