starlingx/distcloud
Code Issues Proposed changes

DC remove firewallrules audit from dcorch

Browse Source 
OAM firewallrules are now managed by Calico GlobalNetworkPolicy configuration
via k8s API (not by sysinv anymore). This update removed firewallrules
audit from dcorch.

Change-Id: I9fab73c016bb4af760c7d78f0db18dcc8bb77057
Closes-Bug: 1844147
Signed-off-by: Andy Ning <andy.ning@windriver.com>
This commit is contained in:
Andy Ning 2019-09-17 11:52:02 -04:00
parent 8c2bd5fa14
commit a7d91e2961
5 changed files with 4 additions and 223 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

8
dcorch/api/proxy/apps/controller.py
View File

@ -394,13 +394,9 @@ class SysinvAPIController(APIController):
request_header = self.get_request_header(environ)
operation_type = proxy_utils.get_operation_type(environ)
resource_type = self._get_resource_type_from_environ(environ)
# Firewall rule and certificate need special processing
# certificate need special processing
p_resource_info = 'suppressed'
if resource_type == consts.RESOURCE_TYPE_SYSINV_FIREWALL_RULES:
resource_info['payload'] = request_body
resource = json.loads(response.body)[resource_type]
resource_id = resource['firewall_sig']
elif resource_type == consts.RESOURCE_TYPE_SYSINV_CERTIFICATE:
if resource_type == consts.RESOURCE_TYPE_SYSINV_CERTIFICATE:
resource_info['payload'] = request_body
resource_info['content_type'] = environ.get('CONTENT_TYPE')
resource = json.loads(response.body)[resource_type]

6
dcorch/api/proxy/common/constants.py
View File

@ -98,10 +98,6 @@ REMOTELOGGING_PATHS = [
'/v1/remotelogging/{uuid}'
]
FIREWALLRULES_PATHS = [
'/v1/firewallrules/import_firewall_rules'
]
CERTIFICATE_PATHS = [
'/v1/certificate/certificate_install'
]
@ -118,7 +114,6 @@ SYSINV_PATH_MAP = {
consts.RESOURCE_TYPE_SYSINV_SNMP_TRAPDEST: TRAP_DEST_PATHS,
consts.RESOURCE_TYPE_SYSINV_SNMP_COMM: COMMUNITY_STRING_PATHS,
consts.RESOURCE_TYPE_SYSINV_REMOTE_LOGGING: REMOTELOGGING_PATHS,
consts.RESOURCE_TYPE_SYSINV_FIREWALL_RULES: FIREWALLRULES_PATHS,
consts.RESOURCE_TYPE_SYSINV_CERTIFICATE: CERTIFICATE_PATHS,
consts.RESOURCE_TYPE_SYSINV_USER: USER_PATHS,
}
@ -334,7 +329,6 @@ ROUTE_METHOD_MAP = {
consts.RESOURCE_TYPE_SYSINV_SNMP_TRAPDEST: ['POST', 'DELETE'],
consts.RESOURCE_TYPE_SYSINV_SNMP_COMM: ['POST', 'DELETE'],
consts.RESOURCE_TYPE_SYSINV_REMOTE_LOGGING: ['PATCH'],
consts.RESOURCE_TYPE_SYSINV_FIREWALL_RULES: ['POST'],
consts.RESOURCE_TYPE_SYSINV_CERTIFICATE: ['POST'],
consts.RESOURCE_TYPE_SYSINV_USER: ['PATCH', 'PUT'],
},

1
dcorch/common/consts.py
View File

@ -83,7 +83,6 @@ ORCH_REQUEST_ABORTED = "aborted"
# SysInv Resources
RESOURCE_TYPE_SYSINV_CERTIFICATE = "certificates"
RESOURCE_TYPE_SYSINV_DNS = "idns"
RESOURCE_TYPE_SYSINV_FIREWALL_RULES = "firewallrules"
RESOURCE_TYPE_SYSINV_NTP = "intp"
RESOURCE_TYPE_SYSINV_PTP = "ptp"
RESOURCE_TYPE_SYSINV_REMOTE_LOGGING = "remotelogging"

72
dcorch/drivers/openstack/sysinv_v1.py
View File

@ -11,9 +11,7 @@
# under the License.
import hashlib
import os
import six
import tsconfig.tsconfig as tsc
from cgtsclient import client as cgts_client
from cgtsclient.exc import HTTPConflict
@ -470,76 +468,6 @@ class SysinvClient(base.DriverBase):
return remotelogging
def get_firewallrules(self):
"""Get the firewallrules for this region
:return: firewallrules
"""
try:
firewallruless = self.client.firewallrules.list()
firewallrules = firewallruless[0]
except Exception as e:
LOG.error("get_firewallrules region={} "
"exception={}".format(self.region_name, e))
raise exceptions.SyncRequestFailedRetry()
if not firewallrules:
LOG.info("firewallrules is None for region: {}".format(
self.region_name))
else:
LOG.info("get_firewallrules uuid=%s firewall_sig=%s" %
(firewallrules.uuid, firewallrules.firewall_sig))
return firewallrules
def _validate_firewallrules(self, firewall_sig, firewallrules):
firewallrules_sig = hashlib.md5(firewallrules).hexdigest()
if firewallrules_sig == firewall_sig:
return True
LOG.info("_validate_firewallrules region={} sig={} mismatch "
"reference firewall_sig={}".format(
self.region_name, firewallrules_sig, firewall_sig))
return False
def update_firewallrules(self,
firewall_sig,
firewallrules=None):
"""Update the firewallrules for this region
:param: firewall_sig
:param: firewallrules
:return: ifirewallrules
"""
if not firewallrules:
# firewallrules not provided, obtain from SystemController
firewall_rules_file = os.path.join(
tsc.CONFIG_PATH,
sysinv_constants.FIREWALL_RULES_FILE)
with open(firewall_rules_file, 'r') as content_file:
firewallrules = content_file.read()
LOG.info("update_firewallrules from shared file={}".format(
firewallrules))
if not self._validate_firewallrules(firewall_sig, firewallrules):
raise exceptions.SyncRequestFailedRetry()
try:
ifirewallrules = self.client.firewallrules.import_firewall_rules(
firewallrules)
LOG.info("region={} firewallrules uuid={} firewall_sig={}".format(
self.region_name, ifirewallrules.get('uuid'), firewall_sig))
except Exception as e:
LOG.error("update_firewallrules exception={}".format(e))
raise exceptions.SyncRequestFailedRetry()
return ifirewallrules
def get_certificates(self):
"""Get the certificates for this region

140
dcorch/engine/sync_services/sysinv.py
View File

@ -44,11 +44,9 @@ class SysinvSyncThread(SyncThread):
SYSINV_ADD_DELETE_RESOURCES = [consts.RESOURCE_TYPE_SYSINV_SNMP_COMM,
consts.RESOURCE_TYPE_SYSINV_SNMP_TRAPDEST]
SYSINV_CREATE_RESOURCES = [consts.RESOURCE_TYPE_SYSINV_FIREWALL_RULES,
consts.RESOURCE_TYPE_SYSINV_CERTIFICATE,
SYSINV_CREATE_RESOURCES = [consts.RESOURCE_TYPE_SYSINV_CERTIFICATE,
consts.RESOURCE_TYPE_SYSINV_FERNET_REPO]
FIREWALL_SIG_NULL = 'NoCustomFirewallRules'
CERTIFICATE_SIG_NULL = 'NoCertificate'
RESOURCE_UUID_NULL = 'NoResourceUUID'
@ -66,8 +64,6 @@ class SysinvSyncThread(SyncThread):
self.sync_snmp_trapdest,
consts.RESOURCE_TYPE_SYSINV_REMOTE_LOGGING:
self.sync_remotelogging,
consts.RESOURCE_TYPE_SYSINV_FIREWALL_RULES:
self.sync_firewallrules,
consts.RESOURCE_TYPE_SYSINV_CERTIFICATE:
self.sync_certificate,
consts.RESOURCE_TYPE_SYSINV_USER: self.sync_user,
@ -81,7 +77,6 @@ class SysinvSyncThread(SyncThread):
self.audit_resources = [
consts.RESOURCE_TYPE_SYSINV_CERTIFICATE,
consts.RESOURCE_TYPE_SYSINV_DNS,
consts.RESOURCE_TYPE_SYSINV_FIREWALL_RULES,
consts.RESOURCE_TYPE_SYSINV_NTP,
consts.RESOURCE_TYPE_SYSINV_PTP,
consts.RESOURCE_TYPE_SYSINV_REMOTE_LOGGING,
@ -531,78 +526,6 @@ class SysinvSyncThread(SyncThread):
iremotelogging.uuid),
extra=self.log_extra)
def update_firewallrules(self, firewall_sig, firewallrules=None):
s_os_client = sdk.OpenStackDriver(self.region_name)
try:
ifirewallrules = s_os_client.sysinv_client.update_firewallrules(
firewall_sig, firewallrules=firewallrules)
return ifirewallrules
except (exceptions.ConnectionRefused, exceptions.NotAuthorized,
exceptions.TimeOut):
LOG.info("update_firewallrules exception Timeout",
extra=self.log_extra)
s_os_client.delete_region_clients(self.region_name)
raise exceptions.SyncRequestTimeout
except (AttributeError, TypeError) as e:
LOG.info("update_firewallrules error {} region_name".format(e),
extra=self.log_extra)
s_os_client.delete_region_clients(self.region_name,
clear_token=True)
raise exceptions.SyncRequestFailedRetry
except Exception as e:
LOG.exception(e)
raise exceptions.SyncRequestFailedRetry
def sync_firewallrules(self, request, rsrc):
# The system is not created with default firewallrules
LOG.info("sync_firewallrules resource_info={}".format(
request.orch_job.resource_info),
extra=self.log_extra)
firewallrules_dict = jsonutils.loads(request.orch_job.resource_info)
payload = firewallrules_dict.get('payload')
# payload is the contents of the POST operation
if not payload:
LOG.info("sync_firewallrules No payload found in resource_info"
"{}".format(request.orch_job.resource_info),
extra=self.log_extra)
return
if isinstance(payload, dict):
firewall_sig = payload.get('firewall_sig')
else:
firewall_sig = rsrc.master_id
LOG.info("firewall_sig from master_id={}".format(firewall_sig))
ifirewallrules = None
if firewall_sig:
ifirewallrules = self.update_firewallrules(firewall_sig)
else:
firewall_sig = rsrc.master_id
if firewall_sig and firewall_sig != self.FIREWALL_SIG_NULL:
ifirewallrules = self.update_firewallrules(
firewall_sig,
firewallrules=payload)
else:
LOG.info("skipping firewall_sig={}".format(firewall_sig))
ifirewallrules_sig = None
try:
ifirewallrules_sig = \
ifirewallrules.get('firewallrules').get('firewall_sig')
except Exception as e:
LOG.warn("No ifirewallrules={} unknown e={}".format(
ifirewallrules, e))
# Ensure subcloud resource is persisted to the DB for later
subcloud_rsrc_id = self.persist_db_subcloud_resource(
rsrc.id, firewall_sig)
LOG.info("firewallrules {} {} [{}/{}] updated".format(rsrc.id,
subcloud_rsrc_id, ifirewallrules_sig, firewall_sig),
extra=self.log_extra)
def update_certificate(self, signature, certificate=None, data=None):
s_os_client = sdk.OpenStackDriver(self.region_name)
@ -869,8 +792,6 @@ class SysinvSyncThread(SyncThread):
return self.get_snmp_trapdest_resources(os_client)
elif resource_type == consts.RESOURCE_TYPE_SYSINV_REMOTE_LOGGING:
return [self.get_remotelogging_resource(os_client)]
elif resource_type == consts.RESOURCE_TYPE_SYSINV_FIREWALL_RULES:
return [self.get_firewallrules_resource(os_client)]
elif resource_type == consts.RESOURCE_TYPE_SYSINV_CERTIFICATE:
return self.get_certificates_resources(os_client)
elif resource_type == consts.RESOURCE_TYPE_SYSINV_USER:
@ -896,8 +817,6 @@ class SysinvSyncThread(SyncThread):
return self.get_snmp_trapdest_resources(os_client)
elif resource_type == consts.RESOURCE_TYPE_SYSINV_REMOTE_LOGGING:
return [self.get_remotelogging_resource(os_client)]
elif resource_type == consts.RESOURCE_TYPE_SYSINV_FIREWALL_RULES:
return [self.get_firewallrules_resource(os_client)]
elif resource_type == consts.RESOURCE_TYPE_SYSINV_CERTIFICATE:
return self.get_certificates_resources(os_client)
elif resource_type == consts.RESOURCE_TYPE_SYSINV_USER:
@ -1036,27 +955,6 @@ class SysinvSyncThread(SyncThread):
LOG.exception(e)
return None
def get_firewallrules_resource(self, os_client):
try:
ifirewallrules = os_client.sysinv_client.get_firewallrules()
return ifirewallrules
except (keystone_exceptions.connection.ConnectTimeout,
keystone_exceptions.ConnectFailure) as e:
LOG.info("get_firewallrules: subcloud {} is not reachable [{}]"
.format(self.subcloud_engine.subcloud.region_name,
str(e)), extra=self.log_extra)
# None will force skip of audit
os_client.delete_region_clients(self.region_name)
return None
except (AttributeError, TypeError) as e:
LOG.info("get_firewallrules_resource error {}".format(e),
extra=self.log_extra)
os_client.delete_region_clients(self.region_name, clear_token=True)
return None
except Exception as e:
LOG.exception(e)
return None
def get_certificates_resources(self, os_client):
try:
return os_client.sysinv_client.get_certificates()
@ -1130,21 +1028,6 @@ class SysinvSyncThread(SyncThread):
"community".format(resource),
extra=self.log_extra)
return resource.ip_address
elif resource_type == consts.RESOURCE_TYPE_SYSINV_FIREWALL_RULES:
if hasattr(resource, 'firewall_sig'):
LOG.info("get_resource_id firewall_sig={}".format(
resource.firewall_sig))
if resource.firewall_sig is None:
return self.FIREWALL_SIG_NULL # master_id cannot be None
return resource.firewall_sig
elif hasattr(resource, 'master_id'):
LOG.info("get_resource_id master_id firewall_sig={}".format(
resource.master_id))
if resource.master_id is None:
return self.FIREWALL_SIG_NULL # master_id cannot be None
return resource.master_id
else:
LOG.error("no get_resource_id for firewall")
elif resource_type == consts.RESOURCE_TYPE_SYSINV_CERTIFICATE:
if hasattr(resource, 'signature'):
LOG.info("get_resource_id signature={}".format(
@ -1234,19 +1117,6 @@ class SysinvSyncThread(SyncThread):
i1.transport == i2.transport and
i1.port == i2.port)
def same_firewallrules(self, i1, i2):
LOG.debug("same_firewallrules i1={}, i2={}".format(i1, i2),
extra=self.log_extra)
same = True
if i1.firewall_sig and (i1.firewall_sig != i2.firewall_sig):
if i1.firewall_sig == self.FIREWALL_SIG_NULL:
return True
LOG.info("same_firewallrules differ i1={}, i2={}".format(i1, i2),
extra=self.log_extra)
same = False
return same
def same_certificate(self, i1, i2):
LOG.debug("same_certificate i1={}, i2={}".format(i1, i2),
extra=self.log_extra)
@ -1296,8 +1166,6 @@ class SysinvSyncThread(SyncThread):
return self.same_snmp_trapdest(m_resource, sc_resource)
elif resource_type == consts.RESOURCE_TYPE_SYSINV_REMOTE_LOGGING:
return self.same_remotelogging(m_resource, sc_resource)
elif resource_type == consts.RESOURCE_TYPE_SYSINV_FIREWALL_RULES:
return self.same_firewallrules(m_resource, sc_resource)
elif resource_type == consts.RESOURCE_TYPE_SYSINV_CERTIFICATE:
return self.same_certificate(m_resource, sc_resource)
elif resource_type == consts.RESOURCE_TYPE_SYSINV_USER:
@ -1363,10 +1231,7 @@ class SysinvSyncThread(SyncThread):
resource_id = self.get_resource_id(resource_type, resource)
if finding == AUDIT_RESOURCE_MISSING:
# default action is create for a 'missing' resource
if resource_id == self.FIREWALL_SIG_NULL:
LOG.info("No custom firewall resource to sync")
return num_of_audit_jobs
elif resource_id == self.CERTIFICATE_SIG_NULL:
if resource_id == self.CERTIFICATE_SIG_NULL:
LOG.info("No certificate resource to sync")
return num_of_audit_jobs
elif resource_id == self.RESOURCE_UUID_NULL:
@ -1396,7 +1261,6 @@ class SysinvSyncThread(SyncThread):
consts.RESOURCE_TYPE_SYSINV_SNMP_COMM,
consts.RESOURCE_TYPE_SYSINV_SNMP_TRAPDEST,
consts.RESOURCE_TYPE_SYSINV_REMOTE_LOGGING,
consts.RESOURCE_TYPE_SYSINV_FIREWALL_RULES,
consts.RESOURCE_TYPE_SYSINV_CERTIFICATE,
consts.RESOURCE_TYPE_SYSINV_USER,
]