diff --git a/devstack/plugin.sh b/devstack/plugin.sh index 73a7e8c0c2..a5f9290c8f 100644 --- a/devstack/plugin.sh +++ b/devstack/plugin.sh @@ -34,8 +34,14 @@ fi if [[ $Q_PLUGIN == 'vmware_nsx_v' ]]; then source $dir/lib/vmware_nsx_v if [[ "$1" == "unstack" ]]; then - python $dir/tools/nsxv_cleanup.py --vsm-ip ${NSXV_MANAGER_URI/https:\/\/} --user $NSXV_USER --password $NSXV_PASSWORD + db_connection=$(iniget $NEUTRON_CONF database connection) + python $dir/tools/nsxv_cleanup.py --vsm-ip ${NSXV_MANAGER_URI/https:\/\/} --user $NSXV_USER --password $NSXV_PASSWORD --db-connection $db_connection + elif [[ "$1" == "clean" ]]; then + if is_service_enabled q-svc || is_service_enabled neutron-api; then + python $dir/tools/nsxv_cleanup.py --vsm-ip ${NSXV_MANAGER_URI/https:\/\/} --user $NSXV_USER --password $NSXV_PASSWORD + fi fi + elif [[ $Q_PLUGIN == 'vmware_nsx' ]]; then source $dir/lib/vmware_nsx if [[ "$1" == "stack" && "$2" == "post-config" ]]; then @@ -50,6 +56,7 @@ elif [[ $Q_PLUGIN == 'vmware_nsx_v3' ]]; then if [[ "$1" == "stack" && "$2" == "post-config" ]]; then init_vmware_nsx_v3 elif [[ "$1" == "unstack" ]]; then + db_connection=$(iniget $NEUTRON_CONF database connection) stop_vmware_nsx_v3 # only clean up when q-svc (legacy support) or neutron-api is enabled if is_service_enabled q-svc || is_service_enabled neutron-api; then @@ -57,6 +64,10 @@ elif [[ $Q_PLUGIN == 'vmware_nsx_v3' ]]; then IFS=',' NSX_MANAGER=($NSX_MANAGER) unset IFS + python $dir/tools/nsxv3_cleanup.py --mgr-ip $NSX_MANAGER --user $NSX_USER --password $NSX_PASSWORD --db-connection $db_connection + fi + elif [[ "$1" == 'clean' ]]; then + if is_service_enabled q-svc || is_service_enabled neutron-api; then python $dir/tools/nsxv3_cleanup.py --mgr-ip $NSX_MANAGER --user $NSX_USER --password $NSX_PASSWORD fi fi diff --git a/devstack/tools/nsxv3_cleanup.py b/devstack/tools/nsxv3_cleanup.py index b040cf9fbe..0d00c4b3da 100755 --- a/devstack/tools/nsxv3_cleanup.py +++ b/devstack/tools/nsxv3_cleanup.py @@ -16,20 +16,56 @@ import base64 import optparse import requests +import sqlalchemy as sa from oslo_serialization import jsonutils import six.moves.urllib.parse as urlparse +from vmware_nsx.db import nsx_models requests.packages.urllib3.disable_warnings() +class NeutronNsxDB(object): + def __init__(self, db_connection): + super(NeutronNsxDB, self).__init__() + engine = sa.create_engine(db_connection) + self.session = sa.orm.session.sessionmaker()(bind=engine) + + def query_all(self, column, model): + return list(set([r[column] for r in self.session.query(model).all()])) + + def get_logical_ports(self): + return self.query_all('nsx_port_id', + nsx_models.NeutronNsxPortMapping) + + def get_nsgroups(self): + return self.query_all('nsx_id', + nsx_models.NeutronNsxSecurityGroupMapping) + + def get_firewall_sections(self): + return self.query_all('nsx_id', + nsx_models.NeutronNsxFirewallSectionMapping) + + def get_logical_routers(self): + return self.query_all('nsx_id', + nsx_models.NeutronNsxRouterMapping) + + def get_logical_switches(self): + return self.query_all('nsx_id', + nsx_models.NeutronNsxNetworkMapping) + + def get_logical_dhcp_servers(self): + return self.query_all('nsx_service_id', + nsx_models.NeutronNsxServiceBinding) + + class NSXClient(object): """Base NSX REST client""" API_VERSION = "v1" NULL_CURSOR_PREFIX = '0000' - def __init__(self, host, username, password, *args, **kwargs): + def __init__(self, host, username, password, db_connection): self.host = host self.username = username self.password = password @@ -43,6 +79,8 @@ class NSXClient(object): self.url = None self.headers = None self.api_version = NSXClient.API_VERSION + self.neutron_db = (NeutronNsxDB(db_connection) + if db_connection else None) self.__set_headers() @@ -169,8 +207,12 @@ class NSXClient(object): """ Retrieve all logical ports created from OpenStack """ - lports = self.get_logical_ports() - return self.get_os_resources(lports) + lports = self.get_os_resources( + self.get_logical_ports()) + if self.neutron_db: + db_lports = self.neutron_db.get_logical_ports() + lports = [lp for lp in lports if lp['id'] in db_lports] + return lports def update_logical_port_attachment(self, lports): """ @@ -188,8 +230,7 @@ class NSXClient(object): """ Delete all logical ports created by OpenStack """ - lports = self.get_logical_ports() - os_lports = self.get_os_resources(lports) + os_lports = self.get_os_logical_ports() print("Number of OS Logical Ports to be deleted: %s" % len(os_lports)) # logical port vif detachment self.update_logical_port_attachment(os_lports) @@ -221,8 +262,14 @@ class NSXClient(object): """ Retrieve all logical switches created from OpenStack """ - lswitches = self.get_logical_switches() - return self.get_os_resources(lswitches) + lswitches = self.get_os_resources( + self.get_logical_switches()) + + if self.neutron_db: + db_lswitches = self.neutron_db.get_logical_switches() + lswitches = [ls for ls in lswitches + if ls['id'] in db_lswitches] + return lswitches def get_lswitch_ports(self, ls_id): """ @@ -258,8 +305,13 @@ class NSXClient(object): """ Retrieve all firewall sections created from OpenStack """ - fw_sections = self.get_firewall_sections() - return self.get_os_resources(fw_sections) + fw_sections = self.get_os_resources( + self.get_firewall_sections()) + if self.neutron_db: + db_sections = self.neutron_db.get_firewall_sections() + fw_sections = [fws for fws in fw_sections + if fws['id'] in db_sections] + return fw_sections def get_firewall_section_rules(self, fw_section): """ @@ -306,8 +358,13 @@ class NSXClient(object): """ Retrieve all NSGroups on NSX backend """ - ns_groups = self.get_list_results(endpoint="/ns-groups") - return self.get_os_resources(ns_groups) + ns_groups = self.get_os_resources( + self.get_list_results(endpoint="/ns-groups")) + if self.neutron_db: + db_nsgroups = self.neutron_db.get_nsgroups() + ns_groups = [nsg for nsg in ns_groups + if nsg['id'] in db_nsgroups] + return ns_groups def cleanup_os_ns_groups(self): """ @@ -333,8 +390,11 @@ class NSXClient(object): """ Retrieve all Switching Profiles created from OpenStack """ - sw_profiles = self.get_switching_profiles() - return self.get_os_resources(sw_profiles) + sw_profiles = self.get_os_resources( + self.get_switching_profiles()) + if self.neutron_db: + sw_profiles = [] + return sw_profiles def cleanup_os_switching_profiles(self): """ @@ -362,7 +422,13 @@ class NSXClient(object): endpoint = "/logical-routers?router_type=%s" % tier else: endpoint = "/logical-routers" - return self.get_list_results(endpoint=endpoint) + lrouters = self.get_list_results(endpoint=endpoint) + + if self.neutron_db: + db_routers = self.neutron_db.get_logical_routers() + lrouters = [lr for lr in lrouters + if lr['id'] in db_routers] + return lrouters def get_os_logical_routers(self): """ @@ -438,8 +504,14 @@ class NSXClient(object): """ Retrieve all logical DHCP servers created from OpenStack """ - dhcp_servers = self.get_logical_dhcp_servers() - return self.get_os_resources(dhcp_servers) + dhcp_servers = self.get_os_resources( + self.get_logical_dhcp_servers()) + + if self.neutron_db: + db_dhcp_servers = self.neutron_db.get_logical_dhcp_servers() + dhcp_servers = [srv for srv in dhcp_servers + if srv['id'] in db_dhcp_servers] + return dhcp_servers def cleanup_os_logical_dhcp_servers(self): """ @@ -487,10 +559,13 @@ if __name__ == "__main__": help="NSX Manager username") parser.add_option("-p", "--password", default="default", dest="password", help="NSX Manager password") + parser.add_option("--db-connection", default="", dest="db_connection", + help=("When set, cleaning only backend resources that " + "have db record.")) (options, args) = parser.parse_args() # Get NSX REST client nsx_client = NSXClient(options.mgr_ip, options.username, - options.password) + options.password, options.db_connection) # Clean all objects created by OpenStack nsx_client.cleanup_all() diff --git a/devstack/tools/nsxv_cleanup.py b/devstack/tools/nsxv_cleanup.py index 96ed14226a..cc5d01b221 100644 --- a/devstack/tools/nsxv_cleanup.py +++ b/devstack/tools/nsxv_cleanup.py @@ -52,19 +52,53 @@ Tong Liu import base64 import optparse import requests +import sqlalchemy as sa import sys + from oslo_serialization import jsonutils +from vmware_nsx.db import nsx_models +from vmware_nsx.db import nsxv_models requests.packages.urllib3.disable_warnings() +class NeutronNsxDB(object): + def __init__(self, db_connection): + super(NeutronNsxDB, self).__init__() + engine = sa.create_engine(db_connection) + self.session = sa.orm.session.sessionmaker()(bind=engine) + + def query_all(self, column, model): + return list(set([r[column] for r in self.session.query(model).all()])) + + def query_all_firewall_sections(self): + return self.query_all('ip_section_id', + nsxv_models.NsxvSecurityGroupSectionMapping) + + def query_all_security_groups(self): + return self.query_all('nsx_id', + nsx_models.NeutronNsxSecurityGroupMapping) + + def query_all_logical_switches(self): + return self.query_all('nsx_id', + nsx_models.NeutronNsxNetworkMapping) + + def query_all_spoofguard_policies(self): + return self.query_all('policy_id', + nsxv_models.NsxvSpoofGuardPolicyNetworkMapping) + + def query_all_edges(self): + return self.query_all('edge_id', + nsxv_models.NsxvRouterBinding) + + class VSMClient(object): """Base VSM REST client """ API_VERSION = "2.0" - def __init__(self, host, username, password, *args, **kwargs): - self.force = True if 'force' in kwargs else False + def __init__(self, host, username, password, db_connection, force): + self.force = force self.host = host self.username = username self.password = password @@ -78,7 +112,8 @@ class VSMClient(object): self.url = None self.headers = None self.api_version = VSMClient.API_VERSION - + self.neutron_db = (NeutronNsxDB(db_connection) if db_connection + else None) self.__set_headers() def __set_endpoint(self, endpoint): @@ -192,6 +227,11 @@ class VSMClient(object): temp_lswitches = response.json()['dataPage']['data'] lswitches += temp_lswitches + if self.neutron_db: + db_lswitches = self.neutron_db.query_all_logical_switches() + lswitches = [ls for ls in lswitches + if ls['objectId'] in db_lswitches] + return lswitches def cleanup_logical_switch(self): @@ -224,6 +264,10 @@ class VSMClient(object): print("ERROR: wrong response status code! Exiting...") sys.exit() + if self.neutron_db: + db_sections = self.neutron_db.query_all_firewall_sections() + firewall_sections = [fws for fws in firewall_sections if fws['id'] + in db_sections] return firewall_sections def cleanup_firewall_section(self): @@ -254,6 +298,11 @@ class VSMClient(object): # related to any security group created by OpenStack security_groups = [sg for sg in sg_all if sg['name'] != "Activity Monitoring Data Collection"] + + if self.neutron_db: + db_sgs = self.neutron_db.query_all_security_groups() + security_groups = [sg for sg in security_groups + if sg['objectId'] in db_sgs] return security_groups def cleanup_security_group(self): @@ -280,6 +329,10 @@ class VSMClient(object): sgp_all = response.json() policies = [sgp for sgp in sgp_all['policies'] if sgp['name'] != 'Default Policy'] + + if self.neutron_db: + db_policies = self.neutron_db.query_all_spoofguard_policies() + policies = [p for p in policies if p['policyId'] in db_policies] return policies def cleanup_spoofguard_policies(self): @@ -313,6 +366,10 @@ class VSMClient(object): temp_edges = response.json()['edgePage']['data'] edges += temp_edges + if self.neutron_db: + db_edges = self.neutron_db.query_all_edges() + edges = [e for e in edges if e['id'] in db_edges] + return edges def cleanup_edge(self): @@ -350,20 +407,20 @@ if __name__ == "__main__": help="NSX Manager username") parser.add_option("-p", "--password", default="default", dest="password", help="NSX Manager password") + parser.add_option("--db-connection", dest="db_connection", default="", + help=("When set, cleaning only backend resources that " + "have db record.")) parser.add_option("-f", "--force", dest="force", action="store_true", help="Force cleanup option") (options, args) = parser.parse_args() print("vsm-ip: %s" % options.vsm_ip) print("username: %s" % options.username) print("password: %s" % options.password) + print("db-connection: %s" % options.db_connection) print("force: %s" % options.force) # Get VSM REST client - if options.force: - vsm_client = VSMClient(options.vsm_ip, options.username, - options.password, force=options.force) - else: - vsm_client = VSMClient(options.vsm_ip, options.username, - options.password) + vsm_client = VSMClient(options.vsm_ip, options.username, options.password, + options.db_connection, options.force) # Clean all objects created by OpenStack vsm_client.cleanup_all()