From f5eeee3e35e0b870b96d60e380f9bb129362c3bb Mon Sep 17 00:00:00 2001 From: Joseph Richard Date: Wed, 1 Aug 2018 15:22:50 -0400 Subject: [PATCH] Dynamically load existing puppet plugins This commit moves each of the existing plugins to be dynamically loaded through stevedore, rather than statically listed inside of sysinv puppet handling code. Story: 2003194 Task: 24722 Change-Id: I17abe8c481e71265cb3ae07474788562035a4823 Signed-off-by: Joseph Richard --- sysinv/sysinv/sysinv/setup.cfg | 34 +++ sysinv/sysinv/sysinv/sysinv/puppet/base.py | 4 + sysinv/sysinv/sysinv/sysinv/puppet/cinder.py | 3 + .../sysinv/sysinv/sysinv/puppet/kubernetes.py | 56 ++-- sysinv/sysinv/sysinv/sysinv/puppet/neutron.py | 4 + sysinv/sysinv/sysinv/sysinv/puppet/nfv.py | 3 + .../sysinv/sysinv/sysinv/puppet/platform.py | 4 +- sysinv/sysinv/sysinv/sysinv/puppet/puppet.py | 249 +----------------- sysinv/sysinv/sysinv/sysinv/puppet/smapi.py | 6 + 9 files changed, 87 insertions(+), 276 deletions(-) diff --git a/sysinv/sysinv/sysinv/setup.cfg b/sysinv/sysinv/sysinv/setup.cfg index e4da474291..df1e1af80a 100644 --- a/sysinv/sysinv/sysinv/setup.cfg +++ b/sysinv/sysinv/sysinv/setup.cfg @@ -37,6 +37,40 @@ console_scripts = sysinv-puppet = sysinv.cmd.puppet:main sysinv-helm = sysinv.cmd.helm:main +systemconfig.puppet_plugins = + 001_platform = sysinv.puppet.platform:PlatformPuppet + 002_interface = sysinv.puppet.interface:InterfacePuppet + 003_ovs = sysinv.puppet.ovs:OVSPuppet + 004_networking = sysinv.puppet.networking:NetworkingPuppet + 005_patching = sysinv.puppet.patching:PatchingPuppet + 006_mtce = sysinv.puppet.mtce:MtcePuppet + 007_keystone = sysinv.puppet.keystone:KeystonePuppet + 008_ldap = sysinv.puppet.ldap:LdapPuppet + 009_sysinv = sysinv.puppet.inventory:SystemInventoryPuppet + 010_nfv = sysinv.puppet.nfv:NfvPuppet + 011_ceph = sysinv.puppet.ceph:CephPuppet + 012_device = sysinv.puppet.device:DevicePuppet + 013_nova = sysinv.puppet.nova:NovaPuppet + 014_neutron = sysinv.puppet.neutron:NeutronPuppet + 015_horizon = sysinv.puppet.horizon:HorizonPuppet + 016_glance = sysinv.puppet.glance:GlancePuppet + 017_gnocchi = sysinv.puppet.gnocchi:GnocchiPuppet + 018_cinder = sysinv.puppet.cinder:CinderPuppet + 019_ceilometer = sysinv.puppet.ceilometer:CeilometerPuppet + 020_aodh = sysinv.puppet.aodh:AodhPuppet + 021_heat = sysinv.puppet.heat:HeatPuppet + 022_magnum = sysinv.puppet.magnum:MagnumPuppet + 023_murano = sysinv.puppet.murano:MuranoPuppet + 024_storage = sysinv.puppet.storage:StoragePuppet + 025_ironic = sysinv.puppet.ironic:IronicPuppet + 026_panko = sysinv.puppet.panko:PankoPuppet + 027_dcmanager = sysinv.puppet.dcmanager:DCManagerPuppet + 028_dcorch = sysinv.puppet.dcorch:DCOrchPuppet + 029_kubernetes = sysinv.puppet.kubernetes:KubernetesPuppet + 030_smapi = sysinv.puppet.smapi:SmPuppet + 031_fm = sysinv.puppet.fm:FmPuppet + 032_service_parameter = sysinv.puppet.service_parameter:ServiceParamPuppet + [pbr] autodoc_index_modules = True diff --git a/sysinv/sysinv/sysinv/sysinv/puppet/base.py b/sysinv/sysinv/sysinv/sysinv/puppet/base.py index 428fdf62d5..fbed83f0f4 100644 --- a/sysinv/sysinv/sysinv/sysinv/puppet/base.py +++ b/sysinv/sysinv/sysinv/sysinv/puppet/base.py @@ -44,6 +44,10 @@ class BasePuppet(object): def dbapi(self): return self._operator.dbapi + @property + def config_uuid(self): + return self._operator.config_uuid + @property def context(self): return self._operator.context diff --git a/sysinv/sysinv/sysinv/sysinv/puppet/cinder.py b/sysinv/sysinv/sysinv/sysinv/puppet/cinder.py index 62b3964500..0839a11de6 100644 --- a/sysinv/sysinv/sysinv/sysinv/puppet/cinder.py +++ b/sysinv/sysinv/sysinv/sysinv/puppet/cinder.py @@ -587,6 +587,9 @@ class CinderPuppet(openstack.OpenstackBasePuppet): return config def get_host_config(self, host): + if (constants.CONTROLLER not in utils.get_personalities(host)): + return {} + cinder_device, cinder_size_gib = utils._get_cinder_device_info(self.dbapi, host.id) config = {} if cinder_device: diff --git a/sysinv/sysinv/sysinv/sysinv/puppet/kubernetes.py b/sysinv/sysinv/sysinv/sysinv/puppet/kubernetes.py index c53b8e86a3..a916277d95 100644 --- a/sysinv/sysinv/sysinv/sysinv/puppet/kubernetes.py +++ b/sysinv/sysinv/sysinv/sysinv/puppet/kubernetes.py @@ -65,34 +65,36 @@ class KubernetesPuppet(base.BasePuppet): def get_host_config(self, host): config = {} - if self._kubernetes_enabled(): - if host.personality == constants.COMPUTE: - create_node = False - try: - # Check if this host has already been configured as a - # kubernetes node. - cmd = ['kubectl', - '--kubeconfig=/etc/kubernetes/admin.conf', - 'get', 'node', host.hostname] - subprocess.check_call(cmd) - except subprocess.CalledProcessError: - # The node does not exist - create_node = True + if host.personality != constants.COMPUTE: + return config - if create_node: - try: - # Generate the token and join command for this host. - cmd = ['kubeadm', 'token', 'create', - '--print-join-command', '--description', - 'Bootstrap token for %s' % host.hostname] - join_cmd = subprocess.check_output(cmd) - config.update( - {'platform::kubernetes::worker::params::join_cmd': - join_cmd, - }) - except subprocess.CalledProcessError: - raise exception.SysinvException( - 'Failed to generate bootstrap token') + if self._kubernetes_enabled(): + create_node = False + try: + # Check if this host has already been configured as a + # kubernetes node. + cmd = ['kubectl', + '--kubeconfig=/etc/kubernetes/admin.conf', + 'get', 'node', host.hostname] + subprocess.check_call(cmd) + except subprocess.CalledProcessError: + # The node does not exist + create_node = True + + if create_node: + try: + # Generate the token and join command for this host. + cmd = ['kubeadm', 'token', 'create', + '--print-join-command', '--description', + 'Bootstrap token for %s' % host.hostname] + join_cmd = subprocess.check_output(cmd) + config.update( + {'platform::kubernetes::worker::params::join_cmd': + join_cmd, + }) + except subprocess.CalledProcessError: + raise exception.SysinvException( + 'Failed to generate bootstrap token') return config diff --git a/sysinv/sysinv/sysinv/sysinv/puppet/neutron.py b/sysinv/sysinv/sysinv/sysinv/puppet/neutron.py index 725ebf2a69..9b8ba72409 100644 --- a/sysinv/sysinv/sysinv/sysinv/puppet/neutron.py +++ b/sysinv/sysinv/sysinv/sysinv/puppet/neutron.py @@ -156,6 +156,10 @@ class NeutronPuppet(openstack.OpenstackBasePuppet): } def get_host_config(self, host): + if (constants.CONTROLLER not in utils.get_personalities(host) and + constants.COMPUTE not in utils.get_personalities(host)): + return {} + device_mappings = [] for iface in self.context['interfaces'].values(): if (utils.get_primary_network_type(iface) == diff --git a/sysinv/sysinv/sysinv/sysinv/puppet/nfv.py b/sysinv/sysinv/sysinv/sysinv/puppet/nfv.py index d4c9e1b601..a1dc7c2980 100644 --- a/sysinv/sysinv/sysinv/sysinv/puppet/nfv.py +++ b/sysinv/sysinv/sysinv/sysinv/puppet/nfv.py @@ -5,6 +5,7 @@ # from sysinv.common import constants +from sysinv.common import utils from . import openstack @@ -103,6 +104,8 @@ class NfvPuppet(openstack.OpenstackBasePuppet): } def get_host_config(self, host): + if (constants.CONTROLLER not in utils.get_personalities(host)): + return {} database_dir = "/opt/platform/nfv/vim/%s" % host.software_load return { 'nfv::vim::database_dir': database_dir, diff --git a/sysinv/sysinv/sysinv/sysinv/puppet/platform.py b/sysinv/sysinv/sysinv/sysinv/puppet/platform.py index e7aa5581cb..a348481bce 100644 --- a/sysinv/sysinv/sysinv/sysinv/puppet/platform.py +++ b/sysinv/sysinv/sysinv/sysinv/puppet/platform.py @@ -58,9 +58,9 @@ class PlatformPuppet(base.BasePuppet): config.update(self._get_user_config()) return config - def get_host_config(self, host, config_uuid): + def get_host_config(self, host): config = {} - config.update(self._get_host_platform_config(host, config_uuid)) + config.update(self._get_host_platform_config(host, self.config_uuid)) config.update(self._get_host_ntp_config(host)) config.update(self._get_host_ptp_config(host)) config.update(self._get_host_sysctl_config(host)) diff --git a/sysinv/sysinv/sysinv/sysinv/puppet/puppet.py b/sysinv/sysinv/sysinv/sysinv/puppet/puppet.py index cda3d0a34f..1b890ab613 100644 --- a/sysinv/sysinv/sysinv/sysinv/puppet/puppet.py +++ b/sysinv/sysinv/sysinv/sysinv/puppet/puppet.py @@ -16,44 +16,8 @@ import yaml from oslo_utils import importutils from stevedore import extension -from sysinv.common import constants -from sysinv.common import exception from sysinv.openstack.common import log as logging -from sysinv.openstack.common.gettextutils import _ - -from . import aodh -from . import ceilometer -from . import ceph -from . import cinder -from . import common -from . import dcmanager -from . import dcorch -from . import fm -from . import glance -from . import gnocchi -from . import heat -from . import horizon -from . import interface -from . import inventory -from . import ironic -from . import keystone -from . import ldap -from . import magnum -from . import mtce -from . import murano -from . import networking -from . import neutron -from . import nfv -from . import nova -from . import ovs -from . import panko -from . import patching -from . import platform -from . import storage -from . import device -from . import service_parameter -from . import kubernetes -from . import smapi +from sysinv.puppet import common LOG = logging.getLogger(__name__) @@ -78,38 +42,6 @@ class PuppetOperator(object): self.dbapi = dbapi self.path = path - self.aodh = aodh.AodhPuppet(self) - self.ceilometer = ceilometer.CeilometerPuppet(self) - self.ceph = ceph.CephPuppet(self) - self.cinder = cinder.CinderPuppet(self) - self.dcmanager = dcmanager.DCManagerPuppet(self) - self.dcorch = dcorch.DCOrchPuppet(self) - self.glance = glance.GlancePuppet(self) - self.gnocchi = gnocchi.GnocchiPuppet(self) - self.heat = heat.HeatPuppet(self) - self.horizon = horizon.HorizonPuppet(self) - self.interface = interface.InterfacePuppet(self) - self.keystone = keystone.KeystonePuppet(self) - self.ldap = ldap.LdapPuppet(self) - self.magnum = magnum.MagnumPuppet(self) - self.mtce = mtce.MtcePuppet(self) - self.murano = murano.MuranoPuppet(self) - self.networking = networking.NetworkingPuppet(self) - self.neutron = neutron.NeutronPuppet(self) - self.nfv = nfv.NfvPuppet(self) - self.nova = nova.NovaPuppet(self) - self.ovs = ovs.OVSPuppet(self) - self.panko = panko.PankoPuppet(self) - self.patching = patching.PatchingPuppet(self) - self.platform = platform.PlatformPuppet(self) - self.storage = storage.StoragePuppet(self) - self.sysinv = inventory.SystemInventoryPuppet(self) - self.device = device.DevicePuppet(self) - self.ironic = ironic.IronicPuppet(self) - self.kubernetes = kubernetes.KubernetesPuppet(self) - self.service_parameter = service_parameter.ServiceParamPuppet(self) - self.smapi = smapi.SmPuppet(self) - self.fm = fm.FmPuppet(self) puppet_plugins = extension.ExtensionManager( namespace='systemconfig.puppet_plugins', invoke_on_load=True, invoke_args=(self,)) @@ -139,29 +71,6 @@ class PuppetOperator(object): try: config = {} - config.update(self.platform.get_static_config()) - config.update(self.patching.get_static_config()) - config.update(self.mtce.get_static_config()) - config.update(self.keystone.get_static_config()) - config.update(self.sysinv.get_static_config()) - config.update(self.ceph.get_static_config()) - config.update(self.nova.get_static_config()) - config.update(self.neutron.get_static_config()) - config.update(self.glance.get_static_config()) - config.update(self.gnocchi.get_static_config()) - config.update(self.cinder.get_static_config()) - config.update(self.aodh.get_static_config()) - config.update(self.heat.get_static_config()) - config.update(self.magnum.get_static_config()) - config.update(self.murano.get_static_config()) - config.update(self.ironic.get_static_config()) - config.update(self.panko.get_static_config()) - config.update(self.ldap.get_static_config()) - config.update(self.dcmanager.get_static_config()) - config.update(self.dcorch.get_static_config()) - config.update(self.smapi.get_static_config()) - config.update(self.fm.get_static_config()) - for puppet_plugin in self.puppet_plugins: config.update(puppet_plugin.obj.get_static_config()) @@ -184,32 +93,6 @@ class PuppetOperator(object): try: config = {} - config.update(self.platform.get_secure_static_config()) - config.update(self.ldap.get_secure_static_config()) - config.update(self.patching.get_secure_static_config()) - config.update(self.mtce.get_secure_static_config()) - config.update(self.keystone.get_secure_static_config()) - config.update(self.sysinv.get_secure_static_config()) - config.update(self.nfv.get_secure_static_config()) - config.update(self.ceph.get_secure_static_config()) - config.update(self.nova.get_secure_static_config()) - config.update(self.neutron.get_secure_static_config()) - config.update(self.horizon.get_secure_static_config()) - config.update(self.glance.get_secure_static_config()) - config.update(self.gnocchi.get_secure_static_config()) - config.update(self.cinder.get_secure_static_config()) - config.update(self.ceilometer.get_secure_static_config()) - config.update(self.aodh.get_secure_static_config()) - config.update(self.heat.get_secure_static_config()) - config.update(self.magnum.get_secure_static_config()) - config.update(self.murano.get_secure_static_config()) - config.update(self.ironic.get_secure_static_config()) - config.update(self.panko.get_secure_static_config()) - config.update(self.dcmanager.get_secure_static_config()) - config.update(self.dcorch.get_secure_static_config()) - config.update(self.smapi.get_secure_static_config()) - config.update(self.fm.get_secure_static_config()) - for puppet_plugin in self.puppet_plugins: config.update(puppet_plugin.obj.get_secure_static_config()) @@ -225,36 +108,6 @@ class PuppetOperator(object): try: # NOTE: order is important due to cached context data config = {} - config.update(self.platform.get_system_config()) - config.update(self.networking.get_system_config()) - config.update(self.patching.get_system_config()) - config.update(self.mtce.get_system_config()) - config.update(self.keystone.get_system_config()) - config.update(self.sysinv.get_system_config()) - config.update(self.nfv.get_system_config()) - config.update(self.ceph.get_system_config()) - config.update(self.nova.get_system_config()) - config.update(self.neutron.get_system_config()) - config.update(self.horizon.get_system_config()) - config.update(self.glance.get_system_config()) - config.update(self.gnocchi.get_system_config()) - config.update(self.cinder.get_system_config()) - config.update(self.ceilometer.get_system_config()) - config.update(self.aodh.get_system_config()) - config.update(self.heat.get_system_config()) - config.update(self.magnum.get_system_config()) - config.update(self.murano.get_system_config()) - config.update(self.storage.get_system_config()) - config.update(self.ironic.get_system_config()) - config.update(self.panko.get_system_config()) - config.update(self.dcmanager.get_system_config()) - config.update(self.dcorch.get_system_config()) - config.update(self.kubernetes.get_system_config()) - config.update(self.smapi.get_system_config()) - config.update(self.fm.get_system_config()) - # service_parameter must be last to permit overrides - config.update(self.service_parameter.get_system_config()) - for puppet_plugin in self.puppet_plugins: config.update(puppet_plugin.obj.get_system_config()) @@ -270,25 +123,6 @@ class PuppetOperator(object): try: # NOTE: order is important due to cached context data config = {} - config.update(self.platform.get_secure_system_config()) - config.update(self.keystone.get_secure_system_config()) - config.update(self.sysinv.get_secure_system_config()) - config.update(self.nova.get_secure_system_config()) - config.update(self.neutron.get_secure_system_config()) - config.update(self.glance.get_secure_system_config()) - config.update(self.gnocchi.get_secure_system_config()) - config.update(self.cinder.get_secure_system_config()) - config.update(self.aodh.get_secure_system_config()) - config.update(self.heat.get_secure_system_config()) - config.update(self.magnum.get_secure_system_config()) - config.update(self.murano.get_secure_system_config()) - config.update(self.ironic.get_secure_system_config()) - config.update(self.panko.get_secure_system_config()) - config.update(self.dcmanager.get_secure_system_config()) - config.update(self.dcorch.get_secure_system_config()) - config.update(self.kubernetes.get_secure_system_config()) - config.update(self.fm.get_secure_system_config()) - for puppet_plugin in self.puppet_plugins: config.update(puppet_plugin.obj.get_secure_system_config()) @@ -302,92 +136,13 @@ class PuppetOperator(object): def update_host_config(self, host, config_uuid=None): """Update the host hiera configuration files for the supplied host""" + self.config_uuid = config_uuid config = {} - if host.personality == constants.CONTROLLER: - config = self.update_controller_config(host, config_uuid) - elif host.personality == constants.COMPUTE: - config = self.update_compute_config(host, config_uuid) - elif host.personality == constants.STORAGE: - config = self.update_storage_config(host, config_uuid) - else: - raise exception.SysinvException(_( - "Invalid method call: unsupported personality: %s") % - host.personality) for puppet_plugin in self.puppet_plugins: config.update(puppet_plugin.obj.get_host_config(host)) self._write_host_config(host, config) - def update_controller_config(self, host, config_uuid=None): - """Update the configuration for a specific controller host""" - try: - # NOTE: order is important due to cached context data - config = {} - config.update(self.platform.get_host_config(host, config_uuid)) - config.update(self.interface.get_host_config(host)) - config.update(self.ovs.get_host_config(host)) - config.update(self.networking.get_host_config(host)) - config.update(self.storage.get_host_config(host)) - config.update(self.ldap.get_host_config(host)) - config.update(self.nfv.get_host_config(host)) - config.update(self.ceph.get_host_config(host)) - config.update(self.cinder.get_host_config(host)) - config.update(self.device.get_host_config(host)) - config.update(self.nova.get_host_config(host)) - config.update(self.neutron.get_host_config(host)) - config.update(self.smapi.get_host_config(host)) - config.update(self.fm.get_host_config(host)) - # service_parameter must be last to permit overrides - config.update(self.service_parameter.get_host_config(host)) - - return config - except Exception: - LOG.exception("failed to create host config: %s" % host.uuid) - raise - - def update_compute_config(self, host, config_uuid=None): - """Update the configuration for a specific compute host""" - try: - # NOTE: order is important due to cached context data - config = {} - config.update(self.platform.get_host_config(host, config_uuid)) - config.update(self.interface.get_host_config(host)) - config.update(self.ovs.get_host_config(host)) - config.update(self.networking.get_host_config(host)) - config.update(self.storage.get_host_config(host)) - config.update(self.ceph.get_host_config(host)) - config.update(self.device.get_host_config(host)) - config.update(self.nova.get_host_config(host)) - config.update(self.neutron.get_host_config(host)) - config.update(self.kubernetes.get_host_config(host)) - config.update(self.ldap.get_host_config(host)) - # service_parameter must be last to permit overrides - config.update(self.service_parameter.get_host_config(host)) - - return config - except Exception: - LOG.exception("failed to create host config: %s" % host.uuid) - raise - - def update_storage_config(self, host, config_uuid=None): - """Update the configuration for a specific storage host""" - try: - # NOTE: order is important due to cached context data - config = {} - config.update(self.platform.get_host_config(host, config_uuid)) - config.update(self.interface.get_host_config(host)) - config.update(self.networking.get_host_config(host)) - config.update(self.storage.get_host_config(host)) - config.update(self.ceph.get_host_config(host)) - config.update(self.ldap.get_host_config(host)) - # service_parameter must be last to permit overrides - config.update(self.service_parameter.get_host_config(host)) - - return config - except Exception: - LOG.exception("failed to create host config: %s" % host.uuid) - raise - def remove_host_config(self, host): """Remove the configuration for the supplied host""" try: diff --git a/sysinv/sysinv/sysinv/sysinv/puppet/smapi.py b/sysinv/sysinv/sysinv/sysinv/puppet/smapi.py index 2d86853572..4f9c1d4873 100644 --- a/sysinv/sysinv/sysinv/sysinv/puppet/smapi.py +++ b/sysinv/sysinv/sysinv/sysinv/puppet/smapi.py @@ -4,6 +4,9 @@ # SPDX-License-Identifier: Apache-2.0 # +from sysinv.common import constants +from sysinv.common import utils + from . import openstack @@ -55,6 +58,9 @@ class SmPuppet(openstack.OpenstackBasePuppet): return config def get_host_config(self, host): + if (constants.CONTROLLER not in utils.get_personalities(host)): + return {} + config = { 'platform::smapi::params::bind_ip': host.mgmt_ip, }