Refactoring fencers
Allow freezer-dr to load fencers more generically and give the drivers space to implement their own fencing procedure Change-Id: I5272ca90d806b8ce83055199724abdc14fe414bc
This commit is contained in:
Saad Zaher
parent
4d11ee77d3
commit
e01bf61472
|
|
@ -15,7 +15,6 @@
|
|||
"""Abstract fencer"""
|
||||
|
||||
import abc
|
||||
|
||||
import six
|
||||
|
||||
|
||||
|
|
@ -28,38 +27,29 @@ class FencerBaseDriver(object):
|
|||
needed.
|
||||
"""
|
||||
|
||||
def __init__(self, node, **kwargs):
|
||||
def __init__(self, nodes, fencer_conf):
|
||||
"""Initialize the driver.
|
||||
|
||||
Any fencer driver requires the following parameters to do the api
|
||||
calls. All these parameters can be passed from the configuration
|
||||
file in /etc/freezer/dr.conf (default).
|
||||
|
||||
:param node: dict with all node details. (/etc/freezer/servers.yml) ?
|
||||
:param kwargs: any additional parameters can be passed using this
|
||||
config option.
|
||||
:param nodes: A list of failed nodes to be fenced!
|
||||
:param fencer_conf: dict contains configuration options loaded
|
||||
from the config file.
|
||||
"""
|
||||
self.node = node
|
||||
self.kwargs = kwargs
|
||||
self.nodes = nodes
|
||||
self.fencer_conf = fencer_conf
|
||||
|
||||
def update_nodes(self, nodes):
|
||||
"""Allows changing the nodes during the evacuation..."""
|
||||
self.nodes = nodes
|
||||
|
||||
@abc.abstractmethod
|
||||
def graceful_shutdown(self):
|
||||
"""Gracefully shutdown the compute node to evacuate it."""
|
||||
|
||||
@abc.abstractmethod
|
||||
def force_shutdown(self):
|
||||
"""Force shutdown the compute node to evacuate it.
|
||||
|
||||
May be you can try force shutdown if the graceful shutdown failed.
|
||||
"""
|
||||
|
||||
@abc.abstractmethod
|
||||
def status(self):
|
||||
"""Get compute node status.
|
||||
|
||||
Should return 1 if on and 0 if off or -1 if error or unknown power
|
||||
status.
|
||||
"""
|
||||
def fence(self):
|
||||
"""This function to be implemented by each driver. Each driver will
|
||||
implement its own fencing logic and the manager will just load it and
|
||||
call the fence function"""
|
||||
|
||||
@abc.abstractmethod
|
||||
def get_info(self):
|
||||
|
|
|
|||
|
|
@ -14,8 +14,7 @@
|
|||
from oslo_config import cfg
|
||||
from oslo_log import log
|
||||
from oslo_utils import importutils
|
||||
from freezer_dr.common.yaml_parser import YamlParser
|
||||
from time import sleep
|
||||
|
||||
|
||||
CONF = cfg.CONF
|
||||
LOG = log.getLogger(__name__)
|
||||
|
|
@ -24,51 +23,20 @@ LOG = log.getLogger(__name__)
|
|||
class FencerManager(object):
|
||||
|
||||
def __init__(self, nodes):
|
||||
self.fencer = CONF.get('fencer')
|
||||
self.fencer_conf = CONF.get('fencer')
|
||||
self.nodes = nodes
|
||||
self.parser = YamlParser(self.fencer.get('credentials_file'))
|
||||
|
||||
def fence(self):
|
||||
"""
|
||||
Try to shutdown nodes and wait for configurable amount of times
|
||||
:return: list of nodes and either they are shutdown or failed
|
||||
"""
|
||||
processed_nodes = []
|
||||
for node in self.nodes:
|
||||
node_details = self.parser.find_server_by_ip(node.get('ip')) or\
|
||||
self.parser.find_server_by_hostname(node.get('host'))
|
||||
driver = importutils.import_object(
|
||||
self.fencer.get('driver'),
|
||||
node=node_details,
|
||||
**self.fencer.get('options')
|
||||
)
|
||||
node['status'] = self.do_shutdown_procedure(driver)
|
||||
processed_nodes.append(node)
|
||||
return processed_nodes
|
||||
|
||||
def do_shutdown_procedure(self, driver):
|
||||
for retry in range(0, self.fencer.get('retries', 1)):
|
||||
if driver.status():
|
||||
try:
|
||||
driver.graceful_shutdown()
|
||||
except Exception as e:
|
||||
LOG.error(e)
|
||||
else:
|
||||
return True
|
||||
# try to wait a pre-configured amount of time before redoing
|
||||
# the fence call again :)
|
||||
sleep(self.fencer.get('hold_period', 10))
|
||||
LOG.info('wait for %d seconds before retrying to gracefully '
|
||||
'shutdown' % self.fencer.get('hold_period', 10))
|
||||
LOG.info('Retrying to gracefully shutdown the node.')
|
||||
|
||||
try:
|
||||
driver.force_shutdown()
|
||||
except Exception as e:
|
||||
LOG.error(e)
|
||||
|
||||
if not driver.status():
|
||||
return True
|
||||
|
||||
return False
|
||||
driver_name = self.fencer_conf['driver']
|
||||
driver = importutils.import_object(
|
||||
driver_name,
|
||||
self.fencer_conf
|
||||
)
|
||||
LOG.debug('Loaded fencing driver {0} with config: '
|
||||
'{1}'.format(driver.get_info(), self.fencer_conf))
|
||||
|
||||
return driver.fence()
|
||||
|
|
|
|||
|
|
@ -11,18 +11,29 @@
|
|||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
from freezer_dr.common.yaml_parser import YamlParser
|
||||
from freezer_dr.fencers.common.driver import FencerBaseDriver
|
||||
from freezer_dr.fencers.drivers.ipmi.ipmitool import IpmiInterface
|
||||
from oslo_log import log
|
||||
from oslo_config import cfg
|
||||
import time
|
||||
|
||||
CONF = cfg.CONF
|
||||
LOG = log.getLogger(__name__)
|
||||
|
||||
|
||||
class IpmiDriver(FencerBaseDriver):
|
||||
def __init__(self, node, **kwargs):
|
||||
super(IpmiDriver, self).__init__(node, **kwargs)
|
||||
|
||||
def __init__(self, nodes, fencer_conf):
|
||||
|
||||
super(IpmiDriver, self).__init__(nodes, fencer_conf)
|
||||
self.parser = YamlParser(self.fencer_conf['credentials_file'])
|
||||
|
||||
def prepare_node(self, node):
|
||||
"""Prepares the subprocess to call ``ipmitool`` with the node details!
|
||||
:param node: dict contains node fencing information
|
||||
"""
|
||||
self.interface = IpmiInterface(
|
||||
node.get('fencer-ip'),
|
||||
node.get('fencer-user'),
|
||||
|
|
@ -48,11 +59,57 @@ class IpmiDriver(FencerBaseDriver):
|
|||
def power_on(self):
|
||||
self.interface.power_on()
|
||||
|
||||
def get_node_details(self, node):
|
||||
"""Loads the node's fencing information from ``credentials_file``
|
||||
:param node: a dict contains node ip or hostname
|
||||
:return: a dict contains node fencing information
|
||||
"""
|
||||
node_details = self.parser.find_server_by_ip(node.get('ip')) or \
|
||||
self.parser.find_server_by_hostname(node.get('host'))
|
||||
|
||||
return node_details
|
||||
|
||||
def fence(self):
|
||||
"""Implements the fencing procedure for server fencing using ipmi
|
||||
:return: a list of nodes and weather they're fenced or not!
|
||||
"""
|
||||
fenced_nodes = []
|
||||
for node in self.nodes:
|
||||
LOG.debug("fencing node {0}".format(node))
|
||||
# load node details
|
||||
node_details = self.get_node_details(node)
|
||||
# loop on the node number of n times trying to fence it gently,
|
||||
# if not force it!
|
||||
self.prepare_node(node_details)
|
||||
for retry in range(0, self.fencer_conf['retries']):
|
||||
if self.status():
|
||||
try:
|
||||
self.graceful_shutdown()
|
||||
except Exception as e:
|
||||
LOG.debug(e)
|
||||
else:
|
||||
node['status'] = True
|
||||
break
|
||||
time.sleep(self.fencer_conf['hold_period'])
|
||||
LOG.info('wait for %d seconds before retrying to gracefully '
|
||||
'shutdown' % self.fencer_conf['hold_period'])
|
||||
|
||||
try:
|
||||
self.force_shutdown()
|
||||
except Exception as e:
|
||||
LOG.error(e)
|
||||
|
||||
if not self.status():
|
||||
node['status'] = True
|
||||
else:
|
||||
node['status'] = False
|
||||
fenced_nodes.append(node)
|
||||
|
||||
return fenced_nodes
|
||||
|
||||
def get_info(self):
|
||||
return {
|
||||
'name': 'IPMI Interface driver',
|
||||
'version': 1.0,
|
||||
'author': 'Hewlett-Packard Development Company, L.P'
|
||||
'version': 1.1,
|
||||
'author': 'Hewlett-Packard Enterprise Company, L.P'
|
||||
}
|
||||
|
||||
|
||||
|
|
|
|||
|
|
@ -11,38 +11,90 @@
|
|||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
from freezer_dr.common.yaml_parser import YamlParser
|
||||
from freezer_dr.fencers.common.driver import FencerBaseDriver
|
||||
import libvirt
|
||||
from oslo_log import log
|
||||
from oslo_config import cfg
|
||||
import libvirt
|
||||
import time
|
||||
|
||||
CONF = cfg.CONF
|
||||
LOG = log.getLogger(__name__)
|
||||
|
||||
|
||||
class LibvirtDriver(FencerBaseDriver):
|
||||
def __init__(self, node, **kwargs):
|
||||
super(LibvirtDriver, self).__init__(node, **kwargs)
|
||||
conn_name = kwargs.get('name', None)
|
||||
|
||||
def __init__(self, nodes, fencer_conf):
|
||||
super(LibvirtDriver, self).__init__(nodes, fencer_conf)
|
||||
self.parser = YamlParser(self.fencer_conf['credentials_file'])
|
||||
# initiate libvirt connection
|
||||
conn_name = self.fencer_conf.get('name', None)
|
||||
self.connection = libvirt.open(name=conn_name)
|
||||
|
||||
def force_shutdown(self):
|
||||
target = self.connection.lookupByName(name=self.node.get('domain-name'))
|
||||
def force_shutdown(self, node):
|
||||
target = self.connection.lookupByName(name=node.get('domain-name'))
|
||||
return target.destroy()
|
||||
|
||||
def graceful_shutdown(self):
|
||||
target = self.connection.lookupByName(name=self.node.get('domain-name'))
|
||||
def graceful_shutdown(self, node):
|
||||
target = self.connection.lookupByName(name=node.get('domain-name'))
|
||||
return target.shutdown()
|
||||
|
||||
def status(self):
|
||||
target = self.connection.lookupByName(name=self.node.get('domain-name'))
|
||||
def status(self, node):
|
||||
target = self.connection.lookupByName(name=node.get('domain-name'))
|
||||
return target.isActive()
|
||||
|
||||
def get_node_details(self, node):
|
||||
"""Loads the node's fencing information from ``credentials_file``
|
||||
:param node: a dict contains node ip or hostname
|
||||
:return: a dict contains node fencing information
|
||||
"""
|
||||
node_details = self.parser.find_server_by_ip(node.get('ip')) or \
|
||||
self.parser.find_server_by_hostname(node.get('host'))
|
||||
|
||||
return node_details
|
||||
|
||||
def fence(self):
|
||||
"""Implements the fencing procedure for server fencing using ipmi
|
||||
:return: a list of nodes and weather they're fenced or not!
|
||||
"""
|
||||
fenced_nodes = []
|
||||
for node in self.nodes:
|
||||
LOG.debug("fencing node {0}".format(node))
|
||||
# load node details
|
||||
node_details = self.get_node_details(node)
|
||||
# loop on the node number of n times trying to fence it gently,
|
||||
# if not force it!
|
||||
for retry in range(0, self.fencer_conf['retries']):
|
||||
if self.status(node=node_details):
|
||||
try:
|
||||
self.graceful_shutdown(node=node_details)
|
||||
except Exception as e:
|
||||
LOG.debug(e)
|
||||
else:
|
||||
node['status'] = True
|
||||
break
|
||||
time.sleep(self.fencer_conf['hold_period'])
|
||||
LOG.info('wait for %d seconds before retrying to gracefully '
|
||||
'shutdown' % self.fencer_conf['hold_period'])
|
||||
|
||||
try:
|
||||
self.force_shutdown(node=node_details)
|
||||
except Exception as e:
|
||||
LOG.error(e)
|
||||
|
||||
if not self.status(node=node_details):
|
||||
node['status'] = True
|
||||
else:
|
||||
node['status'] = False
|
||||
fenced_nodes.append(node)
|
||||
|
||||
return fenced_nodes
|
||||
|
||||
def get_info(self):
|
||||
return {
|
||||
'name': 'Libvirt Interface driver',
|
||||
'version': 1.0,
|
||||
'author': 'Hewlett-Packard Development Company, L.P'
|
||||
'version': 1.1,
|
||||
'author': 'Hewlett-Packard Enterprise Company, L.P'
|
||||
}
|
||||
|
||||
|
||||
|
|
|
|||
Loading…
Reference in New Issue