Merge "Move any Fenix generic functionality out from default workflow"

This commit is contained in:
Zuul 2018-10-05 05:25:49 +00:00 committed by Gerrit Code Review
commit e3ceda7232
3 changed files with 183 additions and 149 deletions

48
fenix/utils/time.py Normal file
View File

@ -0,0 +1,48 @@
# Copyright (c) 2018 OpenStack Foundation.
# All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
import datetime
def str_to_datetime(dt_str):
mdate, mtime = dt_str.split()
year, month, day = map(int, mdate.split('-'))
hours, minutes, seconds = map(int, mtime.split(':'))
return datetime.datetime(year, month, day, hours, minutes, seconds)
def reply_time_str(wait):
now = datetime.datetime.utcnow()
reply = now - datetime.timedelta(
seconds=wait)
return (reply.strftime('%Y-%m-%d %H:%M:%S'))
def time_now_str():
return datetime.datetime.utcnow().strftime('%Y-%m-%d %H:%M:%S')
def is_time_after_time(before, after):
if type(before) == str:
time_before = str_to_datetime(before)
else:
time_before = before
if type(after) == str:
time_after = str_to_datetime(after)
else:
time_after = after
if time_before > time_after:
return True
else:
return False

View File

@ -12,12 +12,16 @@
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
import aodhclient.client as aodhclient
from oslo_log import log as logging
import oslo_messaging as messaging
from oslo_service import threadgroup
from threading import Thread
import time
from fenix.utils.identity_auth import get_identity_auth
from fenix.utils.identity_auth import get_session
LOG = logging.getLogger(__name__)
@ -175,6 +179,23 @@ class BaseWorkflow(Thread):
'MAINTENANCE_COMPLETE': 'maintenance_complete',
'MAINTENANCE_DONE': 'maintenance_done',
'FAILED': 'maintenance_failed'}
self.url = "http://%s:%s" % (conf.host, conf.port)
self.auth = get_identity_auth(conf.workflow_user,
conf.workflow_password,
conf.workflow_project)
self.session = get_session(auth=self.auth)
self.aodh = aodhclient.Client('2', self.session)
transport = messaging.get_transport(self.conf)
self.notif_proj = messaging.Notifier(transport,
'maintenance.planned',
driver='messaging',
topics=['notifications'])
self.notif_proj = self.notif_proj.prepare(publisher_id='fenix')
self.notif_admin = messaging.Notifier(transport,
'maintenance.host',
driver='messaging',
topics=['notifications'])
self.notif_admin = self.notif_admin.prepare(publisher_id='fenix')
def _timer_expired(self, name):
LOG.info("%s: timer expired %s" % (self.session_id, name))
@ -230,3 +251,102 @@ class BaseWorkflow(Thread):
time.sleep(1)
# IDLE while session removed
LOG.info("%s: done" % self.session_id)
def projects_listen_alarm(self, match_event):
match_projects = ([str(alarm['project_id']) for alarm in
self.aodh.alarm.list() if
str(alarm['event_rule']['event_type']) ==
match_event])
all_projects_match = True
for project in self.session_data.project_names():
if project not in match_projects:
LOG.error('%s: project %s not '
'listening to %s' %
(self.session_id, project, match_event))
all_projects_match = False
return all_projects_match
def _project_notify(self, project_id, instance_ids, allowed_actions,
actions_at, reply_at, state, metadata):
reply_url = '%s/v1/maintenance/%s/%s' % (self.url,
self.session_id,
project_id)
payload = dict(project_id=project_id,
instance_ids=instance_ids,
allowed_actions=allowed_actions,
state=state,
actions_at=actions_at,
reply_at=reply_at,
session_id=self.session_id,
metadata=metadata,
reply_url=reply_url)
LOG.info('Sending "maintenance.planned" to project: %s' % payload)
self.notif_proj.info({'some': 'context'}, 'maintenance.scheduled',
payload)
def _admin_notify(self, project, host, state, session_id):
payload = dict(project_id=project, host=host, state=state,
session_id=session_id)
LOG.info('Sending "maintenance.host": %s' % payload)
self.notif_admin.info({'some': 'context'}, 'maintenance.host', payload)
def projects_answer(self, state, projects):
state_ack = 'ACK_%s' % state
state_nack = 'NACK_%s' % state
for project in projects:
pstate = project.state
if pstate == state:
break
elif pstate == state_ack:
continue
elif pstate == state_nack:
LOG.error('%s: %s from %s' %
(self.session_id, pstate, project.name))
break
else:
LOG.error('%s: Project %s in invalid state %s' %
(self.session_id, project.name, pstate))
break
return pstate
def _project_names_in_state(self, projects, state):
return ([project.name for project in projects if
project.state == state])
def wait_projects_state(self, state, timer_name):
state_ack = 'ACK_%s' % state
state_nack = 'NACK_%s' % state
projects = self.session_data.get_projects_with_state()
if not projects:
LOG.error('%s: wait_projects_state %s. Emtpy project list' %
(self.session_id, state))
while not self.is_timer_expired(timer_name):
answer = self.projects_answer(state, projects)
if answer == state:
pass
else:
self.stop_timer(timer_name)
if answer == state_ack:
LOG.info('all projects in: %s' % state_ack)
return True
elif answer == state_nack:
pnames = self._projects_in_state(projects, answer)
LOG.error('%s: projects rejected with %s: %s' %
(self.session_id, answer, pnames))
return False
else:
pnames = self._projects_in_state(projects, answer)
LOG.error('%s: projects with invalid state %s: %s' %
(self.session_id, answer, pnames))
return False
time.sleep(1)
LOG.error('%s: timer %s expired waiting answer to state %s' %
(self.session_id, timer_name, state))
pnames = self._projects_in_state(projects, state)
LOG.error('%s: projects not answered: %s' % (self.session_id, pnames))
return False

View File

@ -12,18 +12,17 @@
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
import datetime
import novaclient.client as novaclient
from novaclient.exceptions import BadRequest
from oslo_log import log as logging
import oslo_messaging as messaging
import time
from fenix.utils.identity_auth import get_identity_auth
from fenix.utils.identity_auth import get_session
import aodhclient.client as aodhclient
import novaclient.client as novaclient
from fenix.utils.time import is_time_after_time
from fenix.utils.time import reply_time_str
from fenix.utils.time import str_to_datetime
from fenix.utils.time import time_now_str
from fenix.workflow.workflow import BaseWorkflow
@ -35,24 +34,7 @@ class Workflow(BaseWorkflow):
def __init__(self, conf, session_id, data):
super(Workflow, self).__init__(conf, session_id, data)
self.url = "http://%s:%s" % (conf.host, conf.port)
self.auth = get_identity_auth(conf.workflow_user,
conf.workflow_password,
conf.workflow_project)
self.session = get_session(auth=self.auth)
self.aodh = aodhclient.Client('2', self.session)
self.nova = novaclient.Client(version='2.34', session=self.session)
transport = messaging.get_transport(self.conf)
self.notif_proj = messaging.Notifier(transport,
'maintenance.planned',
driver='messaging',
topics=['notifications'])
self.notif_proj = self.notif_proj.prepare(publisher_id='fenix')
self.notif_admin = messaging.Notifier(transport,
'maintenance.host',
driver='messaging',
topics=['notifications'])
self.notif_admin = self.notif_admin.prepare(publisher_id='fenix')
LOG.info("%s: initialized" % self.session_id)
def cleanup(self):
@ -112,120 +94,6 @@ class Workflow(BaseWorkflow):
ha)
LOG.info(str(self.session_data))
def projects_listen_alarm(self, match_event):
match_projects = ([str(alarm['project_id']) for alarm in
self.aodh.alarm.list() if
str(alarm['event_rule']['event_type']) ==
match_event])
all_projects_match = True
for project in self.session_data.project_names():
if project not in match_projects:
LOG.error('%s: project %s not '
'listening to %s' %
(self.session_id, project, match_event))
all_projects_match = False
return all_projects_match
def str_to_datetime(self, dt_str):
mdate, mtime = dt_str.split()
year, month, day = map(int, mdate.split('-'))
hours, minutes, seconds = map(int, mtime.split(':'))
return datetime.datetime(year, month, day, hours, minutes, seconds)
def reply_time_str(self, wait):
now = datetime.datetime.utcnow()
reply = now - datetime.timedelta(
seconds=wait)
return (reply.strftime('%Y-%m-%d %H:%M:%S'))
def is_time_after_time(self, before, after):
if type(before) == str:
time_before = self.str_to_datetime(before)
else:
time_before = before
if type(after) == str:
time_after = self.str_to_datetime(after)
else:
time_after = after
if time_before > time_after:
return True
else:
return False
def _project_notify(self, project_id, instance_ids, allowed_actions,
actions_at, reply_at, state, metadata):
reply_url = '%s/v1/maintenance/%s/%s' % (self.url,
self.session_id,
project_id)
payload = dict(project_id=project_id,
instance_ids=instance_ids,
allowed_actions=allowed_actions,
state=state,
actions_at=actions_at,
reply_at=reply_at,
session_id=self.session_id,
metadata=metadata,
reply_url=reply_url)
LOG.info('Sending "maintenance.planned" to project: %s' % payload)
self.notif_proj.info({'some': 'context'}, 'maintenance.scheduled',
payload)
def _admin_notify(self, project, host, state, session_id):
payload = dict(project_id=project, host=host, state=state,
session_id=session_id)
LOG.info('Sending "maintenance.host": %s' % payload)
self.notif_admin.info({'some': 'context'}, 'maintenance.host', payload)
def projects_answer(self, state, projects):
state_ack = 'ACK_%s' % state
state_nack = 'NACK_%s' % state
for project in projects:
pstate = project.state
if pstate == state:
break
elif pstate == state_ack:
continue
elif pstate == state_nack:
LOG.error('%s: %s from %s' %
(self.session_id, pstate, project.name))
break
else:
LOG.error('%s: Project %s in invalid state %s' %
(self.session_id, project.name, pstate))
break
return pstate
def wait_projects_state(self, state, timer_name):
state_ack = 'ACK_%s' % state
state_nack = 'NACK_%s' % state
projects = self.session_data.get_projects_with_state()
if not projects:
LOG.error('%s: wait_projects_state %s. Emtpy project list' %
(self.session_id, state))
while not self.is_timer_expired(timer_name):
answer = self.projects_answer(state, projects)
if answer == state:
pass
else:
self.stop_timer(timer_name)
if answer == state_ack:
LOG.info('all projects in: %s' % state_ack)
return True
elif answer == state_nack:
return False
else:
return False
time.sleep(1)
LOG.error('%s: timer %s expired waiting answer to state %s' %
(self.session_id, timer_name, state))
LOG.error('%s: project states' % self.session_id)
return False
def confirm_maintenance(self):
allowed_actions = []
actions_at = self.session_data.maintenance_at
@ -236,8 +104,8 @@ class Workflow(BaseWorkflow):
instance_ids = '%s/v1/maintenance/%s/%s' % (self.url,
self.session_id,
project)
reply_at = self.reply_time_str(self.conf.project_maintenance_reply)
if self.is_time_after_time(reply_at, actions_at):
reply_at = reply_time_str(self.conf.project_maintenance_reply)
if is_time_after_time(reply_at, actions_at):
raise Exception('%s: No time for project to'
' answer in state: %s' %
(self.session_id, state))
@ -250,7 +118,7 @@ class Workflow(BaseWorkflow):
def confirm_scale_in(self):
allowed_actions = []
actions_at = self.reply_time_str(self.conf.project_scale_in_reply)
actions_at = reply_time_str(self.conf.project_scale_in_reply)
reply_at = actions_at
state = 'SCALE_IN'
self.session_data.set_projets_state(state)
@ -351,7 +219,7 @@ class Workflow(BaseWorkflow):
def confirm_host_to_be_emptied(self, host, state):
allowed_actions = ['MIGRATE', 'LIVE_MIGRATE', 'OWN_ACTION']
actions_at = self.reply_time_str(self.conf.project_maintenance_reply)
actions_at = reply_time_str(self.conf.project_maintenance_reply)
reply_at = actions_at
self.session_data.set_projects_state_and_host_instances(state, host)
for project in self.session_data.project_names():
@ -372,7 +240,7 @@ class Workflow(BaseWorkflow):
def confirm_maintenance_complete(self):
state = 'MAINTENANCE_COMPLETE'
metadata = self.session_data.metadata
actions_at = self.reply_time_str(self.conf.project_scale_in_reply)
actions_at = reply_time_str(self.conf.project_scale_in_reply)
reply_at = actions_at
self.session_data.set_projets_state(state)
for project in self.session_data.project_names():
@ -527,11 +395,9 @@ class Workflow(BaseWorkflow):
LOG.info('Empty host found')
self.state = 'START_MAINTENANCE'
maint_at = self.str_to_datetime(
self.session_data.maintenance_at)
maint_at = str_to_datetime(self.session_data.maintenance_at)
if maint_at > datetime.datetime.utcnow():
time_now = (datetime.datetime.utcnow().strftime(
'%Y-%m-%d %H:%M:%S'))
time_now = time_now_str()
LOG.info('Time now: %s maintenance starts: %s....' %
(time_now, self.session_data.maintenance_at))
td = maint_at - datetime.datetime.utcnow()
@ -539,7 +405,7 @@ class Workflow(BaseWorkflow):
while not self.is_timer_expired('MAINTENANCE_START_TIMEOUT'):
time.sleep(1)
time_now = (datetime.datetime.utcnow().strftime('%Y-%m-%d %H:%M:%S'))
time_now = time_now_str()
LOG.info('Time to start maintenance: %s' % time_now)
def scale_in(self):