From be68d2870706e5955911c5c106a580255ffddf12 Mon Sep 17 00:00:00 2001 From: Dmitry Nikishov Date: Tue, 1 Nov 2016 15:39:15 +0300 Subject: [PATCH] Added audit-enforce command This command performs audit run on the environment, associated with the target repo. After that, if there are any outofsync Puppet resources, it performs an enforcement run, that will sync these resources. Change-Id: I82721eb5f20383d2c0b7618050ab27517d21d15d --- fuel_external_git/const.py | 1 + fuel_external_git/fuelclient_audit.py | 149 ++++++++++++++++++++------ setup.cfg | 1 + 3 files changed, 119 insertions(+), 32 deletions(-) diff --git a/fuel_external_git/const.py b/fuel_external_git/const.py index 43d7682..968bbdf 100644 --- a/fuel_external_git/const.py +++ b/fuel_external_git/const.py @@ -10,6 +10,7 @@ # License for the specific language governing permissions and limitations # under the License. +AUDIT_TASK_CHECK_INTERVAL = 20 REPOS_DIR = '/var/lib/fuel_repos' REPO_TTL = 1200 TASK_RETRIES = 10 diff --git a/fuel_external_git/fuelclient_audit.py b/fuel_external_git/fuelclient_audit.py index 8b5067e..8ee6f8b 100644 --- a/fuel_external_git/fuelclient_audit.py +++ b/fuel_external_git/fuelclient_audit.py @@ -12,6 +12,7 @@ from __future__ import absolute_import +import logging import time from cliff import command @@ -28,10 +29,123 @@ from fuelclient.common import data_utils from fuelclient.objects import Environment from fuelclient.objects import Task +from fuel_external_git.const import AUDIT_TASK_CHECK_INTERVAL from fuel_external_git.const import TASK_RETRIES from fuel_external_git.const import TASK_RETRY_DELAY +LOG = logging.getLogger(__name__) + + +class Audit(lister.Lister, command.Command): + columns = () + + @staticmethod + def get_running_task(name): + for retry in xrange(TASK_RETRIES): + tasks = Task.get_all() + try: + task = filter(lambda t: t.data['status'] == 'running' and + t.data['name'] == name, + tasks).pop() + break + except IndexError: + time.sleep(TASK_RETRY_DELAY) + continue + return task + + @staticmethod + def start_noop_run(env): + # Due to how Nailgun handles such tasks, returned + # one will not contain any deployment-related data. + # So we'll have to fetch the last noop task with progress < 100 + env.redeploy_changes(noop_run=True) + return Audit.get_running_task('dry_run_deployment') + + @staticmethod + def get_outofsync(noop_task): + history = noop_task.connection.get_request( + ('transactions/{tid}/' + 'deployment_history?include_summary=1').format(tid=noop_task.id) + ) + changes = [] + changed_tasks = filter(lambda t: t['status'] != 'skipped' and + t.get('summary', {}) and + t['summary']['resources']['out_of_sync'] > 0, + history) + for task in changed_tasks: + name = task['task_name'] + for item in task['summary']['raw_report']: + if 'Would have triggered' not in item['message'] and \ + 'Finished catalog run' not in item['message']: + short_item = item['source'].replace('/Stage[main]/', '') + changes.append({'task_id': name, + 'resource': short_item, + 'node_id': task['node_id']}) + return changes + + def get_parser(self, prog_name): + parser = super(Audit, self).get_parser(prog_name) + group = parser.add_mutually_exclusive_group(required=True) + group.add_argument('--env', + type=int, + help='Associated Repo ID') + group.add_argument('--repo', + type=int, + help='Associated Repo ID') + return parser + + def take_action(self, parsed_args): + env_id = parsed_args.env + if not env_id: + repo_id = parsed_args.repo + repos = fc_client.get_request('/clusters/git-repos/') + env_id = [repo['env_id'] for repo in repos + if repo['id'] == repo_id][0] + + env = Environment(env_id) + + audit_task = Audit.start_noop_run(env) + + LOG.info("Noop task ID is {tid}".format(tid=audit_task.id)) + + while audit_task.status == 'running': + time.sleep(AUDIT_TASK_CHECK_INTERVAL) + LOG.info( + 'Current task progress is {p}'.format(p=audit_task.progress) + ) + + changes = Audit.get_outofsync(audit_task) + + if changes: + changed_tasks = [c['task_id'] for c in changes] + LOG.info( + "Following tasks have outofsync resources: {tasks}".format( + tasks=set(changed_tasks) + ) + ) + LOG.info(("To get the list of changes, run " + "fuel2 audit get outofsync --task " + "{task_id}").format(task_id=audit_task.id)) + LOG.info("Starting enforce run on environment {eid}".format( + eid=env_id + )) + env.redeploy_changes() + enforce_task = Audit.get_running_task('deployment') + LOG.info("Enforce task id is {tid}".format(tid=enforce_task.id)) + while enforce_task.status == 'running': + time.sleep(AUDIT_TASK_CHECK_INTERVAL) + LOG.info( + 'Current task progress is {p}'.format( + p=enforce_task.progress + ) + ) + LOG.info("Enforce task status is {status}".format( + status=enforce_task.status + )) + return ((), {}) + + class AuditRun(lister.Lister, command.Command): columns = ( 'task_id', @@ -57,20 +171,8 @@ class AuditRun(lister.Lister, command.Command): if repo['id'] == repo_id][0] env = Environment(env_id) - # Due to how Nailgun handles such tasks, returned - # one will not contain any deployment-related data. - # So we'll have to fetch the last noop task with progress < 100 - env.redeploy_changes(noop_run=True) - for retry in xrange(TASK_RETRIES): - tasks = Task.get_all() - try: - task = filter(lambda t: t.data['status'] == 'running' and - t.data['name'] == 'dry_run_deployment', - tasks).pop() - break - except IndexError: - time.sleep(TASK_RETRY_DELAY) - continue + task = Audit.start_noop_run(env) + data = {'task_id': task.id} data = data_utils.get_display_data_multi(self.columns, [data]) return (self.columns, data) @@ -107,24 +209,7 @@ class OutOfSyncResources(lister.Lister, command.Command): else: fuel_task = Task(task_id) - history = fuel_task.connection.get_request( - ('transactions/{tid}/' - 'deployment_history?include_summary=1').format(tid=fuel_task.id) - ) - changes = [] - changed_tasks = filter(lambda t: t['status'] != 'skipped' and - t.get('summary', {}) and - t['summary']['resources']['out_of_sync'] > 0, - history) - for task in changed_tasks: - name = task['task_name'] - for item in task['summary']['raw_report']: - if 'Would have triggered' not in item['message'] and \ - 'Finished catalog run' not in item['message']: - short_item = item['source'].replace('/Stage[main]/', '') - changes.append({'task_id': name, - 'resource': short_item, - 'node_id': task['node_id']}) + changes = Audit.get_outofsync(fuel_task) data = data_utils.get_display_data_multi(self.columns, changes) return (self.columns, data) diff --git a/setup.cfg b/setup.cfg index f3cd056..3a122ea 100644 --- a/setup.cfg +++ b/setup.cfg @@ -30,5 +30,6 @@ fuelclient: gitrepo_update = fuel_external_git.fuelclient_gitrepo:UpdateRepo gitrepo_init = fuel_external_git.fuelclient_gitrepo:InitRepo gitrepo_get_configs = fuel_external_git.fuelclient_gitrepo:DownloadConfgs + audit_enforce = fuel_external_git.fuelclient_audit:Audit audit_noop = fuel_external_git.fuelclient_audit:AuditRun audit_list_outofsync = fuel_external_git.fuelclient_audit:OutOfSyncResources