x/entropy
Code Issues Proposed changes

Move code into a scheduler class, part III

Browse Source 
Remove usage of globals.py

Rewrite register-audit and register-repair to avoid using globals.
Get cfg file from engine name and script type instead.

Remove cfg files from git

Change-Id: I8ee119b4ebf55fa18ff4f6a83c0859ddc6699c5f
This commit is contained in:
pran1990
2014-03-16 23:06:30 -07:00
parent 3ac2fde405
commit 616e6c69d0
6 changed files with 39 additions and 92 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
.gitignore vendored
View File

@@ -61,4 +61,4 @@ docs/_build
entropy/test
entropy/logs
entropy/audit/*.json
entropy/cfg
entropy/cfg/*.cfg

70
entropy/__main__.py
View File

@@ -16,7 +16,6 @@
# under the License.
import argparse
from concurrent.futures import ThreadPoolExecutor
import logging
import os
import sys
@@ -27,36 +26,38 @@ sys.path.insert(0, os.path.join(os.path.abspath(os.pardir)))
sys.path.insert(0, os.path.abspath(os.getcwd()))
from engine import Engine
from entropy import globals
from entropy import utils
LOG = logging.getLogger(__name__)
running_audits = []
running_repairs = []
executor = ThreadPoolExecutor(max_workers=globals.MAX_WORKERS)
all_futures = []
entropy_engine = None
# TODO(praneshp): Only hardcoded stuff in the project. Find a way to move
engine_cfg = os.path.join(os.getcwd(), 'entropy', 'cfg', 'engines.cfg')
log_file = os.path.join(os.getcwd(), 'entropy', 'logs', 'entropy.log')
# TODO(praneshp): for next 3 fns, read the right file from engine name and
# type, then modify that file.
def add_to_list(script_type, **kwargs):
if script_type == 'audit':
cfg_file = globals.AUDIT_CFG
else:
cfg_file = globals.REPAIR_CFG
def get_cfg_file(engine, script_type):
cfg_key = {'audit': 'audit_cfg', 'repair': 'repair_cfg'}
engines = utils.load_yaml(engine_cfg)
for engine_value in engines:
if engine_value.keys()[0] == engine:
return engine_value[engine_value.keys()[0]][cfg_key[script_type]]
return None
def add_to_list(engine, script_type, **kwargs):
cfg_file = get_cfg_file(engine, script_type)
if cfg_file is None:
LOG.error('Engine not found')
return
if utils.check_duplicate(kwargs['name'], cfg_file):
LOG.error('%s already exists, not registering', script_type)
return
with open(cfg_file, "a") as cfg:
cfg.write(yaml.dump(kwargs, canonical=False,
default_flow_style=False,
explicit_start=True))
def repair_present(name):
return utils.check_duplicate(name, globals.REPAIR_CFG)
def audit_present(name):
return utils.check_duplicate(name, globals.AUDIT_CFG)
return True
def register_audit(args):
@@ -68,15 +69,10 @@ def register_audit(args):
LOG.error('Need path to script and json')
return
#Check if this one is already present
if audit_present(args.name):
LOG.error('Audit already exists, not registering')
return
#Write to audit file
audit_cfg_args = {'name': args.name,
'conf': os.path.join(os.getcwd(), args.conf)}
add_to_list('audit', **audit_cfg_args)
if add_to_list(args.engine, 'audit', **audit_cfg_args):
LOG.info('Registered audit %s', args.name)
@@ -89,15 +85,10 @@ def register_repair(args):
LOG.error('Need path to script and json')
return
#Check if this one is already present
if repair_present(args.name):
LOG.error('Repair script already exists, not registering')
return
#Write to audit file
repair_cfg_args = {'name': args.name,
'conf': os.path.join(os.getcwd(), args.conf)}
add_to_list('repair', **repair_cfg_args)
if add_to_list(args.engine, 'repair', **repair_cfg_args):
LOG.info('Registered repair script %s', args.name)
@@ -106,18 +97,17 @@ def start_engine(args):
if not (args.name and args.audit_cfg and args.repair_cfg):
LOG.error('Need name, audit_cfg, and repair_cfg')
return
engine_cfg = os.path.join(os.getcwd(), 'entropy', 'cfg', 'test.cfg')
args.log_file = os.path.join(os.getcwd(), args.log_file)
args.audit_cfg = os.path.join(os.getcwd(), args.audit_cfg)
args.repair_cfg = os.path.join(os.getcwd(), args.repair_cfg)
cfg = {'audit': args.audit_cfg, 'repair': args.repair_cfg}
cfg_data = {'log_file': os.path.join(os.getcwd(), args.log_file),
'audit_cfg': os.path.join(os.getcwd(), args.audit_cfg),
'repair_cfg': os.path.join(os.getcwd(), args.repair_cfg)}
cfg = {args.name: cfg_data}
with open(engine_cfg, "w") as cfg_file:
cfg_file.write(yaml.dump(cfg, canonical=False,
default_flow_style=False,
explicit_start=True))
LOG.info('Wrote to engine cfg')
global entropy_engine
entropy_engine = Engine(args)
entropy_engine = Engine(args.name, **cfg_data)
def parse():
@@ -167,6 +157,6 @@ def parse():
if __name__ == '__main__':
#TODO(praneshp): AMQP, json->yaml, reaction scripts(after amqp)
logging.basicConfig(filename=globals.log_file,
logging.basicConfig(filename=log_file,
level=logging.DEBUG)
parse()

9
entropy/cfg/audit.cfg
View File

@@ -1,9 +0,0 @@
---
conf: entropy/audit/vm_count.json
name: vmcount
---
conf: entropy/audit/audit.json
name: audit
---
conf: entropy/audit/vm_count.json
name: test

6
entropy/cfg/repair.cfg
View File

@@ -1,6 +0,0 @@
---
conf: entropy/repair/vm_count_react.json
name: vmcount
---
conf: entropy/repair/react.json
name: react

12
entropy/engine.py
View File

@@ -29,19 +29,19 @@ LOG = logging.getLogger(__name__)
class Engine(object):
def __init__(self, args):
def __init__(self, name, **cfg_data):
# constants
# TODO(praneshp): Hardcode for now, could/should be cmdline input
self.max_workers = 8
self.audit_type = 'audit'
self.repair_type = 'repair'
# engine variables
self.name = args.name
self.audit_cfg = args.audit_cfg
self.repair_cfg = args.repair_cfg
self.name = name
self.audit_cfg = cfg_data['audit_cfg']
self.repair_cfg = cfg_data['repair_cfg']
# TODO(praneshp): Assuming cfg files are in 1 dir. Change later
self.cfg_dir = os.path.dirname(self.audit_cfg)
self.log_file = args.log_file
self.log_file = cfg_data['log_file']
self.executor = ThreadPoolExecutor(max_workers=self.max_workers)
self.running_audits = []
self.running_repairs = []
@@ -50,7 +50,6 @@ class Engine(object):
self.start_scheduler()
def start_scheduler(self):
LOG.debug("Crap")
# Start watchdog thread, which will detect any new audit/react scripts
# TODO(praneshp): Look into how to do this with threadpoolexecutor?
watchdog_thread = self.start_watchdog(self.cfg_dir) # noqa
@@ -80,6 +79,7 @@ class Engine(object):
def start_watchdog(self, dir_to_watch):
event_fn = {self.audit_cfg: self.audit_modified,
self.repair_cfg: self.repair_modified}
LOG.info(event_fn)
return utils.watch_dir_for_change(dir_to_watch, event_fn)
def start_scripts(self, script_type):

28
entropy/globals.py
View File

@@ -1,28 +0,0 @@
# -*- coding: utf-8 -*-
# vim: tabstop=4 shiftwidth=4 softtabstop=4
# Copyright (C) 2013 Yahoo! Inc. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
import os
SCRIPT_REPO = os.path.dirname(__file__)
# Hardcode for now
LOG_REPO = os.path.join(os.getcwd(), 'entropy', 'logs')
CFG_DIR = os.path.join(SCRIPT_REPO, 'cfg')
AUDIT_CFG = os.path.join(CFG_DIR, 'audit.cfg')
REPAIR_CFG = os.path.join(CFG_DIR, 'repair.cfg')
log_file = os.path.join(LOG_REPO, 'entropy.log')
MAX_WORKERS = 8